From a9ecddf99f754ccc322919b683b5ddf8fc3883f9 Mon Sep 17 00:00:00 2001 From: Alix Hamilton Date: Tue, 17 Oct 2017 17:28:08 -0700 Subject: [PATCH] BigQuery: removes LoadJob error for autodetect + schema --- bigquery/google/cloud/bigquery/job.py | 6 ------ bigquery/tests/system.py | 14 +++++++++++--- bigquery/tests/unit/test_job.py | 26 -------------------------- 3 files changed, 11 insertions(+), 35 deletions(-) diff --git a/bigquery/google/cloud/bigquery/job.py b/bigquery/google/cloud/bigquery/job.py index 350ad7ce579b..2472758b4a51 100644 --- a/bigquery/google/cloud/bigquery/job.py +++ b/bigquery/google/cloud/bigquery/job.py @@ -143,9 +143,6 @@ class AutoDetectSchema(_TypedApiResourceProperty): """ def __set__(self, instance, value): self._validate(value) - if instance.schema: - raise ValueError('A schema should not be already defined ' - 'when using schema auto-detection') instance._properties[self.resource_name] = value @@ -638,9 +635,6 @@ def schema(self): def schema(self, value): if not all(isinstance(field, SchemaField) for field in value): raise ValueError('Schema items must be fields') - if self.autodetect: - raise ValueError( - 'Schema can not be set if `autodetect` property is True') self._schema = tuple(value) def to_api_repr(self): diff --git a/bigquery/tests/system.py b/bigquery/tests/system.py index 86082bf31982..767dac96a433 100644 --- a/bigquery/tests/system.py +++ b/bigquery/tests/system.py @@ -466,8 +466,9 @@ def test_load_table_from_storage_then_dump_table(self): self.assertEqual(sorted(row_tuples, key=by_age), sorted(ROWS, key=by_age)) - def test_load_table_from_storage_w_autodetect_schema(self): + def test_load_table_from_storage_w_autodetect_schema_then_get_job(self): from google.cloud.bigquery import SchemaField + from google.cloud.bigquery.job import LoadJob rows = ROWS * 100 # BigQuery internally uses the first 100 rows to detect schema @@ -477,11 +478,12 @@ def test_load_table_from_storage_w_autodetect_schema(self): HEADER_ROW, rows) dataset = self.temp_dataset(_make_dataset_id('load_gcs_then_dump')) table_ref = dataset.table('test_table') + JOB_ID = 'load_table_w_autodetect_{}'.format(str(uuid.uuid4())) config = bigquery.LoadJobConfig() config.autodetect = True - job = Config.CLIENT.load_table_from_storage(gs_url, table_ref, - job_config=config) + job = Config.CLIENT.load_table_from_storage( + gs_url, table_ref, job_config=config, job_id=JOB_ID) # Allow for 90 seconds of "warm up" before rows visible. See # https://cloud.google.com/bigquery/streaming-data-into-bigquery#dataavailability @@ -502,6 +504,12 @@ def test_load_table_from_storage_w_autodetect_schema(self): self.assertEqual( sorted(actual_row_tuples, key=by_age), sorted(rows, key=by_age)) + fetched_job = Config.CLIENT.get_job(JOB_ID) + + self.assertIsInstance(fetched_job, LoadJob) + self.assertEqual(fetched_job.job_id, JOB_ID) + self.assertEqual(fetched_job.autodetect, True) + def _write_csv_to_storage(self, bucket_name, blob_name, header_row, data_rows): from google.cloud._testing import _NamedTemporaryFile diff --git a/bigquery/tests/unit/test_job.py b/bigquery/tests/unit/test_job.py index 0e0b667e704d..0ad4071f8fc6 100644 --- a/bigquery/tests/unit/test_job.py +++ b/bigquery/tests/unit/test_job.py @@ -426,32 +426,6 @@ def test_schema_setter(self): config.schema = [full_name, age] self.assertEqual(config.schema, [full_name, age]) - def test_schema_setter_w_autodetect(self): - from google.cloud.bigquery.schema import SchemaField - - config = LoadJobConfig() - schema = [SchemaField('full_name', 'STRING')] - config.autodetect = False - config.schema = schema - self.assertEqual(config.schema, schema) - - config.schema = [] - config.autodetect = True - with self.assertRaises(ValueError): - config.schema = schema - - def test_autodetect_setter_w_schema(self): - from google.cloud.bigquery.schema import SchemaField - - config = LoadJobConfig() - - config.autodetect = False - config.schema = [SchemaField('full_name', 'STRING')] - self.assertEqual(config.autodetect, False) - - with self.assertRaises(ValueError): - config.autodetect = True - def test_props_set_by_server(self): import datetime from google.cloud._helpers import UTC