1616import datetime
1717import decimal
1818
19- from google .cloud import bigquery
2019from google .cloud .bigquery import enums
21- from google .cloud .bigquery_storage_v1 import types as gapic_types
22- from google .cloud .bigquery_storage_v1 .writer import AppendRowsStream
2320import pandas as pd
24-
2521import pyarrow as pa
2622
23+ from google .cloud import bigquery
24+ from google .cloud .bigquery_storage_v1 import types as gapic_types
25+ from google .cloud .bigquery_storage_v1 .writer import AppendRowsStream
26+
2727TABLE_LENGTH = 100_000
2828
2929BQ_SCHEMA = [
@@ -100,7 +100,10 @@ def make_table(project_id, dataset_id, bq_client):
100100
101101
102102def create_stream (bqstorage_write_client , table ):
103- stream_name = f"projects/{ table .project } /datasets/{ table .dataset_id } /tables/{ table .table_id } /_default"
103+ stream_name = (
104+ f"projects/{ table .project } /datasets/{ table .dataset_id } /"
105+ f"tables/{ table .table_id } /_default"
106+ )
104107 request_template = gapic_types .AppendRowsRequest ()
105108 request_template .write_stream = stream_name
106109
@@ -160,18 +163,63 @@ def generate_pyarrow_table(num_rows=TABLE_LENGTH):
160163
161164
162165def generate_write_requests (pyarrow_table ):
163- # Determine max_chunksize of the record batches. Because max size of
164- # AppendRowsRequest is 10 MB, we need to split the table if it's too big.
165- # See: https://cloud.google.com/bigquery/docs/reference/storage/rpc/google.cloud.bigquery.storage.v1#appendrowsrequest
166- max_request_bytes = 10 * 2 ** 20 # 10 MB
167- chunk_num = int (pyarrow_table .nbytes / max_request_bytes ) + 1
168- chunk_size = int (pyarrow_table .num_rows / chunk_num )
169-
170- # Construct request(s).
171- for batch in pyarrow_table .to_batches (max_chunksize = chunk_size ):
166+ # Maximum size for a single AppendRowsRequest is 10 MB.
167+ # To be safe, we'll aim for a soft limit of 7 MB.
168+ max_request_bytes = 7 * 1024 * 1024 # 7 MB
169+
170+ def _create_request (batches ):
171+ """Helper to create an AppendRowsRequest from a list of batches."""
172+ combined_table = pa .Table .from_batches (batches )
172173 request = gapic_types .AppendRowsRequest ()
173- request .arrow_rows .rows .serialized_record_batch = batch .serialize ().to_pybytes ()
174- yield request
174+ request .arrow_rows .rows .serialized_record_batch = (
175+ combined_table .combine_chunks ().to_batches ()[0 ].serialize ().to_pybytes ()
176+ )
177+ return request
178+
179+ batches = pyarrow_table .to_batches ()
180+
181+ current_batches = []
182+ current_size = 0
183+
184+ while batches :
185+ batch = batches .pop ()
186+ batch_size = batch .nbytes
187+
188+ # If the batch is larger than max_request_bytes, split it into 2 sub batches.
189+ if batch_size > max_request_bytes :
190+ if batch .num_rows > 1 :
191+ # Split the batch into 2 sub batches with identical chunksizes
192+ mid = batch .num_rows // 2
193+ batch_left = batch .slice (offset = 0 , length = mid )
194+ batch_right = batch .slice (offset = mid )
195+
196+ # Append the new batches into the stack and continue poping.
197+ batches .append (batch_left )
198+ batches .append (batch_right )
199+ continue
200+ # If the batch is single row and still larger than max_request_bytes
201+ else :
202+ raise ValueError (
203+ f"A single PyArrow batch of one row is larger than the maximum request size "
204+ f"(batch size: { batch_size } > max request size: { max_request_bytes } ). Cannot proceed."
205+ )
206+ # The current batches are ok to form a request when next batch will exceed the max_request_bytes.
207+ if current_size + batch_size > max_request_bytes :
208+ # Current batches can't be empty otherwise it will suffice batch_size > max_request_bytes above.
209+ yield _create_request (current_batches )
210+
211+ current_batches = []
212+ current_size = 0
213+ batches .append (batch )
214+
215+ # Otherwise, add the batch into current_batches
216+ else :
217+ current_batches .append (batch )
218+ current_size += batch_size
219+
220+ # Flush remaining batches
221+ if current_batches :
222+ yield _create_request (current_batches )
175223
176224
177225def verify_result (client , table , futures ):
@@ -181,14 +229,13 @@ def verify_result(client, table, futures):
181229 assert bq_table .schema == BQ_SCHEMA
182230
183231 # Verify table size.
184- query = client .query (f"SELECT COUNT(1) FROM `{ bq_table } `;" )
232+ query = client .query (f"SELECT DISTINCT int64_col FROM `{ bq_table } `;" )
185233 query_result = query .result ().to_dataframe ()
186234
187- # There might be extra rows due to retries.
188- assert query_result .iloc [0 , 0 ] >= TABLE_LENGTH
235+ assert len (query_result ) == TABLE_LENGTH
189236
190237 # Verify that table was split into multiple requests.
191- assert len (futures ) == 2
238+ assert len (futures ) == 4
192239
193240
194241def main (project_id , dataset ):
0 commit comments