Skip to content

Commit

Permalink
Split snippets into txt and py (apache#26693)
Browse files Browse the repository at this point in the history
move examples that requires expansion-service from snippets.py
  • Loading branch information
RyuSA committed Jun 6, 2023
1 parent ef35550 commit ff777f4
Show file tree
Hide file tree
Showing 3 changed files with 34 additions and 18 deletions.
16 changes: 0 additions & 16 deletions sdks/python/apache_beam/examples/snippets/snippets.py
Original file line number Diff line number Diff line change
Expand Up @@ -1189,13 +1189,6 @@ def model_bigqueryio(
create_disposition=beam.io.BigQueryDisposition.CREATE_IF_NEEDED)
# [END model_bigqueryio_write]

# [START model_bigqueryio_write_with_storage_write_api]
quotes | "WriteTableWithStorageAPI" >> beam.io.WriteToBigQuery(
table_spec,
schema=table_schema,
method=beam.io.WriteToBigQuery.Method.STORAGE_WRITE_API)
# [END model_bigqueryio_write_with_storage_write_api]

# [START model_bigqueryio_write_schema]
table_schema = {
'fields': [{
Expand All @@ -1206,15 +1199,6 @@ def model_bigqueryio(
}
# [END model_bigqueryio_write_schema]

# [START model_bigqueryio_storage_write_api_with_frequency]
# The SDK for Python does not support `withNumStorageWriteApiStreams`
quotes | "StorageWriteAPIWithFrequency" >> beam.io.WriteToBigQuery(
table_spec,
schema=table_schema,
method=beam.io.WriteToBigQuery.Method.STORAGE_WRITE_API,
triggering_frequency=5)
# [END model_bigqueryio_storage_write_api_with_frequency]

# [START model_bigqueryio_write_dynamic_destinations]
fictional_characters_view = beam.pvalue.AsDict(
pipeline | 'CreateCharacters' >> beam.Create([('Yoda', True),
Expand Down
32 changes: 32 additions & 0 deletions sdks/python/apache_beam/examples/snippets/snippets.py.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

# [START model_bigqueryio_storage_write_api_with_frequency]
# The SDK for Python does not support `withNumStorageWriteApiStreams`
quotes | "StorageWriteAPIWithFrequency" >> beam.io.WriteToBigQuery(
table_spec,
schema=table_schema,
method=beam.io.WriteToBigQuery.Method.STORAGE_WRITE_API,
triggering_frequency=5)
# [END model_bigqueryio_storage_write_api_with_frequency]

# [START model_bigqueryio_write_with_storage_write_api]
quotes | "WriteTableWithStorageAPI" >> beam.io.WriteToBigQuery(
table_spec,
schema=table_schema,
method=beam.io.WriteToBigQuery.Method.STORAGE_WRITE_API)
# [END model_bigqueryio_write_with_storage_write_api]
Original file line number Diff line number Diff line change
Expand Up @@ -807,7 +807,7 @@ BigQueryIO.writeTableRows()
);
{{< /highlight >}}
{{< highlight py >}}
{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets.py" model_bigqueryio_write_with_storage_write_api >}}
{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets.py.txt" model_bigqueryio_write_with_storage_write_api >}}
{{< /highlight >}}

If you want to change the behavior of BigQueryIO so that all the BigQuery sinks
Expand Down Expand Up @@ -846,7 +846,7 @@ BigQueryIO.writeTableRows()
);
{{< /highlight >}}
{{< highlight py >}}
{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets.py" model_bigqueryio_storage_write_api_with_frequency >}}
{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets.py.txt" model_bigqueryio_storage_write_api_with_frequency >}}
{{< /highlight >}}

The number of streams defines the parallelism of the BigQueryIO Write transform
Expand Down

0 comments on commit ff777f4

Please sign in to comment.