chore: sync v3 branch with main

.github/.OwlBot.lock.yaml

@@ -1,3 +1,3 @@
 docker:
-  image: gcr.io/repo-automation-bots/owlbot-python:latest
-  digest: sha256:0ffe3bdd6c7159692df5f7744da74e5ef19966288a6bf76023e8e04e0c424d7d
+  image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest
+  digest: sha256:87eee22d276554e4e52863ec9b1cb6a7245815dfae20439712bf644348215a5a

.github/.OwlBot.yaml

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 docker:
-  image: gcr.io/repo-automation-bots/owlbot-python:latest
+  image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest
 
 deep-remove-regex:
   - /owl-bot-staging

.kokoro/continuous/prerelease-deps-3.8.cfg

@@ -0,0 +1,7 @@
+# Format: //devtools/kokoro/config/proto/build.proto
+
+# Only run this nox session.
+env_vars: {
+    key: "NOX_SESSION"
+    value: "prerelease_deps"
+}

CHANGELOG.md

@@ -5,6 +5,33 @@
 [1]: https://pypi.org/project/google-cloud-bigquery/#history
 
 
+### [2.27.1](https://www.github.com/googleapis/python-bigquery/compare/v2.27.0...v2.27.1) (2021-09-27)
+
+
+### Bug Fixes
+
+* remove py.typed since package fails mypy check ([#988](https://www.github.com/googleapis/python-bigquery/issues/988)) ([39030f2](https://www.github.com/googleapis/python-bigquery/commit/39030f26ce081cfacd456b84694c68e3f04ed48d))
+
+## [2.27.0](https://www.github.com/googleapis/python-bigquery/compare/v2.26.0...v2.27.0) (2021-09-24)
+
+
+### Features
+
+* Add py.typed for PEP 561 compliance ([#976](https://www.github.com/googleapis/python-bigquery/issues/976)) ([96e6bee](https://www.github.com/googleapis/python-bigquery/commit/96e6beef3c63b663b7e5879b1458f4dd1a47a5b5))
+* include key metadata in Job representation ([#964](https://www.github.com/googleapis/python-bigquery/issues/964)) ([acca1cb](https://www.github.com/googleapis/python-bigquery/commit/acca1cb7baaa3b00508246c994ade40314d421c3))
+
+
+### Bug Fixes
+
+* Arrow extension-type metadata was not set when calling the REST API or when there are no rows ([#946](https://www.github.com/googleapis/python-bigquery/issues/946)) ([864383b](https://www.github.com/googleapis/python-bigquery/commit/864383bc01636b3774f7da194587b8b7edd0383d))
+* disambiguate missing policy tags from explicitly unset policy tags ([#983](https://www.github.com/googleapis/python-bigquery/issues/983)) ([f83c00a](https://www.github.com/googleapis/python-bigquery/commit/f83c00acead70fc0ce9959eefb133a672d816277))
+* remove default timeout ([#974](https://www.github.com/googleapis/python-bigquery/issues/974)) ([1cef0d4](https://www.github.com/googleapis/python-bigquery/commit/1cef0d4664bf448168b26487a71795144b7f4d6b))
+
+
+### Documentation
+
+* simplify destination table sample with f-strings ([#966](https://www.github.com/googleapis/python-bigquery/issues/966)) ([ab6e76f](https://www.github.com/googleapis/python-bigquery/commit/ab6e76f9489262fd9c1876a1c4f93d7e139aa999))
+
 ## [2.26.0](https://www.github.com/googleapis/python-bigquery/compare/v2.25.2...v2.26.0) (2021-09-01)
 
 

README.rst

@@ -12,7 +12,7 @@ processing power of Google's infrastructure.
 -  `Product Documentation`_
 
 .. |GA| image:: https://img.shields.io/badge/support-GA-gold.svg
-   :target: https://github.com/googleapis/google-cloud-python/blob/master/README.rst#general-availability
+   :target: https://github.com/googleapis/google-cloud-python/blob/main/README.rst#general-availability
 .. |pypi| image:: https://img.shields.io/pypi/v/google-cloud-bigquery.svg
    :target: https://pypi.org/project/google-cloud-bigquery/
 .. |versions| image:: https://img.shields.io/pypi/pyversions/google-cloud-bigquery.svg

docs/conf.py

@@ -363,7 +363,7 @@
     "google-auth": ("https://googleapis.dev/python/google-auth/latest/", None),
     "google.api_core": ("https://googleapis.dev/python/google-api-core/latest/", None,),
     "grpc": ("https://grpc.github.io/grpc/python/", None),
-    "pandas": ("http://pandas.pydata.org/pandas-docs/dev", None),
+    "pandas": ("http://pandas.pydata.org/pandas-docs/stable/", None),
     "geopandas": ("https://geopandas.org/", None),
 }
 

docs/reference.rst

@@ -128,6 +128,7 @@ Schema
     :toctree: generated
 
     schema.SchemaField
+    schema.PolicyTagList
 
 
 Query

google/cloud/bigquery/__init__.py

@@ -87,6 +87,7 @@
 from google.cloud.bigquery.routine import RoutineArgument
 from google.cloud.bigquery.routine import RoutineReference
 from google.cloud.bigquery.routine import RoutineType
+from google.cloud.bigquery.schema import PolicyTagList
 from google.cloud.bigquery.schema import SchemaField
 from google.cloud.bigquery.standard_sql import StandardSqlDataType
 from google.cloud.bigquery.standard_sql import StandardSqlField
@@ -145,6 +146,7 @@
     "RoutineReference",
     # Shared helpers
     "SchemaField",
+    "PolicyTagList",
     "UDFResource",
     "ExternalConfig",
     "BigtableOptions",

google/cloud/bigquery/_helpers.py

@@ -604,8 +604,9 @@ def _get_sub_prop(container, keys, default=None):
         container (Dict):
             A dictionary which may contain other dictionaries as values.
         keys (Iterable):
-            A sequence of keys to attempt to get the value for. Each item in
-            the sequence represents a deeper nesting. The first key is for
+            A sequence of keys to attempt to get the value for. If ``keys`` is a
+            string, it is treated as sequence containing a single string key. Each item
+            in the sequence represents a deeper nesting. The first key is for
             the top level. If there is a dictionary there, the second key
             attempts to get the value within that, and so on.
         default (Optional[object]):
@@ -632,6 +633,9 @@ def _get_sub_prop(container, keys, default=None):
     Returns:
         object: The value if present or the default.
     """
+    if isinstance(keys, str):
+        keys = [keys]
+
     sub_val = container
     for key in keys:
         if key not in sub_val:
@@ -647,8 +651,9 @@ def _set_sub_prop(container, keys, value):
         container (Dict):
             A dictionary which may contain other dictionaries as values.
         keys (Iterable):
-            A sequence of keys to attempt to set the value for. Each item in
-            the sequence represents a deeper nesting. The first key is for
+            A sequence of keys to attempt to set the value for. If ``keys`` is a
+            string, it is treated as sequence containing a single string key. Each item
+            in the sequence represents a deeper nesting. The first key is for
             the top level. If there is a dictionary there, the second key
             attempts to get the value within that, and so on.
         value (object): Value to set within the container.
@@ -675,6 +680,9 @@ def _set_sub_prop(container, keys, value):
         >>> container
         {'key': {'subkey': 'new'}}
     """
+    if isinstance(keys, str):
+        keys = [keys]
+
     sub_val = container
     for key in keys[:-1]:
         if key not in sub_val:

google/cloud/bigquery/_pandas_helpers.py

@@ -183,6 +183,13 @@ def pyarrow_timestamp():
     # the type ID matters, and it's the same for all decimal256 instances.
     pyarrow.decimal256(76, scale=38).id: "BIGNUMERIC",
 }
+BQ_FIELD_TYPE_TO_ARROW_FIELD_METADATA = {
+    "GEOGRAPHY": {
+        b"ARROW:extension:name": b"google:sqlType:geography",
+        b"ARROW:extension:metadata": b'{"encoding": "WKT"}',
+    },
+    "DATETIME": {b"ARROW:extension:name": b"google:sqlType:datetime"},
+}
 
 
 def bq_to_arrow_struct_data_type(field):
@@ -233,7 +240,12 @@ def bq_to_arrow_field(bq_field, array_type=None):
         if array_type is not None:
             arrow_type = array_type  # For GEOGRAPHY, at least initially
         is_nullable = bq_field.mode.upper() == "NULLABLE"
-        return pyarrow.field(bq_field.name, arrow_type, nullable=is_nullable)
+        metadata = BQ_FIELD_TYPE_TO_ARROW_FIELD_METADATA.get(
+            bq_field.field_type.upper() if bq_field.field_type else ""
+        )
+        return pyarrow.field(
+            bq_field.name, arrow_type, nullable=is_nullable, metadata=metadata
+        )
 
     warnings.warn("Unable to determine type for field '{}'.".format(bq_field.name))
     return None

google/cloud/bigquery/client.py

@@ -116,6 +116,8 @@
 
 TIMEOUT_HEADER = "X-Server-Timeout"
 
+TIMEOUT_HEADER = "X-Server-Timeout"
+
 
 class Project(object):
     """Wrapper for resource describing a BigQuery project.

google/cloud/bigquery/job/base.py

@@ -357,6 +357,11 @@ def reservation_usage(self):
     def transaction_info(self) -> Optional[TransactionInfo]:
         """Information of the multi-statement transaction if this job is part of one.
 
+        Since a scripting query job can execute multiple transactions, this
+        property is only expected on child jobs. Use the
+        :meth:`google.cloud.bigquery.client.Client.list_jobs` method with the
+        ``parent_job`` parameter to iterate over child jobs.
+
         .. versionadded:: 2.24.0
         """
         info = self._properties.get("statistics", {}).get("transactionInfo")
@@ -722,6 +727,14 @@ def cancelled(self):
             and self.error_result.get("reason") == _STOPPED_REASON
         )
 
+    def __repr__(self):
+        result = (
+            f"{self.__class__.__name__}<"
+            f"project={self.project}, location={self.location}, id={self.job_id}"
+            ">"
+        )
+        return result
+
 
 class _JobConfig(object):
     """Abstract base class for job configuration objects.

google/cloud/bigquery/model.py

@@ -174,6 +174,20 @@ def label_columns(self) -> Sequence[standard_sql.StandardSqlField]:
         """
         return self._properties.get("labelColumns", [])
 
+    @property
+    def best_trial_id(self) -> Optional[int]:
+        """The best trial_id across all training runs.
+
+        .. deprecated::
+            This property is deprecated!
+
+        Read-only.
+        """
+        value = self._properties.get("bestTrialId")
+        if value is not None:
+            value = int(value)
+        return value
+
     @property
     def expires(self) -> Optional[datetime.datetime]:
         """The datetime when this model expires.

google/cloud/bigquery/retry.py

@@ -60,7 +60,7 @@ def _should_retry(exc):
 pass ``retry=bigquery.DEFAULT_RETRY.with_deadline(30)``.
 """
 
-DEFAULT_TIMEOUT = 5.0 * 60.0
+DEFAULT_TIMEOUT = None
 """The default API timeout.
 
 This is the time to wait per request. To adjust the total wait time, set a