Skip to content

Commit fb7c188

Browse files
author
Jim Fulton
authored
feat: Comment/description support, bug fixes and better test coverage (#138)
- Runs SQLAlchemy dialect-compliance tests (as system tests). - 100% unit-test coverage. - Support for table and column comments/descriptions (requiring SQLAlchemy 1.2 or higher). - Fixes bugs found while debugging tests, including: - Handling of `in` queries. - String literals with special characters. - Use BIGNUMERIC when necessary. - Missing types: BIGINT, SMALLINT, Boolean, REAL, CHAR, NCHAR, VARCHAR, NVARCHAR, TEXT, VARBINARY, DECIMAL - Literal bytes, dates, times, datetimes, timestamps, and arrays. - Get view definitions. - When executing parameterized queries, the new BigQuery DB API parameter syntax is used to pass type information. This is helpful when the DB API can't determine type information from values, or can't determine it correctly.
1 parent 0a3151b commit fb7c188

22 files changed

+2414
-161
lines changed

.coveragerc

+1-3
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,6 @@
1717
# Generated by synthtool. DO NOT EDIT!
1818
[run]
1919
branch = True
20-
omit =
21-
google/cloud/__init__.py
2220

2321
[report]
2422
fail_under = 100
@@ -35,4 +33,4 @@ omit =
3533
*/proto/*.py
3634
*/core/*.py
3735
*/site-packages/*.py
38-
google/cloud/__init__.py
36+
pybigquery/requirements.py

noxfile.py

+57-4
Original file line numberDiff line numberDiff line change
@@ -28,17 +28,18 @@
2828
BLACK_PATHS = ["docs", "pybigquery", "tests", "noxfile.py", "setup.py"]
2929

3030
DEFAULT_PYTHON_VERSION = "3.8"
31-
SYSTEM_TEST_PYTHON_VERSIONS = ["3.8"]
31+
SYSTEM_TEST_PYTHON_VERSIONS = ["3.9"]
3232
UNIT_TEST_PYTHON_VERSIONS = ["3.6", "3.7", "3.8", "3.9"]
3333

3434
CURRENT_DIRECTORY = pathlib.Path(__file__).parent.absolute()
3535

3636
# 'docfx' is excluded since it only needs to run in 'docs-presubmit'
3737
nox.options.sessions = [
38+
"lint",
3839
"unit",
39-
"system",
4040
"cover",
41-
"lint",
41+
"system",
42+
"compliance",
4243
"lint_setup_py",
4344
"blacken",
4445
"docs",
@@ -169,6 +170,58 @@ def system(session):
169170
)
170171

171172

173+
@nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS)
174+
def compliance(session):
175+
"""Run the system test suite."""
176+
constraints_path = str(
177+
CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt"
178+
)
179+
system_test_folder_path = os.path.join("tests", "sqlalchemy_dialect_compliance")
180+
181+
# Check the value of `RUN_SYSTEM_TESTS` env var. It defaults to true.
182+
if os.environ.get("RUN_COMPLIANCE_TESTS", "true") == "false":
183+
session.skip("RUN_COMPLIANCE_TESTS is set to false, skipping")
184+
# Sanity check: Only run tests if the environment variable is set.
185+
if not os.environ.get("GOOGLE_APPLICATION_CREDENTIALS", ""):
186+
session.skip("Credentials must be set via environment variable")
187+
# Install pyopenssl for mTLS testing.
188+
if os.environ.get("GOOGLE_API_USE_CLIENT_CERTIFICATE", "false") == "true":
189+
session.install("pyopenssl")
190+
# Sanity check: only run tests if found.
191+
if not os.path.exists(system_test_folder_path):
192+
session.skip("Compliance tests were not found")
193+
194+
# Use pre-release gRPC for system tests.
195+
session.install("--pre", "grpcio")
196+
197+
# Install all test dependencies, then install this package into the
198+
# virtualenv's dist-packages.
199+
session.install(
200+
"mock",
201+
"pytest",
202+
"pytest-rerunfailures",
203+
"google-cloud-testutils",
204+
"-c",
205+
constraints_path,
206+
)
207+
session.install("-e", ".", "-c", constraints_path)
208+
209+
session.run(
210+
"py.test",
211+
"-vv",
212+
f"--junitxml=compliance_{session.python}_sponge_log.xml",
213+
"--reruns=3",
214+
"--reruns-delay=60",
215+
"--only-rerun="
216+
"403 Exceeded rate limits|"
217+
"409 Already Exists|"
218+
"404 Not found|"
219+
"400 Cannot execute DML over a non-existent table",
220+
system_test_folder_path,
221+
*session.posargs,
222+
)
223+
224+
172225
@nox.session(python=DEFAULT_PYTHON_VERSION)
173226
def cover(session):
174227
"""Run the final coverage report.
@@ -177,7 +230,7 @@ def cover(session):
177230
test runs (not system test runs), and then erases coverage data.
178231
"""
179232
session.install("coverage", "pytest-cov")
180-
session.run("coverage", "report", "--show-missing", "--fail-under=50")
233+
session.run("coverage", "report", "--show-missing", "--fail-under=100")
181234

182235
session.run("coverage", "erase")
183236

pybigquery/requirements.py

+220
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,220 @@
1+
# Copyright (c) 2021 The PyBigQuery Authors
2+
#
3+
# Permission is hereby granted, free of charge, to any person obtaining a copy of
4+
# this software and associated documentation files (the "Software"), to deal in
5+
# the Software without restriction, including without limitation the rights to
6+
# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
7+
# the Software, and to permit persons to whom the Software is furnished to do so,
8+
# subject to the following conditions:
9+
#
10+
# The above copyright notice and this permission notice shall be included in all
11+
# copies or substantial portions of the Software.
12+
#
13+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
15+
# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
16+
# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
17+
# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
18+
# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
19+
"""
20+
This module is used by the compliance tests to control which tests are run
21+
22+
based on database capabilities.
23+
"""
24+
25+
import sqlalchemy.testing.requirements
26+
import sqlalchemy.testing.exclusions
27+
28+
supported = sqlalchemy.testing.exclusions.open
29+
unsupported = sqlalchemy.testing.exclusions.closed
30+
31+
32+
class Requirements(sqlalchemy.testing.requirements.SuiteRequirements):
33+
@property
34+
def index_reflection(self):
35+
return unsupported()
36+
37+
@property
38+
def indexes_with_ascdesc(self):
39+
"""target database supports CREATE INDEX with per-column ASC/DESC."""
40+
return unsupported()
41+
42+
@property
43+
def unique_constraint_reflection(self):
44+
"""target dialect supports reflection of unique constraints"""
45+
return unsupported()
46+
47+
@property
48+
def autoincrement_insert(self):
49+
"""target platform generates new surrogate integer primary key values
50+
when insert() is executed, excluding the pk column."""
51+
return unsupported()
52+
53+
@property
54+
def primary_key_constraint_reflection(self):
55+
return unsupported()
56+
57+
@property
58+
def foreign_keys(self):
59+
"""Target database must support foreign keys."""
60+
61+
return unsupported()
62+
63+
@property
64+
def foreign_key_constraint_reflection(self):
65+
return unsupported()
66+
67+
@property
68+
def on_update_cascade(self):
69+
"""target database must support ON UPDATE..CASCADE behavior in
70+
foreign keys."""
71+
72+
return unsupported()
73+
74+
@property
75+
def named_constraints(self):
76+
"""target database must support names for constraints."""
77+
78+
return unsupported()
79+
80+
@property
81+
def temp_table_reflection(self):
82+
return unsupported()
83+
84+
@property
85+
def temporary_tables(self):
86+
"""target database supports temporary tables"""
87+
return unsupported() # Temporary tables require use of scripts.
88+
89+
@property
90+
def duplicate_key_raises_integrity_error(self):
91+
"""target dialect raises IntegrityError when reporting an INSERT
92+
with a primary key violation. (hint: it should)
93+
94+
"""
95+
return unsupported()
96+
97+
@property
98+
def precision_numerics_many_significant_digits(self):
99+
"""target backend supports values with many digits on both sides,
100+
such as 319438950232418390.273596, 87673.594069654243
101+
102+
"""
103+
return supported()
104+
105+
@property
106+
def date_coerces_from_datetime(self):
107+
"""target dialect accepts a datetime object as the target
108+
of a date column."""
109+
110+
# BigQuery doesn't allow saving a datetime in a date:
111+
# `TYPE_DATE`, Invalid date: '2012-10-15T12:57:18'
112+
113+
return unsupported()
114+
115+
@property
116+
def window_functions(self):
117+
"""Target database must support window functions."""
118+
return supported() # There are no tests for this. <shrug>
119+
120+
@property
121+
def ctes(self):
122+
"""Target database supports CTEs"""
123+
124+
return supported()
125+
126+
@property
127+
def views(self):
128+
"""Target database must support VIEWs."""
129+
130+
return supported()
131+
132+
@property
133+
def schemas(self):
134+
"""Target database must support external schemas, and have one
135+
named 'test_schema'."""
136+
137+
return supported()
138+
139+
@property
140+
def implicit_default_schema(self):
141+
"""target system has a strong concept of 'default' schema that can
142+
be referred to implicitly.
143+
144+
basically, PostgreSQL.
145+
146+
"""
147+
return supported()
148+
149+
@property
150+
def comment_reflection(self):
151+
return supported() # Well, probably not, but we'll try. :)
152+
153+
@property
154+
def unicode_ddl(self):
155+
"""Target driver must support some degree of non-ascii symbol
156+
names.
157+
"""
158+
return supported()
159+
160+
@property
161+
def datetime_literals(self):
162+
"""target dialect supports rendering of a date, time, or datetime as a
163+
literal string, e.g. via the TypeEngine.literal_processor() method.
164+
165+
"""
166+
167+
return supported()
168+
169+
@property
170+
def timestamp_microseconds(self):
171+
"""target dialect supports representation of Python
172+
datetime.datetime() with microsecond objects but only
173+
if TIMESTAMP is used."""
174+
return supported()
175+
176+
@property
177+
def datetime_historic(self):
178+
"""target dialect supports representation of Python
179+
datetime.datetime() objects with historic (pre 1970) values."""
180+
181+
return supported()
182+
183+
@property
184+
def date_historic(self):
185+
"""target dialect supports representation of Python
186+
datetime.datetime() objects with historic (pre 1970) values."""
187+
188+
return supported()
189+
190+
@property
191+
def precision_numerics_enotation_small(self):
192+
"""target backend supports Decimal() objects using E notation
193+
to represent very small values."""
194+
return supported()
195+
196+
@property
197+
def precision_numerics_enotation_large(self):
198+
"""target backend supports Decimal() objects using E notation
199+
to represent very large values."""
200+
return supported()
201+
202+
@property
203+
def update_from(self):
204+
"""Target must support UPDATE..FROM syntax"""
205+
return supported()
206+
207+
@property
208+
def order_by_label_with_expression(self):
209+
"""target backend supports ORDER BY a column label within an
210+
expression.
211+
212+
Basically this::
213+
214+
select data as foo from test order by foo || 'bar'
215+
216+
Lots of databases including PostgreSQL don't support this,
217+
so this is off by default.
218+
219+
"""
220+
return supported()

0 commit comments

Comments
 (0)