Upgrade glean_parser to 1.20.1 (#827)

* Upgrade glean_parser to 1.20.0

* spellcheck

* Fix sample app metrics.yaml

* Upgrade to glean_parser 1.20.1

* Fix syntax

* iso8601 only needed by earlier Pythons

* Upgrade to 1.20.2

* Handle iso8601 as an optional dependency

* spellcheck

* Fix up CHANGELOG

Author: mdboom

Diff for: .circleci/config.yml

@@ -753,6 +753,17 @@ jobs:
       - skip-if-doc-only
       - test-python
 
+  Python 3_5 tests minimum dependencies:
+    docker:
+      - image: circleci/python:3.5.9
+    steps:
+      - checkout
+      - skip-if-doc-only
+      - run:
+          command: |
+              echo "export GLEAN_PYDEPS=min" >> $BASH_ENV
+      - test-python
+
   Python 3_6 tests:
     docker:
       - image: circleci/python:3.6.9
@@ -779,6 +790,17 @@ jobs:
       - skip-if-doc-only
       - test-python
 
+  Python 3_8 tests minimum dependencies:
+    docker:
+      - image: circleci/python:3.8.2
+    steps:
+      - checkout
+      - skip-if-doc-only
+      - run:
+          command: |
+              echo "export GLEAN_PYDEPS=min" >> $BASH_ENV
+      - test-python
+
   Python Windows x86_64 tests:
     docker:
       - image: circleci/python:3.7.6
@@ -1049,9 +1071,11 @@ workflows:
             branches:
               only: master
       - Python 3_5 tests
+      - Python 3_5 tests minimum dependencies
       - Python 3_6 tests
       - Python 3_7 tests
       - Python 3_8 tests
+      - Python 3_8 tests minimum dependencies
       - Python Windows x86_64 tests
       - Python Windows i686 tests
 

Diff for: .dictionary

@@ -1,4 +1,4 @@
-personal_ws-1.1 en 166 utf-8
+personal_ws-1.1 en 167 utf-8
 AAR
 AARs
 APIs
@@ -67,6 +67,7 @@ aspell
 async
 autodetected
 backend
+backport
 barcode
 boolean
 booleans
@@ -115,6 +116,7 @@ lang
 latencies
 lfloor
 lifecycle
+linter
 linux
 logcat
 macOS

Diff for: CHANGELOG.md

@@ -5,8 +5,14 @@
 
 [Full changelog](https://github.com/mozilla/glean/compare/v28.0.0...master)
 
+* General:
+  * The version of glean_parser has been upgraded to v1.20.2:
+    * **Breaking change:** glinter errors found during code generation will now return an error code.
+    * `glean_parser` now produces a linter warning when `user` lifetime metrics are set to expire. See [bug 1604854](https://bugzilla.mozilla.org/show_bug.cgi?id=1604854) for additional context.
 * Python:
   * BUGFIX: Fixed a race condition in the `atexit` handler, that would have resulted in the message "No database found".  See [bug 1634310](https://bugzilla.mozilla.org/show_bug.cgi?id=1634310).
+  * The minimum versions of many secondary dependencies have been lowered to make the Glean SDK compatible with more environments.
+  * Dependencies that depend on the version of Python being used are now specified using the `Declaring platform specific dependencies syntax in setuptools <https://setuptools.readthedocs.io/en/latest/setuptools.html#declaring-platform-specific-dependencies>`__. This means that more recent versions of dependencies are likely to be installed on Python 3.6 and later, and unnecessary backport libraries won't be installed on more recent Python versions.
 
 # v28.0.0 (2020-04-23)
 

Diff for: Makefile

@@ -5,6 +5,7 @@ help:
 	  awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
 
 GLEAN_PYENV := $(shell python3 -c "import sys; print('glean-core/python/.venv' + '.'.join(str(x) for x in sys.version_info[:2]))")
+GLEAN_PYDEPS := ${GLEAN_PYDEPS}
 
 # Setup environments
 
@@ -15,6 +16,11 @@ $(GLEAN_PYENV)/bin/python3:
 	python3 -m venv $(GLEAN_PYENV)
 	$(GLEAN_PYENV)/bin/pip install --upgrade pip
 	$(GLEAN_PYENV)/bin/pip install -r glean-core/python/requirements_dev.txt
+	bash -c "if [ \"$(GLEAN_PYDEPS)\" == \"min\" ]; then \
+		$(GLEAN_PYENV)/bin/pip install requirements-builder; \
+		$(GLEAN_PYENV)/bin/requirements-builder --level=min glean-core/python/setup.py > min_requirements.txt; \
+		$(GLEAN_PYENV)/bin/pip install -r min_requirements.txt; \
+	fi"
 	# black isn't installable on Python 3.5, but we can do without it
 	$(GLEAN_PYENV)/bin/pip install black || true
 

Diff for: docs/dev/python/setting-up-python-build-environment.md

@@ -67,7 +67,9 @@ pip install -r requirements_dev.txt
 
 The default location of the virtual environment used by the make file is `glean-core/python/.venvX.Y`, where `X.Y` is the version of Python in use. This makes it possible to build and test for multiple versions of Python in the same checkout.
 
-> *Note:* If you wish to change the location of the virtual environment that the `Makefile` uses, pass the `GLEAN_PYENV` environment variable: `make test-python GLEAN_PYENV=mypyenv`.
+> *Note:* If you wish to change the location of the virtual environment that the `Makefile` uses, pass the `GLEAN_PYENV` environment variable: `make python-setup GLEAN_PYENV=mypyenv`.
+
+By default, the `Makefile` installs the latest version available of each of Glean's dependencies.  If you wish to install the minimum required versions instead, (useful primarily to ensure that Glean doesn't unintentionally use newer APIs in its dependencies) pass the `GLEAN_PYDEPS=min` environment variable: `make python-setup GLEAN_PYDEPS=min`.
 
 ## Build the Python bindings
 

Diff for: glean-core/ios/sdk_generator.sh

@@ -25,7 +25,7 @@
 
 set -e
 
-GLEAN_PARSER_VERSION=1.19.0
+GLEAN_PARSER_VERSION=1.20.2
 
 # CMDNAME is used in the usage text below.
 # shellcheck disable=SC2034

Diff for: glean-core/python/glean/metrics/datetime.py

@@ -4,12 +4,10 @@
 
 
 import datetime
+import sys
 from typing import List, Optional
 
 
-import iso8601  # type: ignore
-
-
 from .. import _ffi
 from .._dispatcher import Dispatcher
 from ..testing import ErrorType
@@ -19,6 +17,12 @@
 from .timeunit import TimeUnit
 
 
+if sys.version_info < (3, 7):
+    import iso8601  # type: ignore
+else:
+    iso8601 = None
+
+
 class DatetimeMetricType:
     """
     This implements the developer facing API for recording datetime metrics.
@@ -144,7 +148,12 @@ def test_get_value(self, ping_name: Optional[str] = None) -> datetime.datetime:
         Returns:
             value (datetime.datetime): value of the stored metric.
         """
-        return iso8601.parse_date(self.test_get_value_as_str(ping_name))
+        if sys.version_info < (3, 7):
+            return iso8601.parse_date(self.test_get_value_as_str(ping_name))
+        else:
+            return datetime.datetime.fromisoformat(
+                self.test_get_value_as_str(ping_name)
+            )
 
     def test_get_num_recorded_errors(
         self, error_type: ErrorType, ping_name: Optional[str] = None

Diff for: glean-core/python/setup.py

@@ -47,9 +47,9 @@
     version = parsed_toml["package"]["version"]
 
 requirements = [
-    "cffi==1.13.1",
-    "glean_parser==1.19.0",
-    "iso8601==0.1.12",
+    "cffi>=1",
+    "glean_parser==1.20.2",
+    "iso8601>=0.1.10; python_version<='3.6'",
 ]
 
 setup_requirements = []
@@ -75,7 +75,14 @@
 shutil.copyfile("../ffi/glean.h", "glean/glean.h")
 shutil.copyfile("../metrics.yaml", "glean/metrics.yaml")
 shutil.copyfile("../pings.yaml", "glean/pings.yaml")
-shutil.copyfile(shared_object_build_dir + shared_object, "glean/" + shared_object)
+# When running inside of `requirements-builder`, the Rust shared object may not
+# yet exist, so ignore the exception when trying to copy it. Under normal
+# circumstances, this will still show up as an error when running the `build`
+# command as a missing `package_data` file.
+try:
+    shutil.copyfile(shared_object_build_dir + shared_object, "glean/" + shared_object)
+except FileNotFoundError:
+    pass
 
 
 class BinaryDistribution(Distribution):

Diff for: gradle-plugin/src/main/groovy/mozilla/telemetry/glean-gradle-plugin/GleanGradlePlugin.groovy

@@ -34,7 +34,7 @@ class GleanMetricsYamlTransform extends ArtifactTransform {
 @SuppressWarnings("GrPackage")
 class GleanPlugin implements Plugin<Project> {
     // The version of glean_parser to install from PyPI.
-    private String GLEAN_PARSER_VERSION = "1.19.0"
+    private String GLEAN_PARSER_VERSION = "1.20.2"
     // The version of Miniconda is explicitly specified.
     // Miniconda3-4.5.12 is known to not work on Windows.
     private String MINICONDA_VERSION = "4.5.11"

Diff for: samples/android/app/metrics.yaml

@@ -13,94 +13,98 @@ $schema: moz://mozilla.org/schemas/glean/metrics/1-0-0
 browser.engagement:
   click:
     type: event
-    description: >
+    description: |
       Just testing events
     bugs:
       - https://bugzilla.mozilla.org/123456789
     data_reviews:
-      - N/A 
+      - N/A
     notification_emails:
       - [email protected]
     extra_keys:
       key1:
         description: "This is key one"
       key2:
         description: "This is key two"
-    expires: 2100-01-01 
-    
+    expires: 2100-01-01
+
   event_no_keys:
     type: event
-    description: >
+    description: |
       Just testing events without keys
     bugs:
       - https://bugzilla.mozilla.org/123456789
     data_reviews:
-      - N/A 
+      - N/A
     notification_emails:
       - [email protected]
     expires: 2100-01-01
 
 basic:
   os:
     type: string
-    description: >
+    description: |
       The name of the os
     bugs:
       - https://bugzilla.mozilla.org/123456789
     data_reviews:
-      - N/A 
+      - N/A
     notification_emails:
       - [email protected]
     expires: 2100-01-01
 
 test:
   string_list:
     type: string_list
-    description: >
+    description: |
       Testing StringList ping
     send_in_pings:
       - test-string-list
     lifetime: user
     bugs:
       - https://bugzilla.mozilla.org/123456789
     data_reviews:
-      - N/A 
+      - N/A
     notification_emails:
       - [email protected]
     expires: 2100-01-01
+    no_lint:
+      - USER_LIFETIME_EXPIRATION
 
   counter:
     type: counter
-    description: >
+    description: |
       Testing counter
     send_in_pings:
       - test-string-list
     lifetime: user
     bugs:
       - https://bugzilla.mozilla.org/123456789
     data_reviews:
-      - N/A 
+      - N/A
     notification_emails:
       - [email protected]
     expires: 2100-01-01
+    no_lint:
+      - USER_LIFETIME_EXPIRATION
 
   timespan:
     type: timespan
-    description: >
+    description: |
       Testing a timespan
     time_unit: microsecond
     lifetime: application
     bugs:
       - https://bugzilla.mozilla.org/1508948
     data_reviews:
-      - N/A 
+      - N/A
     notification_emails:
       - [email protected]
     expires: 2100-01-01
 
   is_started:
     type: boolean
-    description: >
+    description: |
       Testing boolean
     send_in_pings:
       - baseline
@@ -112,11 +116,13 @@ test:
     notification_emails:
       - [email protected]
     expires: 2100-01-01
+    no_lint:
+      - BASELINE_PING
 
 custom:
   counter:
     type: counter
-    description: >
+    description: |
       A custom counter that goes on a custom ping
     lifetime: ping
     send_in_pings: