From f76cccffa05b9d0ca1c2115a232db2ceb5832afc Mon Sep 17 00:00:00 2001 From: Ashwin Agarwal Date: Thu, 31 Mar 2022 13:04:19 -0400 Subject: [PATCH 01/27] Add Glean metrics, pings, docs, and npm scripts --- docs/metrics.md | 182 +++++++++++++++++++++----- functions/package-lock.json | 60 ++++++++- functions/package.json | 6 +- metrics.yaml | 247 ++++++++++++++++++++++++++++++++++++ pings.yaml | 75 +++++++++++ 5 files changed, 534 insertions(+), 36 deletions(-) create mode 100644 metrics.yaml create mode 100644 pings.yaml diff --git a/docs/metrics.md b/docs/metrics.md index 5cffb6d5..9bbc6d68 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -8,54 +8,174 @@ This means you might have to go searching through the dependency tree to get a f # Pings -- [rs01-event](#rs01-event) +- [deletion-request](#deletion-request) +- [demographics](#demographics) +- [enrollment](#enrollment) +- [events](#events) +- [study-enrollment](#study-enrollment) +- [study-unenrollment](#study-unenrollment) +- [unenrollment](#unenrollment) -## rs01-event +## deletion-request -A ping representing an event sent by the study. -See the `reasons` documentation for additional -information. +This is a built-in ping that is assembled out of the box by the Glean SDK. + +See the Glean SDK documentation for the [`deletion-request` ping](https://mozilla.github.io/glean/book/user/pings/deletion-request.html). + +All Glean pings contain built-in metrics in the [`ping_info`](https://mozilla.github.io/glean/book/user/pings/index.html#the-ping_info-section) and [`client_info`](https://mozilla.github.io/glean/book/user/pings/index.html#the-client_info-section) sections. + +In addition to those built-in metrics, the following metrics are added to the ping: + +| Name | Type | Description | Data reviews | Extras | Expiration | [Data Sensitivity](https://wiki.mozilla.org/Firefox/Data_Collection) | +| --- | --- | --- | --- | --- | --- | --- | +| rally.id |[uuid](https://mozilla.github.io/glean/book/user/metrics/uuid.html) |The id of the Rally client. |[mozilla-rally/rally-core-addon#505](https://github.com/mozilla-rally/rally-core-addon/pull/505#issuecomment-815826426)||never | | + +## demographics + +After a user joins the platform they are asked to fill a +demographic survey, in order to help researchers parse the +data. The survey is optional and can be partially filled: this +ping is submitted right after the survey is filled. + + +This ping is sent if empty. + +**Data reviews for this ping:** + +- + +**Bugs related to this ping:** + +- + +All Glean pings contain built-in metrics in the [`ping_info`](https://mozilla.github.io/glean/book/user/pings/index.html#the-ping_info-section) and [`client_info`](https://mozilla.github.io/glean/book/user/pings/index.html#the-client_info-section) sections. + +In addition to those built-in metrics, the following metrics are added to the ping: + +| Name | Type | Description | Data reviews | Extras | Expiration | [Data Sensitivity](https://wiki.mozilla.org/Firefox/Data_Collection) | +| --- | --- | --- | --- | --- | --- | --- | +| rally.id |[uuid](https://mozilla.github.io/glean/book/user/metrics/uuid.html) |The id of the Rally client. |[mozilla-rally/rally-core-addon#505](https://github.com/mozilla-rally/rally-core-addon/pull/505#issuecomment-815826426)||never | | +| user.age |[labeled_boolean](https://mozilla.github.io/glean/book/user/metrics/labeled_booleans.html) |The user age. |[mozilla-rally/rally-core-addon#139](https://github.com/mozilla-rally/rally-core-addon/pull/139#issuecomment-736024232)|
  • band_19_24
  • band_25_34
  • band_35_44
  • band_45_54
  • band_55_64
  • band_over_65
|never | | +| user.exact_income |[quantity](https://mozilla.github.io/glean/book/user/metrics/quantity.html) |The user household's combined annual income during the past 12 months. This field replaces the previous income field. |[mozilla-rally/rally-core-addon#139](https://github.com/mozilla-rally/rally-core-addon/pull/139#issuecomment-736024232), [mozilla-rally/rally-core-addon#624](https://github.com/mozilla-rally/rally-core-addon/pull/624#issuecomment-850479051)|
  • unit: US Dollars
|never | | +| user.gender |[labeled_boolean](https://mozilla.github.io/glean/book/user/metrics/labeled_booleans.html) |The user gender. |[mozilla-rally/rally-core-addon#139](https://github.com/mozilla-rally/rally-core-addon/pull/139#issuecomment-736024232)|
  • male
  • female
  • neither
  • decline
|never | | +| user.origin |[labeled_boolean](https://mozilla.github.io/glean/book/user/metrics/labeled_booleans.html) |The user origin: Hispanic, Latinx, Spanish or other. |[mozilla-rally/rally-core-addon#139](https://github.com/mozilla-rally/rally-core-addon/pull/139#issuecomment-736024232)|
  • hispanic_latinx_spanish
  • other
|never | | +| user.races |[labeled_boolean](https://mozilla.github.io/glean/book/user/metrics/labeled_booleans.html) |The user race / ethnicity. |[mozilla-rally/rally-core-addon#139](https://github.com/mozilla-rally/rally-core-addon/pull/139#issuecomment-736024232)|
  • am_indian_or_alaska_native
  • asian_indian
  • black_or_african_american
  • chamorro
  • chinese
  • filipino
  • japanese
  • korean
  • native_hawaiian
  • samoan
  • vietnamese
  • white
  • other_asian
  • other_pacific_islander
  • some_other_race
|never | | +| user.school |[labeled_boolean](https://mozilla.github.io/glean/book/user/metrics/labeled_booleans.html) |The highest level of school user has completed. |[mozilla-rally/rally-core-addon#139](https://github.com/mozilla-rally/rally-core-addon/pull/139#issuecomment-736024232)|
  • less_than_high_school
  • some_high_school
  • high_school_grad_or_eq
  • college_degree_in_progress
  • associates_degree
  • bachelors_degree
  • graduate_degree
|never | | +| user.zipcode |[string](https://mozilla.github.io/glean/book/user/metrics/string.html) |The user zip code. |[mozilla-rally/rally-core-addon#139](https://github.com/mozilla-rally/rally-core-addon/pull/139#issuecomment-736024232)||never | | + +## enrollment + +This ping is sent at the end of the user onboarding process, +when and if the user joins the platform. + + +This ping is sent if empty. + +**Data reviews for this ping:** + +- + +**Bugs related to this ping:** + +- + +All Glean pings contain built-in metrics in the [`ping_info`](https://mozilla.github.io/glean/book/user/pings/index.html#the-ping_info-section) and [`client_info`](https://mozilla.github.io/glean/book/user/pings/index.html#the-client_info-section) sections. + +In addition to those built-in metrics, the following metrics are added to the ping: + +| Name | Type | Description | Data reviews | Extras | Expiration | [Data Sensitivity](https://wiki.mozilla.org/Firefox/Data_Collection) | +| --- | --- | --- | --- | --- | --- | --- | +| rally.id |[uuid](https://mozilla.github.io/glean/book/user/metrics/uuid.html) |The id of the Rally client. |[mozilla-rally/rally-core-addon#505](https://github.com/mozilla-rally/rally-core-addon/pull/505#issuecomment-815826426)||never | | + +## events + +This is a built-in ping that is assembled out of the box by the Glean SDK. + +See the Glean SDK documentation for the [`events` ping](https://mozilla.github.io/glean/book/user/pings/events.html). + +All Glean pings contain built-in metrics in the [`ping_info`](https://mozilla.github.io/glean/book/user/pings/index.html#the-ping_info-section) and [`client_info`](https://mozilla.github.io/glean/book/user/pings/index.html#the-client_info-section) sections. + +In addition to those built-in metrics, the following metrics are added to the ping: + +| Name | Type | Description | Data reviews | Extras | Expiration | [Data Sensitivity](https://wiki.mozilla.org/Firefox/Data_Collection) | +| --- | --- | --- | --- | --- | --- | --- | +| rally.id |[uuid](https://mozilla.github.io/glean/book/user/metrics/uuid.html) |The id of the Rally client. |[mozilla-rally/rally-core-addon#505](https://github.com/mozilla-rally/rally-core-addon/pull/505#issuecomment-815826426)||never | | + +## study-enrollment + +This ping is sent when user clicks the Join button for a study +and accepts the study policy. + + +This ping is sent if empty. + +**Data reviews for this ping:** + +- + +**Bugs related to this ping:** + +- + +All Glean pings contain built-in metrics in the [`ping_info`](https://mozilla.github.io/glean/book/user/pings/index.html#the-ping_info-section) and [`client_info`](https://mozilla.github.io/glean/book/user/pings/index.html#the-client_info-section) sections. + +In addition to those built-in metrics, the following metrics are added to the ping: + +| Name | Type | Description | Data reviews | Extras | Expiration | [Data Sensitivity](https://wiki.mozilla.org/Firefox/Data_Collection) | +| --- | --- | --- | --- | --- | --- | --- | +| enrollment.schema_namespace |[string](https://mozilla.github.io/glean/book/user/metrics/string.html) |The schema namespace for the study the user has joined. |[Bug 1663857](https://bugzilla.mozilla.org/show_bug.cgi?id=1663857#c5)||never | | +| enrollment.study_id |[string](https://mozilla.github.io/glean/book/user/metrics/string.html) |The id of the study user has joined. |[Bug 1663857](https://bugzilla.mozilla.org/show_bug.cgi?id=1663857#c5)||never | | +| rally.id |[uuid](https://mozilla.github.io/glean/book/user/metrics/uuid.html) |The id of the Rally client. |[mozilla-rally/rally-core-addon#505](https://github.com/mozilla-rally/rally-core-addon/pull/505#issuecomment-815826426)||never | | + +## study-unenrollment + +This ping is sent when user leaves a study. + + +This ping is sent if empty. **Data reviews for this ping:** -- +- **Bugs related to this ping:** -- +- + +All Glean pings contain built-in metrics in the [`ping_info`](https://mozilla.github.io/glean/book/user/pings/index.html#the-ping_info-section) and [`client_info`](https://mozilla.github.io/glean/book/user/pings/index.html#the-client_info-section) sections. + +In addition to those built-in metrics, the following metrics are added to the ping: -**Reasons this ping may be sent:** +| Name | Type | Description | Data reviews | Extras | Expiration | [Data Sensitivity](https://wiki.mozilla.org/Firefox/Data_Collection) | +| --- | --- | --- | --- | --- | --- | --- | +| rally.id |[uuid](https://mozilla.github.io/glean/book/user/metrics/uuid.html) |The id of the Rally client. |[mozilla-rally/rally-core-addon#505](https://github.com/mozilla-rally/rally-core-addon/pull/505#issuecomment-815826426)||never | | +| unenrollment.schema_namespace |[string](https://mozilla.github.io/glean/book/user/metrics/string.html) |The schema namespace for the study the user has left. |[Bug 1646151](https://bugzilla.mozilla.org/show_bug.cgi?id=1646151#c32)||never | | +| unenrollment.study_id |[string](https://mozilla.github.io/glean/book/user/metrics/string.html) |The id of the study user has left. |[Bug 1646151](https://bugzilla.mozilla.org/show_bug.cgi?id=1646151#c32)||never | | -- `attention`: An attention event is an instance where the user - was actively using the browser in an active tab - in an active window. +## unenrollment -- `audio`: An audio event tells us when an active browser - tab has audio playing. We use this as a proxy - for a user passively consuming audio and video. +This ping is sent when and if the user chooses to leave the platform. + + +This ping is sent if empty. + +**Data reviews for this ping:** + +- + +**Bugs related to this ping:** + +- All Glean pings contain built-in metrics in the [`ping_info`](https://mozilla.github.io/glean/book/user/pings/index.html#the-ping_info-section) and [`client_info`](https://mozilla.github.io/glean/book/user/pings/index.html#the-client_info-section) sections. In addition to those built-in metrics, the following metrics are added to the ping: -| Name | Type | Description | Data reviews | Extras | Expiration | [Data Sensitivity](https://wiki.mozilla.org/Firefox/Data_Collection) | -| ---------------------------------------- | --------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------- | ---------------------------------- | ---------- | -------------------------------------------------------------------- | -| event.duration | [timespan](https://mozilla.github.io/glean/book/user/metrics/timespan.html) | How long the event occurred. | [Bug 1703279](https://bugzilla.mozilla.org/show_bug.cgi?id=1703279#c2) | | never | | -| event.start | [datetime](https://mozilla.github.io/glean/book/user/metrics/datetime.html) | Noting when the event started. If sent in a ping with reason `attention`, this field notes when an inactive tab with a page loaded in it has been given active focus or a new page loads in an already-active tab. If otherwise sent in a ping with reason `audio`, this field notes when an unmuted audio element began playing in the active tab. | [Bug 1703279](https://bugzilla.mozilla.org/show_bug.cgi?id=1703279#c2) | | never | | -| event.stop | [datetime](https://mozilla.github.io/glean/book/user/metrics/datetime.html) | Noting when the event ended. If sent in a ping with reason `attention`, this field notes when a user closed the active tab, switched or closed the active window, or loaded a new page into the active tab which ends the current attention event. If otherwise sent in a ping with reason `audio`, this field notes when an unmuted audio element stopped playing in the active tab. | [Bug 1703279](https://bugzilla.mozilla.org/show_bug.cgi?id=1703279#c2) | | never | | -| event.termination_reason | [string](https://mozilla.github.io/glean/book/user/metrics/string.html) | The reason the user’s attention switched to the current attention event (e.g. changed a tab, loaded a new URL in the currently-active tab, closed a tab, closed a window, created a new tab, created a new window, stopped playing audio). | [Bug 1703279](https://bugzilla.mozilla.org/show_bug.cgi?id=1703279#c2) | | never | | -| page.attention.max_pixel_scroll_depth | [quantity](https://mozilla.github.io/glean/book/user/metrics/quantity.html) | The largest scroll pixel depth reached on the page. | [Bug 1703279](https://bugzilla.mozilla.org/show_bug.cgi?id=1703279#c2) |
  • unit: pixels
| never | | -| page.attention.max_relative_scroll_depth | [quantity](https://mozilla.github.io/glean/book/user/metrics/quantity.html) | The largest depth reach on the page, as a proportion of the total page height. | [Bug 1703279](https://bugzilla.mozilla.org/show_bug.cgi?id=1703279#c2) |
  • unit: percentage
| never | | -| page.attention.scroll_height | [quantity](https://mozilla.github.io/glean/book/user/metrics/quantity.html) | The total scroll height of the page, taken from `document.documentElement.scrollHeight` at the same interval as the other scroll fields. | [Bug 1703279](https://bugzilla.mozilla.org/show_bug.cgi?id=1703279#c2) |
  • unit: pixels
| never | | -| page.id | [string](https://mozilla.github.io/glean/book/user/metrics/string.html) | A unique ID associated with a page visit. Each page ID is 128-bit value, randomly generated with the Web Crypto API and stored as a hexadecimal string. | [Bug 1703279](https://bugzilla.mozilla.org/show_bug.cgi?id=1703279#c2) | | never | | -| page.og_description | [string](https://mozilla.github.io/glean/book/user/metrics/string.html) | The `og:description` meta tag contents (e.g. ``). If this isn't supplied, then attempts to look at the meta description contents (e.g. ``). | [Bug 1703279](https://bugzilla.mozilla.org/show_bug.cgi?id=1703279#c2) | | never | | -| page.og_type | [string](https://mozilla.github.io/glean/book/user/metrics/string.html) | The `og:type` meta tag contents (e.g. ``). | [Bug 1703279](https://bugzilla.mozilla.org/show_bug.cgi?id=1703279#c2) | | never | | -| page.origin | [string](https://mozilla.github.io/glean/book/user/metrics/string.html) | The origin of the URL associated with the page visit. Calculated by applying `new URL(url).origin`. See the documentation for [url.origin](https://developer.mozilla.org/en-US/docs/Web/API/URL/origin). | [Bug 1703279](https://bugzilla.mozilla.org/show_bug.cgi?id=1703279#c2) | | never | | -| page.referrer_origin | [string](https://mozilla.github.io/glean/book/user/metrics/string.html) | The origin of the referrer URL for the page loading in the tab. | [Bug 1703279](https://bugzilla.mozilla.org/show_bug.cgi?id=1703279#c2) | | never | | -| page.title | [string](https://mozilla.github.io/glean/book/user/metrics/string.html) | The contents of the title element in the head of the page. | [Bug 1703279](https://bugzilla.mozilla.org/show_bug.cgi?id=1703279#c2) | | never | | -| page.visit.start | [datetime](https://mozilla.github.io/glean/book/user/metrics/datetime.html) | When did the page visit start. | [Bug 1703279](https://bugzilla.mozilla.org/show_bug.cgi?id=1703279#c2) | | never | | -| page.visit.stop | [datetime](https://mozilla.github.io/glean/book/user/metrics/datetime.html) | When did the page visit stop. | [Bug 1703279](https://bugzilla.mozilla.org/show_bug.cgi?id=1703279#c2) | | never | | +| Name | Type | Description | Data reviews | Extras | Expiration | [Data Sensitivity](https://wiki.mozilla.org/Firefox/Data_Collection) | +| --- | --- | --- | --- | --- | --- | --- | +| rally.id |[uuid](https://mozilla.github.io/glean/book/user/metrics/uuid.html) |The id of the Rally client. |[mozilla-rally/rally-core-addon#505](https://github.com/mozilla-rally/rally-core-addon/pull/505#issuecomment-815826426)||never | | Data categories are [defined here](https://wiki.mozilla.org/Firefox/Data_Collection). + diff --git a/functions/package-lock.json b/functions/package-lock.json index 92c8c8a2..cf3bb8ab 100644 --- a/functions/package-lock.json +++ b/functions/package-lock.json @@ -4,8 +4,8 @@ "requires": true, "packages": { "": { - "name": "functions", "dependencies": { + "@mozilla/glean": "^1.0.0", "cors": "^2.8.5", "firebase-admin": "^9.8.0", "firebase-functions": "^3.15.5" @@ -1576,6 +1576,32 @@ "@jridgewell/sourcemap-codec": "^1.4.10" } }, + "node_modules/@mozilla/glean": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@mozilla/glean/-/glean-1.0.0.tgz", + "integrity": "sha512-2RzkubrxaCV7mkmCXgBmD16XbDuK4SVqlMdLv3zez2lb3WXnLo6j+C+IKIgBke/f/iGgz6O4SISjTAhkaPvgNQ==", + "dependencies": { + "fflate": "^0.7.1", + "jose": "^4.0.4", + "tslib": "^2.3.1", + "uuid": "^8.3.2" + }, + "bin": { + "glean": "dist/cli/cli.js" + }, + "engines": { + "node": ">=12.20.0", + "npm": ">=7.0.0" + } + }, + "node_modules/@mozilla/glean/node_modules/jose": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/jose/-/jose-4.6.0.tgz", + "integrity": "sha512-0hNAkhMBNi4soKSAX4zYOFV+aqJlEz/4j4fregvasJzEVtjDChvWqRjPvHwLqr5hx28Ayr6bsOs1Kuj87V0O8w==", + "funding": { + "url": "https://github.com/sponsors/panva" + } + }, "node_modules/@panva/asn1.js": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/@panva/asn1.js/-/asn1.js-1.0.0.tgz", @@ -3744,6 +3770,11 @@ "bser": "2.1.1" } }, + "node_modules/fflate": { + "version": "0.7.3", + "resolved": "https://registry.npmjs.org/fflate/-/fflate-0.7.3.tgz", + "integrity": "sha512-0Zz1jOzJWERhyhsimS54VTqOteCNwRtIlh8isdL0AXLo0g7xNTfTL7oWrkmCnPhZGocKIkWHBistBrrpoNH3aw==" + }, "node_modules/file-entry-cache": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-6.0.1.tgz", @@ -7833,7 +7864,6 @@ "version": "8.3.2", "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", - "devOptional": true, "bin": { "uuid": "dist/bin/uuid" } @@ -9372,6 +9402,24 @@ "@jridgewell/sourcemap-codec": "^1.4.10" } }, + "@mozilla/glean": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@mozilla/glean/-/glean-1.0.0.tgz", + "integrity": "sha512-2RzkubrxaCV7mkmCXgBmD16XbDuK4SVqlMdLv3zez2lb3WXnLo6j+C+IKIgBke/f/iGgz6O4SISjTAhkaPvgNQ==", + "requires": { + "fflate": "^0.7.1", + "jose": "^4.0.4", + "tslib": "^2.3.1", + "uuid": "^8.3.2" + }, + "dependencies": { + "jose": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/jose/-/jose-4.6.0.tgz", + "integrity": "sha512-0hNAkhMBNi4soKSAX4zYOFV+aqJlEz/4j4fregvasJzEVtjDChvWqRjPvHwLqr5hx28Ayr6bsOs1Kuj87V0O8w==" + } + } + }, "@panva/asn1.js": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/@panva/asn1.js/-/asn1.js-1.0.0.tgz", @@ -11098,6 +11146,11 @@ "bser": "2.1.1" } }, + "fflate": { + "version": "0.7.3", + "resolved": "https://registry.npmjs.org/fflate/-/fflate-0.7.3.tgz", + "integrity": "sha512-0Zz1jOzJWERhyhsimS54VTqOteCNwRtIlh8isdL0AXLo0g7xNTfTL7oWrkmCnPhZGocKIkWHBistBrrpoNH3aw==" + }, "file-entry-cache": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-6.0.1.tgz", @@ -14254,8 +14307,7 @@ "uuid": { "version": "8.3.2", "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", - "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", - "devOptional": true + "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==" }, "v8-compile-cache": { "version": "2.3.0", diff --git a/functions/package.json b/functions/package.json index 29d9fa0e..edbae5d7 100644 --- a/functions/package.json +++ b/functions/package.json @@ -2,11 +2,14 @@ "name": "functions", "scripts": { "lint": "eslint --ext .js,.ts .", - "build": "tsc", + "build": "npm run build:glean && tsc", + "build:glean": "glean translate ../metrics.yaml ../pings.yaml -f typescript -o ./src/generated", "serve": "npm run build && firebase emulators:start --only functions", "shell": "npm run build && firebase functions:shell", "start": "npm run shell", "deploy": "firebase deploy --only functions", + "docs:glean": "glean translate ../metrics.yaml ../pings.yaml -f markdown -o ../docs", + "lint:glean": "glean glinter ../metrics.yaml ../pings.yaml", "logs": "firebase functions:log", "test": "set -a && . ./.testenv && jest --coverage --detectOpenHandles && set +a", "test:coverage": "set -a && . ./.testenv && jest --coverage --collectCoverage --coverageDirectory coverage --detectOpenHandles && set +a", @@ -17,6 +20,7 @@ }, "main": "lib/index.js", "dependencies": { + "@mozilla/glean": "^1.0.0", "cors": "^2.8.5", "firebase-admin": "^9.8.0", "firebase-functions": "^3.15.5" diff --git a/metrics.yaml b/metrics.yaml new file mode 100644 index 00000000..2c7a27a8 --- /dev/null +++ b/metrics.yaml @@ -0,0 +1,247 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# This file defines the metrics that are recorded by the Glean SDK. +# APIs to use these pings are automatically generated at build time using +# the `glean_parser` PyPI package. + +# Metrics in this file may make use of SDK reserved ping names. See +# https://mozilla.github.io/glean/book/dev/core/internal/reserved-ping-names.html +# for additional information. + +--- +$schema: moz://mozilla.org/schemas/glean/metrics/2-0-0 + +rally: + id: + type: uuid + lifetime: user + send_in_pings: + - deletion-request + - enrollment + - unenrollment + - study-enrollment + - study-unenrollment + - demographics + - events + description: | + The id of the Rally client. + bugs: + - https://github.com/mozilla-rally/rally-core-addon/issues/117 + data_reviews: + - https://github.com/mozilla-rally/rally-core-addon/pull/505#issuecomment-815826426 + notification_emails: + - than@mozilla.com + expires: never + +enrollment: + study_id: + type: string + lifetime: ping + send_in_pings: + - study-enrollment + description: | + The id of the study user has joined. + bugs: + - https://github.com/mozilla-rally/rally-core-addon/issues/545 + data_reviews: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1663857#c5 + notification_emails: + - than@mozilla.com + expires: never + + schema_namespace: + type: string + lifetime: ping + send_in_pings: + - study-enrollment + description: | + The schema namespace for the study the user has joined. + bugs: + - https://github.com/mozilla-rally/rally-core-addon/issues/545 + data_reviews: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1663857#c5 + notification_emails: + - than@mozilla.com + expires: never + +unenrollment: + study_id: + type: string + lifetime: ping + send_in_pings: + - study-unenrollment + description: | + The id of the study user has left. + bugs: + - https://github.com/mozilla-rally/rally-core-addon/issues/545 + data_reviews: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1646151#c32 + notification_emails: + - than@mozilla.com + expires: never + + schema_namespace: + type: string + lifetime: ping + send_in_pings: + - study-unenrollment + description: | + The schema namespace for the study the user has left. + bugs: + - https://github.com/mozilla-rally/rally-core-addon/issues/545 + data_reviews: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1646151#c32 + notification_emails: + - than@mozilla.com + expires: never + +user: + age: + type: labeled_boolean + lifetime: ping + send_in_pings: + - demographics + labels: + - band_19_24 + - band_25_34 + - band_35_44 + - band_45_54 + - band_55_64 + - band_over_65 + description: | + The user age. + bugs: + - https://github.com/mozilla-rally/rally-core-addon/issues/545 + data_reviews: + - https://github.com/mozilla-rally/rally-core-addon/pull/139#issuecomment-736024232 + notification_emails: + - than@mozilla.com + expires: never + + gender: + type: labeled_boolean + lifetime: ping + send_in_pings: + - demographics + labels: + - male + - female + - neither + - decline + description: | + The user gender. + bugs: + - https://github.com/mozilla-rally/rally-core-addon/issues/545 + data_reviews: + - https://github.com/mozilla-rally/rally-core-addon/pull/139#issuecomment-736024232 + notification_emails: + - than@mozilla.com + expires: never + + origin: + type: labeled_boolean + lifetime: ping + send_in_pings: + - demographics + labels: + - hispanic_latinx_spanish + - other + description: | + The user origin: Hispanic, Latinx, Spanish or other. + bugs: + - https://github.com/mozilla-rally/rally-core-addon/issues/545 + data_reviews: + - https://github.com/mozilla-rally/rally-core-addon/pull/139#issuecomment-736024232 + notification_emails: + - than@mozilla.com + expires: never + + races: + type: labeled_boolean + lifetime: ping + send_in_pings: + - demographics + labels: + - am_indian_or_alaska_native + - asian_indian + - black_or_african_american + - chamorro + - chinese + - filipino + - japanese + - korean + - native_hawaiian + - samoan + - vietnamese + - white + - other_asian + - other_pacific_islander + - some_other_race + description: | + The user race / ethnicity. + bugs: + - https://github.com/mozilla-rally/rally-core-addon/issues/545 + data_reviews: + - https://github.com/mozilla-rally/rally-core-addon/pull/139#issuecomment-736024232 + notification_emails: + - than@mozilla.com + expires: never + + school: + type: labeled_boolean + lifetime: ping + send_in_pings: + - demographics + labels: + - less_than_high_school + - some_high_school + - high_school_grad_or_eq + - college_degree_in_progress + - associates_degree + - bachelors_degree + - graduate_degree + description: | + The highest level of school user has completed. + bugs: + - https://github.com/mozilla-rally/rally-core-addon/issues/545 + data_reviews: + - https://github.com/mozilla-rally/rally-core-addon/pull/139#issuecomment-736024232 + notification_emails: + - than@mozilla.com + expires: never + + exact_income: + type: quantity + lifetime: ping + unit: US Dollars + send_in_pings: + - demographics + description: | + The user household's combined annual income during the + past 12 months. This field replaces the previous income field. + bugs: + - https://github.com/mozilla-rally/rally-core-addon/issues/545 + - https://github.com/mozilla-rally/rally-core-addon/issues/621 + data_reviews: + - https://github.com/mozilla-rally/rally-core-addon/pull/139#issuecomment-736024232 + - https://github.com/mozilla-rally/rally-core-addon/pull/624#issuecomment-850479051 + notification_emails: + - than@mozilla.com + expires: never + + zipcode: + type: string + lifetime: ping + send_in_pings: + - demographics + description: | + The user zip code. + bugs: + - https://github.com/mozilla-rally/rally-core-addon/issues/545 + data_reviews: + - https://github.com/mozilla-rally/rally-core-addon/pull/139#issuecomment-736024232 + notification_emails: + - than@mozilla.com + expires: never diff --git a/pings.yaml b/pings.yaml new file mode 100644 index 00000000..942c6651 --- /dev/null +++ b/pings.yaml @@ -0,0 +1,75 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# This file defines the pings that are recorded by the Glean SDK. +# Their code APIs is automatically generated, at build time using, +# the `glean_parser` PyPI package. + +--- +$schema: moz://mozilla.org/schemas/glean/pings/2-0-0 + +enrollment: + description: | + This ping is sent at the end of the user onboarding process, + when and if the user joins the platform. + include_client_id: false + send_if_empty: true + bugs: + - https://github.com/mozilla-rally/rally-core-addon/issues/117 + data_reviews: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1663857#c5 + notification_emails: + - than@mozilla.com + +unenrollment: + description: | + This ping is sent when and if the user chooses to leave the platform. + include_client_id: false + send_if_empty: true + bugs: + - https://github.com/mozilla-rally/rally-core-addon/issues/117 + data_reviews: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1663857#c5 + notification_emails: + - than@mozilla.com + +study-enrollment: + description: | + This ping is sent when user clicks the Join button for a study + and accepts the study policy. + include_client_id: false + send_if_empty: true + bugs: + - https://github.com/mozilla-rally/rally-core-addon/issues/117 + data_reviews: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1663857#c5 + notification_emails: + - than@mozilla.com + +study-unenrollment: + description: | + This ping is sent when user leaves a study. + include_client_id: false + send_if_empty: true + bugs: + - https://github.com/mozilla-rally/rally-core-addon/issues/545 + data_reviews: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1646151#c32 + notification_emails: + - than@mozilla.com + +demographics: + description: | + After a user joins the platform they are asked to fill a + demographic survey, in order to help researchers parse the + data. The survey is optional and can be partially filled: this + ping is submitted right after the survey is filled. + include_client_id: false + send_if_empty: true + bugs: + - https://github.com/mozilla-rally/rally-core-addon/issues/545 + data_reviews: + - https://github.com/mozilla-rally/rally-core-addon/pull/139#issuecomment-736024232 + notification_emails: + - than@mozilla.com From 92f37de1761dfd4a285b12f4b2a538e646926ce7 Mon Sep 17 00:00:00 2001 From: Ashwin Agarwal Date: Thu, 31 Mar 2022 16:13:23 -0400 Subject: [PATCH 02/27] Add triggered function stubs (no Glean pings yet) --- functions/package-lock.json | 1 + functions/src/index.ts | 98 +++++++++++++++++++++++++++++++++++++ 2 files changed, 99 insertions(+) diff --git a/functions/package-lock.json b/functions/package-lock.json index cf3bb8ab..141b0f9a 100644 --- a/functions/package-lock.json +++ b/functions/package-lock.json @@ -4,6 +4,7 @@ "requires": true, "packages": { "": { + "name": "functions", "dependencies": { "@mozilla/glean": "^1.0.0", "cors": "^2.8.5", diff --git a/functions/src/index.ts b/functions/src/index.ts index 78b69f8c..2ef5d5d1 100644 --- a/functions/src/index.ts +++ b/functions/src/index.ts @@ -1,9 +1,12 @@ import * as admin from "firebase-admin"; import * as functions from "firebase-functions"; +import { Change, EventContext } from "firebase-functions"; +import { DocumentSnapshot } from "firebase-functions/v1/firestore"; import { v4 as uuidv4 } from "uuid"; import { useAuthentication } from "./authentication"; import { useCors } from "./cors"; import { studies } from "./studies"; +import { isDeepStrictEqual } from "util"; admin.initializeApp({ credential: admin.credential.applicationDefault(), @@ -166,3 +169,98 @@ export const loadFirestore = functions.https.onRequest( response.status(200).send(); } ); + +/* + * Listen for changes to the User document + * and initiate the appropriate Glean ping(s) + */ +export const handleUserChangesImpl = async function ( + change: Change, + context: EventContext +): Promise { + const userID = context.params.userID; + // Get an object with the current document value. + // If the document does not exist, it has been deleted. + const newUser = change.after.exists ? change.after.data() : null; + + // Get the old document, to compare the enrollment state. + const oldUser = change.before.exists ? change.before.data() : null; + + if (!newUser || (oldUser && oldUser.enrolled === true && !newUser.enrolled)) { + // User document has been deleted + functions.logger.info( + `Sending deletion and unenrollment pings for user ID ${userID}` + ); + // TODO send Glean pings + return true; + } + + if ((!oldUser || !oldUser.enrolled) && newUser.enrolled === true) { + // User just enrolled + functions.logger.info(`Sending enrollment ping for user ID ${userID}`); + // TODO send Glean ping + } + + if ( + ((!oldUser || !oldUser.demographicsData) && newUser.demographicsData) || + (oldUser && oldUser.demographicsData && !newUser.demographicsData) || + (oldUser && + newUser && + !isDeepStrictEqual(oldUser.demographicsData, newUser.demographicsData)) + ) { + // User updated demographicsData + functions.logger.info(`Sending demographics ping for user ID ${userID}`); + // TODO send Glean ping + } + + return true; +}; + +exports.handleUserChanges = functions.firestore + .document("users/{userID}") + .onWrite(handleUserChangesImpl); + +/* + * Listen for changes to the Study document + * and initiate the appropriate Glean ping(s) + */ +export const handleUserStudyChangesImpl = async function ( + change: Change, + context: EventContext +): Promise { + const userID = context.params.userID; + const studyID = context.params.studyID; + // Get an object with the current document value. + // If the document does not exist, it has been deleted. + const newStudy = change.after.exists ? change.after.data() : null; + + // Get the old document, to compare the enrollment state. + const oldStudy = change.before.exists ? change.before.data() : null; + + if ( + !newStudy || + (oldStudy && oldStudy.enrolled === true && !newStudy.enrolled) + ) { + // User unenrolled from study + functions.logger.info( + `Sending deletion and unenrollment pings for study with user ID ${userID} with study ID ${studyID}` + ); + // TODO send Glean pings + return true; + } + + if ((!oldStudy || !oldStudy.enrolled) && newStudy.enrolled === true) { + // User just enrolled in this study + functions.logger.info( + `Sending enrollment ping for study with user ID ${userID} with study ID ${studyID}` + ); + // TODO send Glean ping + return true; + } + + return true; +}; + +exports.handleUserStudyChanges = functions.firestore + .document("users/{userID}/studies/{studyID}") + .onWrite(handleUserStudyChangesImpl); From 8efa6ff25f9cc26ff4d2419971950fc1e1bd31a0 Mon Sep 17 00:00:00 2001 From: Ashwin Agarwal Date: Thu, 31 Mar 2022 17:36:13 -0400 Subject: [PATCH 03/27] Add getRallyID function, fix some function naming --- functions/src/__tests__/index.test.ts | 6 ++--- functions/src/index.ts | 38 +++++++++++++++++++++++++-- 2 files changed, 39 insertions(+), 5 deletions(-) diff --git a/functions/src/__tests__/index.test.ts b/functions/src/__tests__/index.test.ts index f5408149..4bbc20a7 100644 --- a/functions/src/__tests__/index.test.ts +++ b/functions/src/__tests__/index.test.ts @@ -4,7 +4,7 @@ import * as admin from "firebase-admin"; import { useCors } from "../cors"; import * as functions from "firebase-functions"; import { - addRallyStudyToFirestoreImpl, + addRallyUserToFirestoreImpl, deleteRallyUserImpl, loadFirestore, rallytoken, @@ -82,7 +82,7 @@ describe("addRallyUserToFirestore and deleteRallyUserImpl", () => { it("empty provider data does not register extension users", async () => { await expect( - addRallyStudyToFirestoreImpl({ ...user, providerData: [] }) + addRallyUserToFirestoreImpl({ ...user, providerData: [] }) ).resolves.toBeFalsy(); const userRecords = await getUserRecords(); @@ -93,7 +93,7 @@ describe("addRallyUserToFirestore and deleteRallyUserImpl", () => { async function createAndValidateUserRecords() { await expect( - addRallyStudyToFirestoreImpl({ + addRallyUserToFirestoreImpl({ ...user, providerData: [{ uid: user.uid } as admin.auth.UserInfo], }) diff --git a/functions/src/index.ts b/functions/src/index.ts index 2ef5d5d1..a9edad5e 100644 --- a/functions/src/index.ts +++ b/functions/src/index.ts @@ -66,7 +66,7 @@ async function generateToken( return rallyToken; } -export const addRallyStudyToFirestoreImpl = async ( +export const addRallyUserToFirestoreImpl = async ( user: admin.auth.UserRecord ): Promise => { functions.logger.info("addRallyUserToFirestore - onCreate fired for user", { @@ -102,7 +102,7 @@ export const addRallyStudyToFirestoreImpl = async ( exports.addRallyUserToFirestore = functions.auth .user() - .onCreate(addRallyStudyToFirestoreImpl); + .onCreate(addRallyUserToFirestoreImpl); export const deleteRallyUserImpl = async function ( user: admin.auth.UserRecord @@ -179,6 +179,15 @@ export const handleUserChangesImpl = async function ( context: EventContext ): Promise { const userID = context.params.userID; + const rallyID = await getRallyIdForUser(userID); + if (!rallyID) { + // Without Rally ID, we can't make any Glean pings + // This is bad and should be flagged for inspection + throw new Error( + `Unable to obtain Rally ID for user ID ${userID}. Aborting Glean ping process.` + ); + } + // Get an object with the current document value. // If the document does not exist, it has been deleted. const newUser = change.after.exists ? change.after.data() : null; @@ -229,6 +238,15 @@ export const handleUserStudyChangesImpl = async function ( context: EventContext ): Promise { const userID = context.params.userID; + const rallyID = await getRallyIdForUser(userID); + if (!rallyID) { + // Without Rally ID, we can't make any Glean pings + // This is bad and should be flagged for inspection + throw new Error( + `Unable to obtain Rally ID for user ID ${userID}. Aborting Glean ping process.` + ); + } + const studyID = context.params.studyID; // Get an object with the current document value. // If the document does not exist, it has been deleted. @@ -264,3 +282,19 @@ export const handleUserStudyChangesImpl = async function ( exports.handleUserStudyChanges = functions.firestore .document("users/{userID}/studies/{studyID}") .onWrite(handleUserStudyChangesImpl); + +async function getRallyIdForUser(userID: string) { + let extensionUserDoc = await admin + .firestore() + .collection("extensionUsers") + .doc(userID) + .get(); + + const data = extensionUserDoc.data(); + + if (data) { + return data.rallyId; + } else { + return null; + } +} From 1813e86b5a0ef5be2f037699f39d8b72441908c6 Mon Sep 17 00:00:00 2001 From: Ashwin Agarwal Date: Fri, 15 Apr 2022 14:11:10 -0400 Subject: [PATCH 04/27] Port /functions from CommonJS to ESM --- functions/{.eslintrc.js => .eslintrc.cjs} | 0 functions/package.json | 8 ++++++-- functions/src/authentication.ts | 6 +++--- functions/src/cors.ts | 2 +- functions/src/index.ts | 16 ++++++++-------- functions/tsconfig.json | 17 +++++++++++++++-- package.json | 2 +- 7 files changed, 34 insertions(+), 17 deletions(-) rename functions/{.eslintrc.js => .eslintrc.cjs} (100%) diff --git a/functions/.eslintrc.js b/functions/.eslintrc.cjs similarity index 100% rename from functions/.eslintrc.js rename to functions/.eslintrc.cjs diff --git a/functions/package.json b/functions/package.json index edbae5d7..158446a3 100644 --- a/functions/package.json +++ b/functions/package.json @@ -1,5 +1,6 @@ { "name": "functions", + "type": "module", "scripts": { "lint": "eslint --ext .js,.ts .", "build": "npm run build:glean && tsc", @@ -13,7 +14,8 @@ "logs": "firebase functions:log", "test": "set -a && . ./.testenv && jest --coverage --detectOpenHandles && set +a", "test:coverage": "set -a && . ./.testenv && jest --coverage --collectCoverage --coverageDirectory coverage --detectOpenHandles && set +a", - "test:watch": "set -a && . ./.testenv && jest --watch && set +a" + "test:watch": "set -a && . ./.testenv && jest --watch && set +a", + "prepare": "ts-patch install -s" }, "engines": { "node": "14" @@ -36,7 +38,9 @@ "eslint-plugin-import": "^2.22.0", "jest": "^27.5.1", "ts-jest": "^27.1.3", - "typescript": "^3.8.0", + "ts-patch": "^2.0.1", + "ts-transform-esm-import": "^0.9.0", + "typescript": "^4.6.3", "uuid": "^8.3.2" }, "private": true diff --git a/functions/src/authentication.ts b/functions/src/authentication.ts index ccd22a15..bd8e37ba 100644 --- a/functions/src/authentication.ts +++ b/functions/src/authentication.ts @@ -1,6 +1,6 @@ -import * as assert from "assert"; -import * as admin from "firebase-admin"; -import * as functions from "firebase-functions"; +import assert from "assert"; +import admin from "firebase-admin"; +import functions from "firebase-functions"; export type AuthenticatedFunction = ( decodedToken: admin.auth.DecodedIdToken diff --git a/functions/src/cors.ts b/functions/src/cors.ts index 582a3a75..1b19e11d 100644 --- a/functions/src/cors.ts +++ b/functions/src/cors.ts @@ -1,4 +1,4 @@ -import * as cors from "cors"; +import cors from "cors"; export const useCors = cors({ origin: true, diff --git a/functions/src/index.ts b/functions/src/index.ts index a9edad5e..959cccc4 100644 --- a/functions/src/index.ts +++ b/functions/src/index.ts @@ -1,5 +1,5 @@ -import * as admin from "firebase-admin"; -import * as functions from "firebase-functions"; +import admin from "firebase-admin"; +import functions from "firebase-functions"; import { Change, EventContext } from "firebase-functions"; import { DocumentSnapshot } from "firebase-functions/v1/firestore"; import { v4 as uuidv4 } from "uuid"; @@ -100,7 +100,7 @@ export const addRallyUserToFirestoreImpl = async ( return true; }; -exports.addRallyUserToFirestore = functions.auth +export const addRallyUserToFirestore = functions.auth .user() .onCreate(addRallyUserToFirestoreImpl); @@ -142,7 +142,7 @@ export const deleteRallyUserImpl = async function ( return true; }; -exports.deleteRallyUser = functions.auth.user().onDelete(deleteRallyUserImpl); +export const deleteRallyUser = functions.auth.user().onDelete(deleteRallyUserImpl); /** * @@ -177,7 +177,7 @@ export const loadFirestore = functions.https.onRequest( export const handleUserChangesImpl = async function ( change: Change, context: EventContext -): Promise { +): Promise { const userID = context.params.userID; const rallyID = await getRallyIdForUser(userID); if (!rallyID) { @@ -225,7 +225,7 @@ export const handleUserChangesImpl = async function ( return true; }; -exports.handleUserChanges = functions.firestore +export const handleUserChanges = functions.firestore .document("users/{userID}") .onWrite(handleUserChangesImpl); @@ -236,7 +236,7 @@ exports.handleUserChanges = functions.firestore export const handleUserStudyChangesImpl = async function ( change: Change, context: EventContext -): Promise { +): Promise { const userID = context.params.userID; const rallyID = await getRallyIdForUser(userID); if (!rallyID) { @@ -279,7 +279,7 @@ export const handleUserStudyChangesImpl = async function ( return true; }; -exports.handleUserStudyChanges = functions.firestore +export const handleUserStudyChanges = functions.firestore .document("users/{userID}/studies/{studyID}") .onWrite(handleUserStudyChangesImpl); diff --git a/functions/tsconfig.json b/functions/tsconfig.json index cc2d5241..c6454c4a 100644 --- a/functions/tsconfig.json +++ b/functions/tsconfig.json @@ -1,13 +1,26 @@ { "compilerOptions": { - "module": "commonjs", + "module": "esnext", + "moduleResolution": "node", + "allowSyntheticDefaultImports": true, "noImplicitReturns": true, "noUnusedLocals": true, "outDir": "lib", "sourceMap": true, "strict": true, "target": "es2017", - "typeRoots": ["node_modules/@types"] + "typeRoots": ["node_modules/@types"], + "plugins": [ + { + "transform": "ts-transform-esm-import", + "after": true, + "afterDeclarations": true, + "type": "config", + "rootDir": "./src", + "outDir": "./lib", + "resolvers": [{ "dir": "./src" }] + } + ] }, "compileOnSave": true, "include": ["src"] diff --git a/package.json b/package.json index cdc65c18..48d3484f 100644 --- a/package.json +++ b/package.json @@ -10,7 +10,7 @@ "build": "npm run pre-build && npm run build:functions && svelte-kit build && npm run compile:sass", "build:functions": "cd functions && npm install && npm run build && cd ..", "build:web:emulator": "npm run pre-build && npm run config:web:demo && svelte-kit build -- --config-emulator-mode", - "watch:functions": "watch 'npm run build:functions && sleep 1 && npm run load:data' ./functions/src > /dev/null", + "watch:functions": "watch 'npm run build:functions && sleep 1 && npm run load:data' ./functions/src --ignoreDirectoryPattern='/generated/g' > /dev/null", "watch:web": "svelte-kit dev -- --config-emulator-mode", "load:data": "curl -s http://localhost:5001/demo-rally/us-central1/loadFirestore", "preview": "svelte-kit preview", From 71cbd8f16dc3f2e8fd383436c9bfb50b22ee0bee Mon Sep 17 00:00:00 2001 From: Ashwin Agarwal Date: Mon, 18 Apr 2022 17:38:09 -0400 Subject: [PATCH 05/27] Fix existing Jest tests to use ESM --- functions/.testenv | 4 +- functions/jest.config.js | 18 +- functions/package-lock.json | 365 +++++++++++++++++- functions/package.json | 2 + .../src/__tests__/authentication.test.ts | 31 +- functions/src/__tests__/index.test.ts | 32 +- functions/src/cors.ts | 3 +- functions/src/index.ts | 70 ++-- functions/tsconfig.json | 2 + 9 files changed, 460 insertions(+), 67 deletions(-) diff --git a/functions/.testenv b/functions/.testenv index 96c733d8..5af1df66 100644 --- a/functions/.testenv +++ b/functions/.testenv @@ -1,2 +1,4 @@ GCLOUD_PROJECT="demo-rally" -FIRESTORE_EMULATOR_HOST="localhost:8080" \ No newline at end of file +FIRESTORE_EMULATOR_HOST="localhost:8080" +FIREBASE_EMULATOR_HUB="localhost:4400" +NODE_OPTIONS=--experimental-vm-modules \ No newline at end of file diff --git a/functions/jest.config.js b/functions/jest.config.js index 01e567e6..786ccb5a 100644 --- a/functions/jest.config.js +++ b/functions/jest.config.js @@ -1,7 +1,19 @@ -module.exports = { - preset: "ts-jest", +export default { + preset: "ts-jest/presets/default-esm", + globals: { + "ts-jest": { + useESM: true, + }, + }, + resolver: "ts-jest-resolver", + moduleNameMapper: { + "^@mozilla/glean/plugins/encryption": + "@mozilla/glean/dist/plugins/encryption.js", + "^@mozilla/glean/uploader": "@mozilla/glean/dist/core/upload/uploader.js", + }, + // extensionsToTreatAsEsm: [".ts"], testRegex: "src(/__tests__/.*|(\\.|/)(test|spec))\\.(jsx?|tsx?)$", testPathIgnorePatterns: ["lib/", "node_modules/", "setupTests.js"], moduleFileExtensions: ["js", "ts", "tsx", "jsx", "json", "node"], - testEnvironment: "node", + testEnvironment: "node", }; diff --git a/functions/package-lock.json b/functions/package-lock.json index 141b0f9a..c751731c 100644 --- a/functions/package-lock.json +++ b/functions/package-lock.json @@ -21,8 +21,12 @@ "eslint-config-google": "^0.14.0", "eslint-plugin-import": "^2.22.0", "jest": "^27.5.1", + "node-fetch": "^3.1.1", "ts-jest": "^27.1.3", - "typescript": "^3.8.0", + "ts-jest-resolver": "^2.0.0", + "ts-patch": "^2.0.1", + "ts-transform-esm-import": "^0.9.0", + "typescript": "^4.6.3", "uuid": "^8.3.2" }, "engines": { @@ -2850,6 +2854,15 @@ "integrity": "sha512-b0tGHbfegbhPJpxpiBPU2sCkigAqtM9O121le6bbOlgyV+NyGyCmVfJ6QW9eRjz8CpNfWEOYBIMIGRYkLwsIYg==", "dev": true }, + "node_modules/data-uri-to-buffer": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.0.tgz", + "integrity": "sha512-Vr3mLBA8qWmcuschSLAOogKgQ/Jwxulv3RNE4FXnYWRGujzrRWQI4m12fQqRkwX06C0KanhLr4hK+GydchZsaA==", + "dev": true, + "engines": { + "node": ">= 12" + } + }, "node_modules/data-urls": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-2.0.0.tgz", @@ -3771,6 +3784,29 @@ "bser": "2.1.1" } }, + "node_modules/fetch-blob": { + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.1.5.tgz", + "integrity": "sha512-N64ZpKqoLejlrwkIAnb9iLSA3Vx/kjgzpcDhygcqJ2KKjky8nCgUQ+dzXtbrLaWZGZNmNfQTsiQ0weZ1svglHg==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "paypal", + "url": "https://paypal.me/jimmywarting" + } + ], + "dependencies": { + "node-domexception": "^1.0.0", + "web-streams-polyfill": "^3.0.3" + }, + "engines": { + "node": "^12.20 || >= 14.13" + } + }, "node_modules/fflate": { "version": "0.7.3", "resolved": "https://registry.npmjs.org/fflate/-/fflate-0.7.3.tgz", @@ -3914,6 +3950,18 @@ "node": ">= 6" } }, + "node_modules/formdata-polyfill": { + "version": "4.0.10", + "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", + "integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==", + "dev": true, + "dependencies": { + "fetch-blob": "^3.1.2" + }, + "engines": { + "node": ">=12.20.0" + } + }, "node_modules/forwarded": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", @@ -4117,6 +4165,32 @@ "node": ">= 6" } }, + "node_modules/global-prefix": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/global-prefix/-/global-prefix-3.0.0.tgz", + "integrity": "sha512-awConJSVCHVGND6x3tmMaKcQvwXLhjdkmomy2W+Goaui8YPgYgXJZewhg3fWC+DlfqqQuWg8AwqjGTD2nAPVWg==", + "dev": true, + "dependencies": { + "ini": "^1.3.5", + "kind-of": "^6.0.2", + "which": "^1.3.1" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/global-prefix/node_modules/which": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz", + "integrity": "sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==", + "dev": true, + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "which": "bin/which" + } + }, "node_modules/globals": { "version": "13.11.0", "resolved": "https://registry.npmjs.org/globals/-/globals-13.11.0.tgz", @@ -4544,6 +4618,12 @@ "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", "devOptional": true }, + "node_modules/ini": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", + "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==", + "dev": true + }, "node_modules/internal-slot": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/internal-slot/-/internal-slot-1.0.3.tgz", @@ -4558,6 +4638,15 @@ "node": ">= 0.4" } }, + "node_modules/interpret": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/interpret/-/interpret-1.4.0.tgz", + "integrity": "sha512-agE4QfB2Lkp9uICn7BAqoscw4SZP9kTE2hxiFI3jBPmXJfdqiahTbUuKGsMoN2GtqL9AxhYioAcVvgsb1HvRbA==", + "dev": true, + "engines": { + "node": ">= 0.10" + } + }, "node_modules/ipaddr.js": { "version": "1.9.1", "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", @@ -5956,6 +6045,15 @@ "safe-buffer": "^5.0.1" } }, + "node_modules/kind-of": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/kleur": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/kleur/-/kleur-3.0.3.tgz", @@ -6292,6 +6390,43 @@ "node": ">= 0.6" } }, + "node_modules/node-domexception": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", + "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "github", + "url": "https://paypal.me/jimmywarting" + } + ], + "engines": { + "node": ">=10.5.0" + } + }, + "node_modules/node-fetch": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.1.1.tgz", + "integrity": "sha512-SMk+vKgU77PYotRdWzqZGTZeuFKlsJ0hu4KPviQKkfY+N3vn2MIzr0rvpnYpR8MtB3IEuhlEcuOLbGvLRlA+yg==", + "dev": true, + "dependencies": { + "data-uri-to-buffer": "^4.0.0", + "fetch-blob": "^3.1.3", + "formdata-polyfill": "^4.0.10" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/node-fetch" + } + }, "node_modules/node-forge": { "version": "0.10.0", "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-0.10.0.tgz", @@ -6898,6 +7033,18 @@ "node": ">= 6" } }, + "node_modules/rechoir": { + "version": "0.6.2", + "resolved": "https://registry.npmjs.org/rechoir/-/rechoir-0.6.2.tgz", + "integrity": "sha1-hSBLVNuoLVdC4oyWdW70OvUOM4Q=", + "dev": true, + "dependencies": { + "resolve": "^1.1.6" + }, + "engines": { + "node": ">= 0.10" + } + }, "node_modules/regexpp": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/regexpp/-/regexpp-3.2.0.tgz", @@ -7146,6 +7293,23 @@ "node": ">=8" } }, + "node_modules/shelljs": { + "version": "0.8.5", + "resolved": "https://registry.npmjs.org/shelljs/-/shelljs-0.8.5.tgz", + "integrity": "sha512-TiwcRcrkhHvbrZbnRcFYMLl30Dfov3HKqzp5tO5b4pt6G/SezKcYhmDg15zXVBswHmctSAQKznqNW2LO5tTDow==", + "dev": true, + "dependencies": { + "glob": "^7.0.0", + "interpret": "^1.0.0", + "rechoir": "^0.6.2" + }, + "bin": { + "shjs": "bin/shjs" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/side-channel": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.4.tgz", @@ -7690,6 +7854,45 @@ } } }, + "node_modules/ts-jest-resolver": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ts-jest-resolver/-/ts-jest-resolver-2.0.0.tgz", + "integrity": "sha512-yr/lgqJtVBUXhnaxD5Es0XFGHoIYT6NgbUW1VUiAPTEDINHByiUfcnfDf6VOK3CRibqaqWyTEAppBBcXeIuGAw==", + "dev": true, + "dependencies": { + "jest-resolve": "^27.2.5" + } + }, + "node_modules/ts-patch": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/ts-patch/-/ts-patch-2.0.1.tgz", + "integrity": "sha512-mP7beU1QkmyDs1+SzXYVaSTD6Xo7ZCibOJ3sZkb/xsQjoAQXvn4oPjk0keC2LfCNAgilqtqgjiWp3pQri1uz4w==", + "dev": true, + "dependencies": { + "chalk": "^4.1.0", + "glob": "^7.1.7", + "global-prefix": "^3.0.0", + "minimist": "^1.2.5", + "resolve": "^1.20.0", + "shelljs": "^0.8.4", + "strip-ansi": "^6.0.0" + }, + "bin": { + "ts-patch": "bin/cli.js" + }, + "peerDependencies": { + "typescript": ">=4.0.0" + } + }, + "node_modules/ts-transform-esm-import": { + "version": "0.9.0", + "resolved": "https://registry.npmjs.org/ts-transform-esm-import/-/ts-transform-esm-import-0.9.0.tgz", + "integrity": "sha512-gexh5PoWwXQTcLkzexqsdPkML1q72b6Syjbzrs7SLfxkCla2SRgQYHIuoZDV4Vc9/Hn9ol9lkSeeXYW8FEi8Pg==", + "dev": true, + "peerDependencies": { + "typescript": "^4.5.0" + } + }, "node_modules/tsconfig-paths": { "version": "3.10.1", "resolved": "https://registry.npmjs.org/tsconfig-paths/-/tsconfig-paths-3.10.1.tgz", @@ -7782,9 +7985,9 @@ } }, "node_modules/typescript": { - "version": "3.9.10", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.9.10.tgz", - "integrity": "sha512-w6fIxVE/H1PkLKcCPsFqKE7Kv7QUwhU8qQY2MueZXWx5cPZdwFupLgKK3vntcK98BtNHZtAF4LA/yl2a7k8R6Q==", + "version": "4.6.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.6.3.tgz", + "integrity": "sha512-yNIatDa5iaofVozS/uQJEl3JRWLKKGJKh6Yaiv0GLGSuhpFJe7P3SbHZ8/yjAHRQwKRoA6YZqlfjXWmVzoVSMw==", "dev": true, "bin": { "tsc": "bin/tsc", @@ -7946,6 +8149,15 @@ "makeerror": "1.0.12" } }, + "node_modules/web-streams-polyfill": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.2.1.tgz", + "integrity": "sha512-e0MO3wdXWKrLbL0DgGnUV7WHVuw9OUvL4hjgnPkIeEvESk74gAITi5G606JtZPp39cd8HA9VQzCIvA49LpPN5Q==", + "dev": true, + "engines": { + "node": ">= 8" + } + }, "node_modules/webidl-conversions": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", @@ -10409,6 +10621,12 @@ } } }, + "data-uri-to-buffer": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.0.tgz", + "integrity": "sha512-Vr3mLBA8qWmcuschSLAOogKgQ/Jwxulv3RNE4FXnYWRGujzrRWQI4m12fQqRkwX06C0KanhLr4hK+GydchZsaA==", + "dev": true + }, "data-urls": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-2.0.0.tgz", @@ -11147,6 +11365,16 @@ "bser": "2.1.1" } }, + "fetch-blob": { + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.1.5.tgz", + "integrity": "sha512-N64ZpKqoLejlrwkIAnb9iLSA3Vx/kjgzpcDhygcqJ2KKjky8nCgUQ+dzXtbrLaWZGZNmNfQTsiQ0weZ1svglHg==", + "dev": true, + "requires": { + "node-domexception": "^1.0.0", + "web-streams-polyfill": "^3.0.3" + } + }, "fflate": { "version": "0.7.3", "resolved": "https://registry.npmjs.org/fflate/-/fflate-0.7.3.tgz", @@ -11263,6 +11491,15 @@ "mime-types": "^2.1.12" } }, + "formdata-polyfill": { + "version": "4.0.10", + "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", + "integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==", + "dev": true, + "requires": { + "fetch-blob": "^3.1.2" + } + }, "forwarded": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", @@ -11405,6 +11642,28 @@ "is-glob": "^4.0.1" } }, + "global-prefix": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/global-prefix/-/global-prefix-3.0.0.tgz", + "integrity": "sha512-awConJSVCHVGND6x3tmMaKcQvwXLhjdkmomy2W+Goaui8YPgYgXJZewhg3fWC+DlfqqQuWg8AwqjGTD2nAPVWg==", + "dev": true, + "requires": { + "ini": "^1.3.5", + "kind-of": "^6.0.2", + "which": "^1.3.1" + }, + "dependencies": { + "which": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz", + "integrity": "sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==", + "dev": true, + "requires": { + "isexe": "^2.0.0" + } + } + } + }, "globals": { "version": "13.11.0", "resolved": "https://registry.npmjs.org/globals/-/globals-13.11.0.tgz", @@ -11719,6 +11978,12 @@ "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", "devOptional": true }, + "ini": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", + "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==", + "dev": true + }, "internal-slot": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/internal-slot/-/internal-slot-1.0.3.tgz", @@ -11730,6 +11995,12 @@ "side-channel": "^1.0.4" } }, + "interpret": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/interpret/-/interpret-1.4.0.tgz", + "integrity": "sha512-agE4QfB2Lkp9uICn7BAqoscw4SZP9kTE2hxiFI3jBPmXJfdqiahTbUuKGsMoN2GtqL9AxhYioAcVvgsb1HvRbA==", + "dev": true + }, "ipaddr.js": { "version": "1.9.1", "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", @@ -12833,6 +13104,12 @@ "safe-buffer": "^5.0.1" } }, + "kind-of": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", + "dev": true + }, "kleur": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/kleur/-/kleur-3.0.3.tgz", @@ -13116,6 +13393,23 @@ "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.2.tgz", "integrity": "sha512-hZXc7K2e+PgeI1eDBe/10Ard4ekbfrrqG8Ep+8Jmf4JID2bNg7NvCPOZN+kfF574pFQI7mum2AUqDidoKqcTOw==" }, + "node-domexception": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", + "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", + "dev": true + }, + "node-fetch": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.1.1.tgz", + "integrity": "sha512-SMk+vKgU77PYotRdWzqZGTZeuFKlsJ0hu4KPviQKkfY+N3vn2MIzr0rvpnYpR8MtB3IEuhlEcuOLbGvLRlA+yg==", + "dev": true, + "requires": { + "data-uri-to-buffer": "^4.0.0", + "fetch-blob": "^3.1.3", + "formdata-polyfill": "^4.0.10" + } + }, "node-forge": { "version": "0.10.0", "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-0.10.0.tgz", @@ -13579,6 +13873,15 @@ "util-deprecate": "^1.0.1" } }, + "rechoir": { + "version": "0.6.2", + "resolved": "https://registry.npmjs.org/rechoir/-/rechoir-0.6.2.tgz", + "integrity": "sha1-hSBLVNuoLVdC4oyWdW70OvUOM4Q=", + "dev": true, + "requires": { + "resolve": "^1.1.6" + } + }, "regexpp": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/regexpp/-/regexpp-3.2.0.tgz", @@ -13767,6 +14070,17 @@ "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", "dev": true }, + "shelljs": { + "version": "0.8.5", + "resolved": "https://registry.npmjs.org/shelljs/-/shelljs-0.8.5.tgz", + "integrity": "sha512-TiwcRcrkhHvbrZbnRcFYMLl30Dfov3HKqzp5tO5b4pt6G/SezKcYhmDg15zXVBswHmctSAQKznqNW2LO5tTDow==", + "dev": true, + "requires": { + "glob": "^7.0.0", + "interpret": "^1.0.0", + "rechoir": "^0.6.2" + } + }, "side-channel": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.4.tgz", @@ -14175,6 +14489,37 @@ "yargs-parser": "20.x" } }, + "ts-jest-resolver": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ts-jest-resolver/-/ts-jest-resolver-2.0.0.tgz", + "integrity": "sha512-yr/lgqJtVBUXhnaxD5Es0XFGHoIYT6NgbUW1VUiAPTEDINHByiUfcnfDf6VOK3CRibqaqWyTEAppBBcXeIuGAw==", + "dev": true, + "requires": { + "jest-resolve": "^27.2.5" + } + }, + "ts-patch": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/ts-patch/-/ts-patch-2.0.1.tgz", + "integrity": "sha512-mP7beU1QkmyDs1+SzXYVaSTD6Xo7ZCibOJ3sZkb/xsQjoAQXvn4oPjk0keC2LfCNAgilqtqgjiWp3pQri1uz4w==", + "dev": true, + "requires": { + "chalk": "^4.1.0", + "glob": "^7.1.7", + "global-prefix": "^3.0.0", + "minimist": "^1.2.5", + "resolve": "^1.20.0", + "shelljs": "^0.8.4", + "strip-ansi": "^6.0.0" + } + }, + "ts-transform-esm-import": { + "version": "0.9.0", + "resolved": "https://registry.npmjs.org/ts-transform-esm-import/-/ts-transform-esm-import-0.9.0.tgz", + "integrity": "sha512-gexh5PoWwXQTcLkzexqsdPkML1q72b6Syjbzrs7SLfxkCla2SRgQYHIuoZDV4Vc9/Hn9ol9lkSeeXYW8FEi8Pg==", + "dev": true, + "requires": {} + }, "tsconfig-paths": { "version": "3.10.1", "resolved": "https://registry.npmjs.org/tsconfig-paths/-/tsconfig-paths-3.10.1.tgz", @@ -14248,9 +14593,9 @@ } }, "typescript": { - "version": "3.9.10", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.9.10.tgz", - "integrity": "sha512-w6fIxVE/H1PkLKcCPsFqKE7Kv7QUwhU8qQY2MueZXWx5cPZdwFupLgKK3vntcK98BtNHZtAF4LA/yl2a7k8R6Q==", + "version": "4.6.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.6.3.tgz", + "integrity": "sha512-yNIatDa5iaofVozS/uQJEl3JRWLKKGJKh6Yaiv0GLGSuhpFJe7P3SbHZ8/yjAHRQwKRoA6YZqlfjXWmVzoVSMw==", "dev": true }, "unbox-primitive": { @@ -14377,6 +14722,12 @@ "makeerror": "1.0.12" } }, + "web-streams-polyfill": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.2.1.tgz", + "integrity": "sha512-e0MO3wdXWKrLbL0DgGnUV7WHVuw9OUvL4hjgnPkIeEvESk74gAITi5G606JtZPp39cd8HA9VQzCIvA49LpPN5Q==", + "dev": true + }, "webidl-conversions": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", diff --git a/functions/package.json b/functions/package.json index 158446a3..48268e6c 100644 --- a/functions/package.json +++ b/functions/package.json @@ -37,7 +37,9 @@ "eslint-config-google": "^0.14.0", "eslint-plugin-import": "^2.22.0", "jest": "^27.5.1", + "node-fetch": "^3.1.1", "ts-jest": "^27.1.3", + "ts-jest-resolver": "^2.0.0", "ts-patch": "^2.0.1", "ts-transform-esm-import": "^0.9.0", "typescript": "^4.6.3", diff --git a/functions/src/__tests__/authentication.test.ts b/functions/src/__tests__/authentication.test.ts index bff51370..9ca275ab 100644 --- a/functions/src/__tests__/authentication.test.ts +++ b/functions/src/__tests__/authentication.test.ts @@ -1,10 +1,11 @@ -jest.mock("firebase-admin"); +import { jest } from "@jest/globals"; -import * as admin from "firebase-admin"; -import * as functions from "firebase-functions"; +import admin from "firebase-admin"; +import functions from "firebase-functions"; import { AuthenticatedFunction, useAuthentication } from "../authentication"; describe("useAuthentication", () => { + const request = { headers: { authorization: "Bearer abc123", @@ -34,7 +35,7 @@ describe("useAuthentication", () => { }); it("throws when request is null", async () => { - const fn = jest.fn(); + const fn: any = jest.fn(); await expect(() => useAuthentication( @@ -48,7 +49,7 @@ describe("useAuthentication", () => { }); it("throws when request is undefined", async () => { - const fn = jest.fn(); + const fn: any = jest.fn(); await expect(() => useAuthentication( @@ -62,7 +63,7 @@ describe("useAuthentication", () => { }); it("throws when response is null", async () => { - const fn = jest.fn(); + const fn: any = jest.fn(); await expect(() => useAuthentication( @@ -76,7 +77,7 @@ describe("useAuthentication", () => { }); it("throws when response is undefined", async () => { - const fn = jest.fn(); + const fn: any = jest.fn(); await expect(() => useAuthentication( @@ -120,7 +121,7 @@ describe("useAuthentication", () => { }); it("throws http error 401 when request is missing", async () => { - const fn = jest.fn(); + const fn: any = jest.fn(); await useAuthentication( ({} as unknown) as functions.https.Request, @@ -134,7 +135,7 @@ describe("useAuthentication", () => { }); it("throws http error 401 when request authorization header is missing", async () => { - const fn = jest.fn(); + const fn: any = jest.fn(); await useAuthentication( { ...request, headers: {} } as functions.https.Request, @@ -148,7 +149,7 @@ describe("useAuthentication", () => { }); it("throws http error 401 when bearer prefix is missing", async () => { - const fn = jest.fn(); + const fn: any = jest.fn(); await useAuthentication( { @@ -165,7 +166,7 @@ describe("useAuthentication", () => { }); it("throws http error 401 when bearer prefix is not pascal cased", async () => { - const fn = jest.fn(); + const fn: any = jest.fn(); await useAuthentication( { @@ -182,7 +183,7 @@ describe("useAuthentication", () => { }); it("throws http error 401 when auth header has more than 2 parts", async () => { - const fn = jest.fn(); + const fn: any = jest.fn(); await useAuthentication( { @@ -201,7 +202,7 @@ describe("useAuthentication", () => { it("throws http error 401 when token is invalid", async () => { fakeAuth.verifyIdToken.mockRejectedValue("Invalid token"); - const fn = jest.fn(); + const fn: any = jest.fn(); await useAuthentication(request, response, fn); expect(response.status).toHaveBeenCalledWith(401); @@ -215,9 +216,9 @@ describe("useAuthentication", () => { uid: "abc123", } as admin.auth.DecodedIdToken; - fakeAuth.verifyIdToken.mockResolvedValue(decryptedToken); + fakeAuth.verifyIdToken.mockReturnValue(decryptedToken); - const innerFn = jest.fn(); + const innerFn: any = jest.fn(); await useAuthentication(request, response, innerFn); expect(innerFn).toHaveBeenCalledWith(decryptedToken); diff --git a/functions/src/__tests__/index.test.ts b/functions/src/__tests__/index.test.ts index 4bbc20a7..0bde5bb0 100644 --- a/functions/src/__tests__/index.test.ts +++ b/functions/src/__tests__/index.test.ts @@ -1,8 +1,7 @@ -jest.mock("../cors"); +import { jest } from "@jest/globals"; -import * as admin from "firebase-admin"; -import { useCors } from "../cors"; -import * as functions from "firebase-functions"; +import admin from "firebase-admin"; +import functions from "firebase-functions"; import { addRallyUserToFirestoreImpl, deleteRallyUserImpl, @@ -10,6 +9,25 @@ import { rallytoken, } from "../index"; import { studies } from "../studies"; +import fetch from "node-fetch"; + +beforeAll(async () => { + await fetch( + "http://" + process.env.FIREBASE_EMULATOR_HUB + '/functions/disableBackgroundTriggers', + { + method: 'PUT' + } + ); +}); + +afterAll(async () => { + await fetch( + "http://" + process.env.FIREBASE_EMULATOR_HUB + '/functions/enableBackgroundTriggers', + { + method: 'PUT' + } + ); +}) describe("loadFirestore", () => { const studyName = Object.keys(studies)[0]; @@ -141,7 +159,7 @@ describe("addRallyUserToFirestore and deleteRallyUserImpl", () => { }); describe("rallytoken tests", () => { - let send: jest.Mock; // eslint-disable-line @typescript-eslint/no-explicit-any + let send: any; // eslint-disable-line @typescript-eslint/no-explicit-any let response: functions.Response; // eslint-disable-line @typescript-eslint/no-explicit-any const uid = "fake-uid"; @@ -169,10 +187,6 @@ describe("rallytoken tests", () => { fakeAuth.verifyIdToken.mockReturnValue({ uid }); fakeAuth.createCustomToken.mockReturnValue(customToken); - - (useCors as jest.Mock).mockImplementation( - async (req, res, fn) => await fn(req, res) - ); }); afterEach(() => { diff --git a/functions/src/cors.ts b/functions/src/cors.ts index 1b19e11d..b3389f97 100644 --- a/functions/src/cors.ts +++ b/functions/src/cors.ts @@ -1,5 +1,6 @@ import cors from "cors"; export const useCors = cors({ - origin: true, + // Don't use CORS in testing mode + origin: process.env.NODE_ENV === "test" ? false : true, }); diff --git a/functions/src/index.ts b/functions/src/index.ts index 959cccc4..578df958 100644 --- a/functions/src/index.ts +++ b/functions/src/index.ts @@ -12,36 +12,42 @@ admin.initializeApp({ credential: admin.credential.applicationDefault(), }); -export const rallytoken = functions.https.onRequest(async (request, response) => - useCors(request, response, async () => { - await useAuthentication(request, response, async (decodedToken) => { - if (request.method !== "POST") { - response.status(500).send("Only POST and OPTIONS methods are allowed."); - return; - } - - functions.logger.info(`body type: ${typeof request.body}`, { - payload: request.body, - }); - - try { - let studyId; - if (typeof request.body === "string") { - const body = JSON.parse(request.body); - studyId = body.studyId; - } else { - studyId = request.body.studyId; - } - - const rallyToken = await generateToken(decodedToken, studyId); - functions.logger.info("OK"); - response.status(200).send({ rallyToken }); - } catch (ex) { - functions.logger.error(ex); - response.status(500).send(); - } - }); - }) +export const rallytoken = functions.https.onRequest( + async (request, response) => + new Promise((resolve) => + useCors(request, response, async () => { + await useAuthentication(request, response, async (decodedToken) => { + if (request.method !== "POST") { + response + .status(500) + .send("Only POST and OPTIONS methods are allowed."); + return; + } + + functions.logger.info(`body type: ${typeof request.body}`, { + payload: request.body, + }); + + try { + let studyId; + if (typeof request.body === "string") { + const body = JSON.parse(request.body); + studyId = body.studyId; + } else { + studyId = request.body.studyId; + } + + const rallyToken = await generateToken(decodedToken, studyId); + functions.logger.info("OK"); + response.status(200).send({ rallyToken }); + } catch (ex) { + functions.logger.error(ex); + response.status(500).send(); + } + }); + resolve(); + }) + ) ); /** @@ -142,7 +148,9 @@ export const deleteRallyUserImpl = async function ( return true; }; -export const deleteRallyUser = functions.auth.user().onDelete(deleteRallyUserImpl); +export const deleteRallyUser = functions.auth + .user() + .onDelete(deleteRallyUserImpl); /** * diff --git a/functions/tsconfig.json b/functions/tsconfig.json index c6454c4a..dc1ef846 100644 --- a/functions/tsconfig.json +++ b/functions/tsconfig.json @@ -3,6 +3,8 @@ "module": "esnext", "moduleResolution": "node", "allowSyntheticDefaultImports": true, + "allowJs": true, + "esModuleInterop": false, "noImplicitReturns": true, "noUnusedLocals": true, "outDir": "lib", From 2580bce172ea9ef71e6c6a01ee4a44812ea51afd Mon Sep 17 00:00:00 2001 From: Ashwin Agarwal Date: Mon, 18 Apr 2022 19:13:14 -0400 Subject: [PATCH 06/27] Implement Glean pings --- .circleci/config.yml | 2 +- functions/.eslintrc.cjs | 1 + functions/jest.config.js | 9 +- functions/package-lock.json | 17 ++ functions/package.json | 1 + .../src/__tests__/authentication.test.ts | 22 +- functions/src/__tests__/index.test.ts | 14 +- functions/src/glean.ts | 226 ++++++++++++++++++ functions/src/index.ts | 35 ++- 9 files changed, 294 insertions(+), 33 deletions(-) create mode 100644 functions/src/glean.ts diff --git a/.circleci/config.yml b/.circleci/config.yml index 3e0a253f..576da06c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -15,7 +15,7 @@ jobs: # You can specify an image from Dockerhub or use one of our Convenience Images from CircleCI's Developer Hub. # A list of available CircleCI Docker Convenience Images are available here: https://circleci.com/developer/images/image/cimg/node docker: - - image: cimg/openjdk:16.0.2-node + - image: cimg/python:3.8.8-node # Then run your tests! # CircleCI will report the results back to your VCS provider. steps: diff --git a/functions/.eslintrc.cjs b/functions/.eslintrc.cjs index 37c50513..4ad5af28 100644 --- a/functions/.eslintrc.cjs +++ b/functions/.eslintrc.cjs @@ -24,5 +24,6 @@ module.exports = { plugins: ["@typescript-eslint", "import"], rules: { quotes: ["error", "double"], + "import/no-unresolved": "off" // too many weird module resoltions in this project }, }; diff --git a/functions/jest.config.js b/functions/jest.config.js index 786ccb5a..6dcc7e56 100644 --- a/functions/jest.config.js +++ b/functions/jest.config.js @@ -7,11 +7,14 @@ export default { }, resolver: "ts-jest-resolver", moduleNameMapper: { - "^@mozilla/glean/plugins/encryption": - "@mozilla/glean/dist/plugins/encryption.js", + "^@mozilla/glean/node": "@mozilla/glean/dist/index/node.js", + "^@mozilla/glean/plugins/(.*)": "@mozilla/glean/dist/plugins/$1.js", "^@mozilla/glean/uploader": "@mozilla/glean/dist/core/upload/uploader.js", + "^@mozilla/glean/private/metrics/(.*)": + "@mozilla/glean/dist/core/metrics/types/$1.js", + "^@mozilla/glean/private/ping": + "@mozilla/glean/dist/core/pings/ping_type.js", }, - // extensionsToTreatAsEsm: [".ts"], testRegex: "src(/__tests__/.*|(\\.|/)(test|spec))\\.(jsx?|tsx?)$", testPathIgnorePatterns: ["lib/", "node_modules/", "setupTests.js"], moduleFileExtensions: ["js", "ts", "tsx", "jsx", "json", "node"], diff --git a/functions/package-lock.json b/functions/package-lock.json index c751731c..afce3f07 100644 --- a/functions/package-lock.json +++ b/functions/package-lock.json @@ -7,6 +7,7 @@ "name": "functions", "dependencies": { "@mozilla/glean": "^1.0.0", + "async-mutex": "^0.3.2", "cors": "^2.8.5", "firebase-admin": "^9.8.0", "firebase-functions": "^3.15.5" @@ -2306,6 +2307,14 @@ "node": ">=8" } }, + "node_modules/async-mutex": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/async-mutex/-/async-mutex-0.3.2.tgz", + "integrity": "sha512-HuTK7E7MT7jZEh1P9GtRW9+aTWiDWWi9InbZ5hjxrnRa39KS4BW04+xLBhYNS2aXhHUIKZSw3gj4Pn1pj+qGAA==", + "dependencies": { + "tslib": "^2.3.1" + } + }, "node_modules/async-retry": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/async-retry/-/async-retry-1.3.3.tgz", @@ -10192,6 +10201,14 @@ "integrity": "sha512-Z7tMw1ytTXt5jqMcOP+OQteU1VuNK9Y02uuJtKQ1Sv69jXQKKg5cibLwGJow8yzZP+eAc18EmLGPal0bp36rvQ==", "dev": true }, + "async-mutex": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/async-mutex/-/async-mutex-0.3.2.tgz", + "integrity": "sha512-HuTK7E7MT7jZEh1P9GtRW9+aTWiDWWi9InbZ5hjxrnRa39KS4BW04+xLBhYNS2aXhHUIKZSw3gj4Pn1pj+qGAA==", + "requires": { + "tslib": "^2.3.1" + } + }, "async-retry": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/async-retry/-/async-retry-1.3.3.tgz", diff --git a/functions/package.json b/functions/package.json index 48268e6c..7ed8ae94 100644 --- a/functions/package.json +++ b/functions/package.json @@ -23,6 +23,7 @@ "main": "lib/index.js", "dependencies": { "@mozilla/glean": "^1.0.0", + "async-mutex": "^0.3.2", "cors": "^2.8.5", "firebase-admin": "^9.8.0", "firebase-functions": "^3.15.5" diff --git a/functions/src/__tests__/authentication.test.ts b/functions/src/__tests__/authentication.test.ts index 9ca275ab..2db46411 100644 --- a/functions/src/__tests__/authentication.test.ts +++ b/functions/src/__tests__/authentication.test.ts @@ -35,7 +35,7 @@ describe("useAuthentication", () => { }); it("throws when request is null", async () => { - const fn: any = jest.fn(); + const fn: any = jest.fn(); // eslint-disable-line @typescript-eslint/no-explicit-any await expect(() => useAuthentication( @@ -49,7 +49,7 @@ describe("useAuthentication", () => { }); it("throws when request is undefined", async () => { - const fn: any = jest.fn(); + const fn: any = jest.fn(); // eslint-disable-line @typescript-eslint/no-explicit-any await expect(() => useAuthentication( @@ -63,7 +63,7 @@ describe("useAuthentication", () => { }); it("throws when response is null", async () => { - const fn: any = jest.fn(); + const fn: any = jest.fn(); // eslint-disable-line @typescript-eslint/no-explicit-any await expect(() => useAuthentication( @@ -77,7 +77,7 @@ describe("useAuthentication", () => { }); it("throws when response is undefined", async () => { - const fn: any = jest.fn(); + const fn: any = jest.fn(); // eslint-disable-line @typescript-eslint/no-explicit-any await expect(() => useAuthentication( @@ -121,7 +121,7 @@ describe("useAuthentication", () => { }); it("throws http error 401 when request is missing", async () => { - const fn: any = jest.fn(); + const fn: any = jest.fn(); // eslint-disable-line @typescript-eslint/no-explicit-any await useAuthentication( ({} as unknown) as functions.https.Request, @@ -135,7 +135,7 @@ describe("useAuthentication", () => { }); it("throws http error 401 when request authorization header is missing", async () => { - const fn: any = jest.fn(); + const fn: any = jest.fn(); // eslint-disable-line @typescript-eslint/no-explicit-any await useAuthentication( { ...request, headers: {} } as functions.https.Request, @@ -149,7 +149,7 @@ describe("useAuthentication", () => { }); it("throws http error 401 when bearer prefix is missing", async () => { - const fn: any = jest.fn(); + const fn: any = jest.fn(); // eslint-disable-line @typescript-eslint/no-explicit-any await useAuthentication( { @@ -166,7 +166,7 @@ describe("useAuthentication", () => { }); it("throws http error 401 when bearer prefix is not pascal cased", async () => { - const fn: any = jest.fn(); + const fn: any = jest.fn(); // eslint-disable-line @typescript-eslint/no-explicit-any await useAuthentication( { @@ -183,7 +183,7 @@ describe("useAuthentication", () => { }); it("throws http error 401 when auth header has more than 2 parts", async () => { - const fn: any = jest.fn(); + const fn: any = jest.fn(); // eslint-disable-line @typescript-eslint/no-explicit-any await useAuthentication( { @@ -202,7 +202,7 @@ describe("useAuthentication", () => { it("throws http error 401 when token is invalid", async () => { fakeAuth.verifyIdToken.mockRejectedValue("Invalid token"); - const fn: any = jest.fn(); + const fn: any = jest.fn(); // eslint-disable-line @typescript-eslint/no-explicit-any await useAuthentication(request, response, fn); expect(response.status).toHaveBeenCalledWith(401); @@ -218,7 +218,7 @@ describe("useAuthentication", () => { fakeAuth.verifyIdToken.mockReturnValue(decryptedToken); - const innerFn: any = jest.fn(); + const innerFn: any = jest.fn(); // eslint-disable-line @typescript-eslint/no-explicit-any await useAuthentication(request, response, innerFn); expect(innerFn).toHaveBeenCalledWith(decryptedToken); diff --git a/functions/src/__tests__/index.test.ts b/functions/src/__tests__/index.test.ts index 0bde5bb0..d6bee6d4 100644 --- a/functions/src/__tests__/index.test.ts +++ b/functions/src/__tests__/index.test.ts @@ -13,21 +13,25 @@ import fetch from "node-fetch"; beforeAll(async () => { await fetch( - "http://" + process.env.FIREBASE_EMULATOR_HUB + '/functions/disableBackgroundTriggers', + "http://" + + process.env.FIREBASE_EMULATOR_HUB + + "/functions/disableBackgroundTriggers", { - method: 'PUT' + method: "PUT", } ); }); afterAll(async () => { await fetch( - "http://" + process.env.FIREBASE_EMULATOR_HUB + '/functions/enableBackgroundTriggers', + "http://" + + process.env.FIREBASE_EMULATOR_HUB + + "/functions/enableBackgroundTriggers", { - method: 'PUT' + method: "PUT", } ); -}) +}); describe("loadFirestore", () => { const studyName = Object.keys(studies)[0]; diff --git a/functions/src/glean.ts b/functions/src/glean.ts new file mode 100644 index 00000000..0c38701d --- /dev/null +++ b/functions/src/glean.ts @@ -0,0 +1,226 @@ +import Glean from "@mozilla/glean/node"; +import PingEncryptionPlugin from "@mozilla/glean/plugins/encryption"; +import axios from "axios"; +import { Mutex, Semaphore, withTimeout } from "async-mutex"; + +import * as rallyMetrics from "./generated/rally.js"; +import * as userMetrics from "./generated/user.js"; +import * as enrollmentMetrics from "./generated/enrollment.js"; +import * as unenrollmentMetrics from "./generated/unenrollment.js"; +import * as rallyPings from "./generated/pings.js"; + +import { + Uploader, + UploadResult, + UploadResultStatus, +} from "@mozilla/glean/uploader"; + +const GLEAN_DEBUG_VIEW_TAG = "MozillaRally"; +const GLEAN_RALLY_APP_ID = "my-app-id"; +const GLEAN_APP_DISPLAY_VERSION = "TODO-rally-firestore-server"; +const GLEAN_ENCRYPTION_JWK = { + crv: "P-256", + kid: "rally-core", + kty: "EC", + x: "m7Gi2YD8DgPg3zxora5iwf0DFL0JFIhjoD2BRLpg7kI", + y: "zo35XIQME7Ct01uHK_LrMi5pZCuYDMhv8MUsSu7Eq08", +}; + +const GLEAN_DEFAULT_TIMEOUT = 5000; +const gleanLock = withTimeout(new Mutex(), GLEAN_DEFAULT_TIMEOUT); // Lock on global Glean instance, metrics, and pings +const sendPingFlag = withTimeout(new Semaphore(1), GLEAN_DEFAULT_TIMEOUT); // Allow Glean to signal once ping is sent + +/* + * platformEnrollment + * Glean ping: enrollment + */ +export async function platformEnrollment(rallyID: string): Promise { + const releaseGlean = await gleanLock.acquire(); + initializeGlean(); + + rallyMetrics.id.set(rallyID); + + await sendPingFlag.acquire(); + rallyPings.enrollment.submit(); + await sendPingFlag.waitForUnlock(); + + releaseGlean(); +} + +/* + * platformUnenrollment + * Glean ping: unenrollment + * + */ +export async function platformUnenrollment(rallyID: string): Promise { + const releaseGlean = await gleanLock.acquire(); + initializeGlean(); + + rallyMetrics.id.set(rallyID); + + await sendPingFlag.acquire(); + rallyPings.unenrollment.submit(); + await sendPingFlag.waitForUnlock(); + + releaseGlean(); +} + +/* + * demographics + * Glean ping: demographics + */ +export async function demographics( + rallyID: string, + demographicsData: Record +): Promise { + const releaseGlean = await gleanLock.acquire(); + initializeGlean(); + + rallyMetrics.id.set(rallyID); + setUserMetrics(demographicsData); + + await sendPingFlag.acquire(); + rallyPings.demographics.submit(); + await sendPingFlag.waitForUnlock(); + + releaseGlean(); +} + +/* + * studyEnrollment + * Glean ping: study-enrollment + */ +export async function studyEnrollment( + rallyID: string, + studyID: string +): Promise { + const releaseGlean = await gleanLock.acquire(); + initializeGlean(); + + rallyMetrics.id.set(rallyID); + enrollmentMetrics.studyId.set(studyID); + + await sendPingFlag.acquire(); + rallyPings.studyEnrollment.submit(); + await sendPingFlag.waitForUnlock(); + + releaseGlean(); +} + +/* + * studyUnenrollment + * Glean ping: study-unenrollment + */ +export async function studyUnenrollment( + rallyID: string, + studyID: string +): Promise { + const releaseGlean = await gleanLock.acquire(); + initializeGlean(); + + rallyMetrics.id.set(rallyID); + unenrollmentMetrics.studyId.set(studyID); + + await sendPingFlag.acquire(); + rallyPings.studyUnenrollment.submit(); + await sendPingFlag.waitForUnlock(); + + releaseGlean(); +} + +/* + * Helper function for initializing Glean + */ +function initializeGlean(): void { + Glean.setDebugViewTag(GLEAN_DEBUG_VIEW_TAG); + Glean.setLogPings(true); + + // Glean.initialize is a no-op if Glean is already initialized + Glean.initialize(GLEAN_RALLY_APP_ID, true, { + appDisplayVersion: GLEAN_APP_DISPLAY_VERSION, + plugins: [new PingEncryptionPlugin(GLEAN_ENCRYPTION_JWK)], + httpClient: new CustomPingUploader(), + }); +} + +/* + * Helper function for setting user metrics + * from demographic data (mapping) + */ +function setUserMetrics(data: any): void { // eslint-disable-line @typescript-eslint/no-explicit-any + if ("age" in data) { + userMetrics.age[`band_${data["age"]}`].set(true); + } + + if ("gender" in data) { + userMetrics.gender[data["gender"]].set(true); + } + + if ("hispanicLatinxSpanishOrigin" in data) { + const label = + data["hispanicLatinxSpanishOrigin"] === "other" + ? "other" + : "hispanic_latinx_spanish"; + userMetrics.origin[label].set(true); + } + + if ("race" in data) { + for (const raceLabel of data["race"]) { + const label = + raceLabel === "american_indian_or_alaska_native" + ? "am_indian_or_alaska_native" + : raceLabel; + userMetrics.races[label].set(true); + } + } + + if ("school" in data) { + const KEY_FIXUP: any = { // eslint-disable-line @typescript-eslint/no-explicit-any + high_school_graduate_or_equivalent: "high_school_grad_or_eq", + some_college_but_no_degree_or_in_progress: "college_degree_in_progress", + }; + + const originalLabel = data["school"]; + const label = + originalLabel in KEY_FIXUP ? KEY_FIXUP[originalLabel] : originalLabel; + userMetrics.school[label].set(true); + } + + if ("exactIncome" in data) { + userMetrics.exactIncome.set(data["exactIncome"]); + } + + if ("zipcode" in data) { + userMetrics.zipcode.set(data["zipcode"]); + } +} + +/** + * Custom Ping Uploader for Glean + * TODO: replace direct POST request with Google Cloud Task + */ +class CustomPingUploader extends Uploader { + async post( + url: string, + body: string | Uint8Array, + headers: Record + ): Promise { + const result = await axios + .post(url, body, { headers: headers }) + .then(function (response) { + return { + status: response.status, + result: UploadResultStatus.Success, + }; + }) + .catch(function (error) { + console.log(error); + return { + status: 500, + result: UploadResultStatus.UnrecoverableFailure, + }; + }); + sendPingFlag.release(); + return result; + } +} diff --git a/functions/src/index.ts b/functions/src/index.ts index 578df958..3844cfa0 100644 --- a/functions/src/index.ts +++ b/functions/src/index.ts @@ -1,12 +1,12 @@ import admin from "firebase-admin"; -import functions from "firebase-functions"; -import { Change, EventContext } from "firebase-functions"; +import functions, { Change, EventContext } from "firebase-functions"; import { DocumentSnapshot } from "firebase-functions/v1/firestore"; import { v4 as uuidv4 } from "uuid"; import { useAuthentication } from "./authentication"; import { useCors } from "./cors"; import { studies } from "./studies"; import { isDeepStrictEqual } from "util"; +import * as gleanPings from "./glean"; admin.initializeApp({ credential: admin.credential.applicationDefault(), @@ -205,17 +205,15 @@ export const handleUserChangesImpl = async function ( if (!newUser || (oldUser && oldUser.enrolled === true && !newUser.enrolled)) { // User document has been deleted - functions.logger.info( - `Sending deletion and unenrollment pings for user ID ${userID}` - ); - // TODO send Glean pings + functions.logger.info(`Sending unenrollment ping for user ID ${userID}`); + await gleanPings.platformUnenrollment(rallyID); return true; } if ((!oldUser || !oldUser.enrolled) && newUser.enrolled === true) { // User just enrolled functions.logger.info(`Sending enrollment ping for user ID ${userID}`); - // TODO send Glean ping + await gleanPings.platformEnrollment(rallyID); } if ( @@ -227,7 +225,7 @@ export const handleUserChangesImpl = async function ( ) { // User updated demographicsData functions.logger.info(`Sending demographics ping for user ID ${userID}`); - // TODO send Glean ping + await gleanPings.demographics(rallyID, newUser.demographicsData); } return true; @@ -246,6 +244,7 @@ export const handleUserStudyChangesImpl = async function ( context: EventContext ): Promise { const userID = context.params.userID; + const firebaseStudyID = context.params.studyID; const rallyID = await getRallyIdForUser(userID); if (!rallyID) { // Without Rally ID, we can't make any Glean pings @@ -255,7 +254,6 @@ export const handleUserStudyChangesImpl = async function ( ); } - const studyID = context.params.studyID; // Get an object with the current document value. // If the document does not exist, it has been deleted. const newStudy = change.after.exists ? change.after.data() : null; @@ -263,15 +261,26 @@ export const handleUserStudyChangesImpl = async function ( // Get the old document, to compare the enrollment state. const oldStudy = change.before.exists ? change.before.data() : null; + const studyID = + (newStudy ? newStudy.studyId : null) || + (oldStudy ? oldStudy.studyId : null); + if (!studyID) { + // Without Study ID, we can't construct study-related Glean pings + // This is bad and should be flagged for inspection + throw new Error( + `Couldn't find Glean Study ID for user ID ${userID} and Firebase study ID ${firebaseStudyID}. Aborting Glean ping process.` + ); + } + if ( !newStudy || (oldStudy && oldStudy.enrolled === true && !newStudy.enrolled) ) { // User unenrolled from study functions.logger.info( - `Sending deletion and unenrollment pings for study with user ID ${userID} with study ID ${studyID}` + `Sending unenrollment ping for study with user ID ${userID} with study ID ${studyID}` ); - // TODO send Glean pings + await gleanPings.studyUnenrollment(rallyID, studyID); return true; } @@ -280,7 +289,7 @@ export const handleUserStudyChangesImpl = async function ( functions.logger.info( `Sending enrollment ping for study with user ID ${userID} with study ID ${studyID}` ); - // TODO send Glean ping + await gleanPings.studyEnrollment(rallyID, studyID); return true; } @@ -292,7 +301,7 @@ export const handleUserStudyChanges = functions.firestore .onWrite(handleUserStudyChangesImpl); async function getRallyIdForUser(userID: string) { - let extensionUserDoc = await admin + const extensionUserDoc = await admin .firestore() .collection("extensionUsers") .doc(userID) From 323b6f127ce636460fbae3fe7c477df1a6d505d3 Mon Sep 17 00:00:00 2001 From: Ashwin Agarwal Date: Tue, 19 Apr 2022 18:00:27 -0400 Subject: [PATCH 07/27] Disable function triggers in integration test; cleanup some unused code --- package.json | 3 +-- tests/hooks.js | 24 ------------------------ tests/integration/ux.test.ts | 24 ++++++++++++++++++++++++ 3 files changed, 25 insertions(+), 26 deletions(-) delete mode 100644 tests/hooks.js diff --git a/package.json b/package.json index 48d3484f..013b60e3 100644 --- a/package.json +++ b/package.json @@ -20,8 +20,7 @@ "lint:functions": "cd functions && npm install && npm run lint && cd ..", "format": "prettier --write --plugin-search-dir=. .", "doc": "jsdoc -c jsdoc.conf.json && mkdir -p docs/images && cp images/rally-logo.png docs/images/", - "test:unit": "jest ./tests/unit/", - "test:integration:jest": "jest ./tests/integration/", + "test:integration:jest": "set -a && . ./functions/.testenv && jest ./tests/integration/ && set +a", "test:integration": "npm run build:web:emulator && npm run build:functions && ./scripts/integration-test.sh", "test:functions": "firebase emulators:exec --project demo-rally --only auth,functions,firestore --ui 'npm run build:functions && sleep 1 && cd functions && npm run test:coverage && cd ..'", "storybook": "start-storybook -s ./static -p 6006", diff --git a/tests/hooks.js b/tests/hooks.js deleted file mode 100644 index 7a8bacf5..00000000 --- a/tests/hooks.js +++ /dev/null @@ -1,24 +0,0 @@ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -const chrome = require("sinon-chrome/extensions"); -// We need to provide the `browser.runtime.id` for sinon-chrome to -// be happy and play nice with webextension-polyfill. See this issue: -// https://github.com/mozilla/webextension-polyfill/issues/218 -chrome.runtime.id = "testid"; -global.chrome = chrome; - -const browser = require("webextension-polyfill"); - -exports.mochaHooks = { - beforeAll() { - global.chrome = chrome; - global.browser = browser; - }, - afterAll() { - chrome.flush(); - delete global.chrome; - delete global.browser; - }, -}; diff --git a/tests/integration/ux.test.ts b/tests/integration/ux.test.ts index 7eaf640f..53398366 100644 --- a/tests/integration/ux.test.ts +++ b/tests/integration/ux.test.ts @@ -47,6 +47,30 @@ jest.setTimeout(60 * 10000); let driver: WebDriver; let screenshotCount = 0; +beforeAll(async () => { + // Disable function triggers for testing (prevents inadvertent Glean pings) + await fetch( + "http://" + + process.env.FIREBASE_EMULATOR_HUB + + "/functions/disableBackgroundTriggers", + { + method: "PUT", + } + ); +}); + +afterAll(async () => { + // Re-enable function triggers for potential future tests + await fetch( + "http://" + + process.env.FIREBASE_EMULATOR_HUB + + "/functions/enableBackgroundTriggers", + { + method: "PUT", + } + ); +}); + describe("Rally Web Platform UX flows", function () { beforeEach(async () => { driver = await webDriver(loadExtension, headlessMode); From 9e4fef6b3c003a80bd461f92334232f2298007ab Mon Sep 17 00:00:00 2001 From: Ashwin Agarwal Date: Wed, 20 Apr 2022 14:00:04 -0400 Subject: [PATCH 08/27] Add APP_ID and VERSION to Glean; rename sendPingFlag to submitPingFlag --- functions/src/cors.ts | 2 +- functions/src/glean.ts | 32 +++++++++++++++++--------------- 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/functions/src/cors.ts b/functions/src/cors.ts index b3389f97..cb54a59d 100644 --- a/functions/src/cors.ts +++ b/functions/src/cors.ts @@ -2,5 +2,5 @@ import cors from "cors"; export const useCors = cors({ // Don't use CORS in testing mode - origin: process.env.NODE_ENV === "test" ? false : true, + origin: process.env.NODE_ENV !== "test" ? true : false, }); diff --git a/functions/src/glean.ts b/functions/src/glean.ts index 0c38701d..d5ddb65e 100644 --- a/functions/src/glean.ts +++ b/functions/src/glean.ts @@ -16,8 +16,8 @@ import { } from "@mozilla/glean/uploader"; const GLEAN_DEBUG_VIEW_TAG = "MozillaRally"; -const GLEAN_RALLY_APP_ID = "my-app-id"; -const GLEAN_APP_DISPLAY_VERSION = "TODO-rally-firestore-server"; +const GLEAN_RALLY_APP_ID = process.env.NODE_ENV !== "test" ? "rally-core" : "test-app-id"; +const GLEAN_APP_DISPLAY_VERSION = require("../../package.json").version; const GLEAN_ENCRYPTION_JWK = { crv: "P-256", kid: "rally-core", @@ -26,9 +26,9 @@ const GLEAN_ENCRYPTION_JWK = { y: "zo35XIQME7Ct01uHK_LrMi5pZCuYDMhv8MUsSu7Eq08", }; -const GLEAN_DEFAULT_TIMEOUT = 5000; +const GLEAN_DEFAULT_TIMEOUT = 10000; const gleanLock = withTimeout(new Mutex(), GLEAN_DEFAULT_TIMEOUT); // Lock on global Glean instance, metrics, and pings -const sendPingFlag = withTimeout(new Semaphore(1), GLEAN_DEFAULT_TIMEOUT); // Allow Glean to signal once ping is sent +const submitPingFlag = withTimeout(new Semaphore(1), GLEAN_DEFAULT_TIMEOUT); // Allow Glean to signal once ping is sent /* * platformEnrollment @@ -40,9 +40,9 @@ export async function platformEnrollment(rallyID: string): Promise { rallyMetrics.id.set(rallyID); - await sendPingFlag.acquire(); + await submitPingFlag.acquire(); rallyPings.enrollment.submit(); - await sendPingFlag.waitForUnlock(); + await submitPingFlag.waitForUnlock(); releaseGlean(); } @@ -58,9 +58,9 @@ export async function platformUnenrollment(rallyID: string): Promise { rallyMetrics.id.set(rallyID); - await sendPingFlag.acquire(); + await submitPingFlag.acquire(); rallyPings.unenrollment.submit(); - await sendPingFlag.waitForUnlock(); + await submitPingFlag.waitForUnlock(); releaseGlean(); } @@ -79,9 +79,9 @@ export async function demographics( rallyMetrics.id.set(rallyID); setUserMetrics(demographicsData); - await sendPingFlag.acquire(); + await submitPingFlag.acquire(); rallyPings.demographics.submit(); - await sendPingFlag.waitForUnlock(); + await submitPingFlag.waitForUnlock(); releaseGlean(); } @@ -100,9 +100,9 @@ export async function studyEnrollment( rallyMetrics.id.set(rallyID); enrollmentMetrics.studyId.set(studyID); - await sendPingFlag.acquire(); + await submitPingFlag.acquire(); rallyPings.studyEnrollment.submit(); - await sendPingFlag.waitForUnlock(); + await submitPingFlag.waitForUnlock(); releaseGlean(); } @@ -121,9 +121,9 @@ export async function studyUnenrollment( rallyMetrics.id.set(rallyID); unenrollmentMetrics.studyId.set(studyID); - await sendPingFlag.acquire(); + await submitPingFlag.acquire(); rallyPings.studyUnenrollment.submit(); - await sendPingFlag.waitForUnlock(); + await submitPingFlag.waitForUnlock(); releaseGlean(); } @@ -220,7 +220,9 @@ class CustomPingUploader extends Uploader { result: UploadResultStatus.UnrecoverableFailure, }; }); - sendPingFlag.release(); + + // Signal to ping function that ping has been submitted + submitPingFlag.release(); return result; } } From 9f70f0405b96f4f7f90ae91c9801d4bfaddb2d56 Mon Sep 17 00:00:00 2001 From: Ashwin Agarwal Date: Wed, 20 Apr 2022 14:35:14 -0400 Subject: [PATCH 09/27] Add minor comment explanation for function test --- functions/src/__tests__/index.test.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/functions/src/__tests__/index.test.ts b/functions/src/__tests__/index.test.ts index d6bee6d4..588344c4 100644 --- a/functions/src/__tests__/index.test.ts +++ b/functions/src/__tests__/index.test.ts @@ -12,6 +12,7 @@ import { studies } from "../studies"; import fetch from "node-fetch"; beforeAll(async () => { + // Disable function triggers for testing (prevents inadvertent Glean pings) await fetch( "http://" + process.env.FIREBASE_EMULATOR_HUB + @@ -23,6 +24,7 @@ beforeAll(async () => { }); afterAll(async () => { + // Re-enable function triggers for potential future tests await fetch( "http://" + process.env.FIREBASE_EMULATOR_HUB + From d739c0fdae222fd3779ecf4f52efdfbd46626463 Mon Sep 17 00:00:00 2001 From: Ashwin Agarwal Date: Wed, 20 Apr 2022 14:59:00 -0400 Subject: [PATCH 10/27] Change package.json version require to import --- functions/src/glean.ts | 2 +- functions/tsconfig.json | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/functions/src/glean.ts b/functions/src/glean.ts index d5ddb65e..5b750b21 100644 --- a/functions/src/glean.ts +++ b/functions/src/glean.ts @@ -17,7 +17,7 @@ import { const GLEAN_DEBUG_VIEW_TAG = "MozillaRally"; const GLEAN_RALLY_APP_ID = process.env.NODE_ENV !== "test" ? "rally-core" : "test-app-id"; -const GLEAN_APP_DISPLAY_VERSION = require("../../package.json").version; +const GLEAN_APP_DISPLAY_VERSION = (await import("../../package.json")).version; const GLEAN_ENCRYPTION_JWK = { crv: "P-256", kid: "rally-core", diff --git a/functions/tsconfig.json b/functions/tsconfig.json index dc1ef846..e2cfca16 100644 --- a/functions/tsconfig.json +++ b/functions/tsconfig.json @@ -7,6 +7,7 @@ "esModuleInterop": false, "noImplicitReturns": true, "noUnusedLocals": true, + "resolveJsonModule": true, "outDir": "lib", "sourceMap": true, "strict": true, From 1c18b7f5304a6c19cb3d1ab9abfc9f8bf6e50c0d Mon Sep 17 00:00:00 2001 From: Ashwin Agarwal Date: Wed, 20 Apr 2022 17:47:07 -0400 Subject: [PATCH 11/27] Rollback version string; was creating import problems --- functions/src/glean.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/functions/src/glean.ts b/functions/src/glean.ts index 5b750b21..2d4c2119 100644 --- a/functions/src/glean.ts +++ b/functions/src/glean.ts @@ -17,7 +17,7 @@ import { const GLEAN_DEBUG_VIEW_TAG = "MozillaRally"; const GLEAN_RALLY_APP_ID = process.env.NODE_ENV !== "test" ? "rally-core" : "test-app-id"; -const GLEAN_APP_DISPLAY_VERSION = (await import("../../package.json")).version; +const GLEAN_APP_DISPLAY_VERSION = "TODO_app_version"; const GLEAN_ENCRYPTION_JWK = { crv: "P-256", kid: "rally-core", From d5ea869d7e857485254ce317593a7ad3620082fb Mon Sep 17 00:00:00 2001 From: Ashwin Agarwal Date: Thu, 21 Apr 2022 11:53:33 -0400 Subject: [PATCH 12/27] Remove unnecessary 'set' from testing script --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 013b60e3..bca9e2ae 100644 --- a/package.json +++ b/package.json @@ -20,7 +20,7 @@ "lint:functions": "cd functions && npm install && npm run lint && cd ..", "format": "prettier --write --plugin-search-dir=. .", "doc": "jsdoc -c jsdoc.conf.json && mkdir -p docs/images && cp images/rally-logo.png docs/images/", - "test:integration:jest": "set -a && . ./functions/.testenv && jest ./tests/integration/ && set +a", + "test:integration:jest": ". ./functions/.testenv && jest ./tests/integration/", "test:integration": "npm run build:web:emulator && npm run build:functions && ./scripts/integration-test.sh", "test:functions": "firebase emulators:exec --project demo-rally --only auth,functions,firestore --ui 'npm run build:functions && sleep 1 && cd functions && npm run test:coverage && cd ..'", "storybook": "start-storybook -s ./static -p 6006", From 66e95526c5d06c1083195beab237580ed8bdde6d Mon Sep 17 00:00:00 2001 From: Ashwin Agarwal Date: Thu, 21 Apr 2022 12:00:35 -0400 Subject: [PATCH 13/27] Remove explicit true/false in cors.ts --- functions/src/cors.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/functions/src/cors.ts b/functions/src/cors.ts index cb54a59d..5d828863 100644 --- a/functions/src/cors.ts +++ b/functions/src/cors.ts @@ -2,5 +2,5 @@ import cors from "cors"; export const useCors = cors({ // Don't use CORS in testing mode - origin: process.env.NODE_ENV !== "test" ? true : false, + origin: process.env.NODE_ENV !== "test", }); From 654164dada60380d69918c800f126276a43ad54e Mon Sep 17 00:00:00 2001 From: Ashwin Agarwal Date: Thu, 21 Apr 2022 12:50:49 -0400 Subject: [PATCH 14/27] Move testenv loading to integration-test.sh --- package.json | 2 +- scripts/integration-test.sh | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index bca9e2ae..e690621e 100644 --- a/package.json +++ b/package.json @@ -20,7 +20,7 @@ "lint:functions": "cd functions && npm install && npm run lint && cd ..", "format": "prettier --write --plugin-search-dir=. .", "doc": "jsdoc -c jsdoc.conf.json && mkdir -p docs/images && cp images/rally-logo.png docs/images/", - "test:integration:jest": ". ./functions/.testenv && jest ./tests/integration/", + "test:integration:jest": "jest ./tests/integration/", "test:integration": "npm run build:web:emulator && npm run build:functions && ./scripts/integration-test.sh", "test:functions": "firebase emulators:exec --project demo-rally --only auth,functions,firestore --ui 'npm run build:functions && sleep 1 && cd functions && npm run test:coverage && cd ..'", "storybook": "start-storybook -s ./static -p 6006", diff --git a/scripts/integration-test.sh b/scripts/integration-test.sh index 864796a3..dcb157c6 100755 --- a/scripts/integration-test.sh +++ b/scripts/integration-test.sh @@ -9,6 +9,10 @@ pushd tests/integration/extension npm install && npm run build && npm run package popd +set -a +source functions/.testenv +set +a + echo "Testing Firefox, no extension" firebase emulators:exec --project demo-rally "npm run load:data && npm run test:integration:jest -- --test_browser=firefox --load_extension=false --headless_mode=true" 2>&1 | tee integration.log echo "Testing Firefox with extension" From 876506c9d727251da52ded2cab2674475bde09a1 Mon Sep 17 00:00:00 2001 From: Ashwin Agarwal Date: Thu, 21 Apr 2022 13:40:47 -0400 Subject: [PATCH 15/27] Keep function triggers enabled in UX integration test. Disable Glean instead. Add basic Glean info to README --- README.md | 6 ++++++ functions/src/__tests__/index.test.ts | 16 +++++++++++----- functions/src/glean.ts | 25 ++++++++++++++++++++----- scripts/integration-test.sh | 4 ---- tests/integration/ux.test.ts | 24 ------------------------ 5 files changed, 37 insertions(+), 38 deletions(-) diff --git a/README.md b/README.md index 8b0fb1ec..bd4aca31 100644 --- a/README.md +++ b/README.md @@ -105,6 +105,12 @@ connectFirestoreEmulator(db, "localhost", 8080); For the Rally Web Platform, this is done in: `./src/lib/stores/initialize-firebase.js` and automatically enabled when built in emulator mode. +## Glean + +The Rally Web Platform uses [Glean](https://docs.telemetry.mozilla.org/concepts/glean/glean.html) pings to send enrollment and demographic information to a secure analysis environment. + +Glean is **disabled by default** when using the Firebase Emulator (i.e. for development and testing). However it can be **explicitly enabled** by setting the `ENABLE_GLEAN` environment variable to `true`. When Glean is enabled in this way, pings will be logged, and will show up in the official Glean Debug Viewer under the [MozillaRally](https://debug-ping-preview.firebaseapp.com/pings/MozillaRally) tag (note that the ping payload is encrypted before being sent to the Debug Viewer; the data will be obfuscated but you can still see ping type and receiving time). + ## Deploying CircleCI is used to generate build artifacts, which are pushed to the `deploy` branch on this repository. diff --git a/functions/src/__tests__/index.test.ts b/functions/src/__tests__/index.test.ts index 588344c4..24c1380b 100644 --- a/functions/src/__tests__/index.test.ts +++ b/functions/src/__tests__/index.test.ts @@ -11,8 +11,7 @@ import { import { studies } from "../studies"; import fetch from "node-fetch"; -beforeAll(async () => { - // Disable function triggers for testing (prevents inadvertent Glean pings) +async function disableFunctionTriggers() { await fetch( "http://" + process.env.FIREBASE_EMULATOR_HUB + @@ -21,10 +20,9 @@ beforeAll(async () => { method: "PUT", } ); -}); +} -afterAll(async () => { - // Re-enable function triggers for potential future tests +async function enableFunctionTriggers() { await fetch( "http://" + process.env.FIREBASE_EMULATOR_HUB + @@ -33,6 +31,14 @@ afterAll(async () => { method: "PUT", } ); +} + +beforeAll(async () => { + await disableFunctionTriggers(); +}); + +afterAll(async () => { + await enableFunctionTriggers(); }); describe("loadFirestore", () => { diff --git a/functions/src/glean.ts b/functions/src/glean.ts index 2d4c2119..31997501 100644 --- a/functions/src/glean.ts +++ b/functions/src/glean.ts @@ -16,7 +16,11 @@ import { } from "@mozilla/glean/uploader"; const GLEAN_DEBUG_VIEW_TAG = "MozillaRally"; -const GLEAN_RALLY_APP_ID = process.env.NODE_ENV !== "test" ? "rally-core" : "test-app-id"; +const ENABLE_GLEAN = + process.env.ENABLE_GLEAN || !process.env.FUNCTIONS_EMULATOR; +const GLEAN_RALLY_APP_ID = process.env.FUNCTIONS_EMULATOR + ? "test-app-id" + : "rally-core"; const GLEAN_APP_DISPLAY_VERSION = "TODO_app_version"; const GLEAN_ENCRYPTION_JWK = { crv: "P-256", @@ -35,6 +39,7 @@ const submitPingFlag = withTimeout(new Semaphore(1), GLEAN_DEFAULT_TIMEOUT); // * Glean ping: enrollment */ export async function platformEnrollment(rallyID: string): Promise { + if (!ENABLE_GLEAN) return; const releaseGlean = await gleanLock.acquire(); initializeGlean(); @@ -53,6 +58,7 @@ export async function platformEnrollment(rallyID: string): Promise { * */ export async function platformUnenrollment(rallyID: string): Promise { + if (!ENABLE_GLEAN) return; const releaseGlean = await gleanLock.acquire(); initializeGlean(); @@ -73,6 +79,7 @@ export async function demographics( rallyID: string, demographicsData: Record ): Promise { + if (!ENABLE_GLEAN) return; const releaseGlean = await gleanLock.acquire(); initializeGlean(); @@ -94,6 +101,7 @@ export async function studyEnrollment( rallyID: string, studyID: string ): Promise { + if (!ENABLE_GLEAN) return; const releaseGlean = await gleanLock.acquire(); initializeGlean(); @@ -115,6 +123,7 @@ export async function studyUnenrollment( rallyID: string, studyID: string ): Promise { + if (!ENABLE_GLEAN) return; const releaseGlean = await gleanLock.acquire(); initializeGlean(); @@ -132,8 +141,12 @@ export async function studyUnenrollment( * Helper function for initializing Glean */ function initializeGlean(): void { - Glean.setDebugViewTag(GLEAN_DEBUG_VIEW_TAG); - Glean.setLogPings(true); + if (!ENABLE_GLEAN) return; + + if (process.env.FUNCTIONS_EMULATOR) { + Glean.setDebugViewTag(GLEAN_DEBUG_VIEW_TAG); + Glean.setLogPings(true); + } // Glean.initialize is a no-op if Glean is already initialized Glean.initialize(GLEAN_RALLY_APP_ID, true, { @@ -147,7 +160,8 @@ function initializeGlean(): void { * Helper function for setting user metrics * from demographic data (mapping) */ -function setUserMetrics(data: any): void { // eslint-disable-line @typescript-eslint/no-explicit-any +function setUserMetrics(data: any): void { + // eslint-disable-line @typescript-eslint/no-explicit-any if ("age" in data) { userMetrics.age[`band_${data["age"]}`].set(true); } @@ -175,7 +189,8 @@ function setUserMetrics(data: any): void { // eslint-disable-line @typescript-es } if ("school" in data) { - const KEY_FIXUP: any = { // eslint-disable-line @typescript-eslint/no-explicit-any + const KEY_FIXUP: any = { + // eslint-disable-line @typescript-eslint/no-explicit-any high_school_graduate_or_equivalent: "high_school_grad_or_eq", some_college_but_no_degree_or_in_progress: "college_degree_in_progress", }; diff --git a/scripts/integration-test.sh b/scripts/integration-test.sh index dcb157c6..864796a3 100755 --- a/scripts/integration-test.sh +++ b/scripts/integration-test.sh @@ -9,10 +9,6 @@ pushd tests/integration/extension npm install && npm run build && npm run package popd -set -a -source functions/.testenv -set +a - echo "Testing Firefox, no extension" firebase emulators:exec --project demo-rally "npm run load:data && npm run test:integration:jest -- --test_browser=firefox --load_extension=false --headless_mode=true" 2>&1 | tee integration.log echo "Testing Firefox with extension" diff --git a/tests/integration/ux.test.ts b/tests/integration/ux.test.ts index 53398366..7eaf640f 100644 --- a/tests/integration/ux.test.ts +++ b/tests/integration/ux.test.ts @@ -47,30 +47,6 @@ jest.setTimeout(60 * 10000); let driver: WebDriver; let screenshotCount = 0; -beforeAll(async () => { - // Disable function triggers for testing (prevents inadvertent Glean pings) - await fetch( - "http://" + - process.env.FIREBASE_EMULATOR_HUB + - "/functions/disableBackgroundTriggers", - { - method: "PUT", - } - ); -}); - -afterAll(async () => { - // Re-enable function triggers for potential future tests - await fetch( - "http://" + - process.env.FIREBASE_EMULATOR_HUB + - "/functions/enableBackgroundTriggers", - { - method: "PUT", - } - ); -}); - describe("Rally Web Platform UX flows", function () { beforeEach(async () => { driver = await webDriver(loadExtension, headlessMode); From 18c39f64469fa950432fd815c0fbd78171c98cb9 Mon Sep 17 00:00:00 2001 From: Ashwin Agarwal Date: Mon, 25 Apr 2022 10:18:07 -0400 Subject: [PATCH 16/27] Add app_display_version for Glean --- functions/src/glean.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/functions/src/glean.ts b/functions/src/glean.ts index 31997501..dcb2c795 100644 --- a/functions/src/glean.ts +++ b/functions/src/glean.ts @@ -2,6 +2,9 @@ import Glean from "@mozilla/glean/node"; import PingEncryptionPlugin from "@mozilla/glean/plugins/encryption"; import axios from "axios"; import { Mutex, Semaphore, withTimeout } from "async-mutex"; +import fs from "fs"; + +const RWP_pkg = JSON.parse(fs.readFileSync("../package.json").toString()); import * as rallyMetrics from "./generated/rally.js"; import * as userMetrics from "./generated/user.js"; @@ -21,7 +24,7 @@ const ENABLE_GLEAN = const GLEAN_RALLY_APP_ID = process.env.FUNCTIONS_EMULATOR ? "test-app-id" : "rally-core"; -const GLEAN_APP_DISPLAY_VERSION = "TODO_app_version"; +const GLEAN_APP_DISPLAY_VERSION = RWP_pkg.version; const GLEAN_ENCRYPTION_JWK = { crv: "P-256", kid: "rally-core", From 6451d39b14d92d4591e65b96211a3359c2ff06fa Mon Sep 17 00:00:00 2001 From: Ashwin Agarwal Date: Mon, 25 Apr 2022 10:24:31 -0400 Subject: [PATCH 17/27] Change function export format to enable better stack traces --- functions/src/index.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/functions/src/index.ts b/functions/src/index.ts index 3844cfa0..32398051 100644 --- a/functions/src/index.ts +++ b/functions/src/index.ts @@ -72,9 +72,9 @@ async function generateToken( return rallyToken; } -export const addRallyUserToFirestoreImpl = async ( +export async function addRallyUserToFirestoreImpl ( user: admin.auth.UserRecord -): Promise => { +): Promise { functions.logger.info("addRallyUserToFirestore - onCreate fired for user", { user, }); @@ -110,7 +110,7 @@ export const addRallyUserToFirestore = functions.auth .user() .onCreate(addRallyUserToFirestoreImpl); -export const deleteRallyUserImpl = async function ( +export async function deleteRallyUserImpl ( user: admin.auth.UserRecord ): Promise { functions.logger.info("deleteRallyUser fired for user:", user); @@ -182,7 +182,7 @@ export const loadFirestore = functions.https.onRequest( * Listen for changes to the User document * and initiate the appropriate Glean ping(s) */ -export const handleUserChangesImpl = async function ( +export async function handleUserChangesImpl ( change: Change, context: EventContext ): Promise { @@ -239,7 +239,7 @@ export const handleUserChanges = functions.firestore * Listen for changes to the Study document * and initiate the appropriate Glean ping(s) */ -export const handleUserStudyChangesImpl = async function ( +export async function handleUserStudyChangesImpl ( change: Change, context: EventContext ): Promise { From e5cc68778a8dd3511efe23eaef5c1ed6fe5d592b Mon Sep 17 00:00:00 2001 From: Ashwin Agarwal Date: Mon, 25 Apr 2022 10:33:09 -0400 Subject: [PATCH 18/27] Tidy up demographics logic --- functions/src/index.ts | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/functions/src/index.ts b/functions/src/index.ts index 32398051..6109625d 100644 --- a/functions/src/index.ts +++ b/functions/src/index.ts @@ -72,7 +72,7 @@ async function generateToken( return rallyToken; } -export async function addRallyUserToFirestoreImpl ( +export async function addRallyUserToFirestoreImpl( user: admin.auth.UserRecord ): Promise { functions.logger.info("addRallyUserToFirestore - onCreate fired for user", { @@ -104,13 +104,13 @@ export async function addRallyUserToFirestoreImpl ( .set(userDoc, { merge: true }); return true; -}; +} export const addRallyUserToFirestore = functions.auth .user() .onCreate(addRallyUserToFirestoreImpl); -export async function deleteRallyUserImpl ( +export async function deleteRallyUserImpl( user: admin.auth.UserRecord ): Promise { functions.logger.info("deleteRallyUser fired for user:", user); @@ -146,7 +146,7 @@ export async function deleteRallyUserImpl ( await admin.firestore().collection("users").doc(user.uid).delete(); return true; -}; +} export const deleteRallyUser = functions.auth .user() @@ -182,7 +182,7 @@ export const loadFirestore = functions.https.onRequest( * Listen for changes to the User document * and initiate the appropriate Glean ping(s) */ -export async function handleUserChangesImpl ( +export async function handleUserChangesImpl( change: Change, context: EventContext ): Promise { @@ -217,11 +217,10 @@ export async function handleUserChangesImpl ( } if ( - ((!oldUser || !oldUser.demographicsData) && newUser.demographicsData) || - (oldUser && oldUser.demographicsData && !newUser.demographicsData) || - (oldUser && - newUser && - !isDeepStrictEqual(oldUser.demographicsData, newUser.demographicsData)) + !isDeepStrictEqual( + oldUser && oldUser.demographicsData, + newUser && newUser.demographicsData + ) ) { // User updated demographicsData functions.logger.info(`Sending demographics ping for user ID ${userID}`); @@ -229,7 +228,7 @@ export async function handleUserChangesImpl ( } return true; -}; +} export const handleUserChanges = functions.firestore .document("users/{userID}") @@ -239,7 +238,7 @@ export const handleUserChanges = functions.firestore * Listen for changes to the Study document * and initiate the appropriate Glean ping(s) */ -export async function handleUserStudyChangesImpl ( +export async function handleUserStudyChangesImpl( change: Change, context: EventContext ): Promise { @@ -294,7 +293,7 @@ export async function handleUserStudyChangesImpl ( } return true; -}; +} export const handleUserStudyChanges = functions.firestore .document("users/{userID}/studies/{studyID}") From 59294da6b1aaf9362016457bb1eb24c423a05f07 Mon Sep 17 00:00:00 2001 From: Ashwin Agarwal Date: Mon, 25 Apr 2022 10:34:10 -0400 Subject: [PATCH 19/27] Fix typo --- functions/.eslintrc.cjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/functions/.eslintrc.cjs b/functions/.eslintrc.cjs index 4ad5af28..65d11270 100644 --- a/functions/.eslintrc.cjs +++ b/functions/.eslintrc.cjs @@ -24,6 +24,6 @@ module.exports = { plugins: ["@typescript-eslint", "import"], rules: { quotes: ["error", "double"], - "import/no-unresolved": "off" // too many weird module resoltions in this project + "import/no-unresolved": "off" // too many weird module resolutions in this project }, }; From 8d7c402be1d4fe126ac49180423755c4e1420eb8 Mon Sep 17 00:00:00 2001 From: Ashwin Agarwal Date: Mon, 25 Apr 2022 11:12:21 -0400 Subject: [PATCH 20/27] Fix up some types --- .../src/__tests__/authentication.test.ts | 22 +++++++++---------- functions/src/glean.ts | 6 ++--- 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/functions/src/__tests__/authentication.test.ts b/functions/src/__tests__/authentication.test.ts index 2db46411..51ed2085 100644 --- a/functions/src/__tests__/authentication.test.ts +++ b/functions/src/__tests__/authentication.test.ts @@ -35,7 +35,7 @@ describe("useAuthentication", () => { }); it("throws when request is null", async () => { - const fn: any = jest.fn(); // eslint-disable-line @typescript-eslint/no-explicit-any + const fn: AuthenticatedFunction = jest.fn(); await expect(() => useAuthentication( @@ -49,7 +49,7 @@ describe("useAuthentication", () => { }); it("throws when request is undefined", async () => { - const fn: any = jest.fn(); // eslint-disable-line @typescript-eslint/no-explicit-any + const fn: AuthenticatedFunction = jest.fn(); await expect(() => useAuthentication( @@ -63,7 +63,7 @@ describe("useAuthentication", () => { }); it("throws when response is null", async () => { - const fn: any = jest.fn(); // eslint-disable-line @typescript-eslint/no-explicit-any + const fn: AuthenticatedFunction = jest.fn(); await expect(() => useAuthentication( @@ -77,7 +77,7 @@ describe("useAuthentication", () => { }); it("throws when response is undefined", async () => { - const fn: any = jest.fn(); // eslint-disable-line @typescript-eslint/no-explicit-any + const fn: AuthenticatedFunction = jest.fn(); await expect(() => useAuthentication( @@ -121,7 +121,7 @@ describe("useAuthentication", () => { }); it("throws http error 401 when request is missing", async () => { - const fn: any = jest.fn(); // eslint-disable-line @typescript-eslint/no-explicit-any + const fn: AuthenticatedFunction = jest.fn(); await useAuthentication( ({} as unknown) as functions.https.Request, @@ -135,7 +135,7 @@ describe("useAuthentication", () => { }); it("throws http error 401 when request authorization header is missing", async () => { - const fn: any = jest.fn(); // eslint-disable-line @typescript-eslint/no-explicit-any + const fn: AuthenticatedFunction = jest.fn(); await useAuthentication( { ...request, headers: {} } as functions.https.Request, @@ -149,7 +149,7 @@ describe("useAuthentication", () => { }); it("throws http error 401 when bearer prefix is missing", async () => { - const fn: any = jest.fn(); // eslint-disable-line @typescript-eslint/no-explicit-any + const fn: AuthenticatedFunction = jest.fn(); await useAuthentication( { @@ -166,7 +166,7 @@ describe("useAuthentication", () => { }); it("throws http error 401 when bearer prefix is not pascal cased", async () => { - const fn: any = jest.fn(); // eslint-disable-line @typescript-eslint/no-explicit-any + const fn: AuthenticatedFunction = jest.fn(); await useAuthentication( { @@ -183,7 +183,7 @@ describe("useAuthentication", () => { }); it("throws http error 401 when auth header has more than 2 parts", async () => { - const fn: any = jest.fn(); // eslint-disable-line @typescript-eslint/no-explicit-any + const fn: AuthenticatedFunction = jest.fn(); await useAuthentication( { @@ -202,7 +202,7 @@ describe("useAuthentication", () => { it("throws http error 401 when token is invalid", async () => { fakeAuth.verifyIdToken.mockRejectedValue("Invalid token"); - const fn: any = jest.fn(); // eslint-disable-line @typescript-eslint/no-explicit-any + const fn: AuthenticatedFunction = jest.fn(); await useAuthentication(request, response, fn); expect(response.status).toHaveBeenCalledWith(401); @@ -218,7 +218,7 @@ describe("useAuthentication", () => { fakeAuth.verifyIdToken.mockReturnValue(decryptedToken); - const innerFn: any = jest.fn(); // eslint-disable-line @typescript-eslint/no-explicit-any + const innerFn: AuthenticatedFunction = jest.fn(); await useAuthentication(request, response, innerFn); expect(innerFn).toHaveBeenCalledWith(decryptedToken); diff --git a/functions/src/glean.ts b/functions/src/glean.ts index dcb2c795..d69b567d 100644 --- a/functions/src/glean.ts +++ b/functions/src/glean.ts @@ -163,8 +163,7 @@ function initializeGlean(): void { * Helper function for setting user metrics * from demographic data (mapping) */ -function setUserMetrics(data: any): void { - // eslint-disable-line @typescript-eslint/no-explicit-any +function setUserMetrics(data: any): void { // eslint-disable-line @typescript-eslint/no-explicit-any if ("age" in data) { userMetrics.age[`band_${data["age"]}`].set(true); } @@ -192,8 +191,7 @@ function setUserMetrics(data: any): void { } if ("school" in data) { - const KEY_FIXUP: any = { - // eslint-disable-line @typescript-eslint/no-explicit-any + const KEY_FIXUP: any = { // eslint-disable-line @typescript-eslint/no-explicit-any high_school_graduate_or_equivalent: "high_school_grad_or_eq", some_college_but_no_degree_or_in_progress: "college_degree_in_progress", }; From df236a24bc600bf8fad50945d2f8f4f848a71ed0 Mon Sep 17 00:00:00 2001 From: Ashwin Agarwal Date: Mon, 25 Apr 2022 14:38:29 -0400 Subject: [PATCH 21/27] tidy up return value --- functions/src/index.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/functions/src/index.ts b/functions/src/index.ts index 6109625d..3bb6bb6e 100644 --- a/functions/src/index.ts +++ b/functions/src/index.ts @@ -308,9 +308,5 @@ async function getRallyIdForUser(userID: string) { const data = extensionUserDoc.data(); - if (data) { - return data.rallyId; - } else { - return null; - } + return (data && data.rallyId) || null; } From 6400587354028c70b0c2569ff8e4740d066157e0 Mon Sep 17 00:00:00 2001 From: Ashwin Agarwal Date: Mon, 25 Apr 2022 17:58:00 -0400 Subject: [PATCH 22/27] Fix demographic logic --- functions/src/glean.ts | 2 ++ functions/src/index.ts | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/functions/src/glean.ts b/functions/src/glean.ts index d69b567d..ba096475 100644 --- a/functions/src/glean.ts +++ b/functions/src/glean.ts @@ -164,6 +164,8 @@ function initializeGlean(): void { * from demographic data (mapping) */ function setUserMetrics(data: any): void { // eslint-disable-line @typescript-eslint/no-explicit-any + if (!data) return; + if ("age" in data) { userMetrics.age[`band_${data["age"]}`].set(true); } diff --git a/functions/src/index.ts b/functions/src/index.ts index 3bb6bb6e..d97d4cae 100644 --- a/functions/src/index.ts +++ b/functions/src/index.ts @@ -224,7 +224,7 @@ export async function handleUserChangesImpl( ) { // User updated demographicsData functions.logger.info(`Sending demographics ping for user ID ${userID}`); - await gleanPings.demographics(rallyID, newUser.demographicsData); + await gleanPings.demographics(rallyID, newUser && newUser.demographicsData); } return true; From 18cc63a1c1c38effdbbeb7fb6aefa722ddb92138 Mon Sep 17 00:00:00 2001 From: Ashwin Agarwal Date: Mon, 25 Apr 2022 17:58:35 -0400 Subject: [PATCH 23/27] Remove allowJS: true, no longer needed --- functions/tsconfig.json | 1 - 1 file changed, 1 deletion(-) diff --git a/functions/tsconfig.json b/functions/tsconfig.json index e2cfca16..bb27a8f2 100644 --- a/functions/tsconfig.json +++ b/functions/tsconfig.json @@ -3,7 +3,6 @@ "module": "esnext", "moduleResolution": "node", "allowSyntheticDefaultImports": true, - "allowJs": true, "esModuleInterop": false, "noImplicitReturns": true, "noUnusedLocals": true, From f37a2486a653f3141208d5116d7c4d94565ad12e Mon Sep 17 00:00:00 2001 From: Ashwin Agarwal Date: Mon, 25 Apr 2022 18:12:29 -0400 Subject: [PATCH 24/27] Use function logger instead of console.log --- functions/src/glean.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/functions/src/glean.ts b/functions/src/glean.ts index ba096475..6dbbd3fb 100644 --- a/functions/src/glean.ts +++ b/functions/src/glean.ts @@ -1,6 +1,7 @@ import Glean from "@mozilla/glean/node"; import PingEncryptionPlugin from "@mozilla/glean/plugins/encryption"; import axios from "axios"; +import functions from "firebase-functions"; import { Mutex, Semaphore, withTimeout } from "async-mutex"; import fs from "fs"; @@ -232,7 +233,7 @@ class CustomPingUploader extends Uploader { }; }) .catch(function (error) { - console.log(error); + functions.logger.error(error); return { status: 500, result: UploadResultStatus.UnrecoverableFailure, From cc5489c88213cca17334804a365c9c39f3b6940f Mon Sep 17 00:00:00 2001 From: Ashwin Agarwal Date: Tue, 26 Apr 2022 17:45:44 -0400 Subject: [PATCH 25/27] Clean up logic --- functions/src/index.ts | 57 ++++++++++++++++++++---------------------- 1 file changed, 27 insertions(+), 30 deletions(-) diff --git a/functions/src/index.ts b/functions/src/index.ts index d97d4cae..2e56190e 100644 --- a/functions/src/index.ts +++ b/functions/src/index.ts @@ -7,6 +7,7 @@ import { useCors } from "./cors"; import { studies } from "./studies"; import { isDeepStrictEqual } from "util"; import * as gleanPings from "./glean"; +import assert from "assert"; admin.initializeApp({ credential: admin.credential.applicationDefault(), @@ -188,13 +189,13 @@ export async function handleUserChangesImpl( ): Promise { const userID = context.params.userID; const rallyID = await getRallyIdForUser(userID); - if (!rallyID) { - // Without Rally ID, we can't make any Glean pings - // This is bad and should be flagged for inspection - throw new Error( - `Unable to obtain Rally ID for user ID ${userID}. Aborting Glean ping process.` - ); - } + + // Without Rally ID, we can't make any Glean pings + // This is bad and should be flagged for inspection + assert( + rallyID, + `Unable to obtain Rally ID for user ID ${userID}. Aborting Glean ping process.` + ); // Get an object with the current document value. // If the document does not exist, it has been deleted. @@ -203,14 +204,14 @@ export async function handleUserChangesImpl( // Get the old document, to compare the enrollment state. const oldUser = change.before.exists ? change.before.data() : null; - if (!newUser || (oldUser && oldUser.enrolled === true && !newUser.enrolled)) { + if (!newUser || (oldUser && oldUser.enrolled && !newUser.enrolled)) { // User document has been deleted functions.logger.info(`Sending unenrollment ping for user ID ${userID}`); await gleanPings.platformUnenrollment(rallyID); return true; } - if ((!oldUser || !oldUser.enrolled) && newUser.enrolled === true) { + if ((!oldUser || !oldUser.enrolled) && newUser.enrolled) { // User just enrolled functions.logger.info(`Sending enrollment ping for user ID ${userID}`); await gleanPings.platformEnrollment(rallyID); @@ -245,13 +246,13 @@ export async function handleUserStudyChangesImpl( const userID = context.params.userID; const firebaseStudyID = context.params.studyID; const rallyID = await getRallyIdForUser(userID); - if (!rallyID) { - // Without Rally ID, we can't make any Glean pings - // This is bad and should be flagged for inspection - throw new Error( - `Unable to obtain Rally ID for user ID ${userID}. Aborting Glean ping process.` - ); - } + + // Without Rally ID, we can't make any Glean pings + // This is bad and should be flagged for inspection + assert( + rallyID, + `Unable to obtain Rally ID for user ID ${userID}. Aborting Glean ping process.` + ); // Get an object with the current document value. // If the document does not exist, it has been deleted. @@ -261,20 +262,16 @@ export async function handleUserStudyChangesImpl( const oldStudy = change.before.exists ? change.before.data() : null; const studyID = - (newStudy ? newStudy.studyId : null) || - (oldStudy ? oldStudy.studyId : null); - if (!studyID) { - // Without Study ID, we can't construct study-related Glean pings - // This is bad and should be flagged for inspection - throw new Error( - `Couldn't find Glean Study ID for user ID ${userID} and Firebase study ID ${firebaseStudyID}. Aborting Glean ping process.` - ); - } + (newStudy && newStudy.studyId) || (oldStudy && oldStudy.studyId); - if ( - !newStudy || - (oldStudy && oldStudy.enrolled === true && !newStudy.enrolled) - ) { + // Without Study ID, we can't construct study-related Glean pings + // This is bad and should be flagged for inspection + assert( + studyID, + `Couldn't find Glean Study ID for user ID ${userID} and Firebase study ID ${firebaseStudyID}. Aborting Glean ping process.` + ); + + if (!newStudy || (oldStudy && oldStudy.enrolled && !newStudy.enrolled)) { // User unenrolled from study functions.logger.info( `Sending unenrollment ping for study with user ID ${userID} with study ID ${studyID}` @@ -283,7 +280,7 @@ export async function handleUserStudyChangesImpl( return true; } - if ((!oldStudy || !oldStudy.enrolled) && newStudy.enrolled === true) { + if ((!oldStudy || !oldStudy.enrolled) && newStudy.enrolled) { // User just enrolled in this study functions.logger.info( `Sending enrollment ping for study with user ID ${userID} with study ID ${studyID}` From cd9923c15dcf32672ea3d1bba8f1b51eff8575e3 Mon Sep 17 00:00:00 2001 From: Ashwin Agarwal Date: Tue, 26 Apr 2022 18:49:31 -0400 Subject: [PATCH 26/27] Change order of extensionUser deletion --- functions/src/__tests__/index.test.ts | 1 - functions/src/index.ts | 11 +++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/functions/src/__tests__/index.test.ts b/functions/src/__tests__/index.test.ts index 24c1380b..06a4c390 100644 --- a/functions/src/__tests__/index.test.ts +++ b/functions/src/__tests__/index.test.ts @@ -166,7 +166,6 @@ describe("addRallyUserToFirestore and deleteRallyUserImpl", () => { const userRecords = await getUserRecords(); expect(userRecords.user.exists).toBeFalsy(); - expect(userRecords.extensionUser.exists).toBeFalsy(); }); }); diff --git a/functions/src/index.ts b/functions/src/index.ts index 2e56190e..fd3604e7 100644 --- a/functions/src/index.ts +++ b/functions/src/index.ts @@ -116,9 +116,6 @@ export async function deleteRallyUserImpl( ): Promise { functions.logger.info("deleteRallyUser fired for user:", user); - // Delete the extension user document. - await admin.firestore().collection("extensionUsers").doc(user.uid).delete(); - // Delete the user studies subcollection. const collectionRef = admin .firestore() @@ -201,11 +198,17 @@ export async function handleUserChangesImpl( // If the document does not exist, it has been deleted. const newUser = change.after.exists ? change.after.data() : null; + if (!newUser) { + // User document was deleted + // Delete the extension user document, now that we've obtained the rallyID + await admin.firestore().collection("extensionUsers").doc(userID).delete(); + } + // Get the old document, to compare the enrollment state. const oldUser = change.before.exists ? change.before.data() : null; if (!newUser || (oldUser && oldUser.enrolled && !newUser.enrolled)) { - // User document has been deleted + // User document has unenrolled functions.logger.info(`Sending unenrollment ping for user ID ${userID}`); await gleanPings.platformUnenrollment(rallyID); return true; From d602f8dc595a07d3a82a204f523d209e10a53af5 Mon Sep 17 00:00:00 2001 From: Ashwin Agarwal Date: Wed, 27 Apr 2022 13:07:39 -0400 Subject: [PATCH 27/27] Change rallytoken tests to use callback instead of await; remove Promise from rallytoken implementation --- functions/src/__tests__/index.test.ts | 97 ++++++++++++++------------- functions/src/index.ts | 66 +++++++++--------- 2 files changed, 81 insertions(+), 82 deletions(-) diff --git a/functions/src/__tests__/index.test.ts b/functions/src/__tests__/index.test.ts index 06a4c390..29b9dc3a 100644 --- a/functions/src/__tests__/index.test.ts +++ b/functions/src/__tests__/index.test.ts @@ -173,6 +173,19 @@ describe("rallytoken tests", () => { let send: any; // eslint-disable-line @typescript-eslint/no-explicit-any let response: functions.Response; // eslint-disable-line @typescript-eslint/no-explicit-any + // Set up callbacks inside response.status.send + function doAfterResponseSend(validateFn: () => void, doneFn: () => void) { + send = jest.fn().mockImplementation(() => { + validateFn(); // Jest assertions + doneFn(); // Complete unit test + }); + + response = ({ + set: jest.fn(), + status: jest.fn().mockReturnValue({ send }), + } as unknown) as functions.Response; // eslint-disable-line @typescript-eslint/no-explicit-any + } + const uid = "fake-uid"; const customToken = "fake-custom-token"; @@ -190,12 +203,6 @@ describe("rallytoken tests", () => { beforeEach(() => { jest.resetAllMocks(); - send = jest.fn(); - response = ({ - set: jest.fn(), - status: jest.fn().mockReturnValue({ send }), - } as unknown) as functions.Response; // eslint-disable-line @typescript-eslint/no-explicit-any - fakeAuth.verifyIdToken.mockReturnValue({ uid }); fakeAuth.createCustomToken.mockReturnValue(customToken); }); @@ -204,8 +211,15 @@ describe("rallytoken tests", () => { jest.resetAllMocks(); }); - it("fails for invalid http verb", async () => { - await rallytoken( + it("fails for invalid http verb", (done) => { + doAfterResponseSend(() => { + expect(response.status).toHaveBeenCalledWith(500); + expect(send).toHaveBeenCalledWith( + "Only POST and OPTIONS methods are allowed." + ); + }, done); + + rallytoken( { method: "PUT", headers: { @@ -213,16 +227,16 @@ describe("rallytoken tests", () => { }, } as functions.Request, // eslint-disable-line @typescript-eslint/no-explicit-any response - ); // eslint-disable-line @typescript-eslint/no-explicit-any - - expect(response.status).toHaveBeenCalledWith(500); - expect(send).toHaveBeenCalledWith( - "Only POST and OPTIONS methods are allowed." ); }); - it("fails when POST is invoked with invalid payload", async () => { - await rallytoken( + it("fails when POST is invoked with invalid payload", (done) => { + doAfterResponseSend(() => { + expect(response.status).toHaveBeenCalledWith(500); + expect(send).toHaveBeenCalled(); + }, done); + + rallytoken( { method: "POST", headers: { @@ -230,32 +244,29 @@ describe("rallytoken tests", () => { }, } as functions.Request, // eslint-disable-line @typescript-eslint/no-explicit-any response - ); // eslint-disable-line @typescript-eslint/no-explicit-any - - expect(response.status).toHaveBeenCalledWith(500); - expect(send).toHaveBeenCalled(); + ); }); - it("handles payload in POST request", async () => { - const idToken = "idToken"; - const studyId = "study1"; + const idToken = "idToken"; + const studyId = "study1"; + const successValidateFn = () => { + expect(response.status).toHaveBeenCalledWith(200); - const validateFn = () => { - expect(response.status).toHaveBeenCalledWith(200); - - expect(fakeAuth.verifyIdToken).toHaveBeenCalledWith(idToken); - expect(fakeAuth.createCustomToken).toHaveBeenCalledWith( - `${studyId}:${uid}`, - { - firebaseUid: uid, - studyId, - } - ); + expect(fakeAuth.verifyIdToken).toHaveBeenCalledWith(idToken); + expect(fakeAuth.createCustomToken).toHaveBeenCalledWith( + `${studyId}:${uid}`, + { + firebaseUid: uid, + studyId, + } + ); - expect(send.mock.calls[0][0]).toEqual({ rallyToken: customToken }); - }; + expect(send.mock.calls[0][0]).toEqual({ rallyToken: customToken }); + }; - await rallytoken( + it("handles payload string in POST request", (done) => { + doAfterResponseSend(successValidateFn, done); + rallytoken( { method: "POST", headers: { @@ -265,15 +276,11 @@ describe("rallytoken tests", () => { } as functions.Request, // eslint-disable-line @typescript-eslint/no-explicit-any response ); + }); - validateFn(); - - (response.status as jest.Mock).mockClear(); - (fakeAuth.verifyIdToken as jest.Mock).mockClear(); - (fakeAuth.createCustomToken as jest.Mock).mockClear(); - send.mockClear(); - - await rallytoken( + it("handles payload JSON in POST request", (done) => { + doAfterResponseSend(successValidateFn, done); + rallytoken( { method: "POST", headers: { @@ -283,7 +290,5 @@ describe("rallytoken tests", () => { } as functions.Request, // eslint-disable-line @typescript-eslint/no-explicit-any response ); - - validateFn(); }); }); diff --git a/functions/src/index.ts b/functions/src/index.ts index fd3604e7..e4b10d96 100644 --- a/functions/src/index.ts +++ b/functions/src/index.ts @@ -13,42 +13,36 @@ admin.initializeApp({ credential: admin.credential.applicationDefault(), }); -export const rallytoken = functions.https.onRequest( - async (request, response) => - new Promise((resolve) => - useCors(request, response, async () => { - await useAuthentication(request, response, async (decodedToken) => { - if (request.method !== "POST") { - response - .status(500) - .send("Only POST and OPTIONS methods are allowed."); - return; - } - - functions.logger.info(`body type: ${typeof request.body}`, { - payload: request.body, - }); - - try { - let studyId; - if (typeof request.body === "string") { - const body = JSON.parse(request.body); - studyId = body.studyId; - } else { - studyId = request.body.studyId; - } - - const rallyToken = await generateToken(decodedToken, studyId); - functions.logger.info("OK"); - response.status(200).send({ rallyToken }); - } catch (ex) { - functions.logger.error(ex); - response.status(500).send(); - } - }); - resolve(); - }) - ) +export const rallytoken = functions.https.onRequest(async (request, response) => + useCors(request, response, async () => { + await useAuthentication(request, response, async (decodedToken) => { + if (request.method !== "POST") { + response.status(500).send("Only POST and OPTIONS methods are allowed."); + return; + } + + functions.logger.info(`body type: ${typeof request.body}`, { + payload: request.body, + }); + + try { + let studyId; + if (typeof request.body === "string") { + const body = JSON.parse(request.body); + studyId = body.studyId; + } else { + studyId = request.body.studyId; + } + + const rallyToken = await generateToken(decodedToken, studyId); + functions.logger.info("OK"); + response.status(200).send({ rallyToken }); + } catch (ex) { + functions.logger.error(ex); + response.status(500).send(); + } + }); + }) ); /**