Skip to content

Commit

Permalink
Pull latest component versions from amphtml repo (#1235)
Browse files Browse the repository at this point in the history
* Use different endpoint to fetch latest component prod versions

* Use extension config from amphtml repo

* update boilerplate error handler to fix validation errors
  • Loading branch information
sebastianbenz committed May 31, 2021
1 parent b02027c commit e85d2a8
Show file tree
Hide file tree
Showing 13 changed files with 101 additions and 31 deletions.
55 changes: 55 additions & 0 deletions packages/optimizer/build/downloadRuntimeParameters.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
/**
* Copyright 2020 The AMP HTML Authors. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS-IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
'mode strict';

const RuntimeVersion = require('@ampproject/toolbox-runtime-version/lib/RuntimeVersion');
const fetch = require('node-fetch');
const fs = require('fs').promises;
const path = require('path');
const fetchRuntimeParameters = require('../lib/fetchRuntimeParameters');

(async () => {
const runtimeParameters = await fetchRuntimeParameters({
runtimeVersion: new RuntimeVersion(fetch),
fetch,
cache: false,
log: console,
});

const runtimeData = {
ampRuntimeStyles: runtimeParameters.ampRuntimeStyles,
ampRuntimeVersion: runtimeParameters.ampRuntimeVersion,
};

fs.writeFile(
path.join(__dirname, '../lib/runtimeData.json'),
JSON.stringify(runtimeData),
'utf-8'
);

const extensionConfigUrl =
'https://raw.githubusercontent.com/ampproject/amphtml/main/build-system/compile/bundles.config.extensions.json';
const response = await fetch(extensionConfigUrl);
if (!response.ok) {
throw new Error(`Failed downloading ${extensionConfigUrl} with status ${response.status}`);
}
const extensionConfig = await response.json();
fs.writeFile(
path.join(__dirname, '../lib/extensionConfig.json'),
JSON.stringify(extensionConfig),
'utf-8'
);
})();
1 change: 1 addition & 0 deletions packages/optimizer/lib/extensionConfig.json
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
[{"name":"amp-3d-gltf","version":"0.1","latestVersion":"0.1"},{"name":"amp-3q-player","version":"0.1","latestVersion":"0.1"},{"name":"amp-access","version":"0.1","latestVersion":"0.1","options":{"hasCss":true}},{"name":"amp-access-laterpay","version":["0.1","0.2"],"latestVersion":"0.2","options":{"hasCss":true}},{"name":"amp-access-poool","version":"0.1","latestVersion":"0.1"},{"name":"amp-access-scroll","version":"0.1","latestVersion":"0.1","options":{"hasCss":true}},{"name":"amp-accordion","version":"0.1","latestVersion":"0.1","options":{"hasCss":true}},{"name":"amp-accordion","version":"1.0","latestVersion":"0.1","options":{"hasCss":true,"npm":true}},{"name":"amp-action-macro","version":"0.1","latestVersion":"0.1"},{"name":"amp-ad","version":"0.1","latestVersion":"0.1","options":{"hasCss":true}},{"name":"amp-ad-custom","version":"0.1","latestVersion":"0.1"},{"name":"amp-ad-exit","version":"0.1","latestVersion":"0.1"},{"name":"amp-ad-network-adsense-impl","version":"0.1","latestVersion":"0.1"},{"name":"amp-ad-network-adzerk-impl","version":"0.1","latestVersion":"0.1"},{"name":"amp-ad-network-doubleclick-impl","version":"0.1","latestVersion":"0.1"},{"name":"amp-ad-network-fake-impl","version":"0.1","latestVersion":"0.1"},{"name":"amp-ad-network-nws-impl","version":"0.1","latestVersion":"0.1"},{"name":"amp-ad-network-valueimpression-impl","version":"0.1","latestVersion":"0.1"},{"name":"amp-addthis","version":"0.1","latestVersion":"0.1","options":{"hasCss":true}},{"name":"amp-analytics","version":"0.1","latestVersion":"0.1"},{"name":"amp-anim","version":"0.1","latestVersion":"0.1"},{"name":"amp-animation","version":"0.1","latestVersion":"0.1"},{"name":"amp-animation-polyfill","version":"0.1","latestVersion":"0.1"},{"name":"amp-apester-media","version":"0.1","latestVersion":"0.1","options":{"hasCss":true}},{"name":"amp-app-banner","version":"0.1","latestVersion":"0.1","options":{"hasCss":true}},{"name":"amp-audio","version":"0.1","latestVersion":"0.1"},{"name":"amp-auto-ads","version":"0.1","latestVersion":"0.1"},{"name":"amp-auto-lightbox","version":"0.1","latestVersion":"0.1"},{"name":"amp-autocomplete","version":"0.1","latestVersion":"0.1","options":{"hasCss":true}},{"name":"amp-base-carousel","version":"0.1","latestVersion":"0.1","options":{"hasCss":true}},{"name":"amp-base-carousel","version":"1.0","latestVersion":"0.1","options":{"hasCss":true,"npm":true}},{"name":"amp-beopinion","version":"0.1","latestVersion":"0.1"},{"name":"amp-bind","version":"0.1","latestVersion":"0.1"},{"name":"amp-bodymovin-animation","version":"0.1","latestVersion":"0.1"},{"name":"amp-brid-player","version":"0.1","latestVersion":"0.1"},{"name":"amp-brightcove","version":"0.1","latestVersion":"0.1"},{"name":"amp-byside-content","version":"0.1","latestVersion":"0.1","options":{"hasCss":true}},{"name":"amp-cache-url","version":"0.1","latestVersion":"0.1"},{"name":"amp-call-tracking","version":"0.1","latestVersion":"0.1"},{"name":"amp-carousel","version":["0.1","0.2"],"latestVersion":"0.1","options":{"hasCss":true}},{"name":"amp-connatix-player","version":"0.1","latestVersion":"0.1"},{"name":"amp-consent","version":"0.1","latestVersion":"0.1","options":{"hasCss":true}},{"name":"amp-crypto-polyfill","version":"0.1","latestVersion":"0.1"},{"name":"amp-dailymotion","version":"0.1","latestVersion":"0.1"},{"name":"amp-date-countdown","version":"0.1","latestVersion":"0.1"},{"name":"amp-date-countdown","version":"1.0","latestVersion":"0.1","options":{"npm":true}},{"name":"amp-date-display","version":"0.1","latestVersion":"0.1"},{"name":"amp-date-display","version":"1.0","latestVersion":"0.1","options":{"npm":true}},{"name":"amp-date-picker","version":"0.1","latestVersion":"0.1","options":{"hasCss":true}},{"name":"amp-delight-player","version":"0.1","latestVersion":"0.1","options":{"hasCss":true}},{"name":"amp-dynamic-css-classes","version":"0.1","latestVersion":"0.1"},{"name":"amp-embedly-card","version":"0.1","latestVersion":"0.1"},{"name":"amp-experiment","version":["0.1","1.0"],"latestVersion":"0.1"},{"name":"amp-facebook","version":"0.1","latestVersion":"0.1"},{"name":"amp-facebook-comments","version":["0.1","1.0"],"latestVersion":"0.1"},{"name":"amp-facebook-like","version":"0.1","latestVersion":"0.1"},{"name":"amp-facebook-page","version":"0.1","latestVersion":"0.1"},{"name":"amp-fit-text","version":"0.1","latestVersion":"0.1","options":{"hasCss":true}},{"name":"amp-fit-text","version":"1.0","latestVersion":"0.1","options":{"hasCss":true,"npm":true}},{"name":"amp-font","version":"0.1","latestVersion":"0.1"},{"name":"amp-form","version":"0.1","latestVersion":"0.1","options":{"hasCss":true}},{"name":"amp-fx-collection","version":"0.1","latestVersion":"0.1"},{"name":"amp-fx-flying-carpet","version":"0.1","latestVersion":"0.1","options":{"hasCss":true}},{"name":"amp-geo","version":"0.1","latestVersion":"0.1"},{"name":"amp-gfycat","version":"0.1","latestVersion":"0.1"},{"name":"amp-gist","version":"0.1","latestVersion":"0.1"},{"name":"amp-google-assistant-assistjs","version":"0.1","latestVersion":"0.1","options":{"hasCss":true}},{"name":"amp-google-document-embed","version":"0.1","latestVersion":"0.1"},{"name":"amp-gwd-animation","version":"0.1","latestVersion":"0.1","options":{"hasCss":true}},{"name":"amp-hulu","version":"0.1","latestVersion":"0.1"},{"name":"amp-iframe","version":"0.1","latestVersion":"0.1"},{"name":"amp-ima-video","version":"0.1","latestVersion":"0.1"},{"name":"amp-image-lightbox","version":"0.1","latestVersion":"0.1","options":{"hasCss":true}},{"name":"amp-image-slider","version":"0.1","latestVersion":"0.1","options":{"hasCss":true}},{"name":"amp-image-viewer","version":"0.1","latestVersion":"0.1","options":{"hasCss":true}},{"name":"amp-imgur","version":"0.1","latestVersion":"0.1"},{"name":"amp-inline-gallery","version":"0.1","latestVersion":"0.1","options":{"hasCss":true,"cssBinaries":["amp-inline-gallery","amp-inline-gallery-captions","amp-inline-gallery-pagination","amp-inline-gallery-slide","amp-inline-gallery-thumbnails"]}},{"name":"amp-inline-gallery","version":"1.0","latestVersion":"0.1","options":{"hasCss":true,"cssBinaries":["amp-inline-gallery-pagination"],"npm":true}},{"name":"amp-inputmask","version":"0.1","latestVersion":"0.1"},{"name":"amp-instagram","version":"0.1","latestVersion":"0.1","options":{"hasCss":true}},{"name":"amp-instagram","version":"1.0","latestVersion":"0.1","options":{"hasCss":true,"npm":true}},{"name":"amp-install-serviceworker","version":"0.1","latestVersion":"0.1"},{"name":"amp-intersection-observer-polyfill","version":"0.1","latestVersion":"0.1"},{"name":"amp-izlesene","version":"0.1","latestVersion":"0.1"},{"name":"amp-jwplayer","version":"0.1","latestVersion":"0.1"},{"name":"amp-kaltura-player","version":"0.1","latestVersion":"0.1"},{"name":"amp-lightbox","version":"0.1","latestVersion":"0.1","options":{"hasCss":true}},{"name":"amp-lightbox","version":"1.0","latestVersion":"0.1","options":{"hasCss":true,"npm":true}},{"name":"amp-lightbox-gallery","version":"0.1","latestVersion":"0.1","options":{"hasCss":true}},{"name":"amp-link-rewriter","version":"0.1","latestVersion":"0.1"},{"name":"amp-list","version":"0.1","latestVersion":"0.1","options":{"hasCss":true}},{"name":"amp-live-list","version":"0.1","latestVersion":"0.1","options":{"hasCss":true}},{"name":"amp-loader","version":"0.1","latestVersion":"0.1","options":{"hasCss":true}},{"name":"amp-mathml","version":"0.1","latestVersion":"0.1","options":{"hasCss":true}},{"name":"amp-mega-menu","version":"0.1","latestVersion":"0.1","options":{"hasCss":true}},{"name":"amp-megaphone","version":"0.1","latestVersion":"0.1"},{"name":"amp-minute-media-player","version":"0.1","latestVersion":"0.1"},{"name":"amp-mowplayer","version":"0.1","latestVersion":"0.1"},{"name":"amp-mraid","version":"0.1","latestVersion":"0.1"},{"name":"amp-mustache","version":["0.1","0.2"],"latestVersion":"0.2"},{"name":"amp-nested-menu","version":"0.1","latestVersion":"0.1","options":{"hasCss":true}},{"name":"amp-next-page","version":["0.1","1.0"],"latestVersion":"1.0","options":{"hasCss":true}},{"name":"amp-nexxtv-player","version":"0.1","latestVersion":"0.1"},{"name":"amp-o2-player","version":"0.1","latestVersion":"0.1"},{"name":"amp-onetap-google","version":"0.1","latestVersion":"0.1","options":{"hasCss":true}},{"name":"amp-ooyala-player","version":"0.1","latestVersion":"0.1"},{"name":"amp-orientation-observer","version":"0.1","latestVersion":"0.1"},{"name":"amp-pan-zoom","version":"0.1","latestVersion":"0.1","options":{"hasCss":true}},{"name":"amp-pinterest","version":"0.1","latestVersion":"0.1","options":{"hasCss":true}},{"name":"amp-playbuzz","version":"0.1","latestVersion":"0.1","options":{"hasCss":true}},{"name":"amp-position-observer","version":"0.1","latestVersion":"0.1"},{"name":"amp-powr-player","version":"0.1","latestVersion":"0.1"},{"name":"amp-reach-player","version":"0.1","latestVersion":"0.1"},{"name":"amp-recaptcha-input","version":"0.1","latestVersion":"0.1","options":{"hasCss":true}},{"name":"amp-redbull-player","version":"0.1","latestVersion":"0.1"},{"name":"amp-reddit","version":"0.1","latestVersion":"0.1"},{"name":"amp-render","version":"1.0","latestVersion":"1.0"},{"name":"amp-resize-observer-polyfill","version":"0.1","latestVersion":"0.1"},{"name":"amp-riddle-quiz","version":"0.1","latestVersion":"0.1"},{"name":"amp-script","version":"0.1","latestVersion":"0.1","options":{"hasCss":true}},{"name":"amp-selector","version":"0.1","latestVersion":"0.1","options":{"hasCss":true}},{"name":"amp-selector","version":"1.0","latestVersion":"0.1","options":{"hasCss":true,"npm":true}},{"name":"amp-shadow-dom-polyfill","version":"0.1","latestVersion":"0.1","options":{"noWrapper":true}},{"name":"amp-sidebar","version":["0.1","0.2","1.0"],"latestVersion":"0.1","options":{"hasCss":true}},{"name":"amp-skimlinks","version":"0.1","latestVersion":"0.1"},{"// 1":"`amp-slides` is deprecated","// 2":"Please use `<amp-carousel type=slides>` instead.","name":"amp-slides","version":"0.1","latestVersion":"0.1"},{"name":"amp-smartlinks","version":"0.1","latestVersion":"0.1"},{"name":"amp-social-share","version":"0.1","latestVersion":"0.1","options":{"hasCss":true}},{"name":"amp-social-share","version":"1.0","latestVersion":"0.1","options":{"hasCss":true}},{"name":"amp-soundcloud","version":"0.1","latestVersion":"0.1"},{"name":"amp-springboard-player","version":"0.1","latestVersion":"0.1"},{"name":"amp-standalone","version":"0.1","latestVersion":"0.1"},{"name":"amp-sticky-ad","version":"1.0","latestVersion":"1.0","options":{"hasCss":true}},{"name":"amp-story","version":"1.0","latestVersion":"1.0","options":{"hasCss":true,"cssBinaries":["amp-story-consent","amp-story-draggable-drawer-header","amp-story-hint","amp-story-info-dialog","amp-story-open-page-attachment","amp-story-share","amp-story-share-menu","amp-story-system-layer","amp-story-tooltip","amp-story-unsupported-browser-layer","amp-story-viewport-warning-layer"]}},{"name":"amp-story-360","version":"0.1","latestVersion":"0.1","options":{"hasCss":true}},{"name":"amp-story-auto-ads","version":"0.1","latestVersion":"0.1","options":{"hasCss":true,"cssBinaries":["amp-story-auto-ads-ad-badge","amp-story-auto-ads-attribution","amp-story-auto-ads-cta-button","amp-story-auto-ads-inabox","amp-story-auto-ads-progress-bar","amp-story-auto-ads-shared"]}},{"name":"amp-story-auto-analytics","version":"0.1","latestVersion":"0.1"},{"name":"amp-story-dev-tools","version":"0.1","latestVersion":"0.1","options":{"hasCss":true}},{"name":"amp-story-education","version":"0.1","latestVersion":"0.1","options":{"hasCss":true}},{"name":"amp-story-interactive","version":"0.1","latestVersion":"0.1","options":{"hasCss":true,"cssBinaries":["amp-story-interactive-binary-poll","amp-story-interactive-poll","amp-story-interactive-quiz","amp-story-interactive-results"]}},{"name":"amp-story-panning-media","version":"0.1","latestVersion":"0.1","options":{"hasCss":true}},{"name":"amp-story-player","version":"0.1","latestVersion":"0.1","options":{"hasCss":true}},{"name":"amp-stream-gallery","version":"0.1","latestVersion":"0.1","options":{"hasCss":true}},{"name":"amp-stream-gallery","version":"1.0","latestVersion":"0.1","options":{"hasCss":true,"npm":true}},{"name":"amp-subscriptions","version":"0.1","latestVersion":"0.1","options":{"hasCss":true}},{"name":"amp-subscriptions-google","version":"0.1","latestVersion":"0.1","options":{"hasCss":true}},{"name":"amp-tiktok","version":"0.1","latestVersion":"0.1","options":{"hasCss":true}},{"name":"amp-timeago","version":"0.1","latestVersion":"0.1"},{"name":"amp-timeago","version":"1.0","latestVersion":"0.1","options":{"npm":true}},{"name":"amp-truncate-text","version":"0.1","latestVersion":"0.1","options":{"hasCss":true,"cssBinaries":["amp-truncate-text","amp-truncate-text-shadow"]}},{"name":"amp-twitter","version":"0.1","latestVersion":"0.1"},{"name":"amp-twitter","version":"1.0","latestVersion":"0.1","options":{"npm":true}},{"name":"amp-user-notification","version":"0.1","latestVersion":"0.1","options":{"hasCss":true}},{"name":"amp-video","version":"0.1","latestVersion":"0.1"},{"name":"amp-video","version":"1.0","latestVersion":"0.1","options":{"hasCss":true}},{"name":"amp-video-docking","version":"0.1","latestVersion":"0.1","options":{"hasCss":true}},{"name":"amp-video-iframe","version":"0.1","latestVersion":"0.1"},{"name":"amp-video-iframe","version":"1.0","latestVersion":"0.1","options":{"hasCss":true}},{"name":"amp-viewer-integration","version":"0.1","latestVersion":"0.1","options":{"// 1":"The viewer integration code needs to run asap, so that viewers","// 2":"can influence document state asap. Otherwise the document may ","// 3":"take a long time to learn that it should start process other","// 4":"extensions faster.","loadPriority":"high"}},{"name":"amp-vimeo","version":"0.1","latestVersion":"0.1"},{"name":"amp-vimeo","version":"1.0","latestVersion":"0.1","options":{"hasCss":true}},{"name":"amp-vine","version":"0.1","latestVersion":"0.1"},{"name":"amp-viqeo-player","version":"0.1","latestVersion":"0.1"},{"name":"amp-viz-vega","version":"0.1","latestVersion":"0.1","options":{"hasCss":true}},{"name":"amp-vk","version":"0.1","latestVersion":"0.1"},{"name":"amp-web-push","version":"0.1","latestVersion":"0.1","options":{"hasCss":true}},{"name":"amp-wistia-player","version":"0.1","latestVersion":"0.1"},{"name":"amp-yotpo","version":"0.1","latestVersion":"0.1"},{"name":"amp-youtube","version":"0.1","latestVersion":"0.1"},{"name":"amp-youtube","version":"1.0","latestVersion":"0.1","options":{"hasCss":true,"npm":true}}]
41 changes: 23 additions & 18 deletions packages/optimizer/lib/fetchRuntimeParameters.js
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,8 @@
*/
'mode strict';

const URL_COMPONENT_VERSIONS = 'https://amp.dev/static/files/component-versions.json';
const URL_COMPONENT_VERSIONS =
'https://raw.githubusercontent.com/ampproject/amphtml/main/build-system/compile/bundles.config.extensions.json';
const validatorRulesProvider = require('@ampproject/toolbox-validator-rules');
const {MaxAge} = require('@ampproject/toolbox-core');
const {
Expand Down Expand Up @@ -45,17 +46,11 @@ async function fetchRuntimeParameters(config, customRuntimeParameters = {}) {
const runtimeParameters = Object.assign({}, customRuntimeParameters);
// Configure the log level
runtimeParameters.verbose = customRuntimeParameters.verbose || config.verbose || false;
// Validation rules can be downloaded in parallel
const validationRulePromise = initValidatorRules(
runtimeParameters,
customRuntimeParameters,
config
);
await initRuntimeVersion(runtimeParameters, customRuntimeParameters, config);
// Runtime Styles depend on the Runtime version
await initRuntimeStyles(runtimeParameters, config);
// Make sure validation rules are downloaded
await validationRulePromise;
// Validation rules depend on runtime version
await initValidatorRules(runtimeParameters, customRuntimeParameters, config);
return runtimeParameters;
}

Expand All @@ -77,29 +72,39 @@ async function initValidatorRules(runtimeParameters, customRuntimeParameters, co
runtimeParameters.componentVersions =
customRuntimeParameters.componentVersions ||
config.componentVersions ||
(await fetchComponentVersionsFromCache_(config));
(await fetchComponentVersionsFromCache_(config, runtimeParameters));
} catch (error) {
config.log.error('Could not fetch latest component versions from amp.dev');
config.log.verbose(error);
runtimeParameters.componentVersions = [];
}
}
async function fetchComponentVersionsFromCache_(config) {
let componentVersions = await readFromCache_(config, 'component-versions');

async function fetchComponentVersionsFromCache_(config, runtimeParameters) {
const cacheKey = `component-versions-${runtimeParameters.ampRuntimeVersion}`;
let componentVersions = await readFromCache_(config, cacheKey);
if (!componentVersions) {
componentVersions = await fetchComponentVersions_(config);
writeToCache_(config, 'component-versions', componentVersions);
try {
componentVersions = await fetchComponentVersions_(config, runtimeParameters);
writeToCache_(config, cacheKey, componentVersions);
} catch (e) {
config.log.warn(e.message);
componentVersions = require('./extensionConfig.json');
}
}
return componentVersions;
}

async function fetchComponentVersions_(config) {
const response = await config.fetch(URL_COMPONENT_VERSIONS);
async function fetchComponentVersions_(config, runtimeParameters) {
// Strip the leading two chars from the version identifier to get the release tag
const releaseTag = runtimeParameters.ampRuntimeVersion.substring(2);
const componentConfigUrl = `https://raw.githubusercontent.com/ampproject/amphtml/${releaseTag}/build-system/compile/bundles.config.extensions.json`;

const response = await config.fetch(componentConfigUrl);
if (!response.ok) {
config.log.error(
throw new Error(
`Failed fetching latest component versions from ${URL_COMPONENT_VERSIONS} with status: ${response.status}`
);
return [];
}
return response.json();
}
Expand Down
Loading

0 comments on commit e85d2a8

Please sign in to comment.