Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Update dependencies #1303

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 2 additions & 4 deletions .eslintrc.yml
Original file line number Diff line number Diff line change
@@ -1,11 +1,9 @@
extends:
- 'wikimedia/server'

plugins:
- json
- jsdoc

rules:
no-shadow:
- off
array-bracket-spacing:
- off
camelcase:
Expand Down
1 change: 0 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ coverage
config.yaml
node_modules
npm-debug.log
package-lock.json

# WebStorm IDE files
.idea/*
Expand Down
1 change: 1 addition & 0 deletions lib/base_feed.js
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ class BaseFeed {

/**
* Extract a very header from any of the result parts.
*
* @param {!Object} result
* @return {string|null}
* @private
Expand Down
47 changes: 34 additions & 13 deletions lib/mwUtil.js
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,8 @@ const mwUtil = {};
/**
* Create an etag value of the form
* "<revision>/<tid>/<optional_suffix>"
* @param {Integer} rev page revision number
*
* @param {number} rev page revision number
* @param {string} tid page render UUID
* @param {string} [suffix] optional suffix
* @return {string} the value of the ETag header
Expand All @@ -31,6 +32,7 @@ mwUtil.makeETag = (rev, tid, suffix) => {

/**
* Normalizes the request.params.title and returns it back
*
* @param {HyperSwitch} hyper hyperswitch
* @param {Object} req request object
* @param {string} title
Expand All @@ -54,6 +56,7 @@ mwUtil.normalizeTitle = (hyper, req, title) => mwUtil.getSiteInfo(hyper, req)
/**
* Parse an etag value of the form
* "<revision>/<tid>/<suffix>"
*
* @param {string} etag tag to parse
* @return {Object} an object with rev, tid and optional suffix properties
*/
Expand Down Expand Up @@ -85,6 +88,7 @@ mwUtil.parseETag = (etag) => {
/**
* Checks whether a language variant could exist for
* a page in a requested language on a particular wiki.
*
* @param {HyperSwitch} hyper
* @param {Object} req
* @param {string} pageLanguage the language of the requested page.
Expand All @@ -98,6 +102,7 @@ mwUtil.canConvertLangVariant = (hyper, req, pageLanguage) => {
/**
* Checks whether a specialized requested language variant exist for
* a page in a requested language on a particular wiki.
*
* @param {HyperSwitch} hyper
* @param {Object} req
* @param {string} pageLang the language of the requested page.
Expand All @@ -124,6 +129,7 @@ mwUtil.shouldConvertLangVariant = (hyper, req, pageLang, acceptLang) => {
/**
* Extract the date from an `etag` header of the form
* "<revision>/<tid>/<suffix>"
*
* @param {string} etag
* @return {Date|null}
*/
Expand Down Expand Up @@ -155,6 +161,7 @@ mwUtil.coerceTid = (tidString, bucket) => {

/**
* Normalizes the order of 'Content-Type' header fields.
*
* @param {Object} res server response
*/
mwUtil.normalizeContentType = (res) => {
Expand All @@ -170,13 +177,15 @@ mwUtil.normalizeContentType = (res) => {

/**
* Checks whether the request is a 'no-cache' request
*
* @param {Object} req
* @return {boolean}
*/
mwUtil.isNoCacheRequest = (req) => req.headers && /no-cache/i.test(req.headers['cache-control']);

/**
* Checks whether the request is a 'no-store' request
*
* @param {Object} req
* @return {boolean}
*/
Expand Down Expand Up @@ -212,6 +221,7 @@ mwUtil.getLimit = (hyper, req) => {
* if access is restricted, no-op otherwise.
*
* Item might be either a revision or a restriction.
*
* @param {Object} revision the revision object
*/
mwUtil.applyAccessChecks = (revision) => {
Expand Down Expand Up @@ -246,6 +256,7 @@ mwUtil.applyAccessChecks = (revision) => {

/**
* Create a json web token.
*
* @param {HyperSwitch} hyper HyperSwitch context*
* @param {Object} object a JSON object to encode
* @return {string}
Expand All @@ -260,6 +271,7 @@ mwUtil.encodePagingToken = (hyper, object) => {

/**
* Decode signed token and decode the orignal token
*
* @param {HyperSwitch} hyper HyperSwitch context
* @param {string} token paging request token
* @return {Object}
Expand Down Expand Up @@ -304,6 +316,7 @@ mwUtil.getQueryString = (req) => {

/**
* Extracts the domain name from the provided URL.
*
* @param {string} url
* @return {string}
*/
Expand All @@ -325,6 +338,7 @@ mwUtil.addQueryString = (location, query) => {

/**
* Create a `location` header value for a relative redirect.
*
* @param {string} path the path pattern from specInfo.
* @param {hyper.request} req the request
* @param {Object} options with possible parameters:
Expand Down Expand Up @@ -360,10 +374,8 @@ mwUtil.createRelativeTitleRedirect = (path, req, options) => {
mwUtil.getQueryString(req);
} else if (/#/.test(newReqParams.title)) {
const titleFragments = encodeURIComponent(newReqParams.title).split(/%23/);
/* eslint-disable prefer-template */
location = backString + titleFragments.shift() + mwUtil.getQueryString(req) +
'#' + titleFragments.join('%23');
/* eslint-enable prefer-template */
location = backString + titleFragments.shift() + // eslint-disable-line prefer-template
mwUtil.getQueryString(req) + '#' + titleFragments.join('%23');
} else {
location = backString + encodeURIComponent(newReqParams.title) + mwUtil.getQueryString(req);
}
Expand All @@ -376,10 +388,11 @@ mwUtil.createRelativeTitleRedirect = (path, req, options) => {
};

/**
* Checks if the request is a CORS request
* @param {Object} req The request to check
* @return {boolean} Whether the request is CORS or not
*/
* Checks if the request is a CORS request
*
* @param {Object} req The request to check
* @return {boolean} Whether the request is CORS or not
*/
mwUtil.isCrossOrigin = (req) => {
return req && req.headers && req.headers.origin && req.headers.origin !== req.params.domain;
};
Expand Down Expand Up @@ -425,6 +438,7 @@ mwUtil.isSelfRedirect = (req, location) => {

/**
* Fetches the summary content from the provided URI
*
* @param {HyperSwitch} hyper HyperSwitch context
* @param {URI} uri summary URI
* @param {headers} headers req header
Expand All @@ -448,6 +462,7 @@ mwUtil.fetchSummary = (hyper, uri, headers) => {
/**
* Traverses through a request body and replaces the given keys with
* the content fetched using the 'fetch' callback
*
* @param {Object} response the response object to hydrate
* @param {Function} fetch the function used to fetch the content if it's not present
* @return {Promise} response
Expand Down Expand Up @@ -494,6 +509,7 @@ mwUtil.hydrateResponse = (response, fetch) => {

/**
* Checks whether the date is today or in the past in UTC-0 timezone
*
* @param {Date} date a date to check
* @return {boolean} true if the date is in the past
*/
Expand All @@ -505,6 +521,7 @@ mwUtil.isHistoric = (date) => {

/**
* Verifies that the date parameter is in proper format.
*
* @param {Object} req the request to check
*/
mwUtil.verifyDateParams = (req) => {
Expand Down Expand Up @@ -543,6 +560,7 @@ mwUtil.verifyDateParams = (req) => {

/**
* Safely builds the Date from request parameters
*
* @param {Object} rp request parameters object
* @return {Date} the requested date.
*/
Expand All @@ -563,6 +581,7 @@ mwUtil.getDateSafe = (rp) => {
/**
* From a list of regexes and strings, constructs a regex that
* matches any of list items
*
* @param {Array.string} list array of strings and regexes
* @return {Object|undefined} either a RegExp object or undefined
*/
Expand All @@ -582,6 +601,7 @@ mwUtil.constructRegex = (list) => {

/**
* Remove entries with duplicate 'title' values from an array of objects
*
* @param {?Array} arr Array of page/article objects
* @param {?Function} upd Optional function to update the original item based on the duplicate.
* Takes two parameters (orig, dupe): orig - the original object,
Expand All @@ -593,9 +613,8 @@ mwUtil.removeDuplicateTitles = (arr, upd) => {
return arr.filter((item, idx, init) => {
if (titles[item.title]) {
if (upd) {
// eslint-disable-next-line no-unused-vars
let orig = init[init.findIndex((el) => el.title === item.title)];
orig = upd(orig, item);
const orig = init[init.findIndex((el) => el.title === item.title)];
upd(orig, item);
}
return false;
}
Expand All @@ -606,13 +625,15 @@ mwUtil.removeDuplicateTitles = (arr, upd) => {

/**
* Checks if the provided path is an HTML route.
*
* @param {string} path the path to check.
* @return {boolean}
*/
mwUtil.isHTMLRoute = (path) => /\/page\/html\//.test(path);

/**
* Extracts the version number from the content-type or accept profile.
*
* @param {string} header the content-type or accept header.
* @return {string|undefined}
*/
Expand Down Expand Up @@ -656,7 +677,7 @@ mwUtil.addVaryHeader = (res, vary) => {

mwUtil.deepCloneObj = (obj) => {
const clonedObj = Array.isArray(obj) ? [] : {};
Object.keys(obj).forEach((key) =>{
Object.keys(obj).forEach((key) => {
const node = obj[key];
clonedObj[key] = (typeof node === 'object' && node !== null) ? mwUtil.deepCloneObj(node) : node;
});
Expand Down
4 changes: 3 additions & 1 deletion lib/security_response_header_filter.js
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ const cspHeaderVariants = [

/**
* Replaces subdomain with a wildcard
*
* @param {string} domain a full domain name (e.g. en.wikipedia.org)
* @return {string} wildcard version (e.g. *.wikipedia.org)
*/
Expand All @@ -43,7 +44,8 @@ function wildcardSubdomain(domain) {

/**
* Adds Content-Security-Policy & related headers to send in response
* @param {HyperSwitch} hyper the HyperSwitch object
*
* @param {Object} hyper the HyperSwitch object
* @param {Object} req the request to handle
* @param {P} next the promise to execute
* @param {Object} options options containing the following fields:
Expand Down
3 changes: 3 additions & 0 deletions maintenance/.eslintrc.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
rules:
no-console:
- off
18 changes: 8 additions & 10 deletions maintenance/generate-wikimedia-domain-config.js
Original file line number Diff line number Diff line change
Expand Up @@ -6,15 +6,15 @@

'use strict';

var preq = require('preq');
var downloadUrl = 'https://en.wikipedia.org/w/api.php?action=sitematrix&format=json';
const preq = require('preq');
const downloadUrl = 'https://en.wikipedia.org/w/api.php?action=sitematrix&format=json';

preq.get({
uri: downloadUrl
})
.then((res) => {
var sm = res.body.sitematrix;
var projects = {
const sm = res.body.sitematrix;
const projects = {
wikipedia: [],
wiktionary: [],
wikiquote: [],
Expand All @@ -28,13 +28,13 @@ preq.get({
};

Object.keys(sm).forEach((k) => {
var lang = sm[k];
const lang = sm[k];
if (lang.site || k === 'specials') {
var sites = lang.site || lang;
const sites = lang.site || lang;
sites.forEach((site) => {
if (site.closed === undefined && site.private === undefined) {
var domain = site.url.replace(/^https?:\/\//, '');
var name = domain.replace(/[^.]+\.(\w+)\.org$/, '$1');
const domain = site.url.replace(/^https?:\/\//, '');
const name = domain.replace(/[^.]+\.(\w+)\.org$/, '$1');
if (projects[name]) {
projects[name].push(domain);
} else {
Expand All @@ -46,10 +46,8 @@ preq.get({
});

Object.keys(projects).forEach((name) => {
// eslint-disable-next-line no-console
console.log(`\n # ${name}`);
projects[name].forEach((domain) => {
// eslint-disable-next-line no-console
console.log(` /{domain:${domain}}: *wp/default/1.0.0`);
});
});
Expand Down
4 changes: 0 additions & 4 deletions maintenance/generate_random_load.js
Original file line number Diff line number Diff line change
Expand Up @@ -18,27 +18,23 @@ function makeCheck() {
return preq.get(`${testRestBASEUri}/page/html/${encodeURIComponent(title)}/${revision}`)
.then((res) => {
const tid = res.headers.etag.match(/\/(.+)"$/)[1];
// eslint-disable-next-line no-console
console.log(`Testing for ${title}/${revision}/${tid}`);
P.delay(Math.floor(Math.random() * 86400000))
.then(() => {
return P.all([
preq.get(`${testRestBASEUri}/page/html/${encodeURIComponent(title)}/${revision}/${tid}`)
.catch((err) => {
// eslint-disable-next-line no-console
console.log(`${new Date()} Failed to fetch HTML ${title}/${revision}/${tid} from test RB: ${err}`);
}),
preq.get(`${testRestBASEUri}/page/html/${encodeURIComponent(title)}/${revision}/${tid}`)
.catch((err) => {
// eslint-disable-next-line no-console
console.log(`${new Date()} Failed to fetch Data-Parsoid ${title}/${revision}/${tid} from test RB: ${err}`);
})
]);
});

return P.delay(500).then(makeCheck);
}, (e) => {
// eslint-disable-next-line no-console
console.log(`${new Date()} Failed to fetch HTML ${title}/${revision} from test RB: ${e}`);
});
});
Expand Down
Loading