From 1f7957bcddd13e223084cbc10cf9ea93ae3059a8 Mon Sep 17 00:00:00 2001 From: Peter Bengtsson Date: Thu, 29 Apr 2021 09:42:23 -0400 Subject: [PATCH] not all http:// links can be https:// Fixes #3654 --- build/flaws/broken-links.js | 45 ++- build/flaws/safe-to-https-domains.json | 505 +++++++++++++++++++++++++ testing/tests/index.test.js | 2 +- 3 files changed, 544 insertions(+), 8 deletions(-) create mode 100644 build/flaws/safe-to-https-domains.json diff --git a/build/flaws/broken-links.js b/build/flaws/broken-links.js index b8056c8a12d8..afba6fd70c3b 100644 --- a/build/flaws/broken-links.js +++ b/build/flaws/broken-links.js @@ -1,8 +1,27 @@ +const fs = require("fs"); +const path = require("path"); + const { Archive, Document, Redirect, Image } = require("../../content"); const { FLAW_LEVELS } = require("../constants"); const { findMatchesInText } = require("../matches-in-text"); const { DEFAULT_LOCALE, VALID_LOCALES } = require("../../libs/constants"); +const _safeToHttpsDomains = new Map(); +function getSafeToHttpDomains() { + if (!_safeToHttpsDomains.size) { + const fileParsed = JSON.parse( + fs.readFileSync( + path.join(__dirname, "safe-to-https-domains.json"), + "utf-8" + ) + ); + Object.entries(fileParsed).forEach(([key, value]) => + _safeToHttpsDomains.set(key, value) + ); + } + return _safeToHttpsDomains; +} + function isHomepageURL(url) { // Return true if the URL is something like `/` or `/en-US` or `/fr/` if (url === "/") { @@ -131,13 +150,25 @@ function getBrokenLinksFlaws(doc, $, { rawContent }, level) { } if (href.startsWith("http://")) { - addBrokenLink( - a, - checked.get(href), - href, - href.replace("http://", "https://"), - "http:// external links are not allowed (will be forced to https:// at build-time)" - ); + const domain = new URL(href).hostname; + // If a URL's domain is in the list that getSafeToHttpDomains() provides, + // that means we've tested that you can turn that into a HTTPS link + // simply by replacing the `http://` for `https://`. + // Using `.get(domain)` is smart because if the domain isn't known you + // get `undefined` otherwise you get `true` or `false`. And we're only + // interested in the `true`. + if (getSafeToHttpDomains().get(domain)) { + addBrokenLink( + a, + checked.get(href), + href, + href.replace("http://", "https://"), + "Is currently http:// but can become https://" + ); + } + // Note! If it's not known that the URL's domain can be turned into https:// + // we do nothing here. No flaw. It's unfortunate that we still have http:// + // links in our content but that's a reality of MDN being 15+ years old. } else if (href.startsWith("https://developer.mozilla.org/")) { // It might be a working 200 OK link but the link just shouldn't // have the full absolute URL part in it. diff --git a/build/flaws/safe-to-https-domains.json b/build/flaws/safe-to-https-domains.json new file mode 100644 index 000000000000..795840f82061 --- /dev/null +++ b/build/flaws/safe-to-https-domains.json @@ -0,0 +1,505 @@ +{ + "www.mozilla.org": true, + "groups.google.com": true, + "schepers.cc": false, + "stackoverflow.com": true, + "ecma-international.org": true, + "www.sencha.com": true, + "schema.org": true, + "scrawl.rikweb.org.uk": true, + "unicode.org": true, + "gpac.wp.mines-telecom.fr": true, + "www.csstutorial.net": true, + "docs.oracle.com": true, + "example.com": true, + "asmjs.org": false, + "www.iana.org": true, + "pouchdb.com": true, + "www.youtube.com": true, + "hsivonen.iki.fi": true, + "webkit.org": true, + "easings.net": true, + "www.videojs.com": true, + "2017.js13kgames.com": true, + "www.w3.org": true, + "www.double.co.nz": false, + "mdn.beonex.com": true, + "html5doctor.com": true, + "blog.danielfriesen.name": true, + "www.quirksmode.org": true, + "0xfe.blogspot.com": true, + "www-archive.mozilla.org": true, + "zimjs.com": true, + "bugzilla.mozilla.org": true, + "aurelio.audero.it": true, + "www.exslt.org": false, + "diveintohtml5.info": false, + "reference.sitepoint.com": true, + "brucelindbloom.com": false, + "web.archive.org": true, + "hacks.mozilla.org": true, + "dlmf.nist.gov": true, + "tokenposts.blogspot.com.au": true, + "videogamejournaliser.com": false, + "meta.wikimedia.org": true, + "glmatrix.net": true, + "www.inkscape.org": true, + "www.inftyreader.org": true, + "www.ietf.org": true, + "selectivizr.com": false, + "en.wikiversity.org": true, + "www.bbc.co.uk": true, + "cve.mitre.org": true, + "searchfox.org": true, + "whereswalden.com": true, + "updates.html5rocks.com": false, + "www.croczilla.com": false, + "gridbyexample.com": true, + "zenphoton.com": true, + "lbsyun.baidu.com": true, + "www.khronos.org": true, + "jsconsole.com": true, + "dev-test.nemikor.com": false, + "www.mergevr.com": true, + "www.html5rocks.com": true, + "www.ecma-international.org": true, + "wiki.codetalks.org": false, + "dbaron.org": true, + "accessfirefox.com": false, + "jwatt.org": true, + "www.fontsquirrel.com": true, + "paulirish.com": true, + "pubs.opengroup.org": true, + "dmitrysoshnikov.com": false, + "www.isg.rhul.ac.uk": true, + "www.smartjava.org": true, + "fitzgeraldnick.com": true, + "www.colorzilla.com": true, + "msopentech.com": false, + "www.flumotion.net": false, + "wiki.csswg.org": true, + "ffmpeg.org": true, + "nginx-rtmp.blogspot.it": true, + "trac.webkit.org": true, + "openid.net": true, + "www.brendangregg.com": false, + "qnimate.com": false, + "lists.w3.org": true, + "www.chromium.org": true, + "greensock.com": true, + "datatracker.ietf.org": true, + "firefogg.org": false, + "wiki.svg.org": false, + "houdini.glitch.me": true, + "svg-whiz.com": false, + "www.mattgreer.org": true, + "jointjs.com": true, + "www.wirfs-brock.com": false, + "tomrothe.de": true, + "raphaeljs.com": true, + "gva.noekeon.org": false, + "www1.chapman.edu": false, + "lawrencemandel.com": true, + "timepicker.co": true, + "razvancaliman.com": false, + "css-tricks.com": true, + "webvr.directory": true, + "sandbox.photonstorm.com": false, + "yuiblog.com": false, + "floating-point-gui.de": true, + "math.etsu.edu": true, + "toolbar.netcraft.com": true, + "dev.opera.com": true, + "www.bebits.com": false, + "www.websiteoptimization.com": false, + "windows.microsoft.com": true, + "mobile.smashingmagazine.com": true, + "www.faqs.org": false, + "usabilitypost.com": true, + "www.go-oo.org": false, + "dev.w3.org": true, + "www.404notfound.fr": false, + "googledevelopers.blogspot.ca": true, + "www.useit.com": false, + "dsheiko.com": true, + "eloquentjavascript.net": true, + "paperjs.org": false, + "forum.komodoide.com": false, + "unicode-table.com": true, + "jinja.pocoo.org": false, + "www.zvon.org": false, + "mdn.github.io": true, + "jquery.com": true, + "webauthndemo.appspot.com": true, + "csrc.nist.gov": true, + "httpd.apache.org": true, + "tinyurl.com": true, + "bikeshed.com": true, + "www.xml.com": true, + "feature-policy-demos.appspot.com": true, + "uxdesign.com": false, + "addons.mozilla.org": true, + "www.opera.com": true, + "jPlayer.org": true, + "maps.google.com": true, + "www.blender.org": true, + "www.samsung.com": true, + "es6console.com": true, + "www.objectplayground.com": false, + "developer.apple.com": true, + "www.voiceage.com": false, + "www.opensearch.org": true, + "rajudasa.blogspot.in": true, + "cedato.com": true, + "api.jquery.com": true, + "www.paulirish.com": true, + "golem.ph.utexas.edu": true, + "lesscss.org": true, + "www.nvda-project.org": false, + "www.rfc-editor.org": true, + "www.csszengarden.com": false, + "lachy.id.au": true, + "www.createjs.com": true, + "java.sun.com": true, + "www.codedread.com": true, + "docs.phaser.io": false, + "www.stevesouders.com": true, + "ogp.me": true, + "gyu.que.jp": false, + "ckeditor.com": true, + "www.alistapart.com": true, + "www.aptana.com": false, + "ted.mielczarek.org": true, + "www.upu.int": true, + "blog.mozilla.com": true, + "ftp.mozilla.org": true, + "blog.agektmr.com": true, + "docs.google.com": true, + "www.dogtagpki.org": true, + "www.pitivi.org": false, + "developer.android.com": true, + "offlinefirst.org": false, + "www.dexie.org": true, + "perfectionkills.com": false, + "sublimevideo.net": true, + "www.the-pope.com": false, + "support.mozillamessaging.com": true, + "mathworld.wolfram.com": true, + "support.mozilla.com": true, + "msdn2.microsoft.com": true, + "gogogadgetscott.info": false, + "wiki.c2.com": true, + "wiki.mozilla.org": true, + "blog.chromium.org": true, + "web.mit.edu": true, + "cssgridgarden.com": true, + "www.base64-image.de": true, + "mediaelementjs.com": false, + "webaim.org": true, + "bugs.firefox-dev.tools": true, + "tracker.com": false, + "www.currency-iso.org": true, + "www.jwplayer.com": true, + "www.ctan.org": true, + "gstreamer.freedesktop.org": true, + "adrianroselli.com": true, + "developer.yahoo.com": true, + "www.songho.ca": true, + "bugzil.la": true, + "threejs.org": true, + "jibbering.com": true, + "accessibleajax.clcworld.net": false, + "support.apple.com": true, + "www.lutanho.net": false, + "www.typescriptlang.org": true, + "fontsgeek.com": true, + "accessify.com": true, + "developers.whatwg.org": true, + "code.google.com": true, + "alistapart.com": true, + "www.bluegriffon.com": false, + "www.sublimevideo.net": true, + "babeljs.io": true, + "www.cnet.com": true, + "plurib.us": true, + "svg-wow.org": false, + "www.commonjs.org": false, + "demo.huihoo.com": true, + "www.unicode.org": true, + "stuff.mit.edu": true, + "ready.to": true, + "www.html5accessibility.com": true, + "crbug.com": true, + "felixniklas.com": true, + "www.squarefree.com": true, + "simpl.info": true, + "www.apple.com": true, + "chromium.googlecode.com": true, + "www.dyn4j.org": true, + "skillsmatter.com": true, + "www.glge.org": false, + "creativecommons.org": true, + "dashif.org": true, + "dvcs.w3.org": true, + "mathscribe.com": true, + "tripleodeon.com": true, + "thejit.org": true, + "www.adobe.com": true, + "peter.sh": true, + "www.jenitennison.com": true, + "net.tutsplus.com": true, + "speak-demo.herokuapp.com": true, + "uk.video.yahoo.com": true, + "clang.llvm.org": true, + "pieisgood.org": true, + "requirejs.org": true, + "hyperaud.io": true, + "www.pixijs.com": true, + "openfontlibrary.org": true, + "www.peachpit.com": true, + "www.cs.tut.fi": true, + "codepen.io": true, + "lbs.qq.com": true, + "www.brettjankord.com": true, + "eilv.cie.co.at": false, + "nchc.dl.sourceforge.net": true, + "audiocogs.org": false, + "download.smedley.info": false, + "webglfundamentals.org": true, + "www.yoctopuce.com": true, + "www.bbc.com": true, + "www.softwareishard.com": false, + "davidwalsh.name": true, + "pixelscommander.com": false, + "www.seamonkey-project.org": true, + "www-106.ibm.com": true, + "learningbabylonjs.com": false, + "htmldog.com": true, + "standardista.com": false, + "www.yappy.it": false, + "www.cablelabs.com": true, + "mathcs.chapman.edu": true, + "www.whatwg.org": true, + "www.openoffice.org": true, + "www.mirovideoconverter.com": false, + "www.2ality.com": true, + "www.nongnu.org": true, + "socketcluster.io": true, + "innolitics.com": true, + "mp4ra.org": true, + "praegnanz.de": true, + "mathiasbynens.be": true, + "jplayer.org": true, + "www.w3clubs.com": true, + "jsonbeautifier.org": true, + "xmlgraphics.apache.org": true, + "www.color.org": true, + "www.example.com": true, + "blogs.msdn.com": true, + "js13kgames.com": true, + "jsxgraph.uni-bayreuth.de": true, + "www.webglacademy.com": false, + "www.redhat.com": true, + "joshondesign.com": true, + "exploringjs.com": true, + "mediapm.edgesuite.net": false, + "brew.sh": true, + "www.htmlhelp.com": true, + "getfirebug.com": true, + "nightly.mozilla.org": true, + "ImageMagick.org": true, + "ie.microsoft.com": false, + "teamgiraffe.co.uk": false, + "labs.jensimmons.com": true, + "hevc.info": false, + "nodejs.org": true, + "go-oo.org": false, + "www.davidflanagan.com": true, + "cubic-bezier.com": true, + "www.leanbackplayer.com": true, + "m.flickr.com": true, + "callbackhell.com": false, + "www.addyosmani.com": true, + "fabricjs.com": false, + "getbem.com": false, + "bennettfeely.com": true, + "a11y-style-guide.com": true, + "pki.fedoraproject.org": true, + "promisesaplus.com": true, + "www.gtalbot.org": true, + "de.slideshare.net": true, + "libreoffice.org": true, + "webglplayground.net": false, + "www.microsoft.com": true, + "www.svgbasics.com": false, + "oos.moxiecode.com": false, + "csstriggers.com": true, + "www.gwhitworth.com": true, + "xregexp.com": true, + "www.themaninblue.com": true, + "software.hixie.ch": true, + "robert.ocallahan.org": true, + "www.streamingmedia.com": true, + "www.unece.org": true, + "www.jitbit.com": true, + "www.totaljs.com": true, + "www.gnome.org": true, + "www.xiph.org": true, + "plyr.io": true, + "www.codicode.com": true, + "www.webkit.org": true, + "www.lukew.com": true, + "slimjs.com": true, + "ondras.zarovi.cz": true, + "bucephalus.org": true, + "engineering.silk.co": true, + "handbrake.fr": true, + "tylerfinck.com": true, + "www.dotnet247.com": true, + "www.codecademy.com": true, + "my.opera.com": true, + "mozakai.blogspot.com": true, + "fedoraproject.org": true, + "inria.fr": true, + "tools.ietf.org": true, + "www.keio.ac.jp": true, + "mango.blender.org": true, + "www.muazkhan.com": true, + "betterexplained.com": true, + "www.sitepoint.com": true, + "www-xray.ast.cam.ac.uk": true, + "www.slideshare.net": true, + "www.lg.com": true, + "www.nczonline.net": true, + "html5demos.com": true, + "www.modernizr.com": true, + "www.widevine.com": true, + "www-itec.uni-klu.ac.at": true, + "help.adobe.com": true, + "modernizr.com": true, + "www.useragentman.com": true, + "wayback.archive.org": true, + "www.delphiki.com": true, + "www.evolt.org": true, + "httpwg.org": true, + "store.steampowered.com": true, + "unstoppablerobotninja.com": true, + "w3.org": true, + "www.kevinroth.com": true, + "curl.haxx.se": true, + "m.facebook.com": true, + "www.oracle.com": true, + "www.kb.cert.org": true, + "download.java.net": true, + "www.getmiro.com": true, + "www.mulberrytech.com": true, + "www.streamingmediaglobal.com": true, + "www.itu.int": true, + "m.youtube.com": true, + "soledadpenades.com": true, + "people.mozilla.org": true, + "dom.spec.whatwg.org": true, + "svg.org": true, + "encoding.spec.whatwg.org": true, + "duriansoftware.com": true, + "ev.buaa.edu.cn": true, + "html5.org": true, + "intercoolerjs.org": true, + "videojs.com": true, + "www.lyx.org": true, + "www.html5canvastutorials.com": true, + "www.firemath.info": true, + "coffeescript.org": true, + "docs.aws.amazon.com": true, + "jvectormap.com": true, + "www.icecast.org": true, + "lea.verou.me": true, + "www.earlevel.com": true, + "mulberrytech.com": true, + "dash-mse-test.appspot.com": true, + "www.sintel.org": true, + "nerget.com": true, + "en.flossmanuals.net": true, + "www.meyerweb.com": true, + "library.gnome.org": true, + "cssmenumaker.com": true, + "www.testufo.com": true, + "m.vid.ly": true, + "wikiversity.org": true, + "tink.uk": true, + "www.via-corp.com": true, + "soundbible.com": true, + "www.dspguide.com": true, + "carto.net": true, + "www.hixie.ch": true, + "emberjs.com": true, + "signalr.net": true, + "www.google.com": true, + "dropshado.ws": true, + "www.thomasfrank.se": true, + "luca.ntop.org": true, + "webglsamples.org": true, + "carlosrafaelgn.com.br": true, + "blog.kylehuey.com": true, + "earth-info.nga.mil": true, + "www6.software.ibm.com": true, + "processingjs.org": true, + "cdn.rawgit.com": true, + "www.mathjax.org": true, + "sass-lang.com": true, + "www.ffmpeg.org": true, + "v2v.cc": true, + "www.peej.co.uk": true, + "webglreport.com": true, + "jsstore.net": true, + "javascriptweblog.wordpress.com": true, + "glitch.com": true, + "angularjs.org": true, + "html.spec.whatwg.org": true, + "couchdb.apache.org": true, + "www.i18nguy.com": false, + "diagramcenter.org": false, + "www-cs-students.stanford.edu": false, + "www.dria.org": false, + "users.skynet.be": false, + "browserify.org": false, + "brackets.io": false, + "velocityjs.org": false, + "1997.webhistory.org": false, + "www.codelobster.com": false, + "smacss.com": false, + "csslint.net": false, + "www.auraluxgame.com": false, + "adaptive-images.com": false, + "www.html.net": false, + "abisource.org": false, + "www.jsondiff.com": false, + "daniemon.com": false, + "qutoric.com": false, + "camendesign.com": false, + "info.cern.ch": false, + "xml.coverpages.org": false, + "www.deafblind.com": false, + "www.passani.it": false, + "browsershots.org": false, + "www.stubbornella.org": false, + "eyeasme.com": false, + "corber.io": false, + "ogldev.atspace.co.uk": false, + "www.chromevox.com": false, + "disruptive-innovations.com": false, + "mkweb.bcgsc.ca": false, + "www.vision-and-eye-health.com": false, + "getskeleton.com": false, + "snapsvg.io": false, + "compass-style.org": false, + "videogamecaster.com": false, + "www.zibbo.com": false, + "www.javascriptkit.com": false, + "www.7zip.org": false, + "flask.pocoo.org": false, + "chaijs.com": false, + "www.schillmania.com": false, + "www.os2bbs.com": false, + "es6-features.org": false +} diff --git a/testing/tests/index.test.js b/testing/tests/index.test.js index a76efeee8792..432389cd6133 100644 --- a/testing/tests/index.test.js +++ b/testing/tests/index.test.js @@ -571,7 +571,7 @@ test("broken links flaws", () => { "#anchor" ); expect(map.get("http://www.mozilla.org").explanation).toBe( - "http:// external links are not allowed (will be forced to https:// at build-time)" + "Is currently http:// but can become https://" ); expect(map.get("http://www.mozilla.org").suggestion).toBe( "https://www.mozilla.org"