Firefox Developer Tools docs is removed from mdn/content

[yin1999]

What is the problem?

Firefox Developer Tools docs in en-US is now redirected to Firefox DevTools User Docs by mdn/content#14140

Should we follow this changes in this repo?

[SphinxKnight]

Thanks for the heads up @yin1999, I guess the workflow for this would be to use yarn content --delete redirect for every current page under tools

[SphinxKnight]

After fixing a missing logic w/ mdn/yari#5814, I believe using the command line with
yarn content delete <slug> <locale> --redirect https://firefox-source-docs.mozilla.org/<sthg>
will work.
For instance, to delete/redirect one of the French page, I'll use
yarn content delete Tools/3D_View fr --redirect https://firefox-source-docs.mozilla.org/devtools-user/3d_view/index.html

(quick reminder: yarn content works from /content directory whereas yarn tool works from /yari directory)

[SphinxKnight]

if it may be of any help to any maintainer out there, here is the script I used for the French "surface" of Tools: https://gist.github.com/SphinxKnight/16d971e073338c5da4403874a3d8227b
I hope this would help reduce the workload (leaving only the pages for Tools that you may have in addition to the ones French had)

[yin1999]

I have wrote two scripts(in go and python) to deal with removing Tools section.

It resolves the Tools section with the following steps:

• List all the file with .html and .md ext, save the path to a string list.
• sort those path with inverse lexicographical order
• get the redirect url by GET "https://developer.mozilla.org/en-US/docs/" + <slug>
• exec command yarn content delete <slug> <locale> --redirect <redirect url> -y at content repo dir (upgrade to @mdn/[email protected])

Some documents need to be resolved manually (then run the script below again). The script won't resolve the tools section in conflicting dir neither.

I have precompiled the executable files below (for go).

source

go

usage

./remove-tools -pwd /path/to/mdn/content -d /path/to/mdn/translated-content/ -locale <locale>

package main

import (
	"bufio"
	"bytes"
	"flag"
	"fmt"
	"io/fs"
	"log"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"sort"
	"strings"
)

var (
	locale   = flag.String("locale", "en-US", "locale for removing 'Tools'")
	redirect map[string]string
)

type pathSlice []string

var _ sort.Interface = pathSlice{}

func (p pathSlice) Len() int {
	return len(p)
}

func (p pathSlice) Less(i, j int) bool {
	return filepath.Dir(p[i]) > filepath.Dir(p[j]) // use inverse lexicographical order
}

func (p pathSlice) Swap(i, j int) {
	p[i], p[j] = p[j], p[i]
}

func main() {
	pwd := flag.String("pwd", ".", "`content` repo dir")
	dir := flag.String("d", ".", "`mdn/translated-content` repo dir")
	flag.Parse()
	if *pwd != "." {
		err := os.Chdir(*pwd)
		if err != nil {
			log.Fatalln(err)
		}
	}
	if *locale == "" {
		log.Fatal("locale cannot be empty\n")
	}
	redirect = loadRedirect("files/en-us/_redirects.txt")
	var paths pathSlice
	*dir = filepath.Join(*dir, "files", strings.ToLower(*locale), "tools")
	filepath.WalkDir(*dir, func(path string, d fs.DirEntry, err error) error {
		if d.IsDir() {
			return nil
		}
		switch filepath.Ext(path) {
		case ".html", ".md":
			paths = append(paths, path)
		}
		return nil
	})
	sort.Sort(paths)
	for _, path := range paths {
		resolve(path)
	}
}

func loadRedirect(name string) map[string]string {
	file, err := os.Open(name)
	if err != nil {
		panic(err)
	}
	defer file.Close()
	scanner := bufio.NewScanner(file)
	redirect := make(map[string]string)
	for scanner.Scan() {
		line := scanner.Text()
		if strings.HasPrefix(line, "#") {
			continue
		}
		parts := strings.SplitN(line, "\t", 2)
		if len(parts) != 2 {
			fmt.Printf("[warn] invalid redirect line: '%s'\n", line)
			continue
		}
		origin := strings.ToLower(parts[0])
		origin = origin[len("/en-US/docs/"):]
		redirect[origin] = parts[1]
	}
	return redirect
}

var client = http.Client{CheckRedirect: func(req *http.Request, via []*http.Request) error {
	return http.ErrUseLastResponse
}}

func resolve(path string) {
	file, err := os.Open(path)
	if err != nil {
		fmt.Printf("[warn] cannot open file, err: %s\n", err.Error())
	}
	reader := bufio.NewReader(file)
	var data []byte
	var slug string
	for {
		data, _, err = reader.ReadLine()
		if err != nil {
			fmt.Fprintf(os.Stderr, "[warn] resolve file: '%s' failed, err: %s\n", path, err.Error())
			file.Close()
			return
		}
		if bytes.HasPrefix(data, []byte("slug:")) {
			slug = string(bytes.TrimSpace(data[5:]))
			if slug == ">-" {
				data, _, err = reader.ReadLine()
				if err != nil {
					fmt.Fprintf(os.Stderr, "[warn] resolve file: '%s' failed, err: %s\n", path, err.Error())
					file.Close()
					return
				}
				slug = string(bytes.TrimSpace(data))
			}
			if slug[0] == '\'' || slug[0] == '"' {
				slug = slug[1 : len(slug)-1]
			}
			break
		}
	}
	file.Close()
	if slug != "" {
		redirect, ok := redirect[strings.Replace(strings.ToLower(slug), ":", "_colon_", -1)]
		if !ok {
			fmt.Fprintf(os.Stderr, "[warn] get redirect url failed for file: '%s'\n", path)
			return
		}
		cmd := exec.Command("yarn",
			"content", "delete",
			slug, *locale,
			"--redirect", redirect,
			"-y",
		)

		output, err := cmd.CombinedOutput()
		if err != nil {
			fmt.Fprintf(os.Stderr, "[warn] exec command on '%s' failed, err: %s\noutput: %s\n", path, err.Error(), output)
		}
	} else {
		fmt.Fprintf(os.Stderr, "[warn] resolve file: '%s' failed, err: no slug found", path)
	}
	return
}

python

usage:

python source.py --pwd /path/to/mdn/content --d /path/to/mdn/translated-content/ --locale <locale>

import argparse
import os
import requests
import subprocess

parser = argparse.ArgumentParser(description="Redirect resolver")
parser.add_argument("--pwd", type=str, default=".", help="`content` repo dir")
parser.add_argument("--d", type=str, default=".", help="`mdn/translate-content` repo dir")
parser.add_argument("--locale", type=str, default="en-US", help="locale for removing 'Tools'")

args = parser.parse_args()

if args.pwd != ".":
	os.chdir(args.pwd)

if args.locale == "":
	raise Exception("locale is empty")

paths = []
p = os.path.join(args.d, "files", args.locale.lower(), "tools")
for subdir, _, files in os.walk(p):
	for path in files:
		if path.endswith(".html") or path.endswith(".md"):
			paths.append(os.path.join(subdir, path))

paths.sort(key=lambda path: os.path.dirname(path), reverse=True)

for path in paths:
	slug = None
	with open(path, 'r') as f:
		for line in f.readlines():
			if line.startswith('slug:'):
				slug = line[5:].strip()
				break
	if slug is None:
		print("[warn] resolve file: '%s' failed, err: no slug found" % path)
		continue
	if slug[0] == '\'' or slug[0] == '"':
		slug = slug[1:-1]
	response = requests.get("https://developer.mozilla.org/en-US/docs/"+slug, allow_redirects=False)
	if not response.ok:
		print("[warn] get redirect url failed for file: '%s', status code: %d" %(path, response.status_code))
		continue
	redirect = response.headers.get('location', None)
	if redirect is None:
		print("[warn] get redirect url failed for file: '%s', err: empty Header 'Location'" % path)
		continue
	exec = subprocess.run('yarn content delete %s %s --redirect "%s" -y' %(slug, args.locale, redirect), shell=True)
	if exec.returncode != 0:
		print("[warn] exec command on '%s' failed, err: %d" % (path, exec.returncode))

precompiled executable files

• remove-tools.exe for windows x64
• remove-tools for linux x64

remove-tools.zip

[cw118]

l10n-fr taken care of in #4841 by @SphinxKnight 🤗

[yin1999]

l10n-zh is resolved by #4844 and #4845.

[myshov]

I have a question. Does anybody know Is there a plan to add localization to https://firefox-source-docs.mozilla.org/devtools-user/index.htm? It seems that a good amount of translations will be lost.

[alattalatta]

l10n-ko just merged: #4849

[yin1999]

l10n-pt-BR resolved in #5101.