From 1b2c449cebd1b3823520a0a8cf09119484e11a26 Mon Sep 17 00:00:00 2001 From: Joshua Seaton Date: Sat, 10 Aug 2024 19:14:36 -0700 Subject: [PATCH] [scripts] Introduce scripts/fetch-toolchains.py This change introduces $LK_ROOT/toolchain as an official installation directory for toolchains, as well as a new script for conveniently installing them there (versus manual GETs + untarring). getting_started.md is updated to suggest its use. --- .gitignore | 1 + docs/getting_started.md | 16 ++- scripts/fetch-toolchains.py | 221 ++++++++++++++++++++++++++++++++++++ 3 files changed, 229 insertions(+), 9 deletions(-) create mode 100755 scripts/fetch-toolchains.py diff --git a/.gitignore b/.gitignore index f04757e96..41787f305 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,4 @@ scripts/toolpaths.local .vscode .cache compile_commands.json +toolchain diff --git a/docs/getting_started.md b/docs/getting_started.md index 9bfb4205a..6d311d010 100644 --- a/docs/getting_started.md +++ b/docs/getting_started.md @@ -6,25 +6,23 @@ ``` mkdir -p lk-work && cd lk-work ``` -2- Clone the repo +2- Clone the repo and change dir to the root ``` git clone https://github.com/littlekernel/lk +cd lk + ``` -3- Download appropriate toolchain and extract it +3- Download appropriate toolchain ``` -wget https://newos.org/toolchains/riscv64-elf-14.2.0-Linux-x86_64.tar.xz - -mkdir -p toolchain -tar xf riscv64-elf-14.2.0-Linux-x86_64.tar.xz -cd .. +# Fetches the latest riscv64-elf toolchain for your host. +scripts/fetch-toolchains.py --prefix riscv64-elf ``` 4- Add toolchain to PATH ``` export PATH=$PWD/toolchain/riscv64-elf-14.2.0-Linux-x86_64/bin:$PATH ``` -5- Change dir to lk to build and find available project +5- Find available project ``` -cd lk ls project/* ``` 6- E.g pick `qemu-virt-riscv64-test` and build kernel diff --git a/scripts/fetch-toolchains.py b/scripts/fetch-toolchains.py new file mode 100755 index 000000000..70935be7c --- /dev/null +++ b/scripts/fetch-toolchains.py @@ -0,0 +1,221 @@ +#!/usr/bin/env python3 + +# A utility for installing LK toolchains. + +from __future__ import annotations + +import argparse +import html.parser +import io +import os +import pathlib +import sys +import tarfile +import threading +import urllib.request +from typing import Self + +BASE_URL = "https://newos.org/toolchains" + +HOST_OS = os.uname().sysname +HOST_CPU = os.uname().machine + +LK_ROOT = pathlib.Path(os.path.realpath(__file__)).parent.parent +DEFAULT_TOOLCHAIN_DIR = LK_ROOT.joinpath("toolchain") + +TAR_EXT = ".tar.xz" + + +def main() -> int: + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + description="Installs the matching LK toolchains from the official host, " + + BASE_URL, + ) + parser.add_argument( + "--list", + help="just list the matching toolchains; don't download them", + action="store_true", + ) + parser.add_argument( + "--prefix", + help="a toolchain prefix on which to match. If none are specified, all prefixes" + " will match", + nargs="*", + ) + parser.add_argument( + "--version", + help='the exact toolchain version to match, or "latest" to specify only the ' + 'latest version, or "all" for all versions', + type=str, + default="latest", + ) + parser.add_argument( + "--install-dir", + help="the directory at which to install the toolchains", + type=pathlib.Path, + default=DEFAULT_TOOLCHAIN_DIR, + ) + parser.add_argument( + "--force", + help="whether to overwrite past installed versions of matching toolchains", + action="store_true", + ) + parser.add_argument( + "--host-os", + help="the toolchains' host OS", + type=str, + default=HOST_OS, + ) + parser.add_argument( + "--host-cpu", + help="the toolchains' host architecture", + type=str, + default=HOST_CPU, + ) + args = parser.parse_args() + + # Get the full list of remote toolchains available for the provided host. + response = urllib.request.urlopen(BASE_URL) + if response.status != 200: + print(f"Error accessing {BASE_URL}: {response.status}") + return 1 + parser = RemoteToolchainHTMLParser(args.host_os, args.host_cpu) + parser.feed(response.read().decode("utf-8")) + toolchains = parser.toolchains + + # Filter them given --prefix and --version selections. + toolchains.sort() + if args.prefix: + toolchains = [t for t in toolchains if t.prefix in args.prefix] + if args.version == "latest": + # Since we sorted lexicographically on (prefix, version tokens), to pick out the + # latest versions we need only iterate through and pick out the last entry for a + # given prefix. + toolchains = [ + toolchains[i] + for i in range(len(toolchains)) + if ( + i == len(toolchains) - 1 + or toolchains[i].prefix != toolchains[i + 1].prefix + ) + ] + elif args.version != "all": + toolchains = [t for t in toolchains if t.version == args.version] + + if not toolchains: + print("No matching toolchains") + return 0 + + if args.list: + print("Matching toolchains:") + for toolchain in toolchains: + print(toolchain.name) + return 0 + + # The download routine for a given toolchain, factored out for + # multithreading below. + def download(toolchain: RemoteToolchain) -> None: + response = urllib.request.urlopen(toolchain.url) + if response.status != 200: + print(f"Error while downloading {toolchain.name}: {response.status}") + return + with tarfile.open(fileobj=io.BytesIO(response.read()), mode="r:xz") as f: + f.extractall(path=args.install_dir, filter="data") + + downloads = [] + for toolchain in toolchains: + local = args.install_dir.joinpath(toolchain.name) + if local.exists() and not args.force: + print( + f"{toolchain.name} already installed; " + "skipping... (pass --force to overwrite)", + ) + continue + print(f"Downloading {toolchain.name} to {local}...") + downloads.append(threading.Thread(target=download, args=(toolchain,))) + downloads[-1].start() + + for thread in downloads: + thread.join() + + return 0 + + +class RemoteToolchain: + def __init__(self, prefix: str, version: str, host_os: str, host_cpu: str) -> None: + self._prefix = prefix + self._version = [int(token) for token in version.split(".")] + self._host = f"{host_os}-{host_cpu}" + + # Orders toolchains lexicographically on (prefix, version tokens). + def __lt__(self, other: Self) -> bool: + return self._prefix < other.prefix or ( + self._prefix == other.prefix and self._version < other._version + ) + + @property + def prefix(self) -> str: + return self._prefix + + @property + def version(self) -> str: + return ".".join(map(str, self._version)) + + @property + def name(self) -> str: + return f"{self._prefix}-{self.version}-{self._host}" + + @property + def url(self) -> str: + return f"{BASE_URL}/{self.name}{TAR_EXT}" + + +# A simple HTML parser for extracting the toolchain names found at BASE_URL. +# +# It expects toolchains to be available as hyperlinks on that page. Once the +# HTML has been passed to feed(), the parsed toolchains will be accessible via +# toolchains(). +class RemoteToolchainHTMLParser(html.parser.HTMLParser): + def __init__(self, host_os: str, host_cpu: str) -> None: + html.parser.HTMLParser.__init__(self) + self._toolchains = [] + self._tags = [] + self._host_os = host_os + self._host_cpu = host_cpu + + # The parsed toolchains. + @property + def toolchains(self) -> list[RemoteToolchain]: + return self._toolchains + + # + # The following methods implement the parsing, overriding those defined in + # the base class. + # + + def handle_starttag(self, tag: str, _: str) -> None: + self._tags.append(tag) + + def handle_endtag(self, _: str) -> None: + self._tags.pop() + + def handle_data(self, data: str) -> None: + # Only process hyperlinks with tarball names. + if not self._tags or self._tags[-1] != "a" or not data.endswith(TAR_EXT): + return + tokens = data.removesuffix(TAR_EXT).split("-") + if len(tokens) != 5: + print(f"Warning: malformed toolchain name: {data}") + return + prefix = tokens[0] + "-" + tokens[1] + version = tokens[2] + host_os = tokens[3] + host_cpu = tokens[4] + if host_os != self._host_os or host_cpu != self._host_cpu: + return + self._toolchains.append(RemoteToolchain(prefix, version, host_os, host_cpu)) + + +if __name__ == "__main__": + sys.exit(main())