Feature request: add support to monorepo style tags for external Git resources

[robsonos]

Hi Platformio Core team,

I am suggesting adding support to monorepo style tags for external Git resources, so the following can be used:

lib_deps =
  https://github.com/username/repo.git#[email protected]
  https://github.com/username/repo.git#libraries/[email protected]

Where [email protected] and libraries/[email protected] are the actual tags.

The following could also be considered:

lib_deps =
  baz=https://github.com/username/repo/archive/refs/tags/[email protected]
  qux =https://github.com/username/repo/archive/refs/tags/libraries/[email protected]

This may be related to #4562 and #4366, and it may provide a solution to #3990.
I am happy to work on a PR if needed.

Cheers,

Robson

[leon0399]

I'd like to see this feature!
Personally, I do prefer 1st approach, looks clean

[robsonos]

Hi @leon0399,

Both suggestions should be considered, especially for the monorepo scenario.
PlatformIO uses autogenerate source code zips for the dependency installation AFAIK. As GIthub autogenerates those source code zips when releases are created (and there is no way to disable or change this behaviour currently), PlatformIO will either need a way (or convention) to identify the lib folder inside the zip source code or we tell it (like in the second suggestion, using release assets rather than source code) what zip to use.

Here is the workaround I am using at the moment:

• Assuming you released [email protected] with a YourLib-1.0.0.zip (notice the - instead of @ on the zip) release asset in https://github.com/ORG/REPO/releases/download/YourLib%401.0.0/YourLib-1.0.0.zip

• Add the following scripts/install_external_lib_deps.py file to your project:

from os.path import join, exists
import subprocess
import requests
from SCons.Script import ARGUMENTS
Import("env")

GITHUB_API_URL = "https://api.github.com/repos/{repo}/releases/{endpoint}"
CHUNK_SIZE = 8192


def github_request(url, headers=None, stream=False):
    """Make a GitHub API request with the provided URL and headers."""
    if headers is None:
        headers = {}
    response = requests.get(url, headers=headers, stream=stream)
    response.raise_for_status()
    return response


def list_github_assets(repo, tag, token=None, verbose=0):
    """List the assets for a given GitHub release tag."""
    url = GITHUB_API_URL.format(repo=repo, endpoint=f"tags/{tag}")
    headers = {"Accept": "application/vnd.github+json"}
    if token:
        headers["Authorization"] = f"Bearer {token}"

    release_data = github_request(url, headers).json()
    assets = release_data.get('assets', [])

    if not assets:
        raise ValueError(f"No assets found for release '{tag}'.")

    if verbose > 0:
        for asset in assets:
            print("\033[93m" +
                  f"Found asset for tag '{tag}': {asset['name']}, id: {asset['id']}")

    return assets


def download_github_asset_by_id(repo, asset_id, dest_dir, token=None, verbose=0):
    """Download a GitHub release asset by its ID."""
    url = GITHUB_API_URL.format(repo=repo, endpoint=f"assets/{asset_id}")
    headers = {"Accept": "application/octet-stream"}
    if token:
        headers["Authorization"] = f"Bearer {token}"

    response = github_request(url, headers=headers, stream=True)
    asset_name = response.headers.get(
        'content-disposition').split('filename=')[-1].strip('"')
    file_path = join(dest_dir, asset_name)

    with open(file_path, 'wb') as file:
        for chunk in response.iter_content(chunk_size=CHUNK_SIZE):
            if chunk:
                file.write(chunk)

    if verbose > 0:
        print("\033[93m" +
              f"'{asset_name}' downloaded successfully to '{file_path}'.")

    return file_path


def tag_exists_in_dest_dir(tag, dest_dir):
    """Check if a file for the given tag already exists in the destination directory."""
    converted_tag = tag.replace('@', '-')
    expected_filename = f"{converted_tag}.zip"
    expected_file_path = join(dest_dir, expected_filename)
    return exists(expected_file_path), expected_file_path


def install_external_lib_deps(env, verbose):
    """Install library dependencies based on the specified tags."""
    verbose = int(verbose)
    repo = "" # your ORG/REPO here
    token = env.get('ENV').get('GH_TOKEN') # GH_TOKEN needs to be set if using this script with GitHub Actions

    if token is None:
        print("\033[93m" +
              "GH_TOKEN not found. Ignoring custom_lib_deps")
        return

    config = env.GetProjectConfig()
    raw_lib_deps = env.GetProjectOption('custom_lib_deps')
    lib_deps = config.parse_multi_values(raw_lib_deps)

    lib_deps_dir = env.get("PROJECT_LIBDEPS_DIR")
    env_type = env.get("PIOENV")
    dest_dir = join(lib_deps_dir, env_type)

    for tag in lib_deps:
        tag = tag.strip()
        if not tag:
            continue

        try:
            tag_exists, file_path = tag_exists_in_dest_dir(tag, dest_dir)
            if tag_exists:
                if verbose > 0:
                    print("\033[93m" +
                          f"Tag '{tag}' already exists in '{dest_dir}'. Skipping installation")
                continue
            else:
                assets = list_github_assets(repo, tag, token, verbose)
                if not assets:
                    raise ValueError(f"No assets found for release '{tag}'.")
                asset_id = assets[0]['id']
                file_path = download_github_asset_by_id(
                    repo, asset_id, dest_dir, token, verbose)

            install_cmd = [
                env.subst("$PYTHONEXE"), "-m", "platformio", "pkg", "install",
                "-l", f"=file://{file_path}", "--no-save"
            ]

            if verbose < 1:
                install_cmd.append("-s")

            subprocess.check_call(install_cmd)
        except Exception as e:
            raise RuntimeError(f"Error processing tag '{tag}': {e}")


VERBOSE = ARGUMENTS.get("PIOVERBOSE", 0)
install_external_lib_deps(env, VERBOSE)

• Add the following to your platformio.ini:

...
extra_scripts =
  pre:scripts/install_external_lib_deps.py
...
custom_lib_deps =
  [email protected]
...

Cheers,

Robson