Merge branch 'dev'

Author: gilesknap

.devcontainer/Dockerfile

@@ -1,31 +1,37 @@
-# ideas from https://www.docker.com/blog/containerized-python-development-part-1/
+# This file is for use as a devcontainer and a runtime container
+#
+# The devcontainer should use the build target and run as root with podman
+# or docker with user namespaces.
+#
+FROM python:3.10 as build
 
-# This file is for use as a .vscode devcontainer as well as a runtime 
-# container. The devcontainer should be rootful and use podman or docker 
-# with user namespaces.
+ARG PIP_OPTIONS
 
-ARG BASE="mcr.microsoft.com/vscode/devcontainers/python:0-3.10-bullseye"
-FROM ${BASE} as base
+# Add any system dependencies for the developer/build environment here e.g.
+# RUN apt-get update && apt-get upgrade -y && \
+#     apt-get install -y --no-install-recommends \
+#     desired-packages \
+#     && rm -rf /var/lib/apt/lists/*
 
-# use root to pin where the packages will install
-USER root
-ENV PATH=/root/.local/bin:$PATH
+# set up a virtual environment and put it in PATH
+RUN python -m venv /venv
+ENV PATH=/venv/bin:$PATH
 
-FROM base as developer
+# Copy any required context for the pip install over
+COPY . /context
+WORKDIR /context
 
-WORKDIR /workspace
-COPY . .
+# install python package into /venv
+RUN pip install ${PIP_OPTIONS}
 
-# install runtime from DIST if there is one
-RUN mkdir -p /root/.local && \
-    if [ -d dist ] ; then \
-    touch requirements.txt && \
-    pip install --user -r requirements.txt dist/*.whl ; \
-    fi
+FROM python:3.10-slim as runtime
 
-FROM base as runtime
+# Add apt-get system dependecies for runtime here if needed
 
-COPY --from=developer /root/.local /root/.local
+# copy the virtual environment from the build stage and put it in PATH
+COPY --from=build /venv/ /venv/
+ENV PATH=/venv/bin:$PATH
 
+# change this entrypoint if it is not the same as the repo
 ENTRYPOINT ["ibek"]
 CMD ["--version"]

.devcontainer/devcontainer.json

@@ -1,68 +1,41 @@
-// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at:
-// https://github.com/microsoft/vscode-dev-containers/tree/v0.231.6/containers/python-3
+// For format details, see https://aka.ms/devcontainer.json
 {
-    "name": "Python 3",
+    "name": "Python 3 Developer Container",
     "build": {
         "dockerfile": "Dockerfile",
-        "target": "developer",
-        "context": "..",
-        "args": {}
+        "target": "build",
+        // Only upgrade pip, we will install the project below
+        "args": {
+            "PIP_OPTIONS": "--upgrade pip"
+        }
     },
     "remoteEnv": {
         "DISPLAY": "${localEnv:DISPLAY}"
     },
     // Set *default* container specific settings.json values on container create.
     "settings": {
-        "python.defaultInterpreterPath": "/usr/local/bin/python",
-        "python.linting.enabled": true,
-        "python.linting.pylintEnabled": true,
-        "python.formatting.autopep8Path": "/usr/local/py-utils/bin/autopep8",
-        "python.formatting.blackPath": "/usr/local/py-utils/bin/black",
-        "python.formatting.yapfPath": "/usr/local/py-utils/bin/yapf",
-        "python.linting.banditPath": "/usr/local/py-utils/bin/bandit",
-        "python.linting.flake8Path": "/usr/local/py-utils/bin/flake8",
-        "python.linting.mypyPath": "/usr/local/py-utils/bin/mypy",
-        "python.linting.pycodestylePath": "/usr/local/py-utils/bin/pycodestyle",
-        "python.linting.pydocstylePath": "/usr/local/py-utils/bin/pydocstyle",
-        "python.linting.pylintPath": "/usr/local/py-utils/bin/pylint"
+        "python.defaultInterpreterPath": "/venv/bin/python"
     },
     // Add the IDs of extensions you want installed when the container is created.
     "extensions": [
         "ms-python.python",
-        "ms-python.vscode-pylance",
-        "streetsidesoftware.code-spell-checker",
-        "ryanluker.vscode-coverage-gutters",
-        "mhutchie.git-graph",
-        "eamodio.gitlens",
-        "gruntfuggly.todo-tree",
-        "redhat.vscode-yaml",
-        "nsd.vscode-epics",
-        "alefragnani.bookmarks"
+        "ms-python.vscode-pylance"
     ],
-    "features": {
-        //"docker-from-docker": "20.10",
-        "git": "os-provided"
-    },
-    // Use 'forwardPorts' to make a list of ports inside the container available locally.
-    // "forwardPorts": [],
-    // Comment out to connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root.
-    // "remoteUser": "vscode",
     // Make sure the files we are mapping into the container exist on the host
-    "initializeCommand": "bash -c 'for i in $HOME/.inputrc $HOME/.bashrc_dev; do [ -f $i ] || touch $i; done'",
+    "initializeCommand": "bash -c 'for i in $HOME/.inputrc; do [ -f $i ] || touch $i; done'",
     "runArgs": [
-        "--privileged",
         "--net=host",
+        "--security-opt=label=type:container_runtime_t",
         "-v=${localEnv:HOME}/.ssh:/root/.ssh",
-        "-v=${localEnv:HOME}/.bashrc_dev:/root/.bashrc",
         "-v=${localEnv:HOME}/.inputrc:/root/.inputrc"
     ],
     "mounts": [
-        // map in home directory - not strictly necessary but may be useful
+        // map in home directory - not strictly necessary but useful
         "source=${localEnv:HOME},target=${localEnv:HOME},type=bind,consistency=cached"
     ],
-    "workspaceMount": "source=${localWorkspaceFolder},target=/workspace,type=bind",
-    "workspaceFolder": "/workspace",
+    // make the workspace folder the same inside and outside of the container
+    "workspaceMount": "source=${localWorkspaceFolder},target=${localWorkspaceFolder},type=bind",
+    "workspaceFolder": "${localWorkspaceFolder}",
     // After the container is created, install the python project in editable form
-    // This installs into the system python of the container
-    "postCreateCommand": "pip install $([ -f requirements_dev.txt ] && echo -r requirements_dev.txt ) -e .[dev]"
+    "postCreateCommand": "pip install -e .[dev]"
 }

.gitattributes

@@ -0,0 +1,35 @@
+Contributing to the project
+===========================
+
+Contributions and issues are most welcome! All issues and pull requests are
+handled through GitHub_. Also, please check for any existing issues before
+filing a new one. If you have a great idea but it involves big changes, please
+file a ticket before making a pull request! We want to make sure you don't spend
+your time coding something that might not fit the scope of the project.
+
+.. _GitHub: https://github.com/DiamondLightSource/python3-pip-skeleton/issues
+
+Issue or Discussion?
+--------------------
+
+Github also offers discussions_ as a place to ask questions and share ideas. If
+your issue is open ended and it is not obvious when it can be "closed", please
+raise it as a discussion instead.
+
+.. _discussions: https://github.com/DiamondLightSource/python3-pip-skeleton/discussions
+
+Code coverage
+-------------
+
+While 100% code coverage does not make a library bug-free, it significantly
+reduces the number of easily caught bugs! Please make sure coverage remains the
+same or is improved by a pull request!
+
+Developer guide
+---------------
+
+The `Developer Guide`_ contains information on setting up a development
+environment, running the tests and what standards the code and documentation
+should follow.
+
+.. _Developer Guide: https://diamondlightsource.github.io/python3-pip-skeleton/main/developer/how-to/contribute.html

.github/CONTRIBUTING.rst

@@ -0,0 +1,57 @@
+name: Install requirements
+description: Run pip install with requirements and upload resulting requirements
+inputs:
+  requirements_file:
+    description: Name of requirements file to use and upload
+    required: true
+  install_options:
+    description: Parameters to pass to pip install
+    required: true
+  python_version:
+    description: Python version to install
+    default: "3.10"
+
+runs:
+  using: composite
+
+  steps:
+    - name: Setup python
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ inputs.python_version }}
+
+    - name: Pip install
+      run: |
+        touch ${{ inputs.requirements_file }}
+        # -c uses requirements.txt as constraints, see 'Validate requirements file'
+        pip install -c ${{ inputs.requirements_file }} ${{ inputs.install_options }}
+      shell: bash
+
+    - name: Create lockfile
+      run: |
+        mkdir -p lockfiles
+        pip freeze --exclude-editable > lockfiles/${{ inputs.requirements_file }}
+        # delete the self referencing line and make sure it isn't blank
+        sed -i '/file:/d' lockfiles/${{ inputs.requirements_file }}
+      shell: bash
+
+    - name: Upload lockfiles
+      uses: actions/upload-artifact@v3
+      with:
+        name: lockfiles
+        path: lockfiles
+
+    # This eliminates the class of problems where the requirements being given no
+    # longer match what the packages themselves dictate. E.g. In the rare instance
+    # where I install some-package which used to depend on vulnerable-dependency
+    # but now uses good-dependency (despite being nominally the same version)
+    # pip will install both if given a requirements file with -r
+    - name: If requirements file exists, check it matches pip installed packages
+      run: |
+        if [ -s ${{ inputs.requirements_file }} ]; then
+          if ! diff -u ${{ inputs.requirements_file }} lockfiles/${{ inputs.requirements_file }}; then
+            echo "Error: ${{ inputs.requirements_file }} need the above changes to be exhaustive"
+            exit 1
+          fi
+        fi
+      shell: bash

.github/actions/install_requirements/action.yml

@@ -0,0 +1,16 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for all configuration options:
+# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
+
+version: 2
+updates:
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+
+  - package-ecosystem: "pip"
+    directory: "/"
+    schedule:
+      interval: "weekly"

.github/dependabot.yml

@@ -0,0 +1,99 @@
+import json
+import logging
+from argparse import ArgumentParser
+from pathlib import Path
+from subprocess import CalledProcessError, check_output
+from typing import List, Optional
+
+
+def report_output(stdout: bytes, label: str) -> List[str]:
+    ret = stdout.decode().strip().split("\n")
+    print(f"{label}: {ret}")
+    return ret
+
+
+def get_branch_contents(ref: str) -> List[str]:
+    """Get the list of directories in a branch."""
+    stdout = check_output(["git", "ls-tree", "-d", "--name-only", ref])
+    return report_output(stdout, "Branch contents")
+
+
+def get_sorted_tags_list() -> List[str]:
+    """Get a list of sorted tags in descending order from the repository."""
+    stdout = check_output(["git", "tag", "-l", "--sort=-v:refname"])
+    return report_output(stdout, "Tags list")
+
+
+def get_versions(ref: str, add: Optional[str], remove: Optional[str]) -> List[str]:
+    """Generate the file containing the list of all GitHub Pages builds."""
+    # Get the directories (i.e. builds) from the GitHub Pages branch
+    try:
+        builds = set(get_branch_contents(ref))
+    except CalledProcessError:
+        builds = set()
+        logging.warning(f"Cannot get {ref} contents")
+
+    # Add and remove from the list of builds
+    if add:
+        builds.add(add)
+    if remove:
+        assert remove in builds, f"Build '{remove}' not in {sorted(builds)}"
+        builds.remove(remove)
+
+    # Get a sorted list of tags
+    tags = get_sorted_tags_list()
+
+    # Make the sorted versions list from main branches and tags
+    versions: List[str] = []
+    for version in ["master", "main"] + tags:
+        if version in builds:
+            versions.append(version)
+            builds.remove(version)
+
+    # Add in anything that is left to the bottom
+    versions += sorted(builds)
+    print(f"Sorted versions: {versions}")
+    return versions
+
+
+def write_json(path: Path, repository: str, versions: str):
+    org, repo_name = repository.split("/")
+    struct = [
+        dict(version=version, url=f"https://{org}.github.io/{repo_name}/{version}/")
+        for version in versions
+    ]
+    text = json.dumps(struct, indent=2)
+    print(f"JSON switcher:\n{text}")
+    path.write_text(text)
+
+
+def main(args=None):
+    parser = ArgumentParser(
+        description="Make a versions.txt file from gh-pages directories"
+    )
+    parser.add_argument(
+        "--add",
+        help="Add this directory to the list of existing directories",
+    )
+    parser.add_argument(
+        "--remove",
+        help="Remove this directory from the list of existing directories",
+    )
+    parser.add_argument(
+        "repository",
+        help="The GitHub org and repository name: ORG/REPO",
+    )
+    parser.add_argument(
+        "output",
+        type=Path,
+        help="Path of write switcher.json to",
+    )
+    args = parser.parse_args(args)
+
+    # Write the versions file
+    versions = get_versions("origin/gh-pages", args.add, args.remove)
+    write_json(args.output, args.repository, versions)
+
+
+if __name__ == "__main__":
+    main()