PLANNING DOCS (rebase me)

This commit is a mess and needs broken up and rebased.

.github/workflows/bump.yml

@@ -51,7 +51,7 @@ jobs:
       - name: "install envsubst"
         run: |
           sudo apt-get update
-          sudo apt-get --yes --no-install-recommends gettext
+          sudo apt-get install --yes --no-install-recommends gettext
       - run: |
           ./scripts/update-versions.sh
       - name: "Create Pull Request"

.github/workflows/dev.yml

@@ -58,17 +58,45 @@ jobs:
               - '!(README.md|LICENSE|design-docs/**|.gitignore|.github/**)'
               - '.github/workflows/dev.yml'
 
-  build:
-    needs: [ check_changes ]
+  commits:
+    name: "Deduce commit history"
+    permissions:
+      contents: "read"
+      pull-requests: "read"
+    runs-on: "ubuntu-latest"
+    outputs:
+      commits: "${{ steps.commits.outputs.commits }}"
+    steps:
+      - name: "install jq"
+        run: |
+          set -euxo pipefail
+          sudo apt-get update
+          sudo apt-get install --yes --no-install-recommends jq
+      - name: "Checkout"
+        uses: "actions/checkout@v4"
+        with:
+          persist-credentials: "false"
+          fetch-depth: "0"
+      - name: "commits"
+        id: "commits"
+        run: |
+          set -euxo pipefail
+          git rev-list --pretty=oneline ${{ github.event.pull_request.base.sha }}..${{ github.event.pull_request.head.sha }} \
+            | sed -e  's/ /", "msg": "/' -e 's/^/{"ref": "/' -e 's/$/"}/' \
+            | jq --null-input --raw-output --compact-output '[inputs] | "commits=" + (. | tostring)' \
+            | tee -a "${GITHUB_OUTPUT}"
+
+  build-and-test:
+    name: "<${{ matrix.rust }}> ${{matrix.commit.msg}} [${{matrix.commit.ref}}]"
+    needs: [ check_changes, commits ]
     if: "${{ needs.check_changes.outputs.devfiles == 'true' }}"
     strategy:
       fail-fast: false
       matrix:
         rust:
           - "stable"
-          - "beta"
           - "nightly"
-    name: "Developer build (Rust ${{ matrix.rust }})"
+        commit: ${{ fromJSON(needs.commits.outputs.commits) }}
     runs-on: "lab"
     timeout-minutes: 45
     steps:
@@ -83,11 +111,12 @@ jobs:
         with:
           toolchain: "${{ matrix.rust }}"
           targets: "x86_64-unknown-linux-gnu,x86_64-unknown-linux-musl"
-      - name: "Checkout"
+      - name: "Checkout ${{matrix.commit.msg}}"
         uses: "actions/checkout@v4"
         with:
+          ref: "${{ matrix.commit.ref }}"
           persist-credentials: "false"
-          fetch-depth: "0"
+          fetch-depth: "1"
       - uses: "cargo-bins/cargo-binstall@main"
       - name: "install just"
         run: |
@@ -106,33 +135,13 @@ jobs:
           sudo apt-get --yes --no-install-recommends install pkg-config openssl libssl-dev colorized-logs
       - name: "install markdown-test-report"
         run: |
-          cargo binstall --no-confirm markdown-test-report
+          cargo install markdown-test-report
       - name: refresh-compile-env
         run: |
           just --yes debug=true refresh-compile-env
       - run: |
           just --yes debug=true fake-nix
 
-      - run: |
-          # Run a simple build for each separate commit (for "pull_request")
-          # or for the HEAD of the branch (other events).
-          set -eu -o pipefail
-          COMMITS=${{ github.sha }}
-          if [[ "${{ github.event_name == 'pull_request' }}" == "true" ]]; then
-            # Get all commits from Pull Request, in chronological order
-            COMMITS=$(git rev-list --reverse ${{ github.event.pull_request.base.sha }}..${{ github.event.pull_request.head.sha }})
-          fi
-          for commit in $COMMITS ; do
-            git checkout $commit || exit 1
-            printf "::group::Build commit %s\n" "$(git log --oneline --no-decorate -n 1)"
-            (just debug=true cargo +${{matrix.rust}} build --locked --profile=dev --target=x86_64-unknown-linux-gnu) || exit 1
-            printf "::endgroup::\n"
-          done
-          printf "::notice::HEAD remains at %s\n" "$(git log --oneline --no-decorate -n 1)"
-        continue-on-error: ${{ matrix.rust.optional }}
-
-      # At this stage, for Pull Requests, we're back to the HEAD of the branch,
-      # start running tests for different configurations.
       - name: "tests: rust ${{ matrix.rust }} profile=dev target=x86_64-unknown-linux-gnu"
         run: |
           export GITHUB_STEP_SUMMARY
@@ -166,7 +175,7 @@ jobs:
       - uses: "actions/upload-artifact@v4"
         if: ${{ always() }}
         with:
-          name: "test-results-${{ matrix.rust }}"
+          name: "results-${{ matrix.rust }}-${{ matrix.commit.ref }}"
           path: "target/nextest/"
 
       - name: "Setup tmate session for debug"
@@ -175,3 +184,17 @@ jobs:
         timeout-minutes: 60
         with:
           limit-access-to-actor: true
+
+
+  summary:
+    name: "Dev Test and Build"
+    if: ${{ always() }}
+    runs-on: "ubuntu-latest"
+    needs:
+      - build-and-test
+    steps:
+      - name: "Flag any build matrix failures"
+        if: ${{ needs.build-and-test.result != 'success' }}
+        run: |
+          >&2 echo "A critical step failed!"
+          exit 1

.github/workflows/sterile.yml

@@ -5,12 +5,12 @@
 name: "sterile.yml"
 
 on:
-  pull_request: {}
+  pull_request: { }
   push:
     branches:
       - "main"
   merge_group:
-    types: ["checks_requested"]
+    types: [ "checks_requested" ]
   workflow_dispatch:
     inputs:
       debug_enabled:
@@ -24,30 +24,34 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  check_changes:
-    name: "Deduce required tests from code changes"
+  commits:
+    name: "Track commits"
     runs-on: "ubuntu-latest"
     outputs:
       devfiles: "${{ steps.changes.outputs.devfiles }}"
+      commits: "${{ steps.commits.outputs.commits }}"
     steps:
+      - name: "install jq"
+        run: |
+          set -euxo pipefail
+          sudo apt-get update
+          sudo apt-get install --yes --no-install-recommends jq
       - name: "Checkout"
-        if: "${{ !github.event.pull_request }}"
         uses: "actions/checkout@v4"
         with:
           persist-credentials: "false"
           fetch-depth: "0"
-      - name: "Check code changes"
-        uses: "dorny/paths-filter@v3"
-        id: "changes"
-        with:
-          filters: |
-            devfiles:
-              - '!(README.md|LICENSE|design-docs/**|.gitignore|.github/**)'
-              - '.github/workflows/sterile.yml'
+      - name: "commits"
+        id: "commits"
+        run: |
+          set -euxo pipefail
+          git rev-list --pretty=oneline ${{ github.event.pull_request.base.sha }}..${{ github.event.pull_request.head.sha }} \
+            | sed -e  's/ /", "msg": "/' -e 's/^/{"ref": "/' -e 's/$/"}/' \
+            | jq --null-input --raw-output --compact-output '[inputs] | "commits=" + (. | tostring)' \
+            | tee -a "${GITHUB_OUTPUT}"
 
   test:
-    needs: [ check_changes ]
-    if: "${{ needs.check_changes.outputs.devfiles == 'true' }}"
+    needs: [ commits ]
     permissions:
       contents: "read"
       packages: "read"
@@ -58,8 +62,8 @@ jobs:
       matrix:
         rust:
           - "stable"
-          - "nightly"
-    name: "Sterile test run (Rust ${{ matrix.rust }})"
+        commit: ${{ fromJSON(needs.commits.outputs.commits) }}
+    name: "${{matrix.commit.msg}} [${{matrix.commit.ref}} rust: ${{ matrix.rust }}]"
     steps:
       - name: "login to ghcr.io"
         uses: "docker/login-action@v3"
@@ -81,6 +85,9 @@ jobs:
 
       - name: "Checkout"
         uses: "actions/checkout@v4"
+        with:
+          ref: ${{ matrix.commit.ref }}
+          fetch-depth: '1'
 
       - name: "dev/gnu sterile test"
         if: ${{ always() }}
@@ -123,7 +130,7 @@ jobs:
             libssl-dev \
             openssl \
             pkg-config
-          cargo binstall --no-confirm markdown-test-report
+          cargo install markdown-test-report
 
       - name: "dev/gnu report"
         if: ${{ always() }}
@@ -156,7 +163,7 @@ jobs:
       - uses: "actions/upload-artifact@v4"
         if: ${{ always() }}
         with:
-          name: 'rust-${{matrix.rust}}-test-results'
+          name: 'rust-${{matrix.rust}}-test-results-${{matrix.commit.ref}}'
           path: "target/nextest"
 
       - name: "Setup tmate session for debug"
@@ -167,8 +174,7 @@ jobs:
           limit-access-to-actor: true
 
   push:
-    needs: [ check_changes ]
-    if: "${{ needs.check_changes.outputs.devfiles == 'true' }}"
+    needs: [ commits ]
     permissions:
       contents: "read"
       packages: "write"
@@ -179,8 +185,7 @@ jobs:
       matrix:
         rust:
           - "stable"
-          - "nightly"
-    name: "Push containers (Rust ${{ matrix.rust }})"
+    name: "push container image [rust ${{ matrix.rust }}]"
     steps:
       - name: "login to ghcr.io"
         uses: "docker/login-action@v3"
@@ -229,3 +234,17 @@ jobs:
         timeout-minutes: 60
         with:
           limit-access-to-actor: true
+
+  summary:
+    name: "Sterile Test and Build"
+    if: ${{ always() }}
+    runs-on: "ubuntu-latest"
+    needs:
+      - push
+      - test
+    steps:
+      - name: "Flag any build matrix failures"
+        if: ${{ needs.push.result != 'success' || needs.test.result != 'success' }}
+        run: |
+          >&2 echo "A critical step failed!"
+          exit 1

design-docs/src/mdbook/book.toml

@@ -6,10 +6,8 @@ src = "src"
 title = "Hedgehog Dataplane Design Notes"
 
 [preprocessor.plantuml]
-#plantuml-cmd = "/lib/openjdk/bin/java -jar /lib/plantuml.jar"
 plantuml-cmd = "/bin/plantuml"
 after = ["links"]
-use-data-uris = false
 
 [preprocessor.mermaid]
 after = ["links"]
@@ -29,7 +27,6 @@ strict = false
 
 [output.html]
 additional-css = []
-additional-js = ["./theme/external/mermaid.min.js", "./theme/external/mermaid-init.js"]
 smart-punctuation = true
 preferrred-dark-theme = "ayu"
 default-theme = "light"

design-docs/src/mdbook/src/SUMMARY.md

@@ -1,5 +1,7 @@
 # Hedgehog Docs
 
+- [LINKS](links.md)
+
 - [Dataplane project: executive summary](./dataplane/executive-summary.md)
   - [Reflections from last time](./dataplane/reflections-from-last-time.md)
   - [Hardware selection for dataplane project](./dataplane/hardware.md)
@@ -11,3 +13,44 @@
   - [fake-nix](./build/fake-nix.md)
   - [Build dataplane](./build/just-cargo-build.md)
   - [Sterile builds](./build/sterile-build.md)
+
+- [Design session](./dataplane/design-session.md)
+
+- [Development Plan](./dataplane/development-plan.md)
+  - [Control plane dev-env](./dataplane/tasks2/control-plane-dev-env.md)
+  - [Create control plane image](./dataplane/tasks2/create-control-plane-image.md)
+  - [FRR Plugin](./dataplane/tasks2/frr-plugin.md)
+    - [Dataplane / Control plane transport](./dataplane/tasks2/dataplane-control-plane-transport.md)
+    - [Dataplane / Control plane protocol](./dataplane/tasks2/dataplane-control-plane-protocol.md)
+    - [Dataplane / Control plane reconcile](./dataplane/tasks2/dataplane-control-plane-reconcile.md)
+  - [Gateway test env](./dataplane/tasks2/gateway-test-env.md)
+  - [Identify local traffic](./dataplane/tasks2/identify-local-traffic.md)
+  - [Configuration Persistence Investigation](./dataplane/tasks2/configuration-persistence-investigation.md)
+  - [Route manager](./dataplane/tasks2/route-manager.md)
+  - [Dataplane worker lifecycle](./dataplane/tasks2/dataplane-worker-lifecycle.md)
+  - [Telemetry (investigation)](./dataplane/tasks2/telemetry-investigation.md)
+  - [Telemetry (basic)](./dataplane/tasks2/telemetry-basic.md)
+  - [Telemetry (integration)](./dataplane/tasks2/telemetry-integration.md)
+  - [Configuration database schema](./dataplane/tasks2/config-db-schema.md)
+  - [Management plane - dataplane interaction](./dataplane/tasks2/management-plane-dataplane-interaction.md)
+  - [VXLAN tunnels](./dataplane/tasks2/vxlan-tunnels.md)
+
+  - [Underlay routing](./dataplane/tasks2/underlay-routing.md)
+  - [Management plane - dataplane interaction](./dataplane/tasks2/management-plane-dataplane-interaction.md)
+  - [Management plane - control plane interaction](./dataplane/tasks2/management-plane-control-plane-interaction.md)
+  - [VPC routing](./dataplane/tasks2/vpc-routing.md)
+  - [Rate limiting investigation](./dataplane/tasks2/rate-limiting-investigation.md)
+  - [VPC rate-limiting](./dataplane/tasks2/vpc-rate-limiting.md)
+  - [NAT44](./dataplane/tasks2/NAT44.md)
+  - [NAT66](./dataplane/tasks2/NAT66.md)
+  - [NAT64 (investigation)](./dataplane/tasks2/NAT64-investigation.md)
+  - [NAT64](./dataplane/tasks2/NAT64.md)
+  - [State sync (design)](./dataplane/tasks2/state-sync-design.md)
+  - [State sync (implementation)](./dataplane/tasks2/state-sync.md)
+  - [Public internet access](./dataplane/tasks2/public-internet-access.md)
+  - [Fault tolerance (implementation)](./dataplane/tasks2/fault-tolerance-implementation.md)
+  - [Fault tolerance (validation)](./dataplane/tasks2/fault-tolerance-validation.md)
+  - [Performance measurement](./dataplane/tasks2/performance-measurement.md)
+  - [Core pinning](./dataplane/tasks2/core-pinning.md)
+  - [One control plane daemon per container](./dataplane/tasks2/one-control-plane-daemon-per-container.md)
+  - [Programattic Control of FRR](./dataplane/tasks2/programmatic-control-of-frr.md)

design-docs/src/mdbook/src/context.toml

@@ -0,0 +1 @@
+siteRoot="/"