add separate CI job for memory usage stats

Author: ibraheemdev

.github/workflows/mypy_primer.yaml

@@ -48,8 +48,6 @@ jobs:
 
       - name: Run mypy_primer
         shell: bash
-        env:
-          TY_MEMORY_REPORT: mypy_primer
         run: |
           cd ruff
 
@@ -101,3 +99,75 @@ jobs:
         with:
           name: pr-number
           path: pr-number
+
+  memory_usage:
+    name: Run memory statistics
+    runs-on: depot-ubuntu-22.04-32
+    timeout-minutes: 20
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          path: ruff
+          fetch-depth: 0
+          persist-credentials: false
+
+      - name: Install the latest version of uv
+        uses: astral-sh/setup-uv@bd01e18f51369d5a26f1651c3cb451d3417e3bba # v6.3.1
+
+      - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8
+        with:
+          workspaces: "ruff"
+
+      - name: Install Rust toolchain
+        run: rustup show
+
+      - name: Run mypy_primer
+        shell: bash
+        env:
+          TY_MAX_PARALLELISM: 1 # for deterministic memory numbers
+          TY_MEMORY_REPORT: mypy_primer
+        run: |
+          cd ruff
+
+          echo "Enabling mypy primer specific configuration overloads (see .github/mypy-primer-ty.toml)"
+          mkdir -p ~/.config/ty
+          cp .github/mypy-primer-ty.toml ~/.config/ty/ty.toml
+
+          PRIMER_SELECTOR="$(paste -s -d'|' crates/ty_python_semantic/resources/primer/memory.txt)"
+
+          echo "new commit"
+          git rev-list --format=%s --max-count=1 "$GITHUB_SHA"
+
+          MERGE_BASE="$(git merge-base "$GITHUB_SHA" "origin/$GITHUB_BASE_REF")"
+          git checkout -b base_commit "$MERGE_BASE"
+          echo "base commit"
+          git rev-list --format=%s --max-count=1 base_commit
+
+          cd ..
+
+          echo "Project selector: $PRIMER_SELECTOR"
+          # Allow the exit code to be 0 or 1, only fail for actual mypy_primer crashes/bugs
+          uvx \
+            --from="git+https://github.com/hauntsaninja/mypy_primer@e5f55447969d33ae3c7ccdb183e2a37101867270" \
+            mypy_primer \
+            --repo ruff \
+            --type-checker ty \
+            --old base_commit \
+            --new "$GITHUB_SHA" \
+            --project-selector "/($PRIMER_SELECTOR)\$" \
+            --output concise \
+            --debug > mypy_primer_memory.diff || [ $? -eq 1 ]
+
+          # Output diff with ANSI color codes
+          cat mypy_primer_memory.diff
+
+          # Remove ANSI color codes before uploading
+          sed -ie 's/\x1b\[[0-9;]*m//g' mypy_primer_memory.diff
+
+          echo ${{ github.event.number }} > pr-number
+
+      - name: Upload diff
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: mypy_primer_memory_diff
+          path: mypy_primer_memory.diff

.github/workflows/mypy_primer_comment.yaml

@@ -45,15 +45,28 @@ jobs:
           if_no_artifact_found: ignore
           allow_forks: true
 
+      - uses: dawidd6/action-download-artifact@20319c5641d495c8a52e688b7dc5fada6c3a9fbc # v8
+        name: "Download mypy_primer memory results"
+        id: download-mypy_primer_memory_diff
+        if: steps.pr-number.outputs.pr-number
+        with:
+          name: mypy_primer_memory_diff
+          workflow: mypy_primer.yaml
+          pr: ${{ steps.pr-number.outputs.pr-number }}
+          path: pr/mypy_primer_memory_diff
+          workflow_conclusion: completed
+          if_no_artifact_found: ignore
+          allow_forks: true
+
       - name: Generate comment content
         id: generate-comment
-        if: steps.download-mypy_primer_diff.outputs.found_artifact == 'true'
+        if: ${{ steps.download-mypy_primer_diff.outputs.found_artifact == 'true' && steps.download-mypy_primer_memory_diff.outputs.found_artifact == 'true' }}
         run: |
           # Guard against malicious mypy_primer results that symlink to a secret
           # file on this runner
-          if [[ -L pr/mypy_primer_diff/mypy_primer.diff ]]
+          if [[ -L pr/mypy_primer_diff/mypy_primer.diff ]] || [[ -L pr/mypy_primer_memory_diff/mypy_primer_memory.diff ]]
           then
-              echo "Error: mypy_primer.diff cannot be a symlink"
+              echo "Error: mypy_primer.diff and mypy_primer_memory.diff cannot be a symlink"
               exit 1
           fi
 
@@ -74,6 +87,18 @@ jobs:
             echo 'No ecosystem changes detected ✅' >> comment.txt
           fi
 
+          if [ -s "pr/mypy_primer_memory_diff/mypy_primer_memory.diff" ]; then
+            echo '<details>' >> comment.txt
+            echo '<summary>Memory usage changes were detected when running on open source projects</summary>' >> comment.txt
+            echo '' >> comment.txt
+            echo '```diff' >> comment.txt
+            cat pr/mypy_primer_memory_diff/mypy_primer_memory.diff >> comment.txt
+            echo '```' >> comment.txt
+            echo '</details>' >> comment.txt
+          else
+            echo 'No memory usage changes detected ✅' >> comment.txt
+          fi
+
           echo 'comment<<EOF' >> "$GITHUB_OUTPUT"
           cat comment.txt >> "$GITHUB_OUTPUT"
           echo 'EOF' >> "$GITHUB_OUTPUT"

crates/ty/src/lib.rs

@@ -296,6 +296,11 @@ impl MainLoop {
                             tracing::warn!("No python files found under the given path(s)");
                         }
 
+                        // TODO: We should have an official flag to silence workspace diagnostics.
+                        if std::env::var("TY_MEMORY_REPORT").as_deref() == Ok("mypy_primer") {
+                            return Ok(ExitStatus::Success);
+                        }
+
                         let mut stdout = stdout().lock();
 
                         if result.is_empty() {

crates/ty_project/src/db.rs

@@ -324,8 +324,8 @@ impl SalsaMemoryDump {
         struct DisplayShort<'a>(&'a SalsaMemoryDump);
 
         fn round_memory(total: usize) -> usize {
-            // Round the number to the nearest power of 1.1. This gives us a
-            // 5% threshold before the memory usage number is considered to have
+            // Round the number to the nearest power of 1.05. This gives us a
+            // 2.5% threshold before the memory usage number is considered to have
             // changed.
             //
             // TODO: Small changes in memory usage may cause the number to be rounded
@@ -334,7 +334,7 @@ impl SalsaMemoryDump {
             // over time that are unrelated to the current change. Ideally we could compare
             // the exact numbers across runs and compute the difference, but we don't have
             // the infrastructure for that currently.
-            const BASE: f64 = 1.1;
+            const BASE: f64 = 1.05;
             BASE.powf(bytes_to_mb(total).log(BASE).round()) as usize
         }
 

crates/ty_python_semantic/resources/primer/memory.txt

@@ -0,0 +1,4 @@
+flake8
+sphinx
+prefect
+trio