tree-sitter-grammars: refactor grammar sources

pkgs/development/python-modules/tree-sitter-grammars/default.nix

@@ -12,8 +12,20 @@
   grammarDrv,
 }:
 let
-  inherit (grammarDrv) version;
-
+  # Map nix style `0-unstable-YYYY-MM-DD` version identifiers to a PEP 440
+  # compatible form (`0+unstableYYYYMMDD`).
+  version = lib.pipe grammarDrv [
+    lib.getVersion
+    (lib.splitString "-")
+    (
+      parts:
+      let
+        version = lib.head parts;
+        metadata = lib.join "" (lib.tail parts);
+      in
+      if (metadata == "") then version else "${version}+${metadata}"
+    )
+  ];
   snakeCaseName = lib.replaceStrings [ "-" ] [ "_" ] name;
   drvPrefix = "python-${name}";
   # If the name of the grammar attribute differs from the grammar's symbol name,

pkgs/development/tools/parsing/tree-sitter/build-grammar.nix

@@ -0,0 +1,118 @@
+{
+  stdenv,
+  nodejs,
+  tree-sitter,
+  jq,
+  lib,
+}:
+
+{
+  language,
+  version,
+  src,
+  generate ? false,
+  ...
+}@args:
+
+stdenv.mkDerivation (
+  {
+    pname = "tree-sitter-${language}";
+
+    inherit version src;
+
+    nativeBuildInputs = [
+      jq
+    ]
+    ++ lib.optionals generate [
+      nodejs
+      tree-sitter
+    ];
+
+    CFLAGS = [
+      "-Isrc"
+      "-O2"
+    ];
+    CXXFLAGS = [
+      "-Isrc"
+      "-O2"
+    ];
+
+    stripDebugList = [ "parser" ];
+
+    # Tree-sitter grammar packages contain a `tree-sitter.json` file at their
+    # root. This provides package metadata that can be used to infer build
+    # details.
+    #
+    # See https://tree-sitter.github.io/tree-sitter/cli/init.html for spec.
+    configurePhase = ''
+      runHook preConfigure
+      if [[ -e tree-sitter.json ]]; then
+        # Check nix package version matches grammar source
+        NIX_VERSION=${lib.head (lib.splitString "+" version)}
+        SRC_VERSION=$(jq -r '.metadata.version' tree-sitter.json)
+        if [[ "$NIX_VERSION" != "$SRC_VERSION" ]]; then
+          nixErrorLog "grammar version ($NIX_VERSION) differs from source ($SRC_VERSION)"
+        fi
+
+        # Check language name matches source
+        GRAMMAR=$(jq -c 'first(.grammars[] | select(.name == env.language))' tree-sitter.json)
+        if [[ -z "$GRAMMAR" ]]; then
+          GRAMMAR=$(jq -c 'first(.grammars[]) // {}' tree-sitter.json)
+          NAME=$(jq -r '.name' <<< "$GRAMMAR")
+          SRC_LANGS=$(jq -r '[.grammars[].name] | join(", ")' tree-sitter.json)
+          nixErrorLog "grammar name ($language) not found in source grammars ($SRC_LANGS), continuing with $NAME"
+        fi
+
+        # Move to the appropriate working directory for build
+        cd -- $(jq -r '.path // "."' <<< $GRAMMAR)
+      else
+        # Older grammars may not contain this file. The tree-sitter CLI provides
+        # a warning rather than fail unless ABI > 14. Mirror that behaviour
+        # while older grammars age out.
+        nixWarnLog "grammar source is missing tree-sitter.json"
+      fi
+      runHook postConfigure
+    '';
+
+    # Optionally regenerate the parser source from the defined grammar. In most
+    # cases this should not be required as convention is to have this checked
+    # in to the source repo.
+    preBuild = lib.optionalString generate ''
+      tree-sitter generate
+    '';
+
+    # When both scanner.{c,cc} exist, we should not link both since they may be the same but in
+    # different languages. Just randomly prefer C++ if that happens.
+    buildPhase = ''
+      runHook preBuild
+      if [[ -e src/scanner.cc ]]; then
+        $CXX -fPIC -c src/scanner.cc -o scanner.o $CXXFLAGS
+      elif [[ -e src/scanner.c ]]; then
+        $CC -fPIC -c src/scanner.c -o scanner.o $CFLAGS
+      fi
+      $CC -fPIC -c src/parser.c -o parser.o $CFLAGS
+      rm -rf parser
+      $CXX -shared -o parser *.o
+      runHook postBuild
+    '';
+
+    installPhase = ''
+      runHook preInstall
+      mkdir $out
+      mv parser $out/
+      if [[ -d queries ]]; then
+        cp -r queries $out
+      fi
+      runHook postInstall
+    '';
+  }
+  # FIXME: neovim and nvim-treesitter currently rely on passing location rather
+  # than a src attribute with a correctly positioned root (e.g. for grammars in
+  # monorepos). Use this if present for now.
+  // (lib.optionalAttrs (args ? location && args.location != null) {
+    setSourceRoot = "sourceRoot=$(echo */${args.location})";
+  })
+  // removeAttrs args [
+    "generate"
+  ]
+)

pkgs/development/tools/parsing/tree-sitter/default.nix

@@ -1,8 +1,9 @@
 {
   lib,
   stdenv,
-  fetchgit,
   fetchFromGitHub,
+  fetchFromGitLab,
+  fetchFromSourcehut,
   nix-update-script,
   runCommand,
   which,
@@ -18,122 +19,55 @@
   enableShared ? !stdenv.hostPlatform.isStatic,
   enableStatic ? stdenv.hostPlatform.isStatic,
   webUISupport ? false,
-  extraGrammars ? { },
 
   # tests
   lunarvim,
 }:
 
 let
-  # to update:
-  # 1) change all these hashes
-  # 2) nix-build -A tree-sitter.updater.update-all-grammars
-  # 3) Set GITHUB_TOKEN env variable to avoid api rate limit (Use a Personal Access Token from https://github.com/settings/tokens It does not need any permissions)
-  # 4) run the ./result script that is output by that (it updates ./grammars)
-  version = "0.25.10";
-  hash = "sha256-aHszbvLCLqCwAS4F4UmM3wbSb81QuG9FM7BDHTu1ZvM=";
-
-  src = fetchFromGitHub {
-    owner = "tree-sitter";
-    repo = "tree-sitter";
-    tag = "v${version}";
-    inherit hash;
-    fetchSubmodules = true;
-  };
+  /**
+    Build a parser grammar and put the resulting shared object in `$out/parser`.
 
-  update-all-grammars = callPackage ./update.nix { };
+    # Example
 
-  fetchGrammar =
-    v:
-    fetchgit {
-      inherit (v)
-        url
-        rev
-        sha256
-        fetchSubmodules
-        ;
+    ```nix
+    tree-sitter-foo = pkgs.tree-sitter.buildGrammar {
+      language = "foo";
+      version = "0.42.0";
+      src = fetchFromGitHub { ... };
     };
+    ```
+  */
+  buildGrammar = callPackage ./build-grammar.nix { };
 
-  grammars = runCommand "grammars" { } (
-    ''
-      mkdir $out
-    ''
-    + (lib.concatStrings (
-      lib.mapAttrsToList (
-        name: grammar: "ln -s ${grammar.src or (fetchGrammar grammar)} $out/${name}\n"
-      ) (import ./grammars { inherit lib; })
-    ))
-  );
+  /**
+    Attrset of grammar sources.
 
-  buildGrammar = callPackage ./grammar.nix { };
+    Values are of the form { language, version, src }. These may be referenced
+    directly by other areas of the tree-sitter ecosystem in nixpkgs. Src will
+    contain all language bindings provided by the upstream grammar.
 
-  builtGrammars =
-    let
-      build =
-        name: grammar:
-        buildGrammar {
-          language = grammar.language or name;
-          inherit version;
-          src = grammar.src or (fetchGrammar grammar);
-          location = grammar.location or null;
-          generate = grammar.generate or false;
-        };
-      grammars' = import ./grammars { inherit lib; } // extraGrammars;
-      grammars =
-        grammars'
-        // {
-          tree-sitter-latex = grammars'.tree-sitter-latex // {
-            generate = true;
-          };
-        }
-        // {
-          tree-sitter-ocaml = grammars'.tree-sitter-ocaml // {
-            location = "grammars/ocaml";
-          };
-        }
-        // {
-          tree-sitter-ocaml-interface = grammars'.tree-sitter-ocaml // {
-            location = "grammars/interface";
-          };
-        }
-        // {
-          tree-sitter-org-nvim = grammars'.tree-sitter-org-nvim // {
-            language = "tree-sitter-org";
-          };
-        }
-        // {
-          tree-sitter-typescript = grammars'.tree-sitter-typescript // {
-            location = "typescript";
-          };
-        }
-        // {
-          tree-sitter-tsx = grammars'.tree-sitter-typescript // {
-            location = "tsx";
-          };
-        }
-        // {
-          tree-sitter-markdown = grammars'.tree-sitter-markdown // {
-            location = "tree-sitter-markdown";
-          };
-        }
-        // {
-          tree-sitter-markdown-inline = grammars'.tree-sitter-markdown // {
-            language = "tree-sitter-markdown_inline";
-            location = "tree-sitter-markdown-inline";
-          };
-        }
-        // {
-          tree-sitter-php = grammars'.tree-sitter-php // {
-            location = "php";
-          };
-        }
-        // {
-          tree-sitter-sql = grammars'.tree-sitter-sql // {
-            generate = true;
-          };
-        };
-    in
-    lib.mapAttrs build grammars;
+    Use pkgs.tree-sitter.grammars.<name> to access.
+  */
+  grammars = import ./grammars {
+    inherit
+      lib
+      nix-update-script
+      fetchFromGitHub
+      fetchFromGitLab
+      fetchFromSourcehut
+      ;
+  };
+
+  /**
+    Attrset of compiled grammars.
+
+    Compiled grammars contain the pre-built shared library and any queries from
+    the grammar source.
+
+    Use pkgs.tree-sitter-grammars.<name> to access.
+  */
+  builtGrammars = lib.mapAttrs (_: lib.makeOverridable buildGrammar) grammars;
 
   # Usage:
   # pkgs.tree-sitter.withPlugins (p: [ p.tree-sitter-c p.tree-sitter-java ... ])
@@ -167,9 +101,17 @@ let
   allGrammars = builtins.attrValues builtGrammars;
 
 in
-rustPlatform.buildRustPackage {
+rustPlatform.buildRustPackage (final: {
   pname = "tree-sitter";
-  inherit src version;
+  version = "0.25.10";
+
+  src = fetchFromGitHub {
+    owner = "tree-sitter";
+    repo = "tree-sitter";
+    tag = "v${final.version}";
+    hash = "sha256-aHszbvLCLqCwAS4F4UmM3wbSb81QuG9FM7BDHTu1ZvM=";
+    fetchSubmodules = true;
+  };
 
   cargoHash = "sha256-4R5Y9yancbg/w3PhACtsWq0+gieUd2j8YnmEj/5eqkg=";
 
@@ -229,9 +171,6 @@ rustPlatform.buildRustPackage {
   doCheck = false;
 
   passthru = {
-    updater = {
-      inherit update-all-grammars;
-    };
     inherit
       grammars
       buildGrammar
@@ -254,7 +193,7 @@ rustPlatform.buildRustPackage {
     homepage = "https://github.com/tree-sitter/tree-sitter";
     description = "Parser generator tool and an incremental parsing library";
     mainProgram = "tree-sitter";
-    changelog = "https://github.com/tree-sitter/tree-sitter/releases/tag/v${version}";
+    changelog = "https://github.com/tree-sitter/tree-sitter/releases/tag/v${final.version}";
     longDescription = ''
       Tree-sitter is a parser generator tool and an incremental parsing library.
       It can build a concrete syntax tree for a source file and efficiently update the syntax tree as the source file is edited.
@@ -273,4 +212,4 @@ rustPlatform.buildRustPackage {
       amaanq
     ];
   };
-}
+})