Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -1,33 +1,54 @@
[
"tree-sitter-bash",
"tree-sitter-beancount",
"tree-sitter-bibtex",
"tree-sitter-c",
"tree-sitter-c-sharp",
"tree-sitter-cpp",
"tree-sitter-clojure",
"tree-sitter-cmake",
"tree-sitter-comment",
"tree-sitter-css",
"tree-sitter-dart",
"tree-sitter-dockerfile",
"tree-sitter-elixir",
"tree-sitter-elm",
"tree-sitter-erlang",
"tree-sitter-fennel",
"tree-sitter-fortran",
"tree-sitter-gdscript",
"tree-sitter-glsl",
"tree-sitter-go",
"tree-sitter-haskell",
"tree-sitter-hcl",
"tree-sitter-heex",
"tree-sitter-html",
"tree-sitter-janet-simple",
"tree-sitter-java",
"tree-sitter-javascript",
"tree-sitter-jsdoc",
"tree-sitter-json",
"tree-sitter-jsonnet",
"tree-sitter-julia",
"tree-sitter-nix",
"tree-sitter-kotlin",
"tree-sitter-llvm",
"tree-sitter-lua",
"tree-sitter-markdown",
"tree-sitter-ocaml",
"tree-sitter-perl",
"tree-sitter-ocaml-interface",
"tree-sitter-pgn",
"tree-sitter-php",
"tree-sitter-prisma",
"tree-sitter-python",
"tree-sitter-r",
"tree-sitter-rst",
"tree-sitter-ruby",
"tree-sitter-rust",
"tree-sitter-scala",
"tree-sitter-scheme",
"tree-sitter-smithy",
"tree-sitter-solidity",
"tree-sitter-svelte",
"tree-sitter-toml",
"tree-sitter-twig",
"tree-sitter-typescript",
"tree-sitter-verilog",
"tree-sitter-yaml",
"tree-sitter-zig"
]
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,12 @@ def check_grammar_exists(nixpkgs: str, grammar: str) -> bool:
)


def check_grammar_broken(nixpkgs: str, grammar: str) -> bool:
return eval_expr(
nixpkgs, f'tree-sitter-grammars.{fmt_grammar(grammar)}.meta.broken'
)


def build_attr(nixpkgs, attr: str) -> str:
return (
subprocess.run(
Expand All @@ -55,7 +61,7 @@ def build_attr(nixpkgs, attr: str) -> str:

if __name__ == "__main__":
cwd = dirname(abspath(__file__))
nixpkgs = abspath(join(cwd, "../../../../../.."))
nixpkgs = abspath(join(cwd, "../../../../../../.."))

src_dir = build_attr(nixpkgs, "emacs.pkgs.tree-sitter-langs.src")

Expand All @@ -65,7 +71,12 @@ def build_attr(nixpkgs, attr: str) -> str:
for g in grammars:
exists = check_grammar_exists(nixpkgs, g)
if exists:
existing.append(fmt_grammar(g))
broken = check_grammar_broken(nixpkgs, g)
if not broken:
existing.append(fmt_grammar(g))
else:
sys.stderr.write("Grammar is broken: " + fmt_grammar(g) + "\n")
sys.stderr.flush()
else:
sys.stderr.write("Missing grammar: " + fmt_grammar(g) + "\n")
sys.stderr.flush()
Comment on lines 71 to 82
Copy link
Member

@amadaluzia amadaluzia Jun 19, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I propose a rewrite:

for g in grammars:
    grammar = fmt_grammar(g)
    if not check_grammar_exists(nixpkgs, g):
        print(f"Missing grammar: {grammar}", file=sys.stderr)
        continue
    
    if check_grammar_broken(nixpkgs, g):
        print(f"Broken grammar: {grammar}", file=sys.stderr)
        continue
    
    existing.append(grammar)

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hi, thanks.

This script is only for Emacs plugins, so I think that would add the grammars tagged as broken back into the emacs ecosystem.

Which is something that I might have forgotten to handle without your feedback.

Expand Down
2 changes: 2 additions & 0 deletions pkgs/development/tools/parsing/tree-sitter/default.nix
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,7 @@ let
src = grammar.src or (fetchGrammar grammar);
location = grammar.location or null;
generate = grammar.generate or false;
broken = grammar.broken or false;
};
grammars' = import ./grammars { inherit lib; } // extraGrammars;
grammars =
Expand Down Expand Up @@ -120,6 +121,7 @@ let
tree-sitter-markdown-inline = grammars'.tree-sitter-markdown // {
language = "tree-sitter-markdown_inline";
location = "tree-sitter-markdown-inline";
broken = true;
};
}
// {
Expand Down
20 changes: 20 additions & 0 deletions pkgs/development/tools/parsing/tree-sitter/grammar.nix
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
stdenv,
nodejs,
tree-sitter,
writableTmpDirAsHomeHook,
lib,
}:

Expand All @@ -14,6 +15,7 @@
src,
location ? null,
generate ? false,
broken ? false,
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

seems like it is brokenTests rather than the whole grammar ? might be best to add overrides for those grammars in an overrides.nix rather than automate it. Among the advantages is that we dont introduce a new setting broken, it would be doCheck = false like for every other package, and it encodes it just in overrides.nix rather than the json + nix.

...
}@args:

Expand Down Expand Up @@ -47,6 +49,21 @@ stdenv.mkDerivation (
tree-sitter generate
'';

doCheck = !broken;

nativeCheckInputs = [
# tree-sitter needs a writable home folder for the checkPhase
writableTmpDirAsHomeHook
];

checkPhase = ''
runHook preCheck

${lib.getExe tree-sitter} test

runHook postCheck
'';

# When both scanner.{c,cc} exist, we should not link both since they may be the same but in
# different languages. Just randomly prefer C++ if that happens.
buildPhase = ''
Expand All @@ -71,6 +88,9 @@ stdenv.mkDerivation (
fi
runHook postInstall
'';
meta = {
inherit broken;
};
}
// removeAttrs args [
"language"
Expand Down
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
{
"url": "https://github.com/uyha/tree-sitter-cmake",
"rev": "e409ae33f00e04cde30f2bcffb979caf1a33562a",
"date": "2024-10-14T03:37:52+03:00",
"path": "/nix/store/hgwv6jwacl1md38yccvvvhigyk6h21s2-tree-sitter-cmake",
"sha256": "1v2kkxf9w0y9mwwgimgyvn1s9cmmp6jg71ckr001w1w8szyqqf7q",
"hash": "sha256-+DiM/deIBx4AyJOF86S5tbKkg93+1fg4r8kDnlyfU+w=",
"rev": "fe48221d4d9842d916d66b5e71ab3c6307ec28b3",
"date": "2025-03-12T21:26:55Z",
"path": "/nix/store/rrbi1s3fyi02shvnqr2idrn9j36kby3c-tree-sitter-cmake",
"sha256": "0g7i4zp3900hy887ljyjqcxln96fygrxpj9czm89sl54x19q8klm",
"hash": "sha256-lU6EU+ikUJ1Q/SzJ2/PzziRLO8PSS3oQ8hCANO4n8Tw=",
"fetchLFS": false,
"fetchSubmodules": false,
"deepClone": false,
Expand Down
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
{
"url": "https://github.com/thehamsta/tree-sitter-commonlisp",
"rev": "25856774aaab983c573bb2f9cc1ebbc97941f7b9",
"date": "2024-04-06T22:22:41+02:00",
"path": "/nix/store/mlji0h6k2x17jni9q3y571dmk5k4xi85-tree-sitter-commonlisp",
"sha256": "12fh2sinasnfp6rfq5d9qq24nfg55bclc5rp8mnw3a2ccyc5icis",
"hash": "sha256-OrJYmGdMqMFtRTcXRtkq5TlLBMapFeyyuc5qZaMW0Ik=",
"rev": "9db594efb43574c5fe4a1f880568b1f0d0ee6057",
"date": "2025-03-16T16:36:54+01:00",
"path": "/nix/store/96phqhn770ds8i0j0l88phldl7d03g72-tree-sitter-commonlisp",
"sha256": "0xg3ay8l62h7s35abkxi4gjfvndzdvvrpgh1z980q1ib5935sxf0",
"hash": "sha256-wHVdRiorBgxQ+gG+m/duv9nt5COxz6XK0AcKQ5FX43U=",
"fetchLFS": false,
"fetchSubmodules": false,
"deepClone": false,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,5 +8,6 @@
"fetchLFS": false,
"fetchSubmodules": false,
"deepClone": false,
"leaveDotGit": false
"leaveDotGit": false,
"broken": true
}
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
{
"url": "https://github.com/thehamsta/tree-sitter-cuda",
"rev": "9cdfe2d453e5f60fd818935a5895f223e2e6feed",
"date": "2025-01-25T13:03:37+01:00",
"path": "/nix/store/xz5zddyszj1sgr3lp17f021n2x13dmj7-tree-sitter-cuda",
"sha256": "0krk5f0g87cp1mkdn2x6sadldqnvhxxy8zsvc8z2bxc50vb4lw9m",
"hash": "sha256-NXFK1gaF9SU+Ylt/5HuH2+JGm9KmC9tmDZcd9IArM08=",
"url": "https://github.com/tree-sitter-grammars/tree-sitter-cuda",
"rev": "014628ae8d2df391b88ddb9fa0260fd97f770829",
"date": "2025-03-16T17:20:16+01:00",
"path": "/nix/store/w0cl4ga4bis992vyfmj8l0pq385z0cka-tree-sitter-cuda",
"sha256": "1qyh00rapch29czvnqs3364bx0bi4gyapfxb0v8m4r2m8kybnlff",
"hash": "sha256-zlG7/ERVZFLRBqu7q/wjcYG+iBlDY7s/SwKyqzIA0OM=",
"fetchLFS": false,
"fetchSubmodules": false,
"deepClone": false,
Expand Down
Original file line number Diff line number Diff line change
@@ -1,12 +1,13 @@
{
"url": "https://github.com/usernobody14/tree-sitter-dart",
"rev": "e81af6ab94a728ed99c30083be72d88e6d56cf9e",
"date": "2024-11-16T21:53:35-07:00",
"path": "/nix/store/jb7wiv90s78p45pj9mqvmbgk06xhjlj5-tree-sitter-dart",
"sha256": "0zl46vkm4p1jmivmnpyyzc58fwhx5frfgi0rfxna43h0qxdv62wy",
"hash": "sha256-nguzW8cADqJsdxnE57IrHXKHCvveX1t3rDJcUuc2hH4=",
"rev": "80e23c07b64494f7e21090bb3450223ef0b192f4",
"date": "2025-02-28T15:18:07-07:00",
"path": "/nix/store/7pfv5380h2fwv0rg0ckdjg8h71vzq71f-tree-sitter-dart",
"sha256": "00wzdgmi6kph4lk988brpy03j43a8vvfcjx9plnnnk07a14l3hbc",
"hash": "sha256-bMFBSVAHTGstvalL5vZGahA5gL95IZQmJfBOE+trnwM=",
"fetchLFS": false,
"fetchSubmodules": false,
"deepClone": false,
"leaveDotGit": false
"leaveDotGit": false,
"broken": true
}
Original file line number Diff line number Diff line change
Expand Up @@ -8,5 +8,6 @@
"fetchLFS": false,
"fetchSubmodules": false,
"deepClone": false,
"leaveDotGit": false
"leaveDotGit": false,
"broken": true
}
Original file line number Diff line number Diff line change
Expand Up @@ -8,5 +8,6 @@
"fetchLFS": false,
"fetchSubmodules": false,
"deepClone": false,
"leaveDotGit": false
"leaveDotGit": false,
"broken": true
}
Original file line number Diff line number Diff line change
Expand Up @@ -8,5 +8,6 @@
"fetchLFS": false,
"fetchSubmodules": false,
"deepClone": false,
"leaveDotGit": false
"leaveDotGit": false,
"broken": true
}
Original file line number Diff line number Diff line change
Expand Up @@ -8,5 +8,6 @@
"fetchLFS": false,
"fetchSubmodules": false,
"deepClone": false,
"leaveDotGit": false
"leaveDotGit": false,
"broken": true
}
Original file line number Diff line number Diff line change
Expand Up @@ -8,5 +8,6 @@
"fetchLFS": false,
"fetchSubmodules": false,
"deepClone": false,
"leaveDotGit": false
"leaveDotGit": false,
"broken": true
}
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
{
"url": "https://github.com/stadelmanma/tree-sitter-fortran",
"rev": "022b032d31299c5d8336cdfd0ece97de20a609c0",
"date": "2025-01-23T13:28:14-05:00",
"path": "/nix/store/vncpfx5db12ish9rzf26phj25373nqs4-tree-sitter-fortran",
"sha256": "1mncdji60qa9r8jbiywmcid714ylc3gniq25l8mxj1p4zq95nd29",
"hash": "sha256-STRbEv7kBtkrokXgaN9g1JNwWmSV+7gkyklhYKJszNY=",
"rev": "64e11001d7ef3e8ac18e55a3a2d811fe36430923",
"date": "2025-03-17T15:53:16-04:00",
"path": "/nix/store/lb6lr9gqrkfxq4q53z4xnihv927lbwfj-tree-sitter-fortran",
"sha256": "0fxlg3ckah5iv9gyc74d76xkr9qdk7xrs8c650gcicsvnmy9qpza",
"hash": "sha256-6l+cfLVbs8geKIYhnfuZDac8uzmNHOZf2rFANdl4tDs=",
"fetchLFS": false,
"fetchSubmodules": false,
"deepClone": false,
Expand Down
Original file line number Diff line number Diff line change
@@ -1,12 +1,13 @@
{
"url": "https://github.com/gleam-lang/tree-sitter-gleam",
"rev": "af6043419f5aa0f8b6c2a26db0187aefa46c7f5f",
"date": "2025-02-07T14:27:03Z",
"path": "/nix/store/k54xiiqza536lzgvdw8yizsp6czdalln-tree-sitter-gleam",
"sha256": "0b7k2ah7a29simvvzm3f01vna5lzvv1mi7idvzpc056wjbgnvrg8",
"hash": "sha256-6OVt35LcFMDu3y2eWMPenxZldwBu1L93jToJdaAS8yw=",
"rev": "99ec4101504452c488b7c835fb65cfef75b090b7",
"date": "2025-03-20T12:15:08Z",
"path": "/nix/store/fyy76dad39xxxlg1mh8f7w8ny2yck8pa-tree-sitter-gleam",
"sha256": "11vvlpwhxr9wa0flq9k9xl4cr97jnvi8r2zaafy86qb71n7rqiql",
"hash": "sha256-FEecjw1nYYO8U+qLjOK28qTMCO1pJkwdUDzlDvmle4c=",
"fetchLFS": false,
"fetchSubmodules": false,
"deepClone": false,
"leaveDotGit": false
"leaveDotGit": false,
"broken": true
}
Original file line number Diff line number Diff line change
Expand Up @@ -8,5 +8,6 @@
"fetchLFS": false,
"fetchSubmodules": false,
"deepClone": false,
"leaveDotGit": false
"leaveDotGit": false,
"broken": true
}
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
{
"url": "https://github.com/thehamsta/tree-sitter-glsl",
"rev": "66aec57f7119c7e8e40665b723cd7af5594f15ee",
"date": "2024-09-12T12:52:04+02:00",
"path": "/nix/store/xzxngsr3nhs1586c47iwdx9k20yaansc-tree-sitter-glsl",
"sha256": "0gp3bn31xz5rq52amx059r9sllk3749f1ajmbs1fkjb833f2kvqh",
"hash": "sha256-EO8p3BhoyemCXlWq4BI5Y1KqU04F9KpEwbn8HoZd4z4=",
"rev": "e47b8b62b59d0e3529f1c31b03e025d6bd475044",
"date": "2025-03-16T18:14:20+01:00",
"path": "/nix/store/87vf9kgi9yzzksj0s7h707ky79kckf5f-tree-sitter-glsl",
"sha256": "0d0ymklms4a91b310f0vwl80yy50sji4qq9sdgly5qh42kyjnijb",
"hash": "sha256-S0Yr/RQE4uLpazphTKLUoHgPEOUbOBDGCkkRXemsHjQ=",
"fetchLFS": false,
"fetchSubmodules": false,
"deepClone": false,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,5 +8,6 @@
"fetchLFS": false,
"fetchSubmodules": false,
"deepClone": false,
"leaveDotGit": false
"leaveDotGit": false,
"broken": true
}
Original file line number Diff line number Diff line change
Expand Up @@ -8,5 +8,6 @@
"fetchLFS": false,
"fetchSubmodules": false,
"deepClone": false,
"leaveDotGit": false
"leaveDotGit": false,
"broken": true
}
Original file line number Diff line number Diff line change
Expand Up @@ -8,5 +8,6 @@
"fetchLFS": false,
"fetchSubmodules": false,
"deepClone": false,
"leaveDotGit": false
"leaveDotGit": false,
"broken": true
}
Original file line number Diff line number Diff line change
Expand Up @@ -8,5 +8,6 @@
"fetchLFS": false,
"fetchSubmodules": false,
"deepClone": false,
"leaveDotGit": false
"leaveDotGit": false,
"broken": true
}
Original file line number Diff line number Diff line change
Expand Up @@ -8,5 +8,6 @@
"fetchLFS": false,
"fetchSubmodules": false,
"deepClone": false,
"leaveDotGit": false
"leaveDotGit": false,
"broken": true
}
Original file line number Diff line number Diff line change
@@ -1,12 +1,13 @@
{
"url": "https://github.com/MunifTanjim/tree-sitter-lua",
"rev": "99fc677e6971c425e8d407f59c77ab897e585c92",
"date": "2024-09-09T11:10:03-04:00",
"path": "/nix/store/iiih0sfdls1h8q7ca12y0rhc7g5jl76w-tree-sitter-lua",
"sha256": "0wrbxmb6j8xyckf5jw14jf97cb9fn7yhalap6xxgsag84ypfsqj3",
"hash": "sha256-Q2LtrifoKf16N1dRBf2xLi12kpMkcFncZL4jaVbtK3M=",
"rev": "534c461d2b75b0887ec968ef9635f4460b0878b7",
"date": "2025-03-21T16:15:02+01:00",
"path": "/nix/store/vcmm6sdh9lfw4bvwx78yx2llzwq40jyb-tree-sitter-lua",
"sha256": "041anx0qirvd4il87whpic8nfdc1nk3kimxdb99m25bfdzm9rn0r",
"hash": "sha256-Gdic6m9uFVFTWq3XOMe0gTVnEYsX8oNoJG3niEG3KhA=",
"fetchLFS": false,
"fetchSubmodules": false,
"deepClone": false,
"leaveDotGit": false
"leaveDotGit": false,
"broken": true
}
Loading
Loading