refactor: move download_tokenizers.sh to a cargo build script

.github/workflows/ci.yaml

@@ -45,13 +45,6 @@ jobs:
       # the login password doesn't matter, but the keyring must be unlocked for the tests to work
       run: gnome-keyring-daemon --components=secrets --daemonize --unlock <<< 'foobar'
 
-    - name: Install UV
-      run: |
-        curl -LsSf https://astral.sh/uv/install.sh | sh
-
-    - name: Run download_tokenizer_files.py
-      run: uv run download_tokenizer_files.py
-
     - name: Set up Rust
       uses: actions-rs/toolchain@v1
       with:

.github/workflows/desktop-app-release.yaml

@@ -44,14 +44,6 @@ jobs:
         restore-keys: |
           ${{ runner.os }}-cargo-build-
 
-    # Install UV and download tokenizer files
-    - name: Install UV
-      run: |
-        curl -LsSf https://astral.sh/uv/install.sh | sh
-
-    - name: Run download_tokenizer_files.py
-      run: uv run download_tokenizer_files.py
-
     # Build Rust Binary
     - name: Build Release Binary
       run: cargo build --release
@@ -68,7 +60,7 @@ jobs:
       working-directory: ui/desktop
       env:
         CERTIFICATE_OSX_APPLICATION: ${{ secrets.CERTIFICATE_OSX_APPLICATION }}
-        CERTIFICATE_PASSWORD: ${{ secrets.CERTIFICATE_PASSWORD }}      
+        CERTIFICATE_PASSWORD: ${{ secrets.CERTIFICATE_PASSWORD }}
 
     - name: Set up Node.js
       uses: actions/setup-node@v2
@@ -91,4 +83,4 @@ jobs:
       uses: actions/upload-artifact@v3
       with:
         name: Goose.zip
-        path: ui/desktop/out/Goose-darwin-arm64/Goose.zip          
+        path: ui/desktop/out/Goose-darwin-arm64/Goose.zip

.gitignore

@@ -23,3 +23,7 @@ Cargo.lock
 # UI
 ./ui/desktop/node_modules
 ./ui/desktop/out
+
+# Hermit
+/.hermit/
+/bin/

README.md

@@ -18,7 +18,6 @@ This is the branch for goose 1.0 WIP: which is a port over from python to rust +
 ## Building
 
 ```sh
-./download_tokenizer_files.sh
 cargo build
 ```
 

crates/goose/Cargo.toml

@@ -7,6 +7,10 @@ license.workspace = true
 repository.workspace = true
 description.workspace = true
 
+[build-dependencies]
+tokio = { version = "1.36", features = ["full"] }
+reqwest = { version = "0.11", features = ["json"] }
+
 [dependencies]
 mcp-core = { path = "../mcp-core" }
 dirs = "5.0.1"
@@ -44,7 +48,11 @@ libc = "=0.2.167"
 lazy_static = "1.5"
 kill_tree = "0.2.4"
 
-keyring = { version = "3.6.1", features = ["apple-native", "windows-native", "sync-secret-service"] }
+keyring = { version = "3.6.1", features = [
+    "apple-native",
+    "windows-native",
+    "sync-secret-service",
+] }
 shellexpand = "3.1.0"
 rust_decimal = "1.36.0"
 rust_decimal_macros = "1.36.0"

crates/goose/build.rs

@@ -0,0 +1,58 @@
+use std::error::Error;
+use std::fs;
+use std::path::Path;
+
+const BASE_DIR: &str = "../../tokenizer_files";
+const MODELS: &[&str] = &[
+    "Xenova/claude-tokenizer",
+    "Xenova/gemma-2-tokenizer",
+    "Xenova/gpt-4o",
+    "Qwen/Qwen2.5-Coder-32B-Instruct",
+];
+
+#[tokio::main]
+async fn main() -> Result<(), Box<dyn Error>> {
+    // Create base directory
+    fs::create_dir_all(BASE_DIR)?;
+    println!("cargo:rerun-if-changed=build.rs");
+    println!("cargo:rerun-if-changed={}", BASE_DIR);
+
+    for model in MODELS {
+        download_tokenizer(model).await?;
+    }
+
+    Ok(())
+}
+
+async fn download_tokenizer(repo_id: &str) -> Result<(), Box<dyn Error>> {
+    let dir_name = repo_id.replace('/', "--");
+    let download_dir = format!("{}/{}", BASE_DIR, dir_name);
+    let file_url = format!(
+        "https://huggingface.co/{}/resolve/main/tokenizer.json",
+        repo_id
+    );
+    let file_path = format!("{}/tokenizer.json", download_dir);
+
+    // Create directory if it doesn't exist
+    fs::create_dir_all(&download_dir)?;
+
+    // Check if file already exists
+    if Path::new(&file_path).exists() {
+        println!("Tokenizer for {} already exists, skipping...", repo_id);
+        return Ok(());
+    }
+
+    println!("Downloading tokenizer for {}...", repo_id);
+
+    // Download the file
+    let response = reqwest::get(&file_url).await?;
+    if !response.status().is_success() {
+        return Err(format!("Failed to download tokenizer for {}", repo_id).into());
+    }
+
+    let content = response.bytes().await?;
+    fs::write(&file_path, content)?;
+
+    println!("Downloaded {} to {}", repo_id, file_path);
+    Ok(())
+}

rust-toolchain.toml

@@ -0,0 +1,3 @@
+[toolchain]
+channel = "stable"
+profile = "default"