chore: don't require UV to download tokenizers

Justfile

@@ -9,15 +9,14 @@ release:
 # Copy binary command
 copy-binary:
     @if [ -f ./target/release/goosed ]; then \
-        echo "Copying goosed binary to ui/desktop/src/bin..."; \
-        cp ./target/release/goosed ./ui/desktop/src/bin/; \
+        echo "Copying goosed binary to ui/desktop/src/bin with permissions preserved..."; \
+        cp -p ./target/release/goosed ./ui/desktop/src/bin/; \
     else \
         echo "Release binary not found."; \
         exit 1; \
     fi
-
 # Run UI with latest
-run-ui:
+run-ui: download-tokenizers
     @just release
     @echo "Running UI..."
     cd ui/desktop && npm install && npm run start-gui
@@ -26,3 +25,8 @@ run-ui:
 run-server:
     @echo "Running server..."
     cargo run -p goose-server
+
+# Download tokenizer files if they don't exist
+download-tokenizers:
+    @echo "Checking and downloading tokenizer files..."
+    ./download_tokenizers.sh

README.md

@@ -18,13 +18,14 @@ This is the branch for goose 1.0 WIP: which is a port over from python to rust +
 ## Building
 
 ```sh
-uv run download_tokenizer_files.py
+./download_tokenizer_files.sh
 cargo build
 ```
 
 ## Running the CLI
 
 ### Configure
+
 ```
 goose configure
 ```
@@ -63,19 +64,19 @@ Usage: cargo run --bin goose -- run -i instructions.md
 
 ## GUI
 
-Goose has an electron based GUI which you can see in `ui/desktop`: 
+Goose has an electron based GUI which you can see in `ui/desktop`:
 
 <img width="732" alt="image" src="https://github.com/user-attachments/assets/17499ae5-7812-46f0-8aae-e4d3d9583c34">
 <img width="739" alt="image" src="https://github.com/user-attachments/assets/13ff2304-8468-47e0-9de8-89d23a62ec26">
 <img width="744" alt="image" src="https://github.com/user-attachments/assets/3a825455-6cd1-406b-a459-e2c73dba024b">
 
-
-
 ## Start sub system server
+
 ```sh
 cd crates/stub-system
 cargo run
 ```
+
 ## Troubleshooting
 
 #### Compiling `tokenizers` library

download_tokenizers.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+
+# Create base directory for tokenizer files
+BASE_DIR="tokenizer_files"
+mkdir -p "$BASE_DIR"
+
+# Function to download a tokenizer file
+download_tokenizer() {
+    local repo_id="$1"
+    local dir_name="${repo_id//\/--}"  # Replace / with -- for directory name
+    local download_dir="$BASE_DIR/${repo_id//\//--}"  # Replace / with -- for directory name, matching Python's replace("/", "--")
+    local file_url="https://huggingface.co/$repo_id/resolve/main/tokenizer.json"
+
+    mkdir -p "$download_dir"
+
+    # Only download if the file doesn't exist
+    if [ ! -f "$download_dir/tokenizer.json" ]; then
+        echo "Downloading tokenizer for $repo_id..."
+        curl -L "$file_url" -o "$download_dir/tokenizer.json"
+        if [ $? -eq 0 ]; then
+            echo "Downloaded $repo_id to $download_dir/tokenizer.json"
+        else
+            echo "Failed to download $repo_id tokenizer"
+            return 1
+        fi
+    else
+        echo "Tokenizer for $repo_id already exists, skipping..."
+    fi
+}
+
+# Download tokenizers for each model
+download_tokenizer "Xenova/gpt-4o"
+download_tokenizer "Xenova/claude-tokenizer"
+download_tokenizer "Qwen/Qwen2.5-Coder-32B-Instruct"