diff --git a/Justfile b/Justfile
index 53951a86cd31..fb17bda34146 100644
--- a/Justfile
+++ b/Justfile
@@ -9,15 +9,14 @@ release:
# Copy binary command
copy-binary:
@if [ -f ./target/release/goosed ]; then \
- echo "Copying goosed binary to ui/desktop/src/bin..."; \
- cp ./target/release/goosed ./ui/desktop/src/bin/; \
+ echo "Copying goosed binary to ui/desktop/src/bin with permissions preserved..."; \
+ cp -p ./target/release/goosed ./ui/desktop/src/bin/; \
else \
echo "Release binary not found."; \
exit 1; \
fi
-
# Run UI with latest
-run-ui:
+run-ui: download-tokenizers
@just release
@echo "Running UI..."
cd ui/desktop && npm install && npm run start-gui
@@ -26,3 +25,8 @@ run-ui:
run-server:
@echo "Running server..."
cargo run -p goose-server
+
+# Download tokenizer files if they don't exist
+download-tokenizers:
+ @echo "Checking and downloading tokenizer files..."
+ ./download_tokenizers.sh
\ No newline at end of file
diff --git a/README.md b/README.md
index 40afbfade440..af185d8b9c25 100644
--- a/README.md
+++ b/README.md
@@ -18,13 +18,14 @@ This is the branch for goose 1.0 WIP: which is a port over from python to rust +
## Building
```sh
-uv run download_tokenizer_files.py
+./download_tokenizer_files.sh
cargo build
```
## Running the CLI
### Configure
+
```
goose configure
```
@@ -63,19 +64,19 @@ Usage: cargo run --bin goose -- run -i instructions.md
## GUI
-Goose has an electron based GUI which you can see in `ui/desktop`:
+Goose has an electron based GUI which you can see in `ui/desktop`:
-
-
## Start sub system server
+
```sh
cd crates/stub-system
cargo run
```
+
## Troubleshooting
#### Compiling `tokenizers` library
diff --git a/download_tokenizers.sh b/download_tokenizers.sh
new file mode 100755
index 000000000000..6658194d4eda
--- /dev/null
+++ b/download_tokenizers.sh
@@ -0,0 +1,34 @@
+#!/bin/bash
+
+# Create base directory for tokenizer files
+BASE_DIR="tokenizer_files"
+mkdir -p "$BASE_DIR"
+
+# Function to download a tokenizer file
+download_tokenizer() {
+ local repo_id="$1"
+ local dir_name="${repo_id//\/--}" # Replace / with -- for directory name
+ local download_dir="$BASE_DIR/${repo_id//\//--}" # Replace / with -- for directory name, matching Python's replace("/", "--")
+ local file_url="https://huggingface.co/$repo_id/resolve/main/tokenizer.json"
+
+ mkdir -p "$download_dir"
+
+ # Only download if the file doesn't exist
+ if [ ! -f "$download_dir/tokenizer.json" ]; then
+ echo "Downloading tokenizer for $repo_id..."
+ curl -L "$file_url" -o "$download_dir/tokenizer.json"
+ if [ $? -eq 0 ]; then
+ echo "Downloaded $repo_id to $download_dir/tokenizer.json"
+ else
+ echo "Failed to download $repo_id tokenizer"
+ return 1
+ fi
+ else
+ echo "Tokenizer for $repo_id already exists, skipping..."
+ fi
+}
+
+# Download tokenizers for each model
+download_tokenizer "Xenova/gpt-4o"
+download_tokenizer "Xenova/claude-tokenizer"
+download_tokenizer "Qwen/Qwen2.5-Coder-32B-Instruct"
\ No newline at end of file