Skip to content
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 50 additions & 2 deletions e2e/helpers/scripts/git_http_backend_server.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,9 @@
from pathlib import Path

class GitHTTPHandler(http.server.BaseHTTPRequestHandler):
def __init__(self, *args, repo_dir=None, **kwargs):
def __init__(self, *args, repo_dir=None, upload_pack_count_file=None, **kwargs):
self.repo_dir = repo_dir
self.upload_pack_count_file = upload_pack_count_file
super().__init__(*args, **kwargs)

def do_GET(self):
Expand Down Expand Up @@ -47,6 +48,14 @@ def handle_git_request(self):
content_length = int(self.headers.get('Content-Length', 0))
request_body = self.rfile.read(content_length) if content_length > 0 else b''

if (
self.command == 'POST'
and path_info.endswith('/git-upload-pack')
and self.upload_pack_count_file is not None
):
with self.upload_pack_count_file.open('a') as count_file:
count_file.write('1\n')

# Set content type if provided
if 'Content-Type' in self.headers:
env['CONTENT_TYPE'] = self.headers['Content-Type']
Expand Down Expand Up @@ -119,6 +128,33 @@ def create_test_repo(repo_path):
remote_task_file.write_text('#!/usr/bin/env bash\necho "remote task executed"\n')
remote_task_file.chmod(0o755)

snapshot_root = Path(repo_path) / 'xtasks' / 'snapshot'
snapshot_parent = snapshot_root / 'parent' / 'snapshot_parent'
snapshot_parent.parent.mkdir(parents=True)
snapshot_parent.write_text(
'#!/usr/bin/env bash\n'
'#MISE depends=["snapshot_child"]\n'
'echo "snapshot parent checkout: ${0%%/xtasks/*}"\n'
)
snapshot_parent.chmod(0o755)

snapshot_child = snapshot_root / 'child' / 'snapshot_child'
snapshot_child.parent.mkdir(parents=True)
snapshot_child.write_text(
'#!/usr/bin/env bash\n'
'echo "snapshot child checkout: ${0%%/xtasks/*}"\n'
)
snapshot_child.chmod(0o755)

snapshot_failure = snapshot_root / 'failure' / 'snapshot_failure'
snapshot_failure.parent.mkdir(parents=True)
snapshot_failure.write_text(
'#!/usr/bin/env bash\n'
'echo "snapshot failure"\n'
'exit 1\n'
)
snapshot_failure.chmod(0o755)

# A toml task file colocated with the executable scripts. Keys are task
# names; values are the run command (or a table). Used by tests covering
# remote toml task includes.
Expand Down Expand Up @@ -165,13 +201,23 @@ def start_server(port=0):
# Create temp directory
temp_dir = Path(tempfile.mkdtemp(prefix='mise_git_http_'))
repo_path = temp_dir / 'repo'
upload_pack_count_path = os.environ.get('MISE_GIT_HTTP_UPLOAD_PACK_COUNT_FILE')
upload_pack_count_file = Path(upload_pack_count_path) if upload_pack_count_path else None
if upload_pack_count_file is not None:
upload_pack_count_file.parent.mkdir(parents=True, exist_ok=True)
upload_pack_count_file.write_text('')

print(f"Creating test repository at {repo_path}")
create_test_repo(str(repo_path))

# Create handler with repo directory
def handler(*args, **kwargs):
return GitHTTPHandler(*args, repo_dir=temp_dir, **kwargs)
return GitHTTPHandler(
*args,
repo_dir=temp_dir,
upload_pack_count_file=upload_pack_count_file,
**kwargs,
)

# Let the OS assign an available port if port=0
# This avoids race conditions between finding and binding
Expand Down Expand Up @@ -206,6 +252,8 @@ def handler(*args, **kwargs):
port_file.unlink(missing_ok=True)
info_file.unlink(missing_ok=True)
ready_file.unlink(missing_ok=True)
if upload_pack_count_file is not None:
upload_pack_count_file.unlink(missing_ok=True)

if __name__ == '__main__':
port = int(sys.argv[1]) if len(sys.argv) > 1 else 0
Expand Down
94 changes: 91 additions & 3 deletions e2e/tasks/test_task_remote_git_includes
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,16 @@
PORT_FILE="$TMPDIR/mise_git_http_port"
READY_FILE="$TMPDIR/mise_git_http_ready"
INFO_FILE="$TMPDIR/mise_git_http_info"
UPLOAD_PACK_COUNT_FILE="$TMPDIR/mise_git_http_upload_pack_count"

# Clean up any previous server files
rm -f "$PORT_FILE" "$READY_FILE" "$INFO_FILE"
rm -f "$PORT_FILE" "$READY_FILE" "$INFO_FILE" "$UPLOAD_PACK_COUNT_FILE"
Comment thread
risu729 marked this conversation as resolved.
Resolved

# Start local git HTTP server with OS-assigned port to avoid race conditions
MISE_GIT_HTTP_PORT_FILE="$PORT_FILE" \
MISE_GIT_HTTP_READY_FILE="$READY_FILE" \
MISE_GIT_HTTP_INFO_FILE="$INFO_FILE" \
MISE_GIT_HTTP_UPLOAD_PACK_COUNT_FILE="$UPLOAD_PACK_COUNT_FILE" \
python3 "${TEST_ROOT}/helpers/scripts/git_http_backend_server.py" 0 &
SERVER_PID=$!

Expand Down Expand Up @@ -50,7 +52,7 @@ git init
# Ensure cleanup on exit
cleanup() {
kill "$SERVER_PID" 2>/dev/null || true
rm -f "$PORT_FILE" "$READY_FILE" "$INFO_FILE"
rm -f "$PORT_FILE" "$READY_FILE" "$INFO_FILE" "$UPLOAD_PACK_COUNT_FILE"
}
trap cleanup EXIT

Expand Down Expand Up @@ -103,7 +105,10 @@ includes = [
]
EOF

assert_succeed "MISE_TASK_REMOTE_NO_CACHE=true mise run remote-task" # Remote task should be redownloaded
INCLUDE_ARTIFACT_TMPDIR="$TMPDIR/remote-git-include-artifacts"
mkdir -p "$INCLUDE_ARTIFACT_TMPDIR"
assert_succeed "TMPDIR=$INCLUDE_ARTIFACT_TMPDIR MISE_TASK_REMOTE_NO_CACHE=true mise run remote-task" # Remote task should be redownloaded
assert_directory_empty "$INCLUDE_ARTIFACT_TMPDIR"
assert_directory_not_exists "${REMOTE_TASKS_DIR}"

assert_succeed "mise run remote-task" # Cache should be used
Expand Down Expand Up @@ -175,4 +180,87 @@ EOF
assert_contains "mise tasks" "standalone_task"
assert_contains "mise run standalone_task" "standalone_task executed"

#################################################################################
# Test no-cache Git task snapshots
#################################################################################

cat <<EOF >mise.toml
[tasks.remote_git_a]
file = "git::${LOCAL_GIT_URL}//xtasks/snapshot/child/snapshot_child"

[tasks.remote_git_b]
file = "git::${LOCAL_GIT_URL}//xtasks/snapshot/child/snapshot_child"
EOF

ARTIFACT_TMPDIR="$TMPDIR/remote-git-artifacts"
mkdir -p "$ARTIFACT_TMPDIR"
output=$(quiet_assert_succeed "TMPDIR=$ARTIFACT_TMPDIR MISE_GIX=0 MISE_TASK_REMOTE_NO_CACHE=true mise run remote_git_a ::: remote_git_b")
path_count=$(grep -o 'snapshot child checkout: .*' <<<"$output" | sort -u | wc -l)
[[ $path_count -eq 2 ]] || fail "no-cache Git tasks did not retain two distinct snapshots"
assert_directory_empty "$ARTIFACT_TMPDIR"

#################################################################################
# Test one coherent no-cache Git include snapshot per command
#################################################################################

cat <<EOF >mise.toml
[task_config]
includes = [
"git::${LOCAL_GIT_URL}//xtasks/snapshot/parent?ref=v2025.1.17",
"git::${LOCAL_GIT_URL}//xtasks/snapshot/child?ref=v2025.1.17",
"git::${LOCAL_GIT_URL}//xtasks/snapshot/failure?ref=v2025.1.17"
]
EOF

upload_packs_before=$(wc -l <"$UPLOAD_PACK_COUNT_FILE")
output=$(quiet_assert_succeed "TMPDIR=$ARTIFACT_TMPDIR MISE_TASK_REMOTE_NO_CACHE=true mise run snapshot_parent")
upload_packs_after=$(wc -l <"$UPLOAD_PACK_COUNT_FILE")
[[ $((upload_packs_after - upload_packs_before)) -eq 1 ]] ||
fail "one command cloned the same Git repository/ref more than once"
assert_contains_text "$output" "snapshot parent checkout:"
assert_contains_text "$output" "snapshot child checkout:"
snapshot_checkout_count=$(
grep -Eo 'snapshot (parent|child) checkout: .*' <<<"$output" |
sed 's/snapshot [^ ]* checkout: //' |
sort -u |
wc -l
)
[[ $snapshot_checkout_count -eq 1 ]] || fail "one command mixed multiple Git include snapshots"
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
assert_directory_empty "$ARTIFACT_TMPDIR"

#################################################################################
# Test cleanup after an ordinary task-loading error
#################################################################################

cat <<EOF >mise.toml
[task_config]
includes = [
"git::${LOCAL_GIT_URL}//xtasks/lint?ref=v2025.1.17",
"git::${LOCAL_GIT_URL}//xtasks/missing?ref=v2025.1.17"
]
EOF

assert_fail "TMPDIR=$ARTIFACT_TMPDIR MISE_TASK_REMOTE_NO_CACHE=true mise tasks"
assert_directory_empty "$ARTIFACT_TMPDIR"

#################################################################################
# Test cleanup after a requested nonzero exit
#################################################################################

cat <<EOF >mise.toml
[task_config]
includes = [
"git::${LOCAL_GIT_URL}//xtasks/snapshot/failure?ref=v2025.1.17"
]
EOF

assert_fail "TMPDIR=$ARTIFACT_TMPDIR MISE_TASK_REMOTE_NO_CACHE=true mise run snapshot_failure"
# A failed run may leave its unrelated tempfile::tempdir (`.tmp*`) for the e2e
# harness; the command-scoped remote checkout itself must be gone.
for artifact in "$ARTIFACT_TMPDIR"/* "$ARTIFACT_TMPDIR"/.[!.]* "$ARTIFACT_TMPDIR"/..?*; do
[[ -e $artifact || -L $artifact ]] || continue
artifact_name=${artifact##*/}
[[ $artifact_name == .tmp* ]] || fail "unexpected remote task artifact remains: $artifact"
done

echo "All tests passed!"
93 changes: 83 additions & 10 deletions src/config/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,9 @@ use crate::config::env_directive::{EnvResolveOptions, EnvResults, ToolsFilter};
use crate::config::tracking::Tracker;
use crate::env::{MISE_DEFAULT_CONFIG_FILENAME, MISE_DEFAULT_TOOL_VERSIONS_FILENAME};
use crate::file::display_path;
use crate::remote_source::RemoteSource;
use crate::shorthands::{Shorthands, get_shorthands};
use crate::task::task_file_providers::TaskFileProvidersBuilder;
use crate::task::task_file_providers::{TaskFileArtifact, TaskFileProvidersBuilder};
use crate::task::task_sources::TaskOutputs;
use crate::task::{
RunEntry, Task, TaskCacheConfig, TaskRustCacheConfig, TaskTemplate, monorepo_scope,
Expand Down Expand Up @@ -62,6 +63,41 @@ use crate::wildcard::Wildcard;
type AliasMap = IndexMap<String, Alias>;
pub(crate) type ConfigMap = IndexMap<PathBuf, Arc<dyn ConfigFile>>;
pub type EnvWithSources = IndexMap<String, (String, PathBuf)>;
type RemoteTaskIncludeKey = (String, Option<String>);
type RemoteTaskIncludeArtifacts = DashMap<RemoteTaskIncludeKey, Arc<OnceCell<TaskFileArtifact>>>;
static REMOTE_TASK_INCLUDE_ARTIFACTS: Lazy<RemoteTaskIncludeArtifacts> = Lazy::new(DashMap::new);

#[cfg(test)]
pub(crate) fn clear_remote_task_include_artifacts() {
drop(take_remote_task_include_artifacts());
}

pub(crate) fn take_remote_task_include_artifacts() -> Vec<Arc<OnceCell<TaskFileArtifact>>> {
let keys = REMOTE_TASK_INCLUDE_ARTIFACTS
.iter()
.map(|entry| entry.key().clone())
.collect_vec();
keys.into_iter()
.filter_map(|key| {
REMOTE_TASK_INCLUDE_ARTIFACTS
.remove(&key)
.map(|(_, artifact)| artifact)
})
.collect()
}

#[cfg(test)]
pub(crate) fn insert_remote_task_include_artifact_for_test() {
REMOTE_TASK_INCLUDE_ARTIFACTS.insert(
("https://example.test/repo.git".into(), None),
Arc::new(OnceCell::new()),
);
}

#[cfg(test)]
pub(crate) fn remote_task_include_artifact_count_for_test() -> usize {
REMOTE_TASK_INCLUDE_ARTIFACTS.len()
}

pub(crate) struct MonorepoUnion {
pub config_files: ConfigMap,
Expand Down Expand Up @@ -4434,16 +4470,49 @@ fn is_mise_config_file_in_task_include(root: &Path, path: &Path) -> bool {
})
}

async fn resolve_git_url_to_path(git_url: &str) -> Result<PathBuf> {
async fn resolve_git_url_to_path(git_url: &str) -> Result<TaskFileArtifact> {
let no_cache = Settings::get().task.remote_no_cache.unwrap_or(false);
let task_file_providers = TaskFileProvidersBuilder::new()
.with_cache(!no_cache)
.build();
if !no_cache {
let task_file_providers = TaskFileProvidersBuilder::new().with_cache(true).build();
return match task_file_providers.get_provider(git_url) {
Some(provider) => provider.get_local_artifact(git_url).await,
None => bail!("No provider found for git URL: {}", git_url),
};
}

match task_file_providers.get_provider(git_url) {
Some(provider) => provider.get_local_path(git_url).await,
None => bail!("No provider found for git URL: {}", git_url),
let source = RemoteSource::parse_git(git_url)
.ok_or_else(|| eyre!("No provider found for git URL: {}", git_url))?;
let cache_key = (source.url.clone(), source.git_ref.clone());
let checkout = REMOTE_TASK_INCLUDE_ARTIFACTS
.entry(cache_key)
.or_insert_with(|| Arc::new(OnceCell::new()))
.clone();
let checkout = checkout
.get_or_try_init(|| async {
let task_file_providers = TaskFileProvidersBuilder::new().with_cache(false).build();
let provider = task_file_providers
.get_provider(git_url)
.ok_or_else(|| eyre!("No provider found for git URL: {}", git_url))?;
let artifact = provider.get_local_artifact(git_url).await?;
let checkout_path = artifact
.cleanup_path()
.ok_or_else(|| eyre!("no cleanup path for no-cache Git task include"))?
.to_path_buf();
Ok::<TaskFileArtifact, eyre::Report>(artifact.with_path(checkout_path))
})
.await?;

let artifact = checkout.with_path(checkout.path.join(source.path));
let metadata = artifact.path.symlink_metadata()?;
if metadata.file_type().is_file() {
file::make_executable(&artifact.path)?;
} else if !metadata.file_type().is_dir() {
bail!(
"remote task path is not a regular file or directory: {}",
display_path(&artifact.path)
);
}
Ok(artifact)
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
}

/// Check if a pattern contains glob metacharacters
Expand Down Expand Up @@ -4869,12 +4938,16 @@ async fn load_task_sources_from_configs(
None
};
for include in &includes {
let paths = if include.starts_with("git::") {
let artifacts = if include.starts_with("git::") {
vec![resolve_git_url_to_path(include).await?]
} else {
expand_task_include(&resolve_dir, include)
.into_iter()
.map(TaskFileArtifact::persistent)
.collect()
};
for p in paths {
for artifact in artifacts {
let p = artifact.path;
let mut loaded = load_tasks_includes(
config,
&p,
Expand Down
Loading
Loading