diff --git a/PROGRESS.md b/PROGRESS.md index 45820d4..5c5701f 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -16,7 +16,7 @@ - [x] Generate faucet and bootstrap accounts - [x] Build genesis - [ ] Docker Build - - [ ] Build Bootstrap Image + - [x] Build Bootstrap Image - [ ] Push Image to registry - [ ] Create & Deploy Secrets - [ ] Bootstrap diff --git a/README.md b/README.md index 2c0d47d..b469538 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ cargo run --bin cluster -- --release-channel # note: MUST include the "v" ``` -#### Build from Local Repo and Configure Genesis +#### Build from Local Repo and Configure Genesis and Bootstrap Validator Image Example: ``` cargo run --bin cluster -- @@ -51,4 +51,9 @@ cargo run --bin cluster -- --max-genesis-archive-unpacked-size --target-lamports-per-signature --slots-per-epoch + # docker config + --registry # e.g. gregcusack + --tag # e.g. v1 + --base-image # e.g. ubuntu:20.04 + --image-name # e.g. cluster-image ``` \ No newline at end of file diff --git a/src/docker.rs b/src/docker.rs new file mode 100644 index 0000000..1abccde --- /dev/null +++ b/src/docker.rs @@ -0,0 +1,190 @@ +use { + crate::{new_spinner_progress_bar, release::DeployMethod, ValidatorType, BUILD}, + log::*, + std::{ + env, + error::Error, + fs, + path::{Path, PathBuf}, + process::{Command, Output, Stdio}, + }, +}; + +pub struct DockerConfig { + pub base_image: String, + pub image_name: String, + pub tag: String, + pub registry: String, + deploy_method: DeployMethod, +} + +impl DockerConfig { + pub fn new( + base_image: String, + image_name: String, + tag: String, + registry: String, + deploy_method: DeployMethod, + ) -> Self { + DockerConfig { + base_image, + image_name, + tag, + registry, + deploy_method, + } + } + + pub fn build_image( + &self, + solana_root_path: &Path, + validator_type: &ValidatorType, + ) -> Result<(), Box> { + match validator_type { + ValidatorType::Bootstrap => (), + ValidatorType::Standard | ValidatorType::RPC | ValidatorType::Client => { + return Err(format!( + "Build docker image for validator type: {validator_type} not supported yet" + ) + .into()); + } + } + let image_name = format!("{validator_type}-{}", self.image_name); + let docker_path = solana_root_path.join(format!("docker-build/{validator_type}")); + match self.create_base_image(solana_root_path, image_name, &docker_path, validator_type) { + Ok(res) => { + if res.status.success() { + info!("Successfully created base Image"); + Ok(()) + } else { + error!("Failed to build base image"); + Err(String::from_utf8_lossy(&res.stderr).into()) + } + } + Err(err) => Err(err), + } + } + + fn create_base_image( + &self, + solana_root_path: &Path, + image_name: String, + docker_path: &PathBuf, + validator_type: &ValidatorType, + ) -> Result> { + self.create_dockerfile(validator_type, docker_path, None)?; + + trace!("Tmp: {}", docker_path.as_path().display()); + trace!("Exists: {}", docker_path.as_path().exists()); + + // We use std::process::Command here because Docker-rs is very slow building dockerfiles + // when they are in large repos. Docker-rs doesn't seem to support the `--file` flag natively. + // so we result to using std::process::Command + let dockerfile = docker_path.join("Dockerfile"); + let context_path = solana_root_path.display().to_string(); + + let progress_bar = new_spinner_progress_bar(); + progress_bar.set_message(format!("{BUILD}Building {validator_type} docker image...",)); + + let command = format!( + "docker build -t {}/{image_name}:{} -f {dockerfile:?} {context_path}", + self.registry, self.tag, + ); + + let output = match Command::new("sh") + .arg("-c") + .arg(&command) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .expect("Failed to execute command") + .wait_with_output() + { + Ok(res) => Ok(res), + Err(err) => Err(Box::new(err) as Box), + }; + progress_bar.finish_and_clear(); + info!("{validator_type} image build complete"); + + output + } + + fn copy_file_to_docker( + source_dir: &Path, + docker_dir: &Path, + file_name: &str, + ) -> std::io::Result<()> { + let source_path = source_dir.join("src/scripts").join(file_name); + let destination_path = docker_dir.join(file_name); + fs::copy(source_path, destination_path)?; + Ok(()) + } + + fn create_dockerfile( + &self, + validator_type: &ValidatorType, + docker_path: &PathBuf, + content: Option<&str>, + ) -> Result<(), Box> { + if docker_path.exists() { + fs::remove_dir_all(docker_path)?; + } + fs::create_dir_all(docker_path)?; + + if let DeployMethod::Local(_) = self.deploy_method { + if validator_type == &ValidatorType::Bootstrap { + let manifest_path = + PathBuf::from(env::var("CARGO_MANIFEST_DIR").expect("$CARGO_MANIFEST_DIR")); + let files_to_copy = ["bootstrap-startup-script.sh", "common.sh"]; + for file_name in files_to_copy.iter() { + Self::copy_file_to_docker(&manifest_path, docker_path, file_name)?; + } + } + } + + let (solana_build_directory, startup_script_directory) = + if let DeployMethod::ReleaseChannel(_) = self.deploy_method { + ("solana-release", "./src/scripts".to_string()) + } else { + ("farf", format!("./docker-build/{validator_type}")) + }; + + let dockerfile = format!( + r#" +FROM {} +RUN apt-get update +RUN apt-get install -y iputils-ping curl vim bzip2 + +RUN useradd -ms /bin/bash solana +RUN adduser solana sudo +USER solana + +RUN mkdir -p /home/solana/k8s-cluster-scripts +# TODO: this needs to be changed for non bootstrap, this should be ./src/scripts/-startup-scripts.sh +COPY {startup_script_directory}/bootstrap-startup-script.sh /home/solana/k8s-cluster-scripts + +RUN mkdir -p /home/solana/ledger +COPY --chown=solana:solana ./config-k8s/bootstrap-validator /home/solana/ledger + +RUN mkdir -p /home/solana/.cargo/bin + +COPY ./{solana_build_directory}/bin/ /home/solana/.cargo/bin/ +COPY ./{solana_build_directory}/version.yml /home/solana/ + +RUN mkdir -p /home/solana/config +ENV PATH="/home/solana/.cargo/bin:${{PATH}}" + +WORKDIR /home/solana + +"#, + self.base_image + ); + + debug!("dockerfile: {dockerfile:?}"); + std::fs::write( + docker_path.join("Dockerfile"), + content.unwrap_or(dockerfile.as_str()), + )?; + Ok(()) + } +} diff --git a/src/genesis.rs b/src/genesis.rs index 129db00..eaa902c 100644 --- a/src/genesis.rs +++ b/src/genesis.rs @@ -250,7 +250,7 @@ impl Genesis { .for_each(|account_type| { args.push( self.config_dir - .join(format!("bootstrap-validator/{}.json", account_type)) + .join(format!("bootstrap-validator/{account_type}.json")) .to_string_lossy() .to_string(), ); @@ -269,22 +269,19 @@ impl Genesis { args } - pub fn setup_spl_args( - &self, - solana_root_path: &PathBuf, - ) -> Result, Box> { + pub fn setup_spl_args(&self, solana_root_path: &Path) -> Result, Box> { let fetch_spl_file = solana_root_path.join("fetch-spl.sh"); fetch_spl(&fetch_spl_file)?; - // add in spl stuff + // add in spl let spl_file = solana_root_path.join("spl-genesis-args.sh"); parse_spl_genesis_file(&spl_file) } pub fn generate( &mut self, - solana_root_path: &PathBuf, - build_path: &PathBuf, + solana_root_path: &Path, + build_path: &Path, ) -> Result<(), Box> { let mut args = self.setup_genesis_flags(); let mut spl_args = self.setup_spl_args(solana_root_path)?; diff --git a/src/lib.rs b/src/lib.rs index 67cfe23..c1ad9cb 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -54,12 +54,14 @@ pub enum ValidatorType { Client, } +pub mod docker; pub mod genesis; pub mod kubernetes; pub mod release; -static SUN: Emoji = Emoji("🌞 ", ""); +static BUILD: Emoji = Emoji("👷 ", ""); static PACKAGE: Emoji = Emoji("📦 ", ""); +static SUN: Emoji = Emoji("🌞 ", ""); static TRUCK: Emoji = Emoji("🚚 ", ""); /// Creates a new process bar for processing that will take an unknown amount of time @@ -78,7 +80,7 @@ pub fn cat_file(path: &PathBuf) -> std::io::Result<()> { let mut file = File::open(path)?; let mut contents = String::new(); file.read_to_string(&mut contents)?; - info!("{:?}:\n{}", path.file_name(), contents); + info!("{:?}:\n{contents}", path.file_name()); Ok(()) } diff --git a/src/main.rs b/src/main.rs index 3319913..1f9bcbe 100644 --- a/src/main.rs +++ b/src/main.rs @@ -4,6 +4,7 @@ use { std::fs, strum::VariantNames, validator_lab::{ + docker::DockerConfig, genesis::{Genesis, GenesisFlags}, kubernetes::Kubernetes, release::{BuildConfig, BuildType, DeployMethod}, @@ -76,7 +77,7 @@ fn parse_matches() -> clap::ArgMatches { Arg::with_name("enable_warmup_epochs") .long("enable-warmup-epochs") .takes_value(true) - .possible_values(&["true", "false"]) + .possible_values(["true", "false"]) .default_value("true") .help("Genesis config. enable warmup epoch. defaults to true"), ) @@ -89,7 +90,7 @@ fn parse_matches() -> clap::ArgMatches { .arg( Arg::with_name("cluster_type") .long("cluster-type") - .possible_values(&["development", "devnet", "testnet", "mainnet-beta"]) + .possible_values(["development", "devnet", "testnet", "mainnet-beta"]) .takes_value(true) .default_value("development") .help( @@ -108,6 +109,40 @@ fn parse_matches() -> clap::ArgMatches { .takes_value(true) .help("Genesis config. bootstrap validator stake sol"), ) + //Docker config + .arg( + Arg::with_name("skip_docker_build") + .long("skip-docker-build") + .help("Skips build Docker images"), + ) + .arg( + Arg::with_name("registry_name") + .long("registry") + .takes_value(true) + .required(true) + .help("Registry to push docker image to"), + ) + .arg( + Arg::with_name("image_name") + .long("image-name") + .takes_value(true) + .default_value("k8s-cluster-image") + .help("Docker image name. Will be prepended with validator_type (bootstrap or validator)"), + ) + .arg( + Arg::with_name("base_image") + .long("base-image") + .takes_value(true) + .default_value("ubuntu:20.04") + .help("Docker base image"), + ) + .arg( + Arg::with_name("image_tag") + .long("tag") + .takes_value(true) + .default_value("latest") + .help("Docker image tag."), + ) .get_matches() } @@ -180,7 +215,12 @@ async fn main() { } } - let build_config = BuildConfig::new(deploy_method, build_type, solana_root.get_root_path()); + let build_config = BuildConfig::new( + deploy_method.clone(), + build_type, + solana_root.get_root_path(), + !matches.is_present("skip_docker_build"), + ); let genesis_flags = GenesisFlags { hashes_per_tick: matches @@ -235,14 +275,14 @@ async fn main() { match build_config.prepare().await { Ok(_) => info!("Validator setup prepared successfully"), Err(err) => { - error!("Error: {}", err); + error!("Error: {err}"); return; } } let mut genesis = Genesis::new(solana_root.get_root_path(), genesis_flags); match genesis.generate_faucet() { - Ok(_) => (), + Ok(_) => info!("Generated faucet account"), Err(err) => { error!("generate faucet error! {err}"); return; @@ -250,7 +290,7 @@ async fn main() { } match genesis.generate_accounts(ValidatorType::Bootstrap, 1) { - Ok(_) => (), + Ok(_) => info!("Generated bootstrap account"), Err(err) => { error!("generate accounts error! {err}"); return; @@ -259,10 +299,36 @@ async fn main() { // creates genesis and writes to binary file match genesis.generate(solana_root.get_root_path(), &build_path) { - Ok(_) => (), + Ok(_) => info!("Created genesis successfully"), Err(err) => { - error!("generate genesis error! {}", err); + error!("generate genesis error! {err}"); return; } } + + //unwraps are safe here. since their requirement is enforced by argmatches + let docker = DockerConfig::new( + matches + .value_of("base_image") + .unwrap_or_default() + .to_string(), + matches.value_of("image_name").unwrap().to_string(), + matches + .value_of("image_tag") + .unwrap_or_default() + .to_string(), + matches.value_of("registry_name").unwrap().to_string(), + deploy_method, + ); + + if build_config.docker_build() { + let image_type = ValidatorType::Bootstrap; + match docker.build_image(solana_root.get_root_path(), &image_type) { + Ok(_) => info!("{image_type} image built successfully"), + Err(err) => { + error!("Error. Failed to build imge: {err}"); + return; + } + } + } } diff --git a/src/release.rs b/src/release.rs index d498bff..49fb08f 100644 --- a/src/release.rs +++ b/src/release.rs @@ -11,7 +11,7 @@ use { strum_macros::{EnumString, IntoStaticStr, VariantNames}, }; -#[derive(Debug, Clone)] +#[derive(Debug, PartialEq, Clone)] pub enum DeployMethod { Local(String), ReleaseChannel(String), @@ -29,6 +29,8 @@ pub struct BuildConfig { deploy_method: DeployMethod, build_type: BuildType, solana_root_path: PathBuf, + docker_build: bool, + build_path: PathBuf, } impl BuildConfig { @@ -36,14 +38,30 @@ impl BuildConfig { deploy_method: DeployMethod, build_type: BuildType, solana_root_path: &Path, + docker_build: bool, ) -> Self { - Self { + let build_path = match deploy_method { + DeployMethod::Local(_) => solana_root_path.join("farf/bin"), + DeployMethod::ReleaseChannel(_) => solana_root_path.join("solana-release/bin"), + }; + + BuildConfig { deploy_method, build_type, solana_root_path: solana_root_path.to_path_buf(), + docker_build, + build_path, } } + pub fn build_path(&self) -> PathBuf { + self.build_path.clone() + } + + pub fn docker_build(&self) -> bool { + self.docker_build + } + pub async fn prepare(&self) -> Result<(), Box> { match &self.deploy_method { DeployMethod::ReleaseChannel(channel) => match self.setup_tar_deploy(channel).await { @@ -64,7 +82,7 @@ impl BuildConfig { async fn setup_tar_deploy(&self, release_channel: &String) -> Result> { let file_name = "solana-release"; let tar_filename = format!("{file_name}.tar.bz2"); - info!("tar file: {}", tar_filename); + info!("tar file: {tar_filename}"); self.download_release_from_channel(&tar_filename, release_channel) .await?; @@ -128,14 +146,14 @@ impl BuildConfig { let tag_object = solana_repo.revparse_single(tag)?.id(); // Check if the commit associated with the tag is the same as the current commit if tag_object == commit { - info!("The current commit is associated with tag: {}", tag); + info!("The current commit is associated with tag: {tag}"); note = tag_object.to_string(); break; } } // Write to branch/tag and commit to version.yml - let content = format!("channel: devbuild {}\ncommit: {}", note, commit); + let content = format!("channel: devbuild {note}\ncommit: {commit}"); std::fs::write(self.solana_root_path.join("farf/version.yml"), content) .expect("Failed to write version.yml"); @@ -148,22 +166,19 @@ impl BuildConfig { tar_filename: &str, release_channel: &String, ) -> Result<(), Box> { - info!("Downloading release from channel: {}", release_channel); + info!("Downloading release from channel: {release_channel}"); let file_path = self.solana_root_path.join(tar_filename); // Remove file if let Err(err) = fs::remove_file(&file_path) { if err.kind() != std::io::ErrorKind::NotFound { - return Err(format!("{}: {:?}", "Error while removing file:", err).into()); + return Err(format!("{err}: {:?}", "Error while removing file:").into()); } } let download_url = format!( - "{}{}{}", - "https://release.solana.com/", - release_channel, - "/solana-release-x86_64-unknown-linux-gnu.tar.bz2" + "https://release.solana.com/{release_channel}/solana-release-x86_64-unknown-linux-gnu.tar.bz2" ); - info!("download_url: {}", download_url); + info!("download_url: {download_url}"); download_to_temp( download_url.as_str(), diff --git a/src/scripts/bootstrap-startup-script.sh b/src/scripts/bootstrap-startup-script.sh new file mode 100644 index 0000000..56e0a7e --- /dev/null +++ b/src/scripts/bootstrap-startup-script.sh @@ -0,0 +1,184 @@ +#!/bin/bash +set -e + +# start faucet +nohup solana-faucet --keypair bootstrap-accounts/faucet.json & + +# Start the bootstrap validator node +# shellcheck disable=SC1091 +source /home/solana/k8s-cluster-scripts/common.sh + +program="agave-validator" + +no_restart=0 + +echo "PROGRAM: $program" + +args=() +while [[ -n $1 ]]; do + if [[ ${1:0:1} = - ]]; then + if [[ $1 = --init-complete-file ]]; then + args+=("$1" "$2") + shift 2 + elif [[ $1 = --gossip-host ]]; then # set with env variables + args+=("$1" "$2") + shift 2 + elif [[ $1 = --gossip-port ]]; then # set with env variables + args+=("$1" "$2") + shift 2 + elif [[ $1 = --dev-halt-at-slot ]]; then # not enabled in net.sh + args+=("$1" "$2") + shift 2 + elif [[ $1 = --dynamic-port-range ]]; then # not enabled in net.sh + args+=("$1" "$2") + shift 2 + elif [[ $1 = --limit-ledger-size ]]; then + args+=("$1" "$2") + shift 2 + elif [[ $1 = --no-rocksdb-compaction ]]; then # not enabled in net.sh + args+=("$1") + shift + elif [[ $1 = --enable-rpc-transaction-history ]]; then # enabled through full-rpc + args+=("$1") + shift + elif [[ $1 = --rpc-pubsub-enable-block-subscription ]]; then # not enabled in net.sh + args+=("$1") + shift + elif [[ $1 = --enable-cpi-and-log-storage ]]; then # not enabled in net.sh + args+=("$1") + shift + elif [[ $1 = --enable-extended-tx-metadata-storage ]]; then # enabled through full-rpc + args+=("$1") + shift + elif [[ $1 = --enable-rpc-bigtable-ledger-storage ]]; then + args+=("$1") + shift + elif [[ $1 = --tpu-disable-quic ]]; then + args+=("$1") + shift + elif [[ $1 = --tpu-enable-udp ]]; then + args+=("$1") + shift + elif [[ $1 = --rpc-send-batch-ms ]]; then # not enabled in net.sh + args+=("$1" "$2") + shift 2 + elif [[ $1 = --rpc-send-batch-size ]]; then # not enabled in net.sh + args+=("$1" "$2") + shift 2 + elif [[ $1 = --skip-poh-verify ]]; then + args+=("$1") + shift + elif [[ $1 = --no-restart ]]; then # not enabled in net.sh + no_restart=1 + shift + elif [[ $1 == --wait-for-supermajority ]]; then + args+=("$1" "$2") + shift 2 + elif [[ $1 == --expected-bank-hash ]]; then + args+=("$1" "$2") + shift 2 + elif [[ $1 == --accounts ]]; then + args+=("$1" "$2") + shift 2 + elif [[ $1 == --maximum-snapshots-to-retain ]]; then # not enabled in net.sh + args+=("$1" "$2") + shift 2 + elif [[ $1 == --no-snapshot-fetch ]]; then + args+=("$1") + shift + elif [[ $1 == --accounts-db-skip-shrink ]]; then + args+=("$1") + shift + elif [[ $1 == --require-tower ]]; then + args+=("$1") + shift + elif [[ $1 = --log-messages-bytes-limit ]]; then # not enabled in net.sh + args+=("$1" "$2") + shift 2 + else + echo "Unknown argument: $1" + $program --help + exit 1 + fi + else + echo "Unknown argument: $1" + $program --help + exit 1 + fi +done + +# These keypairs are created by ./setup.sh and included in the genesis config +identity=bootstrap-accounts/identity.json +vote_account=bootstrap-accounts/vote.json + +ledger_dir=/home/solana/ledger +[[ -d "$ledger_dir" ]] || { + echo "$ledger_dir does not exist" + exit 1 +} + +args+=( + --no-os-network-limits-test \ + --no-wait-for-vote-to-start-leader \ + --snapshot-interval-slots 200 \ + --identity "$identity" \ + --vote-account "$vote_account" \ + --ledger ledger \ + --log - \ + --gossip-host "$MY_POD_IP" \ + --gossip-port 8001 \ + --rpc-port 8899 \ + --rpc-faucet-address "$MY_POD_IP":9900 \ + --no-poh-speed-test \ + --no-incremental-snapshots \ + --full-rpc-api \ + --allow-private-addr \ + --enable-rpc-transaction-history +) + +echo "Bootstrap Args" +for arg in "${args[@]}"; do + echo "$arg" +done + +pid= +kill_node() { + # Note: do not echo anything from this function to ensure $pid is actually + # killed when stdout/stderr are redirected + set +ex + if [[ -n $pid ]]; then + declare _pid=$pid + pid= + kill "$_pid" || true + wait "$_pid" || true + fi +} + +kill_node_and_exit() { + kill_node + exit +} + +trap 'kill_node_and_exit' INT TERM ERR + +while true; do + echo "$program ${args[*]}" + $program "${args[@]}" & + pid=$! + echo "pid: $pid" + + if ((no_restart)); then + wait "$pid" + exit $? + fi + + while true; do + if [[ -z $pid ]] || ! kill -0 "$pid"; then + echo "############## validator exited, restarting ##############" + break + fi + sleep 1 + done + + kill_node +done diff --git a/src/scripts/common.sh b/src/scripts/common.sh new file mode 100644 index 0000000..dd7a32d --- /dev/null +++ b/src/scripts/common.sh @@ -0,0 +1,123 @@ +# |source| this file +# +# Common utilities shared by other scripts in this directory +# +# The following directive disable complaints about unused variables in this +# file: +# shellcheck disable=2034 + +prebuild= +if [[ $1 = "--prebuild" ]]; then + prebuild=true +fi + +if [[ $(uname) != Linux ]]; then + # Protect against unsupported configurations to prevent non-obvious errors + # later. Arguably these should be fatal errors but for now prefer tolerance. + if [[ -n $SOLANA_CUDA ]]; then + echo "Warning: CUDA is not supported on $(uname)" + SOLANA_CUDA= + fi +fi + +if [[ -n $USE_INSTALL || ! -f "$SOLANA_ROOT"/Cargo.toml ]]; then + # echo "define if solana program" + solana_program() { + # echo "call if solana program" + declare program="$1" + if [[ -z $program ]]; then + printf "solana" + else + printf "solana-%s" "$program" + fi + } +else + echo "define else solana program" + solana_program() { + echo "call if solana program" + declare program="$1" + declare crate="$program" + if [[ -z $program ]]; then + crate="cli" + program="solana" + else + program="solana-$program" + fi + + if [[ -n $NDEBUG ]]; then + maybe_release=--release + fi + + # Prebuild binaries so that CI sanity check timeout doesn't include build time + if [[ $prebuild ]]; then + ( + set -x + # shellcheck disable=SC2086 # Don't want to double quote + cargo $CARGO_TOOLCHAIN build $maybe_release --bin $program + ) + fi + + printf "cargo $CARGO_TOOLCHAIN run $maybe_release --bin %s %s -- " "$program" + } +fi + +solana_bench_tps=$(solana_program bench-tps) +solana_faucet=$(solana_program faucet) +solana_validator=$(solana_program validator) +solana_validator_cuda="$solana_validator --cuda" +solana_genesis=$(solana_program genesis) +solana_gossip=$(solana_program gossip) +solana_keygen=$(solana_program keygen) +solana_ledger_tool=$(solana_program ledger-tool) +solana_cli=$(solana_program) + +export RUST_BACKTRACE=1 + +# https://gist.github.com/cdown/1163649 +urlencode() { + declare s="$1" + declare l=$((${#s} - 1)) + for i in $(seq 0 $l); do + declare c="${s:$i:1}" + case $c in + [a-zA-Z0-9.~_-]) + echo -n "$c" + ;; + *) + printf '%%%02X' "'$c" + ;; + esac + done +} + +default_arg() { + declare name=$1 + declare value=$2 + + for arg in "${args[@]}"; do + if [[ $arg = "$name" ]]; then + return + fi + done + + if [[ -n $value ]]; then + args+=("$name" "$value") + else + args+=("$name") + fi +} + +replace_arg() { + declare name=$1 + declare value=$2 + + default_arg "$name" "$value" + + declare index=0 + for arg in "${args[@]}"; do + index=$((index + 1)) + if [[ $arg = "$name" ]]; then + args[$index]="$value" + fi + done +}