diff --git a/api/schema.capnp b/api/schema.capnp index 16a23052..ba32bc74 100644 --- a/api/schema.capnp +++ b/api/schema.capnp @@ -41,6 +41,14 @@ struct OBuilder { # The contents of the OBuilder spec to build. } +struct Secret { + id @0 :Text; + # The secret id. + + value @1 :Text; + # The secret value. +} + struct JobDescr { action :union { dockerBuild @0 :DockerBuild; @@ -57,6 +65,9 @@ struct JobDescr { commits @3 :List(Text); # The commit(s) to use as the context. If the list is empty, there will be no context. # If there are multiple items, they will be merged. + + secrets @5 :List(Secret); + # Secret id-value pairs provided to the job. } interface Job { diff --git a/api/submission.ml b/api/submission.ml index 1245fb97..b5736917 100644 --- a/api/submission.ml +++ b/api/submission.ml @@ -40,7 +40,7 @@ let get_action descr = | Obuilder action -> Obuilder_build (Obuilder_job.Spec.read action) | Undefined x -> Fmt.failwith "Unknown action type %d" x -let submit ?src ?(urgent=false) t ~pool ~action ~cache_hint = +let submit ?src ?(urgent=false) ?(secrets=[]) t ~pool ~action ~cache_hint = let open X.Submit in let module JD = Raw.Builder.JobDescr in let request, params = Capability.Request.create Params.init_pointer in @@ -61,4 +61,10 @@ let submit ?src ?(urgent=false) t ~pool ~action ~cache_hint = let _ : _ Capnp.Array.t = JD.commits_set_list b commits in JD.repository_set b repo; ); + let secrets_array = JD.secrets_init b (List.length secrets) in + List.iteri (fun i (id, value) -> + let secret = Capnp.Array.get secrets_array i in + Raw.Builder.Secret.id_set secret id; + Raw.Builder.Secret.value_set secret value + ) secrets; Capability.call_for_caps t method_id request Results.ticket_get_pipelined diff --git a/bin/client.ml b/bin/client.ml index 6f8bfb8b..6f765cd5 100644 --- a/bin/client.ml +++ b/bin/client.ml @@ -40,6 +40,7 @@ type submit_options_common = { commits : string list; cache_hint : string; urgent : bool; + secrets : (string * string) list; } let get_action = function @@ -55,7 +56,13 @@ let get_action = function Lwt_io.(with_file ~mode:input) path (Lwt_io.read ?count:None) >|= fun spec -> Cluster_api.Submission.obuilder_build spec -let submit { submission_path; pool; repository; commits; cache_hint; urgent } spec = +let read_whole_file path = + let ic = open_in_bin path in + Fun.protect ~finally:(fun () -> close_in ic) @@ fun () -> + let len = in_channel_length ic in + really_input_string ic len + +let submit { submission_path; pool; repository; commits; cache_hint; urgent; secrets } spec = let src = match repository, commits with | None, [] -> None @@ -65,7 +72,8 @@ let submit { submission_path; pool; repository; commits; cache_hint; urgent } sp in run submission_path @@ fun submission_service -> get_action spec >>= fun action -> - Capability.with_ref (Cluster_api.Submission.submit submission_service ~urgent ~pool ~action ~cache_hint ?src) @@ fun ticket -> + let secrets = List.map (fun (id, path) -> id, read_whole_file path) secrets in + Capability.with_ref (Cluster_api.Submission.submit submission_service ~urgent ~pool ~action ~cache_hint ~secrets ?src) @@ fun ticket -> Capability.with_ref (Cluster_api.Ticket.job ticket) @@ fun job -> let result = Cluster_api.Job.result job in Fmt.pr "Tailing log:@."; @@ -195,6 +203,14 @@ let build_args = ~docv:"ARG" ["build-arg"] +let secrets = + (Arg.value @@ + Arg.(opt_all (pair ~sep:':' string file)) [] @@ + Arg.info + ~doc:"Provide a secret under the form id:file" + ~docv:"SECRET" + ["secret"]) + let squash = Arg.value @@ Arg.flag @@ @@ -239,10 +255,10 @@ let build_options = Term.(pure make $ build_args $ squash $ buildkit $ include_git) let submit_options_common = - let make submission_path pool repository commits cache_hint urgent = - { submission_path; pool; repository; commits; cache_hint; urgent } + let make submission_path pool repository commits cache_hint urgent secrets = + { submission_path; pool; repository; commits; cache_hint; urgent; secrets } in - Term.(pure make $ connect_addr $ pool $ repo $ commits $ cache_hint $ urgent) + Term.(pure make $ connect_addr $ pool $ repo $ commits $ cache_hint $ urgent $ secrets) let submit_docker_options = let make dockerfile push_to build_options = diff --git a/obuilder b/obuilder index ca5d9a2f..bb3af255 160000 --- a/obuilder +++ b/obuilder @@ -1 +1 @@ -Subproject commit ca5d9a2f1644cc3d7dbfd1ea0baccf7507f071e8 +Subproject commit bb3af25518afacfaba06a239c8b261bc79a5f591 diff --git a/ocurrent-plugin/connection.ml b/ocurrent-plugin/connection.ml index 6270a3be..957d9efe 100644 --- a/ocurrent-plugin/connection.ml +++ b/ocurrent-plugin/connection.ml @@ -70,7 +70,7 @@ let urgent_if_high = function | `Low -> false (* This is called by [Current.Job] once the confirmation threshold allows the job to be submitted. *) -let submit ~job ~pool ~action ~cache_hint ?src ~urgent t ~priority ~switch:_ = +let submit ~job ~pool ~action ~cache_hint ?src ?secrets ~urgent t ~priority ~switch:_ = let urgent = urgent priority in let limiter_thread = ref None in let stage = ref `Init in @@ -103,7 +103,7 @@ let submit ~job ~pool ~action ~cache_hint ?src ~urgent t ~priority ~switch:_ = Lwt_pool.use (rate_limit t pool urgent) (fun () -> Prometheus.Gauge.dec_one Metrics.queue_rate_limit; - let ticket = Cluster_api.Submission.submit ~urgent ?src sched ~pool ~action ~cache_hint in + let ticket = Cluster_api.Submission.submit ~urgent ?src ?secrets sched ~pool ~action ~cache_hint in let build_job = Cluster_api.Ticket.job ticket in stage := `Get_ticket ticket; (* Allow the user to cancel it now. *) Prometheus.Gauge.inc_one Metrics.queue_get_ticket; @@ -171,5 +171,5 @@ let create ?(max_pipeline=200) sr = let rate_limits = Hashtbl.create 10 in { sr; sched = Lwt.fail_with "init"; rate_limits; max_pipeline } -let pool ~job ~pool ~action ~cache_hint ?src ?(urgent=urgent_if_high) t = - Current.Pool.of_fn ~label:"OCluster" @@ submit ~job ~pool ~action ~cache_hint ~urgent ?src t +let pool ~job ~pool ~action ~cache_hint ?src ?secrets ?(urgent=urgent_if_high) t = + Current.Pool.of_fn ~label:"OCluster" @@ submit ~job ~pool ~action ~cache_hint ~urgent ?src ?secrets t diff --git a/ocurrent-plugin/connection.mli b/ocurrent-plugin/connection.mli index 89ca4f06..72069bbf 100644 --- a/ocurrent-plugin/connection.mli +++ b/ocurrent-plugin/connection.mli @@ -21,6 +21,7 @@ val pool : action:Cluster_api.Submission.action -> cache_hint:string -> ?src:string * string list -> + ?secrets:(string * string) list -> ?urgent:([`High | `Low] -> bool) -> t -> Cluster_api.Raw.Client.Job.t Capnp_rpc_lwt.Capability.t Current.Pool.t diff --git a/ocurrent-plugin/current_ocluster.ml b/ocurrent-plugin/current_ocluster.ml index 4d3e2844..1f2cb7c5 100644 --- a/ocurrent-plugin/current_ocluster.ml +++ b/ocurrent-plugin/current_ocluster.ml @@ -11,6 +11,7 @@ type t = { connection : Connection.t; timeout : Duration.t option; push_auth : (string * string) option; (* Username/password for pushes *) + secrets : (string * string) list; cache_hint : string option; urgent : urgency; } @@ -25,6 +26,7 @@ type docker_build = { } [@@deriving to_yojson] let with_push_auth push_auth t = { t with push_auth } +let with_secrets secrets t = { t with secrets } let with_timeout timeout t = { t with timeout } let with_urgent urgent t = { t with urgent } @@ -112,7 +114,7 @@ module Op = struct | `Never -> false | `Auto -> priority = `High in - let build_pool = Connection.pool ~job ~pool ~action ~cache_hint ~urgent ?src t.connection in + let build_pool = Connection.pool ~job ~pool ~action ~cache_hint ~urgent ?src ~secrets:t.secrets t.connection in let level = match action with | Docker_build { push_to = Some _; _ } -> Current.Level.Above_average @@ -140,8 +142,8 @@ module Build = Current_cache.Make(Op) open Current.Syntax -let v ?timeout ?push_auth ?(urgent=`Auto) connection = - { connection; timeout; push_auth; cache_hint = None; urgent } +let v ?timeout ?push_auth ?(secrets=[]) ?(urgent=`Auto) connection = + { connection; timeout; push_auth; secrets; cache_hint = None; urgent } let component_label label dockerfile pool = let pp_label = Fmt.(option (cut ++ string)) in diff --git a/ocurrent-plugin/current_ocluster.mli b/ocurrent-plugin/current_ocluster.mli index c541aede..f956b444 100644 --- a/ocurrent-plugin/current_ocluster.mli +++ b/ocurrent-plugin/current_ocluster.mli @@ -15,11 +15,13 @@ type urgency = [ val v : ?timeout:Duration.t -> ?push_auth:(string * string) -> + ?secrets:(string * string) list -> ?urgent:urgency -> Connection.t -> t (** [v conn] is a builder that submits jobs using [conn]. @param push_auth : the username and password to use when pushing to the Docker Hub staging area. + @param secrets : secrets to pass to the job as (id, value) pairs. @param timeout : default timeout @param urgent : when to mark builds as urgent (default [`Auto]). *) @@ -31,6 +33,10 @@ val with_push_auth : (string * string) option -> t -> t (** [with_push_auth x t] is a copy of [t] with the specified push settings, but still sharing the same connection. *) +val with_secrets : (string * string) list -> t -> t +(** [with_secrets x t] is a copy of [t] with the specified secrets, but still sharing + the same connection. *) + val with_urgent : urgency -> t -> t (** [with_urgent x t] is a copy of [t] with urgency policy [x]. *) diff --git a/test/mock_builder.ml b/test/mock_builder.ml index 5955d4ca..fe93b05b 100644 --- a/test/mock_builder.ml +++ b/test/mock_builder.ml @@ -34,7 +34,7 @@ let rec await t id = Lwt_condition.wait t.cond >>= fun () -> await t id -let docker_build t ~switch ~log ~src:_ = function +let docker_build t ~switch ~log ~src:_ ~secrets:_ = function | `Obuilder _ -> assert false | `Docker (dockerfile, _options) -> match dockerfile with diff --git a/worker/cluster_worker.ml b/worker/cluster_worker.ml index 6af7ff2d..6d06f474 100644 --- a/worker/cluster_worker.ml +++ b/worker/cluster_worker.ml @@ -72,6 +72,7 @@ type t = { switch:Lwt_switch.t -> log:Log_data.t -> src:string -> + secrets:(string * string) list -> job_spec -> (string, [`Cancelled | `Msg of string]) Lwt_result.t; prune_threshold : float option; (* docker-prune when free space is lower than this (percentage) *) @@ -128,6 +129,7 @@ let docker_push ~switch ~log t hash { Cluster_api.Docker.Spec.target; auth } = let build ~switch ~log t descr = let module R = Cluster_api.Raw.Reader.JobDescr in let cache_hint = R.cache_hint_get descr in + let secrets = R.secrets_get_list descr |> List.map (fun t -> Cluster_api.Raw.Reader.Secret.(id_get t, value_get t)) in begin match Cluster_api.Submission.get_action descr with | Docker_build { dockerfile; options; push_to } -> Log.info (fun f -> @@ -137,7 +139,7 @@ let build ~switch ~log t descr = ); begin Context.with_build_context t.context ~log descr @@ fun src -> - t.build ~switch ~log ~src (`Docker (dockerfile, options)) >>!= fun hash -> + t.build ~switch ~log ~src ~secrets (`Docker (dockerfile, options)) >>!= fun hash -> match push_to with | None -> Lwt_result.return "" | Some target -> @@ -149,7 +151,7 @@ let build ~switch ~log t descr = f "Got request to build (%s):\n%s" cache_hint (String.trim spec) ); Context.with_build_context t.context ~log descr @@ fun src -> - t.build ~switch ~log ~src (`Obuilder (`Contents spec)) + t.build ~switch ~log ~src ~secrets (`Obuilder (`Contents spec)) end >|= function | Error `Cancelled -> @@ -337,9 +339,18 @@ let write_to_file ~path data = Lwt_io.(with_file ~mode:output) ~flags:Unix.[O_TRUNC; O_CREAT; O_RDWR] path @@ fun ch -> Lwt_io.write_from_string_exactly ch data 0 (String.length data) -let default_build ?obuilder ~switch ~log ~src = function +let create_secret_file value = + let file = Filename.temp_file "build-worker-" ".secret" in + write_to_file ~path:file value >|= fun () -> file + +let try_unlink file = + if Sys.file_exists file then Lwt_unix.unlink file + else Lwt.return_unit + +let default_build ?obuilder ~switch ~log ~src ~secrets = function | `Docker (dockerfile, options) -> let iid_file = Filename.temp_file "build-worker-" ".iid" in + Lwt_list.map_p (fun (id, value) -> create_secret_file value >|= fun fname -> id, fname) secrets >>= fun secret_files -> Lwt.finalize (fun () -> begin @@ -358,6 +369,7 @@ let default_build ?obuilder ~switch ~log ~src = function let args = List.concat_map (fun x -> ["--build-arg"; x]) build_args @ (if squash then ["--squash"] else []) + @ (List.map (fun (id, fname) -> ["--secret"; Fmt.str "id=%s,src=%s" id fname]) secret_files |> List.flatten) @ ["--pull"; "--iidfile"; iid_file; "-f"; dockerpath; src] in Log.info (fun f -> f "docker build @[%a@]" Fmt.(list ~sep:sp (quote string)) args); @@ -365,15 +377,14 @@ let default_build ?obuilder ~switch ~log ~src = function Process.check_call ~label:"docker-build" ?env ~switch ~log ("docker" :: "build" :: args) >>!= fun () -> Lwt_result.return (String.trim (read_file iid_file)) ) - (fun () -> - if Sys.file_exists iid_file then Lwt_unix.unlink iid_file - else Lwt.return_unit + (fun () -> try_unlink iid_file >>= fun () -> + secret_files |> List.map snd |> Lwt_list.iter_p try_unlink ) | `Obuilder (`Contents spec) -> let spec = Obuilder.Spec.t_of_sexp (Sexplib.Sexp.of_string spec) in match obuilder with | None -> Fmt.failwith "This worker is not configured for use with OBuilder!" - | Some builder -> Obuilder_build.build builder ~switch ~log ~spec ~src_dir:src + | Some builder -> Obuilder_build.build builder ~switch ~log ~spec ~src_dir:src ~secrets let metrics = function | `Agent -> diff --git a/worker/cluster_worker.mli b/worker/cluster_worker.mli index d8ccbb37..3f3932ae 100644 --- a/worker/cluster_worker.mli +++ b/worker/cluster_worker.mli @@ -14,6 +14,7 @@ val run : ?build:(switch:Lwt_switch.t -> log:Log_data.t -> src:string -> + secrets:(string * string) list -> job_spec -> (string, [`Cancelled | `Msg of string]) Lwt_result.t) -> ?allow_push:string list -> diff --git a/worker/obuilder_build.ml b/worker/obuilder_build.ml index 5fd59cfc..6017c66f 100644 --- a/worker/obuilder_build.ml +++ b/worker/obuilder_build.ml @@ -110,10 +110,10 @@ let check_free_space t = in aux () -let build t ~switch ~log ~spec ~src_dir = +let build t ~switch ~log ~spec ~src_dir ~secrets = check_free_space t >>= fun () -> let log = log_to log in - let context = Obuilder.Context.v ~switch ~log ~src_dir () in + let context = Obuilder.Context.v ~switch ~log ~src_dir ~secrets () in let Builder ((module Builder), builder) = t.builder in Builder.build builder context spec diff --git a/worker/obuilder_build.mli b/worker/obuilder_build.mli index 1b5df2a1..91ebe2a6 100644 --- a/worker/obuilder_build.mli +++ b/worker/obuilder_build.mli @@ -12,6 +12,7 @@ val build : t -> switch:Lwt_switch.t -> log:Log_data.t -> spec:Obuilder.Spec.t -> - src_dir:string -> (string, [ `Cancelled | `Msg of string ]) Lwt_result.t + src_dir:string -> + secrets:(string * string) list -> (string, [ `Cancelled | `Msg of string ]) Lwt_result.t val healthcheck : t -> (unit, [> `Msg of string]) Lwt_result.t