Add support for secrets

api/schema.capnp

@@ -41,6 +41,14 @@ struct OBuilder {
   # The contents of the OBuilder spec to build.
 }
 
+struct Secret {
+  id @0 :Text;
+  # The secret id.
+
+  value @1 :Text;
+  # The secret value.
+}
+
 struct JobDescr {
   action :union {
     dockerBuild @0 :DockerBuild;
@@ -57,6 +65,9 @@ struct JobDescr {
   commits @3 :List(Text);
   # The commit(s) to use as the context. If the list is empty, there will be no context.
   # If there are multiple items, they will be merged.
+
+  secrets @5 :List(Secret);
+  # Secret id-value pairs provided to the job. 
 }
 
 interface Job {

api/submission.ml

@@ -40,7 +40,7 @@ let get_action descr =
   | Obuilder action -> Obuilder_build (Obuilder_job.Spec.read action)
   | Undefined x -> Fmt.failwith "Unknown action type %d" x
 
-let submit ?src ?(urgent=false) t ~pool ~action ~cache_hint =
+let submit ?src ?(urgent=false) ?(secrets=[]) t ~pool ~action ~cache_hint =
   let open X.Submit in
   let module JD = Raw.Builder.JobDescr in
   let request, params = Capability.Request.create Params.init_pointer in
@@ -61,4 +61,10 @@ let submit ?src ?(urgent=false) t ~pool ~action ~cache_hint =
       let _ : _ Capnp.Array.t = JD.commits_set_list b commits in
       JD.repository_set b repo;
     );
+  let secrets_array = JD.secrets_init b (List.length secrets) in
+  List.iteri (fun i (id, value) ->
+    let secret = Capnp.Array.get secrets_array i in
+    Raw.Builder.Secret.id_set secret id;
+    Raw.Builder.Secret.value_set secret value
+  ) secrets;
   Capability.call_for_caps t method_id request Results.ticket_get_pipelined

bin/client.ml

@@ -40,6 +40,7 @@ type submit_options_common = {
   commits : string list;
   cache_hint : string;
   urgent : bool;
+  secrets : (string * string) list;
 }
 
 let get_action = function
@@ -55,7 +56,13 @@ let get_action = function
     Lwt_io.(with_file ~mode:input) path (Lwt_io.read ?count:None) >|= fun spec ->
     Cluster_api.Submission.obuilder_build spec
 
-let submit { submission_path; pool; repository; commits; cache_hint; urgent } spec =
+let read_whole_file path =
+  let ic = open_in_bin path in
+  Fun.protect ~finally:(fun () -> close_in ic) @@ fun () ->
+  let len = in_channel_length ic in
+  really_input_string ic len
+
+let submit { submission_path; pool; repository; commits; cache_hint; urgent; secrets } spec =
   let src =
     match repository, commits with
     | None, [] -> None
@@ -65,7 +72,8 @@ let submit { submission_path; pool; repository; commits; cache_hint; urgent } sp
   in
   run submission_path @@ fun submission_service ->
   get_action spec >>= fun action ->
-  Capability.with_ref (Cluster_api.Submission.submit submission_service ~urgent ~pool ~action ~cache_hint ?src) @@ fun ticket ->
+  let secrets = List.map (fun (id, path) -> id, read_whole_file path) secrets in
+  Capability.with_ref (Cluster_api.Submission.submit submission_service ~urgent ~pool ~action ~cache_hint ~secrets ?src) @@ fun ticket ->
   Capability.with_ref (Cluster_api.Ticket.job ticket) @@ fun job ->
   let result = Cluster_api.Job.result job in
   Fmt.pr "Tailing log:@.";
@@ -195,6 +203,14 @@ let build_args =
     ~docv:"ARG"
     ["build-arg"]
 
+let secrets =
+  (Arg.value @@
+   Arg.(opt_all (pair ~sep:':' string file)) [] @@
+   Arg.info
+     ~doc:"Provide a secret under the form id:file"
+     ~docv:"SECRET"
+     ["secret"])
+
 let squash =
   Arg.value @@
   Arg.flag @@
@@ -239,10 +255,10 @@ let build_options =
   Term.(pure make $ build_args $ squash $ buildkit $ include_git)
 
 let submit_options_common =
-  let make submission_path pool repository commits cache_hint urgent =
-    { submission_path; pool; repository; commits; cache_hint; urgent }
+  let make submission_path pool repository commits cache_hint urgent secrets =
+    { submission_path; pool; repository; commits; cache_hint; urgent; secrets }
   in
-  Term.(pure make $ connect_addr $ pool $ repo $ commits $ cache_hint $ urgent)
+  Term.(pure make $ connect_addr $ pool $ repo $ commits $ cache_hint $ urgent $ secrets)
 
 let submit_docker_options =
   let make dockerfile push_to build_options =

ocurrent-plugin/connection.ml

@@ -70,7 +70,7 @@ let urgent_if_high = function
   | `Low -> false
 
 (* This is called by [Current.Job] once the confirmation threshold allows the job to be submitted. *)
-let submit ~job ~pool ~action ~cache_hint ?src ~urgent t ~priority ~switch:_ =
+let submit ~job ~pool ~action ~cache_hint ?src ?secrets ~urgent t ~priority ~switch:_ =
   let urgent = urgent priority in
   let limiter_thread = ref None in
   let stage = ref `Init in
@@ -103,7 +103,7 @@ let submit ~job ~pool ~action ~cache_hint ?src ~urgent t ~priority ~switch:_ =
              Lwt_pool.use (rate_limit t pool urgent)
                (fun () ->
                   Prometheus.Gauge.dec_one Metrics.queue_rate_limit;
-                  let ticket = Cluster_api.Submission.submit ~urgent ?src sched ~pool ~action ~cache_hint in
+                  let ticket = Cluster_api.Submission.submit ~urgent ?src ?secrets sched ~pool ~action ~cache_hint in
                   let build_job = Cluster_api.Ticket.job ticket in
                   stage := `Get_ticket ticket;       (* Allow the user to cancel it now. *)
                   Prometheus.Gauge.inc_one Metrics.queue_get_ticket;
@@ -171,5 +171,5 @@ let create ?(max_pipeline=200) sr =
   let rate_limits = Hashtbl.create 10 in
   { sr; sched = Lwt.fail_with "init"; rate_limits; max_pipeline }
 
-let pool ~job ~pool ~action ~cache_hint ?src ?(urgent=urgent_if_high) t =
-  Current.Pool.of_fn ~label:"OCluster" @@ submit ~job ~pool ~action ~cache_hint ~urgent ?src t
+let pool ~job ~pool ~action ~cache_hint ?src ?secrets ?(urgent=urgent_if_high) t =
+  Current.Pool.of_fn ~label:"OCluster" @@ submit ~job ~pool ~action ~cache_hint ~urgent ?src ?secrets t

ocurrent-plugin/connection.mli

@@ -21,6 +21,7 @@ val pool :
   action:Cluster_api.Submission.action ->
   cache_hint:string ->
   ?src:string * string list ->
+  ?secrets:(string * string) list ->
   ?urgent:([`High | `Low] -> bool) ->
   t ->
   Cluster_api.Raw.Client.Job.t Capnp_rpc_lwt.Capability.t Current.Pool.t

ocurrent-plugin/current_ocluster.ml

@@ -11,6 +11,7 @@ type t = {
   connection : Connection.t;
   timeout : Duration.t option;
   push_auth : (string * string) option; (* Username/password for pushes *)
+  secrets : (string * string) list;
   cache_hint : string option;
   urgent : urgency;
 }
@@ -25,6 +26,7 @@ type docker_build = {
 } [@@deriving to_yojson]
 
 let with_push_auth push_auth t = { t with push_auth }
+let with_secrets secrets t = { t with secrets }
 let with_timeout timeout t = { t with timeout }
 let with_urgent urgent t = { t with urgent }
 
@@ -112,7 +114,7 @@ module Op = struct
       | `Never -> false
       | `Auto -> priority = `High
     in
-    let build_pool = Connection.pool ~job ~pool ~action ~cache_hint ~urgent ?src t.connection in
+    let build_pool = Connection.pool ~job ~pool ~action ~cache_hint ~urgent ?src ~secrets:t.secrets t.connection in
     let level =
       match action with
       | Docker_build { push_to = Some _; _ } -> Current.Level.Above_average
@@ -140,8 +142,8 @@ module Build = Current_cache.Make(Op)
 
 open Current.Syntax
 
-let v ?timeout ?push_auth ?(urgent=`Auto) connection =
-  { connection; timeout; push_auth; cache_hint = None; urgent }
+let v ?timeout ?push_auth ?(secrets=[]) ?(urgent=`Auto) connection =
+  { connection; timeout; push_auth; secrets; cache_hint = None; urgent }
 
 let component_label label dockerfile pool =
   let pp_label = Fmt.(option (cut ++ string)) in

ocurrent-plugin/current_ocluster.mli

@@ -15,11 +15,13 @@ type urgency = [
 val v :
   ?timeout:Duration.t ->
   ?push_auth:(string * string) ->
+  ?secrets:(string * string) list ->
   ?urgent:urgency ->
   Connection.t ->
   t
 (** [v conn] is a builder that submits jobs using [conn].
     @param push_auth : the username and password to use when pushing to the Docker Hub staging area.
+    @param secrets : secrets to pass to the job as (id, value) pairs.
     @param timeout : default timeout
     @param urgent : when to mark builds as urgent (default [`Auto]). *)
 
@@ -31,6 +33,10 @@ val with_push_auth : (string * string) option -> t -> t
 (** [with_push_auth x t] is a copy of [t] with the specified push settings, but still
     sharing the same connection. *)
 
+val with_secrets : (string * string) list -> t -> t
+(** [with_secrets x t] is a copy of [t] with the specified secrets, but still sharing
+    the same connection. *)
+
 val with_urgent : urgency -> t -> t
 (** [with_urgent x t] is a copy of [t] with urgency policy [x]. *)
 

test/mock_builder.ml

@@ -34,7 +34,7 @@ let rec await t id =
     Lwt_condition.wait t.cond >>= fun () ->
     await t id
 
-let docker_build t ~switch ~log ~src:_ = function
+let docker_build t ~switch ~log ~src:_ ~secrets:_ = function
   | `Obuilder _ -> assert false
   | `Docker (dockerfile, _options) ->
     match dockerfile with

worker/cluster_worker.ml

@@ -72,6 +72,7 @@ type t = {
     switch:Lwt_switch.t ->
     log:Log_data.t ->
     src:string ->
+    secrets:(string * string) list ->
     job_spec ->
     (string, [`Cancelled | `Msg of string]) Lwt_result.t;
   prune_threshold : float option;      (* docker-prune when free space is lower than this (percentage) *)
@@ -128,6 +129,7 @@ let docker_push ~switch ~log t hash { Cluster_api.Docker.Spec.target; auth } =
 let build ~switch ~log t descr =
   let module R = Cluster_api.Raw.Reader.JobDescr in
   let cache_hint = R.cache_hint_get descr in
+  let secrets = R.secrets_get_list descr |> List.map (fun t -> Cluster_api.Raw.Reader.Secret.(id_get t, value_get t)) in
   begin match Cluster_api.Submission.get_action descr with
     | Docker_build { dockerfile; options; push_to } ->
       Log.info (fun f ->
@@ -137,7 +139,7 @@ let build ~switch ~log t descr =
         );
       begin
         Context.with_build_context t.context ~log descr @@ fun src ->
-        t.build ~switch ~log ~src (`Docker (dockerfile, options)) >>!= fun hash ->
+        t.build ~switch ~log ~src ~secrets (`Docker (dockerfile, options)) >>!= fun hash ->
         match push_to with
         | None -> Lwt_result.return ""
         | Some target ->
@@ -149,7 +151,7 @@ let build ~switch ~log t descr =
           f "Got request to build (%s):\n%s" cache_hint (String.trim spec)
         );
       Context.with_build_context t.context ~log descr @@ fun src ->
-      t.build ~switch ~log ~src (`Obuilder (`Contents spec))
+      t.build ~switch ~log ~src ~secrets (`Obuilder (`Contents spec))
   end
   >|= function
   | Error `Cancelled ->
@@ -337,9 +339,18 @@ let write_to_file ~path data =
   Lwt_io.(with_file ~mode:output) ~flags:Unix.[O_TRUNC; O_CREAT; O_RDWR] path @@ fun ch ->
   Lwt_io.write_from_string_exactly ch data 0 (String.length data)
 
-let default_build ?obuilder ~switch ~log ~src = function
+let create_secret_file value =
+  let file = Filename.temp_file "build-worker-" ".secret" in
+  write_to_file ~path:file value >|= fun () -> file
+
+let try_unlink file =
+  if Sys.file_exists file then Lwt_unix.unlink file
+  else Lwt.return_unit
+
+let default_build ?obuilder ~switch ~log ~src ~secrets = function
   | `Docker (dockerfile, options) ->
     let iid_file = Filename.temp_file "build-worker-" ".iid" in
+    Lwt_list.map_p (fun (id, value) -> create_secret_file value >|= fun fname -> id, fname) secrets >>= fun secret_files ->
     Lwt.finalize
       (fun () ->
          begin
@@ -358,22 +369,22 @@ let default_build ?obuilder ~switch ~log ~src = function
          let args =
            List.concat_map (fun x -> ["--build-arg"; x]) build_args
            @ (if squash then ["--squash"] else [])
+           @ (List.map (fun (id, fname) -> ["--secret"; Fmt.str "id=%s,src=%s" id fname]) secret_files |> List.flatten)
            @ ["--pull"; "--iidfile"; iid_file; "-f"; dockerpath; src]
          in
          Log.info (fun f -> f "docker build @[%a@]" Fmt.(list ~sep:sp (quote string)) args);
          let env = if buildkit then Some buildkit_env else None in
          Process.check_call ~label:"docker-build" ?env ~switch ~log ("docker" :: "build" :: args) >>!= fun () ->
          Lwt_result.return (String.trim (read_file iid_file))
       )
-      (fun () ->
-         if Sys.file_exists iid_file then Lwt_unix.unlink iid_file
-         else Lwt.return_unit
+      (fun () -> try_unlink iid_file >>= fun () ->
+        secret_files |> List.map snd |> Lwt_list.iter_p try_unlink
       )
   | `Obuilder (`Contents spec) ->
     let spec = Obuilder.Spec.t_of_sexp (Sexplib.Sexp.of_string spec) in
     match obuilder with
     | None -> Fmt.failwith "This worker is not configured for use with OBuilder!"
-    | Some builder -> Obuilder_build.build builder ~switch ~log ~spec ~src_dir:src
+    | Some builder -> Obuilder_build.build builder ~switch ~log ~spec ~src_dir:src ~secrets
 
 let metrics = function
   | `Agent ->

worker/cluster_worker.mli

@@ -14,6 +14,7 @@ val run :
   ?build:(switch:Lwt_switch.t ->
           log:Log_data.t ->
           src:string ->
+          secrets:(string * string) list ->
           job_spec ->
           (string, [`Cancelled | `Msg of string]) Lwt_result.t) ->
   ?allow_push:string list ->

worker/obuilder_build.ml

@@ -110,10 +110,10 @@ let check_free_space t =
     in
     aux ()
 
-let build t ~switch ~log ~spec ~src_dir =
+let build t ~switch ~log ~spec ~src_dir ~secrets =
   check_free_space t >>= fun () ->
   let log = log_to log in
-  let context = Obuilder.Context.v ~switch ~log ~src_dir () in
+  let context = Obuilder.Context.v ~switch ~log ~src_dir ~secrets () in
   let Builder ((module Builder), builder) = t.builder in
   Builder.build builder context spec
 

worker/obuilder_build.mli

@@ -12,6 +12,7 @@ val build : t ->
   switch:Lwt_switch.t ->
   log:Log_data.t ->
   spec:Obuilder.Spec.t ->
-  src_dir:string -> (string, [ `Cancelled | `Msg of string ]) Lwt_result.t
+  src_dir:string ->
+  secrets:(string * string) list -> (string, [ `Cancelled | `Msg of string ]) Lwt_result.t
 
 val healthcheck : t -> (unit, [> `Msg of string]) Lwt_result.t