diff --git a/README.md b/README.md index ac1b62d1..903799e4 100644 --- a/README.md +++ b/README.md @@ -71,8 +71,6 @@ which should make it easier to generate and consume it automatically. When performing a build, the user gives OBuilder a specification file (as described below), and a source directory, containing files which may be copied into the image using `copy`. -At the moment, multi-stage builds are not supported, so a spec file is just a single stage, of the form: - ```sexp ((from BASE) OP...) ``` @@ -99,6 +97,29 @@ By default: - The workdir is `/`. - The shell is `/bin/bash -c`. +### Multi-stage builds + +You can define nested builds and use the output from them in `copy` operations. +For example: + +```sexp +((build dev + ((from ocaml/opam:alpine-3.12-ocaml-4.11) + (user (uid 1000) (gid 1000)) + (workdir /home/opam) + (run (shell "echo 'print_endline {|Hello, world!|}' > main.ml")) + (run (shell "opam exec -- ocamlopt -ccopt -static -o hello main.ml")))) + (from alpine:3.12) + (shell /bin/sh -c) + (copy (from (build dev)) + (src /home/opam/hello) + (dst /usr/local/bin/hello)) + (run (shell "hello"))) +``` + +At the moment, the `(build ...)` items must appear before the `(from ...)` line. + + ### workdir ```sexp @@ -175,6 +196,7 @@ Currently, no other networks can be used, so the only options are `host` or an i ```sexp (copy + (from ...)? (src SRC...) (dst DST) (exclude EXCL...)?) @@ -206,6 +228,9 @@ Files whose basenames are listed in `exclude` are ignored. If `exclude` is not given, the empty list is used. At present, glob patterns or full paths cannot be used here. +If `(from (build NAME))` is given then the source directory is the root directory of the named nested build. +Otherwise, it is the source directory provided by the user. + Notes: - Unlike Docker's `COPY` operation, OBuilder copies the files using the current diff --git a/example.spec b/example.spec index 4c0954fe..43a14313 100644 --- a/example.spec +++ b/example.spec @@ -6,33 +6,44 @@ ; ; The result can then be found in /tank/HASH/rootfs/ (where HASH is displayed at the end of the build). -((from ocurrent/opam@sha256:27504372f75c847ac82eecc4f21599ba81647d377f844bde25325d6852a65760) - (workdir /src) - (user (uid 1000) (gid 1000)) ; Build as the "opam" user - (run (shell "sudo chown opam /src")) - (env OPAM_HASH "3332c004db65ef784f67efdadc50982f000b718f") ; Fix the version of opam-repository we want - (run - (network host) - (shell - "cd ~/opam-repository \ - && (git cat-file -e $OPAM_HASH || git fetch origin master) \ - && git reset -q --hard $OPAM_HASH \ - && git log --no-decorate -n1 --oneline \ - && opam update -u")) - (copy (src obuilder-spec.opam obuilder.opam) (dst ./)) ; Copy just the opam file first (helps caching) - (run (shell "opam pin add -yn .")) - ; Install OS package dependencies - (run - (network host) - (cache (opam-archives (target /home/opam/.opam/download-cache))) - (shell "opam depext -y obuilder")) - ; Install OCaml dependencies +((build dev + ((from ocurrent/opam@sha256:27504372f75c847ac82eecc4f21599ba81647d377f844bde25325d6852a65760) + (workdir /src) + (user (uid 1000) (gid 1000)) ; Build as the "opam" user + (run (shell "sudo chown opam /src")) + (env OPAM_HASH "3332c004db65ef784f67efdadc50982f000b718f") ; Fix the version of opam-repository we want + (run + (network host) + (shell + "cd ~/opam-repository \ + && (git cat-file -e $OPAM_HASH || git fetch origin master) \ + && git reset -q --hard $OPAM_HASH \ + && git log --no-decorate -n1 --oneline \ + && opam update -u")) + ; Copy just the opam file first (helps caching) + (copy (src obuilder-spec.opam obuilder.opam) (dst ./)) + (run (shell "opam pin add -yn .")) + ; Install OS package dependencies + (run + (network host) + (cache (opam-archives (target /home/opam/.opam/download-cache))) + (shell "opam depext -y obuilder")) + ; Install OCaml dependencies + (run + (network host) + (cache (opam-archives (target /home/opam/.opam/download-cache))) + (shell "opam install --deps-only -t obuilder")) + (copy ; Copy the rest of the source code + (src .) + (dst /src/) + (exclude .git _build)) + (run (shell "opam exec -- dune build @install @runtest")))) ; Build and test + ; Now generate a small runtime image with just the resulting binary: + (from debian:10) (run (network host) - (cache (opam-archives (target /home/opam/.opam/download-cache))) - (shell "opam install --deps-only -t obuilder")) - (copy ; Copy the rest of the source code - (src .) - (dst /src/) - (exclude .git _build)) - (run (shell "opam exec -- dune build @install @runtest && rm -rf _build"))) ; Build and test + (shell "apt-get update && apt-get install -y libsqlite3-0 --no-install-recommends")) + (copy (from (build dev)) + (src /src/_build/default/main.exe) + (dst /usr/local/bin/obuilder)) + (run (shell "obuilder --help"))) diff --git a/lib/build.ml b/lib/build.ml index 1cf5cb67..c5bbdf1b 100644 --- a/lib/build.ml +++ b/lib/build.ml @@ -6,19 +6,25 @@ let ( >>!= ) = Lwt_result.bind let hostname = "builder" +module Scope = Map.Make(String) + module Context = struct type t = { switch : Lwt_switch.t option; env : Os.env; (* Environment in which to run commands. *) src_dir : string; (* Directory with files for copying. *) - user : Obuilder_spec.user; (* Container user to run as. *) + user : Obuilder_spec.user; (* Container user to run as. *) workdir : string; (* Directory in the container namespace for cwd. *) shell : string list; log : S.logger; + scope : string Scope.t; (* Nested builds that are in scope. *) } let v ?switch ?(env=[]) ?(user=Obuilder_spec.root) ?(workdir="/") ?(shell=["/bin/bash"; "-c"]) ~log ~src_dir () = - { switch; env; src_dir; user; workdir; shell; log } + { switch; env; src_dir; user; workdir; shell; log; scope = Scope.empty } + + let with_binding name value t = + { t with scope = Scope.add name value t.scope } end module Saved_context = struct @@ -96,9 +102,22 @@ module Make (Raw_store : S.STORE) (Sandbox : S.SANDBOX) = struct | [item] -> Ok (`Copy_item (item, dst)) | _ -> Fmt.error_msg "When copying multiple items, the destination must end with '/'" - let copy t ~context ~base { Obuilder_spec.src; dst; exclude } = - let { Context.switch; src_dir; workdir; user; log; shell = _; env = _ } = context in + let copy t ~context ~base { Obuilder_spec.from; src; dst; exclude } = + let { Context.switch; src_dir; workdir; user; log; shell = _; env = _; scope } = context in let dst = if Filename.is_relative dst then workdir / dst else dst in + begin + match from with + | `Context -> Lwt_result.return src_dir + | `Build name -> + match Scope.find_opt name scope with + | None -> Fmt.failwith "Unknown build %S" name (* (shouldn't happen; gets caught earlier) *) + | Some id -> + match Store.result t.store id with + | None -> + Lwt_result.fail (`Msg (Fmt.strf "Build result %S not found" id)) + | Some dir -> + Lwt_result.return (dir / "rootfs") + end >>!= fun src_dir -> let src_manifest = sequence (List.map (Manifest.generate ~exclude ~src_dir) src) in match Result.bind src_manifest (to_copy_op ~dst) with | Error _ as e -> Lwt.return e @@ -162,7 +181,7 @@ module Make (Raw_store : S.STORE) (Sandbox : S.SANDBOX) = struct | `User user -> k ~base ~context:{context with user} | `Run { shell = cmd; cache; network } -> let switch, run_input, log = - let { Context.switch; workdir; user; env; shell; log; src_dir = _ } = context in + let { Context.switch; workdir; user; env; shell; log; src_dir = _; scope = _ } = context in (switch, { base; workdir; user; env; cmd; shell; network }, log) in run t ~switch ~log ~cache run_input >>!= fun base -> @@ -235,11 +254,23 @@ module Make (Raw_store : S.STORE) (Sandbox : S.SANDBOX) = struct let { Saved_context.env } = Saved_context.t_of_sexp (Sexplib.Sexp.load_sexp (path / "env")) in Lwt_result.return (id, env) - let build t context { Obuilder_spec.from = base; ops } = + let rec build ~scope t context { Obuilder_spec.child_builds; from = base; ops } = + let rec aux context = function + | [] -> Lwt_result.return context + | (name, child_spec) :: child_builds -> + context.Context.log `Heading Fmt.(strf "(build %S ...)" name); + build ~scope t context child_spec >>!= fun child_result -> + context.Context.log `Note Fmt.(strf "--> finished %S" name); + let context = Context.with_binding name child_result context in + aux context child_builds + in + aux context child_builds >>!= fun context -> get_base t ~log:context.Context.log base >>!= fun (id, env) -> let context = { context with env = context.env @ env } in run_steps t ~context ~base:id ops + let build = build ~scope:[] + let delete ?log t id = Store.delete ?log t.store id diff --git a/lib_spec/docker.ml b/lib_spec/docker.ml index 8a2f7c06..acf14ac7 100644 --- a/lib_spec/docker.ml +++ b/lib_spec/docker.ml @@ -37,16 +37,24 @@ let of_op ~buildkit (acc, ctx) : Spec.op -> Dockerfile.t list * ctx = function in run "%s %s" (String.concat " " mounts) (wrap shell) :: acc, ctx | `Run { cache = _; network = _; shell } -> run "%s" (wrap shell) :: acc, ctx - | `Copy { src; dst; exclude = _ } -> - if ctx.user = Spec.root then copy ~src ~dst () :: acc, ctx + | `Copy { from; src; dst; exclude = _ } -> + let from = match from with + | `Build name -> Some name + | `Context -> None + in + if ctx.user = Spec.root then copy ?from ~src ~dst () :: acc, ctx else ( let { Spec.uid; gid } = ctx.user in let chown = Printf.sprintf "%d:%d" uid gid in - copy ~chown ~src ~dst () :: acc, ctx + copy ?from ~chown ~src ~dst () :: acc, ctx ) | `User ({ uid; gid } as u) -> user "%d:%d" uid gid :: acc, { user = u } | `Env b -> env [b] :: acc, ctx -let dockerfile_of_spec ~buildkit { Spec.from; ops } = +let rec convert ?name ~buildkit { Spec.child_builds; from; ops } = + let stages = child_builds |> List.map (fun (name, spec) -> convert ~name ~buildkit spec) |> List.flatten in let ops', _ctx = List.fold_left (of_op ~buildkit) ([], default_ctx) ops in - Dockerfile.from from @@@ List.rev ops' + stages @ [Dockerfile.from ?alias:name from @@@ List.rev ops'] + +let dockerfile_of_spec ~buildkit t = + Dockerfile.empty @@@ convert ~buildkit t diff --git a/lib_spec/spec.ml b/lib_spec/spec.ml index fd32c571..de5cd59d 100644 --- a/lib_spec/spec.ml +++ b/lib_spec/spec.ml @@ -1,5 +1,11 @@ open Sexplib.Std +module Scope = Set.Make(String) (* Nested builds in scope *) + +type sexp = Sexplib.Sexp.t = + | Atom of string + | List of sexp list + (* Convert fields matched by [p] from (name v1 v2 ...) to (name (v1 v2 ...)) *) let inflate_record p = let open Sexplib.Sexp in function @@ -22,7 +28,22 @@ let deflate_record p = in List (List.map deflate xs) +type data_source = [ + | `Context + | `Build of string +] + +let sexp_of_data_source = function + | `Context -> Atom "context" + | `Build name -> List [Atom "build"; Atom name] + +let data_source_of_sexp = function + | Atom "context" -> `Context + | List [Atom "build"; Atom name] -> `Build name + | x -> Fmt.failwith "Invalid data source: %a" Sexplib.Sexp.pp_hum x + type copy = { + from : data_source [@default `Context] [@sexp_drop_default (=)]; src : string list; dst : string; exclude : string list [@sexp.list]; @@ -79,7 +100,7 @@ let sexp_of_op x : Sexplib.Sexp.t = | _ -> failwith "Inline op must be a record!" else args in - Sexplib.Sexp.List (Sexplib.Sexp.Atom name :: args) + List (Atom name :: args) | x -> Fmt.failwith "Invalid op: %a" Sexplib.Sexp.pp_hum x let op_of_sexp x = @@ -93,46 +114,64 @@ let op_of_sexp x = | x -> Fmt.failwith "Invalid op: %a" Sexplib.Sexp.pp_hum x type t = { + child_builds : (string * t) list; from : string; ops : op list; } -let sexp_of_t { from; ops } = - let open Sexplib.Sexp in - List (List [ Atom "from"; Atom from ] :: List.map sexp_of_op ops) - -let t_of_sexp = function - | Sexplib.Sexp.List (List [ Atom "from"; Atom from ] :: ops) -> - { from; ops = List.map op_of_sexp ops } - | x -> Fmt.failwith "Invalid spec: %a" Sexplib.Sexp.pp_hum x +let rec sexp_of_t { child_builds; from; ops } = + let child_builds = + child_builds |> List.map (fun (name, spec) -> + List [ Atom "build"; Atom name; sexp_of_t spec ] + ) + in + List (child_builds @ List [ Atom "from"; Atom from ] :: List.map sexp_of_op ops) + +let rec t_of_sexp = function + | Atom _ as x -> Fmt.failwith "Invalid spec: %a" Sexplib.Sexp.pp_hum x + | List items -> + let rec aux acc = function + | List [ Atom "build"; Atom name; child_spec ] :: xs -> + let child = (name, t_of_sexp child_spec) in + aux (child :: acc) xs + | List [ Atom "from"; Atom from ] :: ops -> + let child_builds = List.rev acc in + { child_builds; from; ops = List.map op_of_sexp ops } + | x :: _ -> Fmt.failwith "Invalid spec item: %a" Sexplib.Sexp.pp_hum x + | [] -> Fmt.failwith "Invalid spec: missing (from)" + in + aux [] items let comment fmt = fmt |> Printf.ksprintf (fun c -> `Comment c) let workdir x = `Workdir x let shell xs = `Shell xs let run ?(cache=[]) ?(network=[]) fmt = fmt |> Printf.ksprintf (fun x -> `Run { shell = x; cache; network }) -let copy ?(exclude=[]) src ~dst = `Copy { src; dst; exclude } +let copy ?(from=`Context) ?(exclude=[]) src ~dst = `Copy { from; src; dst; exclude } let env k v = `Env (k, v) let user ~uid ~gid = `User { uid; gid } let root = { uid = 0; gid = 0 } -let stage ~from ops = { from; ops } - let rec pp_no_boxes f : Sexplib.Sexp.t -> unit = function | List xs -> Fmt.pf f "(%a)" (Fmt.list ~sep:Fmt.sp pp_no_boxes) xs | Atom _ as a -> Sexplib.Sexp.pp_hum f a let pp_one_line = Fmt.hbox pp_no_boxes -let pp_op_sexp f : Sexplib.Sexp.t -> unit = function - | List (Atom ("copy") as op :: args) -> +let rec pp_op_sexp f : Sexplib.Sexp.t -> unit = function + | List [(Atom "build" as op); (Atom _ as name); List ops] -> + Fmt.pf f "(%a @[%a@,(@[%a@])@])" + Sexplib.Sexp.pp_hum op + Sexplib.Sexp.pp_hum name + (Fmt.list ~sep:Fmt.cut pp_op_sexp) ops + | List (Atom "copy" as op :: args) -> Fmt.pf f "(%a @[%a@])" Sexplib.Sexp.pp_hum op (Fmt.list ~sep:Fmt.sp pp_one_line) args | List (Atom ("run") as op :: args) -> Fmt.pf f "(%a @[%a@])" Sexplib.Sexp.pp_hum op - (Fmt.list ~sep:Fmt.sp pp_one_line) args + (Fmt.list ~sep:Fmt.cut pp_one_line) args | x -> pp_one_line f x let pp f t = @@ -142,3 +181,30 @@ let pp f t = | x -> Sexplib.Sexp.pp_hum f x let pp_op = Fmt.using sexp_of_op pp_op_sexp + +let rec validate ?(scope=Scope.empty) { child_builds; from = _; ops } = + let scope = + List.fold_left (fun scope (name, spec) -> + validate ~scope spec; + Scope.add name scope + ) scope child_builds in + ops |> List.iter (function + | `Copy { from = `Build name; src = _; _ } as copy -> + if not (Scope.mem name scope) then ( + let hints = Scope.elements scope in + let post f () = Fmt.pf f " in %a" pp_op copy in + Fmt.failwith "%a" + Fmt.(did_you_mean ~kind:"build" ~post (quote string)) (name, hints) + ) + | _ -> () + ) + +let stage ?(child_builds=[]) ~from ops = + let t = { child_builds; from; ops } in + validate t; + t + +let t_of_sexp sexp = + let t = t_of_sexp sexp in + validate t; + t diff --git a/lib_spec/spec.mli b/lib_spec/spec.mli index 323a5c8d..cfb9e7a3 100644 --- a/lib_spec/spec.mli +++ b/lib_spec/spec.mli @@ -1,4 +1,5 @@ type copy = { + from : [`Context | `Build of string]; src : string list; dst : string; exclude : string list; @@ -25,18 +26,19 @@ type op = [ | `Env of (string * string) ] [@@deriving sexp] -type t = { +type t = private { + child_builds : (string * t) list; from : string; ops : op list; } [@@deriving sexp] -val stage : from:string -> op list -> t +val stage : ?child_builds:(string * t) list -> from:string -> op list -> t val comment : ('a, unit, string, op) format4 -> 'a val workdir : string -> op val shell : string list -> op val run : ?cache:Cache.t list -> ?network:string list -> ('a, unit, string, op) format4 -> 'a -val copy : ?exclude:string list -> string list -> dst:string -> op +val copy : ?from:[`Context | `Build of string] -> ?exclude:string list -> string list -> dst:string -> op val env : string -> string -> op val user : uid:int -> gid:int -> op diff --git a/main.ml b/main.ml index 75e9c210..60fb5b96 100644 --- a/main.ml +++ b/main.ml @@ -30,7 +30,12 @@ let create_builder ?fast_sync spec = let build fast_sync store spec src_dir = Lwt_main.run begin create_builder ~fast_sync store >>= fun (Builder ((module Builder), builder)) -> - let spec = Obuilder.Spec.t_of_sexp (Sexplib.Sexp.load_sexp spec) in + let spec = + try Obuilder.Spec.t_of_sexp (Sexplib.Sexp.load_sexp spec) + with Failure msg -> + print_endline msg; + exit 1 + in let context = Obuilder.Context.v ~log ~src_dir () in Builder.build builder context spec >>= function | Ok x -> diff --git a/test/test.ml b/test/test.ml index 65301d61..749a9e6e 100644 --- a/test/test.ml +++ b/test/test.ml @@ -444,7 +444,10 @@ let test_sexp () = Alcotest.(check string) name s (Fmt.strf "%a" Spec.pp spec) in test "copy" {| - ((from base) + ((build tools + ((from base) + (run (shell "make tools")))) + (from base) (comment "A test comment") (workdir /src) (run (shell "a command")) @@ -452,6 +455,7 @@ let test_sexp () = (shell "a very very very very very very very very very very very very very very very long command")) (copy (src a b) (dst c)) (copy (src a b) (dst c) (exclude .git _build)) + (copy (from (build tools)) (src binary) (dst /usr/local/bin/)) (env DEBUG 1) (user (uid 1) (gid 2)) )|} @@ -520,6 +524,18 @@ let test_docker () = (env DEBUG 1) (user (uid 1) (gid 2)) (copy (src a b) (dst c)) + ) |}; + test ~buildkit:false "Multi-stage" + {| FROM base as tools + RUN make tools + FROM base + COPY --from=tools binary /usr/local/bin/ + |} {| + ((build tools + ((from base) + (run (shell "make tools")))) + (from base) + (copy (from (build tools)) (src binary) (dst /usr/local/bin/)) ) |} let manifest =