diff --git a/CHANGES.md b/CHANGES.md index 2253b2c0..d919dffa 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -14,6 +14,9 @@ dev `Dockerfile_opam` (@avsm) - Create a Tier 3 for distros for things we do not want to test in the opam repository (@avsm @kit-ty-kate) +- Add an `os_family` type (@MisterDA) +- Add support for parser directives (@MisterDA) +- Add Windows support (@MisterDA) v7.0.0 2020-08-14 Cambridge --------------------------- diff --git a/dockerfile-opam.opam b/dockerfile-opam.opam index f7179a74..37fdd49e 100644 --- a/dockerfile-opam.opam +++ b/dockerfile-opam.opam @@ -8,7 +8,10 @@ awk/sed-style assembly. The opam subpackage provides opam and Linux-specific distribution support for generating dockerfiles.""" maintainer: "Anil Madhavapeddy " -authors: "Anil Madhavapeddy " +authors: [ + "Anil Madhavapeddy " + "Antonin Décimo " +] license: "ISC" tags: ["org:mirage" "org:ocamllabs"] homepage: "https://github.com/avsm/ocaml-dockerfile" diff --git a/src-opam/dockerfile_distro.ml b/src-opam/dockerfile_distro.ml index 513596c4..90cf32b4 100644 --- a/src-opam/dockerfile_distro.ml +++ b/src-opam/dockerfile_distro.ml @@ -27,8 +27,32 @@ type t = [ | `OracleLinux of [ `V7 | `V8 | `Latest ] | `OpenSUSE of [ `V42_1 | `V42_2 | `V42_3 | `V15_0 | `V15_1 | `V15_2 | `Latest ] | `Ubuntu of [ `V12_04 | `V14_04 | `V15_04 | `V15_10 | `V16_04 | `V16_10 | `V17_04 | `V17_10 | `V18_04 | `V18_10 | `V19_04 | `V19_10 | `V20_04 | `V20_10 | `LTS | `Latest ] + | `Windows of [ `V20H2 | `Latest ] ] [@@deriving sexp] +type os_family = [ `Linux | `Windows ] [@@deriving sexp] + +let os_family_of_distro (t:t) : os_family = + match t with + | `Alpine _ | `Archlinux _ | `CentOS _ | `Debian _ | `Fedora _ + | `OracleLinux _ | `OpenSUSE _ | `Ubuntu _ -> `Linux + | `Windows _ -> `Windows + +let os_family_to_string (os:os_family) = + match os with + | `Linux -> "linux" + | `Windows -> "windows" + +let opam_repository (os:os_family) = + match os with + | `Linux -> "git://github.com/ocaml/opam-repository.git" + | `Windows -> "git://github.com/fdopen/opam-repository-mingw.git#opam2" + +let personality os_family arch = + match os_family with + | `Linux when Ocaml_version.arch_is_32bit arch -> Some "/usr/bin/linux32" + | _ -> None + type status = [ | `Deprecated | `Active of [ `Tier1 | `Tier2 | `Tier3 ] @@ -46,7 +70,9 @@ let distros = [ `OpenSUSE `V42_1; `OpenSUSE `V42_2; `OpenSUSE `V42_3; `OpenSUSE `V15_0; `OpenSUSE `V15_1; `OpenSUSE `V15_2; `OpenSUSE `Latest; `Ubuntu `V12_04; `Ubuntu `V14_04; `Ubuntu `V15_04; `Ubuntu `V15_10; `Ubuntu `V16_04; `Ubuntu `V16_10; `Ubuntu `V17_04; `Ubuntu `V17_10; `Ubuntu `V18_04; `Ubuntu `V18_10; `Ubuntu `V19_04; `Ubuntu `V19_10; `Ubuntu `V20_04; `Ubuntu `V20_10; - `Ubuntu `Latest; `Ubuntu `LTS ] + `Ubuntu `Latest; `Ubuntu `LTS; + `Windows `V20H2; `Windows `Latest; +] let distro_status (d:t) : status = match d with | `Alpine (`V3_3 | `V3_4 | `V3_5 | `V3_6 | `V3_7 | `V3_8 | `V3_9 | `V3_10 | `V3_11) -> `Deprecated @@ -78,11 +104,13 @@ let distro_status (d:t) : status = match d with | `Ubuntu ( `V12_04 | `V14_04 | `V15_04 | `V15_10 | `V16_10 | `V17_04 | `V17_10 | `V18_10 | `V19_04 | `V19_10 ) -> `Deprecated | `Ubuntu `LTS -> `Alias (`Ubuntu `V20_04) | `Ubuntu `Latest -> `Alias (`Ubuntu `V20_10) + | `Windows `V20H2 -> `Active `Tier3 + | `Windows `Latest -> `Alias (`Windows `V20H2) let latest_distros = [ `Alpine `Latest; `Archlinux `Latest; `CentOS `Latest; `Debian `Stable; `OracleLinux `Latest; `OpenSUSE `Latest; - `Fedora `Latest; `Ubuntu `Latest; `Ubuntu `LTS ] + `Fedora `Latest; `Ubuntu `Latest; `Ubuntu `LTS; `Windows `Latest; ] let master_distro = `Debian `Stable @@ -179,9 +207,11 @@ let builtin_ocaml_of_distro (d:t) : string option = |`OpenSUSE `V15_2 -> Some "4.05.0" |`OracleLinux `V7 -> Some "4.01.0" |`OracleLinux `V8 -> Some "4.07.0" + |`Windows `V20H2 -> Some "4.11.1" |`Alpine `Latest |`CentOS `Latest |`OracleLinux `Latest |`OpenSUSE `Latest |`Ubuntu `LTS | `Ubuntu `Latest - |`Debian (`Testing | `Unstable | `Stable) |`Fedora `Latest -> assert false + |`Debian (`Testing | `Unstable | `Stable) |`Fedora `Latest + |`Windows `Latest -> assert false (* The Docker tag for this distro *) let tag_of_distro (d:t) = match d with @@ -249,6 +279,8 @@ let tag_of_distro (d:t) = match d with |`OpenSUSE `V15_1 -> "opensuse-15.1" |`OpenSUSE `V15_2 -> "opensuse-15.2" |`OpenSUSE `Latest -> "opensuse" + |`Windows `V20H2 -> "windows-20H2" + |`Windows `Latest -> "windows" let distro_of_tag x : t option = match x with |"ubuntu-12.04" -> Some (`Ubuntu `V12_04) @@ -314,6 +346,8 @@ let distro_of_tag x : t option = match x with |"opensuse-15.1" -> Some (`OpenSUSE `V15_1) |"opensuse-15.2" -> Some (`OpenSUSE `V15_2) |"opensuse" -> Some (`OpenSUSE `Latest) + |"windows-20H2" -> Some (`Windows `V20H2) + |"windows" -> Some (`Windows `Latest) |_ -> None let rec human_readable_string_of_distro (d:t) = @@ -376,8 +410,9 @@ let rec human_readable_string_of_distro (d:t) = |`OpenSUSE `V15_0 -> "OpenSUSE 15.0 (Leap)" |`OpenSUSE `V15_1 -> "OpenSUSE 15.1 (Leap)" |`OpenSUSE `V15_2 -> "OpenSUSE 15.2 (Leap)" + |`Windows `V20H2 -> "Windows 20H2" |`Alpine `Latest | `Ubuntu `Latest | `Ubuntu `LTS | `CentOS `Latest | `Fedora `Latest - |`OracleLinux `Latest | `OpenSUSE `Latest -> alias () + |`OracleLinux `Latest | `OpenSUSE `Latest |`Windows `Latest -> alias () let human_readable_short_string_of_distro (t:t) = match t with @@ -389,6 +424,7 @@ let human_readable_short_string_of_distro (t:t) = |`Alpine _ -> "Alpine" |`Archlinux _ -> "Archlinux" |`OpenSUSE _ -> "OpenSUSE" + |`Windows _ -> "Windows" (* The alias tag for the latest stable version of this distro *) let latest_tag_of_distro (t:t) = @@ -401,8 +437,9 @@ let latest_tag_of_distro (t:t) = |`Alpine _ -> "alpine" |`Archlinux _ -> "archlinux" |`OpenSUSE _ -> "opensuse" + |`Windows _ -> "windows" -type package_manager = [ `Apt | `Yum | `Apk | `Zypper | `Pacman ] [@@deriving sexp] +type package_manager = [ `Apt | `Yum | `Apk | `Zypper | `Pacman | `Cygwin ] [@@deriving sexp] let package_manager (t:t) = match t with @@ -414,6 +451,7 @@ let package_manager (t:t) = |`Alpine _ -> `Apk |`Archlinux _ -> `Pacman |`OpenSUSE _ -> `Zypper + |`Windows _ -> `Cygwin let base_distro_tag ?(arch=`X86_64) d = match resolve_alias d with @@ -516,6 +554,13 @@ let base_distro_tag ?(arch=`X86_64) d = | `Latest -> assert false in "opensuse/leap", tag + | `Windows v -> + let tag = + match v with + | `V20H2 -> "20H2" + | `Latest -> assert false + in + "mcr.microsoft.com/windows", tag let compare a b = String.compare (human_readable_string_of_distro a) (human_readable_string_of_distro b) diff --git a/src-opam/dockerfile_distro.mli b/src-opam/dockerfile_distro.mli index ea4b50f4..51f38eb9 100644 --- a/src-opam/dockerfile_distro.mli +++ b/src-opam/dockerfile_distro.mli @@ -31,9 +31,28 @@ type t = [ | `OracleLinux of [ `V7 | `V8 | `Latest ] | `OpenSUSE of [ `V42_1 | `V42_2 | `V42_3 | `V15_0 | `V15_1 | `V15_2 | `Latest ] | `Ubuntu of [ `V12_04 | `V14_04 | `V15_04 | `V15_10 | `V16_04 | `V16_10 | `V17_04 | `V17_10 | `V18_04 | `V18_10 | `V19_04 | `V19_10 | `V20_04 | `V20_10 | `LTS | `Latest ] + | `Windows of [ `V20H2 | `Latest ] ] [@@deriving sexp] (** Supported Docker container distributions *) +type os_family = [ `Linux | `Windows ] [@@deriving sexp] +(** Supported Docker container operating systems *) + +val os_family_of_distro : t -> os_family +(** [os_family_of_distro t] returns the OS family of the distro. *) + +val os_family_to_string : os_family -> string +(** [os_family_to_string os] returns a string representing the OS + family. *) + +val opam_repository : os_family -> string +(** [opam_repository os_family] returns the git URL to the default + Opam repository. *) + +val personality : os_family -> Ocaml_version.arch -> string option +(** [personality os_family arch] returns the personality associated to + the architecture, if [os_family] is [`Linux]. *) + val compare : t -> t -> int (** [compare a b] is a lexical comparison function for {!t}. *) @@ -70,6 +89,7 @@ type package_manager = [ | `Yum (** Fedora Yum *) | `Zypper (** OpenSUSE Zypper *) | `Pacman (** Archlinux Pacman *) + | `Cygwin (** Cygwin package manager *) ] [@@deriving sexp] (** The package manager used by a distro. *) diff --git a/src-opam/dockerfile_opam.ml b/src-opam/dockerfile_opam.ml index 23060c6c..93dc0a21 100644 --- a/src-opam/dockerfile_opam.ml +++ b/src-opam/dockerfile_opam.ml @@ -19,9 +19,15 @@ open Dockerfile module Linux = Dockerfile_linux +module Windows = Dockerfile_windows module D = Dockerfile_distro module OV = Ocaml_version +let personality ?arch distro = + match arch with + | None -> None + | Some arch -> D.personality (D.os_family_of_distro distro) arch + let run_as_opam fmt = Linux.run_as_user "opam" fmt let install_opam_from_source ?(add_default_link=true) ?(prefix= "/usr/local") ~branch () = @@ -62,35 +68,45 @@ let install_bubblewrap_wrappers = run "chmod a+x /home/opam/opam-sandbox-enable" @@ run "sudo mv /home/opam/opam-sandbox-enable /usr/bin/opam-sandbox-enable" -let header ?arch ?maintainer img tag = +let header ?arch ?maintainer ?img ?tag d = let platform = match arch with | Some `I386 -> Some "386" | Some `Aarch32 -> Some "arm" | _ -> None in let shell = - match arch with - | Some arch when OV.arch_is_32bit arch -> shell ["/usr/bin/linux32";"/bin/sh";"-c"] - | _ -> empty in + match personality ?arch d with + | Some pers -> shell [pers; "/bin/bash"; "-c"] + | None -> empty in let maintainer = match maintainer with - | None -> empty | Some t -> Dockerfile.maintainer "%s" t + | None -> empty in + let escape = + match D.os_family_of_distro d with + | `Windows -> parser_directive (`Escape '`') + | _ -> empty in + let img, tag = + let dimg, dtag = D.base_distro_tag ?arch d in + let value default = function None -> default | Some str -> str in + value dimg img, value dtag tag in + escape @@ comment "Autogenerated by OCaml-Dockerfile scripts" @@ from ?platform ~tag img @@ maintainer @@ shell (* Apk based Dockerfile *) -let apk_opam2 ?(labels=[]) ?arch ~distro ~tag () = - header ?arch distro tag @@ label (("distro_style", "apk") :: labels) +let apk_opam2 ?(labels=[]) ?arch distro () = + let img, tag = D.base_distro_tag ?arch distro in + header ?arch distro @@ label (("distro_style", "apk") :: labels) @@ Linux.Apk.install "build-base bzip2 git tar curl ca-certificates openssl" @@ Linux.Git.init () @@ install_opam_from_source ~add_default_link:false ~branch:"2.0" () @@ install_opam_from_source ~add_default_link:false ~branch:"master" () @@ run "strip /usr/local/bin/opam*" - @@ from ~tag distro + @@ from ~tag img @@ Linux.Apk.add_repository ~tag:"testing" "http://dl-cdn.alpinelinux.org/alpine/edge/testing" @@ copy ~from:"0" ~src:["/usr/local/bin/opam-2.0"] ~dst:"/usr/bin/opam-2.0" () @@ copy ~from:"0" ~src:["/usr/local/bin/opam-master"] ~dst:"/usr/bin/opam-2.1" () @@ -101,14 +117,15 @@ let apk_opam2 ?(labels=[]) ?arch ~distro ~tag () = (* Debian based Dockerfile *) -let apt_opam2 ?(labels=[]) ?arch ~distro ~tag () = - header ?arch distro tag @@ label (("distro_style", "apt") :: labels) +let apt_opam2 ?(labels=[]) ?arch distro () = + let img, tag = D.base_distro_tag ?arch distro in + header ?arch distro @@ label (("distro_style", "apt") :: labels) @@ Linux.Apt.install "build-essential curl git libcap-dev sudo" @@ Linux.Git.init () @@ install_bubblewrap_from_source () @@ install_opam_from_source ~add_default_link:false ~branch:"2.0" () @@ install_opam_from_source ~add_default_link:false ~branch:"master" () - @@ from ~tag distro + @@ from ~tag img @@ copy ~from:"0" ~src:["/usr/local/bin/bwrap"] ~dst:"/usr/bin/bwrap" () @@ copy ~from:"0" ~src:["/usr/local/bin/opam-2.0"] ~dst:"/usr/bin/opam-2.0" () @@ copy ~from:"0" ~src:["/usr/local/bin/opam-master"] ~dst:"/usr/bin/opam-2.1" () @@ -127,14 +144,15 @@ let apt_opam2 ?(labels=[]) ?arch ~distro ~tag () = [enable_powertools] enables the PowerTools repository on CentOS 8 and above. This is needed to get most of *-devel packages frequently used by opam packages. *) -let yum_opam2 ?(labels= []) ?arch ~yum_workaround ~enable_powertools ~distro ~tag () = +let yum_opam2 ?(labels= []) ?arch ~yum_workaround ~enable_powertools distro () = + let img, tag = D.base_distro_tag ?arch distro in let workaround = if yum_workaround then run "touch /var/lib/rpm/*" @@ Linux.RPM.install "yum-plugin-ovl" else empty in - header ?arch distro tag @@ label (("distro_style", "rpm") :: labels) + header ?arch distro @@ label (("distro_style", "rpm") :: labels) @@ run "yum --version || dnf install -y yum" @@ workaround @@ Linux.RPM.update @@ -143,7 +161,7 @@ let yum_opam2 ?(labels= []) ?arch ~yum_workaround ~enable_powertools ~distro ~ta @@ install_bubblewrap_from_source () @@ install_opam_from_source ~prefix:"/usr" ~add_default_link:false ~branch:"2.0" () @@ install_opam_from_source ~prefix:"/usr" ~add_default_link:false ~branch:"master" () - @@ from ~tag distro + @@ from ~tag img @@ run "yum --version || dnf install -y yum" @@ workaround @@ Linux.RPM.update @@ -160,14 +178,15 @@ let yum_opam2 ?(labels= []) ?arch ~yum_workaround ~enable_powertools ~distro ~ta (* Zypper based Dockerfile *) -let zypper_opam2 ?(labels=[]) ?arch ~distro ~tag () = - header ?arch distro tag @@ label (("distro_style", "zypper") :: labels) +let zypper_opam2 ?(labels=[]) ?arch distro () = + let img, tag = D.base_distro_tag ?arch distro in + header ?arch distro @@ label (("distro_style", "zypper") :: labels) @@ Linux.Zypper.dev_packages () @@ Linux.Git.init () @@ install_bubblewrap_from_source () @@ install_opam_from_source ~prefix:"/usr" ~add_default_link:false ~branch:"2.0" () @@ install_opam_from_source ~prefix:"/usr" ~add_default_link:false ~branch:"master" () - @@ from ~tag distro + @@ from ~tag img @@ Linux.Zypper.dev_packages () @@ copy ~from:"0" ~src:["/usr/local/bin/bwrap"] ~dst:"/usr/bin/bwrap" () @@ copy ~from:"0" ~src:["/usr/bin/opam-2.0"] ~dst:"/usr/bin/opam-2.0" () @@ -177,14 +196,15 @@ let zypper_opam2 ?(labels=[]) ?arch ~distro ~tag () = @@ install_bubblewrap_wrappers @@ Linux.Git.init () (* Pacman based Dockerfile *) -let pacman_opam2 ?(labels=[]) ?arch ~distro ~tag () = - header ?arch distro tag @@ label (("distro_style", "pacman") :: labels) +let pacman_opam2 ?(labels=[]) ?arch distro () = + let img, tag = D.base_distro_tag ?arch distro in + header ?arch distro @@ label (("distro_style", "pacman") :: labels) @@ Linux.Pacman.dev_packages () @@ Linux.Git.init () @@ install_opam_from_source ~add_default_link:false ~branch:"2.0" () @@ install_opam_from_source ~add_default_link:false ~branch:"master" () @@ run "strip /usr/local/bin/opam*" - @@ from ~tag distro + @@ from ~tag img @@ copy ~from:"0" ~src:["/usr/local/bin/opam-2.0"] ~dst:"/usr/bin/opam-2.0" () @@ copy ~from:"0" ~src:["/usr/local/bin/opam-master"] ~dst:"/usr/bin/opam-2.1" () @@ run "ln /usr/bin/opam-2.0 /usr/bin/opam" @@ -192,28 +212,51 @@ let pacman_opam2 ?(labels=[]) ?arch ~distro ~tag () = @@ Linux.Pacman.add_user ~uid:1000 ~sudo:true "opam" @@ install_bubblewrap_wrappers @@ Linux.Git.init () + +(* Cygwin based Dockerfile *) +let cygwin_opam2 ?(labels=[]) ?arch distro () = + Windows.Winget.build_form_source ?arch ~distro () + @@ header ?arch distro @@ label (("distro_style", "cygwin") :: labels) + @@ user "ContainerAdministrator" + @@ Windows.install_vc_redist () + @@ Windows.Winget.setup () + @@ Windows.Winget.dev_packages () + @@ Windows.install_visual_studio_build_tools [ + "Microsoft.VisualStudio.Component.VC.Tools.x86.x64"; + "Microsoft.VisualStudio.Component.Windows10SDK.18362"] + @@ Windows.Winget.Git.init () + @@ begin + let extra = Windows.Cygwin.msvc_packages () in + let extra = Windows.Cygwin.mingw_packages ~extra () in + let extra = Windows.Cygwin.cygwin_packages ~extra () in + let extra, t = Windows.Cygwin.ocaml_for_windows_packages ~extra () in + Windows.Cygwin.setup ~extra () @@ t + end + @@ Windows.cleanup () + let gen_opam2_distro ?(clone_opam_repo=true) ?arch ?labels d = - let distro, tag = D.base_distro_tag ?arch d in let fn = match D.package_manager d with - | `Apk -> apk_opam2 ?labels ?arch ~tag ~distro () - | `Apt -> apt_opam2 ?labels ?arch ~tag ~distro () + | `Apk -> apk_opam2 ?labels ?arch d () + | `Apt -> apt_opam2 ?labels ?arch d () | `Yum -> let yum_workaround = match d with `CentOS `V7 -> true | _ -> false in let enable_powertools = match d with `CentOS (`V6 | `V7) -> false | `CentOS _ -> true | _ -> false in - yum_opam2 ?labels ?arch ~yum_workaround ~enable_powertools ~tag ~distro () - | `Zypper -> zypper_opam2 ?labels ?arch ~tag ~distro () - | `Pacman -> pacman_opam2 ?labels ?arch ~tag ~distro () + yum_opam2 ?labels ?arch ~yum_workaround ~enable_powertools d () + | `Zypper -> zypper_opam2 ?labels ?arch d () + | `Pacman -> pacman_opam2 ?labels ?arch d () + | `Cygwin -> cygwin_opam2 ?labels ?arch d () in let clone = if clone_opam_repo then - run "git clone git://github.com/ocaml/opam-repository /home/opam/opam-repository" + let url = Dockerfile_distro.(os_family_of_distro d |> opam_repository) in + run "git clone %S /home/opam/opam-repository" url else empty in - let personality = match arch with - | Some arch when OV.arch_is_32bit arch -> entrypoint_exec ["/usr/bin/linux32"] - | _ -> empty in - (D.tag_of_distro d, fn @@ clone @@ personality) + let pers = match personality ?arch d with + | None -> empty | Some pers -> entrypoint_exec [pers] in + (D.tag_of_distro d, fn @@ clone @@ pers) let all_ocaml_compilers hub_id arch distro = let distro_tag = D.tag_of_distro distro in + let os_family = Dockerfile_distro.os_family_of_distro distro in let compilers = OV.Releases.recent |> List.filter (fun ov -> D.distro_supported_on arch ov distro) |> fun ovs -> @@ -221,23 +264,33 @@ let all_ocaml_compilers hub_id arch distro = if List.exists OV.Releases.is_dev ovs then run "opam repo add beta git://github.com/ocaml/ocaml-beta-repository --set-default" else empty in + let variant = Dockerfile_windows.ocaml_for_windows_compiler_variant os_family arch in List.map (fun t -> run "opam switch create %s %s" - (OV.(to_string (with_patch (with_variant t None) None))) (OV.Opam.V2.name t)) ovs |> + (OV.(to_string (with_patch (with_variant t variant) None))) (OV.Opam.V2.name t)) ovs |> (@@@) add_beta_remote in let d = - let pers = if OV.arch_is_32bit arch then ["/usr/bin/linux32"] else [] in - header ~arch hub_id (Fmt.strf "%s-opam" distro_tag) + let pers = match personality ~arch distro with + | None -> [] | Some pers -> [pers] in + let sandbox = match os_family with + | `Linux -> run "opam-sandbox-disable" + | `Windows -> empty + in + header ~arch ~tag:(Fmt.strf "%s-opam" distro_tag) ~img:hub_id distro @@ workdir "/home/opam/opam-repository" @@ run "git pull origin master" - @@ run "opam-sandbox-disable" - @@ run "opam init -k git -a /home/opam/opam-repository --bare" + @@ sandbox + @@ run "opam init -k git -a /home/opam/opam-repository --bare%s" + (if os_family = `Windows then " --disable-sandboxing" else "") @@ compilers @@ run "opam switch %s" (OV.(to_string (with_patch OV.Releases.latest None))) @@ entrypoint_exec (pers @ ["opam"; "config"; "exec"; "--"]) - @@ run "opam install -y depext" + @@ run "opam install -y depext%s" + (if os_family = `Windows then " depext-cygwinports" else "") @@ env ["OPAMYES","1"] - @@ cmd "bash" + @@ match os_family with + | `Linux -> cmd "bash" + | `Windows -> cmd "CMD" in (Fmt.strf "%s" distro_tag, d) @@ -248,6 +301,7 @@ let tag_of_ocaml_version ov = let separate_ocaml_compilers hub_id arch distro = let distro_tag = D.tag_of_distro distro in + let os_family = Dockerfile_distro.os_family_of_distro distro in OV.Releases.recent_with_dev |> List.filter (fun ov -> D.distro_supported_on arch ov distro) |> List.map (fun ov -> let add_remote = @@ -255,30 +309,40 @@ let separate_ocaml_compilers hub_id arch distro = run "opam repo add beta git://github.com/ocaml/ocaml-beta-repository --set-default" else empty in let default_switch_name = OV.(with_patch (with_variant ov None) None |> to_string) in + let variant = Dockerfile_windows.ocaml_for_windows_compiler_variant os_family arch in let variants = - OV.Opam.V2.switches arch ov |> - List.map (fun t -> run "opam switch create %s %s" (OV.(to_string (with_patch t None))) (OV.Opam.V2.name t)) |> - (@@@) empty + OV.Opam.V2.switches arch ov + |> List.map (fun t -> run "opam switch create %s %s" + (OV.(to_string (with_patch (with_variant t variant) None))) (OV.Opam.V2.name t)) + |> (@@@) empty in let d = - let pers = if OV.arch_is_32bit arch then ["/usr/bin/linux32"] else [] in - header ~arch hub_id (Fmt.strf "%s-opam" distro_tag) + let pers = match personality ~arch distro with + | None -> [] | Some pers -> [pers] in + let sandbox = match os_family with + | `Linux -> run "opam-sandbox-disable" + | `Windows -> empty in + header ~arch ~tag:(Fmt.strf "%s-opam" distro_tag) ~img:hub_id distro @@ workdir "/home/opam/opam-repository" - @@ run "opam-sandbox-disable" - @@ run "opam init -k git -a /home/opam/opam-repository --bare" + @@ sandbox + @@ run "opam init -k git -a /home/opam/opam-repository --bare%s" + (if os_family = `Windows then "--disable-sandboxing" else "") @@ add_remote @@ variants @@ run "opam switch %s" default_switch_name - @@ run "opam install -y depext" + @@ run "opam install -y depext%s" + (if os_family = `Windows then "depext-cygwinports" else "") @@ env ["OPAMYES","1"] @@ entrypoint_exec (pers @ ["opam"; "config"; "exec"; "--"]) - @@ cmd "bash" + @@ match os_family with + | `Linux -> cmd "bash" + | `Windows -> cmd "CMD" in (Fmt.strf "%s-ocaml-%s" distro_tag (tag_of_ocaml_version ov), d) ) let deprecated = - header "alpine" "latest" + header (`Alpine `Latest) @@ run "echo 'This container is now deprecated and no longer supported. Please see https://github.com/ocaml/infrastructure/wiki/Containers for the latest supported tags. Try to use the longer term supported aliases instead of specific distribution versions if you want to avoid seeing this message in the future.' && exit 1" let multiarch_manifest ~target ~platforms = diff --git a/src-opam/dockerfile_windows.ml b/src-opam/dockerfile_windows.ml new file mode 100644 index 00000000..2ee4d9ec --- /dev/null +++ b/src-opam/dockerfile_windows.ml @@ -0,0 +1,180 @@ +(* + * Copyright (c) 2020 Tarides - Antonin Décimo + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + * + *) + +open Dockerfile +open Printf + +let run_cmd fmt = ksprintf (run "cmd /S /C %s") fmt +let run_powershell fmt = ksprintf (run {|powershell -Command "%s"|}) fmt + +let install_vc_redist ?(vs_version="16") () = + add ~src:["https://aka.ms/vs/" ^ vs_version ^ "/release/vc_redist.x64.exe"] ~dst:{|C:\TEMP\|} () + @@ run {|C:\TEMP\vc_redist.x64.exe /install /passive /norestart /log C:\TEMP\vc_redist.log|} + +let install_visual_studio_build_tools ?(vs_version="16") ?(split=false) components = + let install = + let fmt = format_of_string + {|C:\TEMP\Install.cmd C:\TEMP\vs_buildtools.exe --quiet --wait --norestart --nocache ` + --installPath C:\BuildTools --channelUri C:\TEMP\VisualStudio.chman ` + --installChannelUri C:\TEMP\VisualStudio.chman%s|} in + if split then + List.fold_left (fun install component -> + install @@ run fmt (" `\n --add " ^ component)) empty components + else + run fmt (List.fold_left (fun acc component -> + acc ^ " `\n --add " ^ component) "" components) + in + (* https://docs.microsoft.com/en-us/visualstudio/install/advanced-build-tools-container?view=vs-2019#install-script *) + (* FIXME: don't download from here? *) + add ~src:["https://raw.githubusercontent.com/MisterDA/Windows-OCaml-Docker/images/Install.cmd"] + ~dst:{|C:\TEMP\|} () + @@ add ~src:["https://aka.ms/vscollect.exe"] ~dst:{|C:\TEMP\collect.exe|} () + @@ add ~src:["https://aka.ms/vs/" ^ vs_version ^ "/release/channel"] ~dst:{|C:\TEMP\VisualStudio.chman|} () + @@ add ~src:["https://aka.ms/vs/" ^ vs_version ^ "/release/vs_buildtools.exe"] ~dst:{|C:\TEMP\vs_buildtools.exe|} () + @@ install + +let append_path paths = + let paths = String.concat ";" paths in + run {|for /f "tokens=1,2,*" %%a in ('reg query "HKLM\SYSTEM\CurrentControlSet\Control\Session Manager\Environment" /V Path ^| findstr /r "^[^H]"') do ` + reg add "HKLM\SYSTEM\CurrentControlSet\Control\Session Manager\Environment" /V Path /t REG_EXPAND_SZ /f /d "%%c;%s"|} paths + +let ocaml_for_windows_compiler_variant os_family arch = + if os_family = `Windows then + match arch with + | `I386 -> Some "mingw32c" + | `X86_64 -> Some "mingw64c" + | _ -> None + else None + +let cleanup () = + run_powershell {|Remove-Item 'C:\TEMP' -Recurse|} + +module Cygwin = struct + type cyg = { + root : string; + mirror : string; + } + + let default = { + root = {|C:\cygwin64|}; + mirror = "http://mirrors.kernel.org/sourceware/cygwin/"; + } + + let run_sh ?(cyg=default) fmt = ksprintf (run {|%s\bin\bash.exe --login -c "%s"|} cyg.root) fmt + + let cygsetup = {|C:\cygwin-setup-x86_64.exe|} + let cygcache = {|C:\TEMP\cache|} + + let install_cygsympathy_from_source cyg = + let cygsympathy = "dra27" in + run {|mkdir %s\lib\cygsympathy\|} cyg.root + @@ add ~src:["https://raw.githubusercontent.com/" ^ cygsympathy ^ "/cygsympathy/script/cygsympathy.cmd"] + ~dst:(cyg.root ^ {|\lib\cygsympathy\|}) () + @@ add ~src:["https://raw.githubusercontent.com/" ^ cygsympathy ^ "/cygsympathy/script/cygsympathy.sh"] + ~dst:(cyg.root ^ {|\lib\cygsympathy\cygsympathy|}) () + @@ run {|mkdir %s\etc\postinstall\|} cyg.root + @@ run {|mklink %s\etc\postinstall\zp_cygsympathy.sh %s\lib\cygsympathy\cygsympathy|} cyg.root cyg.root + + let install_msvs_tools_from_source ?(version="0.4.1") cyg = + add ~src:["https://github.com/metastack/msvs-tools/archive/" ^ version ^ ".tar.gz"] + ~dst:({|C:\TEMP\msvs-tools.tar.gz|}) () + @@ run_sh ~cyg {|cd /tmp && tar -xf /cygdrive/c/TEMP/msvs-tools.tar.gz && cp msvs-tools-%s/msvs-detect msvs-tools-%s/msvs-promote-path /bin|} version version + + let cygwin ?(cyg=default) fmt = + ksprintf (run {|%s --quiet-mode --no-shortcuts --no-startmenu --no-desktop --only-site ` + --root %s --site %s --local-package-dir %s ` + %s|} cygsetup cyg.root cyg.mirror cygcache) fmt + + let install ?(cyg=default) fmt = + ksprintf (cygwin ~cyg "--packages %s") fmt + + let setup ?(cyg=default) ?(extra=[]) () = + add ~src:["https://www.cygwin.com/setup-x86_64.exe"] ~dst:{|C:\cygwin-setup-x86_64.exe|} () + @@ install_cygsympathy_from_source cyg + @@ cygwin ~cyg "--packages %s" (extra |> List.sort_uniq String.compare |> String.concat ",") + @@ install_msvs_tools_from_source cyg + @@ append_path (List.map ((^) cyg.root) [{|\bin|}]) + @@ workdir {|%s\home\opam|} cyg.root + + let update ?(cyg=default) () = + run {|%s --quiet-mode --no-shortcuts --no-startmenu --no-desktop --only-site --root %s ` + --site %s --local-package-dir %s --upgrade-also|} + cygsetup cyg.root cyg.mirror cygcache + + let cygwin_packages ?(extra=[]) () = "make" :: "diffutils" :: "ocaml" :: "gcc-core" :: "flexdll" :: extra + let mingw_packages ?(extra=[]) () = "make" :: "diffutils" :: "mingw64-x86_64-gcc-core" :: extra + let msvc_packages ?(extra=[]) () = "make" :: "diffutils" :: extra + let ocaml_for_windows_packages ?cyg ?(extra=[]) ?version:(version="0.0.0.2") () = + let packages = "make" :: "diffutils" :: "mingw64-x86_64-gcc-g++" :: "vim" :: "git" + :: "curl" :: "rsync" :: "unzip" :: "patch" :: "m4" :: extra in + let t = + add ~src:["https://github.com/fdopen/opam-repository-mingw/releases/download/" ^ version ^ "/opam64.tar.xz"] + ~dst:{|C:\TEMP\|} () + @@ run_sh ?cyg {|cd /tmp && tar -xf /cygdrive/c/TEMP/opam64.tar.xz && ./opam64/install.sh --prefix=/usr && rm -rf opam64 opam64.tar.xz|} in + packages, t + + module Git = struct + let init ?cyg ?(name="Docker") ?(email="docker@example.com") () = + run_sh ?cyg "git config --global user.email '%s'" email + @@ run_sh ?cyg "git config --global user.name '%s'" name + end +end + +module Winget = struct + let build_form_source ?arch ?(distro=`Windows `Latest) ?(winget_version="master") ?(vs_version="16") () = + let img, tag = Dockerfile_distro.base_distro_tag ?arch distro in + parser_directive (`Escape '`') + @@ from ~alias:"winget-builder" ~tag img + @@ user "ContainerAdministrator" + @@ install_vc_redist ~vs_version () + @@ install_visual_studio_build_tools ~vs_version [ + "Microsoft.VisualStudio.Workload.ManagedDesktopBuildTools"; (* .NET desktop build tools *) + "Microsoft.VisualStudio.Workload.VCTools"; (* C++ build tools *) + "Microsoft.VisualStudio.Workload.UniversalBuildTools"; (* Universal Windows Platform build tools *) + "Microsoft.VisualStudio.Workload.MSBuildTools"; (* MSBuild Tools *) + "Microsoft.VisualStudio.Component.VC.Tools.x86.x64"; (* VS 2019 C++ x64/x86 build tools *) + "Microsoft.VisualStudio.Component.Windows10SDK.18362"; (* Windows 10 SDK (10.0.18362.0) *) + ] + @@ add ~src:["https://github.com/microsoft/winget-cli/archive/" ^ winget_version ^ ".zip"] + ~dst:{|C:\TEMP\winget-cli.zip|} () + @@ run_powershell {|Expand-Archive -LiteralPath C:\TEMP\winget-cli.zip -DestinationPath C:\TEMP\ -Force|} + @@ run {|cd C:\TEMP && rename winget-cli-%s winget-cli|} winget_version + @@ run {|cd C:\BuildTools\VC\Auxiliary\Build && vcvarsall.bat x64 && cd C:\TEMP\winget-cli && msbuild -t:restore -m -p:RestorePackagesConfig=true -p:Configuration=Release src\AppInstallerCLI.sln|} + @@ run {|cd C:\BuildTools\VC\Auxiliary\Build && vcvarsall.bat x64 && cd C:\TEMP\winget-cli && msbuild -p:Configuration=Release src\AppInstallerCLI.sln|} + @@ run {|mkdir "C:\Program Files\winget-cli"|} + @@ run {|move "C:\TEMP\winget-cli\src\x64\Release\AppInstallerCLI\AppInstallerCLI.exe" "C:\Program Files\winget-cli\winget.exe"|} + @@ run {|move "C:\TEMP\winget-cli\src\x64\Release\AppInstallerCLI\resources.pri" "C:\Program Files\winget-cli\"|} + + let setup () = + copy ~from:"winget-builder" ~src:[{|C:\Program Files\winget-cli|}] ~dst:{|C:\Program Files\winget-cli|} () + @@ append_path [{|C:\Program Files\winget-cli|}] + + let install pkgs = + List.fold_left (fun acc pkg -> acc @@ run "winget install %s" pkg) empty pkgs + + let dev_packages ?extra () = + let git = install ["git"] in + match extra with + | None -> git + | Some packages -> git @@ install packages + + module Git = struct + let init ?(name="Docker") ?(email="docker@example.com") () = + run "git config --global user.email %S" email + @@ run "git config --global user.name %S" name + end +end diff --git a/src-opam/dockerfile_windows.mli b/src-opam/dockerfile_windows.mli new file mode 100644 index 00000000..28001ca7 --- /dev/null +++ b/src-opam/dockerfile_windows.mli @@ -0,0 +1,129 @@ +(* + * Copyright (c) 2020 Tarides - Antonin Décimo + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + * + *) + +(** Windows specific Dockerfile utility functions *) + +open Dockerfile + +val run_cmd : ('a, unit, string, t) format4 -> 'a +(** [run_cmd fmt] will execute [cmd /S /C fmt]. *) + +val run_powershell : ('a, unit, string, t) format4 -> 'a +(** [run_powershell fmt] will execute [powershell -Command "fmt"]. *) + +val install_vc_redist : ?vs_version:string -> unit -> t +(** Install Microsoft Visual C++ Redistributable. + @see *) + +val install_visual_studio_build_tools : ?vs_version:string -> ?split:bool -> string list -> t +(** Install Visual Studio Build Tools components. [split] controls + wether the components should be installed simultaneously or + sequentially. Although simlutaneously may be more efficient, it + seems to cause problems with Docker. + @see *) + +val ocaml_for_windows_compiler_variant : Dockerfile_distro.os_family -> + Ocaml_version.arch -> string option +(** Returns the OCaml for Windows specific OCaml compiler variant, if + applicable. *) + +val cleanup : unit -> t +(** Cleanup caches. *) + +(** Rules for Cygwin-based installation *) +module Cygwin : sig + type cyg = { + root : string; (** Cygwin root directory *) + mirror : string; (** Cygwin mirror *) + } + + val default : cyg + (** The default Cygwin root and mirror. *) + + val setup : ?cyg:cyg -> ?extra:string list -> unit -> t + (** Setup Cygwin with CygSymPathy and msvs-tools, and [extra] + Cygwin packages. + @see + @see *) + + val install : ?cyg:cyg -> ('a, unit, string, t) format4 -> 'a + (** Install the supplied Cygwin package list. The packages should be + comma-separated. *) + + val update : ?cyg:cyg -> unit -> t + (** Update Cygwin packages. *) + + val cygwin_packages : ?extra:string list -> unit -> string list + (** [cygwin_packages ?extra ()] will install the base development + tools for the OCaml Cygwin port. Extra packages may also bep + optionally supplied via [extra]. *) + + val mingw_packages : ?extra:string list -> unit -> string list + (** [mingw_packages ?extra ()] will install the base development + tools for the OCaml mingw port. Extra packages may also be + optionally supplied via [extra]. *) + + val msvc_packages : ?extra:string list -> unit -> string list + (** [msvc_packages ?extra ()] will install the base development + tools for the OCaml MSVC port. Extra packages may also be + optionally supplied via [extra]. *) + + val ocaml_for_windows_packages : ?cyg:cyg -> ?extra:string list -> ?version:string -> unit + -> string list * t + (** [ocaml_for_windows_packages ?extra ()] returns the list of + Cygwin packages dependencies, and the installation instructions. + Extra packages may also be optionally supplied via [extra]. + @see *) + + val run_sh : ?cyg:cyg -> ('a, unit, string, t) format4 -> 'a + (** [run_sh ?cyg fmt] will execute in the Cygwin root + [\bin\bash.exe --login -c "fmt"]. *) + + (** Rules for Git *) + module Git : sig + val init : ?cyg:cyg -> ?name:string -> ?email:string -> unit -> t + (** Configure the git name and email variables to sensible defaults *) + end +end + +(** Rules for Winget-based installation. + @see / *) +module Winget : sig + val build_form_source : + ?arch:Ocaml_version.arch -> ?distro:Dockerfile_distro.t -> + ?winget_version:string -> ?vs_version:string -> unit -> t + (** Build Winget from source. This won't send telemetry to + Microsoft. It is build in a separate Docker image, with alias + [winget-builder]. *) + + val setup : unit -> t + (** Setup winget-cli from the [winget-builder] Docker image. *) + + val install : string list -> t + (** [install packages] will install the supplied Winget package list. *) + + val dev_packages : ?extra:string list -> unit -> t + (** [dev_packages ?extra ()] will install the base development + tools. Extra packages may also be optionally supplied via + [extra]. *) + + (** Rules for Git *) + module Git : sig + val init : ?name:string -> ?email:string -> unit -> t + (** Configure the git name and email variables to sensible defaults *) + end +end diff --git a/src/dockerfile.ml b/src/dockerfile.ml index 8129a02e..f03f4433 100644 --- a/src/dockerfile.ml +++ b/src/dockerfile.ml @@ -31,8 +31,13 @@ type from = { alias: string option; platform: string option } [@@deriving sexp] +type parser_directive = + [ `Syntax of string | `Escape of char ] + [@@deriving sexp] + type line = - [ `Comment of string + [ `ParserDirective of parser_directive + | `Comment of string | `From of from | `Maintainer of string | `Run of shell_or_exec @@ -93,18 +98,17 @@ let json_array_of_list sl = sprintf "[ %s ]" (String.concat ", " (List.map quote sl)) -let string_of_shell_or_exec (t: shell_or_exec) = +let string_of_shell_or_exec ~escape:(escape) (t: shell_or_exec) = match t with | `Shell s -> s | `Shells [] -> "" | `Shells [s] -> s - | `Shells l -> String.concat " && \\\n " l + | `Shells l -> String.concat (" && "^escape^"\n ") l | `Exec sl -> json_array_of_list sl -let string_of_env_list = function - | [(k, v)] -> sprintf "%s %s" k v - | el -> String.concat " " (List.map (fun (k, v) -> sprintf "%s=%S" k v) el) +let string_of_env_list el = + String.concat " " (List.map (fun (k, v) -> sprintf "%s=%S" k v) el) let optional name = function @@ -115,18 +119,20 @@ let optional name = function let string_of_sources_to_dest (t: sources_to_dest) = let `From frm, `Src sl, `Dst d, `Chown chown = t in String.concat " " ( - optional "--chown" chown @ - optional "--from" frm @ - sl @ - [d] - ) + optional "--chown" chown + @ optional "--from" frm + @ [json_array_of_list (sl @ [d])]) let string_of_label_list ls = List.map (fun (k, v) -> sprintf "%s=%S" k v) ls |> String.concat " " -let rec string_of_line (t: line) = +let rec string_of_line ~escape:(escape) (t: line) = match t with + | `ParserDirective (`Escape c) -> + let escape = String.make 1 c in + cmd "#" ("escape="^escape) + | `ParserDirective (`Syntax str) -> cmd "#" ("syntax="^str) | `Comment c -> cmd "#" c | `From {image; tag; alias; platform} -> cmd "FROM" (String.concat "" [ @@ -135,22 +141,24 @@ let rec string_of_line (t: line) = (match tag with None -> "" | Some t -> ":"^t); (match alias with None -> "" | Some a -> " as " ^ a)]) | `Maintainer m -> cmd "MAINTAINER" m - | `Run c -> cmd "RUN" (string_of_shell_or_exec c) - | `Cmd c -> cmd "CMD" (string_of_shell_or_exec c) + | `Run c -> cmd "RUN" (string_of_shell_or_exec ~escape c) + | `Cmd c -> cmd "CMD" (string_of_shell_or_exec ~escape c) | `Expose pl -> cmd "EXPOSE" (String.concat " " (List.map string_of_int pl)) | `Env el -> cmd "ENV" (string_of_env_list el) | `Add c -> cmd "ADD" (string_of_sources_to_dest c) | `Copy c -> cmd "COPY" (string_of_sources_to_dest c) | `User u -> cmd "USER" u | `Volume vl -> cmd "VOLUME" (json_array_of_list vl) - | `Entrypoint el -> cmd "ENTRYPOINT" (string_of_shell_or_exec el) + | `Entrypoint el -> cmd "ENTRYPOINT" (string_of_shell_or_exec ~escape el) | `Shell sl -> cmd "SHELL" (json_array_of_list sl) | `Workdir wd -> cmd "WORKDIR" wd - | `Onbuild t -> cmd "ONBUILD" (string_of_line t) + | `Onbuild t -> cmd "ONBUILD" (string_of_line ~escape t) | `Label ls -> cmd "LABEL" (string_of_label_list ls) (* Function interface *) +let parser_directive pd : t = [`ParserDirective pd] + let from ?alias ?tag ?platform image = [`From { image; tag; alias; platform }] @@ -194,6 +202,13 @@ let shell s : t = [`Shell s] let workdir fmt = ksprintf (fun wd -> [`Workdir wd]) fmt -let string_of_t tl = String.concat "\n" (List.map string_of_line tl) +let string_of_t tl = + let rec find_escape = function + | `ParserDirective (`Escape c) :: _ -> c + | `ParserDirective _ :: tl -> find_escape tl + | _ -> '\\' + in + let escape = String.make 1 (find_escape tl) in + String.concat "\n" (List.map (string_of_line ~escape) tl) let pp ppf tl = Fmt.pf ppf "%s" (string_of_t tl) diff --git a/src/dockerfile.mli b/src/dockerfile.mli index c37ec3ef..e522ddad 100644 --- a/src/dockerfile.mli +++ b/src/dockerfile.mli @@ -52,6 +52,15 @@ val maybe : ('a -> t) -> 'a option -> t (** {2 Dockerfile commands} *) +type parser_directive = + [ `Syntax of string | `Escape of char ] + [@@deriving sexp] + +val parser_directive : parser_directive -> t +(** A parser directive. If used, needs to be the first line of the + Dockerfile. + @see . *) + val comment : ('a, unit, string, t) format4 -> 'a (** Adds a comment to the Dockerfile for documentation purposes *)