diff --git a/CHANGES.md b/CHANGES.md index e736566b..a44d280b 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,6 +1,7 @@ unreleased ---------- +- Build and install opam master from source in Windows images. (@MisterDA #140) - Add OpenSUSE 15.4, deprecate OpenSUSE 15.3. (@MisterDA #138) - Update to bubblewrap 0.7.0. (@MisterDA #131) - Add Alpine 3.17 (3.16 is now tier 2 and 3.15 is deprecated). Remove diff --git a/src-opam/opam.ml b/src-opam/opam.ml index 6486ebc0..32b8f002 100644 --- a/src-opam/opam.ml +++ b/src-opam/opam.ml @@ -31,6 +31,12 @@ let maybe_link_opam add_default_link prefix branch = run "ln %s/bin/opam-%s %s/bin/opam" prefix branch prefix else empty +let maybe_link_opam_windows ?cyg add_default_link prefix branch = + if add_default_link then + Windows.Cygwin.run_sh ?cyg "ln %s/bin/opam-%s %s/bin/opam" prefix branch + prefix + else empty + (* Build opam in a separate worktree from an already cloned opam *) let install_opam_from_source ?(add_default_link = true) ?(prefix = "/usr/local") ?(enable_0install_solver = false) ~branch ~hash () = @@ -47,6 +53,38 @@ let install_opam_from_source ?(add_default_link = true) ?(prefix = "/usr/local") prefix branch prefix branch prefix branch branch @@ maybe_link_opam add_default_link prefix branch +(* Build opam in a separate worktree from an already cloned opam *) +let install_opam_from_source_windows ?cyg ?(add_default_link = true) + ?(prefix = "/usr/local") ?(enable_0install_solver = false) ~branch ~hash () + = + let cold_compiler = true and lib_pkg = true in + Windows.Cygwin.run_sh ?cyg + "cd /tmp/opam-sources && cp -P -R -p . ../opam-build-%s && cd \ + ../opam-build-%s && git checkout %s && git config --global --add \ + safe.directory /tmp/opam-build-%s" + branch branch hash branch + @@ (if cold_compiler then + Windows.Cygwin.run_sh ?cyg + {|cd /tmp/opam-build-%s && make cold CONFIGURE_ARGS="--enable-cold-check%s"|} + branch + (if enable_0install_solver then " --with-0install-solver" else "") + else + Windows.Cygwin.run_sh ?cyg "cd /tmp/opam-build-%s && make compiler" + branch + @@ (if lib_pkg then + Windows.Cygwin.run_sh ?cyg "cd /tmp/opam-build-%s && make lib-pkg" + branch + else empty) + @@ Windows.Cygwin.run_sh ?cyg + "cd /tmp/opam-build-%s && ./configure --enable-cold-check%s && make" + branch + (if enable_0install_solver then " --with-0install-solver" else "")) + @@ Windows.Cygwin.run_sh ?cyg + "cd /tmp/opam-build-%s && mkdir -p %s/bin && cp /tmp/opam-build-%s/opam \ + %s/bin/opam-%s && chmod a+x %s/bin/opam-%s" + branch prefix branch prefix branch prefix branch + @@ maybe_link_opam_windows add_default_link prefix branch + let bubblewrap_minimum = (0, 4, 1) let bubblewrap_latest = (0, 7, 0) @@ -176,6 +214,16 @@ type opam_branch = { aliases : string list; } +let opam_master_branch opam_master_hash = + { + branch = "master"; + hash = opam_master_hash; + enable_0install_solver = true; + public_name = "opam-dev"; + aliases = [ "opam-2.2" ]; + (* TODO: Remove/update when opam 2.2 is branched *) + } + let create_opam_branches opam_hashes = let { opam_2_0_hash; opam_2_1_hash; opam_master_hash } = opam_hashes in ( opam_master_hash, @@ -195,16 +243,13 @@ let create_opam_branches opam_hashes = public_name = "opam-2.1"; aliases = []; }; - { - branch = "master"; - hash = opam_master_hash; - enable_0install_solver = true; - public_name = "opam-dev"; - aliases = [ "opam-2.2" ]; - (* TODO: Remove/update when opam 2.2 is branched *) - }; + opam_master_branch opam_master_hash; ] ) +let create_opam_branches_windows opam_hashes = + let { opam_master_hash; _ } = opam_hashes in + (opam_master_hash, [ opam_master_branch opam_master_hash ]) + let install_opams ?prefix opam_master_hash opam_branches = run "git clone https://github.com/ocaml/opam /tmp/opam && cd /tmp/opam && cp \ @@ -213,11 +258,22 @@ let install_opams ?prefix opam_master_hash opam_branches = opam_master_hash @@ List.fold_left (fun acc { branch; hash; enable_0install_solver; _ } -> - let add_default_link = Some false in - let enable_0install_solver = Some enable_0install_solver in acc - @@ install_opam_from_source ?add_default_link ?prefix - ?enable_0install_solver ~branch ~hash ()) + @@ install_opam_from_source ~add_default_link:false ?prefix + ~enable_0install_solver ~branch ~hash ()) + empty opam_branches + +let install_opams_windows ?cyg ?prefix opam_master_hash opam_branches = + Windows.Cygwin.Git.init ?cyg ~repos:[ "/tmp/opam-sources" ] () + @@ Windows.Cygwin.run_sh ?cyg + "git clone https://github.com/ocaml/opam /tmp/opam && cd /tmp/opam && \ + cp -P -R -p . ../opam-sources && git checkout %s" + opam_master_hash + @@ List.fold_left + (fun acc { branch; hash; enable_0install_solver; _ } -> + acc + @@ install_opam_from_source_windows ?cyg ~add_default_link:false + ?prefix ~enable_0install_solver ~branch ~hash ()) empty opam_branches let copy_opams ~src ~dst opam_branches = @@ -233,6 +289,19 @@ let copy_opams ~src ~dst opam_branches = aliases) empty opam_branches +let copy_opams_windows ~src ~dst opam_branches = + List.fold_left + (fun acc { branch; public_name; aliases; _ } -> + acc + @@ copy ~from:"opam-builder" + ~src:[ src ^ "\\opam-" ^ branch ] + ~dst:(dst ^ "\\" ^ public_name) + () + @@@ List.map + (fun alias -> run "mklink %s\\%s %s\\%s" dst alias dst public_name) + aliases) + empty opam_branches + (* Apk based Dockerfile *) let apk_opam2 ?(labels = []) ?arch ~opam_hashes distro () = let opam_master_hash, opam_branches = create_opam_branches opam_hashes in @@ -347,7 +416,11 @@ let pacman_opam2 ?(labels = []) ?arch ~opam_hashes distro () = add an option to enable 0install-solver, and pass ~hash_opam_2_0 ~hash_opam_2_1 like the cygwin one *) (* Native Windows, WinGet, Cygwin based Dockerfiles *) -let windows_opam2 ?win10_revision ?winget ?(labels = []) ?arch distro () = +let windows_opam2 ?win10_revision ?winget ?(labels = []) ?arch ~opam_hashes + distro () = + let opam_master_hash, opam_branches = + create_opam_branches_windows opam_hashes + in let version = match distro with `Windows (_, v) -> v | _ -> assert false in let winget_image, winget_setup = match winget with @@ -358,14 +431,25 @@ let windows_opam2 ?win10_revision ?winget ?(labels = []) ?arch distro () = @@ Windows.Winget.dev_packages ~version () ) | _ -> (empty, empty) in + let opams_image = + Windows.header ~alias:"opam-builder" ?win10_revision ~version () + @@ Windows.sanitize_reg_path () + @@ Windows.Cygwin.( + install_from_release + ~extra: + ("git" :: "patch" :: "mingw64-x86_64-gcc-g++" + :: "mingw64-i686-gcc-g++" :: mingw_packages) + ()) + @@ install_opams_windows opam_master_hash opam_branches + in (* 2022-10-12: Docker Engine 20.10.18 on Windows fails copying C:\cygwin64, so we cannot build Cygwin in a separate image. *) let ocaml_for_windows = let extra, vs_build_tools = match distro with - | `Windows (`Mingw, _) -> (Windows.Cygwin.mingw_packages (), empty) + | `Windows (`Mingw, _) -> (Windows.Cygwin.mingw_packages, empty) | `Windows (`Msvc, _) -> - ( Windows.Cygwin.msvc_packages (), + ( Windows.Cygwin.msvc_packages, Windows.install_visual_studio_build_tools [ "Microsoft.VisualStudio.Component.VC.Tools.x86.x64"; @@ -374,16 +458,21 @@ let windows_opam2 ?win10_revision ?winget ?(labels = []) ?arch distro () = | _ -> invalid_arg "Invalid distribution" in let extra, pkgs = Windows.Cygwin.ocaml_for_windows_packages ~extra () in - Windows.Cygwin.install_from_release ~extra () @@ vs_build_tools @@ pkgs + Windows.Cygwin.install_from_release ~msvs_tools:true ~extra () + @@ vs_build_tools @@ pkgs in - winget_image + let cyg_root = Windows.Cygwin.default.root in + winget_image @@ opams_image @@ header ?win10_revision ?arch distro @@ label (("distro_style", "windows") :: labels) @@ user "ContainerAdministrator" @@ Windows.install_vc_redist () @@ Windows.sanitize_reg_path () - @@ winget_setup @@ ocaml_for_windows @@ Windows.Cygwin.setup () - @@ Windows.Cygwin.Git.init () + @@ winget_setup @@ ocaml_for_windows + @@ copy_opams_windows + ~src:(cyg_root ^ {|\usr\local\bin|}) + ~dst:(cyg_root ^ {|\usr\bin|}) opam_branches + @@ Windows.Cygwin.setup () @@ Windows.Cygwin.Git.init () let gen_opam2_distro ?win10_revision ?winget ?(clone_opam_repo = true) ?arch ?labels ~opam_hashes d = @@ -403,7 +492,8 @@ let gen_opam2_distro ?win10_revision ?winget ?(clone_opam_repo = true) ?arch d () | `Zypper -> zypper_opam2 ?labels ?arch ~opam_hashes d () | `Pacman -> pacman_opam2 ?labels ?arch ~opam_hashes d () - | `Windows -> windows_opam2 ?win10_revision ?winget ?labels ?arch d () + | `Windows -> + windows_opam2 ?win10_revision ?winget ?labels ?arch ~opam_hashes d () | `Cygwin -> failwith "OCaml/opam Docker images with the Cygwin port are not supported." diff --git a/src-opam/windows.ml b/src-opam/windows.ml index affee6d4..4ad71898 100644 --- a/src-opam/windows.ml +++ b/src-opam/windows.ml @@ -120,14 +120,15 @@ let ocaml_for_windows_package_exn ~switch ~port ~arch = let _, pkgver = Ocaml_version.Opam.V2.package switch in ("ocaml-variants", pkgver ^ "+" ^ variant) -let git_init ~name ~email ~opam_repository = +let git_init ~name ~email ~repos = String.concat " && " - [ - sprintf "git config --global user.email '%s'" email; - sprintf "git config --global user.name '%s'" name; - "git config --system core.longpaths true"; - sprintf "git config --global --add safe.directory %s" opam_repository; - ] + (sprintf "git config --global user.email '%s'" email + :: sprintf "git config --global user.name '%s'" name + :: "git config --system core.longpaths true" + :: List.map + (fun repo -> + sprintf "git config --global --add safe.directory %s" repo) + repos) module Cygwin = struct type cyg = { root : string; site : string; args : string list } @@ -212,10 +213,11 @@ module Cygwin = struct let update ?(cyg = default) () = cygsetup ~cyg ~upgrade:true "" |> cleanup let setup_env ~cyg = - env [ ("CYGWIN", "winsymlinks:native") ] + env [ ("CYGWIN", "nodosfilewarning winsymlinks:native") ] @@ prepend_path (List.map (( ^ ) cyg.root) [ {|\bin|} ]) - let install_from_release ?(cyg = default) ?(extra = []) () = + let install_from_release ?(cyg = default) ?(msvs_tools = false) ?(extra = []) + () = setup_env ~cyg @@ add ~src:[ "https://www.cygwin.com/setup-x86_64.exe" ] @@ -223,11 +225,10 @@ module Cygwin = struct () @@ install_cygsympathy_from_source cyg @@ install ~cyg extra - @@ install_msvs_tools_from_source cyg + @@ (if msvs_tools then install_msvs_tools_from_source cyg else empty) @@ run {|awk -i inplace "/(^#)|(^$)/{print;next}{$4=""noacl,""$4; print}" %s\etc\fstab|} cyg.root - @@ remove_system_attribute (cyg.root ^ {|\dev|}) let setup ?(cyg = default) ?from () = (match from with @@ -237,17 +238,22 @@ module Cygwin = struct | None -> empty) @@ workdir {|%s\home\opam|} cyg.root - let cygwin_packages ?(extra = []) ?(flexdll_version = "0.39-1") () = + let cygwin_packages ?(flexdll_version = "0.39-1") () = (* 2021-03-19: flexdll 0.39 is required, but is in Cygwin testing *) - "make" :: "diffutils" :: "ocaml" :: "gcc-core" :: "git" :: "patch" :: "m4" - :: "cygport" - :: ("flexdll=" ^ flexdll_version) - :: extra - - let mingw_packages ?(extra = []) () = - "make" :: "diffutils" :: "mingw64-x86_64-gcc-core" :: extra + [ + "make"; + "diffutils"; + "ocaml"; + "gcc-core"; + "git"; + "patch"; + "m4"; + "cygport"; + "flexdll=" ^ flexdll_version; + ] - let msvc_packages ?(extra = []) () = "make" :: "diffutils" :: extra + let mingw_packages = [ "make"; "diffutils"; "mingw64-x86_64-gcc-core" ] + let msvc_packages = [ "make"; "diffutils" ] let ocaml_for_windows_packages ?cyg ?(extra = []) ?(version = "0.0.0.2") () = let packages = @@ -269,10 +275,9 @@ module Cygwin = struct module Git = struct let init ?(cyg = default) ?(name = "Docker") ?(email = "docker@example.com") - () = + ?(repos = [ "/home/opam/opam-repository" ]) () = env [ ("HOME", cyg.root ^ {|\home\opam|}) ] - @@ run_sh ~cyg "%s" - (git_init ~email ~name ~opam_repository:"/home/opam/opam-repository") + @@ run_sh ~cyg "%s" (git_init ~email ~name ~repos) end end @@ -344,9 +349,8 @@ module Winget = struct | _ -> install [ "Git.Git" ] @@ maybe install extra module Git = struct - let init ?(name = "Docker") ?(email = "docker@example.com") () = - run "%s" - (git_init ~email ~name - ~opam_repository:"C:/cygwin64/home/opam/opam-repository") + let init ?(name = "Docker") ?(email = "docker@example.com") + ?(repos = [ "C:/cygwin64/home/opam/opam-repository" ]) () = + run "%s" (git_init ~email ~name ~repos) end end diff --git a/src-opam/windows.mli b/src-opam/windows.mli index 8215e014..ff0ce733 100644 --- a/src-opam/windows.mli +++ b/src-opam/windows.mli @@ -61,6 +61,10 @@ val ocaml_for_windows_package_exn : [(package_name, package_version)] of the OCaml compiler package in OCaml for Windows, if applicable. *) +val remove_system_attribute : ?recurse:bool -> string -> t +(** Remove the system attribute on a path. Might be useful to copy + data across images when building Docker images. *) + val header : alias:string -> ?win10_revision:Distro.win10_lcu -> @@ -83,8 +87,8 @@ module Cygwin : sig (** The default Cygwin root, mirror, and arguments. *) val install_from_release : - ?cyg:cyg -> ?extra:string list -> unit -> Dockerfile.t - (** Install Cygwin with CygSymPathy and msvs-tools, and [extra] Cygwin + ?cyg:cyg -> ?msvs_tools:bool -> ?extra:string list -> unit -> Dockerfile.t + (** Install Cygwin with CygSymPathy and optionally msvs-tools, and [extra] Cygwin packages (first in a separate Docker image). Sets the [CYGWIN=winsymlinks:native] environment variable. @see @@ -100,21 +104,17 @@ module Cygwin : sig val update : ?cyg:cyg -> unit -> t (** Update Cygwin packages. *) - val cygwin_packages : - ?extra:string list -> ?flexdll_version:string -> unit -> string list - (** [cygwin_packages ?extra ()] will install the base development - tools for the OCaml Cygwin port. Extra packages may also be - optionally supplied via [extra]. *) + val cygwin_packages : ?flexdll_version:string -> unit -> string list + (** [cygwin_packages ?extra ()] is the list of the base development + tools for the OCaml Cygwin port. *) - val mingw_packages : ?extra:string list -> unit -> string list - (** [mingw_packages ?extra ()] will install the base development - tools for the OCaml mingw port. Extra packages may also be - optionally supplied via [extra]. *) + val mingw_packages : string list + (** [mingw_packages] is the list of base development tools for the + Caml mingw port. *) - val msvc_packages : ?extra:string list -> unit -> string list - (** [msvc_packages ?extra ()] will install the base development - tools for the OCaml MSVC port. Extra packages may also be - optionally supplied via [extra]. *) + val msvc_packages : string list + (** [msvc_packages] is the list of base development tools for the + Caml MSVC port. *) val ocaml_for_windows_packages : ?cyg:cyg -> ?extra:string list -> ?version:string -> unit -> string list * t @@ -134,8 +134,18 @@ module Cygwin : sig (** Rules for Git. *) module Git : sig - val init : ?cyg:cyg -> ?name:string -> ?email:string -> unit -> t - (** Configure the git name and email variables to sensible defaults *) + val init : + ?cyg:cyg -> + ?name:string -> + ?email:string -> + ?repos:string list -> + unit -> + t + (** Configure the git name and email variables to sensible defaults. + + @param repos A list of paths to Git repos to mark as safe + directories. Defaults to the default location of the + opam-repository. *) end end @@ -165,7 +175,11 @@ module Winget : sig (** Rules for Git. *) module Git : sig - val init : ?name:string -> ?email:string -> unit -> t - (** Configure the git name and email variables to sensible defaults *) + val init : ?name:string -> ?email:string -> ?repos:string list -> unit -> t + (** Configure the git name and email variables to sensible defaults. + + @param repos A list of paths to Git repos to mark as safe + directories. Defaults to the default location of the + opam-repository. *) end end