Skip to content

Windows #90

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
talex5 merged 9 commits into from
Jul 9, 2021
Merged

Windows #90

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
b20b805
Set secrets path on Windows
MisterDA Mar 8, 2021
f13b532
Don't squash images on Windows, still too experimental (broken)
MisterDA Mar 23, 2021
382754b
Support Cygwin and Windows (Mingw, Msvc) as distributions
MisterDA Jun 4, 2021
65ad6ca
Don't install ocaml-secondary-compiler on Windows
MisterDA Apr 19, 2021
3093d2f
Don't use dev compiler and Opam beta repo on Windows
MisterDA Apr 19, 2021
1b006c8
Build Linux and Windows images by default
MisterDA Mar 25, 2021
9548068
Share parts of Linux and Windows pipelines
MisterDA Jul 8, 2021
56ae924
Mention Windows in README
MisterDA Jun 4, 2021
5b081a5
Use libev to avoid FD limits
talex5 Jul 8, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
FROM ocaml/opam:debian-10-ocaml-4.12@sha256:80e2b0337fe1c44d2f95ac35589b0a2eed4c65d8f12c3e33c12262065106c0c1 AS build
RUN sudo apt-get update && sudo apt-get install capnproto graphviz m4 pkg-config libsqlite3-dev libgmp-dev libssl-dev libffi-dev -y --no-install-recommends
RUN sudo apt-get update && sudo apt-get install libev-dev capnproto graphviz m4 pkg-config libsqlite3-dev libgmp-dev libssl-dev libffi-dev -y --no-install-recommends
RUN cd ~/opam-repository && git pull origin -q master && git reset --hard 1a8b75438a8a47be909af0e6c0768c25384697c6 && opam update
COPY --chown=opam \
ocurrent/current.opam \
Expand Down Expand Up @@ -31,7 +31,7 @@ ADD --chown=opam . .
RUN opam config exec -- dune build ./src/base_images.exe

FROM debian:10
RUN apt-get update && apt-get install curl git graphviz libsqlite3-dev ca-certificates netbase gnupg2 -y --no-install-recommends
RUN apt-get update && apt-get install libev4 curl git graphviz libsqlite3-dev ca-certificates netbase gnupg2 -y --no-install-recommends
RUN curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add -
RUN echo 'deb [arch=amd64] https://download.docker.com/linux/debian buster stable' >> /etc/apt/sources.list
RUN apt-get update && apt-get install docker-ce -y --no-install-recommends
Expand Down
3 changes: 2 additions & 1 deletion README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
This is an [OCurrent][] pipeline that builds Docker images for OCaml, for
various combinations of Linux distribution, OCaml version and architecture.
various combinations of Linux distribution, Windows version, OCaml version
and architecture.

The resulting images can be run as e.g.

Expand Down
1 change: 1 addition & 0 deletions base-images.opam
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ bug-reports: "https://github.com/ocurrent/docker-base-images/issues"
depends: [
"dune" {>= "2.5"}
"ocaml" {>= "4.12"}
"conf-libev" {os != "win32"}
"prometheus-app" {>= "1.0"}
"ppx_sexp_conv"
"ppx_deriving_yojson"
Expand Down
1,218 changes: 1,218 additions & 0 deletions builds.expected
View file
Open in desktop

Large diffs are not rendered by default.

1 change: 1 addition & 0 deletions dune-project
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
(synopsis "Generate Docker base images for OCaml and opam using ocurrent")
(depends
(ocaml (>= 4.12))
(conf-libev (<> :os "win32"))
(prometheus-app (>= 1.0))
ppx_sexp_conv
ppx_deriving_yojson
Expand Down
15 changes: 12 additions & 3 deletions src/conf.ml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,10 @@ let staging_repo = "ocurrent/opam-staging"

let public_repo = "ocaml/opam"

let password_path = "/run/secrets/ocurrent-hub"
let password_path =
let open Fpath in
let root = v (if Sys.win32 then "C:\\ProgramData\\Docker" else "/run") in
Copy link
Contributor

@dra27 dra27 May 25, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why's this in Docker? I'd take the root as "/run/secrets" on Unix and Option.value (Sys.getenv_opt "ProgramData") ~default:"C:\ProgramData") on Windows and then put the secrets in ocurrent-hub under that.

Copy link
Contributor Author

@MisterDA MisterDA May 26, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

My understanding was that these were Docker Swarm secrets (also available via docker compose). The default location for secrets inside containers on Windows is C:\ProgramData\Docker\secrets.
https://docs.docker.com/engine/swarm/secrets/#windows-support

Copy link
Contributor Author

@MisterDA MisterDA Jun 25, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@dra27 we haven't really resolved that conversation, are you convinced by my explanation?

root / "secrets" / "ocurrent-hub" |> to_string

module Capnp = struct
(* Cap'n Proto RPC is enabled by passing --capnp-public-address. These values are hard-coded
Expand Down Expand Up @@ -36,8 +39,11 @@ let pool_name os_family arch =

let switches ~arch ~distro =
let is_tier1 = List.mem distro (Dockerfile_distro.active_tier1_distros arch) in
(* opam-repository-mingw doesn't package the development version of
the compiler. *)
let with_dev = match distro with `Windows _ -> false | _ -> true in
let main_switches =
Ocaml_version.Releases.recent_with_dev
Ocaml_version.Releases.(if with_dev then recent_with_dev else recent)
|> List.filter (fun ov -> Dockerfile_distro.distro_supported_on arch ov distro)
in
if is_tier1 then (
Expand All @@ -47,7 +53,10 @@ let switches ~arch ~distro =
)

(* We can't get the active distros directly, but assume x86_64 is a superset of everything else. *)
let distros = Dockerfile_distro.(active_distros `X86_64 |> List.filter (fun d -> os_family_of_distro d = `Linux))
let distros = Dockerfile_distro.(active_distros `X86_64 |> List.filter (fun d ->
match os_family_of_distro d with
| `Linux | `Windows -> true
| _ -> false))

let arches_for ~distro = Dockerfile_distro.distro_arches Ocaml_version.Releases.latest distro

Expand Down
1 change: 1 addition & 0 deletions src/dump.ml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,7 @@ let run () =
let repos =
Fake.Current.Primitive.const { Git_repositories.
opam_repository_master = Current_git.Commit_id.v ~repo:"opam_repository" ~gref:"master" ~hash:"master";
opam_repository_mingw_opam2 = Current_git.Commit_id.v ~repo:"opam_repository_mingw_opam2" ~gref:"master" ~hash:"master";
opam_2_0 = Current_git.Commit_id.v ~repo:"opam" ~gref:"2.0" ~hash:"2.0";
opam_2_1 = Current_git.Commit_id.v ~repo:"opam" ~gref:"2.1" ~hash:"2.1";
} in
Expand Down
16 changes: 12 additions & 4 deletions src/git_repositories.ml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,13 +16,15 @@ module Repositories = struct

type t = {
opam_repository_master : repo;
opam_repository_mingw_opam2 : repo;
Copy link
Contributor

@dra27 dra27 May 25, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

At some point later this year this will go - might it be better to have a list of repositories here?

Copy link
Contributor Author

@MisterDA MisterDA May 27, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think it doesn't matter much, if the patch is sufficiently clean it can simply be reverted, but maybe a list would help managing multiple repositories. Ok!

Copy link
Contributor Author

@MisterDA MisterDA May 27, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hm, no, it's simpler with a flat structure (no list), because the branch we want to checkout is different for each git repo (master for opam-repository, opam2 for opam-repository-mingw). With a list we won't be able to distinguish the repos, so we'd need to attach the branch to clone. It would induce a somewhat complex refactor, while reverting the commit is simple enough to remove the support for opam-repository-mingw.

opam_2_0 : repo;
opam_2_1 : repo;
}

let digest {opam_repository_master; opam_2_0; opam_2_1} =
let digest {opam_repository_master; opam_repository_mingw_opam2; opam_2_0; opam_2_1} =
let json = `Assoc [
"opam-repository__master", `String opam_repository_master;
"opam-repository-mingw__opam2", `String opam_repository_mingw_opam2;
"opam__2.0", `String opam_2_0;
"opam__2.1", `String opam_2_1;
] in
Expand All @@ -34,6 +36,7 @@ module Repositories = struct

type t = {
opam_repository_master : hash;
opam_repository_mingw_opam2 : hash;
opam_2_0 : hash;
opam_2_1 : hash;
} [@@deriving yojson]
Expand All @@ -52,12 +55,13 @@ module Repositories = struct
Current.Process.check_output ~cwd ~cancellable:true ~job ("", [|"git"; "rev-parse"; "HEAD"|]) >>!= fun hash ->
Lwt.return (Ok (String.trim hash))

let build No_context job {Key.opam_repository_master; opam_2_0; opam_2_1} =
let build No_context job {Key.opam_repository_master; opam_repository_mingw_opam2; opam_2_0; opam_2_1} =
Current.Job.start job ~level:Current.Level.Mostly_harmless >>= fun () ->
get_commit_hash ~job ~repo:opam_repository_master ~branch:"master" >>!= fun opam_repository_master ->
get_commit_hash ~job ~repo:opam_repository_mingw_opam2 ~branch:"opam2" >>!= fun opam_repository_mingw_opam2 ->
get_commit_hash ~job ~repo:opam_2_0 ~branch:"2.0" >>!= fun opam_2_0 ->
get_commit_hash ~job ~repo:opam_2_1 ~branch:"2.1" >>!= fun opam_2_1 ->
Lwt.return (Ok {Value.opam_repository_master; opam_2_0; opam_2_1})
Lwt.return (Ok {Value.opam_repository_master; opam_repository_mingw_opam2; opam_2_0; opam_2_1})

let pp f _ = Fmt.string f "Git repositories"

Expand All @@ -68,6 +72,7 @@ module Cache = Current_cache.Make(Repositories)

type t = {
opam_repository_master : Current_git.Commit_id.t;
opam_repository_mingw_opam2 : Current_git.Commit_id.t;
opam_2_0 : Current_git.Commit_id.t;
opam_2_1 : Current_git.Commit_id.t;
}
Expand All @@ -76,17 +81,20 @@ let get ~schedule =
let key = {
Repositories.Key.
opam_repository_master = "git://github.com/ocaml/opam-repository";
opam_repository_mingw_opam2 = "git://github.com/fdopen/opam-repository-mingw";
opam_2_0 = "git://github.com/ocaml/opam";
opam_2_1 = "git://github.com/ocaml/opam";
} in
let+ {Repositories.Value.opam_repository_master; opam_2_0; opam_2_1} =
let+ {Repositories.Value.opam_repository_master; opam_repository_mingw_opam2; opam_2_0; opam_2_1} =
Current.component "Git-repositories" |>
let> key = Current.return key in
Cache.get ~schedule Repositories.No_context key
in
{
opam_repository_master =
Current_git.Commit_id.v ~repo:key.opam_repository_master ~gref:"master" ~hash:opam_repository_master;
opam_repository_mingw_opam2 =
Current_git.Commit_id.v ~repo:key.opam_repository_mingw_opam2 ~gref:"opam2" ~hash:opam_repository_mingw_opam2;
opam_2_0 =
Current_git.Commit_id.v ~repo:key.opam_2_0 ~gref:"2.0" ~hash:opam_2_0;
opam_2_1 =
Expand Down
1 change: 1 addition & 0 deletions src/git_repositories.mli
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
type t = {
opam_repository_master : Current_git.Commit_id.t;
opam_repository_mingw_opam2 : Current_git.Commit_id.t;
opam_2_0 : Current_git.Commit_id.t;
opam_2_1 : Current_git.Commit_id.t;
}
Expand Down
Loading