Skip to content

Commit

Permalink
Merge pull request #1672 from pypa/disable-host-mount
Browse files Browse the repository at this point in the history
feat: Adds disable_host_mount suboption to container-engine
  • Loading branch information
joerick authored Jan 20, 2024
2 parents 93542c3 + c18f6cb commit 23a6e88
Show file tree
Hide file tree
Showing 10 changed files with 107 additions and 22 deletions.
6 changes: 4 additions & 2 deletions bin/generate_schema.py
Original file line number Diff line number Diff line change
Expand Up @@ -66,9 +66,9 @@
oneOf:
- enum: [docker, podman]
- type: string
pattern: '^docker; ?create_args:'
pattern: '^docker; ?(create_args|disable_host_mount):'
- type: string
pattern: '^podman; ?create_args:'
pattern: '^podman; ?(create_args|disable_host_mount):'
- type: object
additionalProperties: false
required: [name]
Expand All @@ -79,6 +79,8 @@
type: array
items:
type: string
disable-host-mount:
type: boolean
dependency-versions:
default: pinned
description: Specify how cibuildwheel controls the versions of the tools it uses
Expand Down
26 changes: 21 additions & 5 deletions cibuildwheel/oci_container.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,27 +33,43 @@
class OCIContainerEngineConfig:
name: ContainerEngineName
create_args: Sequence[str] = ()
disable_host_mount: bool = False

@staticmethod
def from_config_string(config_string: str) -> OCIContainerEngineConfig:
config_dict = parse_key_value_string(
config_string, ["name"], ["create_args", "create-args"]
config_string,
["name"],
["create_args", "create-args", "disable_host_mount", "disable-host-mount"],
)
name = " ".join(config_dict["name"])
if name not in {"docker", "podman"}:
msg = f"unknown container engine {name}"
raise ValueError(msg)

name = typing.cast(ContainerEngineName, name)
# some flexibility in the option name to cope with TOML conventions
# some flexibility in the option names to cope with TOML conventions
create_args = config_dict.get("create_args") or config_dict.get("create-args") or []
return OCIContainerEngineConfig(name=name, create_args=create_args)
disable_host_mount_options = (
config_dict.get("disable_host_mount") or config_dict.get("disable-host-mount") or []
)
disable_host_mount = (
strtobool(disable_host_mount_options[-1]) if disable_host_mount_options else False
)

return OCIContainerEngineConfig(
name=name, create_args=create_args, disable_host_mount=disable_host_mount
)

def options_summary(self) -> str | dict[str, str]:
if not self.create_args:
return self.name
else:
return {"name": self.name, "create_args": repr(self.create_args)}
return {
"name": self.name,
"create_args": repr(self.create_args),
"disable_host_mount": str(self.disable_host_mount),
}


DEFAULT_ENGINE = OCIContainerEngineConfig("docker")
Expand Down Expand Up @@ -137,7 +153,7 @@ def __enter__(self) -> Self:
"--env=SOURCE_DATE_EPOCH",
f"--name={self.name}",
"--interactive",
"--volume=/:/host", # ignored on CircleCI
*(["--volume=/:/host"] if not self.engine.disable_host_mount else []),
*network_args,
*self.engine.create_args,
self.image,
Expand Down
3 changes: 3 additions & 0 deletions cibuildwheel/options.py
Original file line number Diff line number Diff line change
Expand Up @@ -378,6 +378,9 @@ def _inner_fmt(k: str, v: Any, table: TableFmt) -> Iterator[str]:
for inner_v in v:
qv = quote_function(inner_v)
yield table["item"].format(k=k, v=qv)
elif isinstance(v, bool):
qv = quote_function(str(v))
yield table["item"].format(k=k, v=qv)
else:
qv = quote_function(v)
yield table["item"].format(k=k, v=qv)
Expand Down
7 changes: 5 additions & 2 deletions cibuildwheel/resources/cibuildwheel.schema.json
Original file line number Diff line number Diff line change
Expand Up @@ -172,11 +172,11 @@
},
{
"type": "string",
"pattern": "^docker; ?create_args:"
"pattern": "^docker; ?(create_args|disable_host_mount):"
},
{
"type": "string",
"pattern": "^podman; ?create_args:"
"pattern": "^podman; ?(create_args|disable_host_mount):"
},
{
"type": "object",
Expand All @@ -196,6 +196,9 @@
"items": {
"type": "string"
}
},
"disable-host-mount": {
"type": "boolean"
}
}
}
Expand Down
4 changes: 2 additions & 2 deletions cibuildwheel/util.py
Original file line number Diff line number Diff line change
Expand Up @@ -738,7 +738,7 @@ def parse_key_value_string(
# split by semicolon
fields = [list(group) for k, group in itertools.groupby(parts, lambda x: x == ";") if not k]

result: dict[str, list[str]] = defaultdict(list)
result: defaultdict[str, list[str]] = defaultdict(list)
for field_i, field in enumerate(fields):
# check to see if the option name is specified
field_name, sep, first_value = field[0].partition(":")
Expand All @@ -759,4 +759,4 @@ def parse_key_value_string(

result[field_name] += values

return result
return dict(result)
6 changes: 6 additions & 0 deletions docs/extra.css
Original file line number Diff line number Diff line change
Expand Up @@ -315,3 +315,9 @@ h1, h2, h3, h4, h5, h6 {
.wy-menu-vertical li.current a {

}

/* word wrap in table cells */
.wy-table-responsive table td, .wy-table-responsive table th {
white-space: normal;
line-height: 1.4;
}
2 changes: 1 addition & 1 deletion docs/faq.md
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ Linux wheels are built in [`manylinux`/`musllinux` containers](https://github.co

`cibuildwheel` supports this by providing the [`CIBW_ENVIRONMENT`](options.md#environment) and [`CIBW_BEFORE_ALL`](options.md#before-all) options to setup the build environment inside the running container.

- The project directory is copied into the container as `/project`, the output directory for the wheels to be copied out is `/output`. In general, this is handled transparently by `cibuildwheel`. For a more finegrained level of control however, the root of the host file system is mounted as `/host`, allowing for example to access shared files, caches, etc. on the host file system. Note that `/host` is not available on CircleCI due to their Docker policies.
- The project directory is copied into the container as `/project`, the output directory for the wheels to be copied out is `/output`. In general, this is handled transparently by `cibuildwheel`. For a more finegrained level of control however, the root of the host file system is mounted as `/host`, allowing for example to access shared files, caches, etc. on the host file system. Note that `/host` is not available on CircleCI and GitLab CI due to their Docker policies.

- Alternative Docker images can be specified with the `CIBW_MANYLINUX_*_IMAGE`/`CIBW_MUSLLINUX_*_IMAGE` options to allow for a custom, preconfigured build environment for the Linux builds. See [options](options.md#linux-image) for more details.

Expand Down
22 changes: 15 additions & 7 deletions docs/options.md
Original file line number Diff line number Diff line change
Expand Up @@ -1071,8 +1071,8 @@ Auditwheel detects the version of the manylinux / musllinux standard in the imag
Options:

- `docker[;create_args: ...]`
- `podman[;create_args: ...]`
- `docker[;create_args: ...][;disable_host_mount: true/false]`
- `podman[;create_args: ...][;disable_host_mount: true/false]`

Default: `docker`

Expand All @@ -1081,11 +1081,13 @@ Set the container engine to use. Docker is the default, or you can switch to
running and `docker` available on PATH. To use Podman, it needs to be
installed and `podman` available on PATH.

Arguments can be supplied to the container engine. Currently, the only option
that's customisable is 'create_args'. Parameters to create_args are
space-separated strings, which are passed to the container engine on the
command line when it's creating the container. If you want to include spaces
inside a parameter, use shell-style quoting.
Options can be supplied after the name.

| Option name | Description
|---|---
| `create_args` | Space-separated strings, which are passed to the container engine on the command line when it's creating the container. If you want to include spaces inside a parameter, use shell-style quoting.
| `disable_host_mount` | By default, cibuildwheel will mount the root of the host filesystem as a volume at `/host` in the container. To disable the host mount, pass `true` to this option.


!!! tip

Expand All @@ -1106,6 +1108,9 @@ inside a parameter, use shell-style quoting.

# pass command line options to 'docker create'
CIBW_CONTAINER_ENGINE: "docker; create_args: --gpus all"

# disable the /host mount
CIBW_CONTAINER_ENGINE: "docker; disable_host_mount: true"
```

!!! tab examples "pyproject.toml"
Expand All @@ -1117,6 +1122,9 @@ inside a parameter, use shell-style quoting.

# pass command line options to 'docker create'
container-engine = { name = "docker", create-args = ["--gpus", "all"]}

# disable the /host mount
container-engine = { name = "docker", disable-host-mount = true }
```


Expand Down
28 changes: 27 additions & 1 deletion unit_test/oci_container_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@

from cibuildwheel.environment import EnvironmentAssignmentBash
from cibuildwheel.oci_container import OCIContainer, OCIContainerEngineConfig
from cibuildwheel.util import detect_ci_provider
from cibuildwheel.util import CIProvider, detect_ci_provider

# Test utilities

Expand Down Expand Up @@ -415,3 +415,29 @@ def test_enforce_32_bit(container_engine, image, shell_args):
text=True,
).stdout
assert json.loads(container_args) == shell_args


@pytest.mark.parametrize(
("config", "should_have_host_mount"),
[
("{name}", True),
("{name}; disable_host_mount: false", True),
("{name}; disable_host_mount: true", False),
],
)
def test_disable_host_mount(tmp_path: Path, container_engine, config, should_have_host_mount):
if detect_ci_provider() in {CIProvider.circle_ci, CIProvider.gitlab}:
pytest.skip("Skipping test because docker on this platform does not support host mounts")

engine = OCIContainerEngineConfig.from_config_string(config.format(name=container_engine.name))

sentinel_file = tmp_path / "sentinel"
sentinel_file.write_text("12345")

with OCIContainer(engine=engine, image=DEFAULT_IMAGE) as container:
host_mount_path = "/host" + str(sentinel_file)
if should_have_host_mount:
assert container.call(["cat", host_mount_path], capture_output=True) == "12345"
else:
with pytest.raises(subprocess.CalledProcessError):
container.call(["cat", host_mount_path], capture_output=True)
25 changes: 23 additions & 2 deletions unit_test/options_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -201,41 +201,61 @@ def test_toml_environment_quoting(tmp_path: Path, toml_assignment, result_value)


@pytest.mark.parametrize(
("toml_assignment", "result_name", "result_create_args"),
("toml_assignment", "result_name", "result_create_args", "result_disable_host_mount"),
[
(
'container-engine = "podman"',
"podman",
[],
False,
),
(
'container-engine = {name = "podman"}',
"podman",
[],
False,
),
(
'container-engine = "docker; create_args: --some-option"',
"docker",
["--some-option"],
False,
),
(
'container-engine = {name = "docker", create-args = ["--some-option"]}',
"docker",
["--some-option"],
False,
),
(
'container-engine = {name = "docker", create-args = ["--some-option", "value that contains spaces"]}',
"docker",
["--some-option", "value that contains spaces"],
False,
),
(
'container-engine = {name = "docker", create-args = ["--some-option", "value;that;contains;semicolons"]}',
"docker",
["--some-option", "value;that;contains;semicolons"],
False,
),
(
'container-engine = {name = "docker", disable-host-mount = true}',
"docker",
[],
True,
),
(
'container-engine = {name = "docker", disable_host_mount = true}',
"docker",
[],
True,
),
],
)
def test_container_engine_option(tmp_path: Path, toml_assignment, result_name, result_create_args):
def test_container_engine_option(
tmp_path: Path, toml_assignment, result_name, result_create_args, result_disable_host_mount
):
args = CommandLineArguments.defaults()
args.package_dir = tmp_path

Expand All @@ -253,6 +273,7 @@ def test_container_engine_option(tmp_path: Path, toml_assignment, result_name, r

assert parsed_container_engine.name == result_name
assert parsed_container_engine.create_args == result_create_args
assert parsed_container_engine.disable_host_mount == result_disable_host_mount


def test_environment_pass_references():
Expand Down

0 comments on commit 23a6e88

Please sign in to comment.