Skip to content

Commit

Permalink
Incrementally copy features near the layer they're installed (#382)
Browse files Browse the repository at this point in the history 
* If available, use RUN bind mounts in each layer to reduce image size and avoid invalidating the entire build context when any feature changes

* ensure /tmp/build-features dir exists

* update test

* copy in devcontainer-features.builtin.env

* fix validate generateFeaturesConfig test

* use incremental COPY commands for non-buildkit builds
  • Loading branch information
@trxcllnt
trxcllnt authored Feb 7, 2023
1 parent 77172a2 commit d02379c
Show file tree
Hide file tree
Showing 3 changed files with 49 additions and 10 deletions.
41 changes: 36 additions & 5 deletions src/spec-configuration/containerFeaturesConfiguration.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -226,7 +226,7 @@ export function getContainerFeaturesBaseDockerFile() {
FROM $_DEV_CONTAINERS_BASE_IMAGE AS dev_containers_feature_content_normalize
USER root
COPY --from=dev_containers_feature_content_source {contentSourceRootPath} /tmp/build-features/
COPY --from=dev_containers_feature_content_source {contentSourceRootPath}/devcontainer-features.builtin.env /tmp/build-features/
RUN chmod -R 0700 /tmp/build-features
FROM $_DEV_CONTAINERS_BASE_IMAGE AS dev_containers_target_stage
Expand Down Expand Up @@ -312,7 +312,7 @@ function escapeQuotesForShell(input: string) {
return input.replace(new RegExp(`'`, 'g'), `'\\''`);
}

export function getFeatureLayers(featuresConfig: FeaturesConfig, containerUser: string, remoteUser: string) {
export function getFeatureLayers(featuresConfig: FeaturesConfig, containerUser: string, remoteUser: string, useBuildKitBuildContexts = false, contentSourceRootPath = '/tmp/build-features/') {
let result = `RUN \\
echo "_CONTAINER_USER_HOME=$(getent passwd ${containerUser} | cut -d: -f6)" >> /tmp/build-features/devcontainer-features.builtin.env && \\
echo "_REMOTE_USER_HOME=$(getent passwd ${remoteUser} | cut -d: -f6)" >> /tmp/build-features/devcontainer-features.builtin.env
Expand All @@ -322,22 +322,53 @@ echo "_REMOTE_USER_HOME=$(getent passwd ${remoteUser} | cut -d: -f6)" >> /tmp/bu
// Features version 1
const folders = (featuresConfig.featureSets || []).filter(y => y.internalVersion !== '2').map(x => x.features[0].consecutiveId);
folders.forEach(folder => {
result += `RUN cd /tmp/build-features/${folder} \\
const source = path.posix.join(contentSourceRootPath, folder!);
if (!useBuildKitBuildContexts) {
result += `COPY --chown=root:root --from=dev_containers_feature_content_source ${source} /tmp/build-features/${folder}
RUN chmod -R 0700 /tmp/build-features/${folder} \\
&& cd /tmp/build-features/${folder} \\
&& chmod +x ./install.sh \\
&& ./install.sh
`;
} else {
result += `RUN --mount=type=bind,from=dev_containers_feature_content_source,source=${source},target=/tmp/build-features-src/${folder} \\
cp -ar /tmp/build-features-src/${folder} /tmp/build-features/ \\
&& chmod -R 0700 /tmp/build-features/${folder} \\
&& cd /tmp/build-features/${folder} \\
&& chmod +x ./install.sh \\
&& ./install.sh \\
&& rm -rf /tmp/build-features/${folder}
`;
}
});
// Features version 2
featuresConfig.featureSets.filter(y => y.internalVersion === '2').forEach(featureSet => {
featureSet.features.forEach(feature => {
result += generateContainerEnvs(feature);
result += `
RUN cd /tmp/build-features/${feature.consecutiveId} \\
const source = path.posix.join(contentSourceRootPath, feature.consecutiveId!);
if (!useBuildKitBuildContexts) {
result += `
COPY --chown=root:root --from=dev_containers_feature_content_source ${source} /tmp/build-features/${feature.consecutiveId}
RUN chmod -R 0700 /tmp/build-features/${feature.consecutiveId} \\
&& cd /tmp/build-features/${feature.consecutiveId} \\
&& chmod +x ./devcontainer-features-install.sh \\
&& ./devcontainer-features-install.sh
`;
} else {
result += `
RUN --mount=type=bind,from=dev_containers_feature_content_source,source=${source},target=/tmp/build-features-src/${feature.consecutiveId} \\
cp -ar /tmp/build-features-src/${feature.consecutiveId} /tmp/build-features/ \\
&& chmod -R 0700 /tmp/build-features/${feature.consecutiveId} \\
&& cd /tmp/build-features/${feature.consecutiveId} \\
&& chmod +x ./devcontainer-features-install.sh \\
&& ./devcontainer-features-install.sh \\
&& rm -rf /tmp/build-features/${feature.consecutiveId}
`;
}
});
});
return result;
Expand Down
2 changes: 1 addition & 1 deletion src/spec-node/containerFeatures.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -289,7 +289,7 @@ async function getFeaturesBuildOptions(params: DockerResolverParameters, devCont
.replace('#{nonBuildKitFeatureContentFallback}', useBuildKitBuildContexts ? '' : `FROM ${buildContentImageName} as dev_containers_feature_content_source`)
.replace('{contentSourceRootPath}', contentSourceRootPath)
.replace('#{featureBuildStages}', getFeatureBuildStages(featuresConfig, buildStageScripts, contentSourceRootPath))
.replace('#{featureLayer}', getFeatureLayers(featuresConfig, containerUser, remoteUser))
.replace('#{featureLayer}', getFeatureLayers(featuresConfig, containerUser, remoteUser, useBuildKitBuildContexts, contentSourceRootPath))
.replace('#{containerEnv}', generateContainerEnvs(featuresConfig))
.replace('#{copyFeatureBuildStages}', getCopyFeatureBuildStages(featuresConfig, buildStageScripts))
.replace('#{devcontainerMetadata}', getDevcontainerMetadataLabel(imageMetadata, common.experimentalImageMetadata))
Expand Down
16 changes: 12 additions & 4 deletions src/test/container-features/generateFeaturesConfig.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,11 +75,15 @@ describe('validate generateFeaturesConfig()', function () {
echo "_CONTAINER_USER_HOME=$(getent passwd testContainerUser | cut -d: -f6)" >> /tmp/build-features/devcontainer-features.builtin.env && \\
echo "_REMOTE_USER_HOME=$(getent passwd testRemoteUser | cut -d: -f6)" >> /tmp/build-features/devcontainer-features.builtin.env
RUN cd /tmp/build-features/first_1 \\
COPY --chown=root:root --from=dev_containers_feature_content_source /tmp/build-features/first_1 /tmp/build-features/first_1
RUN chmod -R 0700 /tmp/build-features/first_1 \\
&& cd /tmp/build-features/first_1 \\
&& chmod +x ./install.sh \\
&& ./install.sh
RUN cd /tmp/build-features/second_2 \\
COPY --chown=root:root --from=dev_containers_feature_content_source /tmp/build-features/second_2 /tmp/build-features/second_2
RUN chmod -R 0700 /tmp/build-features/second_2 \\
&& cd /tmp/build-features/second_2 \\
&& chmod +x ./install.sh \\
&& ./install.sh
Expand Down Expand Up @@ -134,12 +138,16 @@ echo "_CONTAINER_USER_HOME=$(getent passwd testContainerUser | cut -d: -f6)" >>
echo "_REMOTE_USER_HOME=$(getent passwd testRemoteUser | cut -d: -f6)" >> /tmp/build-features/devcontainer-features.builtin.env
RUN cd /tmp/build-features/color_3 \\
COPY --chown=root:root --from=dev_containers_feature_content_source /tmp/build-features/color_3 /tmp/build-features/color_3
RUN chmod -R 0700 /tmp/build-features/color_3 \\
&& cd /tmp/build-features/color_3 \\
&& chmod +x ./devcontainer-features-install.sh \\
&& ./devcontainer-features-install.sh
RUN cd /tmp/build-features/hello_4 \\
COPY --chown=root:root --from=dev_containers_feature_content_source /tmp/build-features/hello_4 /tmp/build-features/hello_4
RUN chmod -R 0700 /tmp/build-features/hello_4 \\
&& cd /tmp/build-features/hello_4 \\
&& chmod +x ./devcontainer-features-install.sh \\
&& ./devcontainer-features-install.sh
Expand Down

0 comments on commit d02379c

Please sign in to comment.