Skip to content

Commit

Permalink
Module Lockfile
Browse files Browse the repository at this point in the history 
- Created lockfile K/F/V holding module information and its related flags
- Updated BazelDepGraph function to use the lockfile
- Created type adapters for module, related classes & immutable collections
- Updated tests with Module.bazel to work after enabling the lockfile
- Added new java test class for the lockfile
- Added a flag to enable/disable lockfile (while it's still in progress)
- Added integration tests

PiperOrigin-RevId: 520925174
Change-Id: Ib8da99aa86a1797da44dc9a899d932f58c4a84c9
  • Loading branch information
@SalmaSamy @copybara-github
SalmaSamy authored and copybara-github committed Mar 31, 2023
1 parent 4c85356 commit 3b9ec35
Show file tree
Hide file tree
Showing 28 changed files with 1,181 additions and 36 deletions.
8 changes: 7 additions & 1 deletion src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import com.google.devtools.build.lib.analysis.RuleDefinition;
import com.google.devtools.build.lib.analysis.config.BuildConfigurationValue;
import com.google.devtools.build.lib.bazel.bzlmod.BazelDepGraphFunction;
import com.google.devtools.build.lib.bazel.bzlmod.BazelLockFileFunction;
import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleInspectorFunction;
import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleInspectorValue.AugmentedModule.ResolutionReason;
import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleResolutionFunction;
Expand Down Expand Up @@ -249,7 +250,12 @@ public ResolutionReason getResolutionReason() {
.addSkyFunction(
SkyFunctions.MODULE_FILE,
new ModuleFileFunction(registryFactory, directories.getWorkspace(), builtinModules))
.addSkyFunction(SkyFunctions.BAZEL_DEP_GRAPH, new BazelDepGraphFunction())
.addSkyFunction(
SkyFunctions.BAZEL_DEP_GRAPH,
new BazelDepGraphFunction(directories.getWorkspace(), registryFactory))
.addSkyFunction(
SkyFunctions.BAZEL_LOCK_FILE,
new BazelLockFileFunction(directories.getWorkspace(), registryFactory))
.addSkyFunction(SkyFunctions.BAZEL_MODULE_INSPECTION, new BazelModuleInspectorFunction())
.addSkyFunction(SkyFunctions.BAZEL_MODULE_RESOLUTION, new BazelModuleResolutionFunction())
.addSkyFunction(SkyFunctions.SINGLE_EXTENSION_EVAL, singleExtensionEvalFunction)
Expand Down
15 changes: 12 additions & 3 deletions src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ java_library(
"//src/main/java/com/google/devtools/build/lib/cmdline",
"//src/main/java/net/starlark/java/eval",
"//third_party:auto_value",
"//third_party:gson",
"//third_party:guava",
"//third_party:jsr305",
],
Expand Down Expand Up @@ -52,14 +53,13 @@ java_library(
deps = [
"//src/main/java/com/google/devtools/build/docgen/annot",
"//src/main/java/com/google/devtools/build/lib/cmdline",
"//src/main/java/com/google/devtools/build/lib/events",
"//src/main/java/com/google/devtools/build/lib/packages",
"//src/main/java/net/starlark/java/annot",
"//src/main/java/net/starlark/java/eval",
"//src/main/java/net/starlark/java/syntax",
"//third_party:auto_value",
"//third_party:gson",
"//third_party:guava",
"//third_party:jsr305",
],
)

Expand Down Expand Up @@ -88,7 +88,9 @@ java_library(
"AbridgedModule.java",
"ArchiveOverride.java",
"BazelDepGraphValue.java",
"BazelLockFileValue.java",
"BazelModuleResolutionValue.java",
"BzlmodFlagsAndEnvVars.java",
"GitOverride.java",
"LocalPathOverride.java",
"Module.java",
Expand All @@ -110,7 +112,6 @@ java_library(
":repo_rule_creator",
"//src/main/java/com/google/devtools/build/lib/analysis:blaze_directories",
"//src/main/java/com/google/devtools/build/lib/cmdline",
"//src/main/java/com/google/devtools/build/lib/concurrent",
"//src/main/java/com/google/devtools/build/lib/events",
"//src/main/java/com/google/devtools/build/lib/packages",
"//src/main/java/com/google/devtools/build/lib/skyframe:sky_functions",
Expand All @@ -120,6 +121,7 @@ java_library(
"//src/main/java/net/starlark/java/eval",
"//src/main/java/net/starlark/java/syntax",
"//third_party:auto_value",
"//third_party:gson",
"//third_party:guava",
"//third_party:jsr305",
],
Expand All @@ -129,8 +131,12 @@ java_library(
name = "resolution_impl",
srcs = [
"BazelDepGraphFunction.java",
"BazelLockFileFunction.java",
"BazelModuleResolutionFunction.java",
"BzlmodFlagsAndEnvVars.java",
"DelegateTypeAdapterFactory.java",
"Discovery.java",
"GsonTypeAdapterUtil.java",
"ModuleExtensionContext.java",
"ModuleFileFunction.java",
"ModuleFileGlobals.java",
Expand Down Expand Up @@ -158,13 +164,15 @@ java_library(
"//src/main/java/com/google/devtools/build/lib/cmdline",
"//src/main/java/com/google/devtools/build/lib/events",
"//src/main/java/com/google/devtools/build/lib/packages",
"//src/main/java/com/google/devtools/build/lib/packages/semantics",
"//src/main/java/com/google/devtools/build/lib/rules:repository/repository_directory_value",
"//src/main/java/com/google/devtools/build/lib/rules:repository/repository_function",
"//src/main/java/com/google/devtools/build/lib/skyframe:bzl_load_value",
"//src/main/java/com/google/devtools/build/lib/skyframe:client_environment_function",
"//src/main/java/com/google/devtools/build/lib/skyframe:client_environment_value",
"//src/main/java/com/google/devtools/build/lib/skyframe:precomputed_value",
"//src/main/java/com/google/devtools/build/lib/skyframe:skyframe_cluster",
"//src/main/java/com/google/devtools/build/lib/util",
"//src/main/java/com/google/devtools/build/lib/vfs",
"//src/main/java/com/google/devtools/build/lib/vfs:pathfragment",
"//src/main/java/com/google/devtools/build/skyframe",
Expand All @@ -175,6 +183,7 @@ java_library(
"//src/main/java/net/starlark/java/syntax",
"//src/main/protobuf:failure_details_java_proto",
"//third_party:auto_value",
"//third_party:gson",
"//third_party:guava",
"//third_party:jsr305",
],
Expand Down
142 changes: 121 additions & 21 deletions src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,26 +15,36 @@

package com.google.devtools.build.lib.bazel.bzlmod;

import static com.google.common.base.Strings.nullToEmpty;
import static com.google.common.collect.ImmutableList.toImmutableList;
import static com.google.common.collect.ImmutableMap.toImmutableMap;

import com.google.common.collect.BiMap;
import com.google.common.collect.HashBiMap;
import com.google.common.collect.ImmutableBiMap;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableTable;
import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleResolutionFunction.BazelModuleResolutionFunctionException;
import com.google.devtools.build.lib.bazel.bzlmod.ModuleFileValue.RootModuleFileValue;
import com.google.devtools.build.lib.cmdline.LabelSyntaxException;
import com.google.devtools.build.lib.cmdline.PackageIdentifier;
import com.google.devtools.build.lib.cmdline.RepositoryName;
import com.google.devtools.build.lib.packages.LabelConverter;
import com.google.devtools.build.lib.packages.semantics.BuildLanguageOptions;
import com.google.devtools.build.lib.server.FailureDetails.ExternalDeps.Code;
import com.google.devtools.build.lib.skyframe.ClientEnvironmentFunction;
import com.google.devtools.build.lib.skyframe.ClientEnvironmentValue;
import com.google.devtools.build.lib.skyframe.PrecomputedValue;
import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.lib.vfs.PathFragment;
import com.google.devtools.build.skyframe.SkyFunction;
import com.google.devtools.build.skyframe.SkyFunctionException;
import com.google.devtools.build.skyframe.SkyFunctionException.Transience;
import com.google.devtools.build.skyframe.SkyKey;
import com.google.devtools.build.skyframe.SkyValue;
import javax.annotation.Nullable;
import net.starlark.java.eval.StarlarkSemantics;

/**
* This function runs Bazel module resolution, extracts the dependency graph from it and creates a
Expand All @@ -43,24 +53,123 @@
*/
public class BazelDepGraphFunction implements SkyFunction {

private final Path rootDirectory;
private final RegistryFactory registryFactory;

public BazelDepGraphFunction(Path rootDirectory, RegistryFactory registryFactory) {
this.rootDirectory = rootDirectory;
this.registryFactory = registryFactory;
}

@Override
@Nullable
public SkyValue compute(SkyKey skyKey, Environment env)
throws SkyFunctionException, InterruptedException {

BazelModuleResolutionValue selectionResult =
(BazelModuleResolutionValue) env.getValue(BazelModuleResolutionValue.KEY);
if (env.valuesMissing()) {
RootModuleFileValue root =
(RootModuleFileValue) env.getValue(ModuleFileValue.KEY_FOR_ROOT_MODULE);
if (root == null) {
return null;
}
StarlarkSemantics starlarkSemantics = PrecomputedValue.STARLARK_SEMANTICS.get(env);
if (starlarkSemantics == null) {
return null;
}

ImmutableMap<ModuleKey, Module> depGraph = null;
BzlmodFlagsAndEnvVars flags = null;

// If the module has not changed (has the same contents and flags as the lockfile),
// read the dependency graph from the lock file, else run resolution and update lockfile
if (starlarkSemantics.getBool(BuildLanguageOptions.ENABLE_LOCKFILE)) {
BazelLockFileValue lockFile = (BazelLockFileValue) env.getValue(BazelLockFileValue.KEY);
if (lockFile == null) {
return null;
}
flags = getFlagsAndEnvVars(env);
if (flags == null) { // unable to read environment variables
return null;
}
if (root.getModuleFileHash().equals(lockFile.getModuleFileHash())
&& flags.equals(lockFile.getFlags())) {
depGraph = lockFile.getModuleDepGraph();
}
}

if (depGraph == null) {
BazelModuleResolutionValue selectionResult =
(BazelModuleResolutionValue) env.getValue(BazelModuleResolutionValue.KEY);
if (env.valuesMissing()) {
return null;
}
depGraph = selectionResult.getResolvedDepGraph();
if (starlarkSemantics.getBool(BuildLanguageOptions.ENABLE_LOCKFILE)) {
BazelLockFileFunction.updateLockedModule(
root.getModuleFileHash(), depGraph, rootDirectory, registryFactory, flags);
}
}

ImmutableMap<ModuleKey, Module> depGraph = selectionResult.getResolvedDepGraph();
ImmutableMap<RepositoryName, ModuleKey> canonicalRepoNameLookup =
depGraph.keySet().stream()
.collect(toImmutableMap(ModuleKey::getCanonicalRepoName, key -> key));

// For each extension usage, we resolve (i.e. canonicalize) its bzl file label. Then we can
// group all usages by the label + name (the ModuleExtensionId).
ImmutableTable<ModuleExtensionId, ModuleKey, ModuleExtensionUsage> extensionUsagesById =
getExtensionUsagesById(depGraph);

ImmutableBiMap<String, ModuleExtensionId> extensionUniqueNames =
calculateUniqueNameForUsedExtensionId(extensionUsagesById);

return BazelDepGraphValue.create(
depGraph,
canonicalRepoNameLookup,
depGraph.values().stream().map(AbridgedModule::from).collect(toImmutableList()),
extensionUsagesById,
extensionUniqueNames.inverse());
}

@Nullable
public static BzlmodFlagsAndEnvVars getFlagsAndEnvVars(Environment env)
throws InterruptedException {
ClientEnvironmentValue allowedYankedVersionsFromEnv =
(ClientEnvironmentValue)
env.getValue(
ClientEnvironmentFunction.key(
BazelModuleResolutionFunction.BZLMOD_ALLOWED_YANKED_VERSIONS_ENV));
if (allowedYankedVersionsFromEnv == null) {
return null;
}

ImmutableList<String> registries = ImmutableList.copyOf(ModuleFileFunction.REGISTRIES.get(env));
ImmutableMap<String, String> moduleOverrides =
ModuleFileFunction.MODULE_OVERRIDES.get(env).entrySet().stream()
.collect(
toImmutableMap(e -> e.getKey(), e -> ((LocalPathOverride) e.getValue()).getPath()));

ImmutableList<String> yankedVersions =
ImmutableList.copyOf(BazelModuleResolutionFunction.ALLOWED_YANKED_VERSIONS.get(env));
Boolean ignoreDevDeps = ModuleFileFunction.IGNORE_DEV_DEPS.get(env);
String compatabilityMode =
BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE.get(env).name();
String directDepsMode = BazelModuleResolutionFunction.CHECK_DIRECT_DEPENDENCIES.get(env).name();

String envYanked = allowedYankedVersionsFromEnv.getValue();

return BzlmodFlagsAndEnvVars.create(
registries,
moduleOverrides,
yankedVersions,
nullToEmpty(envYanked),
ignoreDevDeps,
directDepsMode,
compatabilityMode);
}

/**
* For each extension usage, we resolve (i.e. canonicalize) its bzl file label. Then we can group
* all usages by the label + name (the ModuleExtensionId).
*/
private ImmutableTable<ModuleExtensionId, ModuleKey, ModuleExtensionUsage> getExtensionUsagesById(
ImmutableMap<ModuleKey, Module> depGraph) throws BazelModuleResolutionFunctionException {
ImmutableTable.Builder<ModuleExtensionId, ModuleKey, ModuleExtensionUsage>
extensionUsagesTableBuilder = ImmutableTable.builder();
for (Module module : depGraph.values()) {
Expand Down Expand Up @@ -95,21 +204,18 @@ public SkyValue compute(SkyKey skyKey, Environment env)
extensionUsagesTableBuilder.put(moduleExtensionId, module.getKey(), usage);
}
}
ImmutableTable<ModuleExtensionId, ModuleKey, ModuleExtensionUsage> extensionUsagesById =
extensionUsagesTableBuilder.buildOrThrow();
return extensionUsagesTableBuilder.buildOrThrow();
}

private ImmutableBiMap<String, ModuleExtensionId> calculateUniqueNameForUsedExtensionId(
ImmutableTable<ModuleExtensionId, ModuleKey, ModuleExtensionUsage> extensionUsagesById) {
// Calculate a unique name for each used extension id.
BiMap<String, ModuleExtensionId> extensionUniqueNames = HashBiMap.create();
for (ModuleExtensionId id : extensionUsagesById.rowKeySet()) {
// Ensure that the resulting extension name (and thus the repository names derived from it) do
// not start with a tilde.
RepositoryName repository = id.getBzlFileLabel().getRepository();
String nonEmptyRepoPart;
if (repository.isMain()) {
nonEmptyRepoPart = "_main";
} else {
nonEmptyRepoPart = repository.getName();
}
String nonEmptyRepoPart = repository.isMain() ? "_main" : repository.getName();
String bestName = nonEmptyRepoPart + "~" + id.getExtensionName();
if (extensionUniqueNames.putIfAbsent(bestName, id) == null) {
continue;
Expand All @@ -119,12 +225,6 @@ public SkyValue compute(SkyKey skyKey, Environment env)
suffix++;
}
}

return BazelDepGraphValue.create(
depGraph,
canonicalRepoNameLookup,
depGraph.values().stream().map(AbridgedModule::from).collect(toImmutableList()),
extensionUsagesById,
ImmutableMap.copyOf(extensionUniqueNames.inverse()));
return ImmutableBiMap.copyOf(extensionUniqueNames);
}
}
Loading

0 comments on commit 3b9ec35

Please sign in to comment.