Sb bitbake multi layer recipes #516
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,23 @@ | ||
| package com.synopsys.integration.detectable.detectables.bitbake; | ||
|
|
||
| import java.util.List; | ||
| import java.util.Map; | ||
| import java.util.Set; | ||
|
|
||
| public class ShowRecipesResults { | ||
| private final Set<String> layerNames; | ||
| private final Map<String, List<String>> recipesWithLayers; | ||
|
|
||
| public ShowRecipesResults(final Set<String> layerNames, Map<String, List<String>> recipesWithLayers) { | ||
| this.layerNames = layerNames; | ||
| this.recipesWithLayers = recipesWithLayers; | ||
| } | ||
|
|
||
| public Set<String> getLayerNames() { | ||
| return layerNames; | ||
| } | ||
|
|
||
| public Map<String, List<String>> getRecipesWithLayers() { | ||
| return recipesWithLayers; | ||
| } | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,10 +1,12 @@ | ||
| package com.synopsys.integration.detectable.detectables.bitbake.parse; | ||
|
|
||
| import java.util.HashMap; | ||
| import java.util.List; | ||
| import java.util.Map; | ||
| import java.util.Optional; | ||
|
|
||
| import org.jetbrains.annotations.NotNull; | ||
| import org.jetbrains.annotations.Nullable; | ||
| import org.slf4j.Logger; | ||
| import org.slf4j.LoggerFactory; | ||
|
|
||
|
|
@@ -33,21 +35,22 @@ public BitbakeGraphTransformer(ExternalIdFactory externalIdFactory, EnumListFilt | |
| this.dependencyTypeFilter = dependencyTypeFilter; | ||
| } | ||
|
|
||
| public DependencyGraph transform(BitbakeGraph bitbakeGraph, Map<String, List<String>> recipeLayerMap, Map<String, String> imageRecipes) { | ||
| Map<String, Dependency> namesToExternalIds = generateExternalIds(bitbakeGraph, recipeLayerMap, imageRecipes); | ||
| return buildGraph(bitbakeGraph, namesToExternalIds); | ||
| } | ||
|
|
||
| @NotNull | ||
| private Map<String, Dependency> generateExternalIds(BitbakeGraph bitbakeGraph, Map<String, List<String>> recipeLayerMap, Map<String, String> imageRecipes) { | ||
| Map<String, Dependency> namesToExternalIds = new HashMap<>(); | ||
| for (BitbakeNode bitbakeNode : bitbakeGraph.getNodes()) { | ||
| String name = bitbakeNode.getName(); | ||
|
|
||
| if (bitbakeNode.getVersion().isPresent()) { | ||
| String version = bitbakeNode.getVersion().get(); | ||
| Optional<String> actualLayer = bitbakeNode.getLayer(); | ||
| if (dependencyTypeFilter.shouldInclude(BitbakeDependencyType.BUILD) || !isBuildDependency(imageRecipes, name, version)) { | ||
| Optional<Dependency> dependency = generateExternalId(name, version, actualLayer.orElse(null), recipeLayerMap).map(Dependency::new); | ||
| dependency.ifPresent(value -> namesToExternalIds.put(bitbakeNode.getName(), value)); | ||
| } | ||
| } else if (name.startsWith(VIRTUAL_PREFIX)) { | ||
|
|
@@ -117,20 +120,20 @@ private String removeEpochPrefix(String recipeVersion) { | |
| return epochlessRecipeVersion; | ||
| } | ||
|
|
||
| private Optional<ExternalId> generateExternalId(String dependencyName, String dependencyVersion, @Nullable String dependencyLayer, Map<String, List<String>> recipeLayerMap) { | ||
| List<String> recipeLayerNames = recipeLayerMap.get(dependencyName); | ||
| ExternalId externalId = null; | ||
| if (recipeLayerNames != null) { | ||
| dependencyLayer = chooseRecipeLayer(dependencyName, dependencyLayer, recipeLayerNames); | ||
| externalId = externalIdFactory.createYoctoExternalId(dependencyLayer, dependencyName, dependencyVersion); | ||
| } else { | ||
| logger.debug("Failed to find component '{}' in component layer map. [dependencyVersion: {}; dependencyLayer: {}", dependencyName, dependencyVersion, dependencyLayer); | ||
| if (dependencyName.endsWith(NATIVE_SUFFIX)) { | ||
| String alternativeName = dependencyName.replace(NATIVE_SUFFIX, ""); | ||
| logger.debug("Generating alternative component name '{}' for '{}=={}'", alternativeName, dependencyName, dependencyVersion); | ||
| externalId = generateExternalId(alternativeName, dependencyVersion, dependencyLayer, recipeLayerMap).orElse(null); | ||
| } else { | ||
| logger.debug("'{}:{}' is not an actual component. Excluding from graph.", dependencyName, dependencyVersion); | ||
| } | ||
| } | ||
|
|
||
|
|
@@ -140,4 +143,14 @@ private Optional<ExternalId> generateExternalId(String dependencyName, String de | |
|
|
||
| return Optional.ofNullable(externalId); | ||
| } | ||
|
|
||
| private String chooseRecipeLayer(final String dependencyName, @Nullable String dependencyLayer, final List<String> recipeLayerNames) { | ||
| if (dependencyLayer == null) { | ||
| logger.warn("Did not parse a layer for dependency {} from task-depends.dot; falling back to layer {} (first from show-recipes output)", dependencyName, recipeLayerNames.get(0)); | ||
| dependencyLayer = recipeLayerNames.get(0); | ||
| } else { | ||
| logger.trace("For dependency recipe {}: using layer {} parsed from task-depends.dot", dependencyName, dependencyLayer); | ||
| } | ||
| return dependencyLayer; | ||
| } | ||
| } | ||
|
Contributor
Author
There was a problem hiding this comment. I'm half tempted to remove this fallback-to-first-in-list. I'm hoping it's never used. If we did remove it, we could execute bitbake-layers show-layers instead of bitbake-layers show-recipes, and do a whole lot less parsing. I'm hoping that after reading a few customer logs it'll become more clear that we don't need the fallback.
Contributor
There was a problem hiding this comment. I'm perfectly fine with defaulting to the easier approach and waiting for real customer examples to come through via support. For example, assuming something doesn't happen, but throw exception if we detect it and wait for a diagnostic zip with real world example.
Contributor
Author
There was a problem hiding this comment. @JakeMathews I'd love your opinion on this. The bitbake code does not assume it'll find the recipe version in the task-depends.dot label (GraphParseTransformer.getVersionFromNode() has returned an Optional even before I started mucking with things). Given the vast experience you have now, do you believe that caution makes sense? And should it apply to finding the layer there as well? (I realize this is largely guesswork, but I trust your guesses more than mine.) Looking more closely: the detector does not assume that the label attribute is present, but if the label attribute is present, it assumes it will contain the version. My current plan is to do the same for layer.
Contributor
There was a problem hiding this comment. I think if the label is present its pretty safe to assume the layer is in the path. I always prefer to air on the side of caution especially when understanding is murky at best. i think what you have now will likely be the safest approach, but we won’t know if its right until we can collect more data. |
||
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Same thing, you could just bubble these up.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Here we want to just log and continue on to the next target image / codelocation