Skip to content

Commit bb4b2a1

Browse files
dorsJfrogdorsJfrog
authored
RTDEV-58262 - Modify Go pack zip vendor directory (#824)
1 parent 42e9ca2 commit bb4b2a1

File tree

3 files changed

+114
-14
lines changed

3 files changed

+114
-14
lines changed

‎build-info-extractor-go/src/main/java/org/jfrog/build/extractor/go/extractor/GoVersionUtils.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ public class GoVersionUtils {
1515
public static final String INCOMPATIBLE = "+incompatible";
1616
public static final int ZERO_OR_ONE = 0;
1717

18-
private GoVersionUtils() {
18+
public GoVersionUtils() {
1919
}
2020

2121
/**

‎build-info-extractor-go/src/main/java/org/jfrog/build/extractor/go/extractor/GoZipBallStreamer.java‎

Lines changed: 49 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434

3535
package org.jfrog.build.extractor.go.extractor;
3636

37+
import com.github.zafarkhaja.semver.Version;
3738
import org.apache.commons.compress.archivers.ArchiveOutputStream;
3839
import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
3940
import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream;
@@ -48,11 +49,11 @@
4849
import java.io.IOException;
4950
import java.io.InputStream;
5051
import java.nio.charset.StandardCharsets;
52+
import java.util.Arrays;
5153
import java.util.Enumeration;
5254
import java.util.HashSet;
55+
import java.util.Optional;
5356
import java.util.Set;
54-
import java.util.zip.ZipEntry;
55-
5657

5758
/**
5859
* @author BarakH
@@ -66,6 +67,7 @@ public class GoZipBallStreamer implements Closeable {
6667
private final String version;
6768
private final Set<String> excludedDirectories;
6869
private String subModuleName = "";
70+
Version goModVersion = Version.of(1, 12, 0);
6971
private static final String MOD_FILE = "go.mod";
7072
private static final String MOD_FILE_PATH = "/" + MOD_FILE;
7173
private static final String VENDOR = "vendor/";
@@ -200,11 +202,15 @@ private boolean excludeEntry(String entryName) {
200202
* Scanning all the original zip entries and collecting all the relative paths with go.mod files (i.e submodules)
201203
*/
202204
private void scanEntries() {
203-
Enumeration<? extends ZipEntry> entries = zipFile.getEntries();
204-
ZipEntry zipEntry;
205+
Enumeration<? extends ZipArchiveEntry> entries = zipFile.getEntries();
206+
ZipArchiveEntry zipEntry;
205207
Set<String> allDirectories = new HashSet<>();
206208
while (entries.hasMoreElements()) {
207209
zipEntry = entries.nextElement();
210+
if (zipEntry.getName().endsWith(MOD_FILE_PATH) && !isSubModule(zipEntry.getName())) {
211+
// the entry is a go.mod file of the project we are packing (root / submodule / v2)
212+
setGoVersionByModFile(zipEntry);
213+
}
208214
if (!zipEntry.isDirectory() && isSubModule(zipEntry.getName())) {
209215
String subModulePath = zipEntry.getName().replace(MOD_FILE_PATH, "");
210216
excludedDirectories.add(subModulePath);
@@ -293,20 +299,30 @@ private boolean hasRootModFileOfCompatibleModuleFromV2(String majorVersion) {
293299

294300
/**
295301
* Based on the original go client behaviour:
296-
* https://github.com/golang/go/blob/4be6b4a73d2f95752b69f5b6f6bfb4c1a7a57212/src/cmd/go/internal/modfetch
297-
* /coderepo.go which using https://cs.opensource.google/go/x/mod/+/refs/tags/v0.5.1:zip/zip.go (Create function)
298-
* There is a known bug here - see https://golang.org/issue/31562.
302+
* https://github.com/golang/go/blob/release-branch.go1.24/src/cmd/go/internal/modfetch/coderepo.go#L1057
303+
* which using https://cs.opensource.google/go/x/mod/+/9cd0e4c9f675aeac595a4cbb5ba1b46798ce0fdf:zip/zip.go;l=796
304+
* Go is ignoring vendor packages that are not in the root of the zip file.
305+
* From version 1.24, Go is also ignoring the vendor/modules.txt file.
306+
* Go issue: https://go-review.googlesource.com/c/mod/+/584635
299307
*
300308
* @return True if the entry belongs to a vendor package
301309
*/
302310
private boolean isVendorPackage(String entryName) {
303-
int i;
311+
Version go124Version = Version.of(1, 24, 0);
312+
if (goModVersion.isHigherThanOrEquivalentTo(go124Version) && entryName.endsWith("vendor/modules.txt")) {
313+
return true;
314+
}
315+
316+
int i = 0;
304317
if (entryName.startsWith(VENDOR)) {
305318
i = VENDOR.length();
306-
} else if (entryName.contains('/' + VENDOR)) {
307-
i = VENDOR.length() + 1;
308319
} else {
309-
return false;
320+
int j = entryName.indexOf("/" + VENDOR);
321+
if (j >= 0) {
322+
i = j + ("/" + VENDOR).length();
323+
} else {
324+
return false;
325+
}
310326
}
311327
return entryName.substring(i).contains("/");
312328
}
@@ -349,4 +365,26 @@ private String getFirstPathElement(String path) {
349365
path = StringUtils.removeStart(path, "/");
350366
return StringUtils.substringBefore(path, "/");
351367
}
368+
369+
/**
370+
* Sets the Go version based on the content of go.mod file of the project we are packing (root / submodule / v2)
371+
*/
372+
private void setGoVersionByModFile(ZipArchiveEntry entry) {
373+
// Go version 1.13.0 is the version where we should start looking for go.mod file.
374+
if (goModVersion.equals(Version.of(1, 12, 0))) {
375+
try (InputStream inputStream = zipFile.getInputStream(entry)) {
376+
String modFileContent = IOUtils.toString(inputStream, StandardCharsets.UTF_8.name());
377+
if (!StringUtils.isEmpty(modFileContent)) {
378+
String goVersion = Arrays.stream(
379+
modFileContent.split("\n")).filter(line -> line.startsWith("go"))
380+
.findFirst().get().replace("go", "").trim();
381+
goModVersion = Version.parse(goVersion, false);
382+
}
383+
} catch (Exception e) {
384+
log.warn("Failed to read go.mod file '" + entry.getName() + "': " +
385+
ExceptionUtils.getRootCauseMessage(e));
386+
}
387+
}
388+
}
389+
352390
}

‎build-info-extractor-go/src/test/java/org/jfrog/build/extractor/go/extractor/GoZipBallStreamerTest.java‎

Lines changed: 64 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
package org.jfrog.build.extractor.go.extractor;
22

3+
import com.github.zafarkhaja.semver.Version;
34
import org.testng.Assert;
45
import org.testng.annotations.DataProvider;
56
import org.testng.annotations.Test;
6-
7-
import java.io.IOException;
7+
import java.lang.reflect.Method;
88

99

1010
public class GoZipBallStreamerTest {
@@ -27,4 +27,66 @@ private Object[][] testIsSubModuleProvider() {
2727
};
2828
}
2929

30+
@Test(dataProvider = "vendorPackageProvider")
31+
public void testIsVendorPackage(String goModVersion, String entryName, boolean expected) throws Exception {
32+
GoZipBallStreamer streamer = new GoZipBallStreamer(null, "project", "1.0.0", null);
33+
streamer.goModVersion = Version.valueOf(goModVersion);
34+
35+
Method method = GoZipBallStreamer.class.getDeclaredMethod("isVendorPackage", String.class);
36+
method.setAccessible(true);
37+
boolean result = (boolean) method.invoke(streamer, entryName);
38+
39+
Assert.assertEquals(result, expected, "Failed for: " + entryName + " with Go version: " + goModVersion);
40+
}
41+
42+
@DataProvider
43+
public Object[][] vendorPackageProvider() {
44+
return new Object[][]{
45+
// followed by Go tests scenarios:
46+
//https://cs.opensource.google/go/x/mod/+/c8a731972177c6ce4073699c705e55918ee7be09:zip/vendor_test.go
47+
48+
// allVers (pre-1.24 and 1.24+)
49+
{"1.23.0", "vendor/foo/foo.go", true},
50+
{"1.24.0", "vendor/foo/foo.go", true},
51+
{"1.23.0", "pkg/vendor/foo/foo.go", true},
52+
{"1.24.0", "pkg/vendor/foo/foo.go", true},
53+
{"1.23.0", "longpackagename/vendor/foo/foo.go", true},
54+
{"1.24.0", "longpackagename/vendor/foo/foo.go", true},
55+
{"1.23.0", "vendor/vendor.go", false},
56+
{"1.24.0", "vendor/vendor.go", false},
57+
{"1.23.0", "vendor/foo/modules.txt", true},
58+
{"1.24.0", "vendor/foo/modules.txt", true},
59+
{"1.23.0", "modules.txt", false},
60+
{"1.24.0", "modules.txt", false},
61+
{"1.23.0", "vendor/amodules.txt", false},
62+
{"1.24.0", "vendor/amodules.txt", false},
63+
64+
// vendor/modules.txt special cases
65+
{"1.23.0", "vendor/modules.txt", false},
66+
{"1.24.0", "vendor/modules.txt", true},
67+
{"1.25.0", "vendor/modules.txt", true},
68+
{"1.23.0", "foo/vendor/modules.txt", false},
69+
{"1.24.0", "foo/vendor/modules.txt", true},
70+
{"1.25.0", "foo/vendor/modules.txt", true},
71+
72+
// pkg/vendor/vendor.go and longpackagename/vendor/vendor.go
73+
{"1.23.0", "pkg/vendor/vendor.go", false},
74+
{"1.23.0", "pkg/vendor/foo/vendor.text", true},
75+
{"1.24.0", "pkg/vendor/vendor.go", false},
76+
{"1.23.0", "longpackagename/vendor/vendor.go", false},
77+
{"1.24.0", "longpackagename/vendor/vendor.go", false},
78+
79+
{"1.24.0", "src/vendor/github.com/pkg/errors/errors.go", true},
80+
{"1.23.0", "src/vendor/github.com/pkg/errors/errors.go", true},
81+
{"1.24.0", "vendor/errors.go", false},
82+
{"1.24.0", "foo/bar/vendor/pkg/file.go", true},
83+
{"1.24.0", "notvendor/file.go", false},
84+
{"1.24.0", "vendor/file.go", false},
85+
{"1.24.0", "vendor/pkg/file.go", true},
86+
{"1.24.0", "vendor/", false},
87+
{"1.24.0", "foo/vendor/pkg/file.go", true},
88+
{"1.24.0", "foo/vendor/file.go", false}
89+
};
90+
}
91+
3092
}

0 commit comments

Comments
 (0)