From 6349c3ca3a2339f6b08ee5f31a313e48ea224327 Mon Sep 17 00:00:00 2001 From: Lovepreet Singh Date: Wed, 16 Nov 2022 20:18:31 +0000 Subject: [PATCH 01/49] bsd + zstd fallback implementation --- packages/cache/src/internal/constants.ts | 2 + packages/cache/src/internal/tar.ts | 51 +++++++++++++++++++----- 2 files changed, 42 insertions(+), 11 deletions(-) diff --git a/packages/cache/src/internal/constants.ts b/packages/cache/src/internal/constants.ts index e61b46f464..0acea2ef15 100644 --- a/packages/cache/src/internal/constants.ts +++ b/packages/cache/src/internal/constants.ts @@ -24,3 +24,5 @@ export const SocketTimeout = 5000 // The default path of GNUtar on hosted Windows runners export const GnuTarPathOnWindows = `${process.env['PROGRAMFILES']}\\Git\\usr\\bin\\tar.exe` + +export const SystemTarPathOnWindows = `${process.env['SYSTEMDRIVE']}\\Windows\\System32\\tar.exe` \ No newline at end of file diff --git a/packages/cache/src/internal/tar.ts b/packages/cache/src/internal/tar.ts index 426920082b..22dd05bf3f 100644 --- a/packages/cache/src/internal/tar.ts +++ b/packages/cache/src/internal/tar.ts @@ -3,10 +3,11 @@ import * as io from '@actions/io' import {existsSync, writeFileSync} from 'fs' import * as path from 'path' import * as utils from './cacheUtils' -import {CompressionMethod} from './constants' +import {CompressionMethod, GnuTarPathOnWindows, SystemTarPathOnWindows} from './constants' const IS_WINDOWS = process.platform === 'win32' +// Function also mutates the args array. For non-mutation call with passing an empty array. async function getTarPath(args: string[]): Promise { switch (process.platform) { case 'win32': { @@ -14,7 +15,9 @@ async function getTarPath(args: string[]): Promise { const systemTar = `${process.env['windir']}\\System32\\tar.exe` if (gnuTar) { // Use GNUtar as default on windows - args.push('--force-local') + if (args.length > 0) { + args.push('--force-local') + } return gnuTar } else if (existsSync(systemTar)) { return systemTar @@ -25,7 +28,9 @@ async function getTarPath(args: string[]): Promise { const gnuTar = await io.which('gtar', false) if (gnuTar) { // fix permission denied errors when extracting BSD tar archive with GNU tar - https://github.com/actions/cache/issues/527 - args.push('--delay-directory-restore') + if (args.length > 0) { + args.push('--delay-directory-restore') + } return gnuTar } break @@ -49,18 +54,30 @@ function getWorkingDirectory(): string { } // Common function for extractTar and listTar to get the compression method -function getCompressionProgram(compressionMethod: CompressionMethod): string[] { +async function getCompressionProgram(compressionMethod: CompressionMethod): string[] { // -d: Decompress. // unzstd is equivalent to 'zstd -d' // --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit. // Using 30 here because we also support 32-bit self-hosted runners. + const tarPath = await getTarPath([]) + const BSD_TAR_ZSTD = IS_WINDOWS && tarPath === SystemTarPathOnWindows switch (compressionMethod) { case CompressionMethod.Zstd: + if (BSD_TAR_ZSTD) { + return [ + '-O', '|','zstd -d --long=30 -o' + ] + } return [ '--use-compress-program', IS_WINDOWS ? 'zstd -d --long=30' : 'unzstd --long=30' ] case CompressionMethod.ZstdWithoutLong: + if (BSD_TAR_ZSTD) { + return [ + '-O', '|', 'zstd -d --long=30 -o' + ] + } return ['--use-compress-program', IS_WINDOWS ? 'zstd -d' : 'unzstd'] default: return ['-z'] @@ -72,8 +89,8 @@ export async function listTar( compressionMethod: CompressionMethod ): Promise { const args = [ - ...getCompressionProgram(compressionMethod), '-tf', + ...getCompressionProgram(compressionMethod), archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), '-P' ] @@ -88,8 +105,8 @@ export async function extractTar( const workingDirectory = getWorkingDirectory() await io.mkdirP(workingDirectory) const args = [ - ...getCompressionProgram(compressionMethod), '-xf', + ...getCompressionProgram(compressionMethod), archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), '-P', '-C', @@ -117,14 +134,26 @@ export async function createTar( // --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit. // Using 30 here because we also support 32-bit self-hosted runners. // Long range mode is added to zstd in v1.3.2 release, so we will not use --long in older version of zstd. - function getCompressionProgram(): string[] { + async function getCompressionProgram(): string[] { + const tarPath = await getTarPath([]) + const BSD_TAR_ZSTD = IS_WINDOWS && tarPath === SystemTarPathOnWindows switch (compressionMethod) { case CompressionMethod.Zstd: + if (BSD_TAR_ZSTD) { + return [ + '-O', '|', 'zstd -T0 --long=30 -o' + ] + } return [ '--use-compress-program', IS_WINDOWS ? 'zstd -T0 --long=30' : 'zstdmt --long=30' ] case CompressionMethod.ZstdWithoutLong: + if (BSD_TAR_ZSTD) { + return [ + '-O', '|', 'zstd -T0 -o' + ] + } return ['--use-compress-program', IS_WINDOWS ? 'zstd -T0' : 'zstdmt'] default: return ['-z'] @@ -132,16 +161,16 @@ export async function createTar( } const args = [ '--posix', - ...getCompressionProgram(), - '-cf', - cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), '--exclude', cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), '-P', '-C', workingDirectory.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), '--files-from', - manifestFilename + manifestFilename, + ...getCompressionProgram(), + cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), + ] await execTar(args, archiveFolder) } From db3517fe3b63c59c4ab7857d6aa7658b15b99eaa Mon Sep 17 00:00:00 2001 From: Lovepreet Singh Date: Wed, 16 Nov 2022 20:21:40 +0000 Subject: [PATCH 02/49] bsd + zstd fallback implementation --- packages/cache/src/internal/tar.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/cache/src/internal/tar.ts b/packages/cache/src/internal/tar.ts index 22dd05bf3f..f077dc5277 100644 --- a/packages/cache/src/internal/tar.ts +++ b/packages/cache/src/internal/tar.ts @@ -54,7 +54,7 @@ function getWorkingDirectory(): string { } // Common function for extractTar and listTar to get the compression method -async function getCompressionProgram(compressionMethod: CompressionMethod): string[] { +async function getCompressionProgram(compressionMethod: CompressionMethod): Promise { // -d: Decompress. // unzstd is equivalent to 'zstd -d' // --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit. @@ -90,7 +90,7 @@ export async function listTar( ): Promise { const args = [ '-tf', - ...getCompressionProgram(compressionMethod), + ...(await getCompressionProgram(compressionMethod)), archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), '-P' ] @@ -106,7 +106,7 @@ export async function extractTar( await io.mkdirP(workingDirectory) const args = [ '-xf', - ...getCompressionProgram(compressionMethod), + ...(await getCompressionProgram(compressionMethod)), archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), '-P', '-C', @@ -134,7 +134,7 @@ export async function createTar( // --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit. // Using 30 here because we also support 32-bit self-hosted runners. // Long range mode is added to zstd in v1.3.2 release, so we will not use --long in older version of zstd. - async function getCompressionProgram(): string[] { + async function getCompressionProgram(): Promise { const tarPath = await getTarPath([]) const BSD_TAR_ZSTD = IS_WINDOWS && tarPath === SystemTarPathOnWindows switch (compressionMethod) { @@ -168,7 +168,7 @@ export async function createTar( workingDirectory.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), '--files-from', manifestFilename, - ...getCompressionProgram(), + ...(await getCompressionProgram()), cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), ] From 27bf8304ea3aef2975760926d9742b6d1eb1bf56 Mon Sep 17 00:00:00 2001 From: Lovepreet Singh Date: Wed, 16 Nov 2022 20:50:04 +0000 Subject: [PATCH 03/49] Fix tar operations --- packages/cache/src/internal/tar.ts | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/packages/cache/src/internal/tar.ts b/packages/cache/src/internal/tar.ts index f077dc5277..1a7f38fcde 100644 --- a/packages/cache/src/internal/tar.ts +++ b/packages/cache/src/internal/tar.ts @@ -64,9 +64,7 @@ async function getCompressionProgram(compressionMethod: CompressionMethod): Prom switch (compressionMethod) { case CompressionMethod.Zstd: if (BSD_TAR_ZSTD) { - return [ - '-O', '|','zstd -d --long=30 -o' - ] + return ['-a'] // auto-detect compression } return [ '--use-compress-program', @@ -74,9 +72,7 @@ async function getCompressionProgram(compressionMethod: CompressionMethod): Prom ] case CompressionMethod.ZstdWithoutLong: if (BSD_TAR_ZSTD) { - return [ - '-O', '|', 'zstd -d --long=30 -o' - ] + return ['a'] // auto-detect compression } return ['--use-compress-program', IS_WINDOWS ? 'zstd -d' : 'unzstd'] default: @@ -168,6 +164,7 @@ export async function createTar( workingDirectory.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), '--files-from', manifestFilename, + '-cf', ...(await getCompressionProgram()), cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), From e9e146bbf8faa49d1d78bc774d5c65aabb71bdab Mon Sep 17 00:00:00 2001 From: Lovepreet Singh Date: Wed, 16 Nov 2022 21:04:25 +0000 Subject: [PATCH 04/49] Add -v option for testing --- packages/cache/src/internal/tar.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/cache/src/internal/tar.ts b/packages/cache/src/internal/tar.ts index 1a7f38fcde..a178503933 100644 --- a/packages/cache/src/internal/tar.ts +++ b/packages/cache/src/internal/tar.ts @@ -165,6 +165,7 @@ export async function createTar( '--files-from', manifestFilename, '-cf', + '-v', ...(await getCompressionProgram()), cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), From 5f89653f1b9e6c52c7afbabb923cd08fb6a8210e Mon Sep 17 00:00:00 2001 From: Lovepreet Singh Date: Thu, 17 Nov 2022 06:22:13 +0000 Subject: [PATCH 05/49] Fix order of args for tar --- packages/cache/src/internal/tar.ts | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/packages/cache/src/internal/tar.ts b/packages/cache/src/internal/tar.ts index a178503933..2cabe3b48c 100644 --- a/packages/cache/src/internal/tar.ts +++ b/packages/cache/src/internal/tar.ts @@ -68,13 +68,14 @@ async function getCompressionProgram(compressionMethod: CompressionMethod): Prom } return [ '--use-compress-program', - IS_WINDOWS ? 'zstd -d --long=30' : 'unzstd --long=30' + IS_WINDOWS ? 'zstd -d --long=30' : 'unzstd --long=30', + '-tf' ] case CompressionMethod.ZstdWithoutLong: if (BSD_TAR_ZSTD) { return ['a'] // auto-detect compression } - return ['--use-compress-program', IS_WINDOWS ? 'zstd -d' : 'unzstd'] + return ['--use-compress-program', IS_WINDOWS ? 'zstd -d' : 'unzstd', '-tf'] default: return ['-z'] } @@ -85,7 +86,6 @@ export async function listTar( compressionMethod: CompressionMethod ): Promise { const args = [ - '-tf', ...(await getCompressionProgram(compressionMethod)), archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), '-P' @@ -142,7 +142,8 @@ export async function createTar( } return [ '--use-compress-program', - IS_WINDOWS ? 'zstd -T0 --long=30' : 'zstdmt --long=30' + IS_WINDOWS ? 'zstd -T0 --long=30' : 'zstdmt --long=30', + '-cf' ] case CompressionMethod.ZstdWithoutLong: if (BSD_TAR_ZSTD) { @@ -150,7 +151,7 @@ export async function createTar( '-O', '|', 'zstd -T0 -o' ] } - return ['--use-compress-program', IS_WINDOWS ? 'zstd -T0' : 'zstdmt'] + return ['--use-compress-program', IS_WINDOWS ? 'zstd -T0' : 'zstdmt', '-cf'] default: return ['-z'] } @@ -164,8 +165,6 @@ export async function createTar( workingDirectory.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), '--files-from', manifestFilename, - '-cf', - '-v', ...(await getCompressionProgram()), cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), From 6f7397feb68cfc91380a4d5bf4f665d2871f4437 Mon Sep 17 00:00:00 2001 From: Sampark Sharma Date: Thu, 17 Nov 2022 07:26:52 +0000 Subject: [PATCH 06/49] Add GNUtar as default on windows --- packages/cache/__tests__/tar.test.ts | 58 ++++++++++++++--------- packages/cache/src/internal/cacheUtils.ts | 17 +++++-- packages/cache/src/internal/constants.ts | 3 ++ packages/cache/src/internal/tar.ts | 30 +++++------- 4 files changed, 61 insertions(+), 47 deletions(-) diff --git a/packages/cache/__tests__/tar.test.ts b/packages/cache/__tests__/tar.test.ts index e4233bc952..a6cd17252f 100644 --- a/packages/cache/__tests__/tar.test.ts +++ b/packages/cache/__tests__/tar.test.ts @@ -1,7 +1,11 @@ import * as exec from '@actions/exec' import * as io from '@actions/io' import * as path from 'path' -import {CacheFilename, CompressionMethod} from '../src/internal/constants' +import { + CacheFilename, + CompressionMethod, + GnuTarPathOnWindows +} from '../src/internal/constants' import * as tar from '../src/internal/tar' import * as utils from '../src/internal/cacheUtils' // eslint-disable-next-line @typescript-eslint/no-require-imports @@ -28,6 +32,10 @@ beforeAll(async () => { await jest.requireActual('@actions/io').rmRF(getTempDir()) }) +beforeEach(async () => { + jest.restoreAllMocks() +}) + afterAll(async () => { delete process.env['GITHUB_WORKSPACE'] await jest.requireActual('@actions/io').rmRF(getTempDir()) @@ -41,13 +49,14 @@ test('zstd extract tar', async () => { ? `${process.env['windir']}\\fakepath\\cache.tar` : 'cache.tar' const workspace = process.env['GITHUB_WORKSPACE'] + const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath await tar.extractTar(archivePath, CompressionMethod.Zstd) expect(mkdirMock).toHaveBeenCalledWith(workspace) expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledWith( - `"${defaultTarPath}"`, + `"${tarPath}"`, [ '--use-compress-program', IS_WINDOWS ? 'zstd -d --long=30' : 'unzstd --long=30', @@ -74,9 +83,7 @@ test('gzip extract tar', async () => { await tar.extractTar(archivePath, CompressionMethod.Gzip) expect(mkdirMock).toHaveBeenCalledWith(workspace) - const tarPath = IS_WINDOWS - ? `${process.env['windir']}\\System32\\tar.exe` - : defaultTarPath + const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledWith( `"${tarPath}"`, @@ -87,18 +94,19 @@ test('gzip extract tar', async () => { '-P', '-C', IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace - ].concat(IS_MAC ? ['--delay-directory-restore'] : []), + ] + .concat(IS_WINDOWS ? ['--force-local'] : []) + .concat(IS_MAC ? ['--delay-directory-restore'] : []), {cwd: undefined} ) }) -test('gzip extract GNU tar on windows', async () => { +test('gzip extract GNU tar on windows with GNUtar in path', async () => { if (IS_WINDOWS) { - jest.spyOn(fs, 'existsSync').mockReturnValueOnce(false) - + // GNU tar present in path but not at default location const isGnuMock = jest - .spyOn(utils, 'isGnuTarInstalled') - .mockReturnValue(Promise.resolve(true)) + .spyOn(utils, 'getGnuTarPathOnWindows') + .mockReturnValue(Promise.resolve('tar')) const execMock = jest.spyOn(exec, 'exec') const archivePath = `${process.env['windir']}\\fakepath\\cache.tar` const workspace = process.env['GITHUB_WORKSPACE'] @@ -134,9 +142,11 @@ test('zstd create tar', async () => { await tar.createTar(archiveFolder, sourceDirectories, CompressionMethod.Zstd) + const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath + expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledWith( - `"${defaultTarPath}"`, + `"${tarPath}"`, [ '--posix', '--use-compress-program', @@ -170,9 +180,7 @@ test('gzip create tar', async () => { await tar.createTar(archiveFolder, sourceDirectories, CompressionMethod.Gzip) - const tarPath = IS_WINDOWS - ? `${process.env['windir']}\\System32\\tar.exe` - : defaultTarPath + const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledWith( @@ -189,7 +197,9 @@ test('gzip create tar', async () => { IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace, '--files-from', 'manifest.txt' - ].concat(IS_MAC ? ['--delay-directory-restore'] : []), + ] + .concat(IS_WINDOWS ? ['--force-local'] : []) + .concat(IS_MAC ? ['--delay-directory-restore'] : []), { cwd: archiveFolder } @@ -205,9 +215,10 @@ test('zstd list tar', async () => { await tar.listTar(archivePath, CompressionMethod.Zstd) + const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledWith( - `"${defaultTarPath}"`, + `"${tarPath}"`, [ '--use-compress-program', IS_WINDOWS ? 'zstd -d --long=30' : 'unzstd --long=30', @@ -230,9 +241,10 @@ test('zstdWithoutLong list tar', async () => { await tar.listTar(archivePath, CompressionMethod.ZstdWithoutLong) + const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledWith( - `"${defaultTarPath}"`, + `"${tarPath}"`, [ '--use-compress-program', IS_WINDOWS ? 'zstd -d' : 'unzstd', @@ -254,9 +266,7 @@ test('gzip list tar', async () => { await tar.listTar(archivePath, CompressionMethod.Gzip) - const tarPath = IS_WINDOWS - ? `${process.env['windir']}\\System32\\tar.exe` - : defaultTarPath + const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledWith( `"${tarPath}"`, @@ -265,7 +275,9 @@ test('gzip list tar', async () => { '-tf', IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath, '-P' - ].concat(IS_MAC ? ['--delay-directory-restore'] : []), + ] + .concat(IS_WINDOWS ? ['--force-local'] : []) + .concat(IS_MAC ? ['--delay-directory-restore'] : []), {cwd: undefined} ) -}) +}) \ No newline at end of file diff --git a/packages/cache/src/internal/cacheUtils.ts b/packages/cache/src/internal/cacheUtils.ts index c2ace526bf..9d4e4c3d2c 100644 --- a/packages/cache/src/internal/cacheUtils.ts +++ b/packages/cache/src/internal/cacheUtils.ts @@ -7,7 +7,11 @@ import * as path from 'path' import * as semver from 'semver' import * as util from 'util' import {v4 as uuidV4} from 'uuid' -import {CacheFilename, CompressionMethod} from './constants' +import { + CacheFilename, + CompressionMethod, + GnuTarPathOnWindows +} from './constants' // From https://github.com/actions/toolkit/blob/main/packages/tool-cache/src/tool-cache.ts#L23 export async function createTempDirectory(): Promise { @@ -90,7 +94,7 @@ async function getVersion(app: string): Promise { // Use zstandard if possible to maximize cache performance export async function getCompressionMethod(): Promise { - if (process.platform === 'win32' && !(await isGnuTarInstalled())) { + if (process.platform === 'win32' && !(await getGnuTarPathOnWindows())) { // Disable zstd due to bug https://github.com/actions/cache/issues/301 return CompressionMethod.Gzip } @@ -116,9 +120,12 @@ export function getCacheFileName(compressionMethod: CompressionMethod): string { : CacheFilename.Zstd } -export async function isGnuTarInstalled(): Promise { +export async function getGnuTarPathOnWindows(): Promise { + if (fs.existsSync(GnuTarPathOnWindows)) { + return GnuTarPathOnWindows + } const versionOutput = await getVersion('tar') - return versionOutput.toLowerCase().includes('gnu tar') + return versionOutput.toLowerCase().includes('gnu tar') ? io.which('tar') : '' } export function assertDefined(name: string, value?: T): T { @@ -134,4 +141,4 @@ export function isGhes(): boolean { process.env['GITHUB_SERVER_URL'] || 'https://github.com' ) return ghUrl.hostname.toUpperCase() !== 'GITHUB.COM' -} +} \ No newline at end of file diff --git a/packages/cache/src/internal/constants.ts b/packages/cache/src/internal/constants.ts index 2f78d32685..35225b6c3e 100644 --- a/packages/cache/src/internal/constants.ts +++ b/packages/cache/src/internal/constants.ts @@ -21,3 +21,6 @@ export const DefaultRetryDelay = 5000 // over the socket during this period, the socket is destroyed and the download // is aborted. export const SocketTimeout = 5000 + +// The default path of GNUtar on hosted Windows runners +export const GnuTarPathOnWindows = `${process.env['PROGRAMFILES']}\\Git\\usr\\bin\\tar.exe` \ No newline at end of file diff --git a/packages/cache/src/internal/tar.ts b/packages/cache/src/internal/tar.ts index 2e28ca1ade..2c07dc8124 100644 --- a/packages/cache/src/internal/tar.ts +++ b/packages/cache/src/internal/tar.ts @@ -7,21 +7,17 @@ import {CompressionMethod} from './constants' const IS_WINDOWS = process.platform === 'win32' -async function getTarPath( - args: string[], - compressionMethod: CompressionMethod -): Promise { +async function getTarPath(args: string[]): Promise { switch (process.platform) { case 'win32': { + const gnuTar = await utils.getGnuTarPathOnWindows() const systemTar = `${process.env['windir']}\\System32\\tar.exe` - if (compressionMethod !== CompressionMethod.Gzip) { - // We only use zstandard compression on windows when gnu tar is installed due to - // a bug with compressing large files with bsdtar + zstd + if (gnuTar) { + // Use GNUtar as default on windows args.push('--force-local') + return gnuTar } else if (existsSync(systemTar)) { return systemTar - } else if (await utils.isGnuTarInstalled()) { - args.push('--force-local') } break } @@ -40,13 +36,9 @@ async function getTarPath( return await io.which('tar', true) } -async function execTar( - args: string[], - compressionMethod: CompressionMethod, - cwd?: string -): Promise { +async function execTar(args: string[], cwd?: string): Promise { try { - await exec(`"${await getTarPath(args, compressionMethod)}"`, args, {cwd}) + await exec(`"${await getTarPath(args)}"`, args, {cwd}) } catch (error) { throw new Error(`Tar failed with error: ${error?.message}`) } @@ -85,7 +77,7 @@ export async function listTar( archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), '-P' ] - await execTar(args, compressionMethod) + await execTar(args) } export async function extractTar( @@ -103,7 +95,7 @@ export async function extractTar( '-C', workingDirectory.replace(new RegExp(`\\${path.sep}`, 'g'), '/') ] - await execTar(args, compressionMethod) + await execTar(args) } export async function createTar( @@ -151,5 +143,5 @@ export async function createTar( '--files-from', manifestFilename ] - await execTar(args, compressionMethod, archiveFolder) -} + await execTar(args, archiveFolder) +} \ No newline at end of file From 964682b5d49e779cf41130cbb68fde83a1563ca7 Mon Sep 17 00:00:00 2001 From: Sampark Sharma Date: Thu, 17 Nov 2022 07:34:07 +0000 Subject: [PATCH 07/49] Fix test --- packages/cache/__tests__/tar.test.ts | 2 +- packages/cache/src/internal/cacheUtils.ts | 2 +- packages/cache/src/internal/constants.ts | 2 +- packages/cache/src/internal/tar.ts | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/cache/__tests__/tar.test.ts b/packages/cache/__tests__/tar.test.ts index a6cd17252f..31f9bd5e4a 100644 --- a/packages/cache/__tests__/tar.test.ts +++ b/packages/cache/__tests__/tar.test.ts @@ -280,4 +280,4 @@ test('gzip list tar', async () => { .concat(IS_MAC ? ['--delay-directory-restore'] : []), {cwd: undefined} ) -}) \ No newline at end of file +}) diff --git a/packages/cache/src/internal/cacheUtils.ts b/packages/cache/src/internal/cacheUtils.ts index 9d4e4c3d2c..64fef08acd 100644 --- a/packages/cache/src/internal/cacheUtils.ts +++ b/packages/cache/src/internal/cacheUtils.ts @@ -141,4 +141,4 @@ export function isGhes(): boolean { process.env['GITHUB_SERVER_URL'] || 'https://github.com' ) return ghUrl.hostname.toUpperCase() !== 'GITHUB.COM' -} \ No newline at end of file +} diff --git a/packages/cache/src/internal/constants.ts b/packages/cache/src/internal/constants.ts index 35225b6c3e..e61b46f464 100644 --- a/packages/cache/src/internal/constants.ts +++ b/packages/cache/src/internal/constants.ts @@ -23,4 +23,4 @@ export const DefaultRetryDelay = 5000 export const SocketTimeout = 5000 // The default path of GNUtar on hosted Windows runners -export const GnuTarPathOnWindows = `${process.env['PROGRAMFILES']}\\Git\\usr\\bin\\tar.exe` \ No newline at end of file +export const GnuTarPathOnWindows = `${process.env['PROGRAMFILES']}\\Git\\usr\\bin\\tar.exe` diff --git a/packages/cache/src/internal/tar.ts b/packages/cache/src/internal/tar.ts index 2c07dc8124..426920082b 100644 --- a/packages/cache/src/internal/tar.ts +++ b/packages/cache/src/internal/tar.ts @@ -144,4 +144,4 @@ export async function createTar( manifestFilename ] await execTar(args, archiveFolder) -} \ No newline at end of file +} From ea9856079f362753e8dc557c7ff97d2991a6a2a0 Mon Sep 17 00:00:00 2001 From: Lovepreet Singh Date: Thu, 17 Nov 2022 08:53:46 +0000 Subject: [PATCH 08/49] Fix tar tests --- packages/cache/__tests__/tar.test.ts | 19 ++++++++++--------- packages/cache/src/internal/tar.ts | 8 ++++---- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/packages/cache/__tests__/tar.test.ts b/packages/cache/__tests__/tar.test.ts index 31f9bd5e4a..ec2141814f 100644 --- a/packages/cache/__tests__/tar.test.ts +++ b/packages/cache/__tests__/tar.test.ts @@ -149,17 +149,17 @@ test('zstd create tar', async () => { `"${tarPath}"`, [ '--posix', - '--use-compress-program', - IS_WINDOWS ? 'zstd -T0 --long=30' : 'zstdmt --long=30', - '-cf', - IS_WINDOWS ? CacheFilename.Zstd.replace(/\\/g, '/') : CacheFilename.Zstd, '--exclude', IS_WINDOWS ? CacheFilename.Zstd.replace(/\\/g, '/') : CacheFilename.Zstd, '-P', '-C', IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace, '--files-from', - 'manifest.txt' + 'manifest.txt', + '--use-compress-program', + IS_WINDOWS ? 'zstd -T0 --long=30' : 'zstdmt --long=30', + '-cf', + IS_WINDOWS ? CacheFilename.Zstd.replace(/\\/g, '/') : CacheFilename.Zstd ] .concat(IS_WINDOWS ? ['--force-local'] : []) .concat(IS_MAC ? ['--delay-directory-restore'] : []), @@ -187,16 +187,16 @@ test('gzip create tar', async () => { `"${tarPath}"`, [ '--posix', - '-z', - '-cf', - IS_WINDOWS ? CacheFilename.Gzip.replace(/\\/g, '/') : CacheFilename.Gzip, '--exclude', IS_WINDOWS ? CacheFilename.Gzip.replace(/\\/g, '/') : CacheFilename.Gzip, '-P', '-C', IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace, '--files-from', - 'manifest.txt' + 'manifest.txt', + '-z', + '-cf', + IS_WINDOWS ? CacheFilename.Gzip.replace(/\\/g, '/') : CacheFilename.Gzip, ] .concat(IS_WINDOWS ? ['--force-local'] : []) .concat(IS_MAC ? ['--delay-directory-restore'] : []), @@ -281,3 +281,4 @@ test('gzip list tar', async () => { {cwd: undefined} ) }) + diff --git a/packages/cache/src/internal/tar.ts b/packages/cache/src/internal/tar.ts index 2cabe3b48c..450cde3b9a 100644 --- a/packages/cache/src/internal/tar.ts +++ b/packages/cache/src/internal/tar.ts @@ -69,13 +69,12 @@ async function getCompressionProgram(compressionMethod: CompressionMethod): Prom return [ '--use-compress-program', IS_WINDOWS ? 'zstd -d --long=30' : 'unzstd --long=30', - '-tf' ] case CompressionMethod.ZstdWithoutLong: if (BSD_TAR_ZSTD) { return ['a'] // auto-detect compression } - return ['--use-compress-program', IS_WINDOWS ? 'zstd -d' : 'unzstd', '-tf'] + return ['--use-compress-program', IS_WINDOWS ? 'zstd -d' : 'unzstd'] default: return ['-z'] } @@ -87,6 +86,7 @@ export async function listTar( ): Promise { const args = [ ...(await getCompressionProgram(compressionMethod)), + '-tf', archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), '-P' ] @@ -101,8 +101,8 @@ export async function extractTar( const workingDirectory = getWorkingDirectory() await io.mkdirP(workingDirectory) const args = [ - '-xf', ...(await getCompressionProgram(compressionMethod)), + '-xf', archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), '-P', '-C', @@ -153,7 +153,7 @@ export async function createTar( } return ['--use-compress-program', IS_WINDOWS ? 'zstd -T0' : 'zstdmt', '-cf'] default: - return ['-z'] + return ['-z', '-cf'] } } const args = [ From 4fa5b7d133e7c42c32c09fd1b007c49f2a41ef72 Mon Sep 17 00:00:00 2001 From: Lovepreet Singh Date: Thu, 17 Nov 2022 09:12:53 +0000 Subject: [PATCH 09/49] Fix lint issues --- packages/cache/__tests__/tar.test.ts | 3 +-- packages/cache/src/internal/constants.ts | 2 +- packages/cache/src/internal/tar.ts | 27 ++++++++++++------------ 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/packages/cache/__tests__/tar.test.ts b/packages/cache/__tests__/tar.test.ts index ec2141814f..abf8eecb90 100644 --- a/packages/cache/__tests__/tar.test.ts +++ b/packages/cache/__tests__/tar.test.ts @@ -196,7 +196,7 @@ test('gzip create tar', async () => { 'manifest.txt', '-z', '-cf', - IS_WINDOWS ? CacheFilename.Gzip.replace(/\\/g, '/') : CacheFilename.Gzip, + IS_WINDOWS ? CacheFilename.Gzip.replace(/\\/g, '/') : CacheFilename.Gzip ] .concat(IS_WINDOWS ? ['--force-local'] : []) .concat(IS_MAC ? ['--delay-directory-restore'] : []), @@ -281,4 +281,3 @@ test('gzip list tar', async () => { {cwd: undefined} ) }) - diff --git a/packages/cache/src/internal/constants.ts b/packages/cache/src/internal/constants.ts index 0acea2ef15..c6d8a49094 100644 --- a/packages/cache/src/internal/constants.ts +++ b/packages/cache/src/internal/constants.ts @@ -25,4 +25,4 @@ export const SocketTimeout = 5000 // The default path of GNUtar on hosted Windows runners export const GnuTarPathOnWindows = `${process.env['PROGRAMFILES']}\\Git\\usr\\bin\\tar.exe` -export const SystemTarPathOnWindows = `${process.env['SYSTEMDRIVE']}\\Windows\\System32\\tar.exe` \ No newline at end of file +export const SystemTarPathOnWindows = `${process.env['SYSTEMDRIVE']}\\Windows\\System32\\tar.exe` diff --git a/packages/cache/src/internal/tar.ts b/packages/cache/src/internal/tar.ts index 450cde3b9a..182bc7bb8c 100644 --- a/packages/cache/src/internal/tar.ts +++ b/packages/cache/src/internal/tar.ts @@ -3,7 +3,7 @@ import * as io from '@actions/io' import {existsSync, writeFileSync} from 'fs' import * as path from 'path' import * as utils from './cacheUtils' -import {CompressionMethod, GnuTarPathOnWindows, SystemTarPathOnWindows} from './constants' +import {CompressionMethod, SystemTarPathOnWindows} from './constants' const IS_WINDOWS = process.platform === 'win32' @@ -16,7 +16,7 @@ async function getTarPath(args: string[]): Promise { if (gnuTar) { // Use GNUtar as default on windows if (args.length > 0) { - args.push('--force-local') + args.push('--force-local') } return gnuTar } else if (existsSync(systemTar)) { @@ -54,7 +54,9 @@ function getWorkingDirectory(): string { } // Common function for extractTar and listTar to get the compression method -async function getCompressionProgram(compressionMethod: CompressionMethod): Promise { +async function getCompressionProgram( + compressionMethod: CompressionMethod +): Promise { // -d: Decompress. // unzstd is equivalent to 'zstd -d' // --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit. @@ -68,7 +70,7 @@ async function getCompressionProgram(compressionMethod: CompressionMethod): Prom } return [ '--use-compress-program', - IS_WINDOWS ? 'zstd -d --long=30' : 'unzstd --long=30', + IS_WINDOWS ? 'zstd -d --long=30' : 'unzstd --long=30' ] case CompressionMethod.ZstdWithoutLong: if (BSD_TAR_ZSTD) { @@ -136,9 +138,7 @@ export async function createTar( switch (compressionMethod) { case CompressionMethod.Zstd: if (BSD_TAR_ZSTD) { - return [ - '-O', '|', 'zstd -T0 --long=30 -o' - ] + return ['-O', '|', 'zstd -T0 --long=30 -o'] } return [ '--use-compress-program', @@ -147,11 +147,13 @@ export async function createTar( ] case CompressionMethod.ZstdWithoutLong: if (BSD_TAR_ZSTD) { - return [ - '-O', '|', 'zstd -T0 -o' - ] + return ['-O', '|', 'zstd -T0 -o'] } - return ['--use-compress-program', IS_WINDOWS ? 'zstd -T0' : 'zstdmt', '-cf'] + return [ + '--use-compress-program', + IS_WINDOWS ? 'zstd -T0' : 'zstdmt', + '-cf' + ] default: return ['-z', '-cf'] } @@ -166,8 +168,7 @@ export async function createTar( '--files-from', manifestFilename, ...(await getCompressionProgram()), - cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), - + cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/') ] await execTar(args, archiveFolder) } From b3bd482c0f13ddeac82b85fb36e393c86dbfd761 Mon Sep 17 00:00:00 2001 From: Lovepreet Singh Date: Thu, 17 Nov 2022 09:46:01 +0000 Subject: [PATCH 10/49] Fix windows gnutar test case --- packages/cache/__tests__/tar.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cache/__tests__/tar.test.ts b/packages/cache/__tests__/tar.test.ts index abf8eecb90..3166ad6834 100644 --- a/packages/cache/__tests__/tar.test.ts +++ b/packages/cache/__tests__/tar.test.ts @@ -113,7 +113,7 @@ test('gzip extract GNU tar on windows with GNUtar in path', async () => { await tar.extractTar(archivePath, CompressionMethod.Gzip) - expect(isGnuMock).toHaveBeenCalledTimes(1) + expect(isGnuMock).toHaveBeenCalledTimes(2) expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledWith( `"tar"`, From f9dfb05bd2c556223750394ee101ff83c5b71da2 Mon Sep 17 00:00:00 2001 From: Lovepreet Singh Date: Thu, 17 Nov 2022 12:13:49 +0000 Subject: [PATCH 11/49] Temporarily remove thhe condition that prevents zstd usage on windows unless with GNUtar --- packages/cache/src/internal/cacheUtils.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/cache/src/internal/cacheUtils.ts b/packages/cache/src/internal/cacheUtils.ts index 64fef08acd..03c5256135 100644 --- a/packages/cache/src/internal/cacheUtils.ts +++ b/packages/cache/src/internal/cacheUtils.ts @@ -94,10 +94,10 @@ async function getVersion(app: string): Promise { // Use zstandard if possible to maximize cache performance export async function getCompressionMethod(): Promise { - if (process.platform === 'win32' && !(await getGnuTarPathOnWindows())) { - // Disable zstd due to bug https://github.com/actions/cache/issues/301 - return CompressionMethod.Gzip - } + // if (process.platform === 'win32' && !(await getGnuTarPathOnWindows())) { + // // Disable zstd due to bug https://github.com/actions/cache/issues/301 + // return CompressionMethod.Gzip + // } const versionOutput = await getVersion('zstd') const version = semver.clean(versionOutput) From 54eb9b8055b5027dd2907ce2e8ec17b60222c190 Mon Sep 17 00:00:00 2001 From: Sampark Sharma Date: Mon, 21 Nov 2022 12:05:03 +0000 Subject: [PATCH 12/49] Address some comments and correct compression commands --- packages/cache/__tests__/tar.test.ts | 12 +++--- packages/cache/src/internal/cacheUtils.ts | 5 --- packages/cache/src/internal/tar.ts | 47 +++++++++++++---------- 3 files changed, 32 insertions(+), 32 deletions(-) diff --git a/packages/cache/__tests__/tar.test.ts b/packages/cache/__tests__/tar.test.ts index 3166ad6834..bdb79cb8f0 100644 --- a/packages/cache/__tests__/tar.test.ts +++ b/packages/cache/__tests__/tar.test.ts @@ -149,6 +149,8 @@ test('zstd create tar', async () => { `"${tarPath}"`, [ '--posix', + '-cf', + IS_WINDOWS ? CacheFilename.Zstd.replace(/\\/g, '/') : CacheFilename.Zstd, '--exclude', IS_WINDOWS ? CacheFilename.Zstd.replace(/\\/g, '/') : CacheFilename.Zstd, '-P', @@ -157,9 +159,7 @@ test('zstd create tar', async () => { '--files-from', 'manifest.txt', '--use-compress-program', - IS_WINDOWS ? 'zstd -T0 --long=30' : 'zstdmt --long=30', - '-cf', - IS_WINDOWS ? CacheFilename.Zstd.replace(/\\/g, '/') : CacheFilename.Zstd + IS_WINDOWS ? 'zstd -T0 --long=30' : 'zstdmt --long=30' ] .concat(IS_WINDOWS ? ['--force-local'] : []) .concat(IS_MAC ? ['--delay-directory-restore'] : []), @@ -187,6 +187,8 @@ test('gzip create tar', async () => { `"${tarPath}"`, [ '--posix', + '-cf', + IS_WINDOWS ? CacheFilename.Gzip.replace(/\\/g, '/') : CacheFilename.Gzip, '--exclude', IS_WINDOWS ? CacheFilename.Gzip.replace(/\\/g, '/') : CacheFilename.Gzip, '-P', @@ -194,9 +196,7 @@ test('gzip create tar', async () => { IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace, '--files-from', 'manifest.txt', - '-z', - '-cf', - IS_WINDOWS ? CacheFilename.Gzip.replace(/\\/g, '/') : CacheFilename.Gzip + '-z' ] .concat(IS_WINDOWS ? ['--force-local'] : []) .concat(IS_MAC ? ['--delay-directory-restore'] : []), diff --git a/packages/cache/src/internal/cacheUtils.ts b/packages/cache/src/internal/cacheUtils.ts index 03c5256135..ea1e7de6ae 100644 --- a/packages/cache/src/internal/cacheUtils.ts +++ b/packages/cache/src/internal/cacheUtils.ts @@ -94,11 +94,6 @@ async function getVersion(app: string): Promise { // Use zstandard if possible to maximize cache performance export async function getCompressionMethod(): Promise { - // if (process.platform === 'win32' && !(await getGnuTarPathOnWindows())) { - // // Disable zstd due to bug https://github.com/actions/cache/issues/301 - // return CompressionMethod.Gzip - // } - const versionOutput = await getVersion('zstd') const version = semver.clean(versionOutput) diff --git a/packages/cache/src/internal/tar.ts b/packages/cache/src/internal/tar.ts index 182bc7bb8c..99c638ecd0 100644 --- a/packages/cache/src/internal/tar.ts +++ b/packages/cache/src/internal/tar.ts @@ -62,10 +62,10 @@ async function getCompressionProgram( // --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit. // Using 30 here because we also support 32-bit self-hosted runners. const tarPath = await getTarPath([]) - const BSD_TAR_ZSTD = IS_WINDOWS && tarPath === SystemTarPathOnWindows + const BSD_TAR_WINDOWS = IS_WINDOWS && tarPath === SystemTarPathOnWindows switch (compressionMethod) { case CompressionMethod.Zstd: - if (BSD_TAR_ZSTD) { + if (BSD_TAR_WINDOWS) { return ['-a'] // auto-detect compression } return [ @@ -73,7 +73,7 @@ async function getCompressionProgram( IS_WINDOWS ? 'zstd -d --long=30' : 'unzstd --long=30' ] case CompressionMethod.ZstdWithoutLong: - if (BSD_TAR_ZSTD) { + if (BSD_TAR_WINDOWS) { return ['a'] // auto-detect compression } return ['--use-compress-program', IS_WINDOWS ? 'zstd -d' : 'unzstd'] @@ -121,6 +121,9 @@ export async function createTar( // Write source directories to manifest.txt to avoid command length limits const manifestFilename = 'manifest.txt' const cacheFileName = utils.getCacheFileName(compressionMethod) + const tarFile = 'cache.tar' + const tarPath = await getTarPath([]) + const BSD_TAR_WINDOWS = IS_WINDOWS && tarPath === SystemTarPathOnWindows writeFileSync( path.join(archiveFolder, manifestFilename), sourceDirectories.join('\n') @@ -133,42 +136,44 @@ export async function createTar( // Using 30 here because we also support 32-bit self-hosted runners. // Long range mode is added to zstd in v1.3.2 release, so we will not use --long in older version of zstd. async function getCompressionProgram(): Promise { - const tarPath = await getTarPath([]) - const BSD_TAR_ZSTD = IS_WINDOWS && tarPath === SystemTarPathOnWindows switch (compressionMethod) { case CompressionMethod.Zstd: - if (BSD_TAR_ZSTD) { - return ['-O', '|', 'zstd -T0 --long=30 -o'] + if (BSD_TAR_WINDOWS) { + return ['&&', 'zstd -T0 --long=30 -o'] } return [ '--use-compress-program', - IS_WINDOWS ? 'zstd -T0 --long=30' : 'zstdmt --long=30', - '-cf' + IS_WINDOWS ? 'zstd -T0 --long=30' : 'zstdmt --long=30' ] case CompressionMethod.ZstdWithoutLong: - if (BSD_TAR_ZSTD) { - return ['-O', '|', 'zstd -T0 -o'] + if (BSD_TAR_WINDOWS) { + return ['&&', 'zstd -T0 -o'] } - return [ - '--use-compress-program', - IS_WINDOWS ? 'zstd -T0' : 'zstdmt', - '-cf' - ] + return ['--use-compress-program', IS_WINDOWS ? 'zstd -T0' : 'zstdmt'] default: - return ['-z', '-cf'] + return ['-z'] } } const args = [ '--posix', + '-cf', + BSD_TAR_WINDOWS + ? tarFile + : cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), '--exclude', - cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), + BSD_TAR_WINDOWS + ? tarFile + : cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), '-P', '-C', workingDirectory.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), '--files-from', manifestFilename, - ...(await getCompressionProgram()), - cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/') - ] + ...(await getCompressionProgram()) + ].concat( + BSD_TAR_WINDOWS + ? [cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/')] + : [] + ) await execTar(args, archiveFolder) } From 32b95825ba44015ec43f5089391e924e6ef061cd Mon Sep 17 00:00:00 2001 From: Sampark Sharma Date: Mon, 21 Nov 2022 12:19:44 +0000 Subject: [PATCH 13/49] Add windows bsdtar test --- packages/cache/__tests__/tar.test.ts | 44 ++++++++++++++++++++++++++++ packages/cache/src/internal/tar.ts | 2 +- 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/packages/cache/__tests__/tar.test.ts b/packages/cache/__tests__/tar.test.ts index bdb79cb8f0..5d58774e5f 100644 --- a/packages/cache/__tests__/tar.test.ts +++ b/packages/cache/__tests__/tar.test.ts @@ -169,6 +169,50 @@ test('zstd create tar', async () => { ) }) +test('zstd create tar with windows BSDtar', async () => { + if (IS_WINDOWS) { + const execMock = jest.spyOn(exec, 'exec') + const isGnuMock = jest + .spyOn(utils, 'getGnuTarPathOnWindows') + .mockReturnValue(Promise.resolve('')) + + const archiveFolder = getTempDir() + const workspace = process.env['GITHUB_WORKSPACE'] + const sourceDirectories = ['~/.npm/cache', `${workspace}/dist`] + + await fs.promises.mkdir(archiveFolder, {recursive: true}) + + await tar.createTar(archiveFolder, sourceDirectories, CompressionMethod.Zstd) + + const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath + + // expect(isGnuMock).toHaveBeenCalledTimes(1) + expect(execMock).toHaveBeenCalledTimes(1) + expect(execMock).toHaveBeenCalledWith( + `"${tarPath}"`, + [ + '--posix', + '-cf', + IS_WINDOWS ? CacheFilename.Zstd.replace(/\\/g, '/') : CacheFilename.Zstd, + '--exclude', + IS_WINDOWS ? CacheFilename.Zstd.replace(/\\/g, '/') : CacheFilename.Zstd, + '-P', + '-C', + IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace, + '--files-from', + 'manifest.txt', + '--use-compress-program', + IS_WINDOWS ? 'zstd -T0 --long=30' : 'zstdmt --long=30' + ] + .concat(IS_WINDOWS ? ['--force-local'] : []) + .concat(IS_MAC ? ['--delay-directory-restore'] : []), + { + cwd: archiveFolder + } + ) + } +}) + test('gzip create tar', async () => { const execMock = jest.spyOn(exec, 'exec') diff --git a/packages/cache/src/internal/tar.ts b/packages/cache/src/internal/tar.ts index 99c638ecd0..964f643738 100644 --- a/packages/cache/src/internal/tar.ts +++ b/packages/cache/src/internal/tar.ts @@ -74,7 +74,7 @@ async function getCompressionProgram( ] case CompressionMethod.ZstdWithoutLong: if (BSD_TAR_WINDOWS) { - return ['a'] // auto-detect compression + return ['-a'] // auto-detect compression } return ['--use-compress-program', IS_WINDOWS ? 'zstd -d' : 'unzstd'] default: From 32f538163db2fb4a6e13eccad3b4a66a7bc50ab2 Mon Sep 17 00:00:00 2001 From: Sampark Sharma Date: Mon, 21 Nov 2022 12:51:07 +0000 Subject: [PATCH 14/49] Fix windows test --- packages/cache/__tests__/tar.test.ts | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/packages/cache/__tests__/tar.test.ts b/packages/cache/__tests__/tar.test.ts index 5d58774e5f..477dd7dcf3 100644 --- a/packages/cache/__tests__/tar.test.ts +++ b/packages/cache/__tests__/tar.test.ts @@ -4,7 +4,8 @@ import * as path from 'path' import { CacheFilename, CompressionMethod, - GnuTarPathOnWindows + GnuTarPathOnWindows, + SystemTarPathOnWindows } from '../src/internal/constants' import * as tar from '../src/internal/tar' import * as utils from '../src/internal/cacheUtils' @@ -179,33 +180,33 @@ test('zstd create tar with windows BSDtar', async () => { const archiveFolder = getTempDir() const workspace = process.env['GITHUB_WORKSPACE'] const sourceDirectories = ['~/.npm/cache', `${workspace}/dist`] + const tarFilename = "cache.tar" await fs.promises.mkdir(archiveFolder, {recursive: true}) await tar.createTar(archiveFolder, sourceDirectories, CompressionMethod.Zstd) - const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath + const tarPath = SystemTarPathOnWindows - // expect(isGnuMock).toHaveBeenCalledTimes(1) + expect(isGnuMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledWith( `"${tarPath}"`, [ '--posix', '-cf', - IS_WINDOWS ? CacheFilename.Zstd.replace(/\\/g, '/') : CacheFilename.Zstd, + tarFilename.replace(/\\/g, '/'), '--exclude', - IS_WINDOWS ? CacheFilename.Zstd.replace(/\\/g, '/') : CacheFilename.Zstd, + tarFilename.replace(/\\/g, '/'), '-P', '-C', - IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace, + workspace?.replace(/\\/g, '/'), '--files-from', 'manifest.txt', - '--use-compress-program', - IS_WINDOWS ? 'zstd -T0 --long=30' : 'zstdmt --long=30' - ] - .concat(IS_WINDOWS ? ['--force-local'] : []) - .concat(IS_MAC ? ['--delay-directory-restore'] : []), + "&&", + "zstd -T0 --long=30 -o", + CacheFilename.Zstd.replace(/\\/g, '/') + ], { cwd: archiveFolder } From ffde3e4bd50fb0dce86b72c6a56c7249c729d60a Mon Sep 17 00:00:00 2001 From: Sampark Sharma Date: Tue, 22 Nov 2022 07:33:01 +0000 Subject: [PATCH 15/49] Fix test --- packages/cache/__tests__/tar.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cache/__tests__/tar.test.ts b/packages/cache/__tests__/tar.test.ts index 477dd7dcf3..3378121414 100644 --- a/packages/cache/__tests__/tar.test.ts +++ b/packages/cache/__tests__/tar.test.ts @@ -188,7 +188,7 @@ test('zstd create tar with windows BSDtar', async () => { const tarPath = SystemTarPathOnWindows - expect(isGnuMock).toHaveBeenCalledTimes(1) + expect(isGnuMock).toHaveBeenCalledTimes(2) expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledWith( `"${tarPath}"`, From 2f73afa8435b06b211571703a2cc28c4e3750e92 Mon Sep 17 00:00:00 2001 From: Sampark Sharma Date: Wed, 23 Nov 2022 07:39:15 +0000 Subject: [PATCH 16/49] Separate args --- packages/cache/__tests__/tar.test.ts | 25 ++-- packages/cache/src/internal/tar.ts | 204 ++++++++++++++++++++------- 2 files changed, 166 insertions(+), 63 deletions(-) diff --git a/packages/cache/__tests__/tar.test.ts b/packages/cache/__tests__/tar.test.ts index 3378121414..4b28c7cb88 100644 --- a/packages/cache/__tests__/tar.test.ts +++ b/packages/cache/__tests__/tar.test.ts @@ -59,13 +59,13 @@ test('zstd extract tar', async () => { expect(execMock).toHaveBeenCalledWith( `"${tarPath}"`, [ - '--use-compress-program', - IS_WINDOWS ? 'zstd -d --long=30' : 'unzstd --long=30', '-xf', IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath, '-P', '-C', - IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace + IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace, + '--use-compress-program', + IS_WINDOWS ? 'zstd -d --long=30' : 'unzstd --long=30' ] .concat(IS_WINDOWS ? ['--force-local'] : []) .concat(IS_MAC ? ['--delay-directory-restore'] : []), @@ -89,12 +89,12 @@ test('gzip extract tar', async () => { expect(execMock).toHaveBeenCalledWith( `"${tarPath}"`, [ - '-z', '-xf', IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath, '-P', '-C', - IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace + IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace, + '-z' ] .concat(IS_WINDOWS ? ['--force-local'] : []) .concat(IS_MAC ? ['--delay-directory-restore'] : []), @@ -180,11 +180,15 @@ test('zstd create tar with windows BSDtar', async () => { const archiveFolder = getTempDir() const workspace = process.env['GITHUB_WORKSPACE'] const sourceDirectories = ['~/.npm/cache', `${workspace}/dist`] - const tarFilename = "cache.tar" + const tarFilename = 'cache.tar' await fs.promises.mkdir(archiveFolder, {recursive: true}) - await tar.createTar(archiveFolder, sourceDirectories, CompressionMethod.Zstd) + await tar.createTar( + archiveFolder, + sourceDirectories, + CompressionMethod.Zstd + ) const tarPath = SystemTarPathOnWindows @@ -203,9 +207,10 @@ test('zstd create tar with windows BSDtar', async () => { workspace?.replace(/\\/g, '/'), '--files-from', 'manifest.txt', - "&&", - "zstd -T0 --long=30 -o", - CacheFilename.Zstd.replace(/\\/g, '/') + '&&', + 'zstd -T0 --long=30 -o', + CacheFilename.Zstd.replace(/\\/g, '/'), + tarFilename.replace(/\\/g, '/') ], { cwd: archiveFolder diff --git a/packages/cache/src/internal/tar.ts b/packages/cache/src/internal/tar.ts index 964f643738..8647fdc01f 100644 --- a/packages/cache/src/internal/tar.ts +++ b/packages/cache/src/internal/tar.ts @@ -8,16 +8,13 @@ import {CompressionMethod, SystemTarPathOnWindows} from './constants' const IS_WINDOWS = process.platform === 'win32' // Function also mutates the args array. For non-mutation call with passing an empty array. -async function getTarPath(args: string[]): Promise { +async function getTarPath(): Promise { switch (process.platform) { case 'win32': { const gnuTar = await utils.getGnuTarPathOnWindows() - const systemTar = `${process.env['windir']}\\System32\\tar.exe` + const systemTar = SystemTarPathOnWindows if (gnuTar) { // Use GNUtar as default on windows - if (args.length > 0) { - args.push('--force-local') - } return gnuTar } else if (existsSync(systemTar)) { return systemTar @@ -28,9 +25,6 @@ async function getTarPath(args: string[]): Promise { const gnuTar = await io.which('gtar', false) if (gnuTar) { // fix permission denied errors when extracting BSD tar archive with GNU tar - https://github.com/actions/cache/issues/527 - if (args.length > 0) { - args.push('--delay-directory-restore') - } return gnuTar } break @@ -41,9 +35,100 @@ async function getTarPath(args: string[]): Promise { return await io.which('tar', true) } +async function getTarArgs( + compressionMethod: CompressionMethod, + type: string, + archivePath = '' +): Promise { + const args = [] + const manifestFilename = 'manifest.txt' + const cacheFileName = utils.getCacheFileName(compressionMethod) + const tarFile = 'cache.tar' + const tarPath = await getTarPath() + const workingDirectory = getWorkingDirectory() + const BSD_TAR_WINDOWS = IS_WINDOWS && tarPath === SystemTarPathOnWindows + + // Method specific args + switch (type) { + case 'create': + args.push( + '--posix', + '-cf', + BSD_TAR_WINDOWS + ? tarFile + : cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), + '--exclude', + BSD_TAR_WINDOWS + ? tarFile + : cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), + '-P', + '-C', + workingDirectory.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), + '--files-from', + manifestFilename + ) + break + case 'extract': + args.push( + '-xf', + BSD_TAR_WINDOWS + ? tarFile + : archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), + '-P', + '-C', + workingDirectory.replace(new RegExp(`\\${path.sep}`, 'g'), '/') + ) + break + // TODO: Correct the below code especially archivePath for BSD_TAR_WINDOWS + case 'list': + args.push( + '-tf', + BSD_TAR_WINDOWS + ? tarFile + : archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), + '-P' + ) + break + } + + // Platform specific args + switch (process.platform) { + case 'win32': { + const gnuTar = await utils.getGnuTarPathOnWindows() + if (gnuTar) { + // Use GNUtar as default on windows + args.push('--force-local') + } + break + } + case 'darwin': { + const gnuTar = await io.which('gtar', false) + if (gnuTar) { + // fix permission denied errors when extracting BSD tar archive with GNU tar - https://github.com/actions/cache/issues/527 + args.push('--delay-directory-restore') + } + break + } + } + + return args +} + async function execTar(args: string[], cwd?: string): Promise { try { - await exec(`"${await getTarPath(args)}"`, args, {cwd}) + await exec(`"${await getTarPath()}"`, args, {cwd}) + } catch (error) { + throw new Error(`Tar failed with error: ${error?.message}`) + } +} + +async function execCommand( + command: string, + args: string[], + cwd?: string +): Promise { + try { + await exec(command, args, {cwd}) } catch (error) { throw new Error(`Tar failed with error: ${error?.message}`) } @@ -61,12 +146,19 @@ async function getCompressionProgram( // unzstd is equivalent to 'zstd -d' // --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit. // Using 30 here because we also support 32-bit self-hosted runners. - const tarPath = await getTarPath([]) + const tarPath = await getTarPath() + const cacheFileName = utils.getCacheFileName(compressionMethod) + const tarFile = 'cache.tar' const BSD_TAR_WINDOWS = IS_WINDOWS && tarPath === SystemTarPathOnWindows switch (compressionMethod) { case CompressionMethod.Zstd: if (BSD_TAR_WINDOWS) { - return ['-a'] // auto-detect compression + return [ + 'zstd -d --long=30 -o', + tarFile, + cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), + '&&' + ] } return [ '--use-compress-program', @@ -74,7 +166,12 @@ async function getCompressionProgram( ] case CompressionMethod.ZstdWithoutLong: if (BSD_TAR_WINDOWS) { - return ['-a'] // auto-detect compression + return [ + 'zstd -d -o', + tarFile, + cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), + '&&' + ] } return ['--use-compress-program', IS_WINDOWS ? 'zstd -d' : 'unzstd'] default: @@ -86,13 +183,19 @@ export async function listTar( archivePath: string, compressionMethod: CompressionMethod ): Promise { - const args = [ - ...(await getCompressionProgram(compressionMethod)), - '-tf', - archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), - '-P' - ] - await execTar(args) + const tarPath = await getTarPath() + const BSD_TAR_WINDOWS = IS_WINDOWS && tarPath === SystemTarPathOnWindows + const compressionArgs = await getCompressionProgram(compressionMethod) + const tarArgs = await getTarArgs(compressionMethod, 'list', archivePath) + // TODO: Add a test for BSD tar on windows + if (BSD_TAR_WINDOWS) { + const command = compressionArgs[0] + const args = compressionArgs.slice(1).concat(tarArgs) + await execCommand(command, args) + } else { + const args = tarArgs.concat(compressionArgs) + await execTar(args) + } } export async function extractTar( @@ -101,16 +204,19 @@ export async function extractTar( ): Promise { // Create directory to extract tar into const workingDirectory = getWorkingDirectory() + const tarPath = await getTarPath() + const BSD_TAR_WINDOWS = IS_WINDOWS && tarPath === SystemTarPathOnWindows await io.mkdirP(workingDirectory) - const args = [ - ...(await getCompressionProgram(compressionMethod)), - '-xf', - archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), - '-P', - '-C', - workingDirectory.replace(new RegExp(`\\${path.sep}`, 'g'), '/') - ] - await execTar(args) + const tarArgs = await getTarArgs(compressionMethod, 'extract', archivePath) + const compressionArgs = await getCompressionProgram(compressionMethod) + if (BSD_TAR_WINDOWS) { + const command = compressionArgs[0] + const args = compressionArgs.slice(1).concat(tarArgs) + await execCommand(command, args) + } else { + const args = tarArgs.concat(compressionArgs) + await execTar(args) + } } export async function createTar( @@ -122,7 +228,7 @@ export async function createTar( const manifestFilename = 'manifest.txt' const cacheFileName = utils.getCacheFileName(compressionMethod) const tarFile = 'cache.tar' - const tarPath = await getTarPath([]) + const tarPath = await getTarPath() const BSD_TAR_WINDOWS = IS_WINDOWS && tarPath === SystemTarPathOnWindows writeFileSync( path.join(archiveFolder, manifestFilename), @@ -135,11 +241,16 @@ export async function createTar( // --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit. // Using 30 here because we also support 32-bit self-hosted runners. // Long range mode is added to zstd in v1.3.2 release, so we will not use --long in older version of zstd. - async function getCompressionProgram(): Promise { + function getCompressionProgram(): string[] { switch (compressionMethod) { case CompressionMethod.Zstd: if (BSD_TAR_WINDOWS) { - return ['&&', 'zstd -T0 --long=30 -o'] + return [ + '&&', + 'zstd -T0 --long=30 -o', + cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), + tarFile + ] } return [ '--use-compress-program', @@ -147,33 +258,20 @@ export async function createTar( ] case CompressionMethod.ZstdWithoutLong: if (BSD_TAR_WINDOWS) { - return ['&&', 'zstd -T0 -o'] + return [ + '&&', + 'zstd -T0 -o', + cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), + tarFile + ] } return ['--use-compress-program', IS_WINDOWS ? 'zstd -T0' : 'zstdmt'] default: return ['-z'] } } - const args = [ - '--posix', - '-cf', - BSD_TAR_WINDOWS - ? tarFile - : cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), - '--exclude', - BSD_TAR_WINDOWS - ? tarFile - : cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), - '-P', - '-C', - workingDirectory.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), - '--files-from', - manifestFilename, - ...(await getCompressionProgram()) - ].concat( - BSD_TAR_WINDOWS - ? [cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/')] - : [] - ) + const tarArgs = await getTarArgs(compressionMethod, 'create') + const compressionArgs = getCompressionProgram() + const args = tarArgs.concat(compressionArgs) await execTar(args, archiveFolder) } From 39b7a86b23c0f2db28788407bb33741e26aaa18f Mon Sep 17 00:00:00 2001 From: Sampark Sharma Date: Wed, 23 Nov 2022 07:58:46 +0000 Subject: [PATCH 17/49] Fix old tests --- packages/cache/__tests__/tar.test.ts | 71 +++++++++++++--------------- 1 file changed, 32 insertions(+), 39 deletions(-) diff --git a/packages/cache/__tests__/tar.test.ts b/packages/cache/__tests__/tar.test.ts index 4b28c7cb88..f179221778 100644 --- a/packages/cache/__tests__/tar.test.ts +++ b/packages/cache/__tests__/tar.test.ts @@ -63,12 +63,14 @@ test('zstd extract tar', async () => { IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath, '-P', '-C', - IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace, - '--use-compress-program', - IS_WINDOWS ? 'zstd -d --long=30' : 'unzstd --long=30' + IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace ] .concat(IS_WINDOWS ? ['--force-local'] : []) - .concat(IS_MAC ? ['--delay-directory-restore'] : []), + .concat(IS_MAC ? ['--delay-directory-restore'] : []) + .concat([ + '--use-compress-program', + IS_WINDOWS ? 'zstd -d --long=30' : 'unzstd --long=30' + ]), {cwd: undefined} ) }) @@ -93,11 +95,11 @@ test('gzip extract tar', async () => { IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath, '-P', '-C', - IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace, - '-z' + IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace ] .concat(IS_WINDOWS ? ['--force-local'] : []) - .concat(IS_MAC ? ['--delay-directory-restore'] : []), + .concat(IS_MAC ? ['--delay-directory-restore'] : []) + .concat(['-z']), {cwd: undefined} ) }) @@ -119,13 +121,13 @@ test('gzip extract GNU tar on windows with GNUtar in path', async () => { expect(execMock).toHaveBeenCalledWith( `"tar"`, [ - '-z', '-xf', archivePath.replace(/\\/g, '/'), '-P', '-C', workspace?.replace(/\\/g, '/'), - '--force-local' + '--force-local', + '-z' ], {cwd: undefined} ) @@ -158,12 +160,14 @@ test('zstd create tar', async () => { '-C', IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace, '--files-from', - 'manifest.txt', - '--use-compress-program', - IS_WINDOWS ? 'zstd -T0 --long=30' : 'zstdmt --long=30' + 'manifest.txt' ] .concat(IS_WINDOWS ? ['--force-local'] : []) - .concat(IS_MAC ? ['--delay-directory-restore'] : []), + .concat(IS_MAC ? ['--delay-directory-restore'] : []) + .concat([ + '--use-compress-program', + IS_WINDOWS ? 'zstd -T0 --long=30' : 'zstdmt --long=30' + ]), { cwd: archiveFolder } @@ -245,11 +249,11 @@ test('gzip create tar', async () => { '-C', IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace, '--files-from', - 'manifest.txt', - '-z' + 'manifest.txt' ] .concat(IS_WINDOWS ? ['--force-local'] : []) - .concat(IS_MAC ? ['--delay-directory-restore'] : []), + .concat(IS_MAC ? ['--delay-directory-restore'] : []) + .concat(['-z']), { cwd: archiveFolder } @@ -269,15 +273,13 @@ test('zstd list tar', async () => { expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledWith( `"${tarPath}"`, - [ - '--use-compress-program', - IS_WINDOWS ? 'zstd -d --long=30' : 'unzstd --long=30', - '-tf', - IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath, - '-P' - ] + ['-tf', IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath, '-P'] .concat(IS_WINDOWS ? ['--force-local'] : []) - .concat(IS_MAC ? ['--delay-directory-restore'] : []), + .concat(IS_MAC ? ['--delay-directory-restore'] : []) + .concat([ + '--use-compress-program', + IS_WINDOWS ? 'zstd -d --long=30' : 'unzstd --long=30' + ]), {cwd: undefined} ) }) @@ -295,15 +297,10 @@ test('zstdWithoutLong list tar', async () => { expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledWith( `"${tarPath}"`, - [ - '--use-compress-program', - IS_WINDOWS ? 'zstd -d' : 'unzstd', - '-tf', - IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath, - '-P' - ] + ['-tf', IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath, '-P'] .concat(IS_WINDOWS ? ['--force-local'] : []) - .concat(IS_MAC ? ['--delay-directory-restore'] : []), + .concat(IS_MAC ? ['--delay-directory-restore'] : []) + .concat(['--use-compress-program', IS_WINDOWS ? 'zstd -d' : 'unzstd']), {cwd: undefined} ) }) @@ -320,14 +317,10 @@ test('gzip list tar', async () => { expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledWith( `"${tarPath}"`, - [ - '-z', - '-tf', - IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath, - '-P' - ] + ['-tf', IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath, '-P'] .concat(IS_WINDOWS ? ['--force-local'] : []) - .concat(IS_MAC ? ['--delay-directory-restore'] : []), + .concat(IS_MAC ? ['--delay-directory-restore'] : []) + .concat(['-z']), {cwd: undefined} ) }) From 1f3371766a9666538ac3f0128f7a8db4ad60a7c9 Mon Sep 17 00:00:00 2001 From: Sampark Sharma Date: Wed, 23 Nov 2022 10:44:38 +0000 Subject: [PATCH 18/49] Add new tests --- packages/cache/__tests__/tar.test.ts | 72 ++++++++++++++-- packages/cache/src/internal/tar.ts | 118 ++++++++++++++------------- 2 files changed, 128 insertions(+), 62 deletions(-) diff --git a/packages/cache/__tests__/tar.test.ts b/packages/cache/__tests__/tar.test.ts index f179221778..36c0a38d01 100644 --- a/packages/cache/__tests__/tar.test.ts +++ b/packages/cache/__tests__/tar.test.ts @@ -18,7 +18,7 @@ jest.mock('@actions/io') const IS_WINDOWS = process.platform === 'win32' const IS_MAC = process.platform === 'darwin' -const defaultTarPath = process.platform === 'darwin' ? 'gtar' : 'tar' +const defaultTarPath = IS_MAC ? 'gtar' : 'tar' function getTempDir(): string { return path.join(__dirname, '_temp', 'tar') @@ -75,6 +75,41 @@ test('zstd extract tar', async () => { ) }) +test('zstd extract tar with windows BSDtar', async () => { + if (IS_WINDOWS) { + const mkdirMock = jest.spyOn(io, 'mkdirP') + const execMock = jest.spyOn(exec, 'exec') + jest + .spyOn(utils, 'getGnuTarPathOnWindows') + .mockReturnValue(Promise.resolve('')) + + const archivePath = `${process.env['windir']}\\fakepath\\cache.tar` + const workspace = process.env['GITHUB_WORKSPACE'] + const tarPath = SystemTarPathOnWindows + const tarFilename = 'cache.tar' + + await tar.extractTar(archivePath, CompressionMethod.Zstd) + + expect(mkdirMock).toHaveBeenCalledWith(workspace) + expect(execMock).toHaveBeenCalledTimes(1) + expect(execMock).toHaveBeenCalledWith( + 'zstd -d --long=30 -o', + [ + tarFilename.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), + archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), + '&&', + `"${tarPath}"`, + '-xf', + tarFilename.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), + '-P', + '-C', + workspace?.replace(/\\/g, '/') + ], + {cwd: undefined} + ) + } +}) + test('gzip extract tar', async () => { const mkdirMock = jest.spyOn(io, 'mkdirP') const execMock = jest.spyOn(exec, 'exec') @@ -107,7 +142,7 @@ test('gzip extract tar', async () => { test('gzip extract GNU tar on windows with GNUtar in path', async () => { if (IS_WINDOWS) { // GNU tar present in path but not at default location - const isGnuMock = jest + jest .spyOn(utils, 'getGnuTarPathOnWindows') .mockReturnValue(Promise.resolve('tar')) const execMock = jest.spyOn(exec, 'exec') @@ -116,7 +151,6 @@ test('gzip extract GNU tar on windows with GNUtar in path', async () => { await tar.extractTar(archivePath, CompressionMethod.Gzip) - expect(isGnuMock).toHaveBeenCalledTimes(2) expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledWith( `"tar"`, @@ -177,7 +211,7 @@ test('zstd create tar', async () => { test('zstd create tar with windows BSDtar', async () => { if (IS_WINDOWS) { const execMock = jest.spyOn(exec, 'exec') - const isGnuMock = jest + jest .spyOn(utils, 'getGnuTarPathOnWindows') .mockReturnValue(Promise.resolve('')) @@ -196,7 +230,6 @@ test('zstd create tar with windows BSDtar', async () => { const tarPath = SystemTarPathOnWindows - expect(isGnuMock).toHaveBeenCalledTimes(2) expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledWith( `"${tarPath}"`, @@ -284,6 +317,35 @@ test('zstd list tar', async () => { ) }) +test('zstd list tar with windows BSDtar', async () => { + if (IS_WINDOWS) { + const execMock = jest.spyOn(exec, 'exec') + jest + .spyOn(utils, 'getGnuTarPathOnWindows') + .mockReturnValue(Promise.resolve('')) + const archivePath = `${process.env['windir']}\\fakepath\\cache.tar` + + await tar.listTar(archivePath, CompressionMethod.Zstd) + + const tarFilename = 'cache.tar' + const tarPath = SystemTarPathOnWindows + expect(execMock).toHaveBeenCalledTimes(1) + expect(execMock).toHaveBeenCalledWith( + 'zstd -d --long=30 -o', + [ + tarFilename.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), + archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), + '&&', + `"${tarPath}"`, + '-tf', + tarFilename.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), + '-P' + ], + {cwd: undefined} + ) + } +}) + test('zstdWithoutLong list tar', async () => { const execMock = jest.spyOn(exec, 'exec') diff --git a/packages/cache/src/internal/tar.ts b/packages/cache/src/internal/tar.ts index 8647fdc01f..27fcf1cac3 100644 --- a/packages/cache/src/internal/tar.ts +++ b/packages/cache/src/internal/tar.ts @@ -46,7 +46,9 @@ async function getTarArgs( const tarFile = 'cache.tar' const tarPath = await getTarPath() const workingDirectory = getWorkingDirectory() - const BSD_TAR_WINDOWS = IS_WINDOWS && tarPath === SystemTarPathOnWindows + const BSD_TAR_ZSTD = + tarPath === SystemTarPathOnWindows && + compressionMethod !== CompressionMethod.Gzip // Method specific args switch (type) { @@ -54,11 +56,11 @@ async function getTarArgs( args.push( '--posix', '-cf', - BSD_TAR_WINDOWS + BSD_TAR_ZSTD ? tarFile : cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), '--exclude', - BSD_TAR_WINDOWS + BSD_TAR_ZSTD ? tarFile : cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), '-P', @@ -71,7 +73,7 @@ async function getTarArgs( case 'extract': args.push( '-xf', - BSD_TAR_WINDOWS + BSD_TAR_ZSTD ? tarFile : archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), '-P', @@ -79,11 +81,10 @@ async function getTarArgs( workingDirectory.replace(new RegExp(`\\${path.sep}`, 'g'), '/') ) break - // TODO: Correct the below code especially archivePath for BSD_TAR_WINDOWS case 'list': args.push( '-tf', - BSD_TAR_WINDOWS + BSD_TAR_ZSTD ? tarFile : archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), '-P' @@ -149,31 +150,31 @@ async function getCompressionProgram( const tarPath = await getTarPath() const cacheFileName = utils.getCacheFileName(compressionMethod) const tarFile = 'cache.tar' - const BSD_TAR_WINDOWS = IS_WINDOWS && tarPath === SystemTarPathOnWindows + const BSD_TAR_ZSTD = + tarPath === SystemTarPathOnWindows && + compressionMethod !== CompressionMethod.Gzip switch (compressionMethod) { case CompressionMethod.Zstd: - if (BSD_TAR_WINDOWS) { - return [ - 'zstd -d --long=30 -o', - tarFile, - cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), - '&&' - ] - } - return [ - '--use-compress-program', - IS_WINDOWS ? 'zstd -d --long=30' : 'unzstd --long=30' - ] + return BSD_TAR_ZSTD + ? [ + 'zstd -d --long=30 -o', + tarFile, + cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), + '&&' + ] + : [ + '--use-compress-program', + IS_WINDOWS ? 'zstd -d --long=30' : 'unzstd --long=30' + ] case CompressionMethod.ZstdWithoutLong: - if (BSD_TAR_WINDOWS) { - return [ - 'zstd -d -o', - tarFile, - cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), - '&&' - ] - } - return ['--use-compress-program', IS_WINDOWS ? 'zstd -d' : 'unzstd'] + return BSD_TAR_ZSTD + ? [ + 'zstd -d -o', + tarFile, + cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), + '&&' + ] + : ['--use-compress-program', IS_WINDOWS ? 'zstd -d' : 'unzstd'] default: return ['-z'] } @@ -184,13 +185,15 @@ export async function listTar( compressionMethod: CompressionMethod ): Promise { const tarPath = await getTarPath() - const BSD_TAR_WINDOWS = IS_WINDOWS && tarPath === SystemTarPathOnWindows + const BSD_TAR_ZSTD = + tarPath === SystemTarPathOnWindows && + compressionMethod !== CompressionMethod.Gzip const compressionArgs = await getCompressionProgram(compressionMethod) const tarArgs = await getTarArgs(compressionMethod, 'list', archivePath) // TODO: Add a test for BSD tar on windows - if (BSD_TAR_WINDOWS) { + if (BSD_TAR_ZSTD) { const command = compressionArgs[0] - const args = compressionArgs.slice(1).concat(tarArgs) + const args = compressionArgs.slice(1).concat([tarPath]).concat(tarArgs) await execCommand(command, args) } else { const args = tarArgs.concat(compressionArgs) @@ -205,13 +208,15 @@ export async function extractTar( // Create directory to extract tar into const workingDirectory = getWorkingDirectory() const tarPath = await getTarPath() - const BSD_TAR_WINDOWS = IS_WINDOWS && tarPath === SystemTarPathOnWindows + const BSD_TAR_ZSTD = + tarPath === SystemTarPathOnWindows && + compressionMethod !== CompressionMethod.Gzip await io.mkdirP(workingDirectory) const tarArgs = await getTarArgs(compressionMethod, 'extract', archivePath) const compressionArgs = await getCompressionProgram(compressionMethod) - if (BSD_TAR_WINDOWS) { + if (BSD_TAR_ZSTD) { const command = compressionArgs[0] - const args = compressionArgs.slice(1).concat(tarArgs) + const args = compressionArgs.slice(1).concat([tarPath]).concat(tarArgs) await execCommand(command, args) } else { const args = tarArgs.concat(compressionArgs) @@ -229,12 +234,13 @@ export async function createTar( const cacheFileName = utils.getCacheFileName(compressionMethod) const tarFile = 'cache.tar' const tarPath = await getTarPath() - const BSD_TAR_WINDOWS = IS_WINDOWS && tarPath === SystemTarPathOnWindows + const BSD_TAR_ZSTD = + tarPath === SystemTarPathOnWindows && + compressionMethod !== CompressionMethod.Gzip writeFileSync( path.join(archiveFolder, manifestFilename), sourceDirectories.join('\n') ) - const workingDirectory = getWorkingDirectory() // -T#: Compress using # working thread. If # is 0, attempt to detect and use the number of physical CPU cores. // zstdmt is equivalent to 'zstd -T0' @@ -244,28 +250,26 @@ export async function createTar( function getCompressionProgram(): string[] { switch (compressionMethod) { case CompressionMethod.Zstd: - if (BSD_TAR_WINDOWS) { - return [ - '&&', - 'zstd -T0 --long=30 -o', - cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), - tarFile - ] - } - return [ - '--use-compress-program', - IS_WINDOWS ? 'zstd -T0 --long=30' : 'zstdmt --long=30' - ] + return BSD_TAR_ZSTD + ? [ + '&&', + 'zstd -T0 --long=30 -o', + cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), + tarFile + ] + : [ + '--use-compress-program', + IS_WINDOWS ? 'zstd -T0 --long=30' : 'zstdmt --long=30' + ] case CompressionMethod.ZstdWithoutLong: - if (BSD_TAR_WINDOWS) { - return [ - '&&', - 'zstd -T0 -o', - cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), - tarFile - ] - } - return ['--use-compress-program', IS_WINDOWS ? 'zstd -T0' : 'zstdmt'] + return BSD_TAR_ZSTD + ? [ + '&&', + 'zstd -T0 -o', + cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), + tarFile + ] + : ['--use-compress-program', IS_WINDOWS ? 'zstd -T0' : 'zstdmt'] default: return ['-z'] } From 187781e27372d5c8a32e2e199c0abb093b86d704 Mon Sep 17 00:00:00 2001 From: Sampark Sharma Date: Wed, 23 Nov 2022 11:40:11 +0000 Subject: [PATCH 19/49] Fix tests --- packages/cache/__tests__/tar.test.ts | 4 ++-- packages/cache/src/internal/tar.ts | 27 ++++++++++++++++++++------- 2 files changed, 22 insertions(+), 9 deletions(-) diff --git a/packages/cache/__tests__/tar.test.ts b/packages/cache/__tests__/tar.test.ts index 36c0a38d01..5d997758f3 100644 --- a/packages/cache/__tests__/tar.test.ts +++ b/packages/cache/__tests__/tar.test.ts @@ -98,7 +98,7 @@ test('zstd extract tar with windows BSDtar', async () => { tarFilename.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), '&&', - `"${tarPath}"`, + `${tarPath}`, '-xf', tarFilename.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), '-P', @@ -336,7 +336,7 @@ test('zstd list tar with windows BSDtar', async () => { tarFilename.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), '&&', - `"${tarPath}"`, + `${tarPath}`, '-tf', tarFilename.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), '-P' diff --git a/packages/cache/src/internal/tar.ts b/packages/cache/src/internal/tar.ts index 27fcf1cac3..a38302ec27 100644 --- a/packages/cache/src/internal/tar.ts +++ b/packages/cache/src/internal/tar.ts @@ -141,7 +141,8 @@ function getWorkingDirectory(): string { // Common function for extractTar and listTar to get the compression method async function getCompressionProgram( - compressionMethod: CompressionMethod + compressionMethod: CompressionMethod, + archivePath: string ): Promise { // -d: Decompress. // unzstd is equivalent to 'zstd -d' @@ -159,7 +160,7 @@ async function getCompressionProgram( ? [ 'zstd -d --long=30 -o', tarFile, - cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), + archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), '&&' ] : [ @@ -171,7 +172,7 @@ async function getCompressionProgram( ? [ 'zstd -d -o', tarFile, - cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), + archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), '&&' ] : ['--use-compress-program', IS_WINDOWS ? 'zstd -d' : 'unzstd'] @@ -188,12 +189,18 @@ export async function listTar( const BSD_TAR_ZSTD = tarPath === SystemTarPathOnWindows && compressionMethod !== CompressionMethod.Gzip - const compressionArgs = await getCompressionProgram(compressionMethod) + const compressionArgs = await getCompressionProgram( + compressionMethod, + archivePath + ) const tarArgs = await getTarArgs(compressionMethod, 'list', archivePath) // TODO: Add a test for BSD tar on windows if (BSD_TAR_ZSTD) { const command = compressionArgs[0] - const args = compressionArgs.slice(1).concat([tarPath]).concat(tarArgs) + const args = compressionArgs + .slice(1) + .concat([tarPath]) + .concat(tarArgs) await execCommand(command, args) } else { const args = tarArgs.concat(compressionArgs) @@ -213,10 +220,16 @@ export async function extractTar( compressionMethod !== CompressionMethod.Gzip await io.mkdirP(workingDirectory) const tarArgs = await getTarArgs(compressionMethod, 'extract', archivePath) - const compressionArgs = await getCompressionProgram(compressionMethod) + const compressionArgs = await getCompressionProgram( + compressionMethod, + archivePath + ) if (BSD_TAR_ZSTD) { const command = compressionArgs[0] - const args = compressionArgs.slice(1).concat([tarPath]).concat(tarArgs) + const args = compressionArgs + .slice(1) + .concat([tarPath]) + .concat(tarArgs) await execCommand(command, args) } else { const args = tarArgs.concat(compressionArgs) From 0822441ee01f9c599f4946878791dbe57b352d79 Mon Sep 17 00:00:00 2001 From: Sampark Sharma Date: Wed, 23 Nov 2022 11:51:46 +0000 Subject: [PATCH 20/49] Fix lint test --- packages/cache/src/internal/tar.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/cache/src/internal/tar.ts b/packages/cache/src/internal/tar.ts index a38302ec27..041f2b5e88 100644 --- a/packages/cache/src/internal/tar.ts +++ b/packages/cache/src/internal/tar.ts @@ -149,7 +149,6 @@ async function getCompressionProgram( // --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit. // Using 30 here because we also support 32-bit self-hosted runners. const tarPath = await getTarPath() - const cacheFileName = utils.getCacheFileName(compressionMethod) const tarFile = 'cache.tar' const BSD_TAR_ZSTD = tarPath === SystemTarPathOnWindows && From 0fd856d0a0b514fe05fb361f6d938c0bc563ac72 Mon Sep 17 00:00:00 2001 From: Sampark Sharma Date: Mon, 28 Nov 2022 10:24:40 +0000 Subject: [PATCH 21/49] Refactor code --- packages/cache/src/internal/constants.ts | 5 + packages/cache/src/internal/contracts.d.ts | 5 + packages/cache/src/internal/tar.ts | 238 ++++++++++----------- 3 files changed, 118 insertions(+), 130 deletions(-) diff --git a/packages/cache/src/internal/constants.ts b/packages/cache/src/internal/constants.ts index c6d8a49094..7d0eb65de5 100644 --- a/packages/cache/src/internal/constants.ts +++ b/packages/cache/src/internal/constants.ts @@ -11,6 +11,11 @@ export enum CompressionMethod { Zstd = 'zstd' } +export enum ArchiveToolType { + GNU = 'gnu', + BSD = 'bsd' +} + // The default number of retry attempts. export const DefaultRetryAttempts = 2 diff --git a/packages/cache/src/internal/contracts.d.ts b/packages/cache/src/internal/contracts.d.ts index 1b2a13a139..4aa4879140 100644 --- a/packages/cache/src/internal/contracts.d.ts +++ b/packages/cache/src/internal/contracts.d.ts @@ -31,3 +31,8 @@ export interface InternalCacheOptions { compressionMethod?: CompressionMethod cacheSize?: number } + +export interface ArchiveTool { + path: string + type: string +} \ No newline at end of file diff --git a/packages/cache/src/internal/tar.ts b/packages/cache/src/internal/tar.ts index 041f2b5e88..00b14fda30 100644 --- a/packages/cache/src/internal/tar.ts +++ b/packages/cache/src/internal/tar.ts @@ -3,21 +3,26 @@ import * as io from '@actions/io' import {existsSync, writeFileSync} from 'fs' import * as path from 'path' import * as utils from './cacheUtils' -import {CompressionMethod, SystemTarPathOnWindows} from './constants' +import {ArchiveTool} from './contracts' +import { + CompressionMethod, + SystemTarPathOnWindows, + ArchiveToolType +} from './constants' const IS_WINDOWS = process.platform === 'win32' // Function also mutates the args array. For non-mutation call with passing an empty array. -async function getTarPath(): Promise { +async function getTarPath(): Promise { switch (process.platform) { case 'win32': { const gnuTar = await utils.getGnuTarPathOnWindows() const systemTar = SystemTarPathOnWindows if (gnuTar) { // Use GNUtar as default on windows - return gnuTar + return {path: gnuTar, type: ArchiveToolType.GNU} } else if (existsSync(systemTar)) { - return systemTar + return {path: systemTar, type: ArchiveToolType.BSD} } break } @@ -25,30 +30,39 @@ async function getTarPath(): Promise { const gnuTar = await io.which('gtar', false) if (gnuTar) { // fix permission denied errors when extracting BSD tar archive with GNU tar - https://github.com/actions/cache/issues/527 - return gnuTar + return {path: gnuTar, type: ArchiveToolType.GNU} + } else { + return { + path: await io.which('tar', true), + type: ArchiveToolType.BSD + } } - break } default: break } - return await io.which('tar', true) + return { + path: await io.which('tar', true), + type: ArchiveToolType.GNU + } } +// Return arguments for tar as per tarPath, compressionMethod, method type and os async function getTarArgs( + tarPath: ArchiveTool, compressionMethod: CompressionMethod, type: string, archivePath = '' ): Promise { - const args = [] + const args = [tarPath.path] const manifestFilename = 'manifest.txt' const cacheFileName = utils.getCacheFileName(compressionMethod) const tarFile = 'cache.tar' - const tarPath = await getTarPath() const workingDirectory = getWorkingDirectory() const BSD_TAR_ZSTD = - tarPath === SystemTarPathOnWindows && - compressionMethod !== CompressionMethod.Gzip + tarPath.type === ArchiveToolType.BSD && + compressionMethod !== CompressionMethod.Gzip && + IS_WINDOWS // Method specific args switch (type) { @@ -93,45 +107,44 @@ async function getTarArgs( } // Platform specific args - switch (process.platform) { - case 'win32': { - const gnuTar = await utils.getGnuTarPathOnWindows() - if (gnuTar) { - // Use GNUtar as default on windows + if (tarPath.type === ArchiveToolType.GNU) { + switch (process.platform) { + case 'win32': args.push('--force-local') - } - break - } - case 'darwin': { - const gnuTar = await io.which('gtar', false) - if (gnuTar) { - // fix permission denied errors when extracting BSD tar archive with GNU tar - https://github.com/actions/cache/issues/527 + break + case 'darwin': args.push('--delay-directory-restore') - } - break + break } } return args } -async function execTar(args: string[], cwd?: string): Promise { - try { - await exec(`"${await getTarPath()}"`, args, {cwd}) - } catch (error) { - throw new Error(`Tar failed with error: ${error?.message}`) - } -} - -async function execCommand( - command: string, - args: string[], - cwd?: string -): Promise { - try { - await exec(command, args, {cwd}) - } catch (error) { - throw new Error(`Tar failed with error: ${error?.message}`) +async function getArgs( + compressionMethod: CompressionMethod, + type: string, + archivePath = '' +): Promise { + const tarPath = await getTarPath() + const tarArgs = await getTarArgs( + tarPath, + compressionMethod, + type, + archivePath + ) + const compressionArgs = + type !== 'create' + ? await getDecompressionProgram(tarPath, compressionMethod, archivePath) + : await getCompressionProgram(tarPath, compressionMethod) + const BSD_TAR_ZSTD = + tarPath.type === ArchiveToolType.BSD && + compressionMethod !== CompressionMethod.Gzip && + IS_WINDOWS + if (BSD_TAR_ZSTD && type !== 'create') { + return [...compressionArgs, ...tarArgs].join(' ') + } else { + return [...tarArgs, ...compressionArgs].join(' ') } } @@ -140,7 +153,8 @@ function getWorkingDirectory(): string { } // Common function for extractTar and listTar to get the compression method -async function getCompressionProgram( +async function getDecompressionProgram( + tarPath: ArchiveTool, compressionMethod: CompressionMethod, archivePath: string ): Promise { @@ -148,11 +162,11 @@ async function getCompressionProgram( // unzstd is equivalent to 'zstd -d' // --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit. // Using 30 here because we also support 32-bit self-hosted runners. - const tarPath = await getTarPath() const tarFile = 'cache.tar' const BSD_TAR_ZSTD = - tarPath === SystemTarPathOnWindows && - compressionMethod !== CompressionMethod.Gzip + tarPath.type === ArchiveToolType.BSD && + compressionMethod !== CompressionMethod.Gzip && + IS_WINDOWS switch (compressionMethod) { case CompressionMethod.Zstd: return BSD_TAR_ZSTD @@ -180,31 +194,54 @@ async function getCompressionProgram( } } +// -T#: Compress using # working thread. If # is 0, attempt to detect and use the number of physical CPU cores. +// zstdmt is equivalent to 'zstd -T0' +// --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit. +// Using 30 here because we also support 32-bit self-hosted runners. +// Long range mode is added to zstd in v1.3.2 release, so we will not use --long in older version of zstd. +async function getCompressionProgram( + tarPath: ArchiveTool, + compressionMethod: CompressionMethod +): Promise { + const cacheFileName = utils.getCacheFileName(compressionMethod) + const tarFile = 'cache.tar' + const BSD_TAR_ZSTD = + tarPath.type === ArchiveToolType.BSD && + compressionMethod !== CompressionMethod.Gzip && + IS_WINDOWS + switch (compressionMethod) { + case CompressionMethod.Zstd: + return BSD_TAR_ZSTD + ? [ + '&&', + 'zstd -T0 --long=30 -o', + cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), + tarFile + ] + : [ + '--use-compress-program', + IS_WINDOWS ? 'zstd -T0 --long=30' : 'zstdmt --long=30' + ] + case CompressionMethod.ZstdWithoutLong: + return BSD_TAR_ZSTD + ? [ + '&&', + 'zstd -T0 -o', + cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), + tarFile + ] + : ['--use-compress-program', IS_WINDOWS ? 'zstd -T0' : 'zstdmt'] + default: + return ['-z'] + } +} + export async function listTar( archivePath: string, compressionMethod: CompressionMethod ): Promise { - const tarPath = await getTarPath() - const BSD_TAR_ZSTD = - tarPath === SystemTarPathOnWindows && - compressionMethod !== CompressionMethod.Gzip - const compressionArgs = await getCompressionProgram( - compressionMethod, - archivePath - ) - const tarArgs = await getTarArgs(compressionMethod, 'list', archivePath) - // TODO: Add a test for BSD tar on windows - if (BSD_TAR_ZSTD) { - const command = compressionArgs[0] - const args = compressionArgs - .slice(1) - .concat([tarPath]) - .concat(tarArgs) - await execCommand(command, args) - } else { - const args = tarArgs.concat(compressionArgs) - await execTar(args) - } + const args = await getArgs(compressionMethod, 'list', archivePath) + exec(args) } export async function extractTar( @@ -213,27 +250,9 @@ export async function extractTar( ): Promise { // Create directory to extract tar into const workingDirectory = getWorkingDirectory() - const tarPath = await getTarPath() - const BSD_TAR_ZSTD = - tarPath === SystemTarPathOnWindows && - compressionMethod !== CompressionMethod.Gzip await io.mkdirP(workingDirectory) - const tarArgs = await getTarArgs(compressionMethod, 'extract', archivePath) - const compressionArgs = await getCompressionProgram( - compressionMethod, - archivePath - ) - if (BSD_TAR_ZSTD) { - const command = compressionArgs[0] - const args = compressionArgs - .slice(1) - .concat([tarPath]) - .concat(tarArgs) - await execCommand(command, args) - } else { - const args = tarArgs.concat(compressionArgs) - await execTar(args) - } + const args = await getArgs(compressionMethod, 'extract', archivePath) + exec(args) } export async function createTar( @@ -243,51 +262,10 @@ export async function createTar( ): Promise { // Write source directories to manifest.txt to avoid command length limits const manifestFilename = 'manifest.txt' - const cacheFileName = utils.getCacheFileName(compressionMethod) - const tarFile = 'cache.tar' - const tarPath = await getTarPath() - const BSD_TAR_ZSTD = - tarPath === SystemTarPathOnWindows && - compressionMethod !== CompressionMethod.Gzip writeFileSync( path.join(archiveFolder, manifestFilename), sourceDirectories.join('\n') ) - - // -T#: Compress using # working thread. If # is 0, attempt to detect and use the number of physical CPU cores. - // zstdmt is equivalent to 'zstd -T0' - // --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit. - // Using 30 here because we also support 32-bit self-hosted runners. - // Long range mode is added to zstd in v1.3.2 release, so we will not use --long in older version of zstd. - function getCompressionProgram(): string[] { - switch (compressionMethod) { - case CompressionMethod.Zstd: - return BSD_TAR_ZSTD - ? [ - '&&', - 'zstd -T0 --long=30 -o', - cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), - tarFile - ] - : [ - '--use-compress-program', - IS_WINDOWS ? 'zstd -T0 --long=30' : 'zstdmt --long=30' - ] - case CompressionMethod.ZstdWithoutLong: - return BSD_TAR_ZSTD - ? [ - '&&', - 'zstd -T0 -o', - cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), - tarFile - ] - : ['--use-compress-program', IS_WINDOWS ? 'zstd -T0' : 'zstdmt'] - default: - return ['-z'] - } - } - const tarArgs = await getTarArgs(compressionMethod, 'create') - const compressionArgs = getCompressionProgram() - const args = tarArgs.concat(compressionArgs) - await execTar(args, archiveFolder) + const args = await getArgs(compressionMethod, 'create') + await exec(args) } From 34f0143be2d12d6299b67df7c6903695f3de84c3 Mon Sep 17 00:00:00 2001 From: Sampark Sharma Date: Tue, 29 Nov 2022 10:35:22 +0000 Subject: [PATCH 22/49] Address review comments --- packages/cache/__tests__/tar.test.ts | 79 ++++++++++++------------ packages/cache/src/internal/constants.ts | 5 ++ packages/cache/src/internal/tar.ts | 17 +++-- 3 files changed, 54 insertions(+), 47 deletions(-) diff --git a/packages/cache/__tests__/tar.test.ts b/packages/cache/__tests__/tar.test.ts index 5d997758f3..e868363f63 100644 --- a/packages/cache/__tests__/tar.test.ts +++ b/packages/cache/__tests__/tar.test.ts @@ -5,7 +5,9 @@ import { CacheFilename, CompressionMethod, GnuTarPathOnWindows, - SystemTarPathOnWindows + ManifestFilename, + SystemTarPathOnWindows, + TarFilename } from '../src/internal/constants' import * as tar from '../src/internal/tar' import * as utils from '../src/internal/cacheUtils' @@ -57,8 +59,8 @@ test('zstd extract tar', async () => { expect(mkdirMock).toHaveBeenCalledWith(workspace) expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledWith( - `"${tarPath}"`, [ + `"${tarPath}"`, '-xf', IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath, '-P', @@ -70,7 +72,8 @@ test('zstd extract tar', async () => { .concat([ '--use-compress-program', IS_WINDOWS ? 'zstd -d --long=30' : 'unzstd --long=30' - ]), + ]) + .join(' '), {cwd: undefined} ) }) @@ -86,25 +89,24 @@ test('zstd extract tar with windows BSDtar', async () => { const archivePath = `${process.env['windir']}\\fakepath\\cache.tar` const workspace = process.env['GITHUB_WORKSPACE'] const tarPath = SystemTarPathOnWindows - const tarFilename = 'cache.tar' await tar.extractTar(archivePath, CompressionMethod.Zstd) expect(mkdirMock).toHaveBeenCalledWith(workspace) expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledWith( - 'zstd -d --long=30 -o', [ - tarFilename.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), + 'zstd -d --long=30 -o', + TarFilename.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), '&&', `${tarPath}`, '-xf', - tarFilename.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), + TarFilename.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), '-P', '-C', workspace?.replace(/\\/g, '/') - ], + ].join(' '), {cwd: undefined} ) } @@ -124,8 +126,8 @@ test('gzip extract tar', async () => { const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledWith( - `"${tarPath}"`, [ + `"${tarPath}"`, '-xf', IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath, '-P', @@ -134,7 +136,8 @@ test('gzip extract tar', async () => { ] .concat(IS_WINDOWS ? ['--force-local'] : []) .concat(IS_MAC ? ['--delay-directory-restore'] : []) - .concat(['-z']), + .concat(['-z']) + .join(' '), {cwd: undefined} ) }) @@ -153,8 +156,8 @@ test('gzip extract GNU tar on windows with GNUtar in path', async () => { expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledWith( - `"tar"`, [ + `"tar"`, '-xf', archivePath.replace(/\\/g, '/'), '-P', @@ -162,7 +165,7 @@ test('gzip extract GNU tar on windows with GNUtar in path', async () => { workspace?.replace(/\\/g, '/'), '--force-local', '-z' - ], + ].join(' '), {cwd: undefined} ) } @@ -183,8 +186,8 @@ test('zstd create tar', async () => { expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledWith( - `"${tarPath}"`, [ + `"${tarPath}"`, '--posix', '-cf', IS_WINDOWS ? CacheFilename.Zstd.replace(/\\/g, '/') : CacheFilename.Zstd, @@ -194,14 +197,15 @@ test('zstd create tar', async () => { '-C', IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace, '--files-from', - 'manifest.txt' + ManifestFilename ] .concat(IS_WINDOWS ? ['--force-local'] : []) .concat(IS_MAC ? ['--delay-directory-restore'] : []) .concat([ '--use-compress-program', IS_WINDOWS ? 'zstd -T0 --long=30' : 'zstdmt --long=30' - ]), + ]) + .join(' '), { cwd: archiveFolder } @@ -218,7 +222,6 @@ test('zstd create tar with windows BSDtar', async () => { const archiveFolder = getTempDir() const workspace = process.env['GITHUB_WORKSPACE'] const sourceDirectories = ['~/.npm/cache', `${workspace}/dist`] - const tarFilename = 'cache.tar' await fs.promises.mkdir(archiveFolder, {recursive: true}) @@ -232,23 +235,23 @@ test('zstd create tar with windows BSDtar', async () => { expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledWith( - `"${tarPath}"`, [ + `"${tarPath}"`, '--posix', '-cf', - tarFilename.replace(/\\/g, '/'), + TarFilename.replace(/\\/g, '/'), '--exclude', - tarFilename.replace(/\\/g, '/'), + TarFilename.replace(/\\/g, '/'), '-P', '-C', workspace?.replace(/\\/g, '/'), '--files-from', - 'manifest.txt', + ManifestFilename, '&&', 'zstd -T0 --long=30 -o', CacheFilename.Zstd.replace(/\\/g, '/'), - tarFilename.replace(/\\/g, '/') - ], + TarFilename.replace(/\\/g, '/') + ].join(' '), { cwd: archiveFolder } @@ -282,11 +285,12 @@ test('gzip create tar', async () => { '-C', IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace, '--files-from', - 'manifest.txt' + ManifestFilename ] .concat(IS_WINDOWS ? ['--force-local'] : []) .concat(IS_MAC ? ['--delay-directory-restore'] : []) - .concat(['-z']), + .concat(['-z']) + .join(' '), { cwd: archiveFolder } @@ -305,14 +309,14 @@ test('zstd list tar', async () => { const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledWith( - `"${tarPath}"`, - ['-tf', IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath, '-P'] + [`"${tarPath}"`, '-tf', IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath, '-P'] .concat(IS_WINDOWS ? ['--force-local'] : []) .concat(IS_MAC ? ['--delay-directory-restore'] : []) .concat([ '--use-compress-program', IS_WINDOWS ? 'zstd -d --long=30' : 'unzstd --long=30' - ]), + ]) + .join(' '), {cwd: undefined} ) }) @@ -327,20 +331,19 @@ test('zstd list tar with windows BSDtar', async () => { await tar.listTar(archivePath, CompressionMethod.Zstd) - const tarFilename = 'cache.tar' const tarPath = SystemTarPathOnWindows expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledWith( - 'zstd -d --long=30 -o', [ - tarFilename.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), + 'zstd -d --long=30 -o', + TarFilename.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), '&&', `${tarPath}`, '-tf', - tarFilename.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), + TarFilename.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), '-P' - ], + ].join(' '), {cwd: undefined} ) } @@ -358,11 +361,11 @@ test('zstdWithoutLong list tar', async () => { const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledWith( - `"${tarPath}"`, - ['-tf', IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath, '-P'] + [`"${tarPath}"`, '-tf', IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath, '-P'] .concat(IS_WINDOWS ? ['--force-local'] : []) .concat(IS_MAC ? ['--delay-directory-restore'] : []) - .concat(['--use-compress-program', IS_WINDOWS ? 'zstd -d' : 'unzstd']), + .concat(['--use-compress-program', IS_WINDOWS ? 'zstd -d' : 'unzstd']) + .join(' '), {cwd: undefined} ) }) @@ -378,11 +381,11 @@ test('gzip list tar', async () => { const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledWith( - `"${tarPath}"`, - ['-tf', IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath, '-P'] + [`"${tarPath}"`, '-tf', IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath, '-P'] .concat(IS_WINDOWS ? ['--force-local'] : []) .concat(IS_MAC ? ['--delay-directory-restore'] : []) - .concat(['-z']), + .concat(['-z']) + .join(' '), {cwd: undefined} ) }) diff --git a/packages/cache/src/internal/constants.ts b/packages/cache/src/internal/constants.ts index 7d0eb65de5..b2cddf96df 100644 --- a/packages/cache/src/internal/constants.ts +++ b/packages/cache/src/internal/constants.ts @@ -30,4 +30,9 @@ export const SocketTimeout = 5000 // The default path of GNUtar on hosted Windows runners export const GnuTarPathOnWindows = `${process.env['PROGRAMFILES']}\\Git\\usr\\bin\\tar.exe` +// The default path of BSDtar on hosted Windows runners export const SystemTarPathOnWindows = `${process.env['SYSTEMDRIVE']}\\Windows\\System32\\tar.exe` + +export const TarFilename = 'cache.tar' + +export const ManifestFilename = 'manifest.txt' \ No newline at end of file diff --git a/packages/cache/src/internal/tar.ts b/packages/cache/src/internal/tar.ts index 00b14fda30..7549fb140c 100644 --- a/packages/cache/src/internal/tar.ts +++ b/packages/cache/src/internal/tar.ts @@ -7,7 +7,9 @@ import {ArchiveTool} from './contracts' import { CompressionMethod, SystemTarPathOnWindows, - ArchiveToolType + ArchiveToolType, + TarFilename, + ManifestFilename } from './constants' const IS_WINDOWS = process.platform === 'win32' @@ -55,7 +57,6 @@ async function getTarArgs( archivePath = '' ): Promise { const args = [tarPath.path] - const manifestFilename = 'manifest.txt' const cacheFileName = utils.getCacheFileName(compressionMethod) const tarFile = 'cache.tar' const workingDirectory = getWorkingDirectory() @@ -81,7 +82,7 @@ async function getTarArgs( '-C', workingDirectory.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), '--files-from', - manifestFilename + ManifestFilename ) break case 'extract': @@ -162,7 +163,6 @@ async function getDecompressionProgram( // unzstd is equivalent to 'zstd -d' // --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit. // Using 30 here because we also support 32-bit self-hosted runners. - const tarFile = 'cache.tar' const BSD_TAR_ZSTD = tarPath.type === ArchiveToolType.BSD && compressionMethod !== CompressionMethod.Gzip && @@ -172,7 +172,7 @@ async function getDecompressionProgram( return BSD_TAR_ZSTD ? [ 'zstd -d --long=30 -o', - tarFile, + TarFilename, archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), '&&' ] @@ -184,7 +184,7 @@ async function getDecompressionProgram( return BSD_TAR_ZSTD ? [ 'zstd -d -o', - tarFile, + TarFilename, archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), '&&' ] @@ -204,7 +204,6 @@ async function getCompressionProgram( compressionMethod: CompressionMethod ): Promise { const cacheFileName = utils.getCacheFileName(compressionMethod) - const tarFile = 'cache.tar' const BSD_TAR_ZSTD = tarPath.type === ArchiveToolType.BSD && compressionMethod !== CompressionMethod.Gzip && @@ -216,7 +215,7 @@ async function getCompressionProgram( '&&', 'zstd -T0 --long=30 -o', cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), - tarFile + TarFilename ] : [ '--use-compress-program', @@ -228,7 +227,7 @@ async function getCompressionProgram( '&&', 'zstd -T0 -o', cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), - tarFile + TarFilename ] : ['--use-compress-program', IS_WINDOWS ? 'zstd -T0' : 'zstdmt'] default: From 2e5a5174604f49b48c910fb50602fdea8e32a2d4 Mon Sep 17 00:00:00 2001 From: Sampark Sharma Date: Tue, 29 Nov 2022 12:16:17 +0000 Subject: [PATCH 23/49] Fix test --- packages/cache/__tests__/tar.test.ts | 50 +++++++++++++++++----------- packages/cache/src/internal/tar.ts | 7 ++-- 2 files changed, 33 insertions(+), 24 deletions(-) diff --git a/packages/cache/__tests__/tar.test.ts b/packages/cache/__tests__/tar.test.ts index e868363f63..2143ad89aa 100644 --- a/packages/cache/__tests__/tar.test.ts +++ b/packages/cache/__tests__/tar.test.ts @@ -73,8 +73,7 @@ test('zstd extract tar', async () => { '--use-compress-program', IS_WINDOWS ? 'zstd -d --long=30' : 'unzstd --long=30' ]) - .join(' '), - {cwd: undefined} + .join(' ') ) }) @@ -106,8 +105,7 @@ test('zstd extract tar with windows BSDtar', async () => { '-P', '-C', workspace?.replace(/\\/g, '/') - ].join(' '), - {cwd: undefined} + ].join(' ') ) } }) @@ -137,8 +135,7 @@ test('gzip extract tar', async () => { .concat(IS_WINDOWS ? ['--force-local'] : []) .concat(IS_MAC ? ['--delay-directory-restore'] : []) .concat(['-z']) - .join(' '), - {cwd: undefined} + .join(' ') ) }) @@ -165,8 +162,7 @@ test('gzip extract GNU tar on windows with GNUtar in path', async () => { workspace?.replace(/\\/g, '/'), '--force-local', '-z' - ].join(' '), - {cwd: undefined} + ].join(' ') ) } }) @@ -206,6 +202,7 @@ test('zstd create tar', async () => { IS_WINDOWS ? 'zstd -T0 --long=30' : 'zstdmt --long=30' ]) .join(' '), + undefined, // args { cwd: archiveFolder } @@ -252,6 +249,7 @@ test('zstd create tar with windows BSDtar', async () => { CacheFilename.Zstd.replace(/\\/g, '/'), TarFilename.replace(/\\/g, '/') ].join(' '), + undefined, // args { cwd: archiveFolder } @@ -274,8 +272,8 @@ test('gzip create tar', async () => { expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledWith( - `"${tarPath}"`, [ + `"${tarPath}"`, '--posix', '-cf', IS_WINDOWS ? CacheFilename.Gzip.replace(/\\/g, '/') : CacheFilename.Gzip, @@ -291,6 +289,7 @@ test('gzip create tar', async () => { .concat(IS_MAC ? ['--delay-directory-restore'] : []) .concat(['-z']) .join(' '), + undefined, // args { cwd: archiveFolder } @@ -309,15 +308,19 @@ test('zstd list tar', async () => { const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledWith( - [`"${tarPath}"`, '-tf', IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath, '-P'] + [ + `"${tarPath}"`, + '-tf', + IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath, + '-P' + ] .concat(IS_WINDOWS ? ['--force-local'] : []) .concat(IS_MAC ? ['--delay-directory-restore'] : []) .concat([ '--use-compress-program', IS_WINDOWS ? 'zstd -d --long=30' : 'unzstd --long=30' ]) - .join(' '), - {cwd: undefined} + .join(' ') ) }) @@ -343,8 +346,7 @@ test('zstd list tar with windows BSDtar', async () => { '-tf', TarFilename.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), '-P' - ].join(' '), - {cwd: undefined} + ].join(' ') ) } }) @@ -361,12 +363,16 @@ test('zstdWithoutLong list tar', async () => { const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledWith( - [`"${tarPath}"`, '-tf', IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath, '-P'] + [ + `"${tarPath}"`, + '-tf', + IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath, + '-P' + ] .concat(IS_WINDOWS ? ['--force-local'] : []) .concat(IS_MAC ? ['--delay-directory-restore'] : []) .concat(['--use-compress-program', IS_WINDOWS ? 'zstd -d' : 'unzstd']) - .join(' '), - {cwd: undefined} + .join(' ') ) }) @@ -381,11 +387,15 @@ test('gzip list tar', async () => { const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledWith( - [`"${tarPath}"`, '-tf', IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath, '-P'] + [ + `"${tarPath}"`, + '-tf', + IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath, + '-P' + ] .concat(IS_WINDOWS ? ['--force-local'] : []) .concat(IS_MAC ? ['--delay-directory-restore'] : []) .concat(['-z']) - .join(' '), - {cwd: undefined} + .join(' ') ) }) diff --git a/packages/cache/src/internal/tar.ts b/packages/cache/src/internal/tar.ts index 7549fb140c..f9f91d15da 100644 --- a/packages/cache/src/internal/tar.ts +++ b/packages/cache/src/internal/tar.ts @@ -56,7 +56,7 @@ async function getTarArgs( type: string, archivePath = '' ): Promise { - const args = [tarPath.path] + const args = [`"${tarPath.path}"`] const cacheFileName = utils.getCacheFileName(compressionMethod) const tarFile = 'cache.tar' const workingDirectory = getWorkingDirectory() @@ -260,11 +260,10 @@ export async function createTar( compressionMethod: CompressionMethod ): Promise { // Write source directories to manifest.txt to avoid command length limits - const manifestFilename = 'manifest.txt' writeFileSync( - path.join(archiveFolder, manifestFilename), + path.join(archiveFolder, ManifestFilename), sourceDirectories.join('\n') ) const args = await getArgs(compressionMethod, 'create') - await exec(args) + await exec(args, undefined, {cwd: archiveFolder}) } From 424ae62ee752742c2b7cff5fdab46aba4701b545 Mon Sep 17 00:00:00 2001 From: Sampark Sharma Date: Wed, 30 Nov 2022 06:05:34 +0000 Subject: [PATCH 24/49] Fix tar test --- packages/cache/__tests__/tar.test.ts | 4 ++-- packages/cache/src/internal/constants.ts | 2 +- packages/cache/src/internal/contracts.d.ts | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/cache/__tests__/tar.test.ts b/packages/cache/__tests__/tar.test.ts index 2143ad89aa..a669ccea5f 100644 --- a/packages/cache/__tests__/tar.test.ts +++ b/packages/cache/__tests__/tar.test.ts @@ -99,7 +99,7 @@ test('zstd extract tar with windows BSDtar', async () => { TarFilename.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), '&&', - `${tarPath}`, + `"${tarPath}"`, '-xf', TarFilename.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), '-P', @@ -342,7 +342,7 @@ test('zstd list tar with windows BSDtar', async () => { TarFilename.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), '&&', - `${tarPath}`, + `"${tarPath}"`, '-tf', TarFilename.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), '-P' diff --git a/packages/cache/src/internal/constants.ts b/packages/cache/src/internal/constants.ts index b2cddf96df..4dbff574a0 100644 --- a/packages/cache/src/internal/constants.ts +++ b/packages/cache/src/internal/constants.ts @@ -35,4 +35,4 @@ export const SystemTarPathOnWindows = `${process.env['SYSTEMDRIVE']}\\Windows\\S export const TarFilename = 'cache.tar' -export const ManifestFilename = 'manifest.txt' \ No newline at end of file +export const ManifestFilename = 'manifest.txt' diff --git a/packages/cache/src/internal/contracts.d.ts b/packages/cache/src/internal/contracts.d.ts index 4aa4879140..d215e26df3 100644 --- a/packages/cache/src/internal/contracts.d.ts +++ b/packages/cache/src/internal/contracts.d.ts @@ -35,4 +35,4 @@ export interface InternalCacheOptions { export interface ArchiveTool { path: string type: string -} \ No newline at end of file +} From afbc5c0a9e6a290c6b2c574193af2c9f56da4413 Mon Sep 17 00:00:00 2001 From: Sampark Sharma Date: Wed, 30 Nov 2022 06:16:19 +0000 Subject: [PATCH 25/49] Add await to async function calls --- packages/cache/src/internal/tar.ts | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/packages/cache/src/internal/tar.ts b/packages/cache/src/internal/tar.ts index f9f91d15da..db05a9e522 100644 --- a/packages/cache/src/internal/tar.ts +++ b/packages/cache/src/internal/tar.ts @@ -240,7 +240,11 @@ export async function listTar( compressionMethod: CompressionMethod ): Promise { const args = await getArgs(compressionMethod, 'list', archivePath) - exec(args) + try { + await exec(args) + } catch (error) { + throw new Error(`Tar failed with error: ${error?.message}`) + } } export async function extractTar( @@ -251,7 +255,11 @@ export async function extractTar( const workingDirectory = getWorkingDirectory() await io.mkdirP(workingDirectory) const args = await getArgs(compressionMethod, 'extract', archivePath) - exec(args) + try { + await exec(args) + } catch (error) { + throw new Error(`Tar failed with error: ${error?.message}`) + } } export async function createTar( @@ -265,5 +273,9 @@ export async function createTar( sourceDirectories.join('\n') ) const args = await getArgs(compressionMethod, 'create') - await exec(args, undefined, {cwd: archiveFolder}) + try { + await exec(args, undefined, {cwd: archiveFolder}) + } catch (error) { + throw new Error(`Tar failed with error: ${error?.message}`) + } } From 85958314526502ff3572b86f6b237fd406cae6e3 Mon Sep 17 00:00:00 2001 From: Sampark Sharma Date: Wed, 30 Nov 2022 10:11:07 +0000 Subject: [PATCH 26/49] Fix test --- packages/cache/__tests__/tar.test.ts | 8 ++++---- packages/cache/src/internal/tar.ts | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/cache/__tests__/tar.test.ts b/packages/cache/__tests__/tar.test.ts index a669ccea5f..c68d0be831 100644 --- a/packages/cache/__tests__/tar.test.ts +++ b/packages/cache/__tests__/tar.test.ts @@ -71,7 +71,7 @@ test('zstd extract tar', async () => { .concat(IS_MAC ? ['--delay-directory-restore'] : []) .concat([ '--use-compress-program', - IS_WINDOWS ? 'zstd -d --long=30' : 'unzstd --long=30' + IS_WINDOWS ? '"zstd -d --long=30"' : 'unzstd --long=30' ]) .join(' ') ) @@ -199,7 +199,7 @@ test('zstd create tar', async () => { .concat(IS_MAC ? ['--delay-directory-restore'] : []) .concat([ '--use-compress-program', - IS_WINDOWS ? 'zstd -T0 --long=30' : 'zstdmt --long=30' + IS_WINDOWS ? '"zstd -T0 --long=30"' : 'zstdmt --long=30' ]) .join(' '), undefined, // args @@ -318,7 +318,7 @@ test('zstd list tar', async () => { .concat(IS_MAC ? ['--delay-directory-restore'] : []) .concat([ '--use-compress-program', - IS_WINDOWS ? 'zstd -d --long=30' : 'unzstd --long=30' + IS_WINDOWS ? '"zstd -d --long=30"' : 'unzstd --long=30' ]) .join(' ') ) @@ -371,7 +371,7 @@ test('zstdWithoutLong list tar', async () => { ] .concat(IS_WINDOWS ? ['--force-local'] : []) .concat(IS_MAC ? ['--delay-directory-restore'] : []) - .concat(['--use-compress-program', IS_WINDOWS ? 'zstd -d' : 'unzstd']) + .concat(['--use-compress-program', IS_WINDOWS ? '"zstd -d"' : 'unzstd']) .join(' ') ) }) diff --git a/packages/cache/src/internal/tar.ts b/packages/cache/src/internal/tar.ts index db05a9e522..ae55d32166 100644 --- a/packages/cache/src/internal/tar.ts +++ b/packages/cache/src/internal/tar.ts @@ -178,7 +178,7 @@ async function getDecompressionProgram( ] : [ '--use-compress-program', - IS_WINDOWS ? 'zstd -d --long=30' : 'unzstd --long=30' + IS_WINDOWS ? '"zstd -d --long=30"' : 'unzstd --long=30' ] case CompressionMethod.ZstdWithoutLong: return BSD_TAR_ZSTD @@ -188,7 +188,7 @@ async function getDecompressionProgram( archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), '&&' ] - : ['--use-compress-program', IS_WINDOWS ? 'zstd -d' : 'unzstd'] + : ['--use-compress-program', IS_WINDOWS ? '"zstd -d"' : 'unzstd'] default: return ['-z'] } @@ -219,7 +219,7 @@ async function getCompressionProgram( ] : [ '--use-compress-program', - IS_WINDOWS ? 'zstd -T0 --long=30' : 'zstdmt --long=30' + IS_WINDOWS ? '"zstd -T0 --long=30"' : 'zstdmt --long=30' ] case CompressionMethod.ZstdWithoutLong: return BSD_TAR_ZSTD @@ -229,7 +229,7 @@ async function getCompressionProgram( cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), TarFilename ] - : ['--use-compress-program', IS_WINDOWS ? 'zstd -T0' : 'zstdmt'] + : ['--use-compress-program', IS_WINDOWS ? '"zstd -T0"' : 'zstdmt'] default: return ['-z'] } From c207fbd5db733d969c3a0655971d3c81610eae69 Mon Sep 17 00:00:00 2001 From: Sampark Sharma Date: Thu, 1 Dec 2022 10:08:48 +0000 Subject: [PATCH 27/49] Update for beta release --- packages/cache/RELEASES.md | 3 +++ packages/cache/package-lock.json | 4 ++-- packages/cache/package.json | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/cache/RELEASES.md b/packages/cache/RELEASES.md index 73518e1c98..59d9423f87 100644 --- a/packages/cache/RELEASES.md +++ b/packages/cache/RELEASES.md @@ -91,3 +91,6 @@ ### 3.0.6 - Added `@azure/abort-controller` to dependencies to fix compatibility issue with ESM [#1208](https://github.com/actions/toolkit/issues/1208) + +### 3.1.0-beta.1 +- Update actions/cache on windows to use gnu tar and zstd by default and fallback to bsdtar and zstd if gnu tar is not available. ([issue](https://github.com/actions/cache/issues/984)) \ No newline at end of file diff --git a/packages/cache/package-lock.json b/packages/cache/package-lock.json index 48f7f7e0cf..142cdaf479 100644 --- a/packages/cache/package-lock.json +++ b/packages/cache/package-lock.json @@ -1,12 +1,12 @@ { "name": "@actions/cache", - "version": "3.0.6", + "version": "3.1.0-beta.1", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@actions/cache", - "version": "3.0.6", + "version": "3.1.0-beta.1", "license": "MIT", "dependencies": { "@actions/core": "^1.10.0", diff --git a/packages/cache/package.json b/packages/cache/package.json index e2d6082c7f..a248a5e8c2 100644 --- a/packages/cache/package.json +++ b/packages/cache/package.json @@ -1,6 +1,6 @@ { "name": "@actions/cache", - "version": "3.0.6", + "version": "3.1.0-beta.1", "preview": true, "description": "Actions cache lib", "keywords": [ From 61e630822aa9b96df050df4ea6001c34c00997aa Mon Sep 17 00:00:00 2001 From: Sampark Sharma Date: Mon, 5 Dec 2022 06:09:05 +0000 Subject: [PATCH 28/49] Fix audit issues --- packages/cache/package-lock.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/cache/package-lock.json b/packages/cache/package-lock.json index 142cdaf479..8420e2e03e 100644 --- a/packages/cache/package-lock.json +++ b/packages/cache/package-lock.json @@ -457,9 +457,9 @@ } }, "node_modules/minimatch": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", - "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", "dependencies": { "brace-expansion": "^1.1.7" }, @@ -998,9 +998,9 @@ } }, "minimatch": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", - "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", "requires": { "brace-expansion": "^1.1.7" } From 1dae85574671bf3939d2b9c524f1a5cfc3ca8e58 Mon Sep 17 00:00:00 2001 From: Sampark Sharma Date: Tue, 6 Dec 2022 11:38:17 +0000 Subject: [PATCH 29/49] Add fallback to gzip compression if cache not found --- packages/cache/src/cache.ts | 41 +++++++++++++++++++++++++++++-------- 1 file changed, 32 insertions(+), 9 deletions(-) diff --git a/packages/cache/src/cache.ts b/packages/cache/src/cache.ts index 609c7f94cb..d29eeda20c 100644 --- a/packages/cache/src/cache.ts +++ b/packages/cache/src/cache.ts @@ -4,6 +4,8 @@ import * as utils from './internal/cacheUtils' import * as cacheHttpClient from './internal/cacheHttpClient' import {createTar, extractTar, listTar} from './internal/tar' import {DownloadOptions, UploadOptions} from './options' +import {CompressionMethod} from './internal/constants' +import {ArtifactCacheEntry} from './internal/contracts' export class ValidationError extends Error { constructor(message: string) { @@ -85,17 +87,38 @@ export async function restoreCache( checkKey(key) } - const compressionMethod = await utils.getCompressionMethod() + let cacheEntry: ArtifactCacheEntry | null + let compressionMethod = await utils.getCompressionMethod() let archivePath = '' try { - // path are needed to compute version - const cacheEntry = await cacheHttpClient.getCacheEntry(keys, paths, { - compressionMethod - }) - - if (!cacheEntry?.archiveLocation) { - // Cache not found - return undefined + try { + // path are needed to compute version + cacheEntry = await cacheHttpClient.getCacheEntry(keys, paths, { + compressionMethod + }) + + if (!cacheEntry?.archiveLocation) { + // Cache not found + return undefined + } + } catch (error) { + if ( + process.platform == 'win32' && + compressionMethod != CompressionMethod.Gzip + ) { + // On windows, we will try to download the cache entry with the same key + // but with different compression method. This is to support the old cache entry created + // by the old version of the cache action. + compressionMethod = CompressionMethod.Gzip + cacheEntry = await cacheHttpClient.getCacheEntry(keys, paths, { + compressionMethod + }) + if (!cacheEntry?.archiveLocation) { + throw error + } + } else { + throw error + } } archivePath = path.join( From c0085d773988fba7addf6c7f87f97b0a27d47b7f Mon Sep 17 00:00:00 2001 From: Sampark Sharma Date: Tue, 6 Dec 2022 11:52:46 +0000 Subject: [PATCH 30/49] Fix test --- packages/cache/src/cache.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/cache/src/cache.ts b/packages/cache/src/cache.ts index d29eeda20c..0efaecfe1b 100644 --- a/packages/cache/src/cache.ts +++ b/packages/cache/src/cache.ts @@ -103,8 +103,8 @@ export async function restoreCache( } } catch (error) { if ( - process.platform == 'win32' && - compressionMethod != CompressionMethod.Gzip + process.platform === 'win32' && + compressionMethod !== CompressionMethod.Gzip ) { // On windows, we will try to download the cache entry with the same key // but with different compression method. This is to support the old cache entry created From d1094e15235370f00f7bf3ad8250970003935107 Mon Sep 17 00:00:00 2001 From: Sampark Sharma Date: Wed, 7 Dec 2022 07:39:41 +0000 Subject: [PATCH 31/49] Add test --- packages/cache/__tests__/restoreCache.test.ts | 75 +++++++++++++++++++ packages/cache/src/cache.ts | 5 +- 2 files changed, 77 insertions(+), 3 deletions(-) diff --git a/packages/cache/__tests__/restoreCache.test.ts b/packages/cache/__tests__/restoreCache.test.ts index 36ec880129..2c9b4525d1 100644 --- a/packages/cache/__tests__/restoreCache.test.ts +++ b/packages/cache/__tests__/restoreCache.test.ts @@ -161,6 +161,81 @@ test('restore with gzip compressed cache found', async () => { expect(getCompressionMock).toHaveBeenCalledTimes(1) }) +test('restore with zstd as default but gzip compressed cache found on windows', async () => { + if (process.platform === 'win32') { + const paths = ['node_modules'] + const key = 'node-test' + + const cacheEntry: ArtifactCacheEntry = { + cacheKey: key, + scope: 'refs/heads/main', + archiveLocation: 'www.actionscache.test/download' + } + const getCacheMock = jest.spyOn(cacheHttpClient, 'getCacheEntry') + getCacheMock + .mockImplementationOnce(async () => { + return Promise.resolve({}) + }) + .mockImplementationOnce(async () => { + return Promise.resolve(cacheEntry) + }) + + const tempPath = '/foo/bar' + + const createTempDirectoryMock = jest.spyOn( + cacheUtils, + 'createTempDirectory' + ) + createTempDirectoryMock.mockImplementation(async () => { + return Promise.resolve(tempPath) + }) + + const archivePath = path.join(tempPath, CacheFilename.Gzip) + const downloadCacheMock = jest.spyOn(cacheHttpClient, 'downloadCache') + + const fileSize = 142 + const getArchiveFileSizeInBytesMock = jest + .spyOn(cacheUtils, 'getArchiveFileSizeInBytes') + .mockReturnValue(fileSize) + + const extractTarMock = jest.spyOn(tar, 'extractTar') + const unlinkFileMock = jest.spyOn(cacheUtils, 'unlinkFile') + + const compression = CompressionMethod.Zstd + const getCompressionMock = jest + .spyOn(cacheUtils, 'getCompressionMethod') + .mockReturnValue(Promise.resolve(compression)) + + const cacheKey = await restoreCache(paths, key) + + expect(cacheKey).toBe(key) + expect(getCacheMock).toHaveBeenNthCalledWith(1, [key], paths, { + compressionMethod: compression + }) + expect(getCacheMock).toHaveBeenNthCalledWith(2, [key], paths, { + compressionMethod: CompressionMethod.Gzip + }) + expect(createTempDirectoryMock).toHaveBeenCalledTimes(1) + expect(downloadCacheMock).toHaveBeenCalledWith( + cacheEntry.archiveLocation, + archivePath, + undefined + ) + expect(getArchiveFileSizeInBytesMock).toHaveBeenCalledWith(archivePath) + + expect(extractTarMock).toHaveBeenCalledTimes(1) + expect(extractTarMock).toHaveBeenCalledWith( + archivePath, + CompressionMethod.Gzip + ) + + expect(unlinkFileMock).toHaveBeenCalledTimes(1) + expect(unlinkFileMock).toHaveBeenCalledWith(archivePath) + + expect(getCompressionMock).toHaveBeenCalledTimes(1) + } +}) + test('restore with zstd compressed cache found', async () => { const paths = ['node_modules'] const key = 'node-test' diff --git a/packages/cache/src/cache.ts b/packages/cache/src/cache.ts index 0efaecfe1b..c0d0f85a47 100644 --- a/packages/cache/src/cache.ts +++ b/packages/cache/src/cache.ts @@ -106,9 +106,8 @@ export async function restoreCache( process.platform === 'win32' && compressionMethod !== CompressionMethod.Gzip ) { - // On windows, we will try to download the cache entry with the same key - // but with different compression method. This is to support the old cache entry created - // by the old version of the cache action. + // This is to support the old cache entry created + // by the old version of the cache action on windows. compressionMethod = CompressionMethod.Gzip cacheEntry = await cacheHttpClient.getCacheEntry(keys, paths, { compressionMethod From d79a09bc0e4f1dde4d6ebd0ad455f01cad5f1321 Mon Sep 17 00:00:00 2001 From: Sampark Sharma Date: Wed, 7 Dec 2022 08:27:08 +0000 Subject: [PATCH 32/49] Address review comments --- packages/cache/__tests__/restoreCache.test.ts | 2 +- packages/cache/src/cache.ts | 31 ++++++------------- .../cache/src/internal/cacheHttpClient.ts | 26 ++++++++++++++++ 3 files changed, 37 insertions(+), 22 deletions(-) diff --git a/packages/cache/__tests__/restoreCache.test.ts b/packages/cache/__tests__/restoreCache.test.ts index 2c9b4525d1..82b259b9b9 100644 --- a/packages/cache/__tests__/restoreCache.test.ts +++ b/packages/cache/__tests__/restoreCache.test.ts @@ -174,7 +174,7 @@ test('restore with zstd as default but gzip compressed cache found on windows', const getCacheMock = jest.spyOn(cacheHttpClient, 'getCacheEntry') getCacheMock .mockImplementationOnce(async () => { - return Promise.resolve({}) + throw new Error('Cache not found.') }) .mockImplementationOnce(async () => { return Promise.resolve(cacheEntry) diff --git a/packages/cache/src/cache.ts b/packages/cache/src/cache.ts index c0d0f85a47..b2185e5e6e 100644 --- a/packages/cache/src/cache.ts +++ b/packages/cache/src/cache.ts @@ -96,30 +96,19 @@ export async function restoreCache( cacheEntry = await cacheHttpClient.getCacheEntry(keys, paths, { compressionMethod }) - - if (!cacheEntry?.archiveLocation) { - // Cache not found - return undefined - } } catch (error) { - if ( - process.platform === 'win32' && - compressionMethod !== CompressionMethod.Gzip - ) { - // This is to support the old cache entry created - // by the old version of the cache action on windows. - compressionMethod = CompressionMethod.Gzip - cacheEntry = await cacheHttpClient.getCacheEntry(keys, paths, { - compressionMethod - }) - if (!cacheEntry?.archiveLocation) { - throw error - } - } else { - throw error - } + cacheEntry = await cacheHttpClient.getCacheEntryForGzipFallbackOnWindows( + keys, + paths, + compressionMethod, + error + ) } + if (!cacheEntry?.archiveLocation) { + // Cache not found + return undefined + } archivePath = path.join( await utils.createTempDirectory(), utils.getCacheFileName(compressionMethod) diff --git a/packages/cache/src/internal/cacheHttpClient.ts b/packages/cache/src/internal/cacheHttpClient.ts index c66d1a73e7..048826129d 100644 --- a/packages/cache/src/internal/cacheHttpClient.ts +++ b/packages/cache/src/internal/cacheHttpClient.ts @@ -122,6 +122,32 @@ export async function getCacheEntry( return cacheResult } +// This is to support the old cache entry created +// by the old version of the cache action on windows. +export async function getCacheEntryForGzipFallbackOnWindows( + keys: string[], + paths: string[], + compressionMethod: CompressionMethod, + error: unknown +): Promise { + let cacheEntry: ArtifactCacheEntry | null + if ( + process.platform === 'win32' && + compressionMethod !== CompressionMethod.Gzip + ) { + compressionMethod = CompressionMethod.Gzip + cacheEntry = await getCacheEntry(keys, paths, { + compressionMethod + }) + if (!cacheEntry?.archiveLocation) { + throw error + } + } else { + throw error + } + return cacheEntry +} + export async function downloadCache( archiveLocation: string, archivePath: string, From 27f9a7d4619bde4df165846e435d9e2795ab3bea Mon Sep 17 00:00:00 2001 From: Sampark Sharma Date: Wed, 7 Dec 2022 08:41:56 +0000 Subject: [PATCH 33/49] Revert Address review comments --- packages/cache/src/cache.ts | 22 +++++++++++----- .../cache/src/internal/cacheHttpClient.ts | 26 ------------------- 2 files changed, 16 insertions(+), 32 deletions(-) diff --git a/packages/cache/src/cache.ts b/packages/cache/src/cache.ts index b2185e5e6e..3840e4391f 100644 --- a/packages/cache/src/cache.ts +++ b/packages/cache/src/cache.ts @@ -97,12 +97,22 @@ export async function restoreCache( compressionMethod }) } catch (error) { - cacheEntry = await cacheHttpClient.getCacheEntryForGzipFallbackOnWindows( - keys, - paths, - compressionMethod, - error - ) + // This is to support the old cache entry created + // by the old version of the cache action on windows. + if ( + process.platform === 'win32' && + compressionMethod !== CompressionMethod.Gzip + ) { + compressionMethod = CompressionMethod.Gzip + cacheEntry = await cacheHttpClient.getCacheEntry(keys, paths, { + compressionMethod + }) + if (!cacheEntry?.archiveLocation) { + throw error + } + } else { + throw error + } } if (!cacheEntry?.archiveLocation) { diff --git a/packages/cache/src/internal/cacheHttpClient.ts b/packages/cache/src/internal/cacheHttpClient.ts index 048826129d..c66d1a73e7 100644 --- a/packages/cache/src/internal/cacheHttpClient.ts +++ b/packages/cache/src/internal/cacheHttpClient.ts @@ -122,32 +122,6 @@ export async function getCacheEntry( return cacheResult } -// This is to support the old cache entry created -// by the old version of the cache action on windows. -export async function getCacheEntryForGzipFallbackOnWindows( - keys: string[], - paths: string[], - compressionMethod: CompressionMethod, - error: unknown -): Promise { - let cacheEntry: ArtifactCacheEntry | null - if ( - process.platform === 'win32' && - compressionMethod !== CompressionMethod.Gzip - ) { - compressionMethod = CompressionMethod.Gzip - cacheEntry = await getCacheEntry(keys, paths, { - compressionMethod - }) - if (!cacheEntry?.archiveLocation) { - throw error - } - } else { - throw error - } - return cacheEntry -} - export async function downloadCache( archiveLocation: string, archivePath: string, From 8e39d78020aabce129764db03f2e84246a8c0ca7 Mon Sep 17 00:00:00 2001 From: Sampark Sharma Date: Thu, 8 Dec 2022 07:13:15 +0000 Subject: [PATCH 34/49] Release 3.1.0-beta.2 cache package --- packages/cache/RELEASES.md | 5 ++++- packages/cache/package-lock.json | 4 ++-- packages/cache/package.json | 2 +- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/packages/cache/RELEASES.md b/packages/cache/RELEASES.md index 59d9423f87..ddd458a95a 100644 --- a/packages/cache/RELEASES.md +++ b/packages/cache/RELEASES.md @@ -93,4 +93,7 @@ - Added `@azure/abort-controller` to dependencies to fix compatibility issue with ESM [#1208](https://github.com/actions/toolkit/issues/1208) ### 3.1.0-beta.1 -- Update actions/cache on windows to use gnu tar and zstd by default and fallback to bsdtar and zstd if gnu tar is not available. ([issue](https://github.com/actions/cache/issues/984)) \ No newline at end of file +- Update actions/cache on windows to use gnu tar and zstd by default and fallback to bsdtar and zstd if gnu tar is not available. ([issue](https://github.com/actions/cache/issues/984)) + +### 3.1.0-beta.2 +- Added support for fallback to gzip to restore old caches on windows. diff --git a/packages/cache/package-lock.json b/packages/cache/package-lock.json index 8420e2e03e..70f7933d7e 100644 --- a/packages/cache/package-lock.json +++ b/packages/cache/package-lock.json @@ -1,12 +1,12 @@ { "name": "@actions/cache", - "version": "3.1.0-beta.1", + "version": "3.1.0-beta.2", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@actions/cache", - "version": "3.1.0-beta.1", + "version": "3.1.0-beta.2", "license": "MIT", "dependencies": { "@actions/core": "^1.10.0", diff --git a/packages/cache/package.json b/packages/cache/package.json index a248a5e8c2..1fb3fea717 100644 --- a/packages/cache/package.json +++ b/packages/cache/package.json @@ -1,6 +1,6 @@ { "name": "@actions/cache", - "version": "3.1.0-beta.1", + "version": "3.1.0-beta.2", "preview": true, "description": "Actions cache lib", "keywords": [ From d31c2dd88dc1f685a699fca39223eb3ca0913180 Mon Sep 17 00:00:00 2001 From: Sampark Sharma Date: Fri, 9 Dec 2022 07:56:15 +0000 Subject: [PATCH 35/49] Fix issues --- packages/cache/src/cache.ts | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/packages/cache/src/cache.ts b/packages/cache/src/cache.ts index 3840e4391f..c395d44807 100644 --- a/packages/cache/src/cache.ts +++ b/packages/cache/src/cache.ts @@ -92,21 +92,29 @@ export async function restoreCache( let archivePath = '' try { try { + console.log('before first get cache entry') // path are needed to compute version cacheEntry = await cacheHttpClient.getCacheEntry(keys, paths, { compressionMethod }) + console.log('after first get cache entry') + console.log(cacheEntry) } catch (error) { // This is to support the old cache entry created // by the old version of the cache action on windows. + console.log('in first catch block') if ( process.platform === 'win32' && compressionMethod !== CompressionMethod.Gzip ) { + console.log( + "Couldn't find cache entry with zstd compression, falling back to gzip compression" + ) compressionMethod = CompressionMethod.Gzip cacheEntry = await cacheHttpClient.getCacheEntry(keys, paths, { compressionMethod }) + console.log(cacheEntry) if (!cacheEntry?.archiveLocation) { throw error } @@ -148,6 +156,8 @@ export async function restoreCache( return cacheEntry.cacheKey } catch (error) { + console.log('In second catch block') + console.log(error) const typedError = error as Error if (typedError.name === ValidationError.name) { throw error @@ -183,7 +193,8 @@ export async function saveCache( checkPaths(paths) checkKey(key) - const compressionMethod = await utils.getCompressionMethod() + // const compressionMethod = await utils.getCompressionMethod() + const compressionMethod = CompressionMethod.Gzip let cacheId = -1 const cachePaths = await utils.resolvePaths(paths) From 7a532d03f43c81fc08c18339e3eabf3e2c54ead1 Mon Sep 17 00:00:00 2001 From: Sampark Sharma Date: Fri, 9 Dec 2022 09:33:59 +0000 Subject: [PATCH 36/49] Reconfigure catch block --- packages/cache/__tests__/restoreCache.test.ts | 2 +- packages/cache/src/cache.ts | 30 ++++++------------- packages/cache/src/internal/tar.ts | 13 ++++++-- 3 files changed, 21 insertions(+), 24 deletions(-) diff --git a/packages/cache/__tests__/restoreCache.test.ts b/packages/cache/__tests__/restoreCache.test.ts index 82b259b9b9..9cf457994b 100644 --- a/packages/cache/__tests__/restoreCache.test.ts +++ b/packages/cache/__tests__/restoreCache.test.ts @@ -174,7 +174,7 @@ test('restore with zstd as default but gzip compressed cache found on windows', const getCacheMock = jest.spyOn(cacheHttpClient, 'getCacheEntry') getCacheMock .mockImplementationOnce(async () => { - throw new Error('Cache not found.') + return Promise.resolve(null) }) .mockImplementationOnce(async () => { return Promise.resolve(cacheEntry) diff --git a/packages/cache/src/cache.ts b/packages/cache/src/cache.ts index c395d44807..8d8bbd2cf9 100644 --- a/packages/cache/src/cache.ts +++ b/packages/cache/src/cache.ts @@ -91,18 +91,13 @@ export async function restoreCache( let compressionMethod = await utils.getCompressionMethod() let archivePath = '' try { - try { - console.log('before first get cache entry') - // path are needed to compute version - cacheEntry = await cacheHttpClient.getCacheEntry(keys, paths, { - compressionMethod - }) - console.log('after first get cache entry') - console.log(cacheEntry) - } catch (error) { + // path are needed to compute version + cacheEntry = await cacheHttpClient.getCacheEntry(keys, paths, { + compressionMethod + }) + if (!cacheEntry?.archiveLocation) { // This is to support the old cache entry created // by the old version of the cache action on windows. - console.log('in first catch block') if ( process.platform === 'win32' && compressionMethod !== CompressionMethod.Gzip @@ -114,19 +109,15 @@ export async function restoreCache( cacheEntry = await cacheHttpClient.getCacheEntry(keys, paths, { compressionMethod }) - console.log(cacheEntry) if (!cacheEntry?.archiveLocation) { - throw error + return undefined } } else { - throw error + // Cache not found + return undefined } } - if (!cacheEntry?.archiveLocation) { - // Cache not found - return undefined - } archivePath = path.join( await utils.createTempDirectory(), utils.getCacheFileName(compressionMethod) @@ -156,8 +147,6 @@ export async function restoreCache( return cacheEntry.cacheKey } catch (error) { - console.log('In second catch block') - console.log(error) const typedError = error as Error if (typedError.name === ValidationError.name) { throw error @@ -193,8 +182,7 @@ export async function saveCache( checkPaths(paths) checkKey(key) - // const compressionMethod = await utils.getCompressionMethod() - const compressionMethod = CompressionMethod.Gzip + const compressionMethod = await utils.getCompressionMethod() let cacheId = -1 const cachePaths = await utils.resolvePaths(paths) diff --git a/packages/cache/src/internal/tar.ts b/packages/cache/src/internal/tar.ts index ae55d32166..f2f7591856 100644 --- a/packages/cache/src/internal/tar.ts +++ b/packages/cache/src/internal/tar.ts @@ -127,6 +127,8 @@ async function getArgs( type: string, archivePath = '' ): Promise { + let args: string + const tarPath = await getTarPath() const tarArgs = await getTarArgs( tarPath, @@ -142,11 +144,18 @@ async function getArgs( tarPath.type === ArchiveToolType.BSD && compressionMethod !== CompressionMethod.Gzip && IS_WINDOWS + if (BSD_TAR_ZSTD && type !== 'create') { - return [...compressionArgs, ...tarArgs].join(' ') + args = [...compressionArgs, ...tarArgs].join(' ') } else { - return [...tarArgs, ...compressionArgs].join(' ') + args = [...tarArgs, ...compressionArgs].join(' ') + } + + if (BSD_TAR_ZSTD) { + args = ['cmd /c "', args, '"'].join(' ') } + + return args } function getWorkingDirectory(): string { From 0c23c38c68eb4cc62c95231b67c8656a0110b5d9 Mon Sep 17 00:00:00 2001 From: Sampark Sharma Date: Fri, 9 Dec 2022 11:06:42 +0000 Subject: [PATCH 37/49] Add debug logging for gzip fall back --- packages/cache/src/cache.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/cache/src/cache.ts b/packages/cache/src/cache.ts index 8d8bbd2cf9..d15975c311 100644 --- a/packages/cache/src/cache.ts +++ b/packages/cache/src/cache.ts @@ -102,9 +102,6 @@ export async function restoreCache( process.platform === 'win32' && compressionMethod !== CompressionMethod.Gzip ) { - console.log( - "Couldn't find cache entry with zstd compression, falling back to gzip compression" - ) compressionMethod = CompressionMethod.Gzip cacheEntry = await cacheHttpClient.getCacheEntry(keys, paths, { compressionMethod @@ -112,6 +109,10 @@ export async function restoreCache( if (!cacheEntry?.archiveLocation) { return undefined } + + core.debug( + "Couldn't find cache entry with zstd compression, falling back to gzip compression" + ) } else { // Cache not found return undefined From 0690c105158181754fb962f4f23edfeb42543d55 Mon Sep 17 00:00:00 2001 From: Sampark Sharma Date: Fri, 9 Dec 2022 11:47:00 +0000 Subject: [PATCH 38/49] Fix test --- packages/cache/__tests__/tar.test.ts | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/packages/cache/__tests__/tar.test.ts b/packages/cache/__tests__/tar.test.ts index c68d0be831..efd38d8588 100644 --- a/packages/cache/__tests__/tar.test.ts +++ b/packages/cache/__tests__/tar.test.ts @@ -95,6 +95,7 @@ test('zstd extract tar with windows BSDtar', async () => { expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledWith( [ + 'cmd /c "', 'zstd -d --long=30 -o', TarFilename.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), @@ -104,7 +105,8 @@ test('zstd extract tar with windows BSDtar', async () => { TarFilename.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), '-P', '-C', - workspace?.replace(/\\/g, '/') + workspace?.replace(/\\/g, '/'), + '"' // end cmd /c ].join(' ') ) } @@ -233,6 +235,7 @@ test('zstd create tar with windows BSDtar', async () => { expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledWith( [ + 'cmd /c "', `"${tarPath}"`, '--posix', '-cf', @@ -247,7 +250,8 @@ test('zstd create tar with windows BSDtar', async () => { '&&', 'zstd -T0 --long=30 -o', CacheFilename.Zstd.replace(/\\/g, '/'), - TarFilename.replace(/\\/g, '/') + TarFilename.replace(/\\/g, '/'), + '"' // end cmd /c ].join(' '), undefined, // args { @@ -338,6 +342,7 @@ test('zstd list tar with windows BSDtar', async () => { expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledWith( [ + 'cmd /c "', 'zstd -d --long=30 -o', TarFilename.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), @@ -345,7 +350,8 @@ test('zstd list tar with windows BSDtar', async () => { `"${tarPath}"`, '-tf', TarFilename.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), - '-P' + '-P', + '"' // end cmd /c ].join(' ') ) } From d175a181a08ea8fe5433c7b0b147f5e611cd5a03 Mon Sep 17 00:00:00 2001 From: Sampark Sharma Date: Mon, 12 Dec 2022 06:53:11 +0000 Subject: [PATCH 39/49] Add end to end test for cache using bsd on windows and address review comments --- .github/workflows/cache-windows-test.yml | 90 ++++++++++++++++++++++ packages/cache/__tests__/tar.test.ts | 95 +++++++++++++++++------- packages/cache/src/internal/tar.ts | 56 +++++++------- 3 files changed, 182 insertions(+), 59 deletions(-) create mode 100644 .github/workflows/cache-windows-test.yml diff --git a/.github/workflows/cache-windows-test.yml b/.github/workflows/cache-windows-test.yml new file mode 100644 index 0000000000..3868f29603 --- /dev/null +++ b/.github/workflows/cache-windows-test.yml @@ -0,0 +1,90 @@ +name: cache-windows-bsd-unit-tests +on: + push: + branches: + - main + paths-ignore: + - '**.md' + pull_request: + paths-ignore: + - '**.md' + +jobs: + build: + name: Build + + runs-on: windows-latest + + steps: + - name: Checkout + uses: actions/checkout@v2 + + - shell: bash + run: | + rm "C:\Program Files\Git\usr\bin\tar.exe" + + - name: Set Node.js 12.x + uses: actions/setup-node@v1 + with: + node-version: 12.x + + # In order to save & restore cache from a shell script, certain env variables need to be set that are only available in the + # node context. This runs a local action that gets and sets the necessary env variables that are needed + - name: Set env variables + uses: ./packages/cache/__tests__/__fixtures__/ + + # Need root node_modules because certain npm packages like jest are configured for the entire repository and it won't be possible + # without these to just compile the cache package + - name: Install root npm packages + run: npm ci + + - name: Compile cache package + run: | + npm ci + npm run tsc + working-directory: packages/cache + + - name: Generate files in working directory + shell: bash + run: packages/cache/__tests__/create-cache-files.sh ${{ runner.os }} test-cache + + - name: Generate files outside working directory + shell: bash + run: packages/cache/__tests__/create-cache-files.sh ${{ runner.os }} ~/test-cache + + # We're using node -e to call the functions directly available in the @actions/cache package + - name: Save cache using saveCache() + run: | + node -e "Promise.resolve(require('./packages/cache/lib/cache').saveCache(['test-cache','~/test-cache'],'test-${{ runner.os }}-${{ github.run_id }}'))" + + - name: Delete cache folders before restoring + shell: bash + run: | + rm -rf test-cache + rm -rf ~/test-cache + + - name: Restore cache using restoreCache() with http-client + run: | + node -e "Promise.resolve(require('./packages/cache/lib/cache').restoreCache(['test-cache','~/test-cache'],'test-${{ runner.os }}-${{ github.run_id }}',[],{useAzureSdk: false}))" + + - name: Verify cache restored with http-client + shell: bash + run: | + packages/cache/__tests__/verify-cache-files.sh ${{ runner.os }} test-cache + packages/cache/__tests__/verify-cache-files.sh ${{ runner.os }} ~/test-cache + + - name: Delete cache folders before restoring + shell: bash + run: | + rm -rf test-cache + rm -rf ~/test-cache + + - name: Restore cache using restoreCache() with Azure SDK + run: | + node -e "Promise.resolve(require('./packages/cache/lib/cache').restoreCache(['test-cache','~/test-cache'],'test-${{ runner.os }}-${{ github.run_id }}'))" + + - name: Verify cache restored with Azure SDK + shell: bash + run: | + packages/cache/__tests__/verify-cache-files.sh ${{ runner.os }} test-cache + packages/cache/__tests__/verify-cache-files.sh ${{ runner.os }} ~/test-cache diff --git a/packages/cache/__tests__/tar.test.ts b/packages/cache/__tests__/tar.test.ts index efd38d8588..3e8379606d 100644 --- a/packages/cache/__tests__/tar.test.ts +++ b/packages/cache/__tests__/tar.test.ts @@ -73,7 +73,9 @@ test('zstd extract tar', async () => { '--use-compress-program', IS_WINDOWS ? '"zstd -d --long=30"' : 'unzstd --long=30' ]) - .join(' ') + .join(' '), + undefined, + {cwd: undefined} ) }) @@ -92,22 +94,31 @@ test('zstd extract tar with windows BSDtar', async () => { await tar.extractTar(archivePath, CompressionMethod.Zstd) expect(mkdirMock).toHaveBeenCalledWith(workspace) - expect(execMock).toHaveBeenCalledTimes(1) - expect(execMock).toHaveBeenCalledWith( + expect(execMock).toHaveBeenCalledTimes(2) + + expect(execMock).toHaveBeenNthCalledWith( + 1, [ - 'cmd /c "', 'zstd -d --long=30 -o', TarFilename.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), - archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), - '&&', + archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/') + ].join(' '), + undefined, + {cwd: undefined} + ) + + expect(execMock).toHaveBeenNthCalledWith( + 2, + [ `"${tarPath}"`, '-xf', TarFilename.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), '-P', '-C', - workspace?.replace(/\\/g, '/'), - '"' // end cmd /c - ].join(' ') + workspace?.replace(/\\/g, '/') + ].join(' '), + undefined, + {cwd: undefined} ) } }) @@ -137,7 +148,9 @@ test('gzip extract tar', async () => { .concat(IS_WINDOWS ? ['--force-local'] : []) .concat(IS_MAC ? ['--delay-directory-restore'] : []) .concat(['-z']) - .join(' ') + .join(' '), + undefined, + {cwd: undefined} ) }) @@ -164,7 +177,9 @@ test('gzip extract GNU tar on windows with GNUtar in path', async () => { workspace?.replace(/\\/g, '/'), '--force-local', '-z' - ].join(' ') + ].join(' '), + undefined, + {cwd: undefined} ) } }) @@ -232,10 +247,11 @@ test('zstd create tar with windows BSDtar', async () => { const tarPath = SystemTarPathOnWindows - expect(execMock).toHaveBeenCalledTimes(1) - expect(execMock).toHaveBeenCalledWith( + expect(execMock).toHaveBeenCalledTimes(2) + + expect(execMock).toHaveBeenNthCalledWith( + 1, [ - 'cmd /c "', `"${tarPath}"`, '--posix', '-cf', @@ -246,12 +262,20 @@ test('zstd create tar with windows BSDtar', async () => { '-C', workspace?.replace(/\\/g, '/'), '--files-from', - ManifestFilename, - '&&', + ManifestFilename + ].join(' '), + undefined, // args + { + cwd: archiveFolder + } + ) + + expect(execMock).toHaveBeenNthCalledWith( + 2, + [ 'zstd -T0 --long=30 -o', CacheFilename.Zstd.replace(/\\/g, '/'), - TarFilename.replace(/\\/g, '/'), - '"' // end cmd /c + TarFilename.replace(/\\/g, '/') ].join(' '), undefined, // args { @@ -324,7 +348,9 @@ test('zstd list tar', async () => { '--use-compress-program', IS_WINDOWS ? '"zstd -d --long=30"' : 'unzstd --long=30' ]) - .join(' ') + .join(' '), + undefined, + {cwd: undefined} ) }) @@ -340,19 +366,28 @@ test('zstd list tar with windows BSDtar', async () => { const tarPath = SystemTarPathOnWindows expect(execMock).toHaveBeenCalledTimes(1) - expect(execMock).toHaveBeenCalledWith( + + expect(execMock).toHaveBeenNthCalledWith( + 1, [ - 'cmd /c "', 'zstd -d --long=30 -o', TarFilename.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), - archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), - '&&', + archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/') + ].join(' '), + undefined, + {cwd: undefined} + ) + + expect(execMock).toHaveBeenNthCalledWith( + 2, + [ `"${tarPath}"`, '-tf', TarFilename.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), - '-P', - '"' // end cmd /c - ].join(' ') + '-P' + ].join(' '), + undefined, + {cwd: undefined} ) } }) @@ -378,7 +413,9 @@ test('zstdWithoutLong list tar', async () => { .concat(IS_WINDOWS ? ['--force-local'] : []) .concat(IS_MAC ? ['--delay-directory-restore'] : []) .concat(['--use-compress-program', IS_WINDOWS ? '"zstd -d"' : 'unzstd']) - .join(' ') + .join(' '), + undefined, + {cwd: undefined} ) }) @@ -402,6 +439,8 @@ test('gzip list tar', async () => { .concat(IS_WINDOWS ? ['--force-local'] : []) .concat(IS_MAC ? ['--delay-directory-restore'] : []) .concat(['-z']) - .join(' ') + .join(' '), + undefined, + {cwd: undefined} ) }) diff --git a/packages/cache/src/internal/tar.ts b/packages/cache/src/internal/tar.ts index f2f7591856..97d36d3a49 100644 --- a/packages/cache/src/internal/tar.ts +++ b/packages/cache/src/internal/tar.ts @@ -122,12 +122,12 @@ async function getTarArgs( return args } -async function getArgs( +async function getCommands( compressionMethod: CompressionMethod, type: string, archivePath = '' -): Promise { - let args: string +): Promise { + let args const tarPath = await getTarPath() const tarArgs = await getTarArgs( @@ -146,16 +146,16 @@ async function getArgs( IS_WINDOWS if (BSD_TAR_ZSTD && type !== 'create') { - args = [...compressionArgs, ...tarArgs].join(' ') + args = [...compressionArgs, ...tarArgs] } else { - args = [...tarArgs, ...compressionArgs].join(' ') + args = [...tarArgs, ...compressionArgs] } if (BSD_TAR_ZSTD) { - args = ['cmd /c "', args, '"'].join(' ') + return args } - return args + return [args.join(' ')] } function getWorkingDirectory(): string { @@ -182,8 +182,7 @@ async function getDecompressionProgram( ? [ 'zstd -d --long=30 -o', TarFilename, - archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), - '&&' + archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/') ] : [ '--use-compress-program', @@ -194,8 +193,7 @@ async function getDecompressionProgram( ? [ 'zstd -d -o', TarFilename, - archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), - '&&' + archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/') ] : ['--use-compress-program', IS_WINDOWS ? '"zstd -d"' : 'unzstd'] default: @@ -221,7 +219,6 @@ async function getCompressionProgram( case CompressionMethod.Zstd: return BSD_TAR_ZSTD ? [ - '&&', 'zstd -T0 --long=30 -o', cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), TarFilename @@ -233,7 +230,6 @@ async function getCompressionProgram( case CompressionMethod.ZstdWithoutLong: return BSD_TAR_ZSTD ? [ - '&&', 'zstd -T0 -o', cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), TarFilename @@ -244,16 +240,22 @@ async function getCompressionProgram( } } +async function execCommands(commands: string[], cwd?: string): Promise { + for (const command of commands) { + try { + await exec(command, undefined, {cwd}) + } catch (error) { + throw new Error(`${command[0]} failed with error: ${error?.message}`) + } + } +} + export async function listTar( archivePath: string, compressionMethod: CompressionMethod ): Promise { - const args = await getArgs(compressionMethod, 'list', archivePath) - try { - await exec(args) - } catch (error) { - throw new Error(`Tar failed with error: ${error?.message}`) - } + const commands = await getCommands(compressionMethod, 'list', archivePath) + await execCommands(commands) } export async function extractTar( @@ -263,12 +265,8 @@ export async function extractTar( // Create directory to extract tar into const workingDirectory = getWorkingDirectory() await io.mkdirP(workingDirectory) - const args = await getArgs(compressionMethod, 'extract', archivePath) - try { - await exec(args) - } catch (error) { - throw new Error(`Tar failed with error: ${error?.message}`) - } + const commands = await getCommands(compressionMethod, 'extract', archivePath) + await execCommands(commands) } export async function createTar( @@ -281,10 +279,6 @@ export async function createTar( path.join(archiveFolder, ManifestFilename), sourceDirectories.join('\n') ) - const args = await getArgs(compressionMethod, 'create') - try { - await exec(args, undefined, {cwd: archiveFolder}) - } catch (error) { - throw new Error(`Tar failed with error: ${error?.message}`) - } + const commands = await getCommands(compressionMethod, 'create') + await execCommands(commands, archiveFolder) } From bbf5659dfa310b4cf317f6d00b36dddb8e4a5f7a Mon Sep 17 00:00:00 2001 From: Sampark Sharma Date: Mon, 12 Dec 2022 07:04:15 +0000 Subject: [PATCH 40/49] Fix test --- packages/cache/src/internal/tar.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/cache/src/internal/tar.ts b/packages/cache/src/internal/tar.ts index 97d36d3a49..6d48e7b8b2 100644 --- a/packages/cache/src/internal/tar.ts +++ b/packages/cache/src/internal/tar.ts @@ -146,9 +146,9 @@ async function getCommands( IS_WINDOWS if (BSD_TAR_ZSTD && type !== 'create') { - args = [...compressionArgs, ...tarArgs] + args = [[...compressionArgs].join(' '), [...tarArgs].join(' ')] } else { - args = [...tarArgs, ...compressionArgs] + args = [[...tarArgs].join(' '), [...compressionArgs].join(' ')] } if (BSD_TAR_ZSTD) { From 6bc5dc5a23f73476158e721ec19cae1e75e31e33 Mon Sep 17 00:00:00 2001 From: Sampark Sharma Date: Mon, 12 Dec 2022 07:25:36 +0000 Subject: [PATCH 41/49] Fix test --- packages/cache/__tests__/tar.test.ts | 2 +- packages/cache/src/internal/tar.ts | 12 +++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/packages/cache/__tests__/tar.test.ts b/packages/cache/__tests__/tar.test.ts index 3e8379606d..a6a79a3db7 100644 --- a/packages/cache/__tests__/tar.test.ts +++ b/packages/cache/__tests__/tar.test.ts @@ -365,7 +365,7 @@ test('zstd list tar with windows BSDtar', async () => { await tar.listTar(archivePath, CompressionMethod.Zstd) const tarPath = SystemTarPathOnWindows - expect(execMock).toHaveBeenCalledTimes(1) + expect(execMock).toHaveBeenCalledTimes(2) expect(execMock).toHaveBeenNthCalledWith( 1, diff --git a/packages/cache/src/internal/tar.ts b/packages/cache/src/internal/tar.ts index 6d48e7b8b2..6973138100 100644 --- a/packages/cache/src/internal/tar.ts +++ b/packages/cache/src/internal/tar.ts @@ -180,7 +180,7 @@ async function getDecompressionProgram( case CompressionMethod.Zstd: return BSD_TAR_ZSTD ? [ - 'zstd -d --long=30 -o', + 'zstd -d --long=30 --force -o', TarFilename, archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/') ] @@ -191,7 +191,7 @@ async function getDecompressionProgram( case CompressionMethod.ZstdWithoutLong: return BSD_TAR_ZSTD ? [ - 'zstd -d -o', + 'zstd -d --force -o', TarFilename, archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/') ] @@ -219,7 +219,7 @@ async function getCompressionProgram( case CompressionMethod.Zstd: return BSD_TAR_ZSTD ? [ - 'zstd -T0 --long=30 -o', + 'zstd -T0 --long=30 --force -o', cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), TarFilename ] @@ -230,7 +230,7 @@ async function getCompressionProgram( case CompressionMethod.ZstdWithoutLong: return BSD_TAR_ZSTD ? [ - 'zstd -T0 -o', + 'zstd -T0 --force -o', cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), TarFilename ] @@ -245,7 +245,9 @@ async function execCommands(commands: string[], cwd?: string): Promise { try { await exec(command, undefined, {cwd}) } catch (error) { - throw new Error(`${command[0]} failed with error: ${error?.message}`) + throw new Error( + `${command.split(' ')[0]} failed with error: ${error?.message}` + ) } } } From d7ae8cd1efdfc798552e0c2d5bbb18734236d49e Mon Sep 17 00:00:00 2001 From: Sampark Sharma Date: Mon, 12 Dec 2022 07:51:54 +0000 Subject: [PATCH 42/49] Fix tests --- packages/cache/__tests__/tar.test.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/cache/__tests__/tar.test.ts b/packages/cache/__tests__/tar.test.ts index a6a79a3db7..a33f4fab54 100644 --- a/packages/cache/__tests__/tar.test.ts +++ b/packages/cache/__tests__/tar.test.ts @@ -99,7 +99,7 @@ test('zstd extract tar with windows BSDtar', async () => { expect(execMock).toHaveBeenNthCalledWith( 1, [ - 'zstd -d --long=30 -o', + 'zstd -d --long=30 --force -o', TarFilename.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/') ].join(' '), @@ -273,7 +273,7 @@ test('zstd create tar with windows BSDtar', async () => { expect(execMock).toHaveBeenNthCalledWith( 2, [ - 'zstd -T0 --long=30 -o', + 'zstd -T0 --long=30 --force -o', CacheFilename.Zstd.replace(/\\/g, '/'), TarFilename.replace(/\\/g, '/') ].join(' '), @@ -370,7 +370,7 @@ test('zstd list tar with windows BSDtar', async () => { expect(execMock).toHaveBeenNthCalledWith( 1, [ - 'zstd -d --long=30 -o', + 'zstd -d --long=30 --force -o', TarFilename.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/') ].join(' '), From cad074ceef5b3c14783d2d6cef0a05b58680d7eb Mon Sep 17 00:00:00 2001 From: Sampark Sharma Date: Mon, 12 Dec 2022 10:58:58 +0000 Subject: [PATCH 43/49] Add better comments --- packages/cache/src/cache.ts | 5 ++--- packages/cache/src/internal/cacheHttpClient.ts | 2 ++ packages/cache/src/internal/tar.ts | 10 +++++++++- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/packages/cache/src/cache.ts b/packages/cache/src/cache.ts index d15975c311..2ebf44cabd 100644 --- a/packages/cache/src/cache.ts +++ b/packages/cache/src/cache.ts @@ -96,8 +96,7 @@ export async function restoreCache( compressionMethod }) if (!cacheEntry?.archiveLocation) { - // This is to support the old cache entry created - // by the old version of the cache action on windows. + // This is to support the old cache entry created by gzip on windows. if ( process.platform === 'win32' && compressionMethod !== CompressionMethod.Gzip @@ -111,7 +110,7 @@ export async function restoreCache( } core.debug( - "Couldn't find cache entry with zstd compression, falling back to gzip compression" + "Couldn't find cache entry with zstd compression, falling back to gzip compression." ) } else { // Cache not found diff --git a/packages/cache/src/internal/cacheHttpClient.ts b/packages/cache/src/internal/cacheHttpClient.ts index c66d1a73e7..d196b9d5d4 100644 --- a/packages/cache/src/internal/cacheHttpClient.ts +++ b/packages/cache/src/internal/cacheHttpClient.ts @@ -104,6 +104,7 @@ export async function getCacheEntry( httpClient.getJson(getCacheApiUrl(resource)) ) if (response.statusCode === 204) { + // Cache not found return null } if (!isSuccessStatusCode(response.statusCode)) { @@ -113,6 +114,7 @@ export async function getCacheEntry( const cacheResult = response.result const cacheDownloadUrl = cacheResult?.archiveLocation if (!cacheDownloadUrl) { + // Cache achiveLocation not found throw new Error('Cache not found.') } core.setSecret(cacheDownloadUrl) diff --git a/packages/cache/src/internal/tar.ts b/packages/cache/src/internal/tar.ts index 6973138100..0af6a87a9c 100644 --- a/packages/cache/src/internal/tar.ts +++ b/packages/cache/src/internal/tar.ts @@ -14,7 +14,7 @@ import { const IS_WINDOWS = process.platform === 'win32' -// Function also mutates the args array. For non-mutation call with passing an empty array. +// Returns tar path and type: BSD or GNU async function getTarPath(): Promise { switch (process.platform) { case 'win32': { @@ -43,6 +43,7 @@ async function getTarPath(): Promise { default: break } + // Default assumption is GNU tar is present in path return { path: await io.which('tar', true), type: ArchiveToolType.GNU @@ -60,6 +61,7 @@ async function getTarArgs( const cacheFileName = utils.getCacheFileName(compressionMethod) const tarFile = 'cache.tar' const workingDirectory = getWorkingDirectory() + // Speficic args for BSD tar on windows for workaround const BSD_TAR_ZSTD = tarPath.type === ArchiveToolType.BSD && compressionMethod !== CompressionMethod.Gzip && @@ -122,6 +124,7 @@ async function getTarArgs( return args } +// Returns commands to run tar and compression program async function getCommands( compressionMethod: CompressionMethod, type: string, @@ -201,6 +204,7 @@ async function getDecompressionProgram( } } +// Used for creating the archive // -T#: Compress using # working thread. If # is 0, attempt to detect and use the number of physical CPU cores. // zstdmt is equivalent to 'zstd -T0' // --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit. @@ -240,6 +244,7 @@ async function getCompressionProgram( } } +// Executes all commands as separate processes async function execCommands(commands: string[], cwd?: string): Promise { for (const command of commands) { try { @@ -252,6 +257,7 @@ async function execCommands(commands: string[], cwd?: string): Promise { } } +// List the contents of a tar export async function listTar( archivePath: string, compressionMethod: CompressionMethod @@ -260,6 +266,7 @@ export async function listTar( await execCommands(commands) } +// Extract a tar export async function extractTar( archivePath: string, compressionMethod: CompressionMethod @@ -271,6 +278,7 @@ export async function extractTar( await execCommands(commands) } +// Create a tar export async function createTar( archiveFolder: string, sourceDirectories: string[], From e7e19845caa32f4d2b519c8fe4f5e15d57637c98 Mon Sep 17 00:00:00 2001 From: Sampark Sharma Date: Mon, 12 Dec 2022 17:44:34 +0530 Subject: [PATCH 44/49] Update packages/cache/src/internal/cacheHttpClient.ts Co-authored-by: Bishal Prasad --- packages/cache/src/internal/cacheHttpClient.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cache/src/internal/cacheHttpClient.ts b/packages/cache/src/internal/cacheHttpClient.ts index d196b9d5d4..77debfedfc 100644 --- a/packages/cache/src/internal/cacheHttpClient.ts +++ b/packages/cache/src/internal/cacheHttpClient.ts @@ -114,7 +114,7 @@ export async function getCacheEntry( const cacheResult = response.result const cacheDownloadUrl = cacheResult?.archiveLocation if (!cacheDownloadUrl) { - // Cache achiveLocation not found + // Cache achiveLocation not found. This should never happen, and hence bail out. throw new Error('Cache not found.') } core.setSecret(cacheDownloadUrl) From 2d3c79e6fe27dc9096e36f52834b6bdc9fbd9167 Mon Sep 17 00:00:00 2001 From: Sampark Sharma Date: Mon, 12 Dec 2022 12:18:07 +0000 Subject: [PATCH 45/49] Address review comments --- .github/workflows/cache-windows-test.yml | 1 + packages/cache/__tests__/tar.test.ts | 6 +++--- packages/cache/src/internal/tar.ts | 8 ++++---- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/.github/workflows/cache-windows-test.yml b/.github/workflows/cache-windows-test.yml index 3868f29603..c7f7a5a9f9 100644 --- a/.github/workflows/cache-windows-test.yml +++ b/.github/workflows/cache-windows-test.yml @@ -78,6 +78,7 @@ jobs: run: | rm -rf test-cache rm -rf ~/test-cache + rm -f cache.tar - name: Restore cache using restoreCache() with Azure SDK run: | diff --git a/packages/cache/__tests__/tar.test.ts b/packages/cache/__tests__/tar.test.ts index a33f4fab54..a6a79a3db7 100644 --- a/packages/cache/__tests__/tar.test.ts +++ b/packages/cache/__tests__/tar.test.ts @@ -99,7 +99,7 @@ test('zstd extract tar with windows BSDtar', async () => { expect(execMock).toHaveBeenNthCalledWith( 1, [ - 'zstd -d --long=30 --force -o', + 'zstd -d --long=30 -o', TarFilename.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/') ].join(' '), @@ -273,7 +273,7 @@ test('zstd create tar with windows BSDtar', async () => { expect(execMock).toHaveBeenNthCalledWith( 2, [ - 'zstd -T0 --long=30 --force -o', + 'zstd -T0 --long=30 -o', CacheFilename.Zstd.replace(/\\/g, '/'), TarFilename.replace(/\\/g, '/') ].join(' '), @@ -370,7 +370,7 @@ test('zstd list tar with windows BSDtar', async () => { expect(execMock).toHaveBeenNthCalledWith( 1, [ - 'zstd -d --long=30 --force -o', + 'zstd -d --long=30 -o', TarFilename.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/') ].join(' '), diff --git a/packages/cache/src/internal/tar.ts b/packages/cache/src/internal/tar.ts index 0af6a87a9c..0da4e8df17 100644 --- a/packages/cache/src/internal/tar.ts +++ b/packages/cache/src/internal/tar.ts @@ -183,7 +183,7 @@ async function getDecompressionProgram( case CompressionMethod.Zstd: return BSD_TAR_ZSTD ? [ - 'zstd -d --long=30 --force -o', + 'zstd -d --long=30 -o', TarFilename, archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/') ] @@ -194,7 +194,7 @@ async function getDecompressionProgram( case CompressionMethod.ZstdWithoutLong: return BSD_TAR_ZSTD ? [ - 'zstd -d --force -o', + 'zstd -d -o', TarFilename, archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/') ] @@ -223,7 +223,7 @@ async function getCompressionProgram( case CompressionMethod.Zstd: return BSD_TAR_ZSTD ? [ - 'zstd -T0 --long=30 --force -o', + 'zstd -T0 --long=30 -o', cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), TarFilename ] @@ -234,7 +234,7 @@ async function getCompressionProgram( case CompressionMethod.ZstdWithoutLong: return BSD_TAR_ZSTD ? [ - 'zstd -T0 --force -o', + 'zstd -T0 -o', cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), TarFilename ] From 0ff35ed4a11c66461de733d941822268f176469d Mon Sep 17 00:00:00 2001 From: Sampark Sharma Date: Mon, 12 Dec 2022 12:45:28 +0000 Subject: [PATCH 46/49] Update for new beta cache package release --- packages/cache/RELEASES.md | 3 +++ packages/cache/package-lock.json | 4 ++-- packages/cache/package.json | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/cache/RELEASES.md b/packages/cache/RELEASES.md index ddd458a95a..a3b305d561 100644 --- a/packages/cache/RELEASES.md +++ b/packages/cache/RELEASES.md @@ -97,3 +97,6 @@ ### 3.1.0-beta.2 - Added support for fallback to gzip to restore old caches on windows. + +### 3.1.0-beta.3 +- Bug Fixes for fallback to gzip to restore old caches on windows and bsdtar if gnutar is not available. diff --git a/packages/cache/package-lock.json b/packages/cache/package-lock.json index 70f7933d7e..64d306971c 100644 --- a/packages/cache/package-lock.json +++ b/packages/cache/package-lock.json @@ -1,12 +1,12 @@ { "name": "@actions/cache", - "version": "3.1.0-beta.2", + "version": "3.1.0-beta.3", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@actions/cache", - "version": "3.1.0-beta.2", + "version": "3.1.0-beta.3", "license": "MIT", "dependencies": { "@actions/core": "^1.10.0", diff --git a/packages/cache/package.json b/packages/cache/package.json index 1fb3fea717..af562ba3ff 100644 --- a/packages/cache/package.json +++ b/packages/cache/package.json @@ -1,6 +1,6 @@ { "name": "@actions/cache", - "version": "3.1.0-beta.2", + "version": "3.1.0-beta.3", "preview": true, "description": "Actions cache lib", "keywords": [ From a20e7c1a03597849239cf78964a556d80a4597b5 Mon Sep 17 00:00:00 2001 From: Sampark Sharma Date: Wed, 21 Dec 2022 10:53:00 +0000 Subject: [PATCH 47/49] Address bugbash issues --- .github/workflows/cache-windows-test.yml | 1 - packages/cache/src/cache.ts | 2 +- packages/cache/src/internal/tar.ts | 8 ++++---- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/cache-windows-test.yml b/.github/workflows/cache-windows-test.yml index c7f7a5a9f9..3868f29603 100644 --- a/.github/workflows/cache-windows-test.yml +++ b/.github/workflows/cache-windows-test.yml @@ -78,7 +78,6 @@ jobs: run: | rm -rf test-cache rm -rf ~/test-cache - rm -f cache.tar - name: Restore cache using restoreCache() with Azure SDK run: | diff --git a/packages/cache/src/cache.ts b/packages/cache/src/cache.ts index 2ebf44cabd..f928a2b9d5 100644 --- a/packages/cache/src/cache.ts +++ b/packages/cache/src/cache.ts @@ -109,7 +109,7 @@ export async function restoreCache( return undefined } - core.debug( + core.info( "Couldn't find cache entry with zstd compression, falling back to gzip compression." ) } else { diff --git a/packages/cache/src/internal/tar.ts b/packages/cache/src/internal/tar.ts index 0da4e8df17..0af6a87a9c 100644 --- a/packages/cache/src/internal/tar.ts +++ b/packages/cache/src/internal/tar.ts @@ -183,7 +183,7 @@ async function getDecompressionProgram( case CompressionMethod.Zstd: return BSD_TAR_ZSTD ? [ - 'zstd -d --long=30 -o', + 'zstd -d --long=30 --force -o', TarFilename, archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/') ] @@ -194,7 +194,7 @@ async function getDecompressionProgram( case CompressionMethod.ZstdWithoutLong: return BSD_TAR_ZSTD ? [ - 'zstd -d -o', + 'zstd -d --force -o', TarFilename, archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/') ] @@ -223,7 +223,7 @@ async function getCompressionProgram( case CompressionMethod.Zstd: return BSD_TAR_ZSTD ? [ - 'zstd -T0 --long=30 -o', + 'zstd -T0 --long=30 --force -o', cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), TarFilename ] @@ -234,7 +234,7 @@ async function getCompressionProgram( case CompressionMethod.ZstdWithoutLong: return BSD_TAR_ZSTD ? [ - 'zstd -T0 -o', + 'zstd -T0 --force -o', cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), TarFilename ] From 8e69225720da219e58def78d8498753579729f2f Mon Sep 17 00:00:00 2001 From: Sampark Sharma Date: Wed, 21 Dec 2022 11:50:03 +0000 Subject: [PATCH 48/49] Fix tests --- packages/cache/__tests__/tar.test.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/cache/__tests__/tar.test.ts b/packages/cache/__tests__/tar.test.ts index a6a79a3db7..a33f4fab54 100644 --- a/packages/cache/__tests__/tar.test.ts +++ b/packages/cache/__tests__/tar.test.ts @@ -99,7 +99,7 @@ test('zstd extract tar with windows BSDtar', async () => { expect(execMock).toHaveBeenNthCalledWith( 1, [ - 'zstd -d --long=30 -o', + 'zstd -d --long=30 --force -o', TarFilename.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/') ].join(' '), @@ -273,7 +273,7 @@ test('zstd create tar with windows BSDtar', async () => { expect(execMock).toHaveBeenNthCalledWith( 2, [ - 'zstd -T0 --long=30 -o', + 'zstd -T0 --long=30 --force -o', CacheFilename.Zstd.replace(/\\/g, '/'), TarFilename.replace(/\\/g, '/') ].join(' '), @@ -370,7 +370,7 @@ test('zstd list tar with windows BSDtar', async () => { expect(execMock).toHaveBeenNthCalledWith( 1, [ - 'zstd -d --long=30 -o', + 'zstd -d --long=30 --force -o', TarFilename.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/') ].join(' '), From c76b5b3a03849f2d13e9311d7541d4b56072cd5a Mon Sep 17 00:00:00 2001 From: Sampark Sharma Date: Thu, 22 Dec 2022 07:35:02 +0000 Subject: [PATCH 49/49] Release new actions/cache minor version --- packages/cache/RELEASES.md | 5 +++++ packages/cache/package-lock.json | 4 ++-- packages/cache/package.json | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/packages/cache/RELEASES.md b/packages/cache/RELEASES.md index a3b305d561..4a415486ac 100644 --- a/packages/cache/RELEASES.md +++ b/packages/cache/RELEASES.md @@ -100,3 +100,8 @@ ### 3.1.0-beta.3 - Bug Fixes for fallback to gzip to restore old caches on windows and bsdtar if gnutar is not available. + +### 3.1.0 +- Update actions/cache on windows to use gnu tar and zstd by default +- Update actions/cache on windows to fallback to bsdtar and zstd if gnu tar is not available. +- Added support for fallback to gzip to restore old caches on windows. diff --git a/packages/cache/package-lock.json b/packages/cache/package-lock.json index 64d306971c..a34c4e3aea 100644 --- a/packages/cache/package-lock.json +++ b/packages/cache/package-lock.json @@ -1,12 +1,12 @@ { "name": "@actions/cache", - "version": "3.1.0-beta.3", + "version": "3.1.0", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@actions/cache", - "version": "3.1.0-beta.3", + "version": "3.1.0", "license": "MIT", "dependencies": { "@actions/core": "^1.10.0", diff --git a/packages/cache/package.json b/packages/cache/package.json index af562ba3ff..44c677f813 100644 --- a/packages/cache/package.json +++ b/packages/cache/package.json @@ -1,6 +1,6 @@ { "name": "@actions/cache", - "version": "3.1.0-beta.3", + "version": "3.1.0", "preview": true, "description": "Actions cache lib", "keywords": [