Skip to content

Commit

Permalink
Merge remote-tracking branch 'refs/remotes/origin/dev' into next
Browse files Browse the repository at this point in the history
  • Loading branch information
@adrians5j
adrians5j committed Nov 8, 2024
2 parents ad005a9 + 6240bf3 commit 8c8141a
Show file tree
Hide file tree
Showing 16 changed files with 619 additions and 441 deletions.
7 changes: 7 additions & 0 deletions .github/labeler-config.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
# Used by `.github/workflows/labeler.yml` to configure the `actions/labeler` GitHub Action.
# This file cannot be also placed in `.github/workflows` because it is not a workflow file,
# and we do not want GitHub Actions to try to run it as a workflow.

new-admin-ui:
- changed-files:
- any-glob-to-any-file: 'packages/admin-ui/*'
10 changes: 9 additions & 1 deletion .github/workflows/cleanup/aws-nuke.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,16 @@ accounts:
- "s3://webiny-ci"
IAMRole:
- "GitHubActionsWebinyJs"
IAMPolicy:
- "DeployWebinyProjectGroup1Policy"
- "DeployWebinyProjectGroup2Policy"
- "DeployWebinyProjectGroup3Policy"
- "DeployWebinyProjectExtra"
IAMRolePolicyAttachment:
- "GitHubActionsWebinyJs -> AdministratorAccess"
- "GitHubActionsWebinyJs -> DeployWebinyProjectGroup1Policy"
- "GitHubActionsWebinyJs -> DeployWebinyProjectGroup2Policy"
- "GitHubActionsWebinyJs -> DeployWebinyProjectGroup3Policy"
- "GitHubActionsWebinyJs -> DeployWebinyProjectExtra"

resource-types:
# These resource types will be destroyed.
Expand Down
15 changes: 15 additions & 0 deletions .github/workflows/labeler.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
name: "Pull Request Labeler"
on:
- pull_request_target

jobs:
labeler:
permissions:
contents: read
pull-requests: write
runs-on: ubuntu-latest
steps:
- uses: actions/labeler@v5
with:
repo-token: ${{ secrets.GITHUB_TOKEN }}
configuration-path: .github/labeler-config.yml
395 changes: 268 additions & 127 deletions .github/workflows/pullRequests.yml
View file

Large diffs are not rendered by default.

180 changes: 90 additions & 90 deletions .github/workflows/pushDev.yml
View file

Large diffs are not rendered by default.

180 changes: 90 additions & 90 deletions .github/workflows/pushNext.yml
View file

Large diffs are not rendered by default.

208 changes: 82 additions & 126 deletions .github/workflows/wac/pullRequests.wac.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,9 @@ import {
NODE_VERSION,
BUILD_PACKAGES_RUNNER,
listPackagesWithJestTests,
AWS_REGION
AWS_REGION,
runNodeScript,
addToOutputs
} from "./utils";
import {
createGlobalBuildCacheSteps,
Expand All @@ -22,7 +24,12 @@ const yarnCacheSteps = createYarnCacheSteps({ workingDirectory: DIR_WEBINY_JS })
const globalBuildCacheSteps = createGlobalBuildCacheSteps({ workingDirectory: DIR_WEBINY_JS });
const runBuildCacheSteps = createRunBuildCacheSteps({ workingDirectory: DIR_WEBINY_JS });

const createJestTestsJob = (storage: string | null) => {
const createJestTestsJobs = (storage: string | null) => {
const constantsJobName = storage
? `jestTests${storage}Constants`
: "jestTestsNoStorageConstants";
const runJobName = storage ? `jestTests${storage}Run` : "jestTestsNoStorageRun";

const env: Record<string, string> = { AWS_REGION };

if (storage) {
Expand All @@ -39,23 +46,52 @@ const createJestTestsJob = (storage: string | null) => {
}
}

const packages = listPackagesWithJestTests({
const packagesWithJestTests = listPackagesWithJestTests({
storage
});

const job: NormalJob = createJob({
const constantsJob: NormalJob = createJob({
needs: ["constants", "build"],
name: "Create Jest tests constants",
"runs-on": "ubuntu-latest",
outputs: {
"packages-to-jest-test":
"${{ steps.list-packages-to-jest-test.outputs.packages-to-jest-test }}"
},
steps: [
{
name: "List packages to test with Jest",
id: "list-packages-to-jest-test",
run: runNodeScript(
"listPackagesToJestTest",
`[${JSON.stringify(
packagesWithJestTests
)}, \${{ needs.constants.outputs.changed-packages }}]`,
{ outputAs: "packages-to-jest-test" }
)
},
{
name: "Packages to test with Jest",
id: "list-packages",
run: "echo ${{ steps.list-packages-to-jest-test.outputs.packages-to-jest-test }}"
}
]
});

const runJob: NormalJob = createJob({
needs: ["constants", "build", constantsJobName],
name: "${{ matrix.package.cmd }}",
strategy: {
"fail-fast": false,
matrix: {
os: ["ubuntu-latest"],
node: [NODE_VERSION],
package: "${{ fromJson('" + JSON.stringify(packages) + "') }}"
package: `$\{{ fromJson(needs.${constantsJobName}.outputs.packages-to-jest-test) }}`
}
},
"runs-on": "${{ matrix.os }}",
env,
if: `needs.${constantsJobName}.outputs.packages-to-jest-test != '[]'`,
awsAuth: storage === "ddb-es" || storage === "ddb-os",
checkout: { path: DIR_WEBINY_JS },
steps: [
Expand All @@ -73,10 +109,13 @@ const createJestTestsJob = (storage: string | null) => {
// We prevent running of Jest tests if a PR was created from a fork.
// This is because we don't want to expose our AWS credentials to forks.
if (storage === "ddb-es" || storage === "ddb-os") {
job.if = "needs.constants.outputs.is-fork-pr != 'true'";
runJob.if += " && needs.constants.outputs.is-fork-pr != 'true'";
}

return job;
return {
[constantsJobName]: constantsJob,
[runJobName]: runJob
};
};

export const pullRequests = createWorkflow({
Expand Down Expand Up @@ -113,24 +152,51 @@ export const pullRequests = createWorkflow({
outputs: {
"global-cache-key": "${{ steps.global-cache-key.outputs.global-cache-key }}",
"run-cache-key": "${{ steps.run-cache-key.outputs.run-cache-key }}",
"is-fork-pr": "${{ steps.is-fork-pr.outputs.is-fork-pr }}"
"is-fork-pr": "${{ steps.is-fork-pr.outputs.is-fork-pr }}",
"changed-packages": "${{ steps.detect-changed-packages.outputs.changed-packages }}"
},
checkout: false,
steps: [
{
name: "Create global cache key",
id: "global-cache-key",
run: 'echo "global-cache-key=${{ github.base_ref }}-${{ runner.os }}-$(/bin/date -u "+%m%d")-${{ vars.RANDOM_CACHE_KEY_SUFFIX }}" >> $GITHUB_OUTPUT'
run: addToOutputs(
"global-cache-key",
'${{ github.base_ref }}-${{ runner.os }}-$(/bin/date -u "+%m%d")-${{ vars.RANDOM_CACHE_KEY_SUFFIX }}'
)
},
{
name: "Create workflow run cache key",
id: "run-cache-key",
run: 'echo "run-cache-key=${{ github.run_id }}-${{ github.run_attempt }}-${{ vars.RANDOM_CACHE_KEY_SUFFIX }}" >> $GITHUB_OUTPUT'
run: addToOutputs(
"run-cache-key",
"${{ github.run_id }}-${{ github.run_attempt }}-${{ vars.RANDOM_CACHE_KEY_SUFFIX }}"
)
},
{
name: "Is a PR from a fork",
id: "is-fork-pr",
run: 'echo "is-fork-pr=${{ github.event.pull_request.head.repo.fork }}" >> $GITHUB_OUTPUT'
run: addToOutputs(
"is-fork-pr",
"${{ github.event.pull_request.head.repo.fork }}"
)
},
{
name: "Detect changed files",
id: "detect-changed-files",
uses: "dorny/paths-filter@v3",
with: {
filters: "changed:\n - 'packages/**/*'\n",
"list-files": "json"
}
},
{
name: "Detect changed packages",
id: "detect-changed-packages",
run: runNodeScript(
"listChangedPackages",
"${{ steps.detect-changed-files.outputs.changed_files }}",
{ outputAs: "changed-packages" }
)
}
]
}),
Expand Down Expand Up @@ -189,119 +255,9 @@ export const pullRequests = createWorkflow({
)
]
}),
jestTestsNoStorage: createJestTestsJob(null),
jestTestsDdb: createJestTestsJob("ddb"),
jestTestsDdbEs: createJestTestsJob("ddb-es"),
jestTestsDdbOs: createJestTestsJob("ddb-os")

// We commented out these tests because, A) they don't bring much value, and B) these are
// run within "push" workflows anyway (we deploy a Webiny instance and run E2E tests there).
// verdaccioPublish: createJob({
// name: "Publish to Verdaccio",
// needs: ["constants", "build"],
// if: "needs.constants.outputs.is-fork-pr != 'true'",
// checkout: {
// "fetch-depth": 0,
// ref: "${{ github.event.pull_request.head.ref }}",
// path: DIR_WEBINY_JS
// },
// steps: [
// ...yarnCacheSteps,
// ...runBuildCacheSteps,
// ...installBuildSteps,
// ...withCommonParams(
// [
// {
// name: "Start Verdaccio local server",
// run: "npx pm2 start verdaccio -- -c .verdaccio.yaml"
// },
// {
// name: "Configure NPM to use local registry",
// run: "npm config set registry http://localhost:4873"
// },
// {
// name: "Set git email",
// run: 'git config --global user.email "[email protected]"'
// },
// {
// name: "Set git username",
// run: 'git config --global user.name "webiny-bot"'
// },
// {
// name: 'Create ".npmrc" file in the project root, with a dummy auth token',
// run: "echo '//localhost:4873/:_authToken=\"dummy-auth-token\"' > .npmrc"
// },
// {
// name: "Version and publish to Verdaccio",
// run: "yarn release --type=verdaccio"
// }
// ],
// { "working-directory": DIR_WEBINY_JS }
// ),
// {
// name: "Upload verdaccio files",
// uses: "actions/upload-artifact@v4",
// with: {
// name: "verdaccio-files",
// "retention-days": 1,
// "include-hidden-files": true,
// path: [
// DIR_WEBINY_JS + "/.verdaccio/",
// DIR_WEBINY_JS + "/.verdaccio.yaml"
// ].join("\n")
// }
// }
// ]
// }),
// testCreateWebinyProject: createJob({
// name: 'Test "create-webiny-project"',
// needs: "verdaccioPublish",
// strategy: {
// "fail-fast": false,
// matrix: {
// os: ["ubuntu-latest"],
// node: [NODE_VERSION]
// }
// },
// "runs-on": "${{ matrix.os }}",
// checkout: false,
// setupNode: {
// "node-version": "${{ matrix.node }}"
// },
// steps: [
// {
// uses: "actions/download-artifact@v4",
// with: {
// name: "verdaccio-files",
// path: "verdaccio-files"
// }
// },
// {
// name: "Start Verdaccio local server",
// "working-directory": "verdaccio-files",
// run: "yarn add pm2 verdaccio && npx pm2 start verdaccio -- -c .verdaccio.yaml"
// },
// {
// name: "Configure NPM to use local registry",
// run: "npm config set registry http://localhost:4873"
// },
// {
// name: "Set git email",
// run: 'git config --global user.email "[email protected]"'
// },
// {
// name: "Set git username",
// run: 'git config --global user.name "webiny-bot"'
// },
// {
// name: "Disable Webiny telemetry",
// run: 'mkdir ~/.webiny && echo \'{ "id": "ci", "telemetry": false }\' > ~/.webiny/config\n'
// },
// {
// name: "Create a new Webiny project",
// run: 'npx create-webiny-project@local-npm test-project --tag local-npm --no-interactive --assign-to-yarnrc \'{"npmRegistryServer":"http://localhost:4873","unsafeHttpWhitelist":["localhost"]}\' --template-options \'{"region":"eu-central-1"}\'\n'
// }
// ]
// })
...createJestTestsJobs(null),
...createJestTestsJobs("ddb"),
...createJestTestsJobs("ddb-es"),
...createJestTestsJobs("ddb-os")
}
});
3 changes: 3 additions & 0 deletions .github/workflows/wac/utils/addToOutputs.ts
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
export const addToOutputs = (key: string, value: string) => {
return `echo "${key}=${value}" >> $GITHUB_OUTPUT`;
};
2 changes: 2 additions & 0 deletions .github/workflows/wac/utils/index.ts
View file
Original file line number Diff line number Diff line change
@@ -1,2 +1,4 @@
export * from "./constants";
export * from "./listPackagesWithJestTests";
export * from "./runNodeScript";
export * from "./addToOutputs";
10 changes: 8 additions & 2 deletions .github/workflows/wac/utils/listPackagesWithJestTests.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -370,11 +370,17 @@ export const listPackagesWithJestTests = (params: ListPackagesWithJestTestsParam
const packageName = allPackages[i];

if (typeof CUSTOM_HANDLERS[packageName] === "function") {
packagesWithTests.push(...CUSTOM_HANDLERS[packageName]());
const packagesWithPkgName = CUSTOM_HANDLERS[packageName]().map(packageWithJestTests => {
return { ...packageWithJestTests, packageName };
});
packagesWithTests.push(...packagesWithPkgName);
} else {
const testsFolder = path.join("packages", packageName, "__tests__");
if (hasTestFiles(testsFolder)) {
packagesWithTests.push({ cmd: `packages/${packageName}` } as PackageWithTests);
packagesWithTests.push({
cmd: `packages/${packageName}`,
packageName
} as PackageWithTests);
}
}
}
Expand Down
19 changes: 19 additions & 0 deletions .github/workflows/wac/utils/runNodeScript.ts
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
import { addToOutputs } from "./addToOutputs";

interface RunNodeScriptOptions {
outputAs?: string;
}

export const runNodeScript = (
name: string,
params: string = "",
options: RunNodeScriptOptions = {}
) => {
const scriptPath = `.github/workflows/wac/utils/runNodeScripts/${name}.js`;
let cmd = `node ${scriptPath} '${params}'`;
if (options.outputAs) {
cmd = addToOutputs(options.outputAs, `$(${cmd})`);
}

return cmd;
};
13 changes: 13 additions & 0 deletions .github/workflows/wac/utils/runNodeScripts/listChangedPackages.js
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
const args = process.argv.slice(2); // Removes the first two elements
const [detectedChangedFilesString] = args;
const detectedChangedFiles = JSON.parse(detectedChangedFilesString);

const changedPackages = detectedChangedFiles
.filter(path => path.startsWith("packages/"))
.reduce((acc, item) => {
const [, packageName] = item.split("/");
acc.add(packageName);
return acc;
}, new Set());

console.log(JSON.stringify(Array.from(changedPackages)));
10 changes: 10 additions & 0 deletions .github/workflows/wac/utils/runNodeScripts/listPackagesToJestTest.js
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
const args = process.argv.slice(2); // Removes the first two elements
const [paramsString] = args;

const [packagesWithJestTestsList, changedPackages] = JSON.parse(paramsString);

const packagesToJestTest = packagesWithJestTestsList.filter(pkg => {
return changedPackages.includes(pkg.packageName);
});

console.log(JSON.stringify(packagesToJestTest));
Loading

0 comments on commit 8c8141a

Please sign in to comment.