From fe750a48cb2f42646e62db0fcb573b0d99fcc2b7 Mon Sep 17 00:00:00 2001 From: lwolczynski Date: Fri, 1 Nov 2024 16:15:30 -0500 Subject: [PATCH] IWF-232: Refactor --- .../ci-cadence-integ-test-disable-sticky.yml | 26 ++++++++++++++++ .github/workflows/ci-cadence-integ-test.yml | 8 +---- .../ci-temporal-integ-test-disable-sticky.yml | 31 +++++++++++++++++++ .github/workflows/ci-temporal-integ-test.yml | 9 +----- Makefile | 10 +++--- integ/util.go | 14 +++++++-- service/interpreter/cadence/worker.go | 13 ++++++++ service/interpreter/temporal/worker.go | 13 ++++++++ 8 files changed, 101 insertions(+), 23 deletions(-) create mode 100644 .github/workflows/ci-cadence-integ-test-disable-sticky.yml create mode 100644 .github/workflows/ci-temporal-integ-test-disable-sticky.yml diff --git a/.github/workflows/ci-cadence-integ-test-disable-sticky.yml b/.github/workflows/ci-cadence-integ-test-disable-sticky.yml new file mode 100644 index 00000000..0c11cc1b --- /dev/null +++ b/.github/workflows/ci-cadence-integ-test-disable-sticky.yml @@ -0,0 +1,26 @@ +name: Cadence Integration Test +on: + pull_request: + push: + branches: + - 'main' + +jobs: + tests: + name: "Integration testing" + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + test-subset: + - "a-m" + - "n-z" + steps: + - uses: actions/checkout@v3 + - name: "Set up cadence environment" + run: docker compose -f docker-compose/ci-cadence-dependencies.yml up -d + - name: "Test against cadence" + run: make ci-cadence-integ-test-disable-sticky startsWith=${{ matrix['test-subset'] }} + - name: Dump docker logs + if: always() + uses: jwalton/gh-docker-logs@v2 diff --git a/.github/workflows/ci-cadence-integ-test.yml b/.github/workflows/ci-cadence-integ-test.yml index ac49ec80..646bada9 100644 --- a/.github/workflows/ci-cadence-integ-test.yml +++ b/.github/workflows/ci-cadence-integ-test.yml @@ -9,18 +9,12 @@ jobs: tests: name: "Integration testing" runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - test-subset: - - "a-m" - - "n-z" steps: - uses: actions/checkout@v3 - name: "Set up cadence environment" run: docker compose -f docker-compose/ci-cadence-dependencies.yml up -d - name: "Test against cadence" - run: make ci-cadence-integ-test startsWith=${{ matrix['test-subset'] }} + run: make ci-cadence-integ-test - name: Dump docker logs if: always() uses: jwalton/gh-docker-logs@v2 diff --git a/.github/workflows/ci-temporal-integ-test-disable-sticky.yml b/.github/workflows/ci-temporal-integ-test-disable-sticky.yml new file mode 100644 index 00000000..80b85181 --- /dev/null +++ b/.github/workflows/ci-temporal-integ-test-disable-sticky.yml @@ -0,0 +1,31 @@ +name: Temporal Integration Test +on: + pull_request: + push: + branches: + - 'main' + +jobs: + tests: + name: "Integration testing" + runs-on: ubuntu-latest + permissions: + # Give the default GITHUB_TOKEN write permission to commit and push the + # added or changed files to the repository. + contents: write + strategy: + fail-fast: false + matrix: + test-subset: + - "a-m" + - "n-z" + + steps: + - uses: actions/checkout@v3 + - name: "Set up temporal environment" + run: docker compose -f docker-compose/ci-temporal-dependencies.yml up -d + - name: "Test against temporal" + run: make ci-temporal-integ-test-disable-sticky startsWith=${{ matrix['test-subset'] }} + - name: Dump docker logs + if: always() + uses: jwalton/gh-docker-logs@v2 \ No newline at end of file diff --git a/.github/workflows/ci-temporal-integ-test.yml b/.github/workflows/ci-temporal-integ-test.yml index a8980a14..5cfc21ea 100644 --- a/.github/workflows/ci-temporal-integ-test.yml +++ b/.github/workflows/ci-temporal-integ-test.yml @@ -13,19 +13,12 @@ jobs: # Give the default GITHUB_TOKEN write permission to commit and push the # added or changed files to the repository. contents: write - strategy: - fail-fast: false - matrix: - test-subset: - - "a-m" - - "n-z" - steps: - uses: actions/checkout@v3 - name: "Set up temporal environment" run: docker compose -f docker-compose/ci-temporal-dependencies.yml up -d - name: "Test against temporal" - run: make ci-temporal-integ-test startsWith=${{ matrix['test-subset'] }} + run: make ci-temporal-integ-test - name: Dump docker logs if: always() uses: jwalton/gh-docker-logs@v2 \ No newline at end of file diff --git a/Makefile b/Makefile index fe20175b..b6feb0cd 100644 --- a/Makefile +++ b/Makefile @@ -177,18 +177,16 @@ cadenceIntegTests: $Q go test -v ./integ -temporal=false ci-cadence-integ-test: -ifeq (${startsWith},) $Q go test -v ./integ -search=false -temporal=false -dependencyWaitSeconds=180 -disableStickyCache=true -else + +ci-cadence-integ-test-disable-sticky: $Q go test -v ./integ -run "(?i)^Test[${startsWith}]" -search=false -temporal=false -dependencyWaitSeconds=180 -disableStickyCache=true -endif ci-temporal-integ-test: -ifeq (${startsWith},) $Q go test -v ./integ -cover -coverprofile coverage.out -coverpkg ./service/... -search=false -cadence=false -dependencyWaitSeconds=60 -disableStickyCache=true -else + +ci-temporal-integ-test-disable-sticky: $Q go test -v ./integ -run "(?i)^Test[${startsWith}]" -cover -coverprofile coverage.out -coverpkg ./service/... -search=false -cadence=false -dependencyWaitSeconds=60 -disableStickyCache=true -endif ci-all-tests: # Fails CI when used with -coverprofile flag due to tests that panic; see https://go.dev/doc/build-cover#panicprof diff --git a/integ/util.go b/integ/util.go index 58264913..2d6ce998 100644 --- a/integ/util.go +++ b/integ/util.go @@ -124,7 +124,13 @@ func doStartIwfServiceWithClient(config IwfServiceTestConfig) (uclient uclient.U // start iwf interpreter worker interpreter := temporal.NewInterpreterWorker(createTestConfig(config), temporalClient, service.TaskQueue, config.MemoEncryption, dataConverter, uclient) - interpreter.StartWithOptions(temporal.StartOptions{DisableStickyCache: *disableStickyCache}) + + if *disableStickyCache { + interpreter.StartWithStickyCacheDisabledForTest() + } else { + interpreter.Start() + } + return uclient, func() { iwfServer.Close() interpreter.Close() @@ -155,7 +161,11 @@ func doStartIwfServiceWithClient(config IwfServiceTestConfig) (uclient uclient.U // start iwf interpreter worker interpreter := cadence.NewInterpreterWorker(createTestConfig(config), serviceClient, iwf.DefaultCadenceDomain, service.TaskQueue, closeFunc, uclient) - interpreter.StartWithOptions(cadence.StartOptions{DisableStickyCache: *disableStickyCache}) + if *disableStickyCache { + interpreter.StartWithStickyCacheDisabledForTest() + } else { + interpreter.Start() + } return uclient, func() { iwfServer.Close() interpreter.Close() diff --git a/service/interpreter/cadence/worker.go b/service/interpreter/cadence/worker.go index dc22ad73..d4c7dadc 100644 --- a/service/interpreter/cadence/worker.go +++ b/service/interpreter/cadence/worker.go @@ -37,7 +37,15 @@ func (iw *InterpreterWorker) Close() { iw.worker.Stop() } +func (iw *InterpreterWorker) StartWithStickyCacheDisabledForTest() { + iw.start(true) +} + func (iw *InterpreterWorker) Start() { + iw.start(false) +} + +func (iw *InterpreterWorker) start(disableStickyCache bool) { config := env.GetSharedConfig() var options worker.Options @@ -55,6 +63,11 @@ func (iw *InterpreterWorker) Start() { options.MaxConcurrentDecisionTaskPollers = 10 } + // When DisableStickyCache is true it can harm performance; should not be used in production environment + if disableStickyCache { + options.DisableStickyExecution = true + } + iw.worker = worker.New(iw.service, iw.domain, iw.tasklist, options) worker.EnableVerboseLogging(config.Interpreter.VerboseDebug) diff --git a/service/interpreter/temporal/worker.go b/service/interpreter/temporal/worker.go index 17bf0e62..9f11e161 100644 --- a/service/interpreter/temporal/worker.go +++ b/service/interpreter/temporal/worker.go @@ -35,7 +35,15 @@ func (iw *InterpreterWorker) Close() { iw.worker.Stop() } +func (iw *InterpreterWorker) StartWithStickyCacheDisabledForTest() { + iw.start(true) +} + func (iw *InterpreterWorker) Start() { + iw.start(false) +} + +func (iw *InterpreterWorker) start(disableStickyCache bool) { config := env.GetSharedConfig() var options worker.Options @@ -55,6 +63,11 @@ func (iw *InterpreterWorker) Start() { options.MaxConcurrentWorkflowTaskPollers = 10 } + // When DisableStickyCache is true it can harm performance; should not be used in production environment + if disableStickyCache { + worker.SetStickyWorkflowCacheSize(0) + } + iw.worker = worker.New(iw.temporalClient, iw.taskQueue, options) worker.EnableVerboseLogging(config.Interpreter.VerboseDebug)