Skip to content

Adding fix to block processing errors including partial state recovery #224

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
willmeister merged 5 commits into from
Aug 25, 2020
Merged

Adding fix to block processing errors including partial state recovery #224

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
461542b
Adding fix to block processing errors including partial state recovery
willmeister Aug 24, 2020
dd56c60
Upping timeout of test that is timing out consistently
willmeister Aug 24, 2020
572eeb8
fixing broken test
willmeister Aug 24, 2020
425b6bd
adding configurable retry delay in queued-persisted-processor.spec.ts
willmeister Aug 24, 2020
a676adf
merging master, updating CI dependency caching
willmeister Aug 25, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 31 additions & 16 deletions .github/workflows/build-test-lint.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,13 @@ jobs:
path: node_modules
key: ${{ runner.os }}-${{ matrix.node }}-${{ hashFiles('package.json') }}

- name: Cache contracts deps
uses: actions/cache@v1
id: cache_contracts
with:
path: packages/contracts/node_modules
key: ${{ runner.os }}-${{ matrix.node }}-${{ hashFiles('packages/contracts/package.json') }}

- name: Cache core-db deps
uses: actions/cache@v1
id: cache_core-db
Expand All @@ -48,12 +55,19 @@ jobs:
path: packages/core-utils/node_modules
key: ${{ runner.os }}-${{ matrix.node }}-${{ hashFiles('packages/core-utils/package.json') }}

- name: Cache ovm deps
- name: Cache gas-profiler
uses: actions/cache@v1
id: cache_ovm
id: cache_gas-profiler
with:
path: packages/ovm/node_modules
key: ${{ runner.os }}-${{ matrix.node }}-${{ hashFiles('packages/ovm/package.json') }}
path: packages/gas-profiler/node_modules
key: ${{ runner.os }}-${{ matrix.node }}-${{ hashFiles('packages/gas-profiler/package.json') }}

- name: Cache integration-test-utils deps
uses: actions/cache@v1
id: cache_integration-test-utils
with:
path: packages/integration-test-utils/node_modules
key: ${{ runner.os }}-${{ matrix.node }}-${{ hashFiles('packages/integration-test-utils/package.json') }}

- name: Cache optimistic-game-semantics deps
uses: actions/cache@v1
Expand All @@ -62,12 +76,12 @@ jobs:
path: packages/optimistic-game-semantics/node_modules
key: ${{ runner.os }}-${{ matrix.node }}-${{ hashFiles('packages/optimistic-game-semantics/package.json') }}

- name: Cache rollup-full-node deps
- name: Cache ovm toolchain deps
uses: actions/cache@v1
id: cache_rollup-full-node
id: cache_ovm-toolchain
with:
path: packages/rollup-full-node/node_modules
key: ${{ runner.os }}-${{ matrix.node }}-${{ hashFiles('packages/rollup-full-node/package.json') }}
path: packages/ovm-toolchain/node_modules
key: ${{ runner.os }}-${{ matrix.node }}-${{ hashFiles('packages/ovm-toolchain/package.json') }}

- name: Cache rollup-core deps
uses: actions/cache@v1
Expand All @@ -76,19 +90,20 @@ jobs:
path: packages/rollup-core/node_modules
key: ${{ runner.os }}-${{ matrix.node }}-${{ hashFiles('packages/rollup-core/package.json') }}

- name: Cache state-synchronizer deps
- name: Cache rollup-services deps
uses: actions/cache@v1
id: cache_state-synchronizer
id: cache_rollup-services
with:
path: packages/state-synchronizer/node_modules
key: ${{ runner.os }}-${{ matrix.node }}-${{ hashFiles('packages/state-synchronizer/package.json') }}
path: packages/rollup-services/node_modules
key: ${{ runner.os }}-${{ matrix.node }}-${{ hashFiles('packages/rollup-services/package.json') }}

- name: Cache ovm-truffle-provider-wrapper deps
- name: Cache test-rollup-workflow deps
uses: actions/cache@v1
id: cache_ovm-truffle-provider-wrapper
id: cache_test-rollup-workflow
with:
path: packages/ovm-truffle-provider-wrapper/node_modules
key: ${{ runner.os }}-${{ matrix.node }}-${{ hashFiles('packages/ovm-truffle-provider-wrapper/package.json') }}
path: packages/test-rollup-workflow/node_modules
key: ${{ runner.os }}-${{ matrix.node }}-${{ hashFiles('packages/test-rollup-workflow/package.json') }}


# END DEPENDENCY CACHING

Expand Down
8 changes: 4 additions & 4 deletions docker-compose.microservices.example.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,13 +58,13 @@ services:
- CANONICAL_CHAIN_MIN_BATCH_SIZE=50 # The minimum batch size to build -- if fewer than this number of transactions are ready, a batch will not be created (defaults to 10)
- CANONICAL_CHAIN_MAX_BATCH_SIZE=50 # The maximum batch size to build -- if more than this number of transactions are ready, they will be split into multiple batches of at most this size (defaults to 100)
- CANONICAL_CHAIN_BATCH_CREATOR_PERIOD_MILLIS=30000 # The period in millis at which the CanonicalChainBatchCreator should attempt to create Canonical Chain Batches (defaults to 10,000)
# Canonical Transaction Chain Batch Submitter (needs Postgres, L1 Node, L1 Submitters, and CANONICAL_TRANSACTION_CHAIN_CONTRACT_ADDRESS vars above)
# Canonical Transaction Chain Batch Submitter / Finalizer (needs Postgres, L1 Node, L1 Submitters, and CANONICAL_TRANSACTION_CHAIN_CONTRACT_ADDRESS vars above)
- CANONICAL_CHAIN_BATCH_SUBMITTER_PERIOD_MILLIS # The period in millis at which the CanonicalChainBatchCreator should attempt to create Canonical Chain Batches (defaults to 10,000)
# State Commitment Chain Batch Creator (needs Postgres vars above)
- STATE_COMMITMENT_CHAIN_MIN_BATCH_SIZE=40 # The minimum batch size to build -- if fewer than this number of transactions are ready, a batch will not be created (defaults to 10)
- STATE_COMMITMENT_CHAIN_MAX_BATCH_SIZE=500 # The maximum batch size to build -- if more than this number of transactions are ready, they will be split into multiple batches of at most this size (defaults to 100)
- STATE_COMMITMENT_CHAIN_BATCH_CREATOR_PERIOD_MILLIS=30000 # The period in millis at which the StateCommitmentChainBatchCreator should attempt to create StateCommitmentChain Batches (defaults to 10,000)
# State Commitment Chain Batch Submitter (needs Postgres, L1 Node, L1 Submitters, STATE_COMMITMENT_CHAIN_CONTRACT_ADDRESS vars above)
# State Commitment Chain Batch Submitter / Finalizer (needs Postgres, L1 Node, L1 Submitters, STATE_COMMITMENT_CHAIN_CONTRACT_ADDRESS vars above)
- STATE_COMMITMENT_CHAIN_BATCH_SUBMITTER_PERIOD_MILLIS # The period in millis at which the StateCommitmentChainBatchCreator should attempt to create StateCommitmentChain Batches (defaults to 10,000)
# Fraud Detector
- FRAUD_DETECTOR_PERIOD_MILLIS # The period in millis at which the FraudDetector should run (defaults to 10,000)
Expand All @@ -75,9 +75,9 @@ services:
- RUN_GETH_SUBMISSION_QUEUER=1 # Set to anything to run Geth Submission Queuer
- RUN_QUEUED_GETH_SUBMITTER=1 # Set to anything to run Queued Geth Submitter
- RUN_CANONICAL_CHAIN_BATCH_CREATOR=1 # Set to anything to run Canonical Chain Batch Creator
- RUN_CANONICAL_CHAIN_BATCH_SUBMITTER=1 # Set to anything to run Canonical Chain Batch Submitter
- RUN_CANONICAL_CHAIN_BATCH_SUBMITTER=1 # Set to anything to run Canonical Chain Batch Submitter & Finalizer
- RUN_STATE_COMMITMENT_CHAIN_BATCH_CREATOR=1 # Set to anything to run State Commitment Chain Batch Creator
- RUN_STATE_COMMITMENT_CHAIN_BATCH_SUBMITTER=1 # Set to anything to run State Commitment Chain Batch Submitter
- RUN_STATE_COMMITMENT_CHAIN_BATCH_SUBMITTER=1 # Set to anything to run State Commitment Chain Batch Submitter & Finalizer
- RUN_FRAUD_DETECTOR=1 # Set to anything to run Fraud Detector

postgres:
Expand Down
24 changes: 18 additions & 6 deletions packages/core-db/src/app/queued-persisted-processor.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,8 @@ export abstract class BaseQueuedPersistedProcessor<T>
protected constructor(
private readonly db: DB,
private readonly persistenceKey: string,
startIndex: number = 0
startIndex: number = 0,
private readonly retrySleepDelayMillis: number = 1000
) {
this.initialized = false
this.nextIndexToProcess = startIndex
Expand Down Expand Up @@ -70,13 +71,25 @@ export abstract class BaseQueuedPersistedProcessor<T>
return
}

try {
await this.setNextToProcess(index + 1)
} catch (e) {
this.log(`Error setting next to process to ${index + 1}!`, e)
throw e
}

this.setLastProcessed(index).then(async () => {
this.log(
`Attempting to fetch index ${this.nextIndexToProcess} from storage`
)
const nextItem = await this.fetchItem(this.nextIndexToProcess)
if (!!nextItem) {
this.log(
`Index ${this.nextIndexToProcess} was already stored. Handling it now.`
)
await this.handleIfReady(this.nextIndexToProcess, nextItem)
setTimeout(() => {
this.handleIfReady(this.nextIndexToProcess, nextItem)
}, 0)
} else {
this.log(
`Have not received index ${this.nextIndexToProcess} yet. Waiting...`
Expand Down Expand Up @@ -222,13 +235,12 @@ export abstract class BaseQueuedPersistedProcessor<T>
(allowRetries && index === this.nextIndexToProcess - 1))
) {
try {
await this.setNextToProcess(index + 1)
this.log(`Handling index ${index}.`)
await this.handleNextItem(index, item)
} catch (e) {
logError(log, `Error handling item ${index}. Going to retry.`, e)
await sleep(1000)
return this.handleIfReady(index, item, allowRetries)
await sleep(this.retrySleepDelayMillis)
return this.handleIfReady(index, item, true)
}
} else {
this.log(
Expand Down Expand Up @@ -306,7 +318,7 @@ export abstract class BaseQueuedPersistedProcessor<T>
* Sets the next index to process, persisting the updated index in case of failure.
* @param index The index to set Processed to.
*/
private async setNextToProcess(index: number): Promise<void> {
protected async setNextToProcess(index: number): Promise<void> {
await this.db.put(
this.getStorageKey(
BaseQueuedPersistedProcessor.NEXT_INDEX_TO_PROCESS_KEY
Expand Down
Loading