diff --git a/.changeset/clean-hats-search.md b/.changeset/clean-hats-search.md deleted file mode 100644 index 4c656342f8bcb..0000000000000 --- a/.changeset/clean-hats-search.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@eth-optimism/message-relayer': minor ---- - -Rewrites the message-relayer to use the BaseServiceV2. diff --git a/.changeset/fair-bees-promise.md b/.changeset/fair-bees-promise.md deleted file mode 100644 index 243fbb7227dd5..0000000000000 --- a/.changeset/fair-bees-promise.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@eth-optimism/batch-submitter-service': patch ---- - -Add 20% buffer to gas estimation on tx-batch submission to prevent OOG reverts diff --git a/.changeset/forty-badgers-tease.md b/.changeset/forty-badgers-tease.md deleted file mode 100644 index 11e90cfed5f35..0000000000000 --- a/.changeset/forty-badgers-tease.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@eth-optimism/replica-healthcheck': major ---- - -Rewrite replica-healthcheck with BaseServiceV2 diff --git a/.changeset/gentle-brooms-lick.md b/.changeset/gentle-brooms-lick.md deleted file mode 100644 index 919cbd1856057..0000000000000 --- a/.changeset/gentle-brooms-lick.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@eth-optimism/batch-submitter-service': patch ---- - -Adds MIN_L1_TX_SIZE configuration diff --git a/.changeset/perfect-ties-decide.md b/.changeset/perfect-ties-decide.md deleted file mode 100644 index 81cb8dfdc0ba5..0000000000000 --- a/.changeset/perfect-ties-decide.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@eth-optimism/proxyd': patch ---- - -Don't hit Redis when the out of service interval is zero diff --git a/.changeset/purple-peaches-serve.md b/.changeset/purple-peaches-serve.md deleted file mode 100644 index c1ff63feb06a6..0000000000000 --- a/.changeset/purple-peaches-serve.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@eth-optimism/common-ts': patch ---- - -Have BaseServiceV2 gracefully catch exit signals diff --git a/.changeset/slimy-taxis-study.md b/.changeset/slimy-taxis-study.md deleted file mode 100644 index fbad153431ec8..0000000000000 --- a/.changeset/slimy-taxis-study.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@eth-optimism/sdk': patch ---- - -Update package json to include correct repo link diff --git a/.changeset/tiny-chicken-occur.md b/.changeset/tiny-chicken-occur.md deleted file mode 100644 index f0d3d2dea9844..0000000000000 --- a/.changeset/tiny-chicken-occur.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@eth-optimism/contracts': patch ---- - -Minor README update diff --git a/.changeset/twenty-walls-turn.md b/.changeset/twenty-walls-turn.md deleted file mode 100644 index 6d8276cc343a3..0000000000000 --- a/.changeset/twenty-walls-turn.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@eth-optimism/common-ts': patch ---- - -Introduces the new BaseServiceV2 class. diff --git a/.changeset/yellow-apples-march.md b/.changeset/yellow-apples-march.md deleted file mode 100644 index 827d9a84beafa..0000000000000 --- a/.changeset/yellow-apples-march.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@eth-optimism/sdk': patch ---- - -Tighten type restriction on ProviderLike diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000000000..f90fc34c88d8f --- /dev/null +++ b/SECURITY.md @@ -0,0 +1 @@ +Please see our security policy document [here](https://github.com/ethereum-optimism/.github/blob/master/SECURITY.md). diff --git a/go/batch-submitter/CHANGELOG.md b/go/batch-submitter/CHANGELOG.md index 49de8ba93be79..b00e40d32fe30 100644 --- a/go/batch-submitter/CHANGELOG.md +++ b/go/batch-submitter/CHANGELOG.md @@ -1,5 +1,12 @@ # @eth-optimism/batch-submitter-service +## 0.1.7 + +### Patch Changes + +- aca0684e: Add 20% buffer to gas estimation on tx-batch submission to prevent OOG reverts +- 75040ca5: Adds MIN_L1_TX_SIZE configuration + ## 0.1.6 ### Patch Changes diff --git a/go/batch-submitter/package.json b/go/batch-submitter/package.json index 754c34d7963d6..b5ae0afb248e6 100644 --- a/go/batch-submitter/package.json +++ b/go/batch-submitter/package.json @@ -1,6 +1,6 @@ { "name": "@eth-optimism/batch-submitter-service", - "version": "0.1.6", + "version": "0.1.7", "private": true, "devDependencies": {} } diff --git a/go/proxyd/CHANGELOG.md b/go/proxyd/CHANGELOG.md index 3ad08c2bc05ae..932ff4398755b 100644 --- a/go/proxyd/CHANGELOG.md +++ b/go/proxyd/CHANGELOG.md @@ -1,5 +1,11 @@ # @eth-optimism/proxyd +## 3.8.2 + +### Patch Changes + +- ae18cea1: Don't hit Redis when the out of service interval is zero + ## 3.8.1 ### Patch Changes diff --git a/go/proxyd/package.json b/go/proxyd/package.json index bdf48988fdea3..25f5bf1300b3e 100644 --- a/go/proxyd/package.json +++ b/go/proxyd/package.json @@ -1,6 +1,6 @@ { "name": "@eth-optimism/proxyd", - "version": "3.8.1", + "version": "3.8.2", "private": true, "dependencies": {} } diff --git a/integration-tests/package.json b/integration-tests/package.json index 126acc3b3e67c..cb1b8c422316a 100644 --- a/integration-tests/package.json +++ b/integration-tests/package.json @@ -28,9 +28,9 @@ "url": "https://github.com/ethereum-optimism/optimism.git" }, "devDependencies": { - "@eth-optimism/contracts": "0.5.16", + "@eth-optimism/contracts": "0.5.17", "@eth-optimism/core-utils": "0.8.1", - "@eth-optimism/sdk": "1.0.0", + "@eth-optimism/sdk": "1.0.1", "@ethersproject/abstract-provider": "^5.5.1", "@ethersproject/providers": "^5.5.3", "@ethersproject/transactions": "^5.5.0", diff --git a/packages/common-ts/CHANGELOG.md b/packages/common-ts/CHANGELOG.md index e16f9e8d89080..c4c75b3cba21e 100644 --- a/packages/common-ts/CHANGELOG.md +++ b/packages/common-ts/CHANGELOG.md @@ -1,5 +1,12 @@ # @eth-optimism/common-ts +## 0.2.2 + +### Patch Changes + +- b3f9bdef: Have BaseServiceV2 gracefully catch exit signals +- e53b5783: Introduces the new BaseServiceV2 class. + ## 0.2.1 ### Patch Changes diff --git a/packages/common-ts/package.json b/packages/common-ts/package.json index e9d2d77205768..b2892bde133ae 100644 --- a/packages/common-ts/package.json +++ b/packages/common-ts/package.json @@ -1,6 +1,6 @@ { "name": "@eth-optimism/common-ts", - "version": "0.2.1", + "version": "0.2.2", "description": "[Optimism] Advanced typescript tooling used by various services", "main": "dist/index", "types": "dist/index", diff --git a/packages/contracts/CHANGELOG.md b/packages/contracts/CHANGELOG.md index 36c442b745ef5..44ab83a406837 100644 --- a/packages/contracts/CHANGELOG.md +++ b/packages/contracts/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## 0.5.17 + +### Patch Changes + +- 175ae0bf: Minor README update + ## 0.5.16 ### Patch Changes diff --git a/packages/contracts/package.json b/packages/contracts/package.json index ecf8c0cfe5447..4a490ff20653f 100644 --- a/packages/contracts/package.json +++ b/packages/contracts/package.json @@ -1,6 +1,6 @@ { "name": "@eth-optimism/contracts", - "version": "0.5.16", + "version": "0.5.17", "description": "[Optimism] L1 and L2 smart contracts for Optimism", "main": "dist/index", "types": "dist/index", diff --git a/packages/data-transport-layer/CHANGELOG.md b/packages/data-transport-layer/CHANGELOG.md index c7424f8b23477..9869bf5a318fc 100644 --- a/packages/data-transport-layer/CHANGELOG.md +++ b/packages/data-transport-layer/CHANGELOG.md @@ -1,5 +1,15 @@ # data transport layer +## 0.5.21 + +### Patch Changes + +- Updated dependencies [b3f9bdef] +- Updated dependencies [175ae0bf] +- Updated dependencies [e53b5783] + - @eth-optimism/common-ts@0.2.2 + - @eth-optimism/contracts@0.5.17 + ## 0.5.20 ### Patch Changes diff --git a/packages/data-transport-layer/package.json b/packages/data-transport-layer/package.json index 1343483a1e2db..2a3c7b98305d1 100644 --- a/packages/data-transport-layer/package.json +++ b/packages/data-transport-layer/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "@eth-optimism/data-transport-layer", - "version": "0.5.20", + "version": "0.5.21", "description": "[Optimism] Service for shuttling data from L1 into L2", "main": "dist/index", "types": "dist/index", @@ -36,8 +36,8 @@ "url": "https://github.com/ethereum-optimism/optimism.git" }, "dependencies": { - "@eth-optimism/common-ts": "0.2.1", - "@eth-optimism/contracts": "0.5.16", + "@eth-optimism/common-ts": "0.2.2", + "@eth-optimism/contracts": "0.5.17", "@eth-optimism/core-utils": "0.8.1", "@ethersproject/providers": "^5.5.3", "@ethersproject/transactions": "^5.5.0", diff --git a/packages/message-relayer/CHANGELOG.md b/packages/message-relayer/CHANGELOG.md index 18af4d9f9913f..26c9ac126322e 100644 --- a/packages/message-relayer/CHANGELOG.md +++ b/packages/message-relayer/CHANGELOG.md @@ -1,5 +1,20 @@ # @eth-optimism/message-relayer +## 0.4.0 + +### Minor Changes + +- 860fef46: Rewrites the message-relayer to use the BaseServiceV2. + +### Patch Changes + +- Updated dependencies [b3f9bdef] +- Updated dependencies [7ae1c67f] +- Updated dependencies [e53b5783] +- Updated dependencies [47e5d118] + - @eth-optimism/common-ts@0.2.2 + - @eth-optimism/sdk@1.0.1 + ## 0.3.2 ### Patch Changes diff --git a/packages/message-relayer/package.json b/packages/message-relayer/package.json index 9838e7dc33b90..c669ca51b4ecc 100644 --- a/packages/message-relayer/package.json +++ b/packages/message-relayer/package.json @@ -1,6 +1,6 @@ { "name": "@eth-optimism/message-relayer", - "version": "0.3.2", + "version": "0.4.0", "description": "[Optimism] Service for automatically relaying L2 to L1 transactions", "main": "dist/index", "types": "dist/index", @@ -29,9 +29,9 @@ "url": "https://github.com/ethereum-optimism/optimism.git" }, "dependencies": { - "@eth-optimism/common-ts": "0.2.1", + "@eth-optimism/common-ts": "0.2.2", "@eth-optimism/core-utils": "0.8.1", - "@eth-optimism/sdk": "1.0.0", + "@eth-optimism/sdk": "1.0.1", "ethers": "^5.5.4" }, "devDependencies": { diff --git a/packages/replica-healthcheck/CHANGELOG.md b/packages/replica-healthcheck/CHANGELOG.md index 62b9633219e85..0851a8cf6b26d 100644 --- a/packages/replica-healthcheck/CHANGELOG.md +++ b/packages/replica-healthcheck/CHANGELOG.md @@ -1,5 +1,17 @@ # @eth-optimism/replica-healthcheck +## 1.0.0 + +### Major Changes + +- e264f03f: Rewrite replica-healthcheck with BaseServiceV2 + +### Patch Changes + +- Updated dependencies [b3f9bdef] +- Updated dependencies [e53b5783] + - @eth-optimism/common-ts@0.2.2 + ## 0.3.11 ### Patch Changes diff --git a/packages/replica-healthcheck/package.json b/packages/replica-healthcheck/package.json index 89fb3e86227d3..0e6302178d636 100644 --- a/packages/replica-healthcheck/package.json +++ b/packages/replica-healthcheck/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "@eth-optimism/replica-healthcheck", - "version": "0.3.11", + "version": "1.0.0", "description": "[Optimism] Service for monitoring the health of replica nodes", "main": "dist/index", "types": "dist/index", @@ -31,7 +31,7 @@ "url": "https://github.com/ethereum-optimism/optimism.git" }, "dependencies": { - "@eth-optimism/common-ts": "0.2.1", + "@eth-optimism/common-ts": "0.2.2", "@eth-optimism/core-utils": "0.8.1", "@ethersproject/abstract-provider": "^5.5.1" }, diff --git a/packages/sdk/CHANGELOG.md b/packages/sdk/CHANGELOG.md index 9865125e79c37..68edf141797e8 100644 --- a/packages/sdk/CHANGELOG.md +++ b/packages/sdk/CHANGELOG.md @@ -1,5 +1,14 @@ # @eth-optimism/sdk +## 1.0.1 + +### Patch Changes + +- 7ae1c67f: Update package json to include correct repo link +- 47e5d118: Tighten type restriction on ProviderLike +- Updated dependencies [175ae0bf] + - @eth-optimism/contracts@0.5.17 + ## 1.0.0 ### Major Changes diff --git a/packages/sdk/package.json b/packages/sdk/package.json index f4ee27def52a0..45ad1b730a9ed 100644 --- a/packages/sdk/package.json +++ b/packages/sdk/package.json @@ -1,6 +1,6 @@ { "name": "@eth-optimism/sdk", - "version": "1.0.0", + "version": "1.0.1", "description": "[Optimism] Tools for working with Optimism", "main": "dist/index", "types": "dist/index", @@ -65,7 +65,7 @@ "typescript": "^4.3.5" }, "dependencies": { - "@eth-optimism/contracts": "0.5.16", + "@eth-optimism/contracts": "0.5.17", "@eth-optimism/core-utils": "0.8.1", "lodash": "^4.17.21", "merkletreejs": "^0.2.27", diff --git a/audits/2020_10-Rollup-TrailOfBits.pdf b/technical-documents/audits/2020_10-Rollup-TrailOfBits.pdf similarity index 100% rename from audits/2020_10-Rollup-TrailOfBits.pdf rename to technical-documents/audits/2020_10-Rollup-TrailOfBits.pdf diff --git a/audits/2020_11-Dapphub-ECDSA_Wallet.pdf b/technical-documents/audits/2020_11-Dapphub-ECDSA_Wallet.pdf similarity index 100% rename from audits/2020_11-Dapphub-ECDSA_Wallet.pdf rename to technical-documents/audits/2020_11-Dapphub-ECDSA_Wallet.pdf diff --git a/audits/2021_03-OVM_and_Rollup-OpenZeppelin.pdf b/technical-documents/audits/2021_03-OVM_and_Rollup-OpenZeppelin.pdf similarity index 100% rename from audits/2021_03-OVM_and_Rollup-OpenZeppelin.pdf rename to technical-documents/audits/2021_03-OVM_and_Rollup-OpenZeppelin.pdf diff --git a/audits/2021_03-SafetyChecker-ConsenSysDiligence.pdf b/technical-documents/audits/2021_03-SafetyChecker-ConsenSysDiligence.pdf similarity index 100% rename from audits/2021_03-SafetyChecker-ConsenSysDiligence.pdf rename to technical-documents/audits/2021_03-SafetyChecker-ConsenSysDiligence.pdf diff --git a/technical-documents/postmortems/2022-02-02-inflation-vuln.md b/technical-documents/postmortems/2022-02-02-inflation-vuln.md new file mode 100644 index 0000000000000..e4b833bb94ab2 --- /dev/null +++ b/technical-documents/postmortems/2022-02-02-inflation-vuln.md @@ -0,0 +1,303 @@ +# Self-Destruct Inflation Vulnerability Postmortem + +This document describes a critical bug in our system which we were alerted to on February 2nd, 2022. +It also details our response, lessons learned, and subsequent changes to our processes. + +## Incident Summary + +A vulnerability in Optimism’s fork of go-ethereum ([L2-geth](../../l2geth/README.md)) was reported +to us by [Jay Freeman](https://twitter.com/saurik) (AKA saurik) on February 2nd, 2022. If exploited, +this vulnerability would allow anyone to mint an unbounded amount of ETH on Optimism. + +We confirmed the issue, implemented a fix, and upgraded our infrastructure within 3 hours. We then +alerted infrastructure providers running Optimism, as well as other network operators who are using +a fork of our software. + +All parties were running a patched version of L2-geth within 32 hours of the initial report. + +## Lead up + +saurik had been engaging with our code, and +[opening issues](https://github.com/ethereum-optimism/optimism/issues?q=is%3Aissue+author%3Asaurik+) +for several months prior to identifying this bug. + +We launched our [Immunefi-hosted bug bounty program](https://immunefi.com/bounty/optimism/) on +January 13th, 2022, slightly more than 2 weeks before his report. The maximum payout for the program +was $2,000,042. According to saurik, his decision to hunt for bugs in our code had two motivations. +One was the financial reward, the other was needing a topic to speak about at the upcoming ETHDenver +conference. + +## The Vulnerability + +Contract balances were improperly zeroed during self-destruction, so that the contract address would +still have a balance after it had been self-destructed. This could have enabled an attacker to run a +loop which doubled the balance of a contract each time, resulting in massive inflation and issuance +directly to the attacker. + +A thorough description can be found in saurik's [writeup](https://www.saurik.com/optimism.html). + +## Impact + +The issue was not exploited, so there was no impact to ordinary users. However, the issue required +node operators to update as quickly as possible. Infrastructure providers were especially impacted, +since they had to roll out an emergency patch version. + +## Detection + +Jay Freeman (a.k.a. saurik) reported the bug to us via security@optimism.io. He first attempted to +report via our Immunefi bounty program, but decided to email us since it does not explicitly include +our L2-Geth code. + +## Recovery + +The recovery process was executed by a small subset of the team in a private slack channel. The +timeline and activities were as follows: + +### Timeline (UTC) + +(Using github handles as identifiers) + +- 2022-02-02 1625: smartcontracts receives an e-mail from saurik claiming to have found a critical + issue in L2Geth. E-mail was sent to securityoptimism.io. +- 2022-02-02 X: saurik messaged smartcontracts on Discord to make sure we checked the e-mail since + he knew we had a prior problem where security advisories went to spam. +- 2022-02-02 1650: Huddle begins in #security on Slack. +- 2022-02-02 1758: tynes and smartcontracts confirm the issue on the huddle. +- 2022-02-02 1812: mslipper joins the huddle and alerts infrastructure providers that there is a + live security issue and that an emergency update will be required. +- 2022-02-02 1906: tynes cuts the following builds: + - Mainnet: `0.5.8_b6f79171` + - Kovan: `0.5.9_d4c6d824` +- 2022-02-20 1930: optimisticben deploys to Kovan and mainnet. +- 2022-02-02 2021: mslipper gives instructions to infra providers on how to upgrade. +- 2022-02-02 2150: Infura upgrades both Kovan and mainnet. +- 2022-02-03 0457: Alchemy upgrades both Kovan and mainnet. +- 2022-02-03 2309: Quicknode upgrades mainnet. +- 2022-02-03 1432: Quicknode upgrades Kovan. +- 2022-02-03 1945: smartcontracts alerts Boba. +- 2022-02-03 2300: Boba patches mainnet. +- 2022-02-03 2300: smartcontracts alerts Metis. They patched mainnet at sometime overnight. +- 2022-02-04 1617: smartcontracts opens + [PR #2146](https://github.com/ethereum-optimism/optimism/pull/2146), which we will use to sneak in + the fix without publicly disclosing it. +- 2022-02-06 0250: mslipper merges the finalized patch into PR #2146 after testing, and cuts the + release of l2geth version `0.5.11`. + +## How it was fixed + +The +[fix](https://github.com/ethereum-optimism/optimism/pull/2146/files#diff-20d698ae9b1041792b702bf7015d0beb3cca36701495eaa45b0b8f587b9ae286R887-R889) +itself is only 3 lines long, it ensures that when the `SELFDESTRUCT` operation is called in an +account, its balance (in OVM_ETH) is also immediately set to zero. + +## Lessons learned + +In this section we outline the lessons learned, and how our processes have changed or will change as +a result. These lessons are the product of an internal retrospective, as well as many informal +discussions which have occurred since the incident. + +### On overly-optimistic code reviews + +This bug was (obviously) not caught by our code review process. Naturally we want to understand why +that is, by looking at the PR that introduced it, and the organization context of the time. + +#### The PR + +It was introduced in [PR #1363](https://github.com/ethereum-optimism/optimism/pull/1363), on +2021-07-20, and merged 3 days later. It includes changes to 21 files, (14 in geth code, 6 in test +files). The diff added 217 lines, and removed 149 lines. + +The PR was well scoped, and all of the changes were relevant according to its description: + +> Refactors the usage of OVM_ETH so we can get most remaining integration tests working again. Also +> reworks `vm.UsingOVM` to be `rcfg.UsingOVM` where `rcfg` is a new package within the rollup +> folder. Was required in order to avoid an import cycle. + +The PR was reviewed at least twice, with inline comments that indicate attention to detail, although +the +[comments in the buggy code](https://github.com/ethereum-optimism/optimism/pull/1363/files#diff-11f5b63c52e9c7c30e4e599f96f84db5f08121e8eb623aa1176c2801389487b9) +itself were sparse, and fairly high level. + +Notably, the eventual fix to the bug was made in `instructions.go`, a **file which was completely +untouched by the PR**. + +#### Code and organizational context + +The PR #1363 was one small part of a major architectural update (which we refer as a 'regenesis') to +Optimism. The regenesis removed the OVM contracts, and enabled EVM equivalence. + +The total size of the update can be seen in the +[regenesis 0.5.0 PR](https://github.com/ethereum-optimism/optimism/pull/1594/commits), which +included the commits from the PR above. This was a massive upgrade, as we can see from the size of +the PR (36,311 lines added, 47,430 lines removed), which consumed the attention of our entire +engineering team with a sense of urgency for several months. + +An additional factor contributing to this bug was the significant complexity of the +[L2Geth](https://github.com/ethereum-optimism/optimism/tree/master/l2geth) codebase, which is a fork +of [Geth](https://github.com/ethereum/go-ethereum). Geth itself is already a very complex codebase. +The changes introduced to L2Geth in order to support the OVM made it much more complex, such that +very few people properly understood how it worked. + +The changes made for this regenesis mostly removed this complexity, and moved the behavior of L2Geth +closer that of Geth. Unfortunately L2Geth had already diverged significantly, and the abstractions +of the OVM leaked in enough that a change made in one part of the code could have major consequences +elsewhere. + +More specifically: the OVM used `OVM_ETH`, and ERC20 token rather than native ETH, meaning that +account balances were no longer kept in the state trie. However the EVM's `SELFDESTRUCT` works by +deleting the balance in the trie. In addition, `SELFDESTRUCT` was not implemented in the OVM, +meaning it was not present to remind us that it needed updating in the EVM. + +#### No standard Tests + +The changes outlined above broke many of the common +[Ethereum tests](https://github.com/ethereum/tests) (though not unexpectedly). Modifying the tests +to work with L2Geth and run in CI would have been a major undertaking, but also would have caught +this bug. + +#### Lack of specification + +The fix for this bug might also have been identified by putting more thought into the specification +and security risks associated with the change. Doing so would have a reasonable chance of initiating +the following line of reasoning: + +1. This impacts the way that balances and value transfers happen in the EVM. +1. Several opcodes refer to balance and value transfer. +1. SELFDESTRUCT involves value transfer. +1. Does SELFDESTRUCT behave the same way after the change? + +#### Lack of auditing + +We did not have an audit on the changes made to the regenesis. The rationale for this was that: + +1. the changes were mostly deleting code and simplifying the system by removing the OVM, and +1. the availability of qualified auditors was extremely constrained. + +#### Conclusion regarding the introduction of the bug + +Multiple factors contributed to this bug. Firstly, the pre-existing codebase was heavily modified +from an upstream project (Geth) which very few people fully understand. Arguably the author and the +reviewer were the only people who had a proper grasp of the full scope of changes. + +Perhaps most importantly, because the actual location of the bug was in a file outside of the PR, it +was not considered. This is an unavoidable reality of working in any codebase of a non-trivial size, +but it is not a problem easily solved simply by "reviewing more carefully". + +In order to catch an issue like this, we as reviewers will need to adopt an adversarial mindset, and +we will need a process which enforces this mindset. Such a process would require a reviewer to +explicitly define how an attacker might try to take advantage of a particular change, and to outline +the various risks they considered. + +**Actions planned:** + +- Our forthcoming network upgrade + ([Optimism: Bedrock](https://github.com/ethereum-optimism/optimistic-specs)) will use a + [fresh fork of Geth](https://github.com/ethereum-optimism/reference-optimistic-geth), with a + minimal set of changes which can be easily rebased to track the upstream Geth repository. +- We will ensure the common Ethereum tests are run against Bedrock. +- We are redesigning our code review process, to introduce measure which will: + 1. encourage authors to + 1. clearly state the motivation and specification for the change + 1. explicitly state the risks considered and the associated mitigations they incorporated + during development + 1. encourage reviewers to: + 1. consider areas of the system which are not touched by the PR + 1. view the change from the perspective of an adversary + 1. explicitly define the risks and attacks they considered during their review +- We will build out a threat model which can be used by developers, reviewers and auditors. +- We will make it a hard requirement not to deploy high risk code without an audit. + +### Maximizing the effectiveness of our bug reporting channels + +Our bounty program page on Immunefi did not list Geth as in scope, which led saurik to report +through our security@optimism.io email. Additionally, not all members of the team are in the habit +of checking email at the start of. This caused some delay in the initial incident response process. + +**Actions taken:** + +1. We have extended the scope of the Immunefi program to include our Geth fork. + +**Actions planned:** + +1. Ensure that instructions for reporting a vulnerability are easily discoverable on any of our web + properties, including websites, chat forums, and github repos. +1. We will set up automated alerts for new reports which claim to be critical. +1. We will review who has access to both the email and Immunefi reporting channel, and ensure the + group is limited to those who need to know. + +### Adhering to the principle of least privilege + +Early in the process, the existence of the issue was openly discussed in a public slack channel, +although the details of the vulnerability and exploit path were not described. This violates the +[principle of least priviledge](https://en.wikipedia.org/wiki/Principle_of_least_privilege), as well +as our already existing incident response protocols + +**Action taken:** + +Our incident response documentation is now easier to locate. It explicitly prescribes the use of a +private slack channel, and the principle of least privilege in general. + +### Communicating with the whitehat + +Communication with saurik was initially done mostly in a direct message with a single team member. +This added communication overhead, and reduced saurik's ability to participate in the response +process. + +Another lesson came when we received a review of the fix from saurik, who was able to suggest a +better approach. Consulting with saurik on the fix before implementing would have saved time. + +Keeping the whitehat better informed should also help to build trust with them. + +**Action taken:** + +Our incident response process now requires establishing a private channel with the whitehat and the +full response team, as well as keeping them up to date as the situation progresses. + +### Disclosing to infrastructure operators and forks + +The distribution of patched code to infrastructure operators and forks went relatively smoothly, +still there are opportunities to better document the proper process internally. + +**Actions taken:** + +- We have established an internal database of users to be notified. + +**Actions planned:** + +- We will create internal documentation for building a patched client for infrastructure operators + with a non-standard build target. + +### Public disclosure + +Moving forward, we will adopt a process similar to the Geth team’s +[silent patch policy](https://geth.ethereum.org/docs/vulnerabilities/vulnerabilities#why-silent-patches). + +This means that we reserve the right to hide the fix, and delay the public announcement. We also +reserve the right to directly notify a subset of downstream users prior to the public announcement. + +**Action taken:** This disclosure process is now documented in our +[Security Policies page](https://github.com/ethereum-optimism/.github/blob/master/SECURITY.md). + +### Defensive measures during an incident + +We were fortunate to be informed of this vulnerability without it having been exploited. However +this incident has also revealed that we do not have a clear criteria for deciding whether or not to +disable the sequencer, or pause smart contracts. + +Ultimately this will be a decision made in the moment with the full available context. Although it +is not possible to anticipate all scenarios we outline some basic criteria to inform the decision. + +**Action taken:** + +We've established guiding criteria for disabling the system: + +- If an attack is ongoing: we should disable or pause in order to prevent further damage. +- If we suspect that a vulnerability might be widely known: we should disable or pause proactively. +- Otherwise: we should not disable or pause the system. + +### Alerting + +We would not have automatically detected this bug if it had been exploited. + +**Action planned:** We will expand the set of monitoring and alerting checks we run on the system, +so that we will be alerted to events such as this.