diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 00000000..d4726f16 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,48 @@ +# Clojure CircleCI 2.0 configuration file +# +# Check https://circleci.com/docs/2.0/language-clojure/ for more details +# +version: 2 +jobs: + build: + docker: + # specify the version you desire here + - image: circleci/clojure:lein-2.8.1-browsers + + # Specify service dependencies here if necessary + # CircleCI maintains a library of pre-built images + # documented at https://circleci.com/docs/2.0/circleci-images/ + # - image: circleci/postgres:9.4 + - image: circleci/postgres:9.6 + environment: + - POSTGRES_USER=commiteth + - POSTGRES_DB=commiteth + + working_directory: ~/repo + + environment: + LEIN_ROOT: "true" + # Customize the JVM maximum heap limit + JVM_OPTS: -Xmx3200m + + steps: + - checkout + + # Download and cache dependencies + - restore_cache: + keys: + - v1-dependencies-{{ checksum "project.clj" }} + # fallback to using the latest cache if no exact match is found + - v1-dependencies- + + - run: lein deps + + - save_cache: + paths: + - ~/.m2 + key: v1-dependencies-{{ checksum "project.clj" }} + + - run: echo $ETH_WALLET_JSON > $ETH_WALLET_FILE + + # run tests! + - run: lein test diff --git a/.gitignore b/.gitignore index 39c20db3..b57d8abd 100644 --- a/.gitignore +++ b/.gitignore @@ -26,4 +26,3 @@ node_modules /config-prod.edn /config-dev.edn /config-test.edn -/src/java diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000..017daf0d --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,28 @@ +This document describes process guidelines to be followed when contributing to Status Open Bounty repo. + +First, make sure to familiarize yourself with the [README](https://github.com/status-im/open-bounty/blob/develop/README.md) and [Testing](https://github.com/status-im/open-bounty/blob/develop/doc/testing.md) documents in order to setup the project properly. + +# Issues + - Issues should have type, priority and size (difficulty) assigned via corresponding labels + - Issue descriptions should include the following fields: + - **Summary** + - **Type** + - (*Features or enhancements only*) **User story** + - (*Bugs only*) **Expected behavior** + - (*Bugs only*) **Actual behavior** + - **Additional information** + +# Pull requests + - Branch names should include: + - prefixes indicating issue type (`bug`, `feature`, `doc`, `test`) + - short description in lisp-case + - and include associated issue number + + For instance, `bug/messy-problem-#1234` + - Start the title of the PR with [FIX #NNN], where #NNN is the issue number + - Always include `Status:` in the PR description to indicate whether PR is `WIP` or `Finished`. + - PR description should include the following sections: + - **Summary** + - **Notes** + - **Status** + - Merges into `develop` branch should be approved by at least 1 person, into `master` - by 2. diff --git a/Dockerfile b/Dockerfile index ce2947aa..83164d02 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,7 +23,12 @@ RUN lein uberjar FROM clojure WORKDIR /root/ +RUN apt-get update +RUN apt-get -y install xvfb +RUN apt-get -y install wkhtmltopdf + COPY --from=builder /usr/src/app/target/uberjar/commiteth.jar . +COPY html2png.sh . CMD [""] ENTRYPOINT ["/usr/bin/java", "-Duser.timezone=UTC", "-Dconf=config-test.edn", "-jar", "/root/commiteth.jar"] diff --git a/Jenkinsfile b/Jenkinsfile index ec6b860c..4b24c0ea 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -25,11 +25,15 @@ def dockerreponame = "statusim/openbounty-app" } stage('Deploy') { - build job: 'status-openbounty/openbounty-cluster', parameters: [[$class: 'StringParameterValue', name: 'DEPLOY_ENVIRONMENT', value: "dev"], [$class: 'StringParameterValue', name: 'BRANCH', value: env.BRANCH_NAME]] + if ( currentBuild.rawBuild.getCauses()[0].toString().contains('UserIdCause') ){ + build job: 'status-openbounty/openbounty-cluster', parameters: [[$class: 'StringParameterValue', name: 'DEPLOY_ENVIRONMENT', value: "dev"], [$class: 'StringParameterValue', name: 'BRANCH', value: env.BRANCH_NAME]] + } else { + echo "No deployment on automatic trigger, go to Jenkins and push build button to deliver it." + } } } catch (e) { // slackSend color: 'bad', message: REPO + ":" + BRANCH_NAME + ' failed to build. ' + env.BUILD_URL throw e } -} \ No newline at end of file +} diff --git a/README.md b/README.md index 98216f55..66ddc015 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,5 @@ # Status Open Bounty +[![Riot Chat Badge](https://img.shields.io/badge/join%20%23openbounty-riot-green.svg)](https://chat.status.im/#/room/#openbounty:status.im) Allows you to set bounties for Github issues, paid out in Ether or any ERC-20 token. @@ -13,6 +14,16 @@ Live testnet (Ropsten) version: https://openbounty.status.im:444 The `develop` branch is automatically deployed here. +## Table of contents +- [Prerequisites](#prerequisites) +- [Application config](#application-config) +- [GitHub integration](#github-integration) +- [Contracts](#contracts) +- [Running](#running) +- [Testing](#testing) +- [More info](#more-info) + + ## Prerequisites @@ -23,36 +34,10 @@ You will need [Leiningen](https://github.com/technomancy/leiningen) 2.0 or above Make sure you install [PostgreSQL](https://www.postgresql.org/) and properly set it up: ``` -sudo -u postgres psql -c "CREATE USER commiteth WITH PASSWORD 'commiteth';" -sudo -u postgres createdb commiteth -``` - -## Running - -Launch following commands each in its own shell: - -``` -lein run -lein figwheel -lein less auto +psql postgres -c "CREATE USER commiteth WITH PASSWORD 'commiteth';" +psql postgres -c "CREATE DATABASE commiteth;" ``` - -Make sure you install [PostgreSQL](https://www.postgresql.org/) and properly set it up: - -``` -sudo -u postgres psql -c "CREATE USER commiteth WITH PASSWORD 'commiteth';" -sudo -u postgres createdb commiteth -``` - -### solc - -Solidity compiler [0.4.15](https://github.com/ethereum/solidity/releases/tag/v0.4.15) is required and needs to be in $PATH. - -### web3j - -Web3j [2.3.0](https://github.com/web3j/web3j/releases/tag/v2.3.0) is required and the command line tools need to be in $PATH. - ## Application config Make sure to create `/config-dev.edn` and populate it correctly, which is based on `env/dev/resources/config.edn`. Description of config fields is given below: @@ -68,6 +53,7 @@ eth-account | Ethereum account ID for the bot eth-password | Ethereum account password for the bot eth-rpc-url | RPC URL to Ethereum node, e.g. Geth. Either local or remote eth-wallet-file | Location of wallet file. If Geth is run with the parameters as given below, it will reside under `$HOME/.ropsten/keystore` +offline-signing | Specifies whether to sign transactions locally before sending. Default is true. Set to false when connecting to local Geth node that unlocks accounts tokenreg-base-format | Should be set to `:status` github-client-id | Related to OAuth. Copied from GitHub account Settings->Developer settings->OAuth Apps github-client-secret | Related to OAuth. Copied from GitHub account Settings->Developer settings->OAuth Apps @@ -86,22 +72,52 @@ Follow the steps [here](https://developer.github.com/apps/building-oauth-apps/cr ### GitHub App Follow the steps [here](https://developer.github.com/apps/building-github-apps/creating-a-github-app/). Be sure to specify `:server-address` + `/webhook-app` as "Webhook URL", and `:webhook-secret` as "Webhook Secret". +## Contracts + +All information related to development of OpenBounty smart contracts can be found in [`contracts/`](/contracts/) + ## Running -Lauch a local geth node with the bot account unlocked: +### Ethereum node +There are two options for connecting to an Ethereum node: either run a local node with an unlocked account, or connect to a remote Geth node or Infura. We will be connecting to Ropsten which is a test Ethereum network. + +#### Local + +In order to launch a local geth node with the bot account unlocked issue the following command: ``` #!/bin/bash geth --fast --testnet --cache=1024 --datadir=$HOME/.ropsten --verbosity 4 --port 50100 --ipcpath ~/.ropsten/geth.ipc --rpc --rpcaddr 127.0.0.1 --rpcport 8545 --rpcapi db,eth,net,web3,personal --rpccorsdomain "https://wallet.ethereum.org" --unlock "0xYOUR_ADDR" --password <(echo "YOUR_PASSPHRASE") ``` +#### Remote +Register at [Infura](https://infura.io/signup). You will receive an email with provider URLs. Paste an URL for the Ropsten network into `config.edn` under `:eth-rpc-url` key, and set `:offline-signing` to true. + + +### CSS auto-compilation Launch the following command in a separate shell: ``` lein less auto ``` -Next you want to start a REPL on the backend and the frontend. +### Solidity compilation +Invoke `build-contracts` Leiningen task to compile Solidity files into Java classes: +``` +lein build-contracts +``` + +### Clojure app without REPL +Launch following commands each in its own shell: + +``` +lein run +lein figwheel +``` + +### Clojure app with REPL + +You'll have to start a REPL on the backend and the frontend. ``` lein repl @@ -131,7 +147,10 @@ To create a standalone uberjar: lein uberjar ``` -This creates `target/uberjar/commiteth-.jar` +This creates `target/uberjar/commiteth.jar`. You can run it with the following command from within project root: +``` +java -Dconf= -jar target/uberjar/commiteth.jar +``` ## Testing @@ -160,7 +179,21 @@ lein with-profile test figwheel devcards Open http://localhost:3449/cards.html -### Update landing page +### CircleCI + +We use CircleCI to run unit tests. The following env vars need to be set for this to work: + +> These env vars override configuration parameters that are usually set using the `config.edn` file. + +- `ETH_ACCOUNT` - as in `config.edn` +- `ETH_PASSWORD` - as in `config.edn` +- `ETH_RPC_URL` - as in `config.edn` +- `ETH_WALLET_FILE` - as in `config.edn` +- `ETH_WALLET_JSON` - contents of this will be written to `ETH_WALLET_FILE` + +:bulb: Ideally we'd create those parameters in a script. PR welcome. + +## Update landing page Landing page is static and different CSS and JS due to time constraints. @@ -170,8 +203,8 @@ Landing page is static and different CSS and JS due to time constraints. This copies over necessary artifacts to `resources` dir. -### Troubleshooting -See the [Cookbook](doc/cookbook.md). +## More info +Detailed information on code structure, troubleshooting, etc. can be found [here](doc/README.md). ## License diff --git a/contracts/README.md b/contracts/README.md new file mode 100644 index 00000000..f92dc0e3 --- /dev/null +++ b/contracts/README.md @@ -0,0 +1,31 @@ +# OpenBounty Contracts + +This directory contains all the underlying smart contracts used by the OpenBounty platform. + +- A script `contracts/build.sh` is part of this repository and can be used to +compile the contracts and copy Java interfaces into `src/java/`. + +In order to run the script the following dependencies have to be met: + +- [solc](#solc) +- [web3j](#web3j) + +### solc + +Solidity compiler [0.4.15](https://github.com/ethereum/solidity/releases/tag/v0.4.15) is required and needs to be in $PATH. +Detailed [installation instructions for various platforms](https://solidity.readthedocs.io/en/develop/installing-solidity.html) can be found in the official Solidity documentation. + +``` +brew install https://raw.githubusercontent.com/ethereum/homebrew-ethereum/de1da16f7972a899fc8dd1f3f04299eced6f4312/solidity.rb +brew pin solidity +``` + +### web3j + +Web3j [2.3.0](https://github.com/web3j/web3j/releases/tag/v2.3.0) is required and the command line tools need to be in $PATH. +Installation instructions for the command line tools can be found in the [Web3j Command Line Tools documentation](https://docs.web3j.io/command_line.html). + +``` +brew install https://raw.githubusercontent.com/web3j/homebrew-web3j/881cf369b551a5f2557bd8fb02fa8b7b970256ca/web3j.rb +brew pin web3j +``` \ No newline at end of file diff --git a/build_contracts.sh b/contracts/build.sh similarity index 100% rename from build_contracts.sh rename to contracts/build.sh diff --git a/doc/README.md b/doc/README.md new file mode 100644 index 00000000..a53a2d0d --- /dev/null +++ b/doc/README.md @@ -0,0 +1,9 @@ +# Table of contents + +- [Testing guide](testing.md) +- [Troubleshooting](cookbook.md) +- [Deployment flow](deployment_flow.md) +- [Development workflow](development_workflow.md) +- [Deployment flow](deployment_flow.md) +- [Payout flow](payout_flow.md) +- [Common sync issues](sync_issues.md) diff --git a/doc/core_testing_workflow.md b/doc/core_testing_workflow.md new file mode 100644 index 00000000..fc98cc66 --- /dev/null +++ b/doc/core_testing_workflow.md @@ -0,0 +1,31 @@ +# Testing pull requests in Open Bounty + +All new functionality and features both are being delivered by pull requests (hereinafter PRs). +How to test PR? Steps below could help a bit! + +### Prerequisites +Requirements for PRs to be tested: +* should be in `To test` column in `Pipeline For Pull Requests` project +* should be updated to last develop. If button `Update` is avaivable on particular PR, click it and wait until it will be built. +* shouldn't have conflicts with `develop` branch +* should have a successful build in Jenkins [status-openbounty-app](https://jenkins.status.im/job/status-openbounty/job/status-openbounty-app/view/change-requests/) + + +### Deployment +In order to deploy feature to [testing env](https://testing.openbounty.status.im/) you should **rebuild** PR you are about to test (all open PRs are in [Jenkins](https://jenkins.status.im/job/status-openbounty/job/status-openbounty-app/view/change-requests/)). + +When PR is successfully build and deployed, [testing env](https://testing.openbounty.status.im/) with deployed PR is accessible in 2-3 mins. + +Only one at one time PR can be deployed on [testing env](https://testing.openbounty.status.im/) + +Fresh develop branch with last changes is deployed automatically on [staging env](https://openbounty.status.im:444) + +### Testing +1) Move appropriate PR card to IN TESTING on the [Board](https://github.com/status-im/open-bounty/projects/3) and let people know you are on it - assign it to yourself! :) +2) Сheck the functionality current PR fixes / delivers (positive/negative tests related to the feature). In curtain cases it's worth to look in 'Files changed' tab in GitHub to check the list of what was changed to get understanding of the test coverage or "weak" places that have to be covered. Ask PR-author in #openbounty channel in slack what was changed if it's not clear from the notes in PR. +3) Check reasonable regression using [SOB-general test suite](https://ethstatus.testrail.net/index.php?/suites/view/27&group_by=cases:section_id&group_order=asc) +4) No issues? Perfect! Put appropriate label to the PR (`Tested - OK`), merge it to develop (using `Rebase-Merge`) and move the PR instance to `Merged to develop`. +5) Found issues? Check for duplicates before adding one. Hint: make sure the issue is really introduced by current PR - check latest `develop` branch on [staging env](https://openbounty.status.im:444) . Issue exists in develop? Check existing issues list and make sure you are not adding duplicates before creating your own bug :) +**All PR-specific issues should be added as comments to tested PR.** +Once all issues are logged put label `Tested-issues` to the PR and notify developer that there are several problems that are preventing the PR to merge. Move the PR to `Reviewing, waiting for contributor` on the board if PR is developed by external contributor, and to `Developing` - if it is presented by core contributor. + diff --git a/doc/decisions/0001-record-decisions.md b/doc/decisions/0001-record-decisions.md new file mode 100644 index 00000000..8d32aefb --- /dev/null +++ b/doc/decisions/0001-record-decisions.md @@ -0,0 +1,40 @@ +# 1. Record Decisions + +| Date | Tags | +|------------|---------| +| 2018-02-16 | process | + +## Status + +Proposed + +## Context + +We make a lot of decisions during the life of a project and +documenting those decisions would help new team members and outside +contributors follow our thinking. It also opens up an opportunity for +constructive criticism around those decisions which will result +in better decision making. + +We want to develop a product in the open. Opening up our decision making process for outside contributors is an important part in providing the opportunity to take ownership and impact the project in non-trivial ways. + +Recording decisions will also be a useful form of documentation in general as it does not get out of date. All that's documented is that a decision has been made at a certain point in time. + +## Decision + +We will adopt a format similar to [Architecture Decision +Records](http://thinkrelevance.com/blog/2011/11/15/documenting-architecture-decisions) (ADR) +to propose and document notable decisions. In contrast to ADRs we will try to embrace this tool for more than just architectural concerns. Decisions regarding development practices and product direction are just as important. + +What decisions are notable and which are not is left to common-sense and informal consensus among the team. + +Decisions may be proposed and discussed informally, should however eventually end up — with all relevant discussion summarized — in the `doc/decisions/` directory of this repository. + +Decisions are numbered to have a clear identifier. + +## Consequences + +- We need to document that we record decisions this way and where people can find those documents. +- We should probably provide some pointers around tooling to better work with decision records. [`adr-tools`](https://github.com/npryce/adr-tools) comes to mind. +- Since we deviate from ADRs a bit (we document more than architecture-related decisions) we might need to provide different templates than what comes with tools like `adr-tools` by default. + A template like this may also be useful to guide decision authors to provide appropriate context. \ No newline at end of file diff --git a/doc/decisions/0002-sign-commits-with-gpg.md b/doc/decisions/0002-sign-commits-with-gpg.md new file mode 100644 index 00000000..324ca3e3 --- /dev/null +++ b/doc/decisions/0002-sign-commits-with-gpg.md @@ -0,0 +1,52 @@ +# 2. Sign Commits With GPG + +| Date | Tags | +|------------|-------------------| +| 2018-02-16 | process, security | + + +## Status + +Proposed + +## Context + +OpenBounty is a system which has value flowing through it. +Naturally security is a concern that should be taken into consideration. + +Currently an attacker might get access to an account of a team member +and pose as that developer, merging PRs and pushing changes. + +Status.im as a company is also encouraging the use of GPG signing and +has a Pull Request check in place on Github. This check will mark PRs +as failing if the commits come from an organization member and have not +been GPG-signed. + +## Decision + +In order to verify that commits in the repository are actually authored by the specified +author we adopt [GPG signing of Git commits](https://git-scm.com/book/id/v2/Git-Tools-Signing-Your-Work). + +This will allow us to verify authenticity of the author information saved in +a Git commit and make workflows like deploying on push safer. + +It also introduces some complexity because contributors who want to sign +their commits need to set up the appropriate tooling. Due to that we will +not require outside contributors to sign their commits for now. + +Adopting GPG signing for contributors will also make our PR checks pass +allowing us to more easily discern actually broken and working PRs. + +## Consequences + +GPG signing is only making things safer if we have a trusted way of +exchanging public keys. In the scenario outlined above a user who got access +to GitHub could simply upload an additional key. + +This is currently a work-in-progress within the wider Status organization +and we'll have to wait to see what comes out of that. + +## Appendix + +- [GitHub's instructions for setting up GPG signing](https://help.github.com/articles/signing-commits-using-gpg/) +- More discussion around the usefulness of GPG signing in [issue #285](https://github.com/status-im/open-bounty/issues/285) diff --git a/doc/decisions/0003-include-compiled-web3j-contracts-in-git.md b/doc/decisions/0003-include-compiled-web3j-contracts-in-git.md new file mode 100644 index 00000000..89c5bfbb --- /dev/null +++ b/doc/decisions/0003-include-compiled-web3j-contracts-in-git.md @@ -0,0 +1,44 @@ +# 0003. Include Compiled Web3j Contracts in Git + +| Date | Tags | +|---|---| +| 2018-03-20 | contracts, tooling | + + +## Status + +Accepted + +## Context + +The OpenBounty project utilizes smart contracts to make certain core +aspects of it's product work without centralized trust. + +In order to develop the platform a Java interface to these contracts needs to +be built beforehand. To create those interfaces various tools are required +(`web3j` & `solc`), often in specific versions that are not easily available +via widespread package managers. + +This hurdle also applies to any other situations where the application is set +up from scratch, e.g. continuous integration. + +## Decision + +Instead of forcing every contributor to install those tools we will include +the compiled Java interfaces in our Git repository. This removes a significant +setup cost and hopefully allows people to get going much faster. + +Installing `web3j` and `solc` will only be required when hacking on the +contracts itself which are much more stable than the majority of the code. + +An alternative would be implementing scripts that install those tools in a +platform independent manner but this would require more work. Once we have +the time or someone wants to work on creating those scripts we can easily +revert the decision outlined in this document. + +## Consequences + +- The compiled Java interfaces may get out of date. + This could perhaps be addressed by some clever use of checksums. +- By having changes to the contract interfaces be part of a changeset it may + be easier to spot what changes are required/how APIs are changing. diff --git a/doc/decisions/README.md b/doc/decisions/README.md new file mode 100644 index 00000000..0c501a75 --- /dev/null +++ b/doc/decisions/README.md @@ -0,0 +1,14 @@ +# Decisions + +We record decisions. More context on why and how we do that can be found in [DR-0001](doc/decisions/0001-record-decisions.md). + +The remainder of this document is intended to document tooling around this process. + +### Utilities to Write Decision Records + +- `doc/decisions/templates/template.md` - contains a template for a decision record. +- When recording a decision follow the established file name format + `XXXX-some-title.md` where XXXX is the ever increasing count of decisions + we've made. + +> More to come. Perhaps document how we can use [`adr-tools`](https://github.com/npryce/adr-tools) \ No newline at end of file diff --git a/doc/decisions/templates/template.md b/doc/decisions/templates/template.md new file mode 100644 index 00000000..01c4d407 --- /dev/null +++ b/doc/decisions/templates/template.md @@ -0,0 +1,31 @@ +# NUMBER. TITLE + +| Date | Tags | +|---|---| +| DATE | e.g: architecture, database | + + +## Status + +STATUS + +Common states: proposed, accepted, rejected, deprecated, superseded + +## Context + +Describe the context in which the decision has been made. +Some questions that should give some ideas what to write here: + +- What problems have you encountered that this decision impacts? +- What is the current state of things that are related to this decision? + +## Decision + +Describe your decision. + +- In what ways does it solve the aforementioned problems or improves something? +- What tradeoffs does this decision make? + +## Consequences + +Expand on tradeoffs and other side-effects of this decision. diff --git a/doc/deployment_flow.md b/doc/deployment_flow.md new file mode 100644 index 00000000..8a46f7f5 --- /dev/null +++ b/doc/deployment_flow.md @@ -0,0 +1,16 @@ +# Deployment flow + +This briefly describes events that occur when an issue is labeled as bounty and a new contract has to be deployed. + +1. Issue is labeled +2. Event is received via GitHub App webhook +3. Contract is deployed +4. GitHub issue comment "Deploying contract..." is posted +5. `transaction_hash` is stored in the `issues` table + +The following items execute in scheduler threads that run each minute, so up to 60 sec delay can be expected. + +6. `update-issue-contract-address` scheduler thread fetches transaction receipt, updates `contract_address` and updates GitHub comment with a new image and current balance +7. `deploy-pending-contracts` scheduler thread checks if there are issues that did not have corresponding contracts deployed and attempts to redeploy +8. `update-balances` scheduler thread checks balances and updates GitHub comment accordingly +9. `update-contract-internal-balances` scheduler threads updates internal ERC20 token balances for all deployed contracts. This is required by current contract code diff --git a/doc/development_workflow.md b/doc/development_workflow.md new file mode 100644 index 00000000..f9bdb883 --- /dev/null +++ b/doc/development_workflow.md @@ -0,0 +1,74 @@ +# Development process in Status Open Bounty + +We have a continuously deployed version tracking the `develop` branch live at [test environment](https://openbounty.status.im:444). It uses the [Ropsten](https://ropsten.io/) Ethereum testnet. Any one is welcome to use it and contribute to Open Bounty. + +Any help is greatly appreciated! +Currently we use two projects - `Pipeline For Issues` and `Pipeline For Pull Requests`, issues and pull requests in our repository are passing through all or several stages described below. +Whole team is responsible to keep the projects with accurate information. + +**If issue or pull request marked with `Blocked` label, it means that it is blocked on some stage, reason of blocking have to be in comment.** + +## Pipeline For Issues + +Team is working only on issues that included in this project. +Issues can be added to this project by any core team member. + +#### To define +This is backlog for all features/issues/enhancements which we want to include to development process. + +All issues here should be marked with: + +* **type** - `bug`, `tech-debt`,` enhancement` labels. Issues with `proposal` label should be converted to `bug`, `tech-debt`,` enhancement` before addidng to project. +* **priority** - `Prio: high`, `Prio: med`, `Prio: low` labels. On the board inside issues with same priority sorting from higher to lower priority is applied. + +#### Defining +The column is intended for issues not completely clear or for features, that should be splitted to smaller issues in order to go ahead. +After defining all issues that are intended to develop should have size label + +* `Size: XS` - 1-2 hours, +* `Size: S` - 2-4 hours, +* `Size: M` - 4-8 hours, +* `Size: L` - 8-20 hours, +* `Size: XL` - 20-40 hours, +* `Size: XXL` - 40-60 hours. + +#### To design +It is used for issues that are already defined and require designing process. +#### Designing +It shows up that issues are currently in designing process. +#### To develop +It stores issues which are ready for development. +They are explained, clear, designed and **small enough to create one pull request per issue.** +#### In Bounty +The store with issues which are open bounties. When we put funds to issue and it shows up in [status open bounty](https://openbounty.status.im), we move issue here. +#### Developing +This is for issues that are currently developing, so pull requests assosiated with issues have to be placed in `Pipeline for Pull Request` project. +#### Done +It stores issues with merged to `master` pull requests. +## Pipeline For Pull Requests + +#### Developing +Contains all open pull requests (hereinafter PRs) that already assosiated with issues. +#### Reviewing, waiting for contributor +It keeps all PRs from external contributors which should pass `Reviewing` stage. +#### Reviewing +The storage for all PRs that pass reviewing process. +Review process is discussed [here](https://github.com/status-im/open-bounty/issues/221) +The number of reviewers should be proportional to the complexity of the change and may vary from PR to PR. +Recommended: +* PR is trivial (from core contributor) - 1 approval from core contributor +* PR is normal (or from external contributor) - 2 or more approvals from core contributors +* PR need to be based on and opened against the `develop` branch. +* If a PR has undergone review and requires changes from author, move it back to `Contributor` column +#### To test +All PRs, that are already developed, reviewed, and haven't conflicts with `develop` branch, so ready to be tested. +In case if PR has conflict - it is moved to `Reviewing, waiting for contributor` for external contributors or to `Developing` for core contributors. +#### Testing +Contains all PRs that are currently should pass through testing process, which is described in `core_testing_workflow.md`. +After testing two scenarios possible: +* no issues assosiated with PR - `Tested: OK` label, merge to develop (using `Merge` button) to `Merged to develop` +* issues found - all of them are created as comments to current PR and PR is moved to `Developing` +#### Merged to develop +Keeps PRs that should be merged to `master` and deployed to [prod environment](https://openbounty.status.im/) +#### Done +Stores all merged and closed PRs. diff --git a/doc/payout_flow.md b/doc/payout_flow.md new file mode 100644 index 00000000..315eee9f --- /dev/null +++ b/doc/payout_flow.md @@ -0,0 +1,57 @@ +# Payout flow + +This describes the sequence of events happening once a PR for an issue with a bounty was merged by repo maintainer. + +### Quick info on transaction hashes +In the sequence described below, several types of hashes are used. SOB checks presence of different hashes on records in the `issues` table to decide which action to take on an issue and associated contract. These hashes are: +- `transaction_hash`: set when contract is deployed to the labeled issue +- `execute_hash`: set when PR for an issue with a bounty was merged +- `confirm_hash`: fetched from receipt from transaction invoked in previous step +- `payout_hash`: set when payout was confirmed via `Manage Payouts` + +The event flow is given below. For the bounty to be paid, each issue has to go through the steps in given order. + +### 1. PR closed and merged +- app receives notification via GitHub App webhook (endpoint: `/webhook-app`) +- `handle-claim` fn is invoked which will: +- save PR in the `pull_requests` DB table, where state=1 for merged PRs +- update issue in the DB with commit SHA, if PR was merged + +Afterwards two interleaving sequences of actions come into play: scheduler threads and manual user interaction in the `Manage Payouts` tab. + +### 2. `self-sign-bounty` scheduler thread +- input query name: `pending-bounties`. This selects pending bounties (where `execute_hash` is nil and `commit_sha` is set) +- execute payout transactions +- store `execute_hash` and `winner_login` in `issues` DB table +- update GitHub comment to "Pending maintainer confirmation" +### 3. `update-confirm-hash` scheduler thread +- input query name: `pending-payouts`. This selects bounties with `execute_hash` set and no `confirm_hash` +- fetch `confirm_hash` from transaction receipt +- store `confirm_hash` in `issues` DB table + +### 4. Manage Payouts view +In order to confirm a payout, following conditions have to be met for an issue: +- it is merged +- not paid yet (meaning its `payout_hash` is nil) +- not being confirmed at the moment (`:confirming?` flag is true) + OR +- already confirmed by a scheduler thread(`confirm_hash` is not nil) + +Note that `confirm_hash` issue field and confirmation action in the UI are different things, albeit identically named. In order to confirm a payout from the UI, `confirm_hash` has to be already set by scheduler thread (see above). + +Payout confirmation action results in a `:confirm-payout` event. Its handler will +- use `confirm_hash` to construct transaction payload +- set `:confirming?` flag to `true` +- execute `confirmTransaction()` call +- pass transaction callback to `confirmTransaction()`. Once invoked, the callback will: + - get `payout_hash` passed as an argument + - dispatch `:save-payout-hash` event. Its handler will: + - POST to /api/user/bounty/%s/payout + - This will update `payout_hash` in `issues` DB table + - if POST is successful, dispatch `:payout-confirmed` + - `:payout-confirmed` will update `:confirmed?` to `true` and remove `:confirming?` flag + +### 5. `update-payout-receipt` scheduler thread +- input query name: `confirmed-payouts`. This selects confirmed payouts (the ones that have `payout_hash` and do not have `payout_receipt` set) +- store `payout_receipt` in `issues` DB table +- and update GitHub comment to "Paid to:..." diff --git a/doc/sync_issues.md b/doc/sync_issues.md new file mode 100644 index 00000000..9c3ff6e9 --- /dev/null +++ b/doc/sync_issues.md @@ -0,0 +1,11 @@ +# Common sync issues + +- Repo rename. Related issue: https://github.com/status-im/open-bounty/issues/219. +- Transaction callback not executed, hence payout-hash not set. +- App downtime, multiple GitHub App deliveries failing. These can be afterwards replayed from GitHub App's Advanced tab. +- Geth issue. Not solvable on app end, Geth restart usually fixes these. +- Bot out of gas. Relevant issue: https://github.com/status-im/open-bounty/issues/195. +- Sometimes repos are disabled in DB (state=0), probably only happens to repos that were added earlier through OAuth App. +- PRs might end up in state=0 (opened) instead of state=1(merged), if PR did not have a proper text ("Fixes #..."). This one needs more investigation and more cases to reproduce. +- Sometimes contract_address cannot be fetched for a long period of time. + diff --git a/doc/testing.md b/doc/testing.md index f9dc7a1b..45404d75 100644 --- a/doc/testing.md +++ b/doc/testing.md @@ -8,7 +8,7 @@ For testing you will need: * a web browser (Chrome is known to work, testing with others appreciated) * an Ethereum account on the Ropsten testnet * a Github account with administrative access to one or more repositories -* for approving bounty payouts you will additionally need access to an Ethereum wallet. So far, Mist and [MetaMask](https://metamask.io/) have been used, but anything that provides the web3 javascript interface should work. +* for approving bounty payouts you will additionally need access to an Ethereum wallet. ([MetaMask](https://metamask.io/) plugin) The developers can be reached on the `#openbounty` channel in the [Status slack](http://slack.status.im/). diff --git a/env/dev/clj/user.clj b/env/dev/clj/user.clj index b318a866..a70555af 100644 --- a/env/dev/clj/user.clj +++ b/env/dev/clj/user.clj @@ -1,15 +1,34 @@ (ns user (:require [mount.core :as mount] - [commiteth.figwheel :refer [start-fw stop-fw cljs]])) + [commiteth.figwheel :refer [start-fw stop-fw cljs]] + [clojure.tools.namespace.repl :as repl])) -(defn start [] +(repl/set-refresh-dirs "src" "dev" "test") + +(defn start + "Start all the application components" + [] (require 'commiteth.core) (mount/start-without (ns-resolve 'commiteth.core 'repl-server))) -(defn stop [] +(defn stop + "Stop all the application components" + [] (require 'commiteth.core) (mount/stop-except (ns-resolve 'commiteth.core 'repl-server))) +(defn refresh + "Reload the latest namespace definitions" + [] + (repl/refresh)) + +(defn reset + "Restart application after refreshing namespace definitions" + [] + (stop) + (repl/refresh :after 'user/start)) + (defn restart [] + "Restart without refreshing namespace definitions" (stop) (start)) diff --git a/env/dev/resources/config.edn b/env/dev/resources/config.edn index 571a6aeb..1f8248f7 100644 --- a/env/dev/resources/config.edn +++ b/env/dev/resources/config.edn @@ -14,6 +14,9 @@ :eth-rpc-url "http://localhost:8545" :eth-wallet-file "/some/location" + ;; Specifies whether to sign transactions before sending. Should be false for local Geth, true for remote Infura + :offline-signing true + ;; address of token registry to be used ;; this is the default value for ropsten :tokenreg-addr "0x7d127a3e3b5e72cd8f15e7dee650abe4fcced2b9" diff --git a/externs/web3-externs.js b/externs/web3-externs.js deleted file mode 100644 index 4600afa4..00000000 --- a/externs/web3-externs.js +++ /dev/null @@ -1,12 +0,0 @@ - -/* -externs for web3 injected into js context. needed for things to work with advanced cljs compilation - -TODO: probably not a good idea to maintain this manually in the long run*/ - -var web3 = { - eth: { - accounts: [], - sendTransaction: function() {} - } -}; diff --git a/html2png.sh b/html2png.sh new file mode 100755 index 00000000..781d8d5f --- /dev/null +++ b/html2png.sh @@ -0,0 +1,3 @@ +#!/bin/bash -eu +# need to run wkhtmltoimage in quiet mode because it misuses stdout +xvfb-run -a -s "-screen 0 640x480x16" wkhtmltoimage -q "$@" 2>/dev/null diff --git a/project.clj b/project.clj index ccb500f6..16209a75 100644 --- a/project.clj +++ b/project.clj @@ -5,12 +5,13 @@ :exclusions [joda-time]] [re-frame "0.10.2"] [cljs-ajax "0.7.2"] - [secretary "1.2.3"] + [funcool/bide "1.6.0"] [reagent-utils "0.2.1"] [reagent "0.7.0"] [org.clojure/clojurescript "1.9.946"] - [org.clojure/clojure "1.8.0"] + [org.clojure/clojure "1.9.0"] [selmer "1.11.1"] + [com.taoensso/tufte "1.3.0"] [markdown-clj "1.0.1"] [ring-middleware-format "0.7.2"] [ring/ring-core "1.6.2"] @@ -42,7 +43,6 @@ [cheshire "5.8.0"] [mpg "1.3.0"] [pandect "0.6.1"] - [cljsjs/moment "2.17.1-1"] [org.clojure/tools.nrepl "0.2.13"] [com.cemerick/piggieback "0.2.2"] [jarohen/chime "0.2.2"] @@ -66,20 +66,13 @@ :plugins [[lein-cprop "1.0.1"] [migratus-lein "0.4.1"] [lein-cljsbuild "1.1.7"] - [lein-auto "0.1.2"] [lein-less "1.7.5"] - [lein-shell "0.5.0"] [lein-sha-version "0.1.1"]] :less {:source-paths ["src/less"] :target-path "resources/public/css"} - :auto {"build-contracts" {:file-pattern #"\.(sol)\n" - :paths ["./contracts"]}} - - :aliases {"build-contracts" ["shell" "./build_contracts.sh"]} - :ring {:destroy commiteth.scheduler/stop-scheduler} :uberjar-exclusions [#"public/README.md" #"public/cards.html"] @@ -91,11 +84,10 @@ :css-dirs ["resources/public/css"] :nrepl-middleware [cemerick.piggieback/wrap-cljs-repl]} - :profiles {:uberjar {:jvm-opts ["-server" "-Dconf=config-prod.edn"] :omit-source true - :prep-tasks ["build-contracts" "javac" "compile" ["cljsbuild" "once" "min"] ["less" "once"]] + :prep-tasks ["javac" "compile" ["cljsbuild" "once" "min"] ["less" "once"]] :cljsbuild {:builds {:min @@ -125,6 +117,7 @@ [binaryage/devtools "0.9.7"] [figwheel-sidecar "0.5.14"] [org.clojure/tools.nrepl "0.2.13"] + [org.clojure/tools.namespace "0.2.11"] [com.cemerick/piggieback "0.2.2"] [sablono "0.8.1"]] :plugins [[com.jakemccrary/lein-test-refresh "0.14.0"] @@ -143,7 +136,6 @@ :optimizations :none :pretty-print true}}]} - :prep-tasks ["build-contracts" "javac"] :doo {:build "test"} :source-paths ["env/dev/clj" "test/clj"] :resource-paths ["env/dev/resources"] diff --git a/resources/migrations/20171226182259-hide-user.down.sql b/resources/migrations/20171226182259-hide-user.down.sql new file mode 100644 index 00000000..1b3dcc81 --- /dev/null +++ b/resources/migrations/20171226182259-hide-user.down.sql @@ -0,0 +1,35 @@ +BEGIN; + +-- restore the previous version of the view +CREATE OR REPLACE VIEW "public"."claims_view" AS +SELECT + i.title AS issue_title, + i.issue_number, + r.repo AS repo_name, + r.owner AS repo_owner, + COALESCE(u.name, u.login) AS user_name, + u.avatar_url AS user_avatar_url, + i.payout_receipt, + p.updated, + i.updated AS issue_updated, + i.balance_eth, + i.tokens, + i.value_usd, + p.state AS pr_state, + i.is_open AS issue_open, + (case when u.address IS NULL THEN false ELSE true END) AS user_has_address + FROM issues i, + users u, + repositories r, + pull_requests p + WHERE r.repo_id = i.repo_id + AND p.issue_id = i.issue_id + AND p.user_id = u.id + AND i.contract_address IS NOT NULL + AND i.comment_id IS NOT NULL + ORDER BY p.updated; + +ALTER TABLE users + DROP COLUMN is_hidden_in_hunters; + +COMMIT; diff --git a/resources/migrations/20171226182259-hide-user.up.sql b/resources/migrations/20171226182259-hide-user.up.sql new file mode 100644 index 00000000..a1aa2338 --- /dev/null +++ b/resources/migrations/20171226182259-hide-user.up.sql @@ -0,0 +1,35 @@ +BEGIN; + +ALTER TABLE users + ADD COLUMN is_hidden_in_hunters BOOLEAN NOT NULL DEFAULT FALSE; + +CREATE OR REPLACE VIEW "public"."claims_view" AS +SELECT + i.title AS issue_title, + i.issue_number, + r.repo AS repo_name, + r.owner AS repo_owner, + COALESCE(u.name, u.login) AS user_name, + u.avatar_url AS user_avatar_url, + i.payout_receipt, + p.updated, + i.updated AS issue_updated, + i.balance_eth, + i.tokens, + i.value_usd, + p.state AS pr_state, + i.is_open AS issue_open, + (case when u.address IS NULL THEN false ELSE true END) AS user_has_address + FROM issues i, + users u, + repositories r, + pull_requests p + WHERE r.repo_id = i.repo_id + AND p.issue_id = i.issue_id + AND p.user_id = u.id + AND i.contract_address IS NOT NULL + AND i.comment_id IS NOT NULL + AND NOT u.is_hidden_in_hunters -- added + ORDER BY p.updated; + +COMMIT; diff --git a/resources/sql/queries.sql b/resources/sql/queries.sql index e8a31154..4f866afc 100644 --- a/resources/sql/queries.sql +++ b/resources/sql/queries.sql @@ -17,6 +17,12 @@ WHERE NOT exists(SELECT 1 WHERE id = :id) RETURNING id, login, name, email, avatar_url, address, created; +-- :name user-exists? :? :1 +-- :doc Checks where a user exists in the database. +select 1 +from users u +where u.id = :id; + -- :name update-user! :! :n -- :doc updates an existing user record UPDATE users @@ -96,19 +102,16 @@ WHERE i.repo_id = :repo_id AND i.confirm_hash is null AND i.is_open = true; --- :name update-repo-generic :! :n -/* :require [clojure.string :as string] - [hugsql.parameters :refer [identifier-param-quote]] */ +-- :name update-repo-name :! :n UPDATE repositories -SET -/*~ -(string/join "," - (for [[field _] (:updates params)] - (str (identifier-param-quote (name field) options) - " = :v:updates." (name field)))) -~*/ -where repo_id = :repo_id; +SET repo = :repo_name +WHERE repo_id = :repo_id +AND repo != :repo_name +-- :name update-repo-state :! :n +UPDATE repositories +SET state = :repo_state +WHERE repo_id = :repo_id -- Issues -------------------------------------------------------------------------- @@ -190,20 +193,6 @@ AND i.contract_address IS NULL AND i.transaction_hash IS NOT NULL; --- :name list-failed-deployments :? :* --- :doc retrieves failed contract deployments -SELECT - i.issue_id as issue_id, - i.transaction_hash as transaction_hash, - u.address as owner_address -FROM issues i, users u, repositories r -WHERE r.user_id = u.id -AND i.repo_id = r.repo_id -AND i.contract_address IS NULL -AND i.transaction_hash IS NOT NULL -AND i.updated < now() at time zone 'UTC' - interval '1 hour'; - - -- Pull Requests ------------------------------------------------------------------- -- :name save-pull-request! :! :n @@ -246,7 +235,10 @@ WHERE pr_id = :pr_id; -- :doc bounty issues where deploy contract has failed SELECT i.issue_id AS issue_id, - u.address AS owner_address + i.issue_number AS issue_number, + r.owner AS owner, + u.address AS owner_address, + r.repo AS repo FROM issues i, users u, repositories r WHERE r.user_id = u.id @@ -406,6 +398,14 @@ SELECT exists(SELECT 1 FROM issues WHERE issue_id = :issue_id); +-- :name get-issue :? :1 +-- :doc get issue from DB by repo-id and issue-number +SELECT issue_id, issue_number, is_open, winner_login, commit_sha +FROM issues +WHERE repo_id = :repo_id +AND issue_number = :issue_number; + + -- :name open-bounties :? :* -- :doc all open bounty issues @@ -453,8 +453,9 @@ SELECT i.updated AS updated, i.winner_login AS winner_login, r.repo AS repo_name, - o.address AS owner_address -FROM issues i, users o, repositories r + o.address AS owner_address, + u.address AS payout_address +FROM users o, repositories r, issues i LEFT OUTER JOIN users u ON u.login = i.winner_login WHERE r.repo_id = i.repo_id AND r.user_id = o.id @@ -571,6 +572,7 @@ WHERE pr.commit_sha = i.commit_sha AND u.id = pr.user_id AND i.payout_receipt IS NOT NULL +AND NOT u.is_hidden_in_hunters GROUP BY u.id ORDER BY total_usd DESC LIMIT 5; diff --git a/resources/templates/home.html b/resources/templates/home.html index 590156e4..6afe0ca8 100644 --- a/resources/templates/home.html +++ b/resources/templates/home.html @@ -65,6 +65,7 @@ src="https://www.facebook.com/tr?id=293089407869419&ev=PageView&noscript=1" /> +