diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index aa07a959..00000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,81 +0,0 @@ -version: 2.1 - -workflows: - version: 2.1 - validate: - jobs: - - tests: - matrix: - parameters: - python_version: ["3.8"] - - quality: - matrix: - parameters: - python_version: ["3.8"] -jobs: - tests: - parameters: - python_version: - type: string - default: "3.8" - working_directory: /blockstore/app/ - docker: - - image: python:<< parameters.python_version >>-alpine - - image: mysql:5.7 - command: mysqld --character-set-server=utf8 --collation-server=utf8_general_ci - environment: - MYSQL_ROOT_PASSWORD: "" - MYSQL_ALLOW_EMPTY_PASSWORD: "yes" - - environment: - MYSQL_DATABASE: "blockstore_db" - MYSQL_USER: "root" - MYSQL_HOST: "127.0.0.1" - MYSQL_PORT: 3306 - - steps: - - - checkout - - - run: - name: Install requirements - command: | - apk update && apk upgrade - apk add bash bash-completion build-base git perl mariadb-dev libffi-dev - python<< parameters.python_version >> -m venv /blockstore/venv - source /blockstore/venv/bin/activate - pip install --upgrade pip - make requirements - - - run: - name: Run tests - command: | - source /blockstore/venv/bin/activate - make test - - quality: - parameters: - python_version: - type: string - default: "3.8" - working_directory: /blockstore/app/ - docker: - - image: python:<< parameters.python_version >>-alpine - environment: - DJANGO_SETTINGS_MODULE=blockstore.settings.test - steps: - - checkout - - run: - name: Install requirements - command: | - apk update && apk upgrade - apk add bash bash-completion build-base git perl mariadb-dev libffi-dev - python -m venv /blockstore/venv - source /blockstore/venv/bin/activate - pip install --upgrade pip - make requirements - - run: - name: Run quality - command: | - source /blockstore/venv/bin/activate - make quality diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..13c99fc5 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,94 @@ +name: CI + +on: + push: + branches: [master] + pull_request: + branches: + - '**' + +jobs: + + run_tests: + name: Tests + + runs-on: ${{ matrix.os }} + + strategy: + matrix: + os: [ubuntu-20.04] + python-version: ['3.8'] + + env: + MYSQL_HOST: "127.0.0.1" # For Django DATABASES setting. + VIRTUAL_ENV: ${{ github.workspace }}/venv + + services: + mysql: + image: mysql:5.7 + env: + MYSQL_ALLOW_EMPTY_PASSWORD: "yes" + MYSQL_DATABASE: "blockstore_db" + MYSQL_ROOT_PASSWORD: "" + ports: + - 3306:3306 + + defaults: + run: + working-directory: app + + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 # Need to fetch main branch for diff-cover. + path: app + + - name: Setup python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Create Virtual Environment + run: python3.8 -m venv $VIRTUAL_ENV + + - name: Install Dependencies + run: make requirements-test + + - name: Run Tests + run: make test + + run_quality: + name: Quality + + runs-on: ${{ matrix.os }} + + strategy: + matrix: + os: [ubuntu-20.04] + python-version: ['3.8'] + + env: + VIRTUAL_ENV: ${{ github.workspace }}/venv + + defaults: + run: + working-directory: app + + steps: + - uses: actions/checkout@v2 + with: + path: app + + - name: Setup python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Create Virtual Environment + run: python3.8 -m venv $VIRTUAL_ENV + + - name: Install Dependencies + run: make requirements-test + + - name: Run Quality + run: make quality diff --git a/Dockerfile-3.8 b/Dockerfile-3.8 index 00de85f3..ca588709 100644 --- a/Dockerfile-3.8 +++ b/Dockerfile-3.8 @@ -2,12 +2,21 @@ FROM ubuntu:20.04 -ENV VIRTUAL_ENV=/blockstore/venv - RUN apt-get update -RUN apt-get install libmysqlclient-dev libjpeg-dev libssl-dev libffi-dev python3 python3-venv python3-pip git -y -RUN python3.8 -m venv $VIRTUAL_ENV +# git is required by diff-cover +# build-essential, libmysqlclient-dev and python3-dev are required by mysqlclient. +RUN apt-get install -qy --no-install-recommends \ + build-essential \ + git \ + libmysqlclient-dev \ + make \ + python3-dev \ + python3-pip \ + python3-venv + +ENV VIRTUAL_ENV=/blockstore/venv +RUN python3 -m venv $VIRTUAL_ENV RUN echo 'cd /blockstore/app/' > ~/.bashrc.new RUN echo 'export PATH=$VIRTUAL_ENV/bin:$PATH' >> ~/.bashrc.new diff --git a/Makefile b/Makefile index d7f1372a..eb16c28b 100644 --- a/Makefile +++ b/Makefile @@ -64,13 +64,15 @@ clean: ## Remove all generated files requirements: ## Install requirements for development # We can't add this to requirements. It changes the way pip itself works. - ${VENV_BIN}/pip install wheel + ${VENV_BIN}/pip install -U pip wheel ${VENV_BIN}/pip install -r requirements/local.txt --exists-action w requirements-test: ## Install requirements for testing + ${VENV_BIN}/pip install -U pip wheel ${VENV_BIN}/pip install -r requirements/test.txt --exists-action w production-requirements: + ${VENV_BIN}/pip install -U pip wheel ${VENV_BIN}/pip install -r requirements/production.txt --exists-action w migrate: ## Apply database migrations diff --git a/blockstore/urls.py b/blockstore/urls.py index cb8f8eaa..635c67ce 100644 --- a/blockstore/urls.py +++ b/blockstore/urls.py @@ -47,7 +47,7 @@ urlpatterns += make_docs_urls(api_info) if settings.DEBUG: # pragma: no cover - import debug_toolbar + import debug_toolbar # pylint: disable=import-error urlpatterns.append(url(r'^__debug__/', include(debug_toolbar.urls))) if settings.DEBUG or os.environ['DJANGO_SETTINGS_MODULE'] == 'blockstore.settings.test': diff --git a/docker-compose-3.8.yml b/docker-compose-3.8.yml index 88e0146d..659d3f33 100644 --- a/docker-compose-3.8.yml +++ b/docker-compose-3.8.yml @@ -30,7 +30,6 @@ services: networks: devstack: - external: name: ${OPENEDX_PROJECT_NAME:-devstack}_default volumes: diff --git a/requirements/base.txt b/requirements/base.txt index 4f142204..70f9dbfd 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -18,9 +18,8 @@ coreschema==0.0.4 # via # coreapi # drf-yasg -cryptography==3.3.2 +cryptography==3.4.8 # via - # -c requirements/constraints.txt # pyjwt # social-auth-core defusedxml==0.7.1 @@ -66,7 +65,7 @@ git+https://github.com/alanjds/drf-nested-routers.git@8686392f91b114e01f3781807d # via -r requirements/github.in drf-yasg==1.20.0 # via edx-api-doc-tools -edx-api-doc-tools==1.4.3 +edx-api-doc-tools==1.5.0 # via -r requirements/base.in edx-auth-backends==4.0.0 # via -r requirements/base.in @@ -82,10 +81,8 @@ jinja2==3.0.1 # via coreschema markupsafe==2.0.1 # via jinja2 -mysqlclient==1.3.14 - # via - # -c requirements/constraints.txt - # -r requirements/base.in +mysqlclient==2.0.3 + # via -r requirements/base.in oauthlib==3.1.1 # via # requests-oauthlib @@ -123,15 +120,10 @@ ruamel.yaml.clib==0.2.6 # via ruamel.yaml six==1.16.0 # via - # cryptography # edx-auth-backends # edx-django-release-util # social-auth-app-django social-auth-app-django==4.0.0 - # via - # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt - # edx-auth-backends -social-auth-core==4.0.2 # via # -c requirements/common_constraints.txt # edx-auth-backends @@ -139,7 +131,7 @@ social-auth-core==4.1.0 # via # edx-auth-backends # social-auth-app-django -sqlparse==0.4.1 +sqlparse==0.4.2 # via # -r requirements/base.in # django diff --git a/requirements/constraints.txt b/requirements/constraints.txt index 19376a2e..5d3a58a8 100644 --- a/requirements/constraints.txt +++ b/requirements/constraints.txt @@ -23,14 +23,3 @@ django-cors-headers>=2.2.0,<2.3 django-environ-2==2.1.0 django-filter==2.1.0 - -mysqlclient==1.3.14 - -# We want >=3.3.2 to fix a moderate-severity security issue -# reported here: https://github.com/pyca/cryptography/issues/5615. -# However, we want <3.4 because the usage of Rust in cryptography>=3.4 -# causes the package to fail to install on the Alpine Linux image used for CI. -# When someone has the time to switch that image to Ubuntu, then we can probably -# remove this pin (as far as I can tell, there is no production reason not to -# upgrade to the latest version of this package). -cryptography>=3.3.2,<3.4 diff --git a/requirements/local.txt b/requirements/local.txt index 82589972..2dc786a1 100644 --- a/requirements/local.txt +++ b/requirements/local.txt @@ -66,9 +66,8 @@ coverage==5.5 # via # -r requirements/test.txt # pytest-cov -cryptography==3.3.2 +cryptography==3.4.8 # via - # -c requirements/constraints.txt # -r requirements/test.txt # pyjwt # social-auth-core @@ -131,7 +130,7 @@ drf-yasg==1.20.0 # via # -r requirements/test.txt # edx-api-doc-tools -edx-api-doc-tools==1.4.3 +edx-api-doc-tools==1.5.0 # via -r requirements/test.txt edx-auth-backends==4.0.0 # via -r requirements/test.txt @@ -208,10 +207,8 @@ mypy-extensions==0.4.3 # via # -r requirements/test.txt # mypy -mysqlclient==1.3.14 - # via - # -c requirements/constraints.txt - # -r requirements/test.txt +mysqlclient==2.0.3 + # via -r requirements/test.txt oauthlib==3.1.1 # via # -r requirements/test.txt @@ -338,7 +335,6 @@ six==1.16.0 # via # -r requirements/docs.txt # -r requirements/test.txt - # cryptography # django-dynamic-fixture # edx-auth-backends # edx-django-release-util @@ -373,7 +369,7 @@ sphinxcontrib-websupport==1.2.4 # via # -r requirements/docs.txt # sphinx -sqlparse==0.4.1 +sqlparse==0.4.2 # via # -r requirements/test.txt # django diff --git a/requirements/production.txt b/requirements/production.txt index efb941f2..62c5a258 100644 --- a/requirements/production.txt +++ b/requirements/production.txt @@ -6,11 +6,11 @@ # attrs==21.2.0 # via -r requirements/base.txt -boto3==1.18.34 +boto3==1.18.40 # via # -c requirements/constraints.txt # -r requirements/production.in -botocore==1.21.34 +botocore==1.21.40 # via # boto3 # s3transfer @@ -35,9 +35,8 @@ coreschema==0.0.4 # -r requirements/base.txt # coreapi # drf-yasg -cryptography==3.3.2 +cryptography==3.4.8 # via - # -c requirements/constraints.txt # -r requirements/base.txt # pyjwt # social-auth-core @@ -90,7 +89,7 @@ drf-yasg==1.20.0 # via # -r requirements/base.txt # edx-api-doc-tools -edx-api-doc-tools==1.4.3 +edx-api-doc-tools==1.5.0 # via -r requirements/base.txt edx-auth-backends==4.0.0 # via -r requirements/base.txt @@ -126,10 +125,8 @@ markupsafe==2.0.1 # via # -r requirements/base.txt # jinja2 -mysqlclient==1.3.14 - # via - # -c requirements/constraints.txt - # -r requirements/base.txt +mysqlclient==2.0.3 + # via -r requirements/base.txt newrelic==6.8.1.164 # via -r requirements/production.in oauthlib==3.1.1 @@ -196,7 +193,6 @@ s3transfer==0.5.0 six==1.16.0 # via # -r requirements/base.txt - # cryptography # edx-auth-backends # edx-django-release-util # python-dateutil @@ -212,7 +208,7 @@ social-auth-core==4.1.0 # -r requirements/base.txt # edx-auth-backends # social-auth-app-django -sqlparse==0.4.1 +sqlparse==0.4.2 # via # -r requirements/base.txt # django diff --git a/requirements/test.txt b/requirements/test.txt index 5dd22145..7d71094d 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -49,9 +49,8 @@ coverage==5.5 # via # -r requirements/test.in # pytest-cov -cryptography==3.3.2 +cryptography==3.4.8 # via - # -c requirements/constraints.txt # -r requirements/base.txt # pyjwt # social-auth-core @@ -107,7 +106,7 @@ drf-yasg==1.20.0 # via # -r requirements/base.txt # edx-api-doc-tools -edx-api-doc-tools==1.4.3 +edx-api-doc-tools==1.5.0 # via -r requirements/base.txt edx-auth-backends==4.0.0 # via -r requirements/base.txt @@ -158,10 +157,8 @@ mypy==0.910 # via -r requirements/test.in mypy-extensions==0.4.3 # via mypy -mysqlclient==1.3.14 - # via - # -c requirements/constraints.txt - # -r requirements/base.txt +mysqlclient==2.0.3 + # via -r requirements/base.txt oauthlib==3.1.1 # via # -r requirements/base.txt @@ -262,7 +259,6 @@ ruamel.yaml.clib==0.2.6 six==1.16.0 # via # -r requirements/base.txt - # cryptography # django-dynamic-fixture # edx-auth-backends # edx-django-release-util @@ -279,7 +275,7 @@ social-auth-core==4.1.0 # -r requirements/base.txt # edx-auth-backends # social-auth-app-django -sqlparse==0.4.1 +sqlparse==0.4.2 # via # -r requirements/base.txt # django