From b4645457d9a53263114e88db0e892172ac7d4ad7 Mon Sep 17 00:00:00 2001 From: melvinsoft Date: Wed, 26 Jun 2019 15:14:44 +0200 Subject: [PATCH 1/8] fix ora2 requirement lines --- requirements/edx/base.txt | 2 +- requirements/edx/development.txt | 2 +- requirements/edx/github.in | 2 +- requirements/edx/testing.txt | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index 13b46953addc..8e88184b0270 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -25,7 +25,7 @@ git+https://github.com/mitodl/edx-sga.git@6b2f7aa2a18206023c8407e2c46f86d4b4c3ac git+https://github.com/edx/xblock-lti-consumer.git@v1.1.8#egg=lti_consumer-xblock==1.1.8 git+https://github.com/edx/MongoDBProxy.git@25b99097615bda06bd7cdfe5669ed80dc2a7fed0#egg=MongoDBProxy==0.1.0 -e . -git+https://github.com/edx/edx-ora2.git@2.1.17#egg=ora2==2.1.18 +git+https://github.com/edx/edx-ora2.git@2.1.18#egg=ora2==2.1.18 -e git+https://github.com/dgrtwo/ParsePy.git@7949b9f754d1445eff8e8f20d0e967b9a6420639#egg=parse_rest -e git+https://github.com/appliedsec/pygeoip.git@95e69341cebf5a6a9fbf7c4f5439d458898bdc3b#egg=pygeoip -e git+https://github.com/dementrock/pystache_custom.git@776973740bdaad83a3b029f96e415a7d1e8bec2f#egg=pystache_custom-dev diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index ef7aaf1425c3..049a981c46be 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -28,7 +28,7 @@ git+https://github.com/edx/lettuce.git@7a04591c78ac56dac3eb3e91ca94b15cce844133# git+https://github.com/edx/xblock-lti-consumer.git@v1.1.8#egg=lti_consumer-xblock==1.1.8 git+https://github.com/edx/MongoDBProxy.git@25b99097615bda06bd7cdfe5669ed80dc2a7fed0#egg=MongoDBProxy==0.1.0 -e . -git+https://github.com/edx/edx-ora2.git@2.1.17#egg=ora2==2.1.18 +git+https://github.com/edx/edx-ora2.git@2.1.18#egg=ora2==2.1.18 -e git+https://github.com/dgrtwo/ParsePy.git@7949b9f754d1445eff8e8f20d0e967b9a6420639#egg=parse_rest -e git+https://github.com/appliedsec/pygeoip.git@95e69341cebf5a6a9fbf7c4f5439d458898bdc3b#egg=pygeoip -e git+https://github.com/dementrock/pystache_custom.git@776973740bdaad83a3b029f96e415a7d1e8bec2f#egg=pystache_custom-dev diff --git a/requirements/edx/github.in b/requirements/edx/github.in index 3968319cb835..743b60972016 100644 --- a/requirements/edx/github.in +++ b/requirements/edx/github.in @@ -89,7 +89,7 @@ # Our libraries: -e git+https://github.com/edx/codejail.git@a320d43ce6b9c93b17636b2491f724d9e433be47#egg=codejail -e git+https://github.com/edx/acid-block.git@e46f9cda8a03e121a00c7e347084d142d22ebfb7#egg=acid-xblock --e git+https://github.com/edx/edx-ora2.git@2.1.17#egg=ora2==2.1.18 +-e git+https://github.com/edx/edx-ora2.git@2.1.18#egg=ora2==2.1.18 -e git+https://github.com/edx/RecommenderXBlock.git@1.3.3#egg=recommender-xblock==1.3.3 -e git+https://github.com/solashirai/crowdsourcehinter.git@518605f0a95190949fe77bd39158450639e2e1dc#egg=crowdsourcehinter-xblock==0.1 -e git+https://github.com/edx/RateXBlock.git@367e19c0f6eac8a5f002fd0f1559555f8e74bfff#egg=rate-xblock diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index 449e145d0dbb..47d90de790a2 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -26,7 +26,7 @@ git+https://github.com/edx/lettuce.git@7a04591c78ac56dac3eb3e91ca94b15cce844133# git+https://github.com/edx/xblock-lti-consumer.git@v1.1.8#egg=lti_consumer-xblock==1.1.8 git+https://github.com/edx/MongoDBProxy.git@25b99097615bda06bd7cdfe5669ed80dc2a7fed0#egg=MongoDBProxy==0.1.0 -e . -git+https://github.com/edx/edx-ora2.git@2.1.17#egg=ora2==2.1.18 +git+https://github.com/edx/edx-ora2.git@2.1.18#egg=ora2==2.1.18 -e git+https://github.com/dgrtwo/ParsePy.git@7949b9f754d1445eff8e8f20d0e967b9a6420639#egg=parse_rest -e git+https://github.com/appliedsec/pygeoip.git@95e69341cebf5a6a9fbf7c4f5439d458898bdc3b#egg=pygeoip -e git+https://github.com/dementrock/pystache_custom.git@776973740bdaad83a3b029f96e415a7d1e8bec2f#egg=pystache_custom-dev From abc069f07ba0604e5952d74ff53d47b2eda9fda2 Mon Sep 17 00:00:00 2001 From: Anders Pearson Date: Mon, 24 Jun 2019 14:16:24 +0100 Subject: [PATCH 2/8] TravisCI setup for running pep8 simplest thing we can get TravisCI to do automatically. --- .travis.yml | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 000000000000..a6494efd3571 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,9 @@ +language: python +python: + - "2.7" + +install: + - pip install tox + +script: + - tox -e py27-django111 -- paver run_pep8 From a504e67e396683178aa0a0c8cb1643f8a0153ecf Mon Sep 17 00:00:00 2001 From: Anders Pearson Date: Mon, 24 Jun 2019 14:26:17 +0100 Subject: [PATCH 3/8] fix a pep8 regression --- pavelib/i18n.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pavelib/i18n.py b/pavelib/i18n.py index 18b54fe1a2d1..5273a4f47273 100644 --- a/pavelib/i18n.py +++ b/pavelib/i18n.py @@ -282,6 +282,7 @@ def i18n_release_pull(): resources = find_release_resources() sh("i18n_tool transifex pull " + " ".join(resources)) + @task @needs( "pavelib.i18n.i18n_clean", From a331bfbb08c804058a27867486ca5e1c9679269c Mon Sep 17 00:00:00 2001 From: Anders Pearson Date: Mon, 24 Jun 2019 15:16:20 +0100 Subject: [PATCH 4/8] travis needs some libs installed --- .travis.yml | 5 +++++ requirements/edx/appsembler.txt | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index a6494efd3571..0decadc9244a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,6 +2,11 @@ language: python python: - "2.7" +before_install: + - sudo rm -f /etc/boto.cfg + - travis_retry sudo apt-get update + - travis_retry sudo apt-get install python-dev libxml2-dev libxmlsec1-dev + install: - pip install tox diff --git a/requirements/edx/appsembler.txt b/requirements/edx/appsembler.txt index fe15186a5afc..ab1e4081deea 100644 --- a/requirements/edx/appsembler.txt +++ b/requirements/edx/appsembler.txt @@ -6,7 +6,7 @@ requests==2.9.1 django-anymail==5.0 django-tiers==0.0.19 dj-database-url==0.4.2 -psycopg2==2.6.2 +psycopg2==2.8.3 django-compat==1.0.14 django-hijack==2.1.4 django-hijack-admin==2.1.4 From b2888d65c087ef94cae5e974ef43d01387b89234 Mon Sep 17 00:00:00 2001 From: Anders Pearson Date: Mon, 24 Jun 2019 15:42:16 +0100 Subject: [PATCH 5/8] make a faster pep8 setup bypass paver --- tox.ini | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tox.ini b/tox.ini index 82d060a67cdd..4e041596dd53 100644 --- a/tox.ini +++ b/tox.ini @@ -60,3 +60,9 @@ commands = bash scripts/upgrade_pysqlite.sh # Now perform testing. {posargs} + +[testenv:pep8] +deps = + pycodestyle==2.3.1 +commands = + pycodestyle . From 40d6bf2ce30ee621b1e609388a51d1fce6743367 Mon Sep 17 00:00:00 2001 From: Anders Pearson Date: Mon, 24 Jun 2019 15:52:59 +0100 Subject: [PATCH 6/8] some pep8 fixes somehow `paver run_pep8` misses these, but the plain `pycodestyle .` catches them. --- pavelib/database.py | 6 +++--- pavelib/utils/db_utils.py | 4 ++-- scripts/xblock/xblock_counts.py | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pavelib/database.py b/pavelib/database.py index 06fe07fcafc9..d2274673ebcf 100644 --- a/pavelib/database.py +++ b/pavelib/database.py @@ -92,11 +92,11 @@ def update_local_bokchoy_db_from_s3(options): fingerprints_match = does_fingerprint_on_disk_match(fingerprint) if fingerprints_match: - print ("DB cache files match the current migrations.") + print("DB cache files match the current migrations.") reset_test_db(BOKCHOY_DB_FILES, update_cache_files=False) elif is_fingerprint_in_bucket(fingerprint, CACHE_BUCKET_NAME): - print ("Found updated bokchoy db files at S3.") + print("Found updated bokchoy db files at S3.") refresh_bokchoy_db_cache_from_s3(fingerprint, CACHE_BUCKET_NAME, BOKCHOY_DB_FILES) reset_test_db(BOKCHOY_DB_FILES, update_cache_files=False) @@ -106,7 +106,7 @@ def update_local_bokchoy_db_from_s3(options): "Loading the bokchoy db files from disk", "and running migrations." ) - print (msg) + print(msg) reset_test_db(BOKCHOY_DB_FILES, update_cache_files=True) # Check one last time to see if the fingerprint is present in # the s3 bucket. This could occur because the bokchoy job is diff --git a/pavelib/utils/db_utils.py b/pavelib/utils/db_utils.py index bf3f3e82f3d1..7045c6476442 100644 --- a/pavelib/utils/db_utils.py +++ b/pavelib/utils/db_utils.py @@ -156,7 +156,7 @@ def get_file_from_s3(bucket_name, zipfile_name, path): """ Get the file from s3 and save it to disk. """ - print ("Retrieving {} from bucket {}.".format(zipfile_name, bucket_name)) + print("Retrieving {} from bucket {}.".format(zipfile_name, bucket_name)) conn = boto.connect_s3(anon=True) bucket = conn.get_bucket(bucket_name) key = boto.s3.key.Key(bucket=bucket, name=zipfile_name) @@ -191,7 +191,7 @@ def refresh_bokchoy_db_cache_from_s3(fingerprint, bucket_name, bokchoy_db_files) zipfile_name = '{}.tar.gz'.format(fingerprint) get_file_from_s3(bucket_name, zipfile_name, path) zipfile_path = os.path.join(path, zipfile_name) - print ("Extracting db cache files.") + print("Extracting db cache files.") extract_files_from_zip(bokchoy_db_files, zipfile_path, path) os.remove(zipfile_path) diff --git a/scripts/xblock/xblock_counts.py b/scripts/xblock/xblock_counts.py index 41eb39a46c87..fffeb9b40e98 100644 --- a/scripts/xblock/xblock_counts.py +++ b/scripts/xblock/xblock_counts.py @@ -215,7 +215,7 @@ def _get_course_block_counts(auth_token, block_url): response = requests.get(block_url, headers=headers) if response.status_code != 200: - print ("url {} returned status code {}".format(block_url, response.status_code)) + print("url {} returned status code {}".format(block_url, response.status_code)) return {} response_json = response.json() From e44500a0bebf7286a4fb1b8fce3377cac58039dc Mon Sep 17 00:00:00 2001 From: Anders Pearson Date: Mon, 24 Jun 2019 15:53:48 +0100 Subject: [PATCH 7/8] run the fast pep8 on travis --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 0decadc9244a..5173dbbe9e53 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,4 +11,5 @@ install: - pip install tox script: + - tox -e pep8 - tox -e py27-django111 -- paver run_pep8 From 3e7cd96bbd2d70edf5fb3931ce2496a6b2ec5f04 Mon Sep 17 00:00:00 2001 From: Anders Pearson Date: Tue, 25 Jun 2019 09:47:00 +0100 Subject: [PATCH 8/8] run tox jobs on travis in parallel --- .travis.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5173dbbe9e53..23acc61bdc30 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,6 +10,10 @@ before_install: install: - pip install tox +env: + - TOXENV=pep8 + - TOXENV=py27-django111 + ARGS="-- paver run_pep8" + script: - - tox -e pep8 - - tox -e py27-django111 -- paver run_pep8 + - tox $ARGS