diff --git a/AUTHORS b/AUTHORS index 619251b6cb38..331712930ae0 100644 --- a/AUTHORS +++ b/AUTHORS @@ -153,4 +153,6 @@ Muhammad Ammar Abdallah Nassif Johnny Brown Ben McMorran -Mat Peterson \ No newline at end of file +Mat Peterson +Andreas Dewes +Leo Urbina diff --git a/common/djangoapps/terrain/stubs/http.py b/common/djangoapps/terrain/stubs/http.py index 621b1eb0293b..8013616431f5 100644 --- a/common/djangoapps/terrain/stubs/http.py +++ b/common/djangoapps/terrain/stubs/http.py @@ -221,6 +221,12 @@ def _format_msg(self, format_str, *args): format_str % args ) + def do_HEAD(self): + """ + Respond to an HTTP HEAD request + """ + self.send_response(200) + class StubHttpService(HTTPServer, object): """ diff --git a/pavelib/__init__.py b/pavelib/__init__.py index d2edaa42a227..5185403e1b41 100644 --- a/pavelib/__init__.py +++ b/pavelib/__init__.py @@ -1,4 +1,13 @@ """ paver commands """ -from . import assets, servers, docs, prereqs, quality, tests, js_test +__all__ = [ + "assets_deprecated", "servers_deprecated", "docs_deprecated", + "prereqs_deprecated", "quality_deprecated", "tests_deprecated", + "js_test_deprecated" +] +from . import ( + assets_deprecated, servers_deprecated, docs_deprecated, + prereqs_deprecated, quality_deprecated, tests_deprecated, + js_test_deprecated +) diff --git a/pavelib/assets.py b/pavelib/assets_deprecated.py similarity index 98% rename from pavelib/assets.py rename to pavelib/assets_deprecated.py index 3d98189555bf..14b80b4dc8cf 100644 --- a/pavelib/assets.py +++ b/pavelib/assets_deprecated.py @@ -10,6 +10,8 @@ import traceback from .utils.envs import Env from .utils.cmd import cmd, django_cmd +from .utils.deprecated import deprecated + # setup baseline paths @@ -190,8 +192,9 @@ def watch_assets(options): @task -@needs('pavelib.prereqs.install_prereqs') +@needs('pavelib.prereqs_deprecated.install_prereqs') @consume_args +@deprecated('invoke assets.update') def update_assets(args): """ Compile CoffeeScript and Sass, then collect static assets. diff --git a/pavelib/docs.py b/pavelib/docs_deprecated.py similarity index 93% rename from pavelib/docs.py rename to pavelib/docs_deprecated.py index a6901fefe922..4997c913aa2f 100644 --- a/pavelib/docs.py +++ b/pavelib/docs_deprecated.py @@ -1,7 +1,7 @@ from __future__ import print_function import sys from paver.easy import * - +from .utils.deprecated import deprecated DOC_PATHS = { "dev": "docs/en_us/developers", @@ -54,11 +54,12 @@ def doc_path(options, allow_default=True): @task -@needs('pavelib.prereqs.install_prereqs') +@needs('pavelib.prereqs_deprecated.install_prereqs') @cmdopts([ ("type=", "t", "Type of docs to compile"), ("verbose", "v", "Display verbose output"), ]) +@deprecated('invoke docs.build') def build_docs(options): """ Invoke sphinx 'make build' to generate docs. diff --git a/pavelib/js_test.py b/pavelib/js_test_deprecated.py similarity index 91% rename from pavelib/js_test.py rename to pavelib/js_test_deprecated.py index d69e60da3db2..6865f62218bd 100644 --- a/pavelib/js_test.py +++ b/pavelib/js_test_deprecated.py @@ -5,6 +5,7 @@ from paver.easy import task, cmdopts, needs from pavelib.utils.test.suites import JsTestSuite from pavelib.utils.envs import Env +from .utils.deprecated import deprecated __test__ = False # do not collect @@ -19,6 +20,7 @@ ("mode=", "m", "dev or run"), ("coverage", "c", "Run test under coverage"), ]) +@deprecated('invoke js_test.test_js') def test_js(options): """ Run the JavaScript tests @@ -52,6 +54,7 @@ def test_js(options): ("suite=", "s", "Test suite to run"), ("coverage", "c", "Run test under coverage"), ]) +@deprecated('invoke js_test.test_js_run') def test_js_run(options): """ Run the JavaScript tests and print results to the console @@ -64,6 +67,7 @@ def test_js_run(options): @cmdopts([ ("suite=", "s", "Test suite to run"), ]) +@deprecated('invoke js_test.test_js_dev') def test_js_dev(options): """ Run the JavaScript tests in your default browsers diff --git a/pavelib/prereqs.py b/pavelib/prereqs_deprecated.py similarity index 95% rename from pavelib/prereqs.py rename to pavelib/prereqs_deprecated.py index 014a23e3e5d4..aca5424530f9 100644 --- a/pavelib/prereqs.py +++ b/pavelib/prereqs_deprecated.py @@ -7,7 +7,7 @@ from distutils import sysconfig from paver.easy import * from .utils.envs import Env - +from .utils.deprecated import deprecated PREREQS_MD5_DIR = os.getenv('PREREQ_CACHE_DIR', Env.REPO_ROOT / '.prereqs_cache') NPM_REGISTRY = "http://registry.npmjs.org/" @@ -117,6 +117,7 @@ def python_prereqs_installation(): @task +@deprecated('invoke prereqs.install.ruby') def install_ruby_prereqs(): """ Installs Ruby prereqs @@ -125,6 +126,7 @@ def install_ruby_prereqs(): @task +@deprecated('invoke prereqs.install.node') def install_node_prereqs(): """ Installs Node prerequisites @@ -133,6 +135,7 @@ def install_node_prereqs(): @task +@deprecated('invoke prereqs.install.python') def install_python_prereqs(): """ Installs Python prerequisites @@ -141,6 +144,7 @@ def install_python_prereqs(): @task +@deprecated('invoke prereqs.install') def install_prereqs(): """ Installs Ruby, Node and Python prerequisites diff --git a/pavelib/quality.py b/pavelib/quality_deprecated.py similarity index 100% rename from pavelib/quality.py rename to pavelib/quality_deprecated.py diff --git a/pavelib/servers.py b/pavelib/servers_deprecated.py similarity index 87% rename from pavelib/servers.py rename to pavelib/servers_deprecated.py index 9f2c176cbad5..068363a89472 100644 --- a/pavelib/servers.py +++ b/pavelib/servers_deprecated.py @@ -7,7 +7,7 @@ from paver.easy import * from .utils.cmd import django_cmd from .utils.process import run_process, run_multi_processes - +from .utils.deprecated import deprecated DEFAULT_PORT = {"lms": 8000, "studio": 8001} DEFAULT_SETTINGS = 'dev' @@ -42,12 +42,13 @@ def run_server(system, settings=None, port=None, skip_assets=False): @task -@needs('pavelib.prereqs.install_prereqs') +@needs('pavelib.prereqs_deprecated.install_prereqs') @cmdopts([ ("settings=", "s", "Django settings"), ("port=", "p", "Port"), ("fast", "f", "Skip updating assets") ]) +@deprecated('invoke servers.lms') def lms(options): """ Run the LMS server. @@ -59,12 +60,13 @@ def lms(options): @task -@needs('pavelib.prereqs.install_prereqs') +@needs('pavelib.prereqs_deprecated.install_prereqs') @cmdopts([ ("settings=", "s", "Django settings"), ("port=", "p", "Port"), ("fast", "f", "Skip updating assets") ]) +@deprecated('invoke servers.cms') def studio(options): """ Run the Studio server. @@ -76,8 +78,9 @@ def studio(options): @task -@needs('pavelib.prereqs.install_prereqs') +@needs('pavelib.prereqs_deprecated.install_prereqs') @consume_args +@deprecated('invoke --help servers.devstack') def devstack(args): """ Start the devstack lms or studio server @@ -90,10 +93,11 @@ def devstack(args): @task -@needs('pavelib.prereqs.install_prereqs') +@needs('pavelib.prereqs_deprecated.install_prereqs') @cmdopts([ ("settings=", "s", "Django settings"), ]) +@deprecated('invoke servers.celery') def celery(options): """ Runs Celery workers. @@ -103,7 +107,7 @@ def celery(options): @task -@needs('pavelib.prereqs.install_prereqs') +@needs('pavelib.prereqs_deprecated.install_prereqs') @cmdopts([ ("settings=", "s", "Django settings for both LMS and Studio"), ("worker_settings=", "w", "Celery worker Django settings"), @@ -111,6 +115,8 @@ def celery(options): ("settings_lms=", "l", "Set LMS only, overriding the value from --settings (if provided)"), ("settings_cms=", "c", "Set Studio only, overriding the value from --settings (if provided)"), ]) +# Can't deal with parsing arguments of deprecated functions +@deprecated('invoke servers.run') def run_all_servers(options): """ Runs Celery workers, Studio, and LMS. @@ -137,10 +143,11 @@ def run_all_servers(options): @task -@needs('pavelib.prereqs.install_prereqs') +@needs('pavelib.prereqs_deprecated.install_prereqs') @cmdopts([ ("settings=", "s", "Django settings"), ]) +@deprecated('invoke servers.update_db') def update_db(): """ Runs syncdb and then migrate. @@ -151,8 +158,10 @@ def update_db(): @task -@needs('pavelib.prereqs.install_prereqs') +@needs('pavelib.prereqs_deprecated.install_prereqs') @consume_args +# Can't deal with parsing arguments of deprecated functions +@deprecated('invoke --help servers.check_settings') def check_settings(args): """ Checks settings files. diff --git a/pavelib/tests.py b/pavelib/tests_deprecated.py similarity index 96% rename from pavelib/tests.py rename to pavelib/tests_deprecated.py index 4ec4f6453652..e16d6dc7e553 100644 --- a/pavelib/tests.py +++ b/pavelib/tests_deprecated.py @@ -7,6 +7,7 @@ from pavelib.utils.test import suites from pavelib.utils.envs import Env from optparse import make_option +from .utils.deprecated import deprecated try: from pygments.console import colorize @@ -31,6 +32,7 @@ make_option("-q", "--quiet", action="store_const", const=0, dest="verbosity"), make_option("-v", "--verbosity", action="count", dest="verbosity", default=1), ]) +@deprecated('invoke tests.test_system') def test_system(options): """ Run tests on our djangoapps for lms and cms @@ -75,6 +77,7 @@ def test_system(options): make_option("-q", "--quiet", action="store_const", const=0, dest="verbosity"), make_option("-v", "--verbosity", action="count", dest="verbosity", default=1), ]) +@deprecated('invoke tests.test_lib') def test_lib(options): """ Run tests for common/lib/ @@ -111,6 +114,7 @@ def test_lib(options): make_option("-q", "--quiet", action="store_const", const=0, dest="verbosity"), make_option("-v", "--verbosity", action="count", dest="verbosity", default=1), ]) +@deprecated('invoke tests.test_python') def test_python(options): """ Run all python tests @@ -130,6 +134,7 @@ def test_python(options): 'pavelib.prereqs.install_python_prereqs', 'pavelib.utils.test.utils.clean_reports_dir', ) +@deprecated('invoke tests.test_i18n') def test_i18n(): """ Run all i18n tests @@ -148,6 +153,7 @@ def test_i18n(): make_option("-q", "--quiet", action="store_const", const=0, dest="verbosity"), make_option("-v", "--verbosity", action="count", dest="verbosity", default=1), ]) +@deprecated('invoke tests.test') def test(options): """ Run all tests @@ -170,6 +176,7 @@ def test(options): @cmdopts([ ("compare_branch", "b", "Branch to compare against, defaults to origin/master"), ]) +@deprecated('invoke tests.coverage') def coverage(options): """ Build the html, xml, and diff coverage reports diff --git a/pavelib/utils/deprecated.py b/pavelib/utils/deprecated.py new file mode 100644 index 000000000000..a7c922233783 --- /dev/null +++ b/pavelib/utils/deprecated.py @@ -0,0 +1,17 @@ +from __future__ import print_function +from invoke import run as sh +from pygments.console import colorize + +def deprecated(deprecated_by): + def deprecated_decorator(func): + def wrapper(*args, **kwargs): + sh("pip install -q -r requirements/edx/invoke.txt") + print(colorize("darkred", "Task {name} has been deprecated. Use '{deprecated_by}' instead.".\ + format(name=func.__name__, deprecated_by=deprecated_by))) + sh(deprecated_by, echo=True) + # Copy over the necessary metadata + wrapper.__name__ = func.__name__ + wrapper.__module__ = func.__module__ + wrapper.__doc__ = func.__doc__ + return wrapper + return deprecated_decorator diff --git a/requirements/edx/invoke.txt b/requirements/edx/invoke.txt new file mode 100644 index 000000000000..f52238bcde98 --- /dev/null +++ b/requirements/edx/invoke.txt @@ -0,0 +1,4 @@ +invoke==0.7.0 +lazy==1.1 +path.py==5.1 +wsgiref==0.1.2 diff --git a/tasks/__init__.py b/tasks/__init__.py new file mode 100644 index 000000000000..a2dca4b0eab2 --- /dev/null +++ b/tasks/__init__.py @@ -0,0 +1,24 @@ +from __future__ import print_function +from invoke import Collection + +ns = Collection() + + +from . import assets +from . import clean +from . import db +from . import docs +from . import i18n +from . import prereqs +from . import servers +from . import test + + +ns.add_collection(assets) +ns.add_collection(clean) +ns.add_collection(db) +ns.add_collection(docs) +ns.add_collection(i18n) +ns.add_collection(prereqs) +ns.add_collection(servers) +ns.add_collection(test) diff --git a/tasks/assets.py b/tasks/assets.py new file mode 100644 index 000000000000..c999683b1337 --- /dev/null +++ b/tasks/assets.py @@ -0,0 +1,232 @@ +""" +Asset compilation and collection. +""" +from __future__ import print_function +from invoke import task +from invoke import run as sh +from watchdog.observers import Observer +from watchdog.events import PatternMatchingEventHandler +import glob +import traceback +from path import path +from .utils.envs import Env +from .utils.cmd import cmd, django_cmd +try: + from pygments.console import colorize +except ImportError: + colorize = lambda color, text: text + +COFFEE_DIRS = ['lms', 'cms', 'common'] +SASS_LOAD_PATHS = ['./common/static/sass'] +SASS_UPDATE_DIRS = ['*/static'] +SASS_CACHE_PATH = '/tmp/sass-cache' + + +class CoffeeScriptWatcher(PatternMatchingEventHandler): + """ + Watches for coffeescript changes + """ + ignore_directories = True + patterns = ['*.coffee'] + + def register(self, observer): + """ + register files with observer + """ + dirnames = set() + for filename in sh(coffeescript_files(), hide='stdout').stdout.splitlines(): + dirnames.add(path(filename).abspath().dirname()) + for dirname in dirnames: + observer.schedule(self, dirname) + + def on_modified(self, event): + print('\tCHANGED:', event.src_path) + try: + compile_coffeescript(event.src_path) + except Exception: # pylint: disable=W0703 + traceback.print_exc() + + +class SassWatcher(PatternMatchingEventHandler): + """ + Watches for sass file changes + """ + ignore_directories = True + patterns = ['*.scss'] + ignore_patterns = ['common/static/xmodule/*'] + + def register(self, observer): + """ + register files with observer + """ + for dirname in SASS_LOAD_PATHS + SASS_UPDATE_DIRS + theme_sass_paths(): + paths = [] + if '*' in dirname: + paths.extend(glob.glob(dirname)) + else: + paths.append(dirname) + for dirname in paths: + observer.schedule(self, dirname, recursive=True) + + def on_modified(self, event): + print('\tCHANGED:', event.src_path) + try: + compile_sass() + except Exception: # pylint: disable=W0703 + traceback.print_exc() + + +class XModuleSassWatcher(SassWatcher): + """ + Watches for sass file changes + """ + ignore_directories = True + ignore_patterns = [] + + def register(self, observer): + """ + register files with observer + """ + observer.schedule(self, 'common/lib/xmodule/', recursive=True) + + def on_modified(self, event): + print('\tCHANGED:', event.src_path) + try: + process_xmodule_assets() + except Exception: # pylint: disable=W0703 + traceback.print_exc() + + +def theme_sass_paths(): + """ + Return the a list of paths to the theme's sass assets, + or an empty list if no theme is configured. + """ + edxapp_env = Env() + + if edxapp_env.feature_flags.get('USE_CUSTOM_THEME', False): + theme_name = edxapp_env.env_tokens.get('THEME_NAME', '') + parent_dir = path(edxapp_env.REPO_ROOT).abspath().parent + theme_root = parent_dir / "themes" / theme_name + return [theme_root / "static" / "sass"] + else: + return [] + + +def coffeescript_files(): + """ + return find command for paths containing coffee files + """ + dirs = " ".join([Env.REPO_ROOT / coffee_dir for coffee_dir in COFFEE_DIRS]) + return cmd('find', dirs, '-type f', '-name \"*.coffee\"') + + +def compile_coffeescript(*files): + """ + Compile CoffeeScript to JavaScript. + """ + if not files: + files = ["`{}`".format(coffeescript_files())] + sh(cmd( + "node_modules/.bin/coffee", "--compile", *files + )) + + +def compile_sass(debug=False): + """ + Compile Sass to CSS. + """ + theme_paths = theme_sass_paths() + sh(cmd( + 'sass', '' if debug else '--style compressed', + "--sourcemap", + "--cache-location {cache}".format(cache=SASS_CACHE_PATH), + "--load-path", " ".join(SASS_LOAD_PATHS + theme_paths), + "--update", "-E", "utf-8", " ".join(SASS_UPDATE_DIRS + theme_paths), + )) + + +def compile_templated_sass(systems, settings): + """ + Render Mako templates for Sass files. + `systems` is a list of systems (e.g. 'lms' or 'cms' or both) + `settings` is the Django settings module to use. + """ + for sys in systems: + sh(django_cmd(sys, settings, 'preprocess_assets')) + + +def process_xmodule_assets(): + """ + Process XModule static assets. + """ + sh('xmodule_assets common/static/xmodule') + + +def collect_assets(systems, settings): + """ + Collect static assets, including Django pipeline processing. + `systems` is a list of systems (e.g. 'lms' or 'cms' or both) + `settings` is the Django settings module to use. + """ + for sys in systems: + sh(django_cmd(sys, settings, "collectstatic --noinput > /dev/null")) + + +@task +def watch(background=False, **kwargs): + """ + Watch for changes to asset files, and regenerate js/css + """ + observer = Observer() + + CoffeeScriptWatcher().register(observer) + SassWatcher().register(observer) + XModuleSassWatcher().register(observer) + + print("Starting asset watcher...") + observer.start() + if not background: + # when running as a separate process, the main thread needs to loop + # in order to allow for shutdown by contrl-c + try: + while True: + observer.join(2) + except KeyboardInterrupt: + observer.stop() + print("\nStopped asset watcher.") + + + +@task('prereqs.install', default=True, help={ + "system": "lms or cms", + "settings": "Django settings module", + "debug": "Disable Sass compression", + "skip-collect": "Skip collection of static assets", + "watch": "Watch files for changes", +}) +def update(system=None, watch=False, settings="dev", debug=False, skip_collect=True, **kwargs): + """ + Compile CoffeeScript and Sass, then collect static assets. + """ + + if system is None: + system = ['lms', 'cms'] + else: + system = [system] + + compile_templated_sass(system, settings) + process_xmodule_assets() + compile_coffeescript() + compile_sass(debug) + + if not skip_collect: + collect_assets(system, settings) + print(colorize('white', "Done collecting assets...")) + + if watch: + print(colorize('white', "Starting to watch assets")) + cmd = "invoke assets.watch" + if not debug: + cmd += " --background" + sh(cmd) diff --git a/tasks/clean.py b/tasks/clean.py new file mode 100644 index 000000000000..7cdddfe78ff2 --- /dev/null +++ b/tasks/clean.py @@ -0,0 +1,27 @@ +from __future__ import print_function + +import os +import sys +from distutils.spawn import find_executable + +from path import path +from invoke import task, Collection +from invoke import run as sh +try: + from pygments.console import colorize +except ImportError: + colorize = lambda color, text: text +from .utils.cmd import cmd +from .utils.envs import Env +from .i18n import I18N_REPORT_DIR + +ns = Collection() + + +@task +def clean_reports_dir(): + """Clean coverage files, to ensure that we don't use stale data to generate reports.""" + I18N_REPORT_DIR.rmtree_p() + I18N_REPORT_DIR.makedirs_p() + +ns.add_task(clean_reports_dir, "reports") diff --git a/tasks/db.py b/tasks/db.py new file mode 100644 index 000000000000..c6995dc9c09e --- /dev/null +++ b/tasks/db.py @@ -0,0 +1,25 @@ +from __future__ import print_function +from invoke import task +from invoke import run as sh +try: + from pygments.console import colorize +except ImportError: + colorize = lambda color, text: text +from .utils.cmd import django_cmd + + +@task('prereqs.install', name="update", help={ + "settings": "Django settings", + "verbose": "Display verbose output" +}) +def update_db(settings='dev', verbose=False): + """ + Runs syncdb and then migrate. + """ + hide = None + if not verbose: + hide = 'both' + + sh(django_cmd('lms', settings, 'syncdb', '--traceback', '--pythonpath=.'), hide=hide, echo=True) + sh(django_cmd('lms', settings, 'migrate', '--traceback', '--pythonpath=.'), hide=hide, echo=True) + print(colorize("green", "DB sucessufully updated")) diff --git a/tasks/docs.py b/tasks/docs.py new file mode 100644 index 000000000000..d508324d95f3 --- /dev/null +++ b/tasks/docs.py @@ -0,0 +1,59 @@ +from __future__ import print_function +import sys + +from invoke import task +from invoke import run as sh + +DOC_PATHS = { + "dev": "docs/en_us/developers", + "author": "docs/en_us/course_authors", + "data": "docs/en_us/data", + "default": "docs/en_us" +} + + +def valid_doc_types(): + """ + Return a comma-separated string of valid doc types. + """ + return ", ".join(DOC_PATHS.keys()) + + +def doc_path(doc_type, allow_default=True): + """ + Determine the path of the documentation directory based on the document type. + If the specified path is not one of the valid options, print an error + message and exit. + + If `allow_default` is False, then require that a type is specified, + and exit with an error message if it isn't. + """ + + path = DOC_PATHS.get(doc_type) + + if doc_type == 'default' and not allow_default: + print("You must specify a documentation type using '--type'. " + "Valid options are: {options}".format( + options=valid_doc_types())) + sys.exit(2) + + if path is None: + print("Invalid documentation type '{doc_type}'. " + "Valid options are: {options}".format( + doc_type=doc_type, options=valid_doc_types())) + sys.exit(2) + return path + + +@task('prereqs.install', default=True, help={ + "type": "Type of docs to compile", + "verbose": "Display verbose output" +}) +def build(type='default', verbose=False): + """ + Invoke sphinx 'make build' to generate docs. + """ + + sh("cd {dir}; make html quiet={quiet}" + .format(dir=doc_path(type), + quiet="false" if verbose else "true")) diff --git a/tasks/i18n.py b/tasks/i18n.py new file mode 100644 index 000000000000..61b0baa23e2d --- /dev/null +++ b/tasks/i18n.py @@ -0,0 +1,163 @@ +""" +Internationalization tasks + +NOTE: if a function has **kwargs, it means it is a pretask +for a task taking certain args/options (i.e. **kwargs are ignored) +""" +from __future__ import print_function + +import os +import sys +from distutils.spawn import find_executable + +from path import path +from invoke import task, Collection +from invoke import run as sh +try: + from pygments.console import colorize +except ImportError: + colorize = lambda color, text: text +from .utils.cmd import cmd +from .utils.envs import Env +from .test import test_i18n + +I18N_REPORT_DIR = Env.REPORT_DIR / 'i18n' +I18N_XUNIT_REPORT = I18N_REPORT_DIR.joinpath('nosetests.xml') + +ns = Collection() +ns_validate = Collection('validate') +ns_robot = Collection('robot') + + +@task('i18n.validate.gettext', 'assets.update') +def extract(verbose=False, **kwargs): + """ + Extract localizable strings from sources + Params: + verbose=False Display verbose output + """ + executable = Env.REPO_ROOT / 'i18n/extract.py' + print("Executable", executable) + if verbose: + sh(cmd(executable, '-vv')) + else: + sh(cmd(executable)) + +ns.add_task(extract) + + +@task('i18n.extract') +def generate(strict=False): + """ + Compile localizable strings from sources, extracting strings first. + Params: + strict=False Complain if files are missing + """ + executable = Env.REPO_ROOT / 'i18n/generate.py' + if strict: + sh(cmd(executable, '--strict')) + else: + sh(cmd(executable)) + +ns.add_task(generate, default=True) + + +@task('i18n.extract') +def dummy(): + """ + Simulate international translation by generating dummy strings + corresponding to source strings. + """ + executable = Env.REPO_ROOT / 'i18n/dummy.py' + sh(cmd(executable)) + +ns.add_task(dummy) + + +@task +def validate_gettext(**kwargs): + """Make sure GNU gettext utilities are available""" + if find_executable('xgettext') is None: + err = ( + "Cannot locate GNU gettext utilities, which are required by Django " + "for internationalization.\n See " + "https://docs.djangoproject.com/en/dev/topics/i18n/translation/#message-files\n" + "Try downloading them from http://www.gnu.org/software/gettext/" + ) + print(colorize("darkred", err)) + sys.exit(1) + +ns_validate.add_task(validate_gettext, 'gettext') + +@task +def validate_transifex_config(): + """Make sure config file with username/password exists""" + pathstr = os.environ['HOME'] + '/.transifexrc' + config_file = path(pathstr) + if not (config_file.exists() and config_file.size > 0): + print(colorize("darkred", + "Cannot connect to Transifex, config file is missing or empty: " + "{}\n See http://help.transifex.com/features/client/#transifexrc" + .format(pathstr))) + sys.exit(1) + +ns_validate.add_task(validate_transifex_config, 'transifex') + +@task +def validate_all(): + """ + Validate everything related to i18n + """ + validate_gettext() + validate_transifex_config() + +ns_validate.add_task(validate_all, "all", default=True) + +@task('i18n.validate.transifex') +def transifex_push(): + """Push source strings to Transifex for translation""" + transifex_executable = Env.REPO_ROOT / 'i18n/transifex.py' + sh(cmd(transifex_executable, 'push')) + +ns.add_task(transifex_push, "push") + + +@task('i18n.validate.transifex') +def transifex_pull(): + """Pull translated strings from Transifex""" + transifex_executable = Env.REPO_ROOT / 'i18n/transifex.py' + sh(cmd(transifex_executable, 'pull')) + +ns.add_task(transifex_pull, "pull") + + +@task("i18n.pull", "i18n.extract", "i18n.dummy") +def robot_pull(): + """Pull source strings, generato po and mo files, and validate""" + #XXX: The develop branch of invoke allows for specifying call + #signatures of pre tasks using the `Call` class + sh(cmd("inv", "i18n.generate", "--strict")) + sh(cmd("git", "clean", "-fdX", "conf/locale")) + sh(cmd("inv", "i18n.test")) + sh(cmd("git", "add", "conf/locale")) + sh(cmd("git", "commit", '--message="Update translations (autogenerated message)"','--edit')) + +ns_robot.add_task(robot_pull, "pull", default=True) + +@task("i18n.extract", "i18n.push") +def robot_push(): + """Extract new strings, and push to transifex""" + pass + +ns_robot.add_task(robot_push, "push") + +@task +def test(): + # proxy to `test.i18n` + test_i18n() + +test.__doc__ = test_i18n.__doc__ +ns.add_task(test) + +ns.add_collection(ns_validate) +ns.add_collection(ns_robot) diff --git a/tasks/prereqs.py b/tasks/prereqs.py new file mode 100644 index 000000000000..63dd0a58a8dc --- /dev/null +++ b/tasks/prereqs.py @@ -0,0 +1,175 @@ +""" +Install Python, Ruby, and Node prerequisites. +""" + +import os +import hashlib +from distutils import sysconfig +from invoke import Collection +from invoke import task +from invoke import run as sh +from path import path + +from .utils.envs import Env + +ns = Collection() + + +PREREQS_MD5_DIR = os.getenv('PREREQ_CACHE_DIR', Env.REPO_ROOT / '.prereqs_cache') +PREREQS_MD5_DIR = path(PREREQS_MD5_DIR) +NPM_REGISTRY = "http://registry.npmjs.org/" +PYTHON_REQ_FILES = [ + path('requirements/edx/pre.txt'), + path('requirements/edx/github.txt'), + path('requirements/edx/local.txt'), + path('requirements/edx/base.txt'), + path('requirements/edx/post.txt'), +] + +# Developers can have private requirements, for local copies of github repos, +# or favorite debugging tools, etc. +PRIVATE_REQS = path('requirements/private.txt') +if os.path.exists(PRIVATE_REQS): + PYTHON_REQ_FILES.append(PRIVATE_REQS) + + +def compute_fingerprint(path_list): + """ + Hash the contents of all the files and directories in `path_list`. + Returns the hex digest. + """ + + hasher = hashlib.sha1() + + for p in path_list: + p = path(p) + + # For directories, create a hash based on the modification times + # of first-level subdirectories + if p.isdir(): + for dir in sorted(p.dirs()): + mtime = dir.stat().st_mtime + hasher.update(str(mtime)) + + # For files, hash the contents of the file + if p.isfile(): + hasher.update(p.text()) + + return hasher.hexdigest() + + +def prereq_cache(cache_name, paths, install_func): + """ + Conditionally execute `install_func()` only if the files/directories + specified by `paths` have changed. + + If the code executes successfully (no exceptions are thrown), the cache + is updated with the new hash. + """ + # Retrieve the old hash + cache_filename = cache_name.replace(" ", "_") + cache_file_path = PREREQS_MD5_DIR / (cache_filename + ".sha1") + old_hash = None + if cache_file_path.isfile(): + with open(cache_file_path) as cache_file: + old_hash = cache_file.read() + + # Compare the old hash to the new hash + # If they do not match (either the cache hasn't been created, or the files have changed), + # then execute the code within the block. + new_hash = compute_fingerprint(paths) + if new_hash != old_hash: + install_func() + + # Update the cache with the new hash + # If the code executed within the context fails (throws an exception), + # then this step won't get executed. + PREREQS_MD5_DIR.makedirs_p() + + # recalculate hash + new_hash = compute_fingerprint(paths) + # write to cache file + cache_file_path.write_text(new_hash) + + else: + print('{cache} unchanged, skipping...'.format(cache=cache_name)) + + +@task +def show_cache_hashes(): + """ + Show SHA1 hashes for prereq caches + """ + caches = ("Ruby prereqs", "Node prereqs", "Python prereqs") + for cache in caches: + cache_filename = cache.replace(" ", "_") + cache_file_path = PREREQS_MD5_DIR / (cache_filename + ".sha1") + if cache_file_path.isfile(): + with open(cache_file_path) as cache_file: + prereq_hash = cache_file.read() + print("{cache}: {hash}".format(cache=cache, hash=prereq_hash)) + + +@task +def flush_cache_hashes(): + """ + Flush prereq caches + """ + if not PREREQS_MD5_DIR.exists(): + return + for f in PREREQS_MD5_DIR.files(): + f.remove() + +@task +def install_ruby_prereqs(): + """ + Installs Ruby prereqs + """ + sh('bundle install --quiet') + + +@task +def install_node_prereqs(): + """ + Installs Node prerequisites + """ + sh("npm config set registry {}".format(NPM_REGISTRY)) + sh('npm install') + + +@task +def install_python_prereqs(): + """ + Installs Python prerequisites + """ + for req_file in PYTHON_REQ_FILES: + print(req_file) + sh("pip install --exists-action w -r {req_file}".format(req_file=req_file), hide='stdout') + + +@task(default=True) +def install(**kwargs): + """ + Installs Ruby, Node and Python prerequisites + """ + if os.environ.get("NO_PREREQ_INSTALL", False): + return + + prereq_cache("Ruby prereqs", ["Gemfile"], install_ruby_prereqs) + prereq_cache("Node prereqs", ["package.json"], install_node_prereqs) + prereq_cache("Python prereqs", PYTHON_REQ_FILES + [sysconfig.get_python_lib()], install_python_prereqs) + + +install_ns = Collection('install') +install_ns.add_task(install_ruby_prereqs, 'ruby') +install_ns.add_task(install_node_prereqs, 'node') +install_ns.add_task(install_python_prereqs, 'python') +install_ns.add_task(install, 'all', default=True) +ns.add_collection(install_ns) + +cache_ns = Collection('cache') +cache_ns.add_task(show_cache_hashes, "show", default=True) +cache_ns.add_task(flush_cache_hashes, "flush") +ns.add_collection(cache_ns) + +ns.default = "install.all" diff --git a/tasks/pylint.py b/tasks/pylint.py new file mode 100644 index 000000000000..bed2499e971d --- /dev/null +++ b/tasks/pylint.py @@ -0,0 +1,47 @@ +""" +Run pylint on the code +""" +from __future__ import print_function +import sys +from invoke import task +from invoke import run as sh +from path import path +from .utils import Env + +def run_pylint(system, report_dir=None, flags=""): + apps = [path(system)] + path("{system}/djangoapps".format(system=system)).glob("*") + if system != 'lms': + apps += path("{system}/lib".format(system=system)).glob("*") + + apps = [app.basename().stripext() for app in apps if not app.endswith(".pyc")] + + + from pprint import pprint + pprint(apps) + + + pythonpath = "PYTHONPATH={system}:{system}/djangoapps:{system}/lib:common/djangoapps:common/lib".format(system=system) + + import ipdb + ipdb.set_trace() + sh("{pythonpath} pylint {flags} -f parseable {apps} | tee {report_dir}/pylint.report".format(pythonpath=pythonpath, + flags=flags, + apps=" ".join(apps), + report_dir=report_dir, + )) +def run_pep8(system, report_dir=None): + sh("pep8 {system} | tee {report_dir}/pep8.report") + + + +for system in ['lms', 'cms', 'common']: + report_dir = Env.REPO_ROOT/system + + + + +def main(): + run_pylint("cms") + +if __name__ == '__main__': + main() diff --git a/tasks/servers.py b/tasks/servers.py new file mode 100644 index 000000000000..5c4057b81541 --- /dev/null +++ b/tasks/servers.py @@ -0,0 +1,137 @@ +""" +Run and manage servers for local development. +""" +from __future__ import print_function +import sys +import traceback +from invoke import task, Collection +from invoke import run as sh +try: + from pygments.console import colorize +except ImportError: + colorize = lambda color, text: text + +from .utils.cmd import django_cmd +from .utils.process import run_process, run_multi_processes + +DEFAULT_PORT = {"lms": 8000, "studio": 8001} + + +def run_server(system, settings="dev", port=0, skip_assets=False): + """ + Start the server for the specified `system` (lms or cms). + `settings` is the Django settings module to use; if not provided, use the default. + `port` is the port to run the server on; if not provided, use the default port for the system. + + If `skip_assets` is True, skip the asset compilation step. + """ + if system not in ['lms', 'cms']: + print(colorize("darkred", "System must be either lms or cms", file=sys.stderr)) + exit(1) + + if not skip_assets: + # Local dev settings use staticfiles to serve assets, so we can skip the collecstatic step + sh('invoke assets.update --system={system} --settings={settings}'.format(system=system, settings=settings), echo=True) + + if not port: + port = DEFAULT_PORT[system] + + run_process(django_cmd( + system, settings, 'runserver', '--traceback', + '--pythonpath=.', '0.0.0.0:{}'.format(port))) + + +@task('prereqs.install', help={ + "settings": "Django settings", + "port": "Port", + "fast": "Skip updating assets", +}) +def lms(settings="dev", port=8000, fast=False): + """ + Run the LMS server. + """ + run_server('lms', settings=settings, port=port, skip_assets=fast) + + +@task('prereqs.install', help={ + "settings": "Django settings", + "port": "Port", + "fast": "Skip updating assets", +}) +def cms(settings="dev", port=8001, fast=False): + """ + Run the cms server. + """ + run_server('cms', settings=settings, port=port, skip_assets=fast) + + +@task('prereqs.install', positional=("system"), help={ + "system": "lms or cms", + "fast": "Skip updating assets", +}) +def devstack(system, fast=False): + """ + Start the devstack LMS or CMS server + """ + if system is None: + print(colorize("lightgray", "Usage: invoke servers.devstack --system (lms|cms) [--fast]")) + sys.exit(2) + run_server(system, settings='devstack', skip_assets=fast) + + +@task('prereqs.install', help={ + "settings": "Django settings" +}) +def celery(settings="dev_with_worker"): + """ + Runs Celery workers. + """ + run_process(django_cmd('lms', settings, 'celery', 'worker', '--loglevel=INFO', '--pythonpath=.')) + + +@task('prereqs.install', default=True, help={ + "settings": "Django settings", + "worker_settings": "Celery worker Django settings", + "fast": "Skip updating assets", +}) +def run(settings="dev", worker_settings='dev_with_worker', fast=False): + """ + Runs Celery workers, CMS and LMS. + """ + if not fast: + # This is annoying: invoke does not support calling + # tasks within tasks... + + sh('invoke assets.update --settings={settings} --skip-collect'.format(settings=settings), hide='both', echo=True) + sh('invoke assets.watch --background', hide='both', echo=True) + run_multi_processes([ + django_cmd('lms', settings, 'runserver', '--traceback', '--pythonpath=.', "0.0.0.0:{}".format(DEFAULT_PORT['lms'])), + django_cmd('studio', settings, 'runserver', '--traceback', '--pythonpath=.', "0.0.0.0:{}".format(DEFAULT_PORT['studio'])), + django_cmd('lms', worker_settings, 'celery', 'worker', '--loglevel=INFO', '--pythonpath=.') + ]) + + +@task('prereqs.install', help={ + 'system': "lms or cms", + 'settings': "Django settings", +}) +def check_settings(system=None, settings=None): + """ + Checks settings files. + """ + if system is None or settings is None: + print(colorize("lightgray", +'''Usage: + invoke servers.check_settings --system (lms|cms) --settings +''')) + print("Too few arguments") + sys.exit(2) + + try: + import_cmd = "echo 'import {system}.envs.{settings}'".format(system=system, settings=settings) + django_shell_cmd = django_cmd(system, settings, 'shell', '--plain', '--pythonpath=.') + sh("{import_cmd} | {shell_cmd}".format(import_cmd=import_cmd, shell_cmd=django_shell_cmd), hide='both') + print(colorize("green", "{system} settings for {settings} are ok.".format(system=system, settings=settings))) + except Exception as exc: + traceback.print_exc() + print(colorize("darkred", "Failed to import settings", file=sys.stderr)) diff --git a/tasks/test/__init__.py b/tasks/test/__init__.py new file mode 100644 index 000000000000..621b58998dc7 --- /dev/null +++ b/tasks/test/__init__.py @@ -0,0 +1,211 @@ +""" +Unit test tasks +""" +import os +import sys +from invoke import task, Collection +from invoke import run as sh +from tasks.utils.test import suites +from tasks.utils.envs import Env +from .js import ns as ns_js +from .bok_choy import ns as ns_bok_choy + +ns = Collection() +ns.add_collection(ns_js) +ns.add_collection(ns_bok_choy) + +try: + from pygments.console import colorize +except ImportError: + colorize = lambda color, text: text # pylint: disable-msg=invalid-name + +__test__ = False # do not collect + + +@task('prereqs.install', help={ + "system": "System to act on", + "test_id": "Test id", + "failed": "Run only failed tests", + "fail_fast": "Run only failed tests", + "fasttest": "Run without collectstatic", + "verbosity": "Turn logging up or down", +}) +def test_system( + system=None, test_id=None, failed=None, fail_fast=None, + fasttest=None, verbosity=1 + ): + """ + Run tests on our djangoapps for lms and cms + """ + opts = { + 'failed_only': failed, + 'fail_fast': fail_fast, + 'fasttest': fasttest, + 'verbosity': verbosity, + } + + if test_id: + if not system: + system = test_id.split('/')[0] + opts['test_id'] = test_id + + if test_id or system: + system_tests = [suites.SystemTestSuite(system, **opts)] + else: + system_tests = [] + for syst in ('cms', 'lms'): + system_tests.append(suites.SystemTestSuite(syst, **opts)) + + test_suite = suites.PythonTestSuite('python tests', subsuites=system_tests, **opts) + test_suite.run() + +ns.add_task(test_system, 'system') + + +@task('prereqs.install', help={ + "lib": "lib to test", + "test_id": "Test id", + "failed": "Run only failed tests", + "fail_fast": "Run only failed tests", + "verbosity": "Turn logging up or down", +}) +def test_lib( + lib=None, test_id=None, failed=None, fail_fast=None, + verbosity=1, + ): + """ + Run tests for common/lib/ + """ + test_id = test_id or lib + + opts = { + 'failed_only': failed, + 'fail_fast': fail_fast, + 'verbosity': verbosity, + } + + if test_id: + lib = '/'.join(test_id.split('/')[0:3]) + opts['test_id'] = test_id + lib_tests = [suites.LibTestSuite(lib, **opts)] + else: + lib_tests = [suites.LibTestSuite(d, **opts) for d in Env.LIB_TEST_DIRS] + + test_suite = suites.PythonTestSuite('python tests', subsuites=lib_tests, **opts) + test_suite.run() + +ns.add_task(test_lib, 'lib') + +@task('prereqs.install', help={ + "failed": "Run only failed tests", + "fail_fast": "Run only failed tests", + "verbosity": "Turn logging up or down", +}) +def test_python(failed=None, fail_fast=None, verbosity=1): + """ + Run all python tests + """ + opts = { + 'failed_only': failed, + 'fail_fast': fail_fast, + 'verbosity': verbosity, + } + + python_suite = suites.PythonTestSuite('Python Tests', **opts) + python_suite.run() + +ns.add_task(test_python, 'python') + +@task('prereqs.install.python') +def test_i18n(): + """ + Run all i18n tests + """ + i18n_suite = suites.I18nTestSuite('i18n') + i18n_suite.run() + +ns.add_task(test_i18n, 'i18n') + + +@task('prereqs.install', help={ + "verbosity": "Turn logging up or down" +}) +def test_all(verbosity=1): + """ + Run all tests + """ + opts = { + 'verbosity': verbosity, + } + # Subsuites to be added to the main suite + python_suite = suites.PythonTestSuite('Python Tests', **opts) + i18n_suite = suites.I18nTestSuite('i18n', **opts) + js_suite = suites.JsTestSuite('JS Tests', mode='run', with_coverage=True) + + # Main suite to be run + all_unittests_suite = suites.TestSuite('All Tests', subsuites=[i18n_suite, js_suite, python_suite]) + all_unittests_suite.run() + +ns.add_task(test_all, 'all', default=True) + +@task('prereqs.install', help={ + "compare_branch": "Branch to compare against" +}) +def coverage(compare_branch="origin/master"): + """ + Build the html, xml, and diff coverage reports + """ + for directory in Env.LIB_TEST_DIRS + ['cms', 'lms']: + report_dir = Env.REPORT_DIR / directory + + if (report_dir / '.coverage').isfile(): + # Generate the coverage.py HTML report + sh("coverage html --rcfile={dir}/.coveragerc".format(dir=directory)) + + # Generate the coverage.py XML report + sh("coverage xml -o {report_dir}/coverage.xml --rcfile={dir}/.coveragerc".format( + report_dir=report_dir, + dir=directory + )) + + # Find all coverage XML files (both Python and JavaScript) + xml_reports = [] + + for filepath in Env.REPORT_DIR.walk(): + if filepath.basename() == 'coverage.xml': + xml_reports.append(filepath) + + if not xml_reports: + err_msg = colorize( + 'red', + "No coverage info found. Run `inv test` before running `inv test.coverage`.\n" + ) + sys.stderr.write(err_msg) + else: + xml_report_str = ' '.join(xml_reports) + diff_html_path = os.path.join(Env.REPORT_DIR, 'diff_coverage_combined.html') + + # Generate the diff coverage reports (HTML and console) + + sh("diff-cover {xml_report_str}".format(xml_report_str=xml_report_str)) + + sh( + "diff-cover {xml_report_str} --compare-branch={compare_branch} " + "--html-report {diff_html_path}".format( + xml_report_str=xml_report_str, + compare_branch=compare_branch, + diff_html_path=diff_html_path, + ) + ) + + sh( + "diff-cover {xml_report_str} --compare-branch=" + "{compare_branch}".format( + xml_report_str=xml_report_str, + compare_branch=compare_branch, + ) + ) + + print("\n") + +ns.add_task(coverage, 'coverage') diff --git a/tasks/test/bok_choy.py b/tasks/test/bok_choy.py new file mode 100644 index 000000000000..8e5d9bdbcd42 --- /dev/null +++ b/tasks/test/bok_choy.py @@ -0,0 +1,252 @@ +import os +import memcache +import subprocess +from invoke import task, Collection +from invoke import run as sh +from invoke.exceptions import Failure +try: + from pygments.console import colorize +except ImportError: + colorize = lambda color, text: text + +from ..utils import chdir, singleton_process, wait_for_server +from ..utils.cmd import django_cmd +from ..utils.envs import Env + +ns = Collection('bok_choy') + +# Mongo databases that will be dropped before/after the tests run +BOK_CHOY_MONGO_DATABASE = 'test' + +# Control parallel test execution with environment variables +# Process timeout is the maximum amount of time to wait for results from a particular test case +BOK_CHOY_NUM_PARALLEL = int(os.environ.get('NUM_PARALLEL', 1)) +BOK_CHOY_TEST_TIMEOUT = float(os.environ.get('TEST_TIMEOUT', 300)) + +# Ensure that we have a directory to put logs and reports +BOK_CHOY_DIR = Env.REPO_ROOT / 'common/test/acceptance' +BOK_CHOY_TEST_DIR = BOK_CHOY_DIR / 'tests' +BOK_CHOY_LOG_DIR = Env.REPO_ROOT / 'test_root/log' +BOK_CHOY_LOG_DIR.makedirs_p() + +# Reports +BOK_CHOY_REPORT_DIR = Env.REPORT_DIR / 'bok_choy' +BOK_CHOY_XUNIT_REPORT = BOK_CHOY_REPORT_DIR / 'xunit.xml' +BOK_CHOY_COVERAGE_RC = BOK_CHOY_DIR / '.coveragerc' +BOK_CHOY_REPORT_DIR.makedirs_p() + + +# Directory that videos are served from +VIDEO_SOURCE_DIR = Env.REPO_ROOT / 'test_root/data/video' + +BOK_CHOY_SERVERS = { + 'lms': { + 'port':8003, + 'log': BOK_CHOY_LOG_DIR / 'bok_choy_lms.log' + }, + 'cms': { + 'port': 8031, + 'log': BOK_CHOY_LOG_DIR / 'bok_choy_studio.log' + } +} + +BOK_CHOY_STUBS = { + + 'xqueue': { + 'port': 8040, + 'log': BOK_CHOY_LOG_DIR/'bok_choy_xqueue.log', + 'config': 'register_submission_url=http://0.0.0.0:8041/test/register_submission' + }, + + 'ora': { + 'port': 8041, + 'log': BOK_CHOY_LOG_DIR/'bok_choy_ora.log', + 'config': '' + }, + + 'comments': { + 'port': 4567, + 'log': BOK_CHOY_LOG_DIR/'bok_choy_comments.log' + }, + + 'video': { + 'port': 8777, + 'log': BOK_CHOY_LOG_DIR/'bok_choy_video_sources.log', + 'config': "root_dir={video}".format(video=VIDEO_SOURCE_DIR) + }, + + 'youtube': { + 'port': 9080, + 'log': BOK_CHOY_LOG_DIR/'bok_choy_youtube.log' + } +} + + +# For the time being, stubs are used by both the bok-choy and lettuce acceptance tests +# For this reason, the stubs package is currently located in the Django app called "terrain" +# where other lettuce configuration is stored. +BOK_CHOY_STUB_DIR = Env.REPO_ROOT / 'common/djangoapps/terrain' + +BOK_CHOY_CACHE = memcache.Client(['localhost:11211']) + + +def start_servers(): + '''Start the servers we will run tests on''' + + for service, info in BOK_CHOY_SERVERS.items(): + address = "0.0.0.0:{}".format(info['port']) + cmd = ( + "coverage run --rcfile={rcfile} -m manage {service} " + "--settings bok_choy runserver {address} --traceback " + "--noreload".format( + rcfile=BOK_CHOY_COVERAGE_RC, service=service, address=address, + ) + ) + subprocess.Popen(cmd, shell=True) + + for service, info in BOK_CHOY_STUBS.items(): + with chdir(BOK_CHOY_STUB_DIR): + singleton_process([ + 'python', '-m', 'stubs.start', + service, info['port'], info.get('config', "") + ], logfile=info['log']) + + +def wait_for_test_servers(): + '''Wait until we get a successful response from the servers or time out''' + + for service, info in BOK_CHOY_SERVERS.items() + BOK_CHOY_STUBS.items(): + ready = wait_for_server("http://0.0.0.0", info['port']) + if not ready: + raise RuntimeError('Could not contact {service} test server'.format(service=service)) + + +@task +def check_mongo(): + if not is_mongo_running(): + raise RuntimeError('Mongo is not running locally.') + + +@task +def check_mysql(): + if not is_mysql_running(): + raise RuntimeError('Mysql is not running locally') + + +@task +def check_memcache(): + if not is_memcache_running(): + raise RuntimeError('Memcache is not running locally') + + +@task(check_mongo, check_memcache, check_mysql) +def check_services(): + pass + + +@task(check_mysql, 'prereqs.install') +def bok_choy_setup(): + sh(Env.REPO_ROOT / 'scripts/reset-test-db.sh') + sh("invoke assets.update --settings=bok_choy") + +ns.add_task(bok_choy_setup, 'setup') + + +@task(check_services, 'clean.reports') +def test_bok_choy_fast(spec=None): + clear_mongo() + BOK_CHOY_CACHE.flush_all() + sh(django_cmd('lms', 'bok_choy', 'loaddata', 'common/test/db_fixtures/*.json')) + + # Ensure the test servers are available + print(colorize('green', 'Starting test servers...')) + start_servers() + print(colorize('green', 'Waiting for servers to start...')) + wait_for_test_servers() + + try: + print(colorize('green', 'Running test suite...')) + run_bok_choy(spec) + except: + print(colorize('red', 'Tests failed!')) + finally: + print(colorize('green', 'Cleaning up databases...')) + +ns.add_task(test_bok_choy_fast, 'fast', default=True) + + +@task +def coverage(): + print(colorize('green', 'Combining coverage reports')) + sh('coverage combine --rcfile={}'.format(BOK_CHOY_COVERAGE_RC)) + + print(colorize('green', 'Generating coverage reports')) + sh("coverage html --rcfile={}".format(BOK_CHOY_COVERAGE_RC)) + sh("coverage xml --rcfile={}".format(BOK_CHOY_COVERAGE_RC)) + sh("coverage report --rcfile={}".format(BOK_CHOY_COVERAGE_RC)) + +ns.add_task(coverage, 'coverage') + + +def is_mongo_running(): + ''' + The mongo command will connect to the service, + failing with a non-zero exit code if it cannot connect. + ''' + try: + sh("mongo --eval \"print('running')\"") + except Failure: + return False + return True + + +def is_memcache_running(): + ''' + We use the memcache client to attempt to set a key + in memcache. If we cannot do so because the service is not + available, then it will return 0. + ''' + result = BOK_CHOY_CACHE.set('test', 'test') + return result != 0 + + +def is_mysql_running(): + ''' + We use the MySQL CLI client to list the available databases. + If the mysql server is not up, the command will exit with a non-zero status + ''' + try: + sh('mysql -e "SHOW DATABASES"') + except Failure: + return False + return True + + +def run_bok_choy(test_spec): + ''' + ''' + + # Default to running all tests if no test is specified + if test_spec: + test_spec = BOK_CHOY_TEST_DIR / test_spec + else: + test_spec = BOK_CHOY_TEST_DIR + + cmd = [ + "SCREENSHOT_DIR='{}'".format(BOK_CHOY_LOG_DIR), "nosetests", test_spec, + "--with-xunit", "--with-flaky", "--xunit-file={}".format(BOK_CHOY_XUNIT_REPORT), "--verbosity=2" + ] + + if BOK_CHOY_NUM_PARALLEL > 1: + cmd += ["--processes={}".format(BOK_CHOY_NUM_PARALLEL), "--process-timeout={}".format(BOK_CHOY_TEST_TIMEOUT)] + + sh(' '.join(cmd)) + + +def cleanup(): + sh(django_cmd('lms', 'bok_choy', 'flush', '--no-input')) + clear_mongo() + + +def clear_mongo(): + sh("mongo {} --eval 'db.dropDatabase()'".format(BOK_CHOY_MONGO_DATABASE)) diff --git a/tasks/test/js.py b/tasks/test/js.py new file mode 100644 index 000000000000..5bc1700e7c75 --- /dev/null +++ b/tasks/test/js.py @@ -0,0 +1,55 @@ +""" +Javascript test tasks +""" +from __future__ import print_function +import sys +from invoke import task, Collection +from tasks.utils.test.suites import JsTestSuite +from tasks.utils.test.suites.js_suite import JS_TEST_IDS +from tasks.utils.envs import Env + +__test__ = False # do not collect + +ns = Collection('js') + + +@task('prereqs.install', positional=["suite"], help={ + 'suite': "Test suite to run", + 'mode': "dev or run", + 'coverage': "Run test under coverage", +}) +def test_js(suite=None, mode="run", coverage=False): + """ + Run the JavaScript tests + """ + if not mode in ("dev", "run"): + sys.stderr.write("Invalid mode. Please choose 'dev' or 'run'.") + return + + if mode == 'run': + suite = suite or "all" + + if suite != 'all' and suite not in JS_TEST_IDS: + sys.stderr.write( + "Unknown test suite. Please choose from ({suites})\n".format( + suites=", ".join(JS_TEST_IDS.keys()) + ) + ) + return + + test_suite = JsTestSuite(suite, mode=mode, with_coverage=coverage) + test_suite.run() + +ns.add_task(test_js, 'run', default=True) + + +@task(help={ + 'suite': "Test suite to run", +}) +def test_js_dev(suite): + """ + Run the JavaScript tests in your default browsers + """ + test_js(suite=suite, mode="dev") + +ns.add_task(test_js_dev, "dev") diff --git a/tasks/utils/__init__.py b/tasks/utils/__init__.py new file mode 100644 index 000000000000..16281fe0b66d --- /dev/null +++ b/tasks/utils/__init__.py @@ -0,0 +1 @@ +from .utils import * diff --git a/tasks/utils/cmd.py b/tasks/utils/cmd.py new file mode 100644 index 000000000000..44e83eb65c45 --- /dev/null +++ b/tasks/utils/cmd.py @@ -0,0 +1,23 @@ +""" +Helper functions for constructing shell commands. +""" + +def cmd(*args): + """ + Concatenate the arguments into a space-separated shell command. + """ + return " ".join([str(arg) for arg in args]) + + +def django_cmd(sys, settings, *args): + """ + Construct a Django management command. + + `sys` is either 'lms' or 'studio'. + `settings` is the Django settings module (such as "dev" or "test") + `args` are concatenated to form the rest of the command. + """ + # Maintain backwards compatibility with manage.py, + # which calls "studio" "cms" + sys = 'cms' if sys == 'studio' else sys + return cmd("python manage.py", sys, "--settings={}".format(settings), *args) diff --git a/tasks/utils/envs.py b/tasks/utils/envs.py new file mode 100644 index 000000000000..77137d4c1288 --- /dev/null +++ b/tasks/utils/envs.py @@ -0,0 +1,78 @@ +""" +Helper functions for loading environment settings. +""" +from __future__ import print_function +import os +import sys +import json +from lazy import lazy +from path import path + + +class Env(object): + """ + Load information about the execution environment. + """ + + # Root of the git repository (edx-platform) + REPO_ROOT = path(__file__).abspath().parent.parent.parent + + # Root dir for reports + REPORT_DIR = REPO_ROOT / "reports" + + TEST_DIR = REPO_ROOT / ".testids" + + # Service variant (lms, cms, etc.) configured with an environment variable + # We use this to determine which envs.json file to load. + SERVICE_VARIANT = os.environ.get('SERVICE_VARIANT', None) + + # Directories used for common/lib/ tests + LIB_TEST_DIRS = [] + for item in (REPO_ROOT / "common/lib").listdir(): + if (REPO_ROOT / 'common/lib' / item).isdir(): + LIB_TEST_DIRS.append(path("common/lib") / item.basename()) + + @lazy + def env_tokens(self): + """ + Return a dict of environment settings. + If we couldn't find the JSON file, issue a warning and return an empty dict. + """ + + # Find the env JSON file + if self.SERVICE_VARIANT: + env_path = self.REPO_ROOT.parent / "{service}.env.json".format(service=self.SERVICE_VARIANT) + else: + env_path = path("env.json").abspath() + + # If the file does not exist, here or one level up, + # issue a warning and return an empty dict + if not env_path.isfile(): + env_path = env_path.parent.parent / env_path.basename() + if not env_path.isfile(): + print( + "Warning: could not find environment JSON file " + "at '{path}'".format(path=env_path), + file=sys.stderr, + ) + return dict() + + # Otherwise, load the file as JSON and return the resulting dict + try: + with open(env_path) as env_file: + return json.load(env_file) + + except ValueError: + print( + "Error: Could not parse JSON " + "in {path}".format(path=env_path), + file=sys.stderr, + ) + sys.exit(1) + + @lazy + def feature_flags(self): + """ + Return a dictionary of feature flags configured by the environment. + """ + return self.env_tokens.get('FEATURES', dict()) diff --git a/tasks/utils/process.py b/tasks/utils/process.py new file mode 100644 index 000000000000..e699d3303106 --- /dev/null +++ b/tasks/utils/process.py @@ -0,0 +1,69 @@ +""" +Helper functions for managing processes. +""" +from __future__ import print_function +import sys +import os +import subprocess +import signal +import psutil +from pygments.console import colorize + +def kill_process(proc): + """ + Kill the process `proc` created with `subprocess`. + """ + p1_group = psutil.Process(proc.pid) + + child_pids = p1_group.get_children(recursive=True) + + for child_pid in child_pids: + os.kill(child_pid.pid, signal.SIGKILL) + + +def run_multi_processes(cmd_list, out_log=None, err_log=None): + """ + Run each shell command in `cmd_list` in a separate process, + piping stdout to `out_log` (a path) and stderr to `err_log` (also a path). + + Terminates the processes on CTRL-C and ensures the processes are killed + if an error occurs. + """ + kwargs = {'shell': True, 'cwd': None} + pids = [] + + if out_log: + out_log_file = open(out_log, 'w') + kwargs['stdout'] = out_log_file + + if err_log: + err_log_file = open(err_log, 'w') + kwargs['stderr'] = err_log_file + + try: + for cmd in cmd_list: + pids.extend([subprocess.Popen(cmd, **kwargs)]) + + def _signal_handler(*args): + print(colorize("lightgray", "\nEnding...")) + signal.signal(signal.SIGINT, _signal_handler) + print(colorize("lightgray", "Enter CTL-C to end")) + signal.pause() + print(colorize("lightgray", "Processes stopped")) + + except Exception as err: + print(colorize("darkred", "Error running process {}".format(err), file=sys.stderr)) + + finally: + for pid in pids: + kill_process(pid) + + +def run_process(cmd, out_log=None, err_log=None): + """ + Run the shell command `cmd` in a separate process, + piping stdout to `out_log` (a path) and stderr to `err_log` (also a path). + + Terminates the process on CTRL-C or if an error occurs. + """ + return run_multi_processes([cmd], out_log=out_log, err_log=err_log) diff --git a/tasks/utils/test/__init__.py b/tasks/utils/test/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tasks/utils/test/suites/__init__.py b/tasks/utils/test/suites/__init__.py new file mode 100644 index 000000000000..3aa94bbbfd3f --- /dev/null +++ b/tasks/utils/test/suites/__init__.py @@ -0,0 +1,8 @@ +""" +TestSuite class and subclasses +""" +from .suite import TestSuite +from .nose_suite import NoseTestSuite, SystemTestSuite, LibTestSuite +from .python_suite import PythonTestSuite +from .js_suite import JsTestSuite +from .i18n_suite import I18nTestSuite diff --git a/tasks/utils/test/suites/i18n_suite.py b/tasks/utils/test/suites/i18n_suite.py new file mode 100644 index 000000000000..a83bede944d9 --- /dev/null +++ b/tasks/utils/test/suites/i18n_suite.py @@ -0,0 +1,42 @@ +""" +Classes used for defining and running i18n test suites +""" +from tasks.utils.test.suites import TestSuite +from tasks.utils.envs import Env + +__test__ = False # do not collect + + +class I18nTestSuite(TestSuite): + """ + Run tests for the internationalization library + """ + def __init__(self, *args, **kwargs): + super(I18nTestSuite, self).__init__(*args, **kwargs) + self.report_dir = Env.REPO_ROOT / "i18n" + self.xunit_report = self.report_dir / 'nosetests.xml' + + def __enter__(self): + super(I18nTestSuite, self).__enter__() + self.report_dir.makedirs_p() + + @property + def cmd(self): + pythonpath_prefix = ( + "PYTHONPATH={repo_root}/i18n:$PYTHONPATH".format( + repo_root=Env.REPO_ROOT + ) + ) + + cmd = ( + "{pythonpath_prefix} nosetests {repo_root}/i18n/tests " + "--with-xunit --xunit-file={xunit_report} " + "--verbosity={verbosity}".format( + pythonpath_prefix=pythonpath_prefix, + repo_root=Env.REPO_ROOT, + xunit_report=self.xunit_report, + verbosity=self.verbosity, + ) + ) + + return cmd diff --git a/tasks/utils/test/suites/js_suite.py b/tasks/utils/test/suites/js_suite.py new file mode 100644 index 000000000000..dad8d53d6308 --- /dev/null +++ b/tasks/utils/test/suites/js_suite.py @@ -0,0 +1,70 @@ +""" +Javascript test tasks +""" +from tasks import assets +from tasks.utils.test import utils as test_utils +from tasks.utils.test.suites import TestSuite +from tasks.utils.envs import Env + +__test__ = False # do not collect + + +JS_TEST_IDS = { + 'lms': Env.REPO_ROOT / 'lms/static/js_test.yml', + 'cms': Env.REPO_ROOT / 'cms/static/js_test.yml', + 'cms-squire': Env.REPO_ROOT / 'cms/static/js_test_squire.yml', + 'xmodule': Env.REPO_ROOT / 'common/lib/xmodule/xmodule/js/js_test.yml', + 'common': Env.REPO_ROOT / 'common/static/js_test.yml', +} + + +class JsTestSuite(TestSuite): + """ + A class for running JavaScript tests. + """ + def __init__(self, *args, **kwargs): + super(JsTestSuite, self).__init__(*args, **kwargs) + self.run_under_coverage = kwargs.get('with_coverage', True) + self.mode = kwargs.get('mode', 'run') + + try: + self.test_id = JS_TEST_IDS[self.root] + except (KeyError, ValueError): + self.test_id = " ".join(JS_TEST_IDS.values()) + + self.root = self.root + ' javascript' + self.report_dir = Env.REPORT_DIR / 'javascript' + self.coverage_report = self.report_dir / 'coverage.xml' + self.xunit_report = self.report_dir / 'javascript_xunit.xml' + + def __enter__(self): + super(JsTestSuite, self).__enter__() + self.report_dir.makedirs_p() + test_utils.clean_test_files() + + if self.mode == 'run' and not self.run_under_coverage: + test_utils.clean_dir(self.report_dir) + + assets.compile_coffeescript("`find lms cms common -type f -name \"*.coffee\"`") + + @property + def cmd(self): + """ + Run the tests using js-test-tool. See js-test-tool docs for + description of different command line arguments. + """ + cmd = ( + "js-test-tool {mode} {test_id} --use-firefox --timeout-sec " + "600 --xunit-report {xunit_report}".format( + mode=self.mode, + test_id=self.test_id, + xunit_report=self.xunit_report, + ) + ) + + if self.run_under_coverage: + cmd += " --coverage-xml {report_dir}".format( + report_dir=self.coverage_report + ) + + return cmd diff --git a/tasks/utils/test/suites/nose_suite.py b/tasks/utils/test/suites/nose_suite.py new file mode 100644 index 000000000000..d9bbbb3139e1 --- /dev/null +++ b/tasks/utils/test/suites/nose_suite.py @@ -0,0 +1,165 @@ +""" +Classes used for defining and running nose test suites +""" +import os +from tasks.assets import update as update_assets +from tasks.utils.test import utils as test_utils +from tasks.utils.test.suites import TestSuite +from tasks.utils.envs import Env + +__test__ = False # do not collect + + +class NoseTestSuite(TestSuite): + """ + A subclass of TestSuite with extra methods that are specific + to nose tests + """ + def __init__(self, *args, **kwargs): + super(NoseTestSuite, self).__init__(*args, **kwargs) + self.failed_only = kwargs.get('failed_only', False) + self.fail_fast = kwargs.get('fail_fast', False) + self.run_under_coverage = kwargs.get('with_coverage', True) + self.report_dir = Env.REPORT_DIR / self.root + self.test_id_dir = Env.TEST_DIR / self.root + self.test_ids = self.test_id_dir / 'noseids' + + def __enter__(self): + super(NoseTestSuite, self).__enter__() + self.report_dir.makedirs_p() + self.test_id_dir.makedirs_p() + + def __exit__(self, exc_type, exc_value, traceback): + """ + Cleans mongo afer the tests run. + """ + super(NoseTestSuite, self).__exit__(exc_type, exc_value, traceback) + test_utils.clean_mongo() + + def _under_coverage_cmd(self, cmd): + """ + If self.run_under_coverage is True, it returns the arg 'cmd' + altered to be run under coverage. It returns the command + unaltered otherwise. + """ + if self.run_under_coverage: + cmd0, cmd_rest = cmd.split(" ", 1) + # We use "python -m coverage" so that the proper python + # will run the importable coverage rather than the + # coverage that OS path finds. + + cmd = ( + "python -m coverage run --rcfile={root}/.coveragerc " + "`which {cmd0}` {cmd_rest}".format( + root=self.root, + cmd0=cmd0, + cmd_rest=cmd_rest, + ) + ) + + return cmd + + @property + def test_options_flags(self): + """ + Takes the test options and returns the appropriate flags + for the command. + """ + opts = " " + + # Handle "--failed" as a special case: we want to re-run only + # the tests that failed within our Django apps + # This sets the --failed flag for the nosetests command, so this + # functionality is the same as described in the nose documentation + if self.failed_only: + opts += "--failed" + + # This makes it so we use nose's fail-fast feature in two cases. + # Case 1: --fail_fast is passed as an arg in the paver command + # Case 2: The environment variable TESTS_FAIL_FAST is set as True + env_fail_fast_set = ( + 'TESTS_FAIL_FAST' in os.environ and os.environ['TEST_FAIL_FAST'] + ) + + if self.fail_fast or env_fail_fast_set: + opts += " --stop" + + return opts + + +class SystemTestSuite(NoseTestSuite): + """ + TestSuite for lms and cms nosetests + """ + def __init__(self, *args, **kwargs): + super(SystemTestSuite, self).__init__(*args, **kwargs) + self.test_id = kwargs.get('test_id', self._default_test_id) + self.fasttest = kwargs.get('fasttest', False) + + def __enter__(self): + super(SystemTestSuite, self).__enter__() + update_assets(system=self.root, settings="test", skip_collect=self.fasttest) + + @property + def cmd(self): + cmd = ( + './manage.py {system} test --verbosity={verbosity} ' + '{test_id} {test_opts} --traceback --settings=test'.format( + system=self.root, + verbosity=self.verbosity, + test_id=self.test_id, + test_opts=self.test_options_flags, + ) + ) + + return self._under_coverage_cmd(cmd) + + @property + def _default_test_id(self): + """ + If no test id is provided, we need to limit the test runner + to the Djangoapps we want to test. Otherwise, it will + run tests on all installed packages. We do this by + using a default test id. + """ + # We need to use $DIR/*, rather than just $DIR so that + # django-nose will import them early in the test process, + # thereby making sure that we load any django models that are + # only defined in test files. + default_test_id = "{system}/djangoapps/* common/djangoapps/*".format( + system=self.root + ) + + if self.root in ('lms', 'cms'): + default_test_id += " {system}/lib/*".format(system=self.root) + + if self.root == 'lms': + default_test_id += " {system}/tests.py".format(system=self.root) + + return default_test_id + + +class LibTestSuite(NoseTestSuite): + """ + TestSuite for edx-platform/common/lib nosetests + """ + def __init__(self, *args, **kwargs): + super(LibTestSuite, self).__init__(*args, **kwargs) + self.test_id = kwargs.get('test_id', self.root) + self.xunit_report = self.report_dir / "nosetests.xml" + + @property + def cmd(self): + cmd = ( + "nosetests --id-file={test_ids} {test_id} {test_opts} " + "--with-xunit --xunit-file={xunit_report} " + "--verbosity={verbosity}".format( + test_ids=self.test_ids, + test_id=self.test_id, + test_opts=self.test_options_flags, + xunit_report=self.xunit_report, + verbosity=self.verbosity, + ) + ) + + return self._under_coverage_cmd(cmd) diff --git a/tasks/utils/test/suites/python_suite.py b/tasks/utils/test/suites/python_suite.py new file mode 100644 index 000000000000..7a9182e38eeb --- /dev/null +++ b/tasks/utils/test/suites/python_suite.py @@ -0,0 +1,48 @@ +""" +Classes used for defining and running python test suites +""" +from tasks.utils.test import utils as test_utils +from tasks.utils.test.suites import TestSuite, LibTestSuite, SystemTestSuite +from tasks.utils.envs import Env + +__test__ = False # do not collect + + +class PythonTestSuite(TestSuite): + """ + A subclass of TestSuite with extra setup for python tests + """ + def __init__(self, *args, **kwargs): + super(PythonTestSuite, self).__init__(*args, **kwargs) + self.fasttest = kwargs.get('fasttest', False) + self.failed_only = kwargs.get('failed_only', None) + self.fail_fast = kwargs.get('fail_fast', None) + self.subsuites = kwargs.get('subsuites', self._default_subsuites) + + def __enter__(self): + super(PythonTestSuite, self).__enter__() + if not self.fasttest: + test_utils.clean_test_files() + + @property + def _default_subsuites(self): + """ + The default subsuites to be run. They include lms, cms, + and all of the libraries in common/lib. + """ + opts = { + 'failed_only': self.failed_only, + 'fail_fast': self.fail_fast, + 'fasttest': self.fasttest, + } + + lib_suites = [ + LibTestSuite(d, **opts) for d in Env.LIB_TEST_DIRS + ] + + system_suites = [ + SystemTestSuite('cms', **opts), + SystemTestSuite('lms', **opts), + ] + + return system_suites + lib_suites diff --git a/tasks/utils/test/suites/suite.py b/tasks/utils/test/suites/suite.py new file mode 100644 index 000000000000..caefc693130f --- /dev/null +++ b/tasks/utils/test/suites/suite.py @@ -0,0 +1,124 @@ +""" +A class used for defining and running test suites +""" +import sys +import subprocess +from tasks.utils.process import kill_process + +try: + from pygments.console import colorize +except ImportError: + colorize = lambda color, text: text # pylint: disable-msg=invalid-name + +__test__ = False # do not collect + + +class TestSuite(object): + """ + TestSuite is a class that defines how groups of tests run. + """ + def __init__(self, *args, **kwargs): + self.root = args[0] + self.subsuites = kwargs.get('subsuites', []) + self.failed_suites = [] + self.verbosity = kwargs.get('verbosity', 1) + + def __enter__(self): + """ + This will run before the test suite is run with the run_suite_tests method. + If self.run_test is called directly, it should be run in a 'with' block to + ensure that the proper context is created. + + Specific setup tasks should be defined in each subsuite. + + i.e. Checking for and defining required directories. + """ + print("\nSetting up for {suite_name}".format(suite_name=self.root)) + self.failed_suites = [] + + def __exit__(self, exc_type, exc_value, traceback): + """ + This is run after the tests run with the run_suite_tests method finish. + Specific clean up tasks should be defined in each subsuite. + + If self.run_test is called directly, it should be run in a 'with' block + to ensure that clean up happens properly. + + i.e. Cleaning mongo after the lms tests run. + """ + print("\nCleaning up after {suite_name}".format(suite_name=self.root)) + + @property + def cmd(self): + """ + The command to run tests (as a string). For this base class there is none. + """ + return None + + def run_test(self): + """ + Runs a self.cmd in a subprocess and waits for it to finish. + It returns False if errors or failures occur. Otherwise, it + returns True. + """ + cmd = self.cmd + sys.stdout.write(cmd) + + msg = colorize( + 'green', + '\n{bar}\n Running tests for {suite_name} \n{bar}\n'.format(suite_name=self.root, bar='=' * 40), + ) + + sys.stdout.write(msg) + sys.stdout.flush() + + kwargs = {'shell': True, 'cwd': None} + process = None + + try: + process = subprocess.Popen(cmd, **kwargs) + process.communicate() + except KeyboardInterrupt: + kill_process(process) + sys.exit(1) + else: + return (process.returncode == 0) + + def run_suite_tests(self): + """ + Runs each of the suites in self.subsuites while tracking failures + """ + # Uses __enter__ and __exit__ for context + with self: + # run the tests for this class, and for all subsuites + if self.cmd: + passed = self.run_test() + if not passed: + self.failed_suites.append(self) + + for suite in self.subsuites: + suite.run_suite_tests() + if len(suite.failed_suites) > 0: + self.failed_suites.extend(suite.failed_suites) + + def report_test_results(self): + """ + Writes a list of failed_suites to sys.stderr + """ + if len(self.failed_suites) > 0: + msg = colorize('red', "\n\n{bar}\nTests failed in the following suites:\n* ".format(bar="=" * 48)) + msg += colorize('red', '\n* '.join([s.root for s in self.failed_suites]) + '\n\n') + else: + msg = colorize('green', "\n\n{bar}\nNo test failures ".format(bar="=" * 48)) + + print(msg) + + def run(self): + """ + Runs the tests in the suite while tracking and reporting failures. + """ + self.run_suite_tests() + self.report_test_results() + + if len(self.failed_suites) > 0: + sys.exit(1) diff --git a/tasks/utils/test/utils.py b/tasks/utils/test/utils.py new file mode 100644 index 000000000000..1b1ed0faeebd --- /dev/null +++ b/tasks/utils/test/utils.py @@ -0,0 +1,46 @@ +""" +Helper functions for test tasks +""" +from invoke import task +from invoke import run as sh +from tasks.utils.envs import Env + +__test__ = False # do not collect + + +@task +def clean_test_files(): + """ + Clean fixture files used by tests and .pyc files + """ + sh("git clean -fqdx test_root/logs test_root/data test_root/staticfiles test_root/uploads") + sh("find . -type f -name \"*.pyc\" -delete") + sh("rm -rf test_root/log/auto_screenshots/*") + + +def clean_dir(directory): + """ + Clean coverage files, to ensure that we don't use stale data to generate reports. + """ + # We delete the files but preserve the directory structure + # so that coverage.py has a place to put the reports. + sh('find {dir} -type f -delete'.format(dir=directory)) + + +@task +def clean_reports_dir(): + """ + Clean coverage files, to ensure that we don't use stale data to generate reports. + """ + # We delete the files but preserve the directory structure + # so that coverage.py has a place to put the reports. + reports_dir = Env.REPORT_DIR.makedirs_p() + clean_dir(reports_dir) + + +@task +def clean_mongo(): + """ + Clean mongo test databases + """ + sh("mongo {repo_root}/scripts/delete-mongo-test-dbs.js".format(repo_root=Env.REPO_ROOT)) diff --git a/tasks/utils/utils.py b/tasks/utils/utils.py new file mode 100644 index 000000000000..898e85c06e6c --- /dev/null +++ b/tasks/utils/utils.py @@ -0,0 +1,65 @@ +import re +import os +import time +import requests +import contextlib +import subprocess +from pygments.console import colorize + + +@contextlib.contextmanager +def chdir(dirname): + '''Context manager for changing directories''' + + cwd = os.getcwd() + try: + os.chdir(dirname) + yield + finally: + os.chdir(cwd) + + +def singleton_process(cmd, logfile=None): + cmd = [str(c) for c in cmd] + cmdstr = " ".join(cmd) + if not process_is_running(cmd): + if logfile: + log = open(logfile, 'a') + else: + log = None + + print("Running {command}, redirecting output to {logfile}".format( + command=cmdstr, logfile=log and log.name + )) + subprocess.Popen(cmd, stdout=log) + else: + print(colorize( + "darkblue", + "Process {} already running, skipping".format(cmdstr) + )) + + +def process_is_running(cmd): + '''Checks whether a process is running''' + + if isinstance(cmd, list): + cmd = ' '.join(str(c) for c in cmd) + + s = subprocess.Popen(['ps', '-ef'], stdout=subprocess.PIPE) + for x in s.stdout: + if re.search(cmd, x): + return True + return False + + +def wait_for_server(server, port): + for i in range(20): + attempts = 0 + try: + response = requests.head("{server}:{port}".format(**locals())) + if response.ok: + return True + except: + time.sleep(1) + + return False