From 4df4ab489f4b99d8275f969e0f7d33afb978664d Mon Sep 17 00:00:00 2001 From: Calen Pennington Date: Sat, 20 Jul 2013 07:45:56 -0400 Subject: [PATCH] Add a manage.py command that understands the extra edX options --- CHANGELOG.rst | 1 + README.md | 25 ++--- cms/manage.py | 16 --- common/djangoapps/external_auth/models.py | 6 +- .../management/commands/pearson_transfer.py | 2 +- common/djangoapps/student/models.py | 6 +- doc/development.md | 4 +- doc/discussion.md | 17 ++-- doc/testing.md | 2 +- .../courseware/tests/test_masquerade.py | 2 +- .../instructor/tests/test_download_csv.py | 2 +- lms/djangoapps/notes/README.md | 3 +- lms/djangoapps/open_ended_grading/tests.py | 2 +- lms/envs/README.txt | 16 --- manage.py | 97 +++++++++++++++++++ rakelib/django.rake | 6 +- rakelib/helpers.rb | 3 +- scripts/create-dev-env.sh | 11 +-- scripts/runone.py | 11 +-- 19 files changed, 138 insertions(+), 94 deletions(-) delete mode 100755 cms/manage.py delete mode 100644 lms/envs/README.txt create mode 100755 manage.py diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 468db0607c39..44e85497ad7d 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -5,6 +5,7 @@ These are notable changes in edx-platform. This is a rolling list of changes, in roughly chronological order, most recent first. Add your entries at or near the top. Include a label indicating the component affected. +Common: Add a manage.py that knows about edx-platform specific settings and projects Common: Added *experimental* support for jsinput type. diff --git a/README.md b/README.md index 2208fe1cada1..898e75ea1479 100644 --- a/README.md +++ b/README.md @@ -80,7 +80,7 @@ those files to run edX, mirroring the folder in `/opt/edx/edx-platform`). You may also want to create a super-user with: ``` -$ rake django-admin["createsuperuser"] +$ ./manage.py lms createsuperuser ``` Also note that if you register a new user through the web interface, @@ -229,23 +229,12 @@ or any other process management tool. Configuring Your Project ------------------------ -We use [`rake`](http://rake.rubyforge.org/) to execute common tasks in our -project. The `rake` tasks are defined in the `rakefile`, or you can run `rake -T` -to view a summary. - Before you run your project, you need to create a sqlite database, create -tables in that database, run database migrations, and populate templates for -CMS templates. Fortunately, `rake` will do all of this for you! Just run: - - $ rake django-admin[syncdb] - $ rake django-admin[migrate] +tables in that database, and run database migrations. Fortunately, `django` +will do all of this for you -If you are running these commands using the [`zsh`](http://www.zsh.org/) shell, -zsh will assume that you are doing -[shell globbing](https://en.wikipedia.org/wiki/Glob_%28programming%29), search for -a file in your directory named `django-adminsyncdb` or `django-adminmigrate`, -and fail. To fix this, just surround the argument with quotation marks, so that -you're running `rake "django-admin[syncdb]"`. + $ ./manage.py lms syncdb --migrate + $ ./manage.py cms syncdb --migrate Run Your Project ---------------- @@ -253,6 +242,10 @@ edX has two components: Studio, the course authoring system; and the LMS (learning management system) used by students. These two systems communicate through the MongoDB database, which stores course information. +We use [`rake`](http://rake.rubyforge.org/) to execute common tasks in our +project. The `rake` tasks are defined in the `rakefile`, or you can run `rake -T` +to view a summary. + To run Studio, run: $ rake cms diff --git a/cms/manage.py b/cms/manage.py deleted file mode 100755 index 723fa59da17c..000000000000 --- a/cms/manage.py +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env python -from django.core.management import execute_manager -import imp -try: - imp.find_module('settings') # Assumed to be in the same directory. -except ImportError: - import sys - sys.stderr.write("Error: Can't find the file 'settings.py' in the directory containing %r. " - "It appears you've customized things.\nYou'll have to run django-admin.py, " - "passing it your settings module.\n" % __file__) - sys.exit(1) - -import settings - -if __name__ == "__main__": - execute_manager(settings) diff --git a/common/djangoapps/external_auth/models.py b/common/djangoapps/external_auth/models.py index 6c2f38d8b31c..b40e4274343b 100644 --- a/common/djangoapps/external_auth/models.py +++ b/common/djangoapps/external_auth/models.py @@ -4,9 +4,9 @@ If you make changes to this model, be sure to create an appropriate migration file and check it in at the same time as your model changes. To do that, -1. Go to the mitx dir -2. django-admin.py schemamigration student --auto --settings=lms.envs.dev --pythonpath=. description_of_your_change -3. Add the migration file created in mitx/common/djangoapps/external_auth/migrations/ +1. Go to the edx-platform dir +2. ./manage.py lms schemamigration student --auto description_of_your_change +3. Add the migration file created in edx-platform/common/djangoapps/external_auth/migrations/ """ from django.db import models diff --git a/common/djangoapps/student/management/commands/pearson_transfer.py b/common/djangoapps/student/management/commands/pearson_transfer.py index 75716c744370..39e1e7468e73 100644 --- a/common/djangoapps/student/management/commands/pearson_transfer.py +++ b/common/djangoapps/student/management/commands/pearson_transfer.py @@ -20,7 +20,7 @@ class Command(BaseCommand): files and then uploads over SFTP to Pearson and stuffs the entry in an S3 bucket for archive purposes. - Usage: django-admin.py pearson-transfer --mode [import|export|both] + Usage: ./manage.py pearson-transfer --mode [import|export|both] """ option_list = BaseCommand.option_list + ( diff --git a/common/djangoapps/student/models.py b/common/djangoapps/student/models.py index af93c343173c..8c348f4cad6f 100644 --- a/common/djangoapps/student/models.py +++ b/common/djangoapps/student/models.py @@ -6,9 +6,9 @@ If you make changes to this model, be sure to create an appropriate migration file and check it in at the same time as your model changes. To do that, -1. Go to the mitx dir -2. django-admin.py schemamigration student --auto --settings=lms.envs.dev --pythonpath=. description_of_your_change -3. Add the migration file created in mitx/common/djangoapps/student/migrations/ +1. Go to the edx-platform dir +2. ./manage.py lms schemamigration student --auto description_of_your_change +3. Add the migration file created in edx-platform/common/djangoapps/student/migrations/ """ from datetime import datetime import hashlib diff --git a/doc/development.md b/doc/development.md index e6ab650002dd..5b0e606adbe9 100644 --- a/doc/development.md +++ b/doc/development.md @@ -48,14 +48,14 @@ Both the LMS and Studio can be started using the following shortcut tasks rake lms[cms.dev] # Start LMS to run alongside Studio rake lms[cms.dev_preview] # Start LMS to run alongside Studio in preview mode -Under the hood, this executes `django-admin.py runserver --pythonpath=$WORKING_DIRECTORY --settings=lms.envs.dev`, +Under the hood, this executes `./manage.py {lms|cms} --settings $ENV runserver`, which starts a local development server. Both of these commands take arguments to start the servers in different environments or with additional options: # Start the LMS using the test configuration, on port 5000 - rake lms[test,5000] # Executes django-admin.py runserver --pythonpath=$WORKING_DIRECTORY --setings=lms.envs.test 5000 + rake lms[test,5000] # Executes ./manage.py lms --settings test runserver 5000 *N.B.* You may have to escape the `[` characters, depending on your shell: `rake "lms[test,5000]"` diff --git a/doc/discussion.md b/doc/discussion.md index 752dc6a5e7da..12c139c62d78 100644 --- a/doc/discussion.md +++ b/doc/discussion.md @@ -68,24 +68,19 @@ First make sure that the database is up-to-date: If you have created users in the edx-platform django apps when the comment service was not running, you will need to one-way sync the users into the comment service back end database: - rake django-admin[sync_user_info] - -For convenience, add the following environment variables to the terminal (assuming that you're using configuration set lms.envs.dev): - - export DJANGO_SETTINGS_MODULE=lms.envs.dev - export PYTHONPATH=. + ./manage.py lms sync_user_info Now initialize roles and permissions, providing a course id. See the example below. Note that you do not need to do this for Studio-created courses, as the Studio application does this for you. - django-admin.py seed_permissions_roles "MITx/6.002x/2012_Fall" + ./manage.py lms seed_permissions_roles "MITx/6.002x/2012_Fall" To assign yourself as a moderator, use the following command (assuming your username is "test", and the course id is "MITx/6.002x/2012_Fall"): - django-admin.py assign_role test Moderator "MITx/6.002x/2012_Fall" + ./manage.py lms assign_role test Moderator "MITx/6.002x/2012_Fall" To assign yourself as an administrator, use the following command - django-admin.py assign_role test Administrator "MITx/6.002x/2012_Fall" + ./manage.py lms assign_role test Administrator "MITx/6.002x/2012_Fall" ## Some other useful commands @@ -155,8 +150,8 @@ You can use the following command to launch a console within the service environ Use the following command to see the roles and permissions of a user in a given course (assuming, again, that the username is "test"): - django-admin.py show_permissions moderator + ./manage.py lms show_permissions moderator You need to make sure that the environment variables are exported. Otherwise you would need to do - django-admin.py show_permissions moderator --settings=lms.envs.dev --pythonpath=. + ./manage.py lms show_permissions moderator diff --git a/doc/testing.md b/doc/testing.md index 96f0dcfd9881..cfca297f2b5b 100644 --- a/doc/testing.md +++ b/doc/testing.md @@ -233,7 +233,7 @@ generates HTML and XML (Cobertura format) reports. When testing problems that use a queue server on AWS (e.g. sandbox-xqueue.edx.org), you'll need to run your server on your public IP, like so. -`django-admin.py runserver --settings=lms.envs.dev --pythonpath=. 0.0.0.0:8000` +`./manage.py lms runserver 0.0.0.0:8000` When you connect to the LMS, you need to use the public ip. Use `ifconfig` to figure out the number, and connect e.g. to `http://18.3.4.5:8000/` diff --git a/lms/djangoapps/courseware/tests/test_masquerade.py b/lms/djangoapps/courseware/tests/test_masquerade.py index 3dc3d2b6b1b6..0fc4eae242a2 100644 --- a/lms/djangoapps/courseware/tests/test_masquerade.py +++ b/lms/djangoapps/courseware/tests/test_masquerade.py @@ -5,7 +5,7 @@ Notes for running by hand: -django-admin.py test --settings=lms.envs.test --pythonpath=. lms/djangoapps/courseware +./manage.py lms --settings test test lms/djangoapps/courseware """ from django.test.utils import override_settings diff --git a/lms/djangoapps/instructor/tests/test_download_csv.py b/lms/djangoapps/instructor/tests/test_download_csv.py index fd5bd562badd..5a934d7a2319 100644 --- a/lms/djangoapps/instructor/tests/test_download_csv.py +++ b/lms/djangoapps/instructor/tests/test_download_csv.py @@ -5,7 +5,7 @@ Notes for running by hand: -django-admin.py test --settings=lms.envs.test --pythonpath=. lms/djangoapps/instructor +./manage.py lms --settings test test lms/djangoapps/instructor """ from django.test.utils import override_settings diff --git a/lms/djangoapps/notes/README.md b/lms/djangoapps/notes/README.md index 2e81fa5ec168..233575ddacf9 100644 --- a/lms/djangoapps/notes/README.md +++ b/lms/djangoapps/notes/README.md @@ -30,8 +30,7 @@ Developer Overview ### Quickstart ``` -$ rake django-admin[syncdb] -$ rake django-admin[migrate] +$ ./manage.py lms syncdb --migrate ``` Then follow the steps above to enable the *My Notes* tab or manually add a tab to the policy tab configuration with ```{"type": "notes", "name": "My Notes"}```. diff --git a/lms/djangoapps/open_ended_grading/tests.py b/lms/djangoapps/open_ended_grading/tests.py index b2529e38db16..556a42180813 100644 --- a/lms/djangoapps/open_ended_grading/tests.py +++ b/lms/djangoapps/open_ended_grading/tests.py @@ -1,7 +1,7 @@ """ Tests for open ended grading interfaces -django-admin.py test --settings=lms.envs.test --pythonpath=. lms/djangoapps/open_ended_grading +./manage.py lms --settings test test lms/djangoapps/open_ended_grading """ import json diff --git a/lms/envs/README.txt b/lms/envs/README.txt deleted file mode 100644 index 6d1512e6f6f8..000000000000 --- a/lms/envs/README.txt +++ /dev/null @@ -1,16 +0,0 @@ -Transitional for moving to new settings scheme. - -To use: - rake lms - or - django-admin.py runserver --settings=lms.envs.dev --pythonpath=. - -NOTE: Using manage.py will automatically run mitx/settings.py first, regardless -of what you send it for an explicit --settings flag. It still works, but might -have odd side effects. Using django-admin.py avoids that problem. -django-admin.py is installed by default when you install Django. - -To use with gunicorn_django in debug mode: - - gunicorn_django lms/envs/dev.py - diff --git a/manage.py b/manage.py new file mode 100755 index 000000000000..cde8f27e3816 --- /dev/null +++ b/manage.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python +""" +Usage: manage.py {lms|cms} [--settings env] ... + +Run django management commands. Because edx-platform contains multiple django projects, +the first argument specifies which project to run (cms [Studio] or lms [Learning Management System]). + +By default, those systems run in with a settings file appropriate for development. However, +by passing the --settings flag, you can specify what environment specific settings file to use. + +Any arguments not understood by this manage.py will be passed to django-admin.py +""" + +import os +import sys +import glob2 +import imp +from argparse import ArgumentParser + +def parse_args(): + """Parse edx specific arguments to manage.py""" + parser = ArgumentParser() + subparsers = parser.add_subparsers(title='system', description='edx service to run') + + lms = subparsers.add_parser( + 'lms', + help='Learning Management System', + add_help=False, + usage='%(prog)s [options] ...' + ) + lms.add_argument('-h', '--help', action='store_true', help='show this help message and exit') + lms.add_argument( + '--settings', + help="Which django settings module to use. Will search for a submodule of lms.envs first, and " + "if that module doesn't exist, will search in sys.path. If not provided, the DJANGO_SETTINGS_MODULE " + "environment variable will be used if it is set, otherwise will default to lms.envs.dev") + lms.add_argument( + '-s', '--service-variant', + choices=['lms', 'lms-xml', 'lms-preview'], + default='lms', + help='Which service variant to run, when using the aws environment') + lms.set_defaults( + help_string=lms.format_help(), + settings_base='lms/envs', + default_settings='lms.envs.dev' + ) + + cms = subparsers.add_parser( + 'cms', + help='Studio', + add_help=False, + usage='%(prog)s [options] ...' + ) + cms.add_argument( + '--settings', + help="Which django settings module to use. Will search for a submodule of cms.envs first, and " + "if that module doesn't exist, will search in sys.path. If not provided, the DJANGO_SETTINGS_MODULE " + "environment variable will be used if it is set, otherwise will default to cms.envs.dev") + cms.add_argument('-h', '--help', action='store_true', help='show this help message and exit') + cms.set_defaults( + help_string=cms.format_help(), + settings_base='cms/envs', + default_settings='cms.envs.dev', + service_variant='cms' + ) + + + edx_args, django_args = parser.parse_known_args() + + if edx_args.help: + print "edX:" + print edx_args.help_string + + return edx_args, django_args + + +if __name__ == "__main__": + edx_args, django_args = parse_args() + + if edx_args.settings: + try: + imp.find_module(edx_args.settings, [edx_args.settings_base]) + os.environ["DJANGO_SETTINGS_MODULE"] = edx_args.settings_base.replace('/', '.') + "." + edx_args.settings + except ImportError: + imp.find_module(edx_args.settings) + os.environ["DJANGO_SETTINGS_MODULE"] = edx_args.settings + else: + os.environ.setdefault("DJANGO_SETTINGS_MODULE", edx_args.default_settings) + os.environ.setdefault("SERVICE_VARIANT", edx_args.service_variant) + if edx_args.help: + print "Django:" + # This will trigger django-admin.py to print out its help + django_args.insert(0, '--help') + + from django.core.management import execute_from_command_line + + execute_from_command_line([sys.argv[0]] + django_args) \ No newline at end of file diff --git a/rakelib/django.rake b/rakelib/django.rake index 53c299ac9f34..40005e19e3cc 100644 --- a/rakelib/django.rake +++ b/rakelib/django.rake @@ -11,8 +11,7 @@ end task :fastlms do # this is >2 times faster that rake [lms], and does not need web, good for local dev - django_admin = ENV['DJANGO_ADMIN_PATH'] || select_executable('django-admin.py', 'django-admin') - sh("#{django_admin} runserver --traceback --settings=lms.envs.dev --pythonpath=.") + sh("./manage.py lms runserver --traceback") end # Start :system locally with the specified :env and :options. @@ -36,9 +35,8 @@ end desc "Start #{system} Celery worker" task "#{system}_worker", [:options] => [:predjango] do |t, args| args.with_defaults(:options => default_options[system]) - django_admin = ENV['DJANGO_ADMIN_PATH'] || select_executable('django-admin.py', 'django-admin') command = 'celery worker' - sh("#{django_admin} #{command} --loglevel=INFO --settings=#{system}.envs.dev_with_worker --pythonpath=. #{args.join(' ')}") + sh("./manage.py #{system} --settings dev_with_worker #{command} --loglevel=INFO #{args.join(' ')}") end # Per environment tasks diff --git a/rakelib/helpers.rb b/rakelib/helpers.rb index 925b7e1b2856..6826570a1800 100644 --- a/rakelib/helpers.rb +++ b/rakelib/helpers.rb @@ -12,8 +12,7 @@ def select_executable(*cmds) end def django_admin(system, env, command, *args) - django_admin = ENV['DJANGO_ADMIN_PATH'] || select_executable('django-admin.py', 'django-admin') - return "#{django_admin} #{command} --traceback --settings=#{system}.envs.#{env} --pythonpath=. #{args.join(' ')}" + return "./manage.py #{system} --settings #{env} #{command} --traceback #{args.join(' ')}" end def report_dir_path(dir) diff --git a/scripts/create-dev-env.sh b/scripts/create-dev-env.sh index 8699cc76562b..25e1b7520b4d 100755 --- a/scripts/create-dev-env.sh +++ b/scripts/create-dev-env.sh @@ -495,8 +495,8 @@ mkdir -p "$BASE/log" mkdir -p "$BASE/db" mkdir -p "$BASE/data" -rake django-admin[syncdb,lms,dev,--noinput] -rake django-admin[migrate] +./manage.py lms syncdb --noinput --migrate +./manage.py cms syncdb --noinput --migrate # Configure Git @@ -520,18 +520,13 @@ if [[ ! $quiet ]]; then $ workon mitx - To initialize Django - - $ rake django-admin[syncdb] - $ rake django-admin[migrate] - To start the Django on port 8000 $ rake lms Or to start Django on a different - $ rake django-admin[runserver,lms,dev,] + $ ./manage.py lms runserver If the Django development server starts properly you should see: diff --git a/scripts/runone.py b/scripts/runone.py index a644aa077bd0..b403b09ff957 100755 --- a/scripts/runone.py +++ b/scripts/runone.py @@ -41,18 +41,17 @@ def main(argv): test_py_path = find_full_path(test_py_path) test_spec = "%s:%s.%s" % (test_py_path, test_class, test_method) - settings = None + system = None if test_py_path.startswith('cms'): - settings = 'cms.envs.test' + system = 'cms' elif test_py_path.startswith('lms'): - settings = 'lms.envs.test' + system = 'lms' - if settings: + if system: # Run as a django test suite from django.core import management - django_args = ["django-admin.py", "test", "--pythonpath=."] - django_args.append("--settings=%s" % settings) + django_args = ["./manage.py", system, "--settings", "test", "test"] if args.nocapture: django_args.append("-s") django_args.append(test_spec)