revise submit

liuzh47 · Aug 18, 2020 · 1730ff7 · 1730ff7
1 parent de0b4c9
commit 1730ff7
Show file tree

Hide file tree

Showing 2 changed files with 14 additions and 4 deletions.
diff --git a/run_batch_squad.sh b/run_batch_squad.sh
@@ -5,5 +5,5 @@ python3 tools/batch/submit-job.py \
     --name test_squad_electra_base \
     --work-dir scripts/question_answering \
     --remote https://github.com/zheyuye/gluon-nlp/ \
-    --command 'commands/run_squad2_electra_base.sh | tee stdout.log' \
+    --command 'bash commands/run_squad2_electra_base.sh | tee stdout.log' \
     --wait
diff --git a/tools/batch/submit-job.py b/tools/batch/submit-job.py
@@ -97,15 +97,25 @@ def nowInMillis():
     'c5n.18x': 'gluon-nlp-1-cpu-jobs:2',
 }
 
+job_queues = {
+    'g4dn.4x': 'g4dn',
+    'g4dn.8x': 'g4dn',
+    'g4dn.12x': 'g4dn-multi-gpu',
+    'g4dn.16x': 'g4dn',
+    'p3.2x': 'p3',
+    'p3.8x': 'p3-4gpu',
+    'p3.16x': 'p3-8gpu',
+    'p3dn.24x': 'p3dn-8gpu',
+    'c5n.18x': 'c5n',
+}
+
 def main():
     spin = ['-', '/', '|', '\\', '-', '/', '|', '\\']
     logGroupName = '/aws/batch/job'
 
     jobName = re.sub('[^A-Za-z0-9_\-]', '', args.name)[:128]  # Enforce AWS Batch jobName rules
     jobType = args.job_type
-    jobQueue = jobType.split('.')[0]
-    if jobQueue == 'p3dn':
-        jobQueue = 'p3'
+    jobQueue = job_queues[jobType]
     jobDefinition = job_definitions[jobType]
     command = args.command.split()
     wait = args.wait