analysis.yml

---
  # First Run:                    ansible-playbook analysis.yml -i inventory/analysis -e "exp=demo run=new"
  # Continue Run with Run Id:     ansible-playbook analysis.yml -i inventory/analysis -e "exp=demo run=1611332286"

- name: Playbook 
  hosts: leonhard
  vars:
    n_lsf_monitor_tries: 1
  tasks:
  - name: Load an Experiment Run (init a new run if it does not exist yet)
    include_role:
      name: experiment-state
    vars:
      expstate: load

  - name: Setup experiments environment
    include_role:
      name: analysis-setup-exp
    when: run == 'new' # only submit if it is a new experiment

  - name: Submit the Experiments into the Job-System 
    include_role:
      name: "{{ queue | default('lsf') }}-submit-job"
    when: run == 'new' # only submit if it is a new experiment
    loop: "{{ experiments_ext }}"
    loop_control:
      loop_var: exp_config
      index_var: exp_idx

  - name: Save the updated state of the experiment run (save job ids)
    include_role:
      name: experiment-state
    vars:
      expstate: save
    when: run == 'new' # only persist the state if it is a new experiment

  - name: Monitor the unfinished jobs. (loop up to n_lsf_monitor_tries times (the role has a delay in between), then stop)
    include_role:
      name: "{{ queue | default('lsf') }}-monitor-jobs"
    when: (lsf_job_ids_unfinished | length > 0) or (force_refresh | default(false)) # there are some unfinished jobs left
    loop: "{{ range(0, n_lsf_monitor_tries, 1)|list }}"

  - name: Output the run id of this experiment for convenience
    debug:
      var: run_id
    when: run == 'new'