Merge pull request #3 from eflows4hpc/updates

Update with new dls pipelines

ansible/playbooks/submit_dag_git2ssh_data.yaml

@@ -0,0 +1,73 @@
+---
+
+- name: Set host from env
+  set_fact:
+    host: "{{ host_from_env }}"
+  when:
+    - host_from_env is defined
+    - host_from_env | length > 0
+
+- name: Set git_url
+  set_fact:
+    git_url: "{{ git_url_input }}"
+  when:
+    - git_url_input is defined
+    - git_url_input | length > 0
+
+- name: Set git_repo
+  set_fact:
+    git_repo: "{{ git_repo_input }}"
+  when:
+    - git_repo_input is defined
+    - git_repo_input | length > 0
+
+- name: Set target_path
+  set_fact:
+    target_path: "{{ target_path_input }}"
+  when:
+    - target_path_input is defined
+    - target_path_input | length > 0
+
+- name: Set login
+  set_fact:
+    login: "{{ login_input }}"
+  when:
+    - login_input is defined
+    - login_input | length > 0
+
+- name: Set vault_id
+  set_fact:
+    vault_id: "{{ vault_id_input }}"
+  when:
+    - vault_id_input is defined
+    - vault_id_input | length > 0
+
+- name: Set target conf
+  set_fact:
+    target_conf:
+      gitlab_url: "{{ git_url }}"
+      gitlab_repo: "{{ git_repo }}"
+      target: "{{ target_path }}"
+      host: "{{ host }}"
+      login: "{{ login }}"
+      vault_id: "{{ vault_id }}"
+  no_log: "{{ debug_logs | default('false') == 'false' }}"
+
+
+- name: add extra_conf
+  set_fact:
+    target_conf: "{{ conf | from_json | combine(target_conf) }}"
+  when:
+    - conf is defined
+    - conf | length > 0
+  no_log: "{{ debug_logs | default('false') == 'false' }}"
+
+- name: Set request body
+  set_fact:
+    request_body:
+      dag_run_id: "{{ login }}-{{ 1000000000 | random | to_uuid }}"
+      conf: "{{ target_conf }}"
+  no_log: "{{ debug_logs | default('false') == 'false' }}"
+
+- name: Import Submit DAG RUN to Airflow
+  include_tasks: submit_tasks.yaml

ansible/playbooks/submit_dag_http2ssh_data.yaml

@@ -42,7 +42,7 @@
       force: "{{ force | bool }}"
       target: "{{ target_path }}"
       host: "{{ host }}"
-      port: "{{ port }}"
+      port: "{{ port | int }}"
       login: "{{ login }}"
       vault_id: "{{ vault_id }}"
   no_log: "{{ debug_logs | default('false') == 'false' }}"

ansible/playbooks/submit_dag_mlflow_upload.yaml

@@ -0,0 +1,63 @@
+---
+- name: Set host from env
+  set_fact:
+    host: "{{ host_from_env }}"
+  when:
+    - host_from_env is defined
+    - host_from_env | length > 0
+
+- name: Set location
+  set_fact:
+    location: "{{ location_input }}"
+  when:
+    - location_input is defined
+    - location_input | length > 0
+
+- name: Set location with subfolder
+  set_fact:
+    location: "{{ location_input }}/{{ subfolder}}"
+  when:
+    - subfolder is defined
+    - subfolder | length > 0
+
+- name: Set login
+  set_fact:
+    login: "{{ login_input }}"
+  when:
+    - login_input is defined
+    - login_input | length > 0
+
+- name: Set vault_id
+  set_fact:
+    vault_id: "{{ vault_id_input }}"
+  when:
+    - vault_id_input is defined
+    - vault_id_input | length > 0
+
+- name: Set target conf
+  set_fact:
+    target_conf:
+      location: "{{ location }}"
+      host: "{{ host }}"
+      login: "{{ login }}"
+      vault_id: "{{ vault_id }}"
+  no_log: "{{ debug_logs | default('false') == 'false' }}"
+
+
+- name: add extra_conf
+  set_fact:
+    target_conf: "{{ conf | from_json | combine(target_conf) }}"
+  when:
+    - conf is defined
+    - conf | length > 0
+  no_log: "{{ debug_logs | default('false') == 'false' }}"
+
+- name: Set request body
+  set_fact:
+    request_body:
+      dag_run_id: "{{ login }}-{{ 1000000000 | random | to_uuid }}"
+      conf: "{{ target_conf }}"
+  no_log: "{{ debug_logs | default('false') == 'false' }}"
+
+- name: Import Submit DAG RUN to Airflow
+  include_tasks: submit_tasks.yaml

ansible/playbooks/submit_dag_model_search_upload.yaml

@@ -0,0 +1,64 @@
+---
+- name: Set host from env
+  set_fact:
+    host: "{{ host_from_env }}"
+  when:
+    - host_from_env is defined
+    - host_from_env | length > 0
+
+- name: Set location
+  set_fact:
+    location: "{{ location_input }}"
+  when:
+    - location_input is defined
+    - location_input | length > 0
+
+- name: Set experiment
+  set_fact:
+    experiment: "{{ experiment_input }}"
+  when:
+    - experiment_input is defined
+    - experiment_input | length > 0
+
+- name: Set login
+  set_fact:
+    login: "{{ login_input }}"
+  when:
+    - login_input is defined
+    - login_input | length > 0
+
+- name: Set vault_id
+  set_fact:
+    vault_id: "{{ vault_id_input }}"
+  when:
+    - vault_id_input is defined
+    - vault_id_input | length > 0
+
+- name: Set target conf
+  set_fact:
+    target_conf:
+      location: "{{ location }}"
+      experiment_name: "{{ experiment }}"
+      host: "{{ host }}"
+      login: "{{ login }}"
+      vault_id: "{{ vault_id }}"
+  no_log: "{{ debug_logs | default('false') == 'false' }}"
+
+
+- name: add extra_conf
+  set_fact:
+    target_conf: "{{ conf | from_json | combine(target_conf) }}"
+  when:
+    - conf is defined
+    - conf | length > 0
+  no_log: "{{ debug_logs | default('false') == 'false' }}"
+
+- name: Set request body
+  set_fact:
+    request_body:
+      dag_run_id: "{{ login }}-{{ 1000000000 | random | to_uuid }}"
+      conf: "{{ target_conf }}"
+  no_log: "{{ debug_logs | default('false') == 'false' }}"
+
+- name: Import Submit DAG RUN to Airflow
+  include_tasks: submit_tasks.yaml

ansible/playbooks/submit_dag_model_stagein.yaml

@@ -0,0 +1,72 @@
+---
+- name: Set host from env
+  set_fact:
+    host: "{{ host_from_env }}"
+  when:
+    - host_from_env is defined
+    - host_from_env | length > 0
+
+- name: Set location
+  set_fact:
+    location: "{{ location_input }}"
+  when:
+    - location_input is defined
+    - location_input | length > 0
+
+- name: Set experiment
+  set_fact:
+    experiment: "{{ experiment_input }}"
+  when:
+    - experiment_input is defined
+    - experiment_input | length > 0
+
+- name: Set model_path
+  set_fact:
+    model_path: "{{ model_path_input }}"
+  when:
+    - model_path_input is defined
+    - model_path_input | length > 0
+
+- name: Set login
+  set_fact:
+    login: "{{ login_input }}"
+  when:
+    - login_input is defined
+    - login_input | length > 0
+
+- name: Set vault_id
+  set_fact:
+    vault_id: "{{ vault_id_input }}"
+  when:
+    - vault_id_input is defined
+    - vault_id_input | length > 0
+
+- name: Set target conf
+  set_fact:
+    target_conf:
+      location: "{{ location }}"
+      mlflow_runid: "{{ experiment }}"
+      mlflow_modelpath: "{{ model_path }}"
+      host: "{{ host }}"
+      login: "{{ login }}"
+      vault_id: "{{ vault_id }}"
+  no_log: "{{ debug_logs | default('false') == 'false' }}"
+
+
+- name: add extra_conf
+  set_fact:
+    target_conf: "{{ conf | from_json | combine(target_conf) }}"
+  when:
+    - conf is defined
+    - conf | length > 0
+  no_log: "{{ debug_logs | default('false') == 'false' }}"
+
+- name: Set request body
+  set_fact:
+    request_body:
+      dag_run_id: "{{ login }}-{{ 1000000000 | random | to_uuid }}"
+      conf: "{{ target_conf }}"
+  no_log: "{{ debug_logs | default('false') == 'false' }}"
+
+- name: Import Submit DAG RUN to Airflow
+  include_tasks: submit_tasks.yaml

ansible/playbooks/submit_dag_ssh2ssh_data.yaml

@@ -0,0 +1,103 @@
+---
+
+- name: Set source_host from env
+  set_fact:
+    source_host: "{{ source_host_from_env }}"
+  when:
+    - source_host_from_env is defined
+    - source_host_from_env | length > 0
+
+- name: Set targer_host from env
+  set_fact:
+    target_host: "{{ target_host_from_env }}"
+  when:
+    - target_host_from_env is defined
+    - target_host_from_env | length > 0
+
+- name: Set source_path
+  set_fact:
+    source_path: "{{ source_path_input }}"
+  when:
+    - source_path_input is defined
+    - source_path_input | length > 0
+
+- name: Set target_path
+  set_fact:
+    target_path: "{{ target_path_input }}"
+  when:
+    - target_path_input is defined
+    - target_path_input | length > 0
+
+- name: Set source login
+  set_fact:
+    source_login: "{{ source_login_input }}"
+  when:
+    - source_login_input is defined
+    - source_login_input | length > 0
+
+- name: Set target login
+  set_fact:
+    target_login: "{{ target_login_input }}"
+  when:
+    - target_login_input is defined
+    - target_login_input | length > 0
+
+- name: Set source vault_id
+  set_fact:
+    source_vault_id: "{{ source_vault_id_input }}"
+  when:
+    - source_vault_id_input is defined
+    - source_vault_id_input | length > 0
+
+- name: Set target vault_id
+  set_fact:
+    target_vault_id: "{{ target_vault_id_input }}"
+  when:
+    - target_vault_id_input is defined
+    - target_vault_id_input | length > 0
+
+- name: Set source vault_id as target vault_id when it is not defined
+  set_fact:
+    source_vault_id: "{{ target_vault_id }}"
+  when:
+    - target_vault_id is defined
+    - source_vault_id | length == 0
+
+- name: Set source login when not defined
+  set_fact:
+    source_login: "{{ target_login }}"
+  when:
+    - target_login is defined
+    - source_login | length == 0
+
+- name: Set target conf
+  set_fact:
+    target_conf:
+      source_path: "{{ source_path }}"
+      source_host: "{{ source_host }}"
+      source_login: "{{ source_login }}"
+      source_vault_id: "{{ source_vault_id }}"
+      target_path: "{{ target_path }}"
+      target_host: "{{ target_host }}"
+      target_login: "{{ target_login }}"
+      target_vault_id: "{{ target_vault_id }}"
+  no_log: "{{ debug_logs | default('false') == 'false' }}"
+
+
+- name: add extra_conf
+  set_fact:
+    target_conf: "{{ conf | from_json | combine(target_conf) }}"
+  when:
+    - conf is defined
+    - conf | length > 0
+  no_log: "{{ debug_logs | default('false') == 'false' }}"
+
+- name: Set request body
+  set_fact:
+    request_body:
+      dag_run_id: "{{ target_login }}-{{ 1000000000 | random | to_uuid }}"
+      conf: "{{ target_conf }}"
+  no_log: "{{ debug_logs | default('false') == 'false' }}"
+
+- name: Import Submit DAG RUN to Airflow
+  include_tasks: submit_tasks.yaml

ansible/playbooks/submit_dag_stagein_data.yaml

@@ -41,7 +41,6 @@
       oid: "{{ oid }}"
       target: "{{ target_path }}"
       host: "{{ host }}"
-      port: "{{ port }}"
       login: "{{ login }}"
       vault_id: "{{ vault_id }}"
   no_log: "{{ debug_logs | default('false') == 'false' }}"