moved import out of type check

.github/configs/mr-test-example.yaml

@@ -32,7 +32,11 @@
 ## include:
 ##  - dataset: ["Carbon Bot", "Sara"]
 ##    config: ["all"]
-
+##
+## Shortcuts:
+## You can use the "all" shortcut to include all available configurations or datasets.
+## You can use the "all-nlu" shortcut to include all available NLU configurations or datasets.
+## You can use the "all-core" shortcut to include all available core configurations or datasets.
 
 include:
   - dataset: ["Carbon Bot"]

.github/runner/github-runner-deployment.yaml.tmpl

@@ -22,7 +22,7 @@ spec:
     spec:
       priorityClassName: high-priority
       automountServiceAccountToken: false
-      terminationGracePeriodSeconds: 180
+      terminationGracePeriodSeconds: 720
       containers:
       - name: github-runner
         image: {{getenv "GH_RUNNER_IMAGE"}}:latest

.github/scripts/mr_generate_summary.py

@@ -6,28 +6,46 @@
 SUMMARY_FILE = os.environ["SUMMARY_FILE"]
 CONFIG = os.environ["CONFIG"]
 DATASET = os.environ["DATASET_NAME"]
+TYPE = os.environ["TYPE"]
+IS_EXTERNAL = os.environ["IS_EXTERNAL"]
+CONFIG_REPOSITORY = "training-data"
+CONFIG_REPOSITORY_BRANCH = os.environ["DATASET_REPOSITORY_BRANCH"]
 DATASET_REPOSITORY_BRANCH = os.environ["DATASET_REPOSITORY_BRANCH"]
 task_mapping = {
     "intent_report.json": "intent_classification",
     "CRFEntityExtractor_report.json": "entity_prediction",
     "DIETClassifier_report.json": "entity_prediction",
     "response_selection_report.json": "response_selection",
+    "story_report.json": "story_prediction",
 }
 
 
 def generate_json(file, task, data):
+    global IS_EXTERNAL
+    global DATASET_REPOSITORY_BRANCH
+
     if not DATASET in data:
         data = {DATASET: {CONFIG: {}}, **data}
     elif not CONFIG in data[DATASET]:
         data[DATASET] = {CONFIG: {}, **data[DATASET]}
 
+    if str(IS_EXTERNAL).lower() in ("yes", "true", "t", "1"):
+        IS_EXTERNAL = True
+        DATASET_REPOSITORY_BRANCH = os.environ["EXTERNAL_DATASET_REPOSITORY_BRANCH"]
+    else:
+        IS_EXTERNAL = False
+
     data[DATASET][CONFIG] = {
+        "external_dataset_repository": IS_EXTERNAL,
         "dataset_repository_branch": DATASET_REPOSITORY_BRANCH,
+        "config_repository": CONFIG_REPOSITORY,
+        "config_repository_branch": CONFIG_REPOSITORY_BRANCH,
         "dataset_commit": os.environ["DATASET_COMMIT"],
         "accelerator_type": os.environ["ACCELERATOR_TYPE"],
         "test_run_time": os.environ["TEST_RUN_TIME"],
         "train_run_time": os.environ["TRAIN_RUN_TIME"],
         "total_run_time": os.environ["TOTAL_RUN_TIME"],
+        "type": TYPE,
         **data[DATASET][CONFIG],
     }
 
@@ -40,7 +58,7 @@ def read_results(file):
     with open(file) as json_file:
         data = json.load(json_file)
 
-        keys = ["accuracy", "weighted avg", "macro avg", "micro avg"]
+        keys = ["accuracy", "weighted avg", "macro avg", "micro avg", "conversation_accuracy"]
         result = {key: data[key] for key in keys if key in data}
 
     return result

.github/scripts/mr_publish_results.py

@@ -5,28 +5,48 @@
 import json
 import os
 
+IS_EXTERNAL = os.environ["IS_EXTERNAL"]
+DATASET_REPOSITORY_BRANCH = os.environ["DATASET_REPOSITORY_BRANCH"]
+EXTERNAL_DATASET_REPOSITORY_BRANCH = None
+CONFIG_REPOSITORY = "training-data"
+CONFIG_REPOSITORY_BRANCH = os.environ["DATASET_REPOSITORY_BRANCH"]
+
 analytics.write_key = os.environ["SEGMENT_TOKEN"]
 
 task_mapping = {
     "intent_report.json": "Intent Classification",
     "CRFEntityExtractor_report.json": "Entity Prediction",
     "DIETClassifier_report.json": "Entity Prediction",
     "response_selection_report.json": "Response Selection",
+    "story_report.json": "Story Prediction",
 }
 
 
 def send_to_segment(context):
+    global IS_EXTERNAL
+    global DATASET_REPOSITORY_BRANCH
+
     jobID = os.environ["GITHUB_RUN_ID"]
 
     analytics.identify(
         jobID, {"name": "model-regression-tests", "created_at": datetime.datetime.now()}
     )
 
+    if str(IS_EXTERNAL).lower() in ("yes", "true", "t", "1"):
+        IS_EXTERNAL = True
+        DATASET_REPOSITORY_BRANCH = os.environ["EXTERNAL_DATASET_REPOSITORY_BRANCH"]
+    else:
+        IS_EXTERNAL = False
+
     analytics.track(
         jobID,
         "results",
         {
             "dataset": os.environ["DATASET_NAME"],
+            "dataset_repository_branch": DATASET_REPOSITORY_BRANCH,
+            "external_dataset_repository": IS_EXTERNAL,
+            "config_repository": CONFIG_REPOSITORY,
+            "config_repository_branch": CONFIG_REPOSITORY_BRANCH,
             "dataset_repository_branch": os.environ["DATASET_REPOSITORY_BRANCH"],
             "dataset_commit": os.environ["DATASET_COMMIT"],
             "workflow": os.environ["GITHUB_WORKFLOW"],
@@ -39,6 +59,7 @@ def send_to_segment(context):
             "github_run_id": os.environ["GITHUB_RUN_ID"],
             "github_sha": os.environ["GITHUB_SHA"],
             "github_event": os.environ["GITHUB_EVENT_NAME"],
+            "type": os.environ["TYPE"],
             **context,
         },
     )
@@ -48,7 +69,7 @@ def read_results(file):
     with open(file) as json_file:
         data = json.load(json_file)
 
-        keys = ["accuracy", "weighted avg", "macro avg", "micro avg"]
+        keys = ["accuracy", "weighted avg", "macro avg", "micro avg", "conversation_accuracy"]
         result = {key: data[key] for key in keys if key in data}
 
     return result

.github/templates/configuration_variables.tmpl

@@ -8,9 +8,36 @@ which contain paths to file/directory.
  */ -}}
 {{- $mapping := (datasource "mapping") -}}
 {{- $dataset := (index $mapping.datasets (getenv "DATASET_NAME")) -}}
-{{- $config := (index $mapping.configurations (getenv "CONFIG_NAME")) -}}
-
+{{- $config := $mapping.configurations -}}
+{{- if has $dataset "repository" }}
+export DATASET="{{ $dataset.repository }}"
+export IS_EXTERNAL="true"
+echo "::add-mask::{{ $dataset.repository }}"
+{{ if has $dataset "repository_branch" }}
+export EXTERNAL_DATASET_REPOSITORY_BRANCH="{{ $dataset.repository_branch }}"
+{{ else }}
+export EXTERNAL_DATASET_REPOSITORY_BRANCH="main"
+{{ end }}
+{{- else if has $dataset "path" }}
 export DATASET="{{ $dataset.path }}"
-export CONFIG="{{ $dataset.language }}/{{ $config }}"
+export IS_EXTERNAL="false"
 echo "::add-mask::{{ $dataset.path }}"
-echo "::add-mask::{{ $dataset.language }}/{{ $config }}"
+{{ end }}
+
+{{- if has $dataset "train" }}
+export TRAIN_DIR="{{ $dataset.train }}"
+{{ end }}
+{{- if has $dataset "test" }}
+export TEST_DIR="{{ $dataset.test }}"
+{{ end }}
+{{- if has $dataset "domain" }}
+export DOMAIN_FILE="{{ $dataset.domain }}"
+{{ end }}
+
+{{- if (has $config.nlu (getenv "CONFIG_NAME")) }}
+export CONFIG="{{ $dataset.language }}/nlu/{{ index $config.nlu (getenv "CONFIG_NAME") }}"
+echo "::add-mask::{{ $dataset.language }}/nlu/{{ index $config.nlu (getenv "CONFIG_NAME") }}"
+{{ else if (has $config.core (getenv "CONFIG_NAME")) }}
+export CONFIG="{{ $dataset.language }}/core/{{ index $config.core (getenv "CONFIG_NAME") }}"
+echo "::add-mask::{{ $dataset.language }}/core/{{ index $config.core (getenv "CONFIG_NAME") }}"
+{{ end -}}

.github/templates/model_regression_test_config_comment.tmpl

@@ -9,6 +9,12 @@ Comment with a help message is added if a PR doesn't contain a comment
 with a configuration for the model regression tests.
 
  */ -}}
+{{ define "check_available_configuration" -}}
+NLU
+{{- if has .dataset "domain" -}}
+, Core
+{{- end -}}
+{{- end -}}
 Hey @{{ .Env.GITHUB_ACTOR }}! :wave: To run model regression tests, comment with the `/modeltest` command and a configuration.
 
 _Tips :bulb:: The model regression test will be run on `push` events. You can re-run the tests by re-add `status:model-regression-tests` label or use a `Re-run jobs` button in Github Actions workflow._
@@ -23,12 +29,17 @@ You can copy this in your comment and customize:
 >##########
 >## Available datasets
 >##########
-{{range (coll.Keys (datasource "mapping").datasets)}}># - "{{.}}"{{"\n"}}{{ end -}}
+{{range (coll.Keys (datasource "mapping").datasets)}}># - "{{ . }}" ({{ template "check_available_configuration" (dict "dataset" (index (datasource "mapping").datasets .)) }}){{"\n"}}{{ end -}}
 >
 >##########
->## Available configurations
+>## Available NLU configurations
 >##########
-{{range (coll.Keys (datasource "mapping").configurations)}}># - "{{.}}"{{"\n"}}{{ end -}}
+{{range (coll.Keys (datasource "mapping").configurations.nlu)}}># - "{{.}}"{{"\n"}}{{ end -}}
+>
+>##########
+>## Available Core configurations
+>##########
+{{range (coll.Keys (datasource "mapping").configurations.core)}}># - "{{.}}"{{"\n"}}{{ end -}}
 >
 {{range split (file.Read ".github/configs/mr-test-example.yaml") "\n"}}>{{.}}{{"\n"}}{{ end -}}
 >```

.github/templates/model_regression_test_config_to_json.tmpl

@@ -3,6 +3,13 @@
 The template reads an issue/a PR comment and transforms a YAML code block into JSON.
 
 */ -}}
+{{ define "check_config_type" -}}
+{{- if has (datasource "mapping").configurations.nlu . -}}
+nlu
+{{- else if has (datasource "mapping").configurations.core . -}}
+core
+{{- end -}}
+{{- end -}}
 {{- $config := ((datasource "github").body | regexp.Find "```(?s)(.*)```" | regexp.ReplaceLiteral "```.*|\r" "" | yaml | toJSON | json) -}}
 {{- $dataset_branch := "main" -}}
 {{- /* if a branch name for dataset repository is not defined use the main branch */ -}}
@@ -16,18 +23,44 @@ The template reads an issue/a PR comment and transforms a YAML code block into J
 {{- /* use all available datasets if value is equal to all */ -}}
 {{- if eq (index $pair.dataset 0) "all" -}}
 {{ $dataset = (coll.Keys (datasource "mapping").datasets) }}
+{{- else if eq (index $pair.dataset 0) "all-core" -}}
+{{- range $dataset_name, $dataset_spec := (datasource "mapping").datasets -}}
+{{- if has $dataset_spec "domain" -}}
+{{ $dataset = (coll.Append $dataset_name $dataset) -}}
+{{- end -}}
+{{- end -}}
+{{- else if eq (index $pair.dataset 0) "all-nlu" -}}
+{{- range $dataset_name, $dataset_spec := (datasource "mapping").datasets -}}
+{{- if not (has $dataset_spec "domain") -}}
+{{ $dataset = (coll.Append $dataset_name $dataset) -}}
+{{- end -}}
+{{- end -}}
 {{- else -}}
 {{- $dataset = $pair.dataset -}}
 {{- end -}}
 {{- range $index_dataset, $value_dataset := $dataset -}}
 {{- range $index_config, $value_config := $pair.config -}}
 {{- /* use all available configurations if value is equal to all */ -}}
 {{- if eq $value_config "all" -}}
-{{- range $config_name, $config_file := (datasource "mapping").configurations -}}
-{{ $inc = (coll.Append (dict "dataset_branch" $dataset_branch "dataset" $value_dataset "config" $config_name | toJSON) $inc) -}}
+{{- range $config_type := (coll.Keys (datasource "mapping").configurations) -}}
+{{- range $config_name, $config_file := (index (datasource "mapping").configurations $config_type ) -}}
+{{ $inc = (coll.Append (dict "dataset_branch" $dataset_branch "dataset" $value_dataset "config" $config_name "type" $config_type | toJSON) $inc) -}}
+{{- end -}}
+{{- end -}}
+{{- else if eq $value_config "all-core" -}}
+{{- range $config_name, $config_file := (datasource "mapping").configurations.core -}}
+{{ $inc = (coll.Append (dict "dataset_branch" $dataset_branch "dataset" $value_dataset "config" $config_name "type" "core" | toJSON) $inc) -}}
+{{- end -}}
+{{- else if eq $value_config "all-nlu" -}}
+{{- range $config_name, $config_file := (datasource "mapping").configurations.nlu -}}
+{{ $inc = (coll.Append (dict "dataset_branch" $dataset_branch "dataset" $value_dataset "config" $config_name "type" "nlu" | toJSON) $inc) -}}
 {{- end -}}
 {{- else -}}
-{{ $inc = (coll.Append (dict "dataset_branch" $dataset_branch "dataset" $value_dataset "config" $value_config | toJSON) $inc) -}}
+{{- if has (datasource "mapping").configurations.nlu $value_config -}}
+{{ $inc = (coll.Append (dict "dataset_branch" $dataset_branch "dataset" $value_dataset "config" $value_config "type" "nlu" | toJSON) $inc) -}}
+{{- else if has (datasource "mapping").configurations.core $value_config -}}
+{{ $inc = (coll.Append (dict "dataset_branch" $dataset_branch "dataset" $value_dataset "config" $value_config "type" "core" | toJSON) $inc) -}}
+{{- end -}}
 {{- end -}}
 {{- end -}}
 {{- end -}}