Upstream merge 16 apr 25

.github/ISSUE_TEMPLATE/new_meta.yaml

@@ -37,7 +37,7 @@ body:
   - type: textarea
     attributes:
       label: Tasking
-      value: "```[tasklist]\n### Meta Tasks\n- [ ] Provide Week 1 Update Comment\n- [ ] Provide Week 2 Update or Closeout Comment\n```"
+      value: "\n### Meta Tasks\n- [ ] Provide Week 1 Update Comment\n- [ ] Provide Week 2 Update or Closeout Comment\n"
       render:
 
   - type: textarea

.github/workflows/attack-coverage-update.yml

@@ -45,7 +45,7 @@ jobs:
          git add docs-dev/"ATT\&CK-coverage.md"
 
       - name: Create Pull Request
-        uses: peter-evans/create-pull-request@v7.0.3
+        uses: peter-evans/create-pull-request@6cd32fd93684475c31847837f87bb135d40a2b79 # v7.0.3
         with:
           assignees: '${{github.actor}}'
           delete-branch: true

.github/workflows/docs-build.yml

@@ -0,0 +1,23 @@
+name: docs-build
+
+on:
+  push:
+    branches:
+      - main
+  pull_request_target: ~
+
+jobs:
+  preview:
+    uses: elastic/docs-builder/.github/workflows/preview-build.yml@main
+    with:
+      continue-on-error: false
+      strict: true
+      path-pattern: |
+        docs/**
+        rules/**
+        rules_building_block/**
+    permissions:
+      deployments: write
+      id-token: write
+      contents: read
+      pull-requests: read

.github/workflows/docs-cleanup.yml

@@ -0,0 +1,14 @@
+name: docs-cleanup
+
+on:
+  pull_request_target:
+    types:
+      - closed
+
+jobs:
+  preview:
+    uses: elastic/docs-builder/.github/workflows/preview-cleanup.yml@main
+    permissions:
+      contents: none
+      id-token: write
+      deployments: write

.github/workflows/kibana-mitre-update.yml

@@ -16,7 +16,7 @@ jobs:
 
       - name: Get MITRE Attack changed files
         id: changed-attack-files
-        uses: tj-actions/changed-files@v44
+        uses: tj-actions/changed-files@2f7c5bfce28377bc069a65ba478de0a74aa0ca32 # v46.0.1
         with:
           files: detection_rules/etc/attack-v*.json.gz
 

.github/workflows/lock-versions.yml

@@ -6,7 +6,7 @@ on:
           description: 'List of branches to lock versions (ordered, comma separated)'
           required: true
           # 7.17 was intentionally skipped because it was added late and was bug fix only
-          default: '8.12,8.13,8.14,8.15,8.16,8.17'
+          default: '8.14,8.15,8.16,8.17,8.18,9.0'
 
 jobs:
   pr:

.github/workflows/pythonpackage.yml

@@ -2,7 +2,7 @@ name: Unit Tests
 
 on:
   push:
-    branches: [ "main", "7.*", "8.*" ]
+    branches: [ "main", "7.*", "8.*", "9.*" ]
   pull_request:
     branches: [ "*" ]
 

.github/workflows/react-tests-dispatcher.yml

@@ -22,6 +22,7 @@ on:
       - '!rules/integrations/o365/*.toml'
       - '!rules/integrations/okta/*.toml'
       - '!rules/integrations/problemchild/*.toml'
+      - '!rules/integrations/pad/*.toml'
 
 jobs:
   dispatch:

.github/workflows/version-code-and-release.yml

@@ -92,7 +92,7 @@ jobs:
           git push origin "dev-v$version"
 
       - name: Run Release Drafter
-        uses: release-drafter/release-drafter@v6
+        uses: release-drafter/release-drafter@b1476f6e6eb133afa41ed8589daba6dc69b4d3f5 # v6.1.0
         with:
           config-name: release-drafter.yml
         env:

CLI.md

@@ -265,6 +265,7 @@ Options:
   -e, --overwrite-exceptions      Overwrite exceptions in existing rules
   -ac, --overwrite-action-connectors
                                   Overwrite action connectors in existing rules
+  -nt, --no-tactic-filename       Allow rule filenames without tactic prefix. Use this if rules have been exported with this flag.
   -h, --help                      Show this message and exit.
 ```
 
@@ -481,7 +482,7 @@ Options:
 
 ### Exporting rules
 
-This command should be run with the `CUSTOM_RULES_DIR` envvar set, that way proper validation is applied to versioning when the rules are downloaded. See the [custom rules docs](docs-dev/custom-rules.md) for more information.
+This command should be run with the `CUSTOM_RULES_DIR` envvar set, that way proper validation is applied to versioning when the rules are downloaded. See the [custom rules docs](docs-dev/custom-rules-management.md) for more information.
 
 ```
 python -m detection_rules kibana export-rules -h
@@ -520,6 +521,7 @@ Options:
   -e, --export-exceptions         Include exceptions in export
   -s, --skip-errors               Skip errors when exporting rules
   -sv, --strip-version            Strip the version fields from all rules
+  -nt, --no-tactic-filename       Exclude tactic prefix in exported filenames for rules. Use same flag for import-rules to prevent warnings and disable its unit test.
   -h, --help                      Show this message and exit.
 
 ```

detection_rules/cli_utils.py

@@ -23,6 +23,9 @@
                           dict_filter)
 from .schemas import definitions
 from .utils import clear_caches, rulename_to_filename
+from .config import parse_rules_config
+
+RULES_CONFIG = parse_rules_config()
 
 
 def single_collection(f):
@@ -66,11 +69,15 @@ def multi_collection(f):
     @click.option("--directory", "-d", multiple=True, type=click.Path(file_okay=False), required=False,
                   help="Recursively load rules from a directory")
     @click.option("--rule-id", "-id", multiple=True, required=False)
+    @click.option("--no-tactic-filename", "-nt", is_flag=True, required=False,
+                  help="Allow rule filenames without tactic prefix. "
+                  "Use this if rules have been exported with this flag.")
     @functools.wraps(f)
     def get_collection(*args, **kwargs):
         rule_id: List[str] = kwargs.pop("rule_id", [])
         rule_files: List[str] = kwargs.pop("rule_file")
         directories: List[str] = kwargs.pop("directory")
+        no_tactic_filename: bool = kwargs.pop("no_tactic_filename", False)
 
         rules = RuleCollection()
 
@@ -99,7 +106,10 @@ def get_collection(*args, **kwargs):
         for rule in rules:
             threat = rule.contents.data.get("threat")
             first_tactic = threat[0].tactic.name if threat else ""
-            rule_name = rulename_to_filename(rule.contents.data.name, tactic_name=first_tactic)
+            # Check if flag or config is set to not include tactic in the filename
+            no_tactic_filename = no_tactic_filename or RULES_CONFIG.no_tactic_filename
+            tactic_name = None if no_tactic_filename else first_tactic
+            rule_name = rulename_to_filename(rule.contents.data.name, tactic_name=tactic_name)
             if rule.path.name != rule_name:
                 click.secho(
                     f"WARNING: Rule path does not match required path: {rule.path.name} != {rule_name}", fg="yellow"
@@ -210,18 +220,11 @@ def rule_prompt(path=None, rule_type=None, required_only=True, save=True, verbos
     # DEFAULT_PREBUILT_RULES_DIRS[0] is a required directory just as a suggestion
     suggested_path = Path(DEFAULT_PREBUILT_RULES_DIRS[0]) / contents['name']
     path = Path(path or input(f'File path for rule [{suggested_path}]: ') or suggested_path).resolve()
-    # Inherit maturity from the rule already exists
-    maturity = "development"
-    if path.exists():
-        rules = RuleCollection()
-        rules.load_file(path)
-        if rules:
-            maturity = rules.rules[0].contents.metadata.maturity
-
+    # Inherit maturity and optionally local dates from the rule if it already exists
     meta = {
-        "creation_date": creation_date,
-        "updated_date": creation_date,
-        "maturity": maturity,
+        "creation_date": kwargs.get("creation_date") or creation_date,
+        "updated_date": kwargs.get("updated_date") or creation_date,
+        "maturity": "development" or kwargs.get("maturity"),
     }
 
     try:

detection_rules/config.py

@@ -193,6 +193,7 @@ class RulesConfig:
     exception_dir: Optional[Path] = None
     normalize_kql_keywords: bool = True
     bypass_optional_elastic_validation: bool = False
+    no_tactic_filename: bool = False
 
     def __post_init__(self):
         """Perform post validation on packages.yaml file."""
@@ -311,6 +312,10 @@ def parse_rules_config(path: Optional[Path] = None) -> RulesConfig:
     if contents['bypass_optional_elastic_validation']:
         set_all_validation_bypass(contents['bypass_optional_elastic_validation'])
 
+    # no_tactic_filename
+    contents['no_tactic_filename'] = loaded.get('no_tactic_filename', False)
+
+    # return the config
     try:
         rules_config = RulesConfig(test_config=test_config, **contents)
     except (ValueError, TypeError) as e:

detection_rules/custom_rules.py

@@ -15,7 +15,7 @@
 from .utils import ROOT_DIR, get_etc_path, load_etc_dump
 
 DEFAULT_CONFIG_PATH = Path(get_etc_path('_config.yaml'))
-CUSTOM_RULES_DOC_PATH = Path(ROOT_DIR).joinpath(REPO_DOCS_DIR, 'custom-rules.md')
+CUSTOM_RULES_DOC_PATH = Path(ROOT_DIR).joinpath(REPO_DOCS_DIR, 'custom-rules-management.md')
 
 
 @root.group('custom-rules')

detection_rules/etc/_config.yaml

@@ -72,3 +72,8 @@ normalize_kql_keywords: False
 # If set in this file, the path should be relative to the location of this config. If passed as an environment variable,
 #    it should be the full path
 # Note: Using the `custom-rules setup-config <name>` command will generate a config called `test_config.yaml`
+
+# To prevent the tactic prefix from being added to the rule filename, set the line below to True
+# This config line can be used instead of specifying the `--no-tactic-filename` flag in the CLI
+# Mind that for unit tests, you also want to disable the filename test in the test_config.yaml
+# no_tactic_filename: True