Generate an integrations package from a release

detection_rules/devtools.py

@@ -137,7 +137,7 @@ def kibana_commit(ctx, local_repo, github_repo, ssh, kibana_directory, base_bran
     """Prep a commit and push to Kibana."""
     git_exe = shutil.which("git")
 
-    package_name = load_dump(PACKAGE_FILE)['package']["name"]
+    package_name = Package.load_configs()['package']["name"]
     release_dir = os.path.join(RELEASE_DIR, package_name)
     message = message or f"[Detection Rules] Add {package_name} rules"
 

detection_rules/misc.py

@@ -482,7 +482,10 @@ def _convert_type(_val):
                         break
                     else:
                         return []
-            return [_convert_type(r) for r in result_list]
+            if required and value is None:
+                continue
+            else:
+                return [_convert_type(r) for r in result_list]
         else:
             if _check_type(result):
                 return _convert_type(result)

detection_rules/packaging.py

@@ -12,9 +12,10 @@
 import shutil
 from collections import defaultdict, OrderedDict
 from pathlib import Path
-from typing import List, Tuple
+from typing import List, Optional, Tuple
 
 import click
+import yaml
 
 from . import rule_loader
 from .misc import JS_LICENSE, cached
@@ -24,6 +25,7 @@
 RELEASE_DIR = get_path("releases")
 PACKAGE_FILE = get_etc_path('packages.yml')
 NOTICE_FILE = get_path('NOTICE.txt')
+# CHANGELOG_FILE = Path(get_etc_path('rules-changelog.json'))
 
 
 def filter_rule(rule: Rule, config_filter: dict, exclude_fields: dict) -> bool:
@@ -150,13 +152,17 @@ def manage_versions(rules: list, deprecated_rules: list = None, current_versions
 class Package(object):
     """Packaging object for siem rules and releases."""
 
-    def __init__(self, rules, name, deprecated_rules=None, release=False, current_versions=None, min_version=None,
-                 max_version=None, update_version_lock=False, verbose=True):
+    def __init__(self, rules: List[Rule], name: str, deprecated_rules: Optional[List[Rule]] = None,
+                 release: Optional[bool] = False, current_versions: Optional[dict] = None,
+                 min_version: Optional[int] = None, max_version: Optional[int] = None,
+                 update_version_lock: Optional[bool] = False, registry_data: Optional[dict] = None,
+                 verbose: Optional[bool] = True):
         """Initialize a package."""
-        self.rules: List[Rule] = [r.copy() for r in rules]
+        self.rules = [r.copy() for r in rules]
         self.name = name
-        self.deprecated_rules: List[Rule] = [r.copy() for r in deprecated_rules or []]
+        self.deprecated_rules = [r.copy() for r in deprecated_rules or []]
         self.release = release
+        self.registry_data = registry_data or {}
 
         self.changed_rule_ids, self.new_rules_ids, self.removed_rule_ids = self._add_versions(current_versions,
                                                                                               update_version_lock,
@@ -171,6 +177,11 @@ def _add_versions(self, current_versions, update_versions_lock=False, verbose=Tr
         return manage_versions(self.rules, deprecated_rules=self.deprecated_rules, current_versions=current_versions,
                                save_changes=update_versions_lock, verbose=verbose)
 
+    @classmethod
+    def load_configs(cls):
+        """Load configs from packages.yml."""
+        return load_etc_dump(PACKAGE_FILE)['package']
+
     @staticmethod
     def _package_kibana_notice_file(save_dir):
         """Convert and save notice file with package."""
@@ -257,6 +268,7 @@ def save(self, verbose=True):
         self._package_kibana_index_file(rules_dir)
 
         if self.release:
+            self._generate_registry_package(save_dir)
             self.save_release_files(extras_dir, self.changed_rule_ids, self.new_rules_ids, self.removed_rule_ids)
 
             # zip all rules only and place in extras
@@ -461,6 +473,34 @@ def generate_xslx(self, path):
         doc.populate()
         doc.close()
 
+    def _generate_registry_package(self, save_dir):
+        """Generate the artifact for the oob package-storage."""
+        from .schemas.registry_package import RegistryPackageManifest
+
+        manifest = RegistryPackageManifest.from_dict(self.registry_data)
+
+        package_dir = Path(save_dir).joinpath(manifest.version)
+        docs_dir = package_dir / 'docs'
+        rules_dir = package_dir / 'kibana' / 'rules'
+
+        docs_dir.mkdir(parents=True)
+        rules_dir.mkdir(parents=True)
+
+        manifest_file = package_dir.joinpath('manifest.yml')
+        readme_file = docs_dir.joinpath('README.md')
+
+        manifest_file.write_text(yaml.safe_dump(manifest.asdict()))
+        # shutil.copyfile(CHANGELOG_FILE, str(rules_dir.joinpath('CHANGELOG.json')))
+
+        for rule in self.rules:
+            rule.save(new_path=str(rules_dir.joinpath(f'rule-{rule.id}.json')))
+
+        readme_text = ('# Detection rules\n\n'
+                       'The detection rules package stores all the security rules '
+                       'for the detection engine within the Elastic Security application.\n\n')
+
+        readme_file.write_text(readme_text)
+
     def bump_versions(self, save_changes=False, current_versions=None):
         """Bump the versions of all production rules included in a release and optionally save changes."""
         return manage_versions(self.rules, current_versions=current_versions, save_changes=save_changes)

detection_rules/rule.py

@@ -58,7 +58,7 @@ def __ne__(self, other):
     def __hash__(self):
         return hash(self.get_hash())
 
-    def copy(self):
+    def copy(self) -> 'Rule':
         return Rule(path=self.path, contents={'rule': self.contents.copy(), 'metadata': self.metadata.copy()})
 
     @property

detection_rules/rule_loader.py

@@ -119,7 +119,7 @@ def load_rules(file_lookup=None, verbose=True, error=True):
 
         except Exception as e:
             failed = True
-            err_msg = "Invalid rule file in {}\n{}".format(rule_file, click.style(e.args[0], fg='red'))
+            err_msg = "Invalid rule file in {}\n{}".format(rule_file, click.style(str(e), fg='red'))
             errors.append(err_msg)
             if error:
                 if verbose:

detection_rules/schemas/base.py

@@ -10,16 +10,12 @@
 import jsl
 import jsonschema
 
+from .definitions import (
+    DATE_PATTERN, MATURITY_LEVELS, OS_OPTIONS, UUID_PATTERN, VERSION_PATTERN, BRANCH_PATTERN
+)
 from ..utils import cached
 
 
-DATE_PATTERN = r'\d{4}/\d{2}/\d{2}'
-MATURITY_LEVELS = ['development', 'experimental', 'beta', 'production', 'deprecated']
-OS_OPTIONS = ['windows', 'linux', 'macos', 'solaris']
-UUID_PATTERN = r'[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}'
-VERSION_PATTERN = r'\d+\.\d+\.\d+|master'
-
-
 class MarkdownField(jsl.StringField):
     """Helper class for noting which fields are markdown."""
 
@@ -69,14 +65,14 @@ def strip_additional_properties(cls, document, role=None):
 
 
 class TomlMetadata(GenericSchema):
-    """Schema for siem rule toml metadata."""
+    """Schema for rule toml metadata."""
 
     creation_date = jsl.StringField(required=True, pattern=DATE_PATTERN, default=time.strftime('%Y/%m/%d'))
 
     # rule validated against each ecs schema contained
     beats_version = jsl.StringField(pattern=VERSION_PATTERN, required=False)
     comments = jsl.StringField(required=False)
-    ecs_versions = jsl.ArrayField(jsl.StringField(pattern=VERSION_PATTERN, required=True), required=False)
+    ecs_versions = jsl.ArrayField(jsl.StringField(pattern=BRANCH_PATTERN, required=True), required=False)
     maturity = jsl.StringField(enum=MATURITY_LEVELS, default='development', required=True)
 
     os_type_list = jsl.ArrayField(jsl.StringField(enum=OS_OPTIONS), required=False)

detection_rules/schemas/definitions.py

@@ -0,0 +1,45 @@
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License
+# 2.0; you may not use this file except in compliance with the Elastic License
+# 2.0.
+
+"""Custom shared definitions for schemas."""
+
+from typing import ClassVar, Type
+
+import marshmallow
+import marshmallow_dataclass
+from marshmallow_dataclass import NewType
+from marshmallow import validate
+
+
+DATE_PATTERN = r'\d{4}/\d{2}/\d{2}'
+MATURITY_LEVELS = ['development', 'experimental', 'beta', 'production', 'deprecated']
+OS_OPTIONS = ['windows', 'linux', 'macos', 'solaris']
+PR_PATTERN = r'^$|\d+'
+SHA256_PATTERN = r'[a-fA-F0-9]{64}'
+UUID_PATTERN = r'[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}'
+
+_version = r'\d+\.\d+(\.\d+[\w-]*)*'
+CONDITION_VERSION_PATTERN = rf'^\^{_version}$'
+VERSION_PATTERN = f'^{_version}$'
+BRANCH_PATTERN = f'{VERSION_PATTERN}|^master$'
+
+ConditionSemVer = NewType('ConditionSemVer', str, validate=validate.Regexp(CONDITION_VERSION_PATTERN))
+Date = NewType('Date', str, validate=validate.Regexp(DATE_PATTERN))
+SemVer = NewType('SemVer', str, validate=validate.Regexp(VERSION_PATTERN))
+Sha256 = NewType('Sha256', str, validate=validate.Regexp(SHA256_PATTERN))
+UUIDString = NewType('UUIDString', str, validate=validate.Regexp(UUID_PATTERN))
+
+
+@marshmallow_dataclass.dataclass
+class BaseMarshmallowDataclass:
+    """Base marshmallow dataclass configs."""
+
+    class Meta:
+        ordered = True
+
+    Schema: ClassVar[Type[marshmallow.Schema]] = marshmallow.Schema
+
+    def dump(self) -> dict:
+        return self.Schema().dump(self)

detection_rules/schemas/registry_package.py

@@ -0,0 +1,47 @@
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License
+# 2.0; you may not use this file except in compliance with the Elastic License
+# 2.0.
+
+"""Definitions for packages destined for the registry."""
+
+from dataclasses import dataclass, field
+from typing import Dict, List, Type
+
+from marshmallow import Schema, validate
+from marshmallow_dataclass import class_schema
+
+from .definitions import ConditionSemVer, SemVer
+
+
+@dataclass
+class RegistryPackageManifest:
+    """Base class for registry packages."""
+
+    conditions: Dict[str, ConditionSemVer]
+    version: SemVer
+
+    categories: List[str] = field(default_factory=lambda: ['security'])
+    description: str = 'Rules for the detection engine in the Security application.'
+    format_version: SemVer = field(metadata=dict(validate=validate.Equal('1.0.0')), default='1.0.0')
+    icons: list = field(default_factory=list)
+    internal: bool = True
+    license: str = 'basic'
+    name: str = 'detection_rules'
+    owner: Dict[str, str] = field(default_factory=lambda: dict(github='elastic/protections'))
+    policy_templates: list = field(default_factory=list)
+    release: str = 'experimental'
+    screenshots: list = field(default_factory=list)
+    title: str = 'Detection rules'
+    type: str = 'integration'
+
+    @classmethod
+    def get_schema(cls) -> Type[Schema]:
+        return class_schema(cls)
+
+    @classmethod
+    def from_dict(cls, obj: dict) -> 'RegistryPackageManifest':
+        return cls.get_schema()().load(obj)
+
+    def asdict(self) -> dict:
+        return self.get_schema()().dump(self)

detection_rules/semver.py

@@ -5,14 +5,14 @@
 
 """Helper functionality for comparing semantic versions."""
 import re
+from typing import Iterable, Union
 
 
 class Version(tuple):
 
-    def __new__(cls, version):
+    def __new__(cls, version: Union[Iterable, str]) -> 'Version':
         if not isinstance(version, (int, list, tuple)):
             version = tuple(int(a) if a.isdigit() else a for a in re.split(r'[.-]', version))
-
         return version if isinstance(version, int) else tuple.__new__(cls, version)
 
     def bump(self):

etc/packages.yml

@@ -2,22 +2,31 @@
 package:
   name: "7.13"
   release: true
-# exclude rules which have any of the following index <-> field pairs
-#  exclude_fields:
-#    # special field to apply to all indexes
-#    any:
-#    - process.args
-#    - network.direction
-#    logs-endpoint.events.*:
-#    - file.name
+  # exclude rules which have any of the following index <-> field pairs
+  #  exclude_fields:
+  #    # special field to apply to all indexes
+  #    any:
+  #    - process.args
+  #    - network.direction
+  #    logs-endpoint.events.*:
+  #    - file.name
   filter:
     # ecs_version:
     # - 1.4.0
     # - 1.5.0
     maturity:
     - production
-# log deprecated rules in summary and change logs
+  # log deprecated rules in summary and change logs
   log_deprecated: true
-# rule version scoping
-#  min_version: 1
-#  max_version: 5
+  # rule version scoping
+  #  min_version: 1
+  #  max_version: 5
+
+  # Integrations registry
+  registry_data:
+    # integration package schema version
+    format_version: "1.0.0"
+    conditions:
+      kibana_version: "^7.13.0"
+    # this determines the version for the package-storage generated artifact
+    version: "0.0.1-dev.1"

requirements.txt

@@ -8,6 +8,8 @@ PyYAML~=5.3
 eql==0.9.9
 elasticsearch~=7.9
 XlsxWriter==1.3.6
+marshmallow==3.6.1
+marshmallow-dataclass==8.3.1
 
 # test deps
 pyflakes==2.2.0

tests/test_packages.py

@@ -6,12 +6,14 @@
 """Test that the packages are built correctly."""
 import unittest
 import uuid
-import yaml
 
 from detection_rules import rule_loader
 from detection_rules.packaging import PACKAGE_FILE, Package
 
 
+package_configs = Package.load_configs()
+
+
 class TestPackages(unittest.TestCase):
     """Test package building and saving."""
 
@@ -48,10 +50,7 @@ def test_package_loader_production_config(self):
 
     def test_package_loader_default_configs(self):
         """Test configs in etc/packages.yml."""
-        with open(PACKAGE_FILE) as f:
-            configs = yaml.safe_load(f)['package']
-
-        package = Package.from_config(configs)
+        package = Package.from_config(package_configs)
         for rule in package.rules:
             rule.contents.pop('version')
             rule.validate(as_rule=True)
@@ -148,3 +147,26 @@ def test_version_filter(self):
 
         package = Package(rules, 'test', current_versions=version_info, min_version=2, max_version=2)
         self.assertEqual(1, len(package.rules), msg)
+
+
+class TestRegistryPackage(unittest.TestCase):
+    """Test the OOB registry package."""
+
+    @classmethod
+    def setUpClass(cls) -> None:
+        from detection_rules.schemas.registry_package import RegistryPackageManifest
+
+        assert 'registry_data' in package_configs, f'Missing registry_data in {PACKAGE_FILE}'
+        cls.registry_config = package_configs['registry_data']
+        RegistryPackageManifest.from_dict(cls.registry_config)
+
+    def test_registry_package_config(self):
+        """Test that the registry package is validating properly."""
+        from marshmallow import ValidationError
+        from detection_rules.schemas.registry_package import RegistryPackageManifest
+
+        registry_config = self.registry_config.copy()
+        registry_config['version'] += '7.1.1.'
+
+        with self.assertRaises(ValidationError):
+            RegistryPackageManifest.from_dict(registry_config)