Generate an integrations package from a release

detection_rules/misc.py

@@ -480,7 +480,10 @@ def _convert_type(_val):
                         break
                     else:
                         return []
-            return [_convert_type(r) for r in result_list]
+            if required and value is None:
+                continue
+            else:
+                return [_convert_type(r) for r in result_list]
         else:
             if _check_type(result):
                 return _convert_type(result)

detection_rules/packaging.py

@@ -23,6 +23,7 @@
 RELEASE_DIR = get_path("releases")
 PACKAGE_FILE = get_etc_path('packages.yml')
 NOTICE_FILE = get_path('NOTICE.txt')
+CHANGELOG_FILE = Path(get_etc_path('rules-changelog.json'))
 
 
 def filter_rule(rule: Rule, config_filter: dict, exclude_fields: dict) -> bool:
@@ -149,13 +150,15 @@ def manage_versions(rules: list, deprecated_rules: list = None, current_versions
 class Package(object):
     """Packaging object for siem rules and releases."""
 
-    def __init__(self, rules, name, deprecated_rules=None, release=False, current_versions=None, min_version=None,
-                 max_version=None, update_version_lock=False, verbose=True):
+    def __init__(self, rules: List[Rule], name, deprecated_rules: List[Rule] = None, release=False,
+                 current_versions: dict = None, min_version: int = None, max_version: int = None,
+                 update_version_lock=False, registry_data: dict = None, verbose=True):
         """Initialize a package."""
-        self.rules: List[Rule] = [r.copy() for r in rules]
+        self.rules = [r.copy() for r in rules]
         self.name = name
-        self.deprecated_rules: List[Rule] = [r.copy() for r in deprecated_rules or []]
+        self.deprecated_rules = [r.copy() for r in deprecated_rules or []]
         self.release = release
+        self.registry_data = registry_data or {}
 
         self.changed_rule_ids, self.new_rules_ids, self.removed_rule_ids = self._add_versions(current_versions,
                                                                                               update_version_lock,
@@ -256,6 +259,9 @@ def save(self, verbose=True):
         self._package_kibana_index_file(rules_dir)
 
         if self.release:
+            if self.registry_data:
+                self._generate_registry_package(save_dir)
+
             self.save_release_files(extras_dir, self.changed_rule_ids, self.new_rules_ids, self.removed_rule_ids)
 
             # zip all rules only and place in extras
@@ -460,6 +466,39 @@ def generate_xslx(self, path):
         doc.populate()
         doc.close()
 
+    def _generate_registry_package(self, save_dir):
+        """Generate the artifact for the oob package-storage."""
+        from .schemas.registry_package import get_manifest
+
+        assert self.registry_data
+
+        registry_manifest = get_manifest(self.registry_data['format_version'])
+        manifest = registry_manifest.Schema().load(self.registry_data)
+
+        package_dir = Path(save_dir).joinpath(manifest.version)
+        docs_dir = package_dir.joinpath('docs')
+        rules_dir = package_dir.joinpath('kibana', 'rules')
+
+        docs_dir.mkdir(parents=True)
+        rules_dir.mkdir(parents=True)
+
+        manifest_file = package_dir.joinpath('manifest.yml')
+        readme_file = docs_dir.joinpath('README.md')
+
+        manifest_file.write_text(json.dumps(manifest.dump(), indent=2, sort_keys=True))
+        shutil.copyfile(CHANGELOG_FILE, str(rules_dir.joinpath('CHANGELOG.json')))
+
+        for rule in self.rules:
+            rule.save(new_path=str(rules_dir.joinpath(f'rule-{rule.id}.json')))
+
+        readme_text = '# Detection rules\n'
+        readme_text += '\n'
+        readme_text += 'The detection rules package is a non-integration package to store all the rules and '
+        readme_text += 'dependencies (e.g. ML jobs) for the detection engine within the Elastic Security application.\n'
+        readme_text += '\n'
+
+        readme_file.write_text(readme_text)
+
     def bump_versions(self, save_changes=False, current_versions=None):
         """Bump the versions of all production rules included in a release and optionally save changes."""
         return manage_versions(self.rules, current_versions=current_versions, save_changes=save_changes)

detection_rules/rule_loader.py

@@ -118,7 +118,7 @@ def load_rules(file_lookup=None, verbose=True, error=True):
 
         except Exception as e:
             failed = True
-            err_msg = "Invalid rule file in {}\n{}".format(rule_file, click.style(e.args[0], fg='red'))
+            err_msg = "Invalid rule file in {}\n{}".format(rule_file, click.style(str(e), fg='red'))
             errors.append(err_msg)
             if error:
                 if verbose:

detection_rules/schemas/base.py

@@ -9,16 +9,12 @@
 import jsl
 import jsonschema
 
+from .definitions import (
+    DATE_PATTERN, MATURITY_LEVELS, OS_OPTIONS, UUID_PATTERN, VERSION_PATTERN, VERSION_W_MASTER_PATTERN
+)
 from ..utils import cached
 
 
-DATE_PATTERN = r'\d{4}/\d{2}/\d{2}'
-MATURITY_LEVELS = ['development', 'experimental', 'beta', 'production', 'deprecated']
-OS_OPTIONS = ['windows', 'linux', 'macos', 'solaris']
-UUID_PATTERN = r'[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}'
-VERSION_PATTERN = r'\d+\.\d+\.\d+|master'
-
-
 class MarkdownField(jsl.StringField):
     """Helper class for noting which fields are markdown."""
 
@@ -68,14 +64,14 @@ def strip_additional_properties(cls, document, role=None):
 
 
 class TomlMetadata(GenericSchema):
-    """Schema for siem rule toml metadata."""
+    """Schema for rule toml metadata."""
 
     creation_date = jsl.StringField(required=True, pattern=DATE_PATTERN, default=time.strftime('%Y/%m/%d'))
 
     # rule validated against each ecs schema contained
     beats_version = jsl.StringField(pattern=VERSION_PATTERN, required=False)
     comments = jsl.StringField(required=False)
-    ecs_versions = jsl.ArrayField(jsl.StringField(pattern=VERSION_PATTERN, required=True), required=False)
+    ecs_versions = jsl.ArrayField(jsl.StringField(pattern=VERSION_W_MASTER_PATTERN, required=True), required=False)
     maturity = jsl.StringField(enum=MATURITY_LEVELS, default='development', required=True)
 
     os_type_list = jsl.ArrayField(jsl.StringField(enum=OS_OPTIONS), required=False)

detection_rules/schemas/definitions.py

@@ -0,0 +1,44 @@
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+
+"""Custom shared definitions for schemas."""
+
+from typing import ClassVar, Type
+
+import marshmallow
+import marshmallow_dataclass
+from marshmallow_dataclass import NewType
+from marshmallow import validate
+
+
+DATE_PATTERN = r'\d{4}/\d{2}/\d{2}'
+MATURITY_LEVELS = ['development', 'experimental', 'beta', 'production', 'deprecated']
+OS_OPTIONS = ['windows', 'linux', 'macos', 'solaris']
+PR_PATTERN = r'^$|\d+'
+SHA256_PATTERN = r'[a-fA-F0-9]{64}'
+UUID_PATTERN = r'[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}'
+
+_version = r'\d+\.\d+(\.\d+[\w-]*)*'
+CONDITION_VERSION_PATTERN = rf'^\^{_version}$'
+VERSION_PATTERN = f'^{_version}$'
+VERSION_W_MASTER_PATTERN = f'{VERSION_PATTERN}|^master$'
+
+ConditionSemVer = NewType('ConditionSemVer', str, validate=validate.Regexp(CONDITION_VERSION_PATTERN))
+Date = NewType('Date', str, validate=validate.Regexp(DATE_PATTERN))
+SemVer = NewType('SemVer', str, validate=validate.Regexp(VERSION_PATTERN))
+Sha256 = NewType('Sha256', str, validate=validate.Regexp(SHA256_PATTERN))
+Uuid = NewType('Uuid', str, validate=validate.Regexp(UUID_PATTERN))
+
+
+@marshmallow_dataclass.dataclass
+class BaseMarshmallowDataclass:
+    """Base marshmallow dataclass configs."""
+
+    class Meta:
+        ordered = True
+
+    Schema: ClassVar[Type[marshmallow.Schema]] = marshmallow.Schema
+
+    def dump(self) -> dict:
+        return self.Schema().dump(self)

detection_rules/schemas/registry_package.py

@@ -0,0 +1,53 @@
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+
+"""Definitions for packages destined for the registry."""
+
+import dataclasses
+from typing import Dict, Union, Type
+
+import marshmallow_dataclass
+from marshmallow import validate
+
+from .definitions import BaseMarshmallowDataclass, ConditionSemVer, SemVer
+
+
+@marshmallow_dataclass.dataclass
+class BaseManifest(BaseMarshmallowDataclass):
+    """Base class for registry packages."""
+
+    conditions: Dict[str, ConditionSemVer]
+    version: SemVer
+    format_version: SemVer
+
+    categories: list = dataclasses.field(default_factory=lambda: ['security'].copy())
+    description: str = 'Rules for the detection engine in the Security application.'
+    icons: list = dataclasses.field(default_factory=list)
+    license: str = 'basic'
+    name: str = 'detection_rules'
+    owner: dict = dataclasses.field(default_factory=lambda: dict(github='elastic/protections').copy())
+    policy_templates: list = dataclasses.field(default_factory=list)
+    release: str = 'experimental'
+    screenshots: list = dataclasses.field(default_factory=list)
+    title: str = 'Detection rules'
+    type: str = 'rules'
+
+
+@marshmallow_dataclass.dataclass
+class ManifestV1Dot0(BaseManifest):
+    """Integrations registry package schema."""
+
+    format_version: SemVer = dataclasses.field(metadata=dict(validate=validate.Equal('1.0.0')), default='1.0.0')
+
+
+manifests = [
+    ManifestV1Dot0
+]
+
+
+def get_manifest(format_version: str) -> Union[Type[ManifestV1Dot0]]:
+    """Retrieve a manifest class by format_version."""
+    for manifest in manifests:
+        if manifest.format_version == format_version:
+            return manifest

detection_rules/semver.py

@@ -4,14 +4,21 @@
 
 """Helper functionality for comparing semantic versions."""
 import re
+from typing import Iterable, Union
 
 
 class Version(tuple):
 
-    def __new__(cls, version):
+    def __new__(cls, version: Union[Iterable, str], pad: int = None) -> 'Version':
         if not isinstance(version, (int, list, tuple)):
             version = tuple(int(a) if a.isdigit() else a for a in re.split(r'[.-]', version))
 
+            if pad:
+                width = len(version)
+
+                if pad > width:
+                    version = version + (0,) * (pad - width)
+
         return version if isinstance(version, int) else tuple.__new__(cls, version)
 
     def bump(self):

detection_rules/utils.py

@@ -190,6 +190,10 @@ def cached(f):
 
     @functools.wraps(f)
     def wrapped(*args, **kwargs):
+        bypass_cache = kwargs.pop('bypass_cache', None)
+        if bypass_cache:
+            return f(*args, **kwargs)
+
         _cache.setdefault(func_key, {})
         cache_key = freeze(args), freeze(kwargs)
 

etc/packages.yml

@@ -2,22 +2,31 @@
 package:
   name: "7.13"
   release: true
-# exclude rules which have any of the following index <-> field pairs
-#  exclude_fields:
-#    # special field to apply to all indexes
-#    any:
-#    - process.args
-#    - network.direction
-#    logs-endpoint.events.*:
-#    - file.name
+  # exclude rules which have any of the following index <-> field pairs
+  #  exclude_fields:
+  #    # special field to apply to all indexes
+  #    any:
+  #    - process.args
+  #    - network.direction
+  #    logs-endpoint.events.*:
+  #    - file.name
   filter:
     # ecs_version:
     # - 1.4.0
     # - 1.5.0
     maturity:
     - production
-# log deprecated rules in summary and change logs
+  # log deprecated rules in summary and change logs
   log_deprecated: true
-# rule version scoping
-#  min_version: 1
-#  max_version: 5
+  # rule version scoping
+  #  min_version: 1
+  #  max_version: 5
+
+  # Integrations registry
+  registry_data:
+    # integration package schema version
+    format_version: "1.0.0"
+    conditions:
+      kibana_version: "^7.13.0"
+    # this determines the version for the package-storage generated artifact
+    version: "0.0.1-dev.1"

etc/rules-changelog.json

@@ -0,0 +1,3 @@
+{
+  "changelog": {}
+}

requirements.txt

@@ -8,6 +8,8 @@ PyYAML~=5.3
 eql==0.9.9
 elasticsearch~=7.9
 XlsxWriter==1.3.6
+marshmallow==3.6.1
+marshmallow-dataclass==8.3.1
 
 # test deps
 pyflakes==2.2.0

tests/test_packages.py

@@ -11,6 +11,10 @@
 from detection_rules.packaging import PACKAGE_FILE, Package
 
 
+with open(PACKAGE_FILE) as f:
+    package_configs = yaml.safe_load(f)['package']
+
+
 class TestPackages(unittest.TestCase):
     """Test package building and saving."""
 
@@ -47,10 +51,7 @@ def test_package_loader_production_config(self):
 
     def test_package_loader_default_configs(self):
         """Test configs in etc/packages.yml."""
-        with open(PACKAGE_FILE) as f:
-            configs = yaml.safe_load(f)['package']
-
-        package = Package.from_config(configs)
+        package = Package.from_config(package_configs)
         for rule in package.rules:
             rule.contents.pop('version')
             rule.validate(as_rule=True)
@@ -147,3 +148,33 @@ def test_version_filter(self):
 
         package = Package(rules, 'test', current_versions=version_info, min_version=2, max_version=2)
         self.assertEqual(1, len(package.rules), msg)
+
+
+class TestRegistryPackage(unittest.TestCase):
+    """Test the OOB registry package."""
+
+    @classmethod
+    def setUpClass(cls) -> None:
+        from detection_rules.schemas.registry_package import get_manifest
+
+        assert 'registry_data' in package_configs, f'Missing registry_data in {PACKAGE_FILE}'
+        cls.registry_config = package_configs['registry_data']
+        assert 'format_version' in cls.registry_config, f'format_version missing from registry_data in {PACKAGE_FILE}'
+
+        cls.format_version = cls.registry_config['format_version']
+        cls.registry_manifest = get_manifest(cls.format_version)
+        assert cls.registry_manifest is not None, f'No registry package schema available for {cls.format_version}'
+
+        cls.registry_manifest.Schema().load(cls.registry_config)
+
+    def test_registry_package_config(self):
+        """Test that the registry package is validating properly."""
+        from marshmallow import ValidationError
+        from detection_rules.schemas.registry_package import get_manifest
+
+        registry_manifest = get_manifest('1.0.0')
+        registry_config = self.registry_config.copy()
+        registry_config['version'] += '7.1.1.'
+
+        with self.assertRaises(ValidationError):
+            registry_manifest.Schema().load(registry_config)