diff --git a/detection_rules/devtools.py b/detection_rules/devtools.py index 8480a05931c..2726fe2fd73 100644 --- a/detection_rules/devtools.py +++ b/detection_rules/devtools.py @@ -137,7 +137,7 @@ def kibana_commit(ctx, local_repo, github_repo, ssh, kibana_directory, base_bran """Prep a commit and push to Kibana.""" git_exe = shutil.which("git") - package_name = load_dump(PACKAGE_FILE)['package']["name"] + package_name = Package.load_configs()['package']["name"] release_dir = os.path.join(RELEASE_DIR, package_name) message = message or f"[Detection Rules] Add {package_name} rules" diff --git a/detection_rules/misc.py b/detection_rules/misc.py index e75bb00b242..afedc5a58bb 100644 --- a/detection_rules/misc.py +++ b/detection_rules/misc.py @@ -482,7 +482,10 @@ def _convert_type(_val): break else: return [] - return [_convert_type(r) for r in result_list] + if required and value is None: + continue + else: + return [_convert_type(r) for r in result_list] else: if _check_type(result): return _convert_type(result) diff --git a/detection_rules/packaging.py b/detection_rules/packaging.py index 10823e47f94..21865bd306d 100644 --- a/detection_rules/packaging.py +++ b/detection_rules/packaging.py @@ -12,9 +12,10 @@ import shutil from collections import defaultdict, OrderedDict from pathlib import Path -from typing import List, Tuple +from typing import List, Optional, Tuple import click +import yaml from . import rule_loader from .misc import JS_LICENSE, cached @@ -24,6 +25,7 @@ RELEASE_DIR = get_path("releases") PACKAGE_FILE = get_etc_path('packages.yml') NOTICE_FILE = get_path('NOTICE.txt') +# CHANGELOG_FILE = Path(get_etc_path('rules-changelog.json')) def filter_rule(rule: Rule, config_filter: dict, exclude_fields: dict) -> bool: @@ -150,13 +152,17 @@ def manage_versions(rules: list, deprecated_rules: list = None, current_versions class Package(object): """Packaging object for siem rules and releases.""" - def __init__(self, rules, name, deprecated_rules=None, release=False, current_versions=None, min_version=None, - max_version=None, update_version_lock=False, verbose=True): + def __init__(self, rules: List[Rule], name: str, deprecated_rules: Optional[List[Rule]] = None, + release: Optional[bool] = False, current_versions: Optional[dict] = None, + min_version: Optional[int] = None, max_version: Optional[int] = None, + update_version_lock: Optional[bool] = False, registry_data: Optional[dict] = None, + verbose: Optional[bool] = True): """Initialize a package.""" - self.rules: List[Rule] = [r.copy() for r in rules] + self.rules = [r.copy() for r in rules] self.name = name - self.deprecated_rules: List[Rule] = [r.copy() for r in deprecated_rules or []] + self.deprecated_rules = [r.copy() for r in deprecated_rules or []] self.release = release + self.registry_data = registry_data or {} self.changed_rule_ids, self.new_rules_ids, self.removed_rule_ids = self._add_versions(current_versions, update_version_lock, @@ -171,6 +177,11 @@ def _add_versions(self, current_versions, update_versions_lock=False, verbose=Tr return manage_versions(self.rules, deprecated_rules=self.deprecated_rules, current_versions=current_versions, save_changes=update_versions_lock, verbose=verbose) + @classmethod + def load_configs(cls): + """Load configs from packages.yml.""" + return load_etc_dump(PACKAGE_FILE)['package'] + @staticmethod def _package_kibana_notice_file(save_dir): """Convert and save notice file with package.""" @@ -257,6 +268,7 @@ def save(self, verbose=True): self._package_kibana_index_file(rules_dir) if self.release: + self._generate_registry_package(save_dir) self.save_release_files(extras_dir, self.changed_rule_ids, self.new_rules_ids, self.removed_rule_ids) # zip all rules only and place in extras @@ -461,6 +473,34 @@ def generate_xslx(self, path): doc.populate() doc.close() + def _generate_registry_package(self, save_dir): + """Generate the artifact for the oob package-storage.""" + from .schemas.registry_package import RegistryPackageManifest + + manifest = RegistryPackageManifest.from_dict(self.registry_data) + + package_dir = Path(save_dir).joinpath(manifest.version) + docs_dir = package_dir / 'docs' + rules_dir = package_dir / 'kibana' / 'rules' + + docs_dir.mkdir(parents=True) + rules_dir.mkdir(parents=True) + + manifest_file = package_dir.joinpath('manifest.yml') + readme_file = docs_dir.joinpath('README.md') + + manifest_file.write_text(yaml.safe_dump(manifest.asdict())) + # shutil.copyfile(CHANGELOG_FILE, str(rules_dir.joinpath('CHANGELOG.json'))) + + for rule in self.rules: + rule.save(new_path=str(rules_dir.joinpath(f'rule-{rule.id}.json'))) + + readme_text = ('# Detection rules\n\n' + 'The detection rules package stores all the security rules ' + 'for the detection engine within the Elastic Security application.\n\n') + + readme_file.write_text(readme_text) + def bump_versions(self, save_changes=False, current_versions=None): """Bump the versions of all production rules included in a release and optionally save changes.""" return manage_versions(self.rules, current_versions=current_versions, save_changes=save_changes) diff --git a/detection_rules/rule.py b/detection_rules/rule.py index dfcca19d524..dc9b8a3c2f2 100644 --- a/detection_rules/rule.py +++ b/detection_rules/rule.py @@ -58,7 +58,7 @@ def __ne__(self, other): def __hash__(self): return hash(self.get_hash()) - def copy(self): + def copy(self) -> 'Rule': return Rule(path=self.path, contents={'rule': self.contents.copy(), 'metadata': self.metadata.copy()}) @property diff --git a/detection_rules/rule_loader.py b/detection_rules/rule_loader.py index c9e9533281a..b8d7d8f63f4 100644 --- a/detection_rules/rule_loader.py +++ b/detection_rules/rule_loader.py @@ -119,7 +119,7 @@ def load_rules(file_lookup=None, verbose=True, error=True): except Exception as e: failed = True - err_msg = "Invalid rule file in {}\n{}".format(rule_file, click.style(e.args[0], fg='red')) + err_msg = "Invalid rule file in {}\n{}".format(rule_file, click.style(str(e), fg='red')) errors.append(err_msg) if error: if verbose: diff --git a/detection_rules/schemas/base.py b/detection_rules/schemas/base.py index 1df4696f3c6..0768089d665 100644 --- a/detection_rules/schemas/base.py +++ b/detection_rules/schemas/base.py @@ -10,16 +10,12 @@ import jsl import jsonschema +from .definitions import ( + DATE_PATTERN, MATURITY_LEVELS, OS_OPTIONS, UUID_PATTERN, VERSION_PATTERN, BRANCH_PATTERN +) from ..utils import cached -DATE_PATTERN = r'\d{4}/\d{2}/\d{2}' -MATURITY_LEVELS = ['development', 'experimental', 'beta', 'production', 'deprecated'] -OS_OPTIONS = ['windows', 'linux', 'macos', 'solaris'] -UUID_PATTERN = r'[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}' -VERSION_PATTERN = r'\d+\.\d+\.\d+|master' - - class MarkdownField(jsl.StringField): """Helper class for noting which fields are markdown.""" @@ -69,14 +65,14 @@ def strip_additional_properties(cls, document, role=None): class TomlMetadata(GenericSchema): - """Schema for siem rule toml metadata.""" + """Schema for rule toml metadata.""" creation_date = jsl.StringField(required=True, pattern=DATE_PATTERN, default=time.strftime('%Y/%m/%d')) # rule validated against each ecs schema contained beats_version = jsl.StringField(pattern=VERSION_PATTERN, required=False) comments = jsl.StringField(required=False) - ecs_versions = jsl.ArrayField(jsl.StringField(pattern=VERSION_PATTERN, required=True), required=False) + ecs_versions = jsl.ArrayField(jsl.StringField(pattern=BRANCH_PATTERN, required=True), required=False) maturity = jsl.StringField(enum=MATURITY_LEVELS, default='development', required=True) os_type_list = jsl.ArrayField(jsl.StringField(enum=OS_OPTIONS), required=False) diff --git a/detection_rules/schemas/definitions.py b/detection_rules/schemas/definitions.py new file mode 100644 index 00000000000..355b5cbcfbb --- /dev/null +++ b/detection_rules/schemas/definitions.py @@ -0,0 +1,45 @@ +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License +# 2.0; you may not use this file except in compliance with the Elastic License +# 2.0. + +"""Custom shared definitions for schemas.""" + +from typing import ClassVar, Type + +import marshmallow +import marshmallow_dataclass +from marshmallow_dataclass import NewType +from marshmallow import validate + + +DATE_PATTERN = r'\d{4}/\d{2}/\d{2}' +MATURITY_LEVELS = ['development', 'experimental', 'beta', 'production', 'deprecated'] +OS_OPTIONS = ['windows', 'linux', 'macos', 'solaris'] +PR_PATTERN = r'^$|\d+' +SHA256_PATTERN = r'[a-fA-F0-9]{64}' +UUID_PATTERN = r'[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}' + +_version = r'\d+\.\d+(\.\d+[\w-]*)*' +CONDITION_VERSION_PATTERN = rf'^\^{_version}$' +VERSION_PATTERN = f'^{_version}$' +BRANCH_PATTERN = f'{VERSION_PATTERN}|^master$' + +ConditionSemVer = NewType('ConditionSemVer', str, validate=validate.Regexp(CONDITION_VERSION_PATTERN)) +Date = NewType('Date', str, validate=validate.Regexp(DATE_PATTERN)) +SemVer = NewType('SemVer', str, validate=validate.Regexp(VERSION_PATTERN)) +Sha256 = NewType('Sha256', str, validate=validate.Regexp(SHA256_PATTERN)) +UUIDString = NewType('UUIDString', str, validate=validate.Regexp(UUID_PATTERN)) + + +@marshmallow_dataclass.dataclass +class BaseMarshmallowDataclass: + """Base marshmallow dataclass configs.""" + + class Meta: + ordered = True + + Schema: ClassVar[Type[marshmallow.Schema]] = marshmallow.Schema + + def dump(self) -> dict: + return self.Schema().dump(self) diff --git a/detection_rules/schemas/registry_package.py b/detection_rules/schemas/registry_package.py new file mode 100644 index 00000000000..bfd30e14ed1 --- /dev/null +++ b/detection_rules/schemas/registry_package.py @@ -0,0 +1,47 @@ +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License +# 2.0; you may not use this file except in compliance with the Elastic License +# 2.0. + +"""Definitions for packages destined for the registry.""" + +from dataclasses import dataclass, field +from typing import Dict, List, Type + +from marshmallow import Schema, validate +from marshmallow_dataclass import class_schema + +from .definitions import ConditionSemVer, SemVer + + +@dataclass +class RegistryPackageManifest: + """Base class for registry packages.""" + + conditions: Dict[str, ConditionSemVer] + version: SemVer + + categories: List[str] = field(default_factory=lambda: ['security']) + description: str = 'Rules for the detection engine in the Security application.' + format_version: SemVer = field(metadata=dict(validate=validate.Equal('1.0.0')), default='1.0.0') + icons: list = field(default_factory=list) + internal: bool = True + license: str = 'basic' + name: str = 'detection_rules' + owner: Dict[str, str] = field(default_factory=lambda: dict(github='elastic/protections')) + policy_templates: list = field(default_factory=list) + release: str = 'experimental' + screenshots: list = field(default_factory=list) + title: str = 'Detection rules' + type: str = 'integration' + + @classmethod + def get_schema(cls) -> Type[Schema]: + return class_schema(cls) + + @classmethod + def from_dict(cls, obj: dict) -> 'RegistryPackageManifest': + return cls.get_schema()().load(obj) + + def asdict(self) -> dict: + return self.get_schema()().dump(self) diff --git a/detection_rules/semver.py b/detection_rules/semver.py index fe8d35fa985..b032bff0626 100644 --- a/detection_rules/semver.py +++ b/detection_rules/semver.py @@ -5,14 +5,14 @@ """Helper functionality for comparing semantic versions.""" import re +from typing import Iterable, Union class Version(tuple): - def __new__(cls, version): + def __new__(cls, version: Union[Iterable, str]) -> 'Version': if not isinstance(version, (int, list, tuple)): version = tuple(int(a) if a.isdigit() else a for a in re.split(r'[.-]', version)) - return version if isinstance(version, int) else tuple.__new__(cls, version) def bump(self): diff --git a/etc/packages.yml b/etc/packages.yml index b5229200bff..39ee364bb58 100644 --- a/etc/packages.yml +++ b/etc/packages.yml @@ -2,22 +2,31 @@ package: name: "7.13" release: true -# exclude rules which have any of the following index <-> field pairs -# exclude_fields: -# # special field to apply to all indexes -# any: -# - process.args -# - network.direction -# logs-endpoint.events.*: -# - file.name + # exclude rules which have any of the following index <-> field pairs + # exclude_fields: + # # special field to apply to all indexes + # any: + # - process.args + # - network.direction + # logs-endpoint.events.*: + # - file.name filter: # ecs_version: # - 1.4.0 # - 1.5.0 maturity: - production -# log deprecated rules in summary and change logs + # log deprecated rules in summary and change logs log_deprecated: true -# rule version scoping -# min_version: 1 -# max_version: 5 + # rule version scoping + # min_version: 1 + # max_version: 5 + + # Integrations registry + registry_data: + # integration package schema version + format_version: "1.0.0" + conditions: + kibana_version: "^7.13.0" + # this determines the version for the package-storage generated artifact + version: "0.0.1-dev.1" diff --git a/requirements.txt b/requirements.txt index 3edd9e01b93..e49414e7883 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,6 +8,8 @@ PyYAML~=5.3 eql==0.9.9 elasticsearch~=7.9 XlsxWriter==1.3.6 +marshmallow==3.6.1 +marshmallow-dataclass==8.3.1 # test deps pyflakes==2.2.0 diff --git a/tests/test_packages.py b/tests/test_packages.py index ab9b6b07ab3..8bd2bfb6ce9 100644 --- a/tests/test_packages.py +++ b/tests/test_packages.py @@ -6,12 +6,14 @@ """Test that the packages are built correctly.""" import unittest import uuid -import yaml from detection_rules import rule_loader from detection_rules.packaging import PACKAGE_FILE, Package +package_configs = Package.load_configs() + + class TestPackages(unittest.TestCase): """Test package building and saving.""" @@ -48,10 +50,7 @@ def test_package_loader_production_config(self): def test_package_loader_default_configs(self): """Test configs in etc/packages.yml.""" - with open(PACKAGE_FILE) as f: - configs = yaml.safe_load(f)['package'] - - package = Package.from_config(configs) + package = Package.from_config(package_configs) for rule in package.rules: rule.contents.pop('version') rule.validate(as_rule=True) @@ -148,3 +147,26 @@ def test_version_filter(self): package = Package(rules, 'test', current_versions=version_info, min_version=2, max_version=2) self.assertEqual(1, len(package.rules), msg) + + +class TestRegistryPackage(unittest.TestCase): + """Test the OOB registry package.""" + + @classmethod + def setUpClass(cls) -> None: + from detection_rules.schemas.registry_package import RegistryPackageManifest + + assert 'registry_data' in package_configs, f'Missing registry_data in {PACKAGE_FILE}' + cls.registry_config = package_configs['registry_data'] + RegistryPackageManifest.from_dict(cls.registry_config) + + def test_registry_package_config(self): + """Test that the registry package is validating properly.""" + from marshmallow import ValidationError + from detection_rules.schemas.registry_package import RegistryPackageManifest + + registry_config = self.registry_config.copy() + registry_config['version'] += '7.1.1.' + + with self.assertRaises(ValidationError): + RegistryPackageManifest.from_dict(registry_config)