From 52db8e2785a57d34580942898da73ee36d685d0b Mon Sep 17 00:00:00 2001
From: brokensound77 <brokensound77@users.noreply.github.com>
Date: Fri, 5 Mar 2021 02:35:57 -0900
Subject: [PATCH 01/11] Generate an integrations package from a release

---
 detection_rules/misc.py                     |  5 +-
 detection_rules/packaging.py                | 47 ++++++++++++++++--
 detection_rules/rule_loader.py              |  2 +-
 detection_rules/schemas/base.py             | 14 ++----
 detection_rules/schemas/definitions.py      | 44 +++++++++++++++++
 detection_rules/schemas/registry_package.py | 54 +++++++++++++++++++++
 detection_rules/semver.py                   |  9 +++-
 detection_rules/utils.py                    |  4 ++
 etc/packages.yml                            | 33 ++++++++-----
 etc/rules-changelog.json                    |  3 ++
 requirements.txt                            |  2 +
 11 files changed, 189 insertions(+), 28 deletions(-)
 create mode 100644 detection_rules/schemas/definitions.py
 create mode 100644 detection_rules/schemas/registry_package.py
 create mode 100644 etc/rules-changelog.json

diff --git a/detection_rules/misc.py b/detection_rules/misc.py
index 35d5da54fc4..63dffd57a61 100644
--- a/detection_rules/misc.py
+++ b/detection_rules/misc.py
@@ -480,7 +480,10 @@ def _convert_type(_val):
                         break
                     else:
                         return []
-            return [_convert_type(r) for r in result_list]
+            if required and value is None:
+                continue
+            else:
+                return [_convert_type(r) for r in result_list]
         else:
             if _check_type(result):
                 return _convert_type(result)
diff --git a/detection_rules/packaging.py b/detection_rules/packaging.py
index 63fe1e4b321..4f20ac88f99 100644
--- a/detection_rules/packaging.py
+++ b/detection_rules/packaging.py
@@ -23,6 +23,7 @@
 RELEASE_DIR = get_path("releases")
 PACKAGE_FILE = get_etc_path('packages.yml')
 NOTICE_FILE = get_path('NOTICE.txt')
+CHANGELOG_FILE = Path(get_etc_path('rules-changelog.json'))
 
 
 def filter_rule(rule: Rule, config_filter: dict, exclude_fields: dict) -> bool:
@@ -149,13 +150,15 @@ def manage_versions(rules: list, deprecated_rules: list = None, current_versions
 class Package(object):
     """Packaging object for siem rules and releases."""
 
-    def __init__(self, rules, name, deprecated_rules=None, release=False, current_versions=None, min_version=None,
-                 max_version=None, update_version_lock=False, verbose=True):
+    def __init__(self, rules: List[Rule], name, deprecated_rules: List[Rule] = None, release=False,
+                 current_versions: dict = None, min_version: int = None, max_version: int = None,
+                 update_version_lock=False, registry_data: dict = None, verbose=True):
         """Initialize a package."""
-        self.rules: List[Rule] = [r.copy() for r in rules]
+        self.rules = [r.copy() for r in rules]
         self.name = name
-        self.deprecated_rules: List[Rule] = [r.copy() for r in deprecated_rules or []]
+        self.deprecated_rules = [r.copy() for r in deprecated_rules or []]
         self.release = release
+        self.registry_data = registry_data or {}
 
         self.changed_rule_ids, self.new_rules_ids, self.removed_rule_ids = self._add_versions(current_versions,
                                                                                               update_version_lock,
@@ -256,6 +259,9 @@ def save(self, verbose=True):
         self._package_kibana_index_file(rules_dir)
 
         if self.release:
+            if self.registry_data:
+                self._generate_registry_package(save_dir)
+
             self.save_release_files(extras_dir, self.changed_rule_ids, self.new_rules_ids, self.removed_rule_ids)
 
             # zip all rules only and place in extras
@@ -460,6 +466,39 @@ def generate_xslx(self, path):
         doc.populate()
         doc.close()
 
+    def _generate_registry_package(self, save_dir):
+        """Generate the artifact for the oob package-storage."""
+        from .schemas.registry_package import get_manifest
+
+        assert self.registry_data
+
+        registry_manifest = get_manifest(self.registry_data['format_version'])
+        manifest = registry_manifest.Schema().load(self.registry_data)
+
+        package_dir = Path(save_dir).joinpath(manifest.version)
+        docs_dir = package_dir.joinpath('docs')
+        rules_dir = package_dir.joinpath('kibana', 'rules')
+
+        docs_dir.mkdir(parents=True)
+        rules_dir.mkdir(parents=True)
+
+        manifest_file = package_dir.joinpath('manifest.yml')
+        readme_file = docs_dir.joinpath('README.md')
+
+        manifest_file.write_text(json.dumps(manifest.dump(), indent=2, sort_keys=True))
+        shutil.copyfile(CHANGELOG_FILE, str(rules_dir.joinpath('CHANGELOG.json')))
+
+        for rule in self.rules:
+            rule.save(new_path=str(rules_dir.joinpath(f'rule-{rule.id}.json')))
+
+        readme_text = '# Detection rules\n'
+        readme_text += '\n'
+        readme_text += 'The detection rules package is a non-integration package to store all the rules and '
+        readme_text += 'dependencies (e.g. ML jobs) for the detection engine within the Elastic Security application.\n'
+        readme_text += '\n'
+
+        readme_file.write_text(readme_text)
+
     def bump_versions(self, save_changes=False, current_versions=None):
         """Bump the versions of all production rules included in a release and optionally save changes."""
         return manage_versions(self.rules, current_versions=current_versions, save_changes=save_changes)
diff --git a/detection_rules/rule_loader.py b/detection_rules/rule_loader.py
index 5abe09a8eff..832e8db0719 100644
--- a/detection_rules/rule_loader.py
+++ b/detection_rules/rule_loader.py
@@ -118,7 +118,7 @@ def load_rules(file_lookup=None, verbose=True, error=True):
 
         except Exception as e:
             failed = True
-            err_msg = "Invalid rule file in {}\n{}".format(rule_file, click.style(e.args[0], fg='red'))
+            err_msg = "Invalid rule file in {}\n{}".format(rule_file, click.style(str(e), fg='red'))
             errors.append(err_msg)
             if error:
                 if verbose:
diff --git a/detection_rules/schemas/base.py b/detection_rules/schemas/base.py
index d44840f389a..3bb22430364 100644
--- a/detection_rules/schemas/base.py
+++ b/detection_rules/schemas/base.py
@@ -9,16 +9,12 @@
 import jsl
 import jsonschema
 
+from .definitions import (
+    DATE_PATTERN, MATURITY_LEVELS, OS_OPTIONS, UUID_PATTERN, VERSION_PATTERN, VERSION_W_MASTER_PATTERN
+)
 from ..utils import cached
 
 
-DATE_PATTERN = r'\d{4}/\d{2}/\d{2}'
-MATURITY_LEVELS = ['development', 'experimental', 'beta', 'production', 'deprecated']
-OS_OPTIONS = ['windows', 'linux', 'macos', 'solaris']
-UUID_PATTERN = r'[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}'
-VERSION_PATTERN = r'\d+\.\d+\.\d+|master'
-
-
 class MarkdownField(jsl.StringField):
     """Helper class for noting which fields are markdown."""
 
@@ -68,14 +64,14 @@ def strip_additional_properties(cls, document, role=None):
 
 
 class TomlMetadata(GenericSchema):
-    """Schema for siem rule toml metadata."""
+    """Schema for rule toml metadata."""
 
     creation_date = jsl.StringField(required=True, pattern=DATE_PATTERN, default=time.strftime('%Y/%m/%d'))
 
     # rule validated against each ecs schema contained
     beats_version = jsl.StringField(pattern=VERSION_PATTERN, required=False)
     comments = jsl.StringField(required=False)
-    ecs_versions = jsl.ArrayField(jsl.StringField(pattern=VERSION_PATTERN, required=True), required=False)
+    ecs_versions = jsl.ArrayField(jsl.StringField(pattern=VERSION_W_MASTER_PATTERN, required=True), required=False)
     maturity = jsl.StringField(enum=MATURITY_LEVELS, default='development', required=True)
 
     os_type_list = jsl.ArrayField(jsl.StringField(enum=OS_OPTIONS), required=False)
diff --git a/detection_rules/schemas/definitions.py b/detection_rules/schemas/definitions.py
new file mode 100644
index 00000000000..cf38961a865
--- /dev/null
+++ b/detection_rules/schemas/definitions.py
@@ -0,0 +1,44 @@
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+
+"""Custom shared definitions for schemas."""
+
+from typing import ClassVar, Type
+
+import marshmallow
+import marshmallow_dataclass
+from marshmallow_dataclass import NewType
+from marshmallow import validate
+
+
+DATE_PATTERN = r'\d{4}/\d{2}/\d{2}'
+MATURITY_LEVELS = ['development', 'experimental', 'beta', 'production', 'deprecated']
+OS_OPTIONS = ['windows', 'linux', 'macos', 'solaris']
+PR_PATTERN = r'^$|\d+'
+SHA256_PATTERN = r'[a-fA-F0-9]{64}'
+UUID_PATTERN = r'[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}'
+
+_version = r'\d+\.\d+(\.\d+[\w-]*)*'
+CONDITION_VERSION_PATTERN = rf'^\^{_version}$'
+VERSION_PATTERN = f'^{_version}$'
+VERSION_W_MASTER_PATTERN = f'{VERSION_PATTERN}|^master$'
+
+ConditionSemVer = NewType('ConditionSemVer', str, validate=validate.Regexp(CONDITION_VERSION_PATTERN))
+Date = NewType('Date', str, validate=validate.Regexp(DATE_PATTERN))
+SemVer = NewType('SemVer', str, validate=validate.Regexp(VERSION_PATTERN))
+Sha256 = NewType('Sha256', str, validate=validate.Regexp(SHA256_PATTERN))
+Uuid = NewType('Uuid', str, validate=validate.Regexp(UUID_PATTERN))
+
+
+@marshmallow_dataclass.dataclass
+class BaseMarshmallowDataclass:
+    """Base marshmallow dataclass configs."""
+
+    class Meta:
+        ordered = True
+
+    Schema: ClassVar[Type[marshmallow.Schema]] = marshmallow.Schema
+
+    def dump(self) -> dict:
+        return self.Schema().dump(self)
diff --git a/detection_rules/schemas/registry_package.py b/detection_rules/schemas/registry_package.py
new file mode 100644
index 00000000000..539d18cebd7
--- /dev/null
+++ b/detection_rules/schemas/registry_package.py
@@ -0,0 +1,54 @@
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+
+"""Definitions for packages destined for the registry."""
+
+import dataclasses
+from typing import Dict, Union, Type
+
+import marshmallow_dataclass
+from marshmallow import validate
+
+from .definitions import BaseMarshmallowDataclass, ConditionSemVer, SemVer
+
+
+@marshmallow_dataclass.dataclass
+class BaseManifest(BaseMarshmallowDataclass):
+    """Base class for registry packages."""
+
+    conditions: Dict[str, ConditionSemVer]
+    version: SemVer
+    format_version: SemVer
+
+    categories: list = dataclasses.field(default_factory=lambda: ['security'].copy())
+    description: str = 'Rules for the detection engine in the Security application.'
+    icons: list = dataclasses.field(default_factory=list)
+    license: str = 'basic'
+    name: str = 'detection_rules'
+    owner: dict = dataclasses.field(default_factory=lambda: dict(github='elastic/protections').copy())
+    policy_templates: list = dataclasses.field(default_factory=list)
+    release: str = 'experimental'
+    screenshots: list = dataclasses.field(default_factory=list)
+    title: str = 'Detection rules'
+    type: str = 'rules'
+
+
+@marshmallow_dataclass.dataclass
+class ManifestV1Dot0(BaseManifest):
+    """Integrations registry package schema."""
+
+    format_version: SemVer = dataclasses.field(metadata=dict(validate=validate.Equal('1.0.0')), default='1.0.0')
+
+
+manifests = [
+    ManifestV1Dot0
+]
+MANIFESTS = [Type[t] for t in manifests]
+
+
+def get_manifest(format_version: str) -> Union[MANIFESTS]:
+    """Retrieve a manifest class by format_version."""
+    for manifest in manifests:
+        if manifest.format_version == format_version:
+            return manifest
diff --git a/detection_rules/semver.py b/detection_rules/semver.py
index 1cc9ea3e70c..556ac4f5aa2 100644
--- a/detection_rules/semver.py
+++ b/detection_rules/semver.py
@@ -4,14 +4,21 @@
 
 """Helper functionality for comparing semantic versions."""
 import re
+from typing import Iterable, Union
 
 
 class Version(tuple):
 
-    def __new__(cls, version):
+    def __new__(cls, version: Union[Iterable, str], pad: int = None) -> 'Version':
         if not isinstance(version, (int, list, tuple)):
             version = tuple(int(a) if a.isdigit() else a for a in re.split(r'[.-]', version))
 
+            if pad:
+                width = len(version)
+
+                if pad > width:
+                    version = version + (0,) * (pad - width)
+
         return version if isinstance(version, int) else tuple.__new__(cls, version)
 
     def bump(self):
diff --git a/detection_rules/utils.py b/detection_rules/utils.py
index b461b7f9d1f..3a37bb4857a 100644
--- a/detection_rules/utils.py
+++ b/detection_rules/utils.py
@@ -190,6 +190,10 @@ def cached(f):
 
     @functools.wraps(f)
     def wrapped(*args, **kwargs):
+        bypass_cache = kwargs.pop('bypass_cache', None)
+        if bypass_cache:
+            return f(*args, **kwargs)
+
         _cache.setdefault(func_key, {})
         cache_key = freeze(args), freeze(kwargs)
 
diff --git a/etc/packages.yml b/etc/packages.yml
index b5229200bff..26d0070dd0c 100644
--- a/etc/packages.yml
+++ b/etc/packages.yml
@@ -2,22 +2,31 @@
 package:
   name: "7.13"
   release: true
-# exclude rules which have any of the following index <-> field pairs
-#  exclude_fields:
-#    # special field to apply to all indexes
-#    any:
-#    - process.args
-#    - network.direction
-#    logs-endpoint.events.*:
-#    - file.name
+  # exclude rules which have any of the following index <-> field pairs
+  #  exclude_fields:
+  #    # special field to apply to all indexes
+  #    any:
+  #    - process.args
+  #    - network.direction
+  #    logs-endpoint.events.*:
+  #    - file.name
   filter:
     # ecs_version:
     # - 1.4.0
     # - 1.5.0
     maturity:
     - production
-# log deprecated rules in summary and change logs
+  # log deprecated rules in summary and change logs
   log_deprecated: true
-# rule version scoping
-#  min_version: 1
-#  max_version: 5
+  # rule version scoping
+  #  min_version: 1
+  #  max_version: 5
+
+  # Integrations registry
+  registry_data:
+    # integration package schema version
+    format_version: "1.0.0"
+    conditions:
+      kibana_version: "^7.13.0"
+    # this determines the version for the package-storage generated artifact
+    version: "0.0.1-dev.1"
\ No newline at end of file
diff --git a/etc/rules-changelog.json b/etc/rules-changelog.json
new file mode 100644
index 00000000000..d7d299c421d
--- /dev/null
+++ b/etc/rules-changelog.json
@@ -0,0 +1,3 @@
+{
+  "changelog": {}
+}
\ No newline at end of file
diff --git a/requirements.txt b/requirements.txt
index 3edd9e01b93..e49414e7883 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -8,6 +8,8 @@ PyYAML~=5.3
 eql==0.9.9
 elasticsearch~=7.9
 XlsxWriter==1.3.6
+marshmallow==3.6.1
+marshmallow-dataclass==8.3.1
 
 # test deps
 pyflakes==2.2.0

From b3676baa408a0322bbdbf308376209dac88f866a Mon Sep 17 00:00:00 2001
From: brokensound77 <brokensound77@users.noreply.github.com>
Date: Fri, 5 Mar 2021 03:48:09 -0900
Subject: [PATCH 02/11] add tests

---
 detection_rules/schemas/registry_package.py |  3 +-
 tests/test_packages.py                      | 39 ++++++++++++++++++---
 2 files changed, 36 insertions(+), 6 deletions(-)

diff --git a/detection_rules/schemas/registry_package.py b/detection_rules/schemas/registry_package.py
index 539d18cebd7..1ff6acd721a 100644
--- a/detection_rules/schemas/registry_package.py
+++ b/detection_rules/schemas/registry_package.py
@@ -44,10 +44,9 @@ class ManifestV1Dot0(BaseManifest):
 manifests = [
     ManifestV1Dot0
 ]
-MANIFESTS = [Type[t] for t in manifests]
 
 
-def get_manifest(format_version: str) -> Union[MANIFESTS]:
+def get_manifest(format_version: str) -> Union[Type[ManifestV1Dot0]]:
     """Retrieve a manifest class by format_version."""
     for manifest in manifests:
         if manifest.format_version == format_version:
diff --git a/tests/test_packages.py b/tests/test_packages.py
index 608d04f7b15..f6cca20fd96 100644
--- a/tests/test_packages.py
+++ b/tests/test_packages.py
@@ -11,6 +11,10 @@
 from detection_rules.packaging import PACKAGE_FILE, Package
 
 
+with open(PACKAGE_FILE) as f:
+    package_configs = yaml.safe_load(f)['package']
+
+
 class TestPackages(unittest.TestCase):
     """Test package building and saving."""
 
@@ -47,10 +51,7 @@ def test_package_loader_production_config(self):
 
     def test_package_loader_default_configs(self):
         """Test configs in etc/packages.yml."""
-        with open(PACKAGE_FILE) as f:
-            configs = yaml.safe_load(f)['package']
-
-        package = Package.from_config(configs)
+        package = Package.from_config(package_configs)
         for rule in package.rules:
             rule.contents.pop('version')
             rule.validate(as_rule=True)
@@ -147,3 +148,33 @@ def test_version_filter(self):
 
         package = Package(rules, 'test', current_versions=version_info, min_version=2, max_version=2)
         self.assertEqual(1, len(package.rules), msg)
+
+
+class TestRegistryPackage(unittest.TestCase):
+    """Test the OOB registry package."""
+
+    @classmethod
+    def setUpClass(cls) -> None:
+        from detection_rules.schemas.registry_package import get_manifest
+
+        assert 'registry_data' in package_configs, f'Missing registry_data in {PACKAGE_FILE}'
+        cls.registry_config = package_configs['registry_data']
+        assert 'format_version' in cls.registry_config, f'format_version missing from registry_data in {PACKAGE_FILE}'
+
+        cls.format_version = cls.registry_config['format_version']
+        cls.registry_manifest = get_manifest(cls.format_version)
+        assert cls.registry_manifest is not None, f'No registry package schema available for {cls.format_version}'
+
+        cls.registry_manifest.Schema().load(cls.registry_config)
+
+    def test_registry_package_config(self):
+        """Test that the registry package is validating properly."""
+        from marshmallow import ValidationError
+        from detection_rules.schemas.registry_package import get_manifest
+
+        registry_manifest = get_manifest('1.0.0')
+        registry_config = self.registry_config.copy()
+        registry_config['version'] += '7.1.1.'
+
+        with self.assertRaises(ValidationError):
+            registry_manifest.Schema().load(registry_config)

From 3f5833248752caad0ef2b21217105ca958773052 Mon Sep 17 00:00:00 2001
From: brokensound77 <brokensound77@users.noreply.github.com>
Date: Fri, 5 Mar 2021 03:56:56 -0900
Subject: [PATCH 03/11] add trailing newline in packages.yml

---
 etc/packages.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/etc/packages.yml b/etc/packages.yml
index 26d0070dd0c..39ee364bb58 100644
--- a/etc/packages.yml
+++ b/etc/packages.yml
@@ -29,4 +29,4 @@ package:
     conditions:
       kibana_version: "^7.13.0"
     # this determines the version for the package-storage generated artifact
-    version: "0.0.1-dev.1"
\ No newline at end of file
+    version: "0.0.1-dev.1"

From 61ec159e22c74550037b5a332a2d252ad38fad85 Mon Sep 17 00:00:00 2001
From: brokensound77 <brokensound77@users.noreply.github.com>
Date: Fri, 5 Mar 2021 03:58:07 -0900
Subject: [PATCH 04/11] add trailing newlines to files

---
 etc/packages.yml         | 2 +-
 etc/rules-changelog.json | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/etc/packages.yml b/etc/packages.yml
index 26d0070dd0c..39ee364bb58 100644
--- a/etc/packages.yml
+++ b/etc/packages.yml
@@ -29,4 +29,4 @@ package:
     conditions:
       kibana_version: "^7.13.0"
     # this determines the version for the package-storage generated artifact
-    version: "0.0.1-dev.1"
\ No newline at end of file
+    version: "0.0.1-dev.1"
diff --git a/etc/rules-changelog.json b/etc/rules-changelog.json
index d7d299c421d..e36f0d193fe 100644
--- a/etc/rules-changelog.json
+++ b/etc/rules-changelog.json
@@ -1,3 +1,3 @@
 {
   "changelog": {}
-}
\ No newline at end of file
+}

From 4117d744d2686492ed2c432fe23770abacaa2466 Mon Sep 17 00:00:00 2001
From: brokensound77 <brokensound77@users.noreply.github.com>
Date: Fri, 5 Mar 2021 12:57:33 -0900
Subject: [PATCH 05/11] replace marshmallow_dataclass, single manifest class,
 and feedback

---
 detection_rules/devtools.py                 |  2 +-
 detection_rules/packaging.py                | 20 ++++----
 detection_rules/schemas/definitions.py      |  2 +-
 detection_rules/schemas/registry_package.py | 53 +++++++++------------
 detection_rules/semver.py                   |  4 +-
 detection_rules/utils.py                    |  4 --
 etc/rules-changelog.json                    |  3 --
 tests/test_packages.py                      | 19 ++------
 8 files changed, 44 insertions(+), 63 deletions(-)
 delete mode 100644 etc/rules-changelog.json

diff --git a/detection_rules/devtools.py b/detection_rules/devtools.py
index 0edaba793de..983244b1421 100644
--- a/detection_rules/devtools.py
+++ b/detection_rules/devtools.py
@@ -136,7 +136,7 @@ def kibana_commit(ctx, local_repo, github_repo, ssh, kibana_directory, base_bran
     """Prep a commit and push to Kibana."""
     git_exe = shutil.which("git")
 
-    package_name = load_dump(PACKAGE_FILE)['package']["name"]
+    package_name = Package.load_configs()['package']["name"]
     release_dir = os.path.join(RELEASE_DIR, package_name)
     message = message or f"[Detection Rules] Add {package_name} rules"
 
diff --git a/detection_rules/packaging.py b/detection_rules/packaging.py
index 4f20ac88f99..3eecdb79df4 100644
--- a/detection_rules/packaging.py
+++ b/detection_rules/packaging.py
@@ -23,7 +23,7 @@
 RELEASE_DIR = get_path("releases")
 PACKAGE_FILE = get_etc_path('packages.yml')
 NOTICE_FILE = get_path('NOTICE.txt')
-CHANGELOG_FILE = Path(get_etc_path('rules-changelog.json'))
+# CHANGELOG_FILE = Path(get_etc_path('rules-changelog.json'))
 
 
 def filter_rule(rule: Rule, config_filter: dict, exclude_fields: dict) -> bool:
@@ -173,6 +173,11 @@ def _add_versions(self, current_versions, update_versions_lock=False, verbose=Tr
         return manage_versions(self.rules, deprecated_rules=self.deprecated_rules, current_versions=current_versions,
                                save_changes=update_versions_lock, verbose=verbose)
 
+    @classmethod
+    def load_configs(cls):
+        """Load configs from packages.yml."""
+        return load_etc_dump(PACKAGE_FILE)['package']
+
     @staticmethod
     def _package_kibana_notice_file(save_dir):
         """Convert and save notice file with package."""
@@ -468,16 +473,15 @@ def generate_xslx(self, path):
 
     def _generate_registry_package(self, save_dir):
         """Generate the artifact for the oob package-storage."""
-        from .schemas.registry_package import get_manifest
+        from .schemas.registry_package import RegistryPackageManifest
 
         assert self.registry_data
 
-        registry_manifest = get_manifest(self.registry_data['format_version'])
-        manifest = registry_manifest.Schema().load(self.registry_data)
+        manifest = RegistryPackageManifest.from_dict(self.registry_data)
 
         package_dir = Path(save_dir).joinpath(manifest.version)
-        docs_dir = package_dir.joinpath('docs')
-        rules_dir = package_dir.joinpath('kibana', 'rules')
+        docs_dir = package_dir / 'docs'
+        rules_dir = package_dir / 'kibana' / 'rules'
 
         docs_dir.mkdir(parents=True)
         rules_dir.mkdir(parents=True)
@@ -486,14 +490,14 @@ def _generate_registry_package(self, save_dir):
         readme_file = docs_dir.joinpath('README.md')
 
         manifest_file.write_text(json.dumps(manifest.dump(), indent=2, sort_keys=True))
-        shutil.copyfile(CHANGELOG_FILE, str(rules_dir.joinpath('CHANGELOG.json')))
+        # shutil.copyfile(CHANGELOG_FILE, str(rules_dir.joinpath('CHANGELOG.json')))
 
         for rule in self.rules:
             rule.save(new_path=str(rules_dir.joinpath(f'rule-{rule.id}.json')))
 
         readme_text = '# Detection rules\n'
         readme_text += '\n'
-        readme_text += 'The detection rules package is a non-integration package to store all the rules and '
+        readme_text += 'The detection rules package is package to store all the security rules and '
         readme_text += 'dependencies (e.g. ML jobs) for the detection engine within the Elastic Security application.\n'
         readme_text += '\n'
 
diff --git a/detection_rules/schemas/definitions.py b/detection_rules/schemas/definitions.py
index cf38961a865..fadcba13d8c 100644
--- a/detection_rules/schemas/definitions.py
+++ b/detection_rules/schemas/definitions.py
@@ -28,7 +28,7 @@
 Date = NewType('Date', str, validate=validate.Regexp(DATE_PATTERN))
 SemVer = NewType('SemVer', str, validate=validate.Regexp(VERSION_PATTERN))
 Sha256 = NewType('Sha256', str, validate=validate.Regexp(SHA256_PATTERN))
-Uuid = NewType('Uuid', str, validate=validate.Regexp(UUID_PATTERN))
+UUIDString = NewType('UUIDString', str, validate=validate.Regexp(UUID_PATTERN))
 
 
 @marshmallow_dataclass.dataclass
diff --git a/detection_rules/schemas/registry_package.py b/detection_rules/schemas/registry_package.py
index 1ff6acd721a..d842308044c 100644
--- a/detection_rules/schemas/registry_package.py
+++ b/detection_rules/schemas/registry_package.py
@@ -4,50 +4,43 @@
 
 """Definitions for packages destined for the registry."""
 
-import dataclasses
-from typing import Dict, Union, Type
+from dataclasses import dataclass, field
+from typing import Dict, List, Type
 
-import marshmallow_dataclass
-from marshmallow import validate
+from marshmallow import Schema, validate
+from marshmallow_dataclass import class_schema
 
-from .definitions import BaseMarshmallowDataclass, ConditionSemVer, SemVer
+from .definitions import ConditionSemVer, SemVer
 
 
-@marshmallow_dataclass.dataclass
-class BaseManifest(BaseMarshmallowDataclass):
+@dataclass
+class RegistryPackageManifest:
     """Base class for registry packages."""
 
     conditions: Dict[str, ConditionSemVer]
     version: SemVer
-    format_version: SemVer
 
-    categories: list = dataclasses.field(default_factory=lambda: ['security'].copy())
+    categories: List[str] = field(default_factory=lambda: ['security'])
     description: str = 'Rules for the detection engine in the Security application.'
-    icons: list = dataclasses.field(default_factory=list)
+    format_version: SemVer = field(metadata=dict(validate=validate.Equal('1.0.0')), default='1.0.0')
+    icons: list = field(default_factory=list)
+    internal: bool = True
     license: str = 'basic'
     name: str = 'detection_rules'
-    owner: dict = dataclasses.field(default_factory=lambda: dict(github='elastic/protections').copy())
-    policy_templates: list = dataclasses.field(default_factory=list)
+    owner: Dict[str, str] = field(default_factory=lambda: dict(github='elastic/protections').copy())
+    policy_templates: list = field(default_factory=list)
     release: str = 'experimental'
-    screenshots: list = dataclasses.field(default_factory=list)
+    screenshots: list = field(default_factory=list)
     title: str = 'Detection rules'
-    type: str = 'rules'
+    type: str = 'integration'
 
+    @classmethod
+    def get_schema(cls) -> Type[Schema]:
+        return class_schema(cls)
 
-@marshmallow_dataclass.dataclass
-class ManifestV1Dot0(BaseManifest):
-    """Integrations registry package schema."""
+    @classmethod
+    def from_dict(cls, obj: dict) -> 'RegistryPackageManifest':
+        return cls.get_schema()().load(obj)
 
-    format_version: SemVer = dataclasses.field(metadata=dict(validate=validate.Equal('1.0.0')), default='1.0.0')
-
-
-manifests = [
-    ManifestV1Dot0
-]
-
-
-def get_manifest(format_version: str) -> Union[Type[ManifestV1Dot0]]:
-    """Retrieve a manifest class by format_version."""
-    for manifest in manifests:
-        if manifest.format_version == format_version:
-            return manifest
+    def dump(self) -> dict:
+        return self.get_schema()().dump(self)
diff --git a/detection_rules/semver.py b/detection_rules/semver.py
index 556ac4f5aa2..70911a40dfc 100644
--- a/detection_rules/semver.py
+++ b/detection_rules/semver.py
@@ -4,12 +4,12 @@
 
 """Helper functionality for comparing semantic versions."""
 import re
-from typing import Iterable, Union
+from typing import Iterable, Optional, Union
 
 
 class Version(tuple):
 
-    def __new__(cls, version: Union[Iterable, str], pad: int = None) -> 'Version':
+    def __new__(cls, version: Union[Iterable, str], pad: Optional[int] = None) -> 'Version':
         if not isinstance(version, (int, list, tuple)):
             version = tuple(int(a) if a.isdigit() else a for a in re.split(r'[.-]', version))
 
diff --git a/detection_rules/utils.py b/detection_rules/utils.py
index 3a37bb4857a..b461b7f9d1f 100644
--- a/detection_rules/utils.py
+++ b/detection_rules/utils.py
@@ -190,10 +190,6 @@ def cached(f):
 
     @functools.wraps(f)
     def wrapped(*args, **kwargs):
-        bypass_cache = kwargs.pop('bypass_cache', None)
-        if bypass_cache:
-            return f(*args, **kwargs)
-
         _cache.setdefault(func_key, {})
         cache_key = freeze(args), freeze(kwargs)
 
diff --git a/etc/rules-changelog.json b/etc/rules-changelog.json
deleted file mode 100644
index e36f0d193fe..00000000000
--- a/etc/rules-changelog.json
+++ /dev/null
@@ -1,3 +0,0 @@
-{
-  "changelog": {}
-}
diff --git a/tests/test_packages.py b/tests/test_packages.py
index f6cca20fd96..d2f0b4e8941 100644
--- a/tests/test_packages.py
+++ b/tests/test_packages.py
@@ -5,14 +5,12 @@
 """Test that the packages are built correctly."""
 import unittest
 import uuid
-import yaml
 
 from detection_rules import rule_loader
 from detection_rules.packaging import PACKAGE_FILE, Package
 
 
-with open(PACKAGE_FILE) as f:
-    package_configs = yaml.safe_load(f)['package']
+package_configs = Package.load_configs()
 
 
 class TestPackages(unittest.TestCase):
@@ -155,26 +153,19 @@ class TestRegistryPackage(unittest.TestCase):
 
     @classmethod
     def setUpClass(cls) -> None:
-        from detection_rules.schemas.registry_package import get_manifest
+        from detection_rules.schemas.registry_package import RegistryPackageManifest
 
         assert 'registry_data' in package_configs, f'Missing registry_data in {PACKAGE_FILE}'
         cls.registry_config = package_configs['registry_data']
-        assert 'format_version' in cls.registry_config, f'format_version missing from registry_data in {PACKAGE_FILE}'
-
-        cls.format_version = cls.registry_config['format_version']
-        cls.registry_manifest = get_manifest(cls.format_version)
-        assert cls.registry_manifest is not None, f'No registry package schema available for {cls.format_version}'
-
-        cls.registry_manifest.Schema().load(cls.registry_config)
+        RegistryPackageManifest.from_dict(cls.registry_config)
 
     def test_registry_package_config(self):
         """Test that the registry package is validating properly."""
         from marshmallow import ValidationError
-        from detection_rules.schemas.registry_package import get_manifest
+        from detection_rules.schemas.registry_package import RegistryPackageManifest
 
-        registry_manifest = get_manifest('1.0.0')
         registry_config = self.registry_config.copy()
         registry_config['version'] += '7.1.1.'
 
         with self.assertRaises(ValidationError):
-            registry_manifest.Schema().load(registry_config)
+            RegistryPackageManifest.from_dict(registry_config)

From 1756c947e999a120bacb1241dbbe53eabed5f403 Mon Sep 17 00:00:00 2001
From: Ross Wolf <31489089+rw-access@users.noreply.github.com>
Date: Tue, 9 Mar 2021 11:40:06 -0700
Subject: [PATCH 06/11] License update

---
 detection_rules/schemas/definitions.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/detection_rules/schemas/definitions.py b/detection_rules/schemas/definitions.py
index fadcba13d8c..f6d0d470186 100644
--- a/detection_rules/schemas/definitions.py
+++ b/detection_rules/schemas/definitions.py
@@ -1,6 +1,7 @@
 # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
-# or more contributor license agreements. Licensed under the Elastic License;
-# you may not use this file except in compliance with the Elastic License.
+# or more contributor license agreements. Licensed under the Elastic License
+# 2.0; you may not use this file except in compliance with the Elastic License
+# 2.0.
 
 """Custom shared definitions for schemas."""
 

From 9766145ce52ee1d8a33ddcb982bee26245049413 Mon Sep 17 00:00:00 2001
From: Ross Wolf <31489089+rw-access@users.noreply.github.com>
Date: Tue, 9 Mar 2021 11:41:22 -0700
Subject: [PATCH 07/11] License update

---
 detection_rules/schemas/registry_package.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/detection_rules/schemas/registry_package.py b/detection_rules/schemas/registry_package.py
index d842308044c..00f161c1507 100644
--- a/detection_rules/schemas/registry_package.py
+++ b/detection_rules/schemas/registry_package.py
@@ -1,6 +1,7 @@
 # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
-# or more contributor license agreements. Licensed under the Elastic License;
-# you may not use this file except in compliance with the Elastic License.
+# or more contributor license agreements. Licensed under the Elastic License
+# 2.0; you may not use this file except in compliance with the Elastic License
+# 2.0.
 
 """Definitions for packages destined for the registry."""
 

From 49b5c7e576257a012d198d1541ff042173a4c2d3 Mon Sep 17 00:00:00 2001
From: brokensound77 <brokensound77@users.noreply.github.com>
Date: Tue, 9 Mar 2021 12:40:31 -0900
Subject: [PATCH 08/11] update typing and other feedback tweaks

---
 detection_rules/packaging.py                | 18 ++++++++----------
 detection_rules/rule.py                     |  2 +-
 detection_rules/schemas/registry_package.py |  4 ++--
 detection_rules/semver.py                   | 11 ++---------
 4 files changed, 13 insertions(+), 22 deletions(-)

diff --git a/detection_rules/packaging.py b/detection_rules/packaging.py
index 3eecdb79df4..1112442ee54 100644
--- a/detection_rules/packaging.py
+++ b/detection_rules/packaging.py
@@ -11,7 +11,7 @@
 import shutil
 from collections import defaultdict, OrderedDict
 from pathlib import Path
-from typing import List, Tuple
+from typing import List, Optional, Tuple
 
 import click
 
@@ -150,9 +150,11 @@ def manage_versions(rules: list, deprecated_rules: list = None, current_versions
 class Package(object):
     """Packaging object for siem rules and releases."""
 
-    def __init__(self, rules: List[Rule], name, deprecated_rules: List[Rule] = None, release=False,
-                 current_versions: dict = None, min_version: int = None, max_version: int = None,
-                 update_version_lock=False, registry_data: dict = None, verbose=True):
+    def __init__(self, rules: List[Rule], name: str, deprecated_rules: Optional[List[Rule]] = None,
+                 release: Optional[bool] = False, current_versions: Optional[dict] = None,
+                 min_version: Optional[int] = None, max_version: Optional[int] = None,
+                 update_version_lock: Optional[bool] = False, registry_data: Optional[dict] = None,
+                 verbose: Optional[bool] = True):
         """Initialize a package."""
         self.rules = [r.copy() for r in rules]
         self.name = name
@@ -264,9 +266,7 @@ def save(self, verbose=True):
         self._package_kibana_index_file(rules_dir)
 
         if self.release:
-            if self.registry_data:
-                self._generate_registry_package(save_dir)
-
+            self._generate_registry_package(save_dir)
             self.save_release_files(extras_dir, self.changed_rule_ids, self.new_rules_ids, self.removed_rule_ids)
 
             # zip all rules only and place in extras
@@ -475,8 +475,6 @@ def _generate_registry_package(self, save_dir):
         """Generate the artifact for the oob package-storage."""
         from .schemas.registry_package import RegistryPackageManifest
 
-        assert self.registry_data
-
         manifest = RegistryPackageManifest.from_dict(self.registry_data)
 
         package_dir = Path(save_dir).joinpath(manifest.version)
@@ -489,7 +487,7 @@ def _generate_registry_package(self, save_dir):
         manifest_file = package_dir.joinpath('manifest.yml')
         readme_file = docs_dir.joinpath('README.md')
 
-        manifest_file.write_text(json.dumps(manifest.dump(), indent=2, sort_keys=True))
+        manifest_file.write_text(json.dumps(manifest.asdict(), indent=2, sort_keys=True))
         # shutil.copyfile(CHANGELOG_FILE, str(rules_dir.joinpath('CHANGELOG.json')))
 
         for rule in self.rules:
diff --git a/detection_rules/rule.py b/detection_rules/rule.py
index 841e23bf421..9003ac02ff5 100644
--- a/detection_rules/rule.py
+++ b/detection_rules/rule.py
@@ -57,7 +57,7 @@ def __ne__(self, other):
     def __hash__(self):
         return hash(self.get_hash())
 
-    def copy(self):
+    def copy(self) -> 'Rule':
         return Rule(path=self.path, contents={'rule': self.contents.copy(), 'metadata': self.metadata.copy()})
 
     @property
diff --git a/detection_rules/schemas/registry_package.py b/detection_rules/schemas/registry_package.py
index d842308044c..67437307328 100644
--- a/detection_rules/schemas/registry_package.py
+++ b/detection_rules/schemas/registry_package.py
@@ -27,7 +27,7 @@ class RegistryPackageManifest:
     internal: bool = True
     license: str = 'basic'
     name: str = 'detection_rules'
-    owner: Dict[str, str] = field(default_factory=lambda: dict(github='elastic/protections').copy())
+    owner: Dict[str, str] = field(default_factory=lambda: dict(github='elastic/protections'))
     policy_templates: list = field(default_factory=list)
     release: str = 'experimental'
     screenshots: list = field(default_factory=list)
@@ -42,5 +42,5 @@ def get_schema(cls) -> Type[Schema]:
     def from_dict(cls, obj: dict) -> 'RegistryPackageManifest':
         return cls.get_schema()().load(obj)
 
-    def dump(self) -> dict:
+    def asdict(self) -> dict:
         return self.get_schema()().dump(self)
diff --git a/detection_rules/semver.py b/detection_rules/semver.py
index 70911a40dfc..c14e1ece563 100644
--- a/detection_rules/semver.py
+++ b/detection_rules/semver.py
@@ -4,21 +4,14 @@
 
 """Helper functionality for comparing semantic versions."""
 import re
-from typing import Iterable, Optional, Union
+from typing import Iterable, Union
 
 
 class Version(tuple):
 
-    def __new__(cls, version: Union[Iterable, str], pad: Optional[int] = None) -> 'Version':
+    def __new__(cls, version: Union[Iterable, str]) -> 'Version':
         if not isinstance(version, (int, list, tuple)):
             version = tuple(int(a) if a.isdigit() else a for a in re.split(r'[.-]', version))
-
-            if pad:
-                width = len(version)
-
-                if pad > width:
-                    version = version + (0,) * (pad - width)
-
         return version if isinstance(version, int) else tuple.__new__(cls, version)
 
     def bump(self):

From c2e22d12ad047c31760a5624432e063d099301d6 Mon Sep 17 00:00:00 2001
From: brokensound77 <brokensound77@users.noreply.github.com>
Date: Tue, 9 Mar 2021 12:57:26 -0900
Subject: [PATCH 09/11] dump manifest as yaml instead of json

---
 detection_rules/packaging.py           | 3 ++-
 detection_rules/schemas/base.py        | 4 ++--
 detection_rules/schemas/definitions.py | 2 +-
 3 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/detection_rules/packaging.py b/detection_rules/packaging.py
index f3cb405255a..64d33d728f2 100644
--- a/detection_rules/packaging.py
+++ b/detection_rules/packaging.py
@@ -15,6 +15,7 @@
 from typing import List, Optional, Tuple
 
 import click
+import yaml
 
 from . import rule_loader
 from .misc import JS_LICENSE, cached
@@ -488,7 +489,7 @@ def _generate_registry_package(self, save_dir):
         manifest_file = package_dir.joinpath('manifest.yml')
         readme_file = docs_dir.joinpath('README.md')
 
-        manifest_file.write_text(json.dumps(manifest.asdict(), indent=2, sort_keys=True))
+        manifest_file.write_text(yaml.safe_dump(manifest.asdict()))
         # shutil.copyfile(CHANGELOG_FILE, str(rules_dir.joinpath('CHANGELOG.json')))
 
         for rule in self.rules:
diff --git a/detection_rules/schemas/base.py b/detection_rules/schemas/base.py
index 21f7b362ca7..0768089d665 100644
--- a/detection_rules/schemas/base.py
+++ b/detection_rules/schemas/base.py
@@ -11,7 +11,7 @@
 import jsonschema
 
 from .definitions import (
-    DATE_PATTERN, MATURITY_LEVELS, OS_OPTIONS, UUID_PATTERN, VERSION_PATTERN, VERSION_W_MASTER_PATTERN
+    DATE_PATTERN, MATURITY_LEVELS, OS_OPTIONS, UUID_PATTERN, VERSION_PATTERN, BRANCH_PATTERN
 )
 from ..utils import cached
 
@@ -72,7 +72,7 @@ class TomlMetadata(GenericSchema):
     # rule validated against each ecs schema contained
     beats_version = jsl.StringField(pattern=VERSION_PATTERN, required=False)
     comments = jsl.StringField(required=False)
-    ecs_versions = jsl.ArrayField(jsl.StringField(pattern=VERSION_W_MASTER_PATTERN, required=True), required=False)
+    ecs_versions = jsl.ArrayField(jsl.StringField(pattern=BRANCH_PATTERN, required=True), required=False)
     maturity = jsl.StringField(enum=MATURITY_LEVELS, default='development', required=True)
 
     os_type_list = jsl.ArrayField(jsl.StringField(enum=OS_OPTIONS), required=False)
diff --git a/detection_rules/schemas/definitions.py b/detection_rules/schemas/definitions.py
index f6d0d470186..355b5cbcfbb 100644
--- a/detection_rules/schemas/definitions.py
+++ b/detection_rules/schemas/definitions.py
@@ -23,7 +23,7 @@
 _version = r'\d+\.\d+(\.\d+[\w-]*)*'
 CONDITION_VERSION_PATTERN = rf'^\^{_version}$'
 VERSION_PATTERN = f'^{_version}$'
-VERSION_W_MASTER_PATTERN = f'{VERSION_PATTERN}|^master$'
+BRANCH_PATTERN = f'{VERSION_PATTERN}|^master$'
 
 ConditionSemVer = NewType('ConditionSemVer', str, validate=validate.Regexp(CONDITION_VERSION_PATTERN))
 Date = NewType('Date', str, validate=validate.Regexp(DATE_PATTERN))

From cbc569ac49e354c30a81d29b0f9a91855ec7c0f2 Mon Sep 17 00:00:00 2001
From: brokensound77 <brokensound77@users.noreply.github.com>
Date: Tue, 9 Mar 2021 13:09:48 -0900
Subject: [PATCH 10/11] update registry package readme text

---
 detection_rules/packaging.py | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/detection_rules/packaging.py b/detection_rules/packaging.py
index 64d33d728f2..21865bd306d 100644
--- a/detection_rules/packaging.py
+++ b/detection_rules/packaging.py
@@ -495,11 +495,9 @@ def _generate_registry_package(self, save_dir):
         for rule in self.rules:
             rule.save(new_path=str(rules_dir.joinpath(f'rule-{rule.id}.json')))
 
-        readme_text = '# Detection rules\n'
-        readme_text += '\n'
-        readme_text += 'The detection rules package is package to store all the security rules and '
-        readme_text += 'dependencies (e.g. ML jobs) for the detection engine within the Elastic Security application.\n'
-        readme_text += '\n'
+        readme_text = ('# Detection rules\n\n'
+                       'The detection rules package stores all the security rules '
+                       'for the detection engine within the Elastic Security application.\n\n')
 
         readme_file.write_text(readme_text)
 

From d2c90f3244aa42f23f00ff98881495d451165ecf Mon Sep 17 00:00:00 2001
From: Ross Wolf <31489089+rw-access@users.noreply.github.com>
Date: Tue, 9 Mar 2021 15:25:20 -0700
Subject: [PATCH 11/11] Trigger update?