inital commit of regression tests into main branch

Author: csanders-git

util/regression-tests/CRS_Tests.py

@@ -0,0 +1,57 @@
+from ftw import ruleset, logchecker, testrunner
+import pytest
+import pdb
+import sys
+import re
+import os
+import ConfigParser
+
+def test_crs(ruleset, test, logchecker_obj):
+    runner = testrunner.TestRunner()
+    for stage in test.stages:
+        runner.run_stage(stage, logchecker_obj)
+        
+class FooLogChecker(logchecker.LogChecker):
+    
+    def reverse_readline(self, filename):
+        with open(filename) as f:
+            f.seek(0, os.SEEK_END)
+            position = f.tell()
+            line = ''
+            while position >= 0:
+                f.seek(position)
+                next_char = f.read(1)
+                if next_char == "\n":
+                    yield line[::-1]
+                    line = ''
+                else:
+                    line += next_char
+                position -= 1
+            yield line[::-1]    
+
+    def get_logs(self):
+        import datetime
+        config = ConfigParser.ConfigParser()
+        config.read("settings.ini")
+        log_location = config.get('settings', 'log_location')
+        our_logs = []
+        pattern = re.compile(r"\[([A-Z][a-z]{2} [A-z][a-z]{2} \d{1,2} \d{1,2}\:\d{1,2}\:\d{1,2}\.\d+? \d{4})\]")
+        for lline in self.reverse_readline(log_location):
+            # Extract dates from each line
+            match = re.match(pattern,lline)
+            if match:
+                log_date = match.group(1)
+                # Convert our date
+                log_date = datetime.datetime.strptime(log_date, "%a %b %d %H:%M:%S.%f %Y")
+                ftw_start = self.start
+                ftw_end = self.end
+                # If we have a log date in range
+                if log_date <= ftw_end and log_date >= ftw_start:
+                    our_logs.append(lline)
+                # If our log is from before FTW started stop
+                if(log_date < ftw_start):
+                    break
+        return our_logs
+@pytest.fixture
+def logchecker_obj():
+    return FooLogChecker()

util/regression-tests/CRS_Tests_Journal.py

@@ -0,0 +1,57 @@
+from ftw import ruleset, logchecker, testrunner
+import pytest
+import sys
+import re
+import os
+import ConfigParser
+
+def test_crs(ruleset, test, logchecker_obj, with_journal, tablename):
+    runner = testrunner.TestRunner()
+    for stage in test.stages:
+        runner.run_stage_with_journal(test.ruleset_meta['name'], test, with_journal, tablename, logchecker_obj)
+
+class FooLogChecker(logchecker.LogChecker):
+
+    def reverse_readline(self, filename):
+        with open(filename) as f:
+            f.seek(0, os.SEEK_END)
+            position = f.tell()
+            line = ''
+            while position >= 0:
+                f.seek(position)
+                next_char = f.read(1)
+                if next_char == "\n":
+                    yield line[::-1]
+                    line = ''
+                else:
+                    line += next_char
+                position -= 1
+            yield line[::-1]
+
+    def get_logs(self):
+        import datetime
+        config = ConfigParser.ConfigParser()
+        config.read("settings.ini")
+        log_location = config.get('settings', 'log_location')
+        our_logs = []
+        pattern = re.compile(r"\[([A-Z][a-z]{2} [A-z][a-z]{2} \d{1,2} \d{1,2}\:\d{1,2}\:\d{1,2}\.\d+? \d{4})\]")
+        for lline in self.reverse_readline(log_location):
+            # Extract dates from each line
+            match = re.match(pattern,lline)
+            if match:
+                log_date = match.group(1)
+                # Convert our date
+                log_date = datetime.datetime.strptime(log_date, "%a %b %d %H:%M:%S.%f %Y")
+                ftw_start = self.start
+                ftw_end = self.end
+                # If we have a log date in range
+                if log_date <= ftw_end and log_date >= ftw_start:
+                    our_logs.append(lline)
+                # If our log is from before FTW started stop
+                if(log_date < ftw_start):
+                    break
+        return our_logs
+
+@pytest.fixture
+def logchecker_obj():
+    return FooLogChecker()

util/regression-tests/OWASP-CRS-regressions

@@ -0,0 +1,58 @@
+=====================
+OWASP-CRS-regressions
+=====================
+
+Introduction
+============
+Welcome to the OWASP Core Rule Set regression testing suite. This suite is meant to test specific rules in OWASP CRS version 3. The suite is designed to uses preconfigured IDs that are specific to this version of CRS. The tests themselves can be run without CRS and one would expect the same elements to be blocked, however one must override the default Output parameter in the tests. 
+
+Installation
+============
+The OWASP Core Rule Set project was part of the effort to develop FTW, the Framework for Testing WAFs. As a result, we use this project in order to run our regression testing. FTW is designed to use existing Python testing frameworks to allow for easy to read web based testing, provided in YAML. You can install FTW by from the repository (at https://github.com/fastly/ftw) or by running pip.
+
+```pip install -r requirements.txt```
+
+This will install FTW as a library. It can also be run natively, see the FTW documentation for more detail.
+
+Requirements
+============
+There are Three requirements for running the OWASP CRS regressions.
+
+1. You must have ModSecurity specify the location of your error.log, this is done in the settings.ini file
+2. ModSecurity must be in DetectionOnly (or anomaly scoring) mode
+3. You must disable IP blocking based on previous events
+
+To accomplish 2. and 3. you may use the following rule in your setup.conf:
+
+```
+SecAction "id:900005, \ 
+  phase:1,\
+  nolog, \
+  pass, \
+  ctl:ruleEngine=DetectionOnly,\ 
+  ctl:ruleRemoveById=910000,\
+  setvar:tx.paranoia_level=4,\
+  setvar:tx.crs_validate_utf8_encoding=1,\
+  setvar:tx.arg_name_length=100, \
+  setvar:tx.arg_length=400"
+```
+
+Once these requirements have been met the tests can be run by using pytest.
+
+Running The Tests
+=================
+
+On Windows this will look like:
+-------------------------------
+Single Rule File:
+```py.test.exe -v CRS_Tests.py --rule=tests/test.yaml```
+The Whole Suite:
+```py.test.exe -v CRS_Tests.py --ruledir_recurse=tests/```
+
+On Linux this will look like:
+-----------------------------
+Single Rule File:
+```py.test -v CRS_Tests.py --rule=tests/test.yaml```
+The Whole Suite:
+```py.test -v CRS_Tests.py --ruledir_recurse=tests/```
+

util/regression-tests/README

@@ -0,0 +1 @@
+ftw

util/regression-tests/README.md

@@ -0,0 +1,3 @@
+[settings]
+#log_location = /var/log/httpd/error_log
+log_location = C:\Apache24\logs\error.log