TTP Testing: Add Default Dry-Run Test Case (#475)

Summary:
Pull Request resolved: #475

* Buck will now validate all TTP schema fields (requirements, arguments, metadata, etc) for TTPForge and ForgeArmory. It will validate `steps:` where possible, though explicit test case definition is required in some cases
* Buck will also run all integration tests defined with the TTP `tests:` feature for both TTPForge and ForgeArmory
* TTPForge will auto-generate a dry-run test case for every TTP that does not need arguments specified.
* This replaces the  schema validation prototype previously implemented in ForgeArmory, as with the `--dry-run` flag that I implemented previously we now can validate directly against the TTPForge source-of-truth schema without any need for synchronization to an external schema.

Reviewed By: nicolagiacchetta

Differential Revision: D51896839

fbshipit-source-id: fcfaf8da402382766ac994f83f0e394540e625c9

cmd/test.go

@@ -26,6 +26,7 @@ import (
 	"os/exec"
 	"time"
 
+	"github.com/facebookincubator/ttpforge/pkg/blocks"
 	"github.com/facebookincubator/ttpforge/pkg/logging"
 	"github.com/facebookincubator/ttpforge/pkg/preprocess"
 
@@ -35,14 +36,18 @@ import (
 )
 
 type testCase struct {
-	Name   string            `yaml:"name"`
-	Args   map[string]string `yaml:"args"`
-	DryRun bool              `yaml:"dry_run"`
+	Name        string            `yaml:"name"`
+	Description string            `yaml:"description"`
+	Args        map[string]string `yaml:"args"`
+	DryRun      bool              `yaml:"dry_run"`
 }
 
-type ttpFields struct {
-	Name  string     `yaml:"name"`
-	Cases []testCase `yaml:"tests"`
+// We want to verify everything but the steps themselves against
+// our schema. The steps will, in turn, be validated
+// by the subsequent invocation of the `ttpforge run` command.
+type ttpNonStepFields struct {
+	blocks.PreambleFields `yaml:",inline"`
+	Cases                 []testCase `yaml:"tests"`
 }
 
 // Note - this command cannot be unit tested
@@ -78,6 +83,9 @@ func buildTestCommand(cfg *Config) *cobra.Command {
 }
 
 func runTestsForTTP(ttpAbsPath string, timeoutSeconds int) error {
+	logging.DividerThick()
+	logging.L().Infof("TESTING TTP FILE:")
+	logging.L().Info(ttpAbsPath)
 	// preprocess to separate out the `tests:` section from the `steps:`
 	// section and avoid YAML parsing errors associated with template syntax
 	contents, err := afero.ReadFile(afero.NewOsFs(), ttpAbsPath)
@@ -89,13 +97,19 @@ func runTestsForTTP(ttpAbsPath string, timeoutSeconds int) error {
 		return err
 	}
 
-	// load the test cases - we don't want to load the entire ttp
-	// because that's one of the parts of the code that this
-	// command is trying to test
-	var ttpf ttpFields
+	// load the test cases and preamble fields - we don't load
+	// the steps themselves because that process will
+	// be tested when we call `ttpforge run`
+	var ttpf ttpNonStepFields
 	err = yaml.Unmarshal(preprocessResult.PreambleBytes, &ttpf)
 	if err != nil {
-		return fmt.Errorf("failed to parse `test:` section of TTP file %v: %w", ttpAbsPath, err)
+		return fmt.Errorf("failed to parse TTP file %v: %w", ttpAbsPath, err)
+	}
+
+	// validate as many fields as we can prior to actually
+	// invoking `ttpforge run`
+	if err := ttpf.PreambleFields.Validate(); err != nil {
+		return fmt.Errorf("invalid TTP file %v: %w", ttpAbsPath, err)
 	}
 
 	// look up the path of this binary (ttpforge)
@@ -104,16 +118,28 @@ func runTestsForTTP(ttpAbsPath string, timeoutSeconds int) error {
 		return fmt.Errorf("could not resolve self path (path to current ttpforge binary): %w", err)
 	}
 
+	var testCases []testCase
 	if len(ttpf.Cases) == 0 {
-		logging.L().Warnf("No tests defined in TTP file %v; exiting...", ttpAbsPath)
-		return nil
+		if len(ttpf.ArgSpecs) == 0 {
+			// since this TTP doesn't accept arguments, it doesn't need a test case
+			// to validate its steps - so we can add an implicit dry run case
+			testCases = append(testCases, testCase{
+				Name:        "auto_generated_dry_run",
+				Description: "Auto-generated dry run test case",
+				DryRun:      true,
+			})
+		} else {
+			logging.L().Warnf("No tests defined in TTP file %v; exiting...", ttpAbsPath)
+			return nil
+		}
+	} else {
+		testCases = append(testCases, ttpf.Cases...)
 	}
 
 	// run all cases
 	logging.DividerThick()
-	logging.L().Infof("TTP: %q", ttpf.Name)
-	logging.L().Infof("EXECUTING %v TEST CASE(S)", len(ttpf.Cases))
-	for tcIdx, tc := range ttpf.Cases {
+	logging.L().Infof("EXECUTING %v TEST CASE(S)", len(testCases))
+	for tcIdx, tc := range testCases {
 		logging.DividerThin()
 		logging.L().Infof("RUNNING TEST CASE #%d: %q", tcIdx+1, tc.Name)
 		logging.DividerThin()

integration-tests.sh

@@ -23,4 +23,4 @@ ttpforge_binary=$(realpath "${ttpforge_binary}")
   --arg run_second_step=true
 
 # run all the TTP test cases
-./run-all-ttp-tests.sh "${ttpforge_binary}"
+./run-all-ttp-tests.sh "${ttpforge_binary}" "example-ttps"

pkg/blocks/preamble.go

@@ -0,0 +1,59 @@
+/*
+Copyright © 2023-present, Meta Platforms, Inc. and affiliates
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+package blocks
+
+import (
+	"fmt"
+
+	"github.com/facebookincubator/ttpforge/pkg/args"
+)
+
+// PreambleFields are TTP fields that can be parsed
+// prior to rendering the TTP steps with `text/template`
+//
+// **Attributes:**
+//
+// Name: The name of the TTP.
+// Description: A description of the TTP.
+// MitreAttackMapping: A MitreAttack object containing mappings to the MITRE ATT&CK framework.
+// Requirements: The Requirements to run the TTP
+// ArgSpecs: An slice of argument specifications for the TTP.
+type PreambleFields struct {
+	Name               string              `yaml:"name,omitempty"`
+	Description        string              `yaml:"description"`
+	MitreAttackMapping *MitreAttack        `yaml:"mitre,omitempty"`
+	Requirements       *RequirementsConfig `yaml:"requirements,omitempty"`
+	ArgSpecs           []args.Spec         `yaml:"args,omitempty,flow"`
+}
+
+// Validate validates the preamble fields.
+// It is used by both `ttpforge run` and `ttpforge test`
+func (pf *PreambleFields) Validate() error {
+	// validate MITRE mapping
+	if pf.MitreAttackMapping != nil && len(pf.MitreAttackMapping.Tactics) == 0 {
+		return fmt.Errorf("TTP '%s' has a MitreAttackMapping but no Tactic is defined", pf.Name)
+	}
+
+	// validate requirements
+	if err := pf.Requirements.Validate(); err != nil {
+		return fmt.Errorf("TTP '%s' has an invalid requirements section: %w", pf.Name, err)
+	}
+	return nil
+}

pkg/blocks/ttps.go

@@ -26,7 +26,6 @@ import (
 	"runtime"
 	"time"
 
-	"github.com/facebookincubator/ttpforge/pkg/args"
 	"github.com/facebookincubator/ttpforge/pkg/checks"
 	"github.com/facebookincubator/ttpforge/pkg/logging"
 	"github.com/facebookincubator/ttpforge/pkg/platforms"
@@ -39,21 +38,13 @@ import (
 //
 // **Attributes:**
 //
-// Name: The name of the TTP.
-// Description: A description of the TTP.
-// MitreAttackMapping: A MitreAttack object containing mappings to the MITRE ATT&CK framework.
 // Environment: A map of environment variables to be set for the TTP.
 // Steps: An slice of steps to be executed for the TTP.
-// ArgSpecs: An slice of argument specifications for the TTP.
 // WorkDir: The working directory for the TTP.
 type TTP struct {
-	Name               string              `yaml:"name,omitempty"`
-	Description        string              `yaml:"description"`
-	MitreAttackMapping *MitreAttack        `yaml:"mitre,omitempty"`
-	Environment        map[string]string   `yaml:"env,flow,omitempty"`
-	Requirements       *RequirementsConfig `yaml:"requirements,omitempty"`
-	Steps              []Step              `yaml:"steps,omitempty,flow"`
-	ArgSpecs           []args.Spec         `yaml:"args,omitempty,flow"`
+	PreambleFields `yaml:",inline"`
+	Environment    map[string]string `yaml:"env,flow,omitempty"`
+	Steps          []Step            `yaml:"steps,omitempty,flow"`
 	// Omit WorkDir, but expose for testing.
 	WorkDir string `yaml:"-"`
 }
@@ -137,16 +128,13 @@ func reduceIndentation(b []byte, n int) []byte {
 func (t *TTP) Validate(execCtx TTPExecutionContext) error {
 	logging.L().Debugf("Validating TTP %q...", t.Name)
 
-	// validate MITRE mapping
-	if t.MitreAttackMapping != nil && len(t.MitreAttackMapping.Tactics) == 0 {
-		return fmt.Errorf("TTP '%s' has a MitreAttackMapping but no Tactic is defined", t.Name)
-	}
-
-	// validate requirements
-	if err := t.Requirements.Validate(); err != nil {
-		return fmt.Errorf("TTP '%s' has an invalid requirements section: %w", t.Name, err)
+	// Validate preamble fields
+	err := t.PreambleFields.Validate()
+	if err != nil {
+		return err
 	}
 
+	// Validate steps
 	for _, step := range t.Steps {
 		stepCopy := step
 		if err := stepCopy.Validate(execCtx); err != nil {
@@ -189,7 +177,6 @@ func (t *TTP) chdir() (func(), error) {
 // *StepResultsRecord: A StepResultsRecord containing the results of each step.
 // error: An error if any of the steps fail to execute.
 func (t *TTP) Execute(execCtx *TTPExecutionContext) (*StepResultsRecord, error) {
-	logging.DividerThick()
 	logging.L().Infof("RUNNING TTP: %v", t.Name)
 
 	// verify that we actually meet the necessary requirements to execute this TTP

run-all-ttp-tests.sh

@@ -1,5 +1,26 @@
 #!/bin/bash
 set -e
 
-cd "$(dirname "$0")"
-"$1" test example-ttps/**/*.yaml
+# validate first argument
+TTPFORGE_BINARY="$1"
+if [ ! -f "${TTPFORGE_BINARY}" ]
+then
+  echo "Invalid TTPForge Binary Path Specified!"
+  exit 1
+fi
+TTPFORGE_BINARY=$(realpath "${TTPFORGE_BINARY}")
+
+# validate second argument
+TTP_DIR="$2"
+if [ ! -d "${TTP_DIR}" ]
+then
+  echo "Invalid TTP Directory Specified!"
+  exit 1
+fi
+TTP_DIR=$(realpath "${TTP_DIR}")
+
+TTP_FILE_LIST="$(find "${TTP_DIR}" -name "*.yaml")"
+for TTP_FILE in ${TTP_FILE_LIST}
+do
+    ${TTPFORGE_BINARY} test "${TTP_FILE}"
+done