diff --git a/pkg/catalog/index/index.go b/pkg/catalog/index/index.go index aec8c742c3..f498640fdd 100644 --- a/pkg/catalog/index/index.go +++ b/pkg/catalog/index/index.go @@ -20,7 +20,7 @@ const ( // IndexVersion is the schema version for cache invalidation on breaking // changes. - IndexVersion = 1 + IndexVersion = 2 // DefaultMaxSize is the default maximum number of templates to cache. DefaultMaxSize = 50000 @@ -72,6 +72,8 @@ func NewIndex(cacheDir string) (*Index, error) { weight += len(value.Severity) weight += len(value.ProtocolType) weight += len(value.TemplateVerifier) + weight += len(value.VerifierFingerprint) + weight += len(value.ContentDigest) for _, author := range value.Authors { weight += len(author) diff --git a/pkg/catalog/index/metadata.go b/pkg/catalog/index/metadata.go index d1563a6479..b181a78543 100644 --- a/pkg/catalog/index/metadata.go +++ b/pkg/catalog/index/metadata.go @@ -54,6 +54,13 @@ type Metadata struct { // TemplateVerifier is the verifier used for the template. TemplateVerifier string `gob:"verifier,omitempty"` + // VerifierFingerprint identifies the public key that verified the template. + VerifierFingerprint [32]byte `gob:"verifier_fingerprint,omitempty"` + + // ContentDigest binds the cached verification result to the content that + // was verified. + ContentDigest [32]byte `gob:"content_digest,omitempty"` + // Validation records how the built-in parser validated this metadata's // source before it was cached. Validation ValidationMode `gob:"validation,omitempty"` @@ -108,8 +115,10 @@ func NewMetadataFromTemplate(path string, tpl *templates.Template) *Metadata { ProtocolType: tpl.Type().String(), - Verified: tpl.Verified, - TemplateVerifier: tpl.TemplateVerifier, + Verified: tpl.Verified, + TemplateVerifier: tpl.TemplateVerifier, + VerifierFingerprint: tpl.VerifierFingerprint(), + ContentDigest: tpl.ContentDigest(), } } diff --git a/pkg/catalog/loader/loader.go b/pkg/catalog/loader/loader.go index 2b825b424c..f66b5f5023 100644 --- a/pkg/catalog/loader/loader.go +++ b/pkg/catalog/loader/loader.go @@ -272,8 +272,10 @@ func getTemplateVerification(metadataIndex *index.Index, templatePath string) *p } return &protocols.TemplateVerification{ - Verified: metadata.Verified, - Verifier: metadata.TemplateVerifier, + Verified: metadata.Verified, + Verifier: metadata.TemplateVerifier, + VerifierFingerprint: metadata.VerifierFingerprint, + ContentDigest: metadata.ContentDigest, } } @@ -891,7 +893,12 @@ func (store *Store) LoadTemplatesWithTags(templatesList, tags []string) ([]*temp if loaded { parsed, err := templates.Parse(templatePath, store.preprocessor, store.config.ExecutorOptions) - if parsed != nil && !metadataReusable { + verificationChanged := parsed != nil && (metadata == nil || + metadata.Verified != parsed.Verified || + metadata.TemplateVerifier != parsed.TemplateVerifier || + metadata.VerifierFingerprint != parsed.VerifierFingerprint() || + metadata.ContentDigest != parsed.ContentDigest()) + if parsed != nil && (!metadataReusable || verificationChanged) { if store.metadataIndex != nil { metadata = store.cacheValidatedMetadata(templatePath, parsed) } else { diff --git a/pkg/catalog/loader/loader_test.go b/pkg/catalog/loader/loader_test.go index a2c71c8b01..2ec0a4fe12 100644 --- a/pkg/catalog/loader/loader_test.go +++ b/pkg/catalog/loader/loader_test.go @@ -6,6 +6,7 @@ import ( "path/filepath" "reflect" "testing" + "time" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/gologger/formatter" @@ -500,6 +501,77 @@ javascript: require.Equal(t, initialUnverified, stats.GetValue(templates.SkippedUnverifiedTemplateStats)) } +func TestLoadTemplatesReverifiesCachedJavascriptTemplate(t *testing.T) { + templatePath := filepath.Join(t.TempDir(), "cached-javascript.yaml") + verifiedModTime := time.Now().Add(-time.Hour) + require.NoError(t, os.WriteFile(templatePath, []byte(`id: cached-javascript + +info: + name: Cached Javascript + author: pdteam + severity: info + +javascript: + - init: | + set("init-status", "executed") + code: | + Export("cached-javascript") +`), 0o600)) + require.NoError(t, os.Chtimes(templatePath, verifiedModTime, verifiedModTime)) + fileInfo, err := os.Stat(templatePath) + require.NoError(t, err) + + metadataIndex, err := metadataindex.NewIndex(t.TempDir()) + require.NoError(t, err) + metadataIndex.Set(templatePath, &metadataindex.Metadata{ + ID: "cached-javascript", + FilePath: templatePath, + ModTime: fileInfo.ModTime(), + Name: "Cached Javascript", + Authors: []string{"pdteam"}, + Severity: "info", + ProtocolType: "javascript", + Verified: true, + TemplateVerifier: "projectdiscovery/nuclei-templates", + ContentDigest: [32]byte{1}, + Validation: metadataindex.ValidationStrict, + }) + + options := testutils.DefaultOptions.Copy() + options.Logger = &gologger.Logger{} + options.ExecutionId = "loader-cached-javascript" + options.DisableUnsignedTemplates = false + options.TemplateLoadingConcurrency = 1 + testutils.Init(options) + t.Cleanup(func() { + testutils.Cleanup(options) + }) + + catalog := disk.NewCatalog("") + executerOpts := testutils.NewMockExecuterOptions(options, nil) + executerOpts.Catalog = catalog + executerOpts.Parser = templates.NewParser() + executerOpts.Logger = options.Logger + + workflowLoader, err := workflow.NewLoader(executerOpts) + require.NoError(t, err) + executerOpts.WorkflowLoader = workflowLoader + + loaderConfig := NewConfig(options, catalog, executerOpts) + loaderConfig.MetadataIndex = metadataIndex + store, err := New(loaderConfig) + require.NoError(t, err) + + loaded, err := store.LoadTemplates([]string{templatePath}) + require.NoError(t, err) + require.Empty(t, loaded, "mtime-only metadata must not authorize javascript execution") + + refreshedMetadata, found := metadataIndex.Get(templatePath) + require.True(t, found) + require.False(t, refreshedMetadata.Verified) + require.NotEqual(t, [32]byte{1}, refreshedMetadata.ContentDigest) +} + func TestLoadTemplatesTreatsMixedTemplateWithJavascriptAsJavascriptSensitive(t *testing.T) { templatePath := filepath.Join(t.TempDir(), "mixed-javascript.yaml") err := os.WriteFile(templatePath, []byte(`id: mixed-javascript-template diff --git a/pkg/protocols/javascript/js.go b/pkg/protocols/javascript/js.go index 53eaa3796a..9d063528fa 100644 --- a/pkg/protocols/javascript/js.go +++ b/pkg/protocols/javascript/js.go @@ -142,7 +142,18 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error { } } + var initCompiled *goja.Program + if request.Init != "" { + // Validate init syntax for every template, but do not execute unsigned + // template code during compilation. + initCompiled, err = compiler.SourceAutoMode(request.Init, false) + if err != nil { + return errkit.Newf("could not compile init code: %s", err) + } + } + + if initCompiled != nil && request.options.Verified { // execute init code if any if request.options.Options.Debug || request.options.Options.DebugRequests { gologger.Debug().Msgf("[%s] Executing Template Init\n", request.TemplateID) @@ -220,10 +231,6 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error { // proceed with whatever args we have args.Args, _, _ = request.evaluateArgs(allVars, options, true) - initCompiled, err := compiler.SourceAutoMode(request.Init, false) - if err != nil { - return errkit.Newf("could not compile init code: %s", err) - } result, err := request.options.JsCompiler.ExecuteWithOptions(context.Background(), initCompiled, args, opts) if err != nil { return errkit.Newf("could not execute pre-condition: %s", err) @@ -285,6 +292,10 @@ func (request *Request) GetID() string { // ExecuteWithResults executes the protocol requests and returns results instead of writing them. func (request *Request) ExecuteWithResults(target *contextargs.Context, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error { + if request.options == nil || !request.options.Verified { + return errkit.New("refusing to execute unverified javascript template; sign it (-sign) or run a verified template") + } + // Get default port(s) if specified in template ports := request.getPorts() if len(ports) == 0 { diff --git a/pkg/protocols/javascript/js_test.go b/pkg/protocols/javascript/js_test.go index 6c6bb27ba8..9c424b8d2e 100644 --- a/pkg/protocols/javascript/js_test.go +++ b/pkg/protocols/javascript/js_test.go @@ -6,6 +6,7 @@ import ( "testing" "time" + "github.com/projectdiscovery/nuclei/v3/internal/tests/testutils" "github.com/projectdiscovery/nuclei/v3/pkg/catalog/config" "github.com/projectdiscovery/nuclei/v3/pkg/catalog/disk" "github.com/projectdiscovery/nuclei/v3/pkg/loader/workflow" @@ -15,7 +16,6 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs" javascript "github.com/projectdiscovery/nuclei/v3/pkg/protocols/javascript" "github.com/projectdiscovery/nuclei/v3/pkg/templates" - "github.com/projectdiscovery/nuclei/v3/internal/tests/testutils" "github.com/projectdiscovery/ratelimit" "github.com/stretchr/testify/require" ) @@ -89,6 +89,7 @@ func TestExecuteWithResultsReturnsArgEvaluationErrorWithoutPanic(t *testing.T) { executorOptions := testutils.NewMockExecuterOptions(options, tmplInfo) executorOptions.JsCompiler = templates.GetJsCompiler() + executorOptions.Verified = true request := &javascript.Request{ Args: map[string]interface{}{ @@ -108,3 +109,23 @@ func TestExecuteWithResultsReturnsArgEvaluationErrorWithoutPanic(t *testing.T) { }) require.ErrorContains(t, err, `failed to evaluate expression "base64()"`) } + +func TestExecuteWithResultsRejectsUnverifiedTemplate(t *testing.T) { + options := testutils.DefaultOptions.Copy() + testutils.Init(options) + t.Cleanup(func() { + testutils.Cleanup(options) + }) + + executorOptions := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ID: "unverified-javascript"}) + executorOptions.JsCompiler = templates.GetJsCompiler() + + request := &javascript.Request{Code: `module.exports = { success: true, response: "unexpected" }`} + require.NoError(t, request.Compile(executorOptions)) + + target := contextargs.NewWithInput(context.Background(), "https://example.com:443") + err := request.ExecuteWithResults(target, nil, nil, func(*output.InternalWrappedEvent) { + t.Fatal("unexpected callback for unverified javascript template") + }) + require.ErrorContains(t, err, "refusing to execute unverified javascript template") +} diff --git a/pkg/protocols/protocols.go b/pkg/protocols/protocols.go index 8f7fb4cf71..6f0da903ed 100644 --- a/pkg/protocols/protocols.go +++ b/pkg/protocols/protocols.go @@ -59,8 +59,13 @@ type Executer interface { // TemplateVerification holds cached verification information for a template. type TemplateVerification struct { - Verified bool - Verifier string + Verified bool + Verifier string + VerifierFingerprint [32]byte + + // ContentDigest binds the cached result to the verified template and + // imported-file contents. + ContentDigest [32]byte } // ExecutorOptions contains the configuration options for executer clients @@ -73,9 +78,8 @@ type ExecutorOptions struct { TemplateInfo model.Info // TemplateVerifier is the verifier for the template TemplateVerifier string - // Verified reports whether the template's signature was successfully - // verified by a trusted verifier. It is checked by the code protocol at - // execution time. + // Verified reports whether a trusted verifier verified the template's + // signature. Code and JavaScript protocols check it at execution time. Verified bool // TemplateVerificationCallback returns cached verification info for a template path. // If it returns nil, verification should be computed normally. diff --git a/pkg/templates/compile.go b/pkg/templates/compile.go index 0478828476..00d423559b 100644 --- a/pkg/templates/compile.go +++ b/pkg/templates/compile.go @@ -1,6 +1,7 @@ package templates import ( + "crypto/sha256" "fmt" "io" "reflect" @@ -177,6 +178,7 @@ func Parse(filePath string, preprocessor Preprocessor, options *protocols.Execut newBase.TemplatePath = tplCopy.Options.TemplatePath newBase.TemplateInfo = tplCopy.Options.TemplateInfo newBase.TemplateVerifier = tplCopy.Options.TemplateVerifier + newBase.Verified = tplCopy.Options.Verified newBase.RawTemplate = tplCopy.Options.RawTemplate if tplCopy.Options.Variables.Len() > 0 { @@ -482,7 +484,9 @@ func ParseTemplateFromReader(reader io.Reader, preprocessor Preprocessor, option // add generated constants to constants map and executer options template.Constants = generators.MergeMaps(template.Constants, generatedConstants) template.Options.Constants = template.Constants - applyTemplateVerification(template, data) + if err := verifyAndCompileTemplate(template, data); err != nil { + return nil, err + } if !template.Verified && len(template.Workflows) == 0 { // workflows are not signed by default @@ -500,18 +504,30 @@ func parseTemplate(data []byte, srcOptions *protocols.ExecutorOptions) (*Templat if err != nil { return nil, err } - applyTemplateVerification(template, data) + + if err := verifyAndCompileTemplate(template, data); err != nil { + return nil, err + } return template, nil } +// verifyAndCompileTemplate keeps signature verification before protocol +// compilation because JavaScript init blocks execute during compilation. +func verifyAndCompileTemplate(template *Template, data []byte) error { + applyTemplateVerification(template, data) + + return compileTemplate(template) +} + // parseTemplateNoVerify parses the template without applying any verification. func parseTemplateNoVerify(data []byte, srcOptions *protocols.ExecutorOptions) (*Template, error) { // Create a copy of the options specifically for this template options := srcOptions.Copy() - template := &Template{} + var err error + switch config.GetTemplateFormatFromExt(template.Path) { case config.JSON: err = json.Unmarshal(data, template) @@ -523,6 +539,7 @@ func parseTemplateNoVerify(data []byte, srcOptions *protocols.ExecutorOptions) ( return nil, err } } + if err != nil { return nil, errkit.Wrapf(err, "failed to parse %s", template.Path) } @@ -530,6 +547,7 @@ func parseTemplateNoVerify(data []byte, srcOptions *protocols.ExecutorOptions) ( if utils.IsBlank(template.Info.Name) { return nil, errors.New("no template name field provided") } + if template.Info.Authors.IsEmpty() { return nil, errors.New("no template author field provided") } @@ -569,13 +587,13 @@ func parseTemplateNoVerify(data []byte, srcOptions *protocols.ExecutorOptions) ( options.CreateTemplateCtxStore() options.ProtocolType = template.Type() options.Constants = template.Constants - // initialize the js compiler if missing if options.JsCompiler == nil { options.JsCompiler = GetJsCompiler() // this is a singleton } template.Options = options + // If no requests, and it is also not a workflow, return error. if template.Requests() == 0 { return nil, fmt.Errorf("no requests defined for %s", template.ID) @@ -587,22 +605,31 @@ func parseTemplateNoVerify(data []byte, srcOptions *protocols.ExecutorOptions) ( return nil, errkit.Wrapf(err, "failed to load file refs for %s", template.ID) } + return template, nil +} + +// compileTemplate prepares a parsed template for execution after its signature +// verification result is available to protocol compilers. +func compileTemplate(template *Template) error { if err := template.compileProtocolRequests(template.Options); err != nil { - return nil, err + return err } if template.Executer != nil { if err := template.Executer.Compile(); err != nil { - return nil, errors.Wrap(err, "could not compile request") + return errors.Wrap(err, "could not compile request") } + template.TotalRequests = template.Executer.Requests() } + if template.Executer == nil && template.CompiledWorkflow == nil { - return nil, ErrCreateTemplateExecutor + return ErrCreateTemplateExecutor } + template.parseSelfContainedRequests() - return template, nil + return nil } // applyTemplateVerification verifies a parsed template against the provided data. @@ -612,36 +639,47 @@ func applyTemplateVerification(template *Template, data []byte) { } options := template.Options + verificationDigest, digestErr := templateVerificationDigest(data, template) + template.verificationDigest = verificationDigest + // check if the template is verified // only valid templates can be verified or signed - if options.TemplateVerificationCallback != nil && options.TemplatePath != "" { + if digestErr == nil && options.TemplateVerificationCallback != nil && options.TemplatePath != "" { if cached := options.TemplateVerificationCallback(options.TemplatePath); cached != nil { - template.Verified = cached.Verified - template.TemplateVerifier = cached.Verifier - options.TemplateVerifier = cached.Verifier - // mirror the verification result onto options for the code protocol. - options.Verified = cached.Verified - //nolint - if !(template.Verified && template.TemplateVerifier == "projectdiscovery/nuclei-templates") { - template.Options.RawTemplate = data + if cached.ContentDigest == verificationDigest && cached.ContentDigest != ([sha256.Size]byte{}) && cachedTemplateVerificationIsTrusted(cached) { + template.Verified = cached.Verified + template.TemplateVerifier = cached.Verifier + template.verifierFingerprint = cached.VerifierFingerprint + options.TemplateVerifier = cached.Verifier + // Mirror verification onto options for execution-time checks. + options.Verified = cached.Verified + //nolint + if !(template.Verified && template.TemplateVerifier == "projectdiscovery/nuclei-templates") { + template.Options.RawTemplate = data + } + + return } - return } } var verifier *signer.TemplateSigner + for _, verifier = range signer.DefaultTemplateVerifiers { template.Verified, _ = verifier.Verify(data, template) if config.DefaultConfig.LogAllEvents { gologger.Verbose().Msgf("template %v verified by %s : %v", template.ID, verifier.Identifier(), template.Verified) } + if template.Verified { template.TemplateVerifier = verifier.Identifier() + template.verifierFingerprint = verifier.Fingerprint() break } } + options.TemplateVerifier = template.TemplateVerifier - // mirror the verification result onto options for the code protocol. + // Mirror verification onto options for code and JavaScript execution. options.Verified = template.Verified //nolint @@ -650,6 +688,50 @@ func applyTemplateVerification(template *Template, data []byte) { } } +// ContentDigest returns the digest used to bind cached signature +// verification to template and imported-file contents. +func (template *Template) ContentDigest() [sha256.Size]byte { + return template.verificationDigest +} + +// VerifierFingerprint returns the public-key fingerprint for the verifier that +// authenticated this template. +func (template *Template) VerifierFingerprint() [sha256.Size]byte { + return template.verifierFingerprint +} + +func cachedTemplateVerificationIsTrusted(cached *protocols.TemplateVerification) bool { + if !cached.Verified || cached.VerifierFingerprint == ([sha256.Size]byte{}) { + return false + } + + for _, verifier := range signer.DefaultTemplateVerifiers { + if verifier.Identifier() == cached.Verifier && verifier.Fingerprint() == cached.VerifierFingerprint { + return true + } + } + + return false +} + +func templateVerificationDigest(data []byte, template *Template) ([sha256.Size]byte, error) { + componentDigests := make([]byte, 0, sha256.Size*(len(template.GetFileImports())+1)) + dataDigest := sha256.Sum256(data) + componentDigests = append(componentDigests, dataDigest[:]...) + + importedContents, complete := template.GetFileImportContents() + if !complete { + return [sha256.Size]byte{}, errors.New("imported-file content snapshot is incomplete") + } + + for _, contents := range importedContents { + fileDigest := sha256.Sum256(contents) + componentDigests = append(componentDigests, fileDigest[:]...) + } + + return sha256.Sum256(componentDigests), nil +} + // isCachedTemplateValid validates that a cached template is still usable after // option updates func isCachedTemplateValid(template *Template) bool { diff --git a/pkg/templates/compile_test.go b/pkg/templates/compile_test.go index 68dd65827a..9d0d8ba0ca 100644 --- a/pkg/templates/compile_test.go +++ b/pkg/templates/compile_test.go @@ -2,17 +2,21 @@ package templates_test import ( "context" + "crypto/sha256" "fmt" + "io" "log" netHttp "net/http" "net/http/httptest" "os" "path/filepath" + "strings" "testing" "time" "github.com/julienschmidt/httprouter" "github.com/projectdiscovery/nuclei/v3/internal/tests/testutils" + "github.com/projectdiscovery/nuclei/v3/pkg/catalog" "github.com/projectdiscovery/nuclei/v3/pkg/catalog/config" "github.com/projectdiscovery/nuclei/v3/pkg/catalog/disk" "github.com/projectdiscovery/nuclei/v3/pkg/loader/workflow" @@ -28,6 +32,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/variables" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/http" "github.com/projectdiscovery/nuclei/v3/pkg/templates" + templatesigner "github.com/projectdiscovery/nuclei/v3/pkg/templates/signer" "github.com/projectdiscovery/nuclei/v3/pkg/utils/stats" "github.com/projectdiscovery/nuclei/v3/pkg/workflows" "github.com/projectdiscovery/ratelimit" @@ -333,6 +338,265 @@ workflows: require.Equal(t, initialUnverified, stats.GetValue(templates.SkippedUnverifiedTemplateStats)) } +func TestParseTemplateExecutesJavascriptInitAfterVerification(t *testing.T) { + options := testutils.DefaultOptions.Copy() + options.ExecutionId = "parse-verified-javascript-init" + testutils.Init(options) + t.Cleanup(func() { + testutils.Cleanup(options) + }) + + executerOptions := testutils.NewMockExecuterOptions(options, nil) + executerOptions.TemplatePath = "verified-javascript-init.yaml" + templateSource := `id: verified-javascript-init + +info: + name: Verified Javascript Init + author: pdteam + severity: info + +javascript: + - init: | + set("init-status", "executed") + code: | + Export("verified-javascript-init") +` + executerOptions.TemplateVerificationCallback = func(templatePath string) *protocols.TemplateVerification { + require.Equal(t, executerOptions.TemplatePath, templatePath) + return trustedVerificationForTest(templateSource) + } + + template, err := templates.ParseTemplateFromReader(strings.NewReader(templateSource), nil, executerOptions) + require.NoError(t, err) + require.True(t, template.Verified) + require.True(t, template.Options.Verified) + require.Equal(t, "executed", template.RequestsJavascript[0].Args["init-status"]) +} + +func TestParseTemplateExecutesPreprocessedJavascriptInitAfterVerification(t *testing.T) { + options := testutils.DefaultOptions.Copy() + options.ExecutionId = "parse-verified-preprocessed-javascript-init" + testutils.Init(options) + t.Cleanup(func() { + testutils.Cleanup(options) + }) + + executerOptions := testutils.NewMockExecuterOptions(options, nil) + executerOptions.TemplatePath = "verified-preprocessed-javascript-init.yaml" + templateSource := `id: verified-preprocessed-javascript-init + +info: + name: Verified Preprocessed Javascript Init {{randstr}} + author: pdteam + severity: info + +javascript: + - init: | + set("init-status", "{{randstr}}") + code: | + Export("verified-preprocessed-javascript-init") +` + executerOptions.TemplateVerificationCallback = func(templatePath string) *protocols.TemplateVerification { + require.Equal(t, executerOptions.TemplatePath, templatePath) + return trustedVerificationForTest(templateSource) + } + + template, err := templates.ParseTemplateFromReader(strings.NewReader(templateSource), nil, executerOptions) + require.NoError(t, err) + require.True(t, template.Verified) + require.True(t, template.Options.Verified) + require.NotEmpty(t, template.RequestsJavascript[0].Args["init-status"]) + require.NotEqual(t, "{{randstr}}", template.RequestsJavascript[0].Args["init-status"]) +} + +func verificationDigestForTest(data string, importedContents ...string) [sha256.Size]byte { + dataDigest := sha256.Sum256([]byte(data)) + componentDigests := append([]byte(nil), dataDigest[:]...) + for _, contents := range importedContents { + importDigest := sha256.Sum256([]byte(contents)) + componentDigests = append(componentDigests, importDigest[:]...) + } + return sha256.Sum256(componentDigests) +} + +func trustedVerificationForTest(data string, importedContents ...string) *protocols.TemplateVerification { + verifier := templatesigner.DefaultTemplateVerifiers[0] + return &protocols.TemplateVerification{ + Verified: true, + Verifier: verifier.Identifier(), + VerifierFingerprint: verifier.Fingerprint(), + ContentDigest: verificationDigestForTest(data, importedContents...), + } +} + +func TestParseTemplateVerificationUsesLoadedImportContents(t *testing.T) { + options := testutils.DefaultOptions.Copy() + loadedCode := `Export("loaded-import")` + diskCode := `Export("disk-import")` + importPath := filepath.Join(t.TempDir(), "import.js") + require.NoError(t, os.WriteFile(importPath, []byte(diskCode), 0o600)) + options.LoadHelperFileFunction = func(helperFile, _ string, _ catalog.Catalog) (io.ReadCloser, error) { + require.Equal(t, importPath, helperFile) + return io.NopCloser(strings.NewReader(loadedCode)), nil + } + testutils.Init(options) + t.Cleanup(func() { + testutils.Cleanup(options) + }) + + executerOptions := testutils.NewMockExecuterOptions(options, nil) + executerOptions.TemplatePath = "loaded-import.yaml" + templateSource := fmt.Sprintf(`id: loaded-import + +info: + name: Loaded Import + author: pdteam + severity: info + +javascript: + - code: %q +`, importPath) + executerOptions.TemplateVerificationCallback = func(templatePath string) *protocols.TemplateVerification { + require.Equal(t, executerOptions.TemplatePath, templatePath) + return trustedVerificationForTest(templateSource, loadedCode) + } + + template, err := templates.ParseTemplateFromReader(strings.NewReader(templateSource), nil, executerOptions) + require.NoError(t, err) + require.True(t, template.Verified) + require.Equal(t, loadedCode, template.RequestsJavascript[0].Code) +} + +func TestParseTemplateRejectsCachedVerificationWithMismatchedVerifierFingerprint(t *testing.T) { + options := testutils.DefaultOptions.Copy() + testutils.Init(options) + t.Cleanup(func() { + testutils.Cleanup(options) + }) + + executerOptions := testutils.NewMockExecuterOptions(options, nil) + executerOptions.TemplatePath = "revoked-verifier.yaml" + templateSource := `id: revoked-verifier + +info: + name: Revoked Verifier + author: pdteam + severity: info + +javascript: + - init: | + set("init-status", "executed") + code: | + Export("revoked-verifier") +` + executerOptions.TemplateVerificationCallback = func(templatePath string) *protocols.TemplateVerification { + require.Equal(t, executerOptions.TemplatePath, templatePath) + verifier := templatesigner.DefaultTemplateVerifiers[0] + rotatedFingerprint := verifier.Fingerprint() + rotatedFingerprint[0] ^= 0xff + return &protocols.TemplateVerification{ + Verified: true, + Verifier: verifier.Identifier(), + VerifierFingerprint: rotatedFingerprint, + ContentDigest: verificationDigestForTest(templateSource), + } + } + + template, err := templates.ParseTemplateFromReader(strings.NewReader(templateSource), nil, executerOptions) + require.NoError(t, err) + require.False(t, template.Verified) + require.NotContains(t, template.RequestsJavascript[0].Args, "init-status") +} + +func TestParseTemplateCompilesUnsignedJavascriptInit(t *testing.T) { + options := testutils.DefaultOptions.Copy() + testutils.Init(options) + t.Cleanup(func() { + testutils.Cleanup(options) + }) + + executerOptions := testutils.NewMockExecuterOptions(options, nil) + template, err := templates.ParseTemplateFromReader(strings.NewReader(`id: unsigned-malformed-javascript-init + +info: + name: Unsigned Malformed Javascript Init + author: pdteam + severity: info + +javascript: + - init: | + { + code: | + Export("unsigned-malformed-javascript-init") +`), nil, executerOptions) + require.Nil(t, template) + require.ErrorContains(t, err, "could not compile init code") +} + +func TestParseTemplateDoesNotExecuteUnsignedJavascriptInit(t *testing.T) { + options := testutils.DefaultOptions.Copy() + testutils.Init(options) + t.Cleanup(func() { + testutils.Cleanup(options) + }) + + executerOptions := testutils.NewMockExecuterOptions(options, nil) + template, err := templates.ParseTemplateFromReader(strings.NewReader(`id: unsigned-javascript-init + +info: + name: Unsigned Javascript Init + author: pdteam + severity: info + +javascript: + - init: | + set("init-status", "executed") + code: | + Export("unsigned-javascript-init") +`), nil, executerOptions) + require.NoError(t, err) + require.False(t, template.Verified) + require.NotContains(t, template.RequestsJavascript[0].Args, "init-status") +} + +func TestParseCachedTemplatePreservesVerification(t *testing.T) { + options := testutils.DefaultOptions.Copy() + testutils.Init(options) + t.Cleanup(func() { + testutils.Cleanup(options) + }) + + templateSource := `id: cached-verified-javascript + +info: + name: Cached Verified Javascript + author: pdteam + severity: info + +javascript: + - code: | + Export("cached-verified-javascript") +` + templatePath := filepath.Join(t.TempDir(), "cached-verified-javascript.yaml") + require.NoError(t, os.WriteFile(templatePath, []byte(templateSource), 0o600)) + + executerOptions := testutils.NewMockExecuterOptions(options, nil) + executerOptions.Parser = templates.NewParser() + executerOptions.TemplateVerificationCallback = func(path string) *protocols.TemplateVerification { + require.Equal(t, templatePath, path) + return trustedVerificationForTest(templateSource) + } + + first, err := templates.Parse(templatePath, nil, executerOptions) + require.NoError(t, err) + require.True(t, first.Options.Verified) + + cached, err := templates.Parse(templatePath, nil, executerOptions) + require.NoError(t, err) + require.True(t, cached.Verified) + require.True(t, cached.Options.Verified) +} + func Test_WrongTemplate(t *testing.T) { setup() diff --git a/pkg/templates/fuzz_harness.go b/pkg/templates/fuzz_harness.go index 6c1d59a8a4..9a8dfa8359 100644 --- a/pkg/templates/fuzz_harness.go +++ b/pkg/templates/fuzz_harness.go @@ -140,6 +140,9 @@ func compileFuzzTemplate(data []byte) (*Template, error) { if template == nil { return nil, errors.New("nil compiled template") } + if err := compileTemplate(template); err != nil { + return nil, err + } exerciseFuzzParsedTemplate(template) return template, nil } diff --git a/pkg/templates/signer/tmpl_signer.go b/pkg/templates/signer/tmpl_signer.go index 962dd13119..fccada24af 100644 --- a/pkg/templates/signer/tmpl_signer.go +++ b/pkg/templates/signer/tmpl_signer.go @@ -46,6 +46,14 @@ type SignableTemplate interface { HasCodeProtocol() bool } +type fileImportContentProvider interface { + GetFileImportContents() ([][]byte, bool) +} + +type javascriptSignableTemplate interface { + HasJavascriptRequest(...int) bool +} + type TemplateSigner struct { sync.Once handler *KeyHandler @@ -96,43 +104,44 @@ func (t *TemplateSigner) Sign(data []byte, tmpl SignableTemplate) (string, error existingSignature, content := ExtractSignatureAndContent(data) content = normalizeTemplateContentForSignature(content) - // while re-signing template check if it has a code protocol - // if it does then verify that it is signed by current signer - // if not then return error - if tmpl.HasCodeProtocol() { + // Executable templates can only be re-signed by the current signer. + hasJavascript := false + if javascriptTemplate, ok := tmpl.(javascriptSignableTemplate); ok { + hasJavascript = javascriptTemplate.HasJavascriptRequest() + } + + if tmpl.HasCodeProtocol() || hasJavascript { if len(existingSignature) > 0 { arr := strings.SplitN(string(existingSignature), ":", 3) if len(arr) == 2 { // signature has no fragment - return "", errkit.New("re-signing code templates are not allowed for security reasons.") + return "", errkit.New("re-signing executable templates are not allowed for security reasons.") } + if len(arr) == 3 { // signature has fragment verify if it is equal to current fragment fragment, err := t.userFragment() if err != nil { return "", err } + if fragment != arr[2] { - return "", errkit.New("re-signing code templates are not allowed for security reasons.") + return "", errkit.New("re-signing executable templates are not allowed for security reasons.") } } } } - buff := bytes.NewBuffer(content) - // if file has any imports process them - for _, file := range tmpl.GetFileImports() { - bin, err := os.ReadFile(file) - if err != nil { - return "", err - } - buff.WriteRune('\n') - buff.Write(bin) + buff, err := templateContentWithImports(content, tmpl) + if err != nil { + return "", err } + signatureData, err := t.sign(buff.Bytes()) if err != nil { return "", err } + return signatureData, nil } @@ -141,18 +150,22 @@ func (t *TemplateSigner) Sign(data []byte, tmpl SignableTemplate) (string, error // in templates are not processed use template.SignTemplate() instead func (t *TemplateSigner) sign(data []byte) (string, error) { dataHash := sha256.Sum256(data) + ecdsaSignature, err := ecdsa.SignASN1(rand.Reader, t.handler.ecdsaKey, dataHash[:]) if err != nil { return "", err } + var signatureData bytes.Buffer if err := gob.NewEncoder(&signatureData).Encode(ecdsaSignature); err != nil { return "", err } + fragment, err := t.userFragment() if err != nil { return "", err } + return fmt.Sprintf(SignatureFmt, signatureData.Bytes(), fragment), nil } @@ -168,12 +181,15 @@ func (t *TemplateSigner) Verify(data []byte, tmpl SignableTemplate) (bool, error } digestData := bytes.TrimSpace(bytes.TrimPrefix(signature, []byte(SignaturePattern))) + fragment, err := t.userFragment() if err != nil { return false, err } + // remove fragment from digest as it is used for re-signing purposes only digestString := strings.TrimSuffix(string(digestData), ":"+fragment) + digest, err := hex.DecodeString(digestString) if err != nil { return false, err @@ -181,18 +197,53 @@ func (t *TemplateSigner) Verify(data []byte, tmpl SignableTemplate) (bool, error content = normalizeTemplateContentForSignature(content) + buff, err := templateContentWithImports(content, tmpl) + if err != nil { + return false, err + } + + return t.verify(buff.Bytes(), digest) +} + +func templateContentWithImports(content []byte, tmpl SignableTemplate) (*bytes.Buffer, error) { buff := bytes.NewBuffer(content) - // if file has any imports process them - for _, file := range tmpl.GetFileImports() { + imports := tmpl.GetFileImports() + + if provider, ok := tmpl.(fileImportContentProvider); ok { + if importedContents, captured := provider.GetFileImportContents(); captured { + if len(importedContents) != len(imports) { + return nil, fmt.Errorf("imported-file content count does not match imported-file path count") + } + + for _, importedContent := range importedContents { + buff.WriteRune('\n') + buff.Write(importedContent) + } + + return buff, nil + } + } + + for _, file := range imports { bin, err := os.ReadFile(file) if err != nil { - return false, err + return nil, err } + buff.WriteRune('\n') buff.Write(bin) } - return t.verify(buff.Bytes(), digest) + return buff, nil +} + +// Fingerprint returns the SHA-256 fingerprint of the signer's public key. +func (t *TemplateSigner) Fingerprint() [sha256.Size]byte { + if t == nil || t.handler == nil || t.handler.cert == nil { + return [sha256.Size]byte{} + } + + return sha256.Sum256(t.handler.cert.RawSubjectPublicKeyInfo) } func normalizeTemplateContentForSignature(content []byte) []byte { @@ -209,13 +260,16 @@ func (t *TemplateSigner) verify(data, signatureData []byte) (bool, error) { if err := gob.NewDecoder(bytes.NewReader(signatureData)).Decode(&signature); err != nil { return false, err } + return ecdsa.VerifyASN1(t.handler.ecdsaPubKey, dataHash[:], signature), nil } // NewTemplateSigner creates a new signer for signing templates func NewTemplateSigner(cert, privateKey []byte) (*TemplateSigner, error) { handler := &KeyHandler{} + var err error + if cert != nil || privateKey != nil { handler.UserCert = cert handler.PrivateKey = privateKey @@ -225,15 +279,18 @@ func NewTemplateSigner(cert, privateKey []byte) (*TemplateSigner, error) { err = handler.ReadPrivateKey(PrivateKeyEnvName, config.DefaultConfig.GetKeysDir()) } } + if err != nil && !SkipGeneratingKeys { if err != ErrNoCertificate && err != ErrNoPrivateKey { gologger.Info().Msgf("Invalid user cert found : %s\n", err) } + // generating new keys handler.GenerateKeyPair() if err := handler.SaveToDisk(config.DefaultConfig.GetKeysDir()); err != nil { gologger.Fatal().Msgf("could not save generated keys to disk: %s\n", err) } + // do not continue further let user re-run the command os.Exit(0) } else if err != nil && SkipGeneratingKeys { @@ -243,9 +300,11 @@ func NewTemplateSigner(cert, privateKey []byte) (*TemplateSigner, error) { if err := handler.ParseUserCert(); err != nil { return nil, err } + if err := handler.ParsePrivateKey(); err != nil { return nil, err } + return &TemplateSigner{ handler: handler, }, nil @@ -257,10 +316,12 @@ func NewTemplateSignerFromFiles(cert, privKey string) (*TemplateSigner, error) { if err != nil { return nil, err } + privKeyData, err := os.ReadFile(privKey) if err != nil { return nil, err } + return NewTemplateSigner(certData, privKeyData) } @@ -274,9 +335,11 @@ func NewTemplateSigVerifier(cert []byte) (*TemplateSigner, error) { return nil, err } } + if err := handler.ParseUserCert(); err != nil { return nil, err } + return &TemplateSigner{ handler: handler, }, nil diff --git a/pkg/templates/signer/tmpl_signer_test.go b/pkg/templates/signer/tmpl_signer_test.go index 15299c47c8..7034353562 100644 --- a/pkg/templates/signer/tmpl_signer_test.go +++ b/pkg/templates/signer/tmpl_signer_test.go @@ -22,8 +22,15 @@ const ( ) type mockSignableTemplate struct { - imports []string - hasCode bool + imports []string + hasCode bool + hasJavascript bool +} + +type snapshotSignableTemplate struct { + imports []string + importContents [][]byte + captured bool } func (m *mockSignableTemplate) GetFileImports() []string { @@ -34,6 +41,22 @@ func (m *mockSignableTemplate) HasCodeProtocol() bool { return m.hasCode } +func (m *mockSignableTemplate) HasJavascriptRequest(...int) bool { + return m.hasJavascript +} + +func (m *snapshotSignableTemplate) GetFileImports() []string { + return m.imports +} + +func (m *snapshotSignableTemplate) GetFileImportContents() ([][]byte, bool) { + return m.importContents, m.captured +} + +func (m *snapshotSignableTemplate) HasCodeProtocol() bool { + return false +} + var signer, _ = NewTemplateSignerFromFiles(testCertFile, testKeyFile) func TestPublicKeyFragmentTrimsLeadingZeroXCoordinate(t *testing.T) { @@ -175,3 +198,40 @@ func TestTemplateSignerSignAndVerify(t *testing.T) { }) } } + +func TestTemplateSignerUsesImportedContentSnapshot(t *testing.T) { + importPath := filepath.Join(t.TempDir(), "import.js") + require.NoError(t, os.WriteFile(importPath, []byte("disk content before signing"), 0o600)) + + tmpl := &snapshotSignableTemplate{ + imports: []string{importPath}, + importContents: [][]byte{[]byte("loaded content")}, + captured: true, + } + templateData := []byte("id: imported-content-snapshot") + signature, err := signer.Sign(templateData, tmpl) + require.NoError(t, err) + + require.NoError(t, os.WriteFile(importPath, []byte("disk content after signing"), 0o600)) + signedData := append(templateData, []byte("\n"+signature)...) + verified, err := signer.Verify(signedData, tmpl) + require.NoError(t, err) + require.True(t, verified) +} + +func TestTemplateSignerRejectsMismatchedImportedContentSnapshot(t *testing.T) { + importPath := filepath.Join(t.TempDir(), "import.js") + require.NoError(t, os.WriteFile(importPath, []byte("disk content"), 0o600)) + + tmpl := &snapshotSignableTemplate{imports: []string{importPath}, captured: true} + _, err := signer.Sign([]byte("id: incomplete-import-snapshot"), tmpl) + require.ErrorContains(t, err, "imported-file content count does not match imported-file path count") +} + +func TestTemplateSignerRejectsResigningJavascriptWithForeignSigner(t *testing.T) { + tmpl := &mockSignableTemplate{hasJavascript: true} + templateData := []byte("id: javascript-template\n# digest: 00:foreign-signer") + + _, err := signer.Sign(templateData, tmpl) + require.ErrorContains(t, err, "re-signing executable templates are not allowed") +} diff --git a/pkg/templates/template_sign_test.go b/pkg/templates/template_sign_test.go new file mode 100644 index 0000000000..893262ea4e --- /dev/null +++ b/pkg/templates/template_sign_test.go @@ -0,0 +1,27 @@ +package templates + +import ( + "os" + "path/filepath" + "testing" + + "github.com/projectdiscovery/nuclei/v3/pkg/templates/signer" + "github.com/stretchr/testify/require" +) + +func TestTemplateSignerSupportsProgrammaticImportedFiles(t *testing.T) { + templateSigner, err := signer.NewTemplateSignerFromFiles("signer/testdata/ci.crt", "signer/testdata/ci-private-key.pem") + require.NoError(t, err) + + importPath := filepath.Join(t.TempDir(), "import.js") + require.NoError(t, os.WriteFile(importPath, []byte("programmatic import"), 0o600)) + template := &Template{ImportedFiles: []string{importPath}} + templateData := []byte("id: programmatic-import") + + signature, err := templateSigner.Sign(templateData, template) + require.NoError(t, err) + signedData := append(templateData, []byte("\n"+signature)...) + verified, err := templateSigner.Verify(signedData, template) + require.NoError(t, err) + require.True(t, verified) +} diff --git a/pkg/templates/templates.go b/pkg/templates/templates.go index 0c5a99ad2a..bd46c9883d 100644 --- a/pkg/templates/templates.go +++ b/pkg/templates/templates.go @@ -157,11 +157,18 @@ type Template struct { Verified bool `yaml:"-" json:"-"` // TemplateVerifier is identifier verifier used to verify the template (default nuclei-templates have projectdiscovery/nuclei-templates) TemplateVerifier string `yaml:"-" json:"-"` + // verificationDigest binds cached verification to the verified template and imported-file contents. + verificationDigest [32]byte + // verifierFingerprint binds cached verification to the verifier's public key. + verifierFingerprint [32]byte // RequestsQueue contains all template requests in order (both protocol & request order) RequestsQueue []protocols.Request `yaml:"-" json:"-"` // ImportedFiles contains list of files whose contents are imported after template was compiled ImportedFiles []string `yaml:"-" json:"-"` + // importedFileContents contains the immutable contents loaded for ImportedFiles. + importedFileContents [][]byte + importedFileContentsCaptured bool } // HasCodeProtocol returns true if the template has a code protocol section @@ -412,6 +419,7 @@ func (template *Template) UnmarshalYAML(unmarshal func(interface{}) error) error // instead of actual javascript / engine code if so it loads the file contents and replaces the reference func (template *Template) ImportFileRefs(options *protocols.ExecutorOptions) error { var errs []error + template.importedFileContentsCaptured = true loadFile := func(source string) (string, bool) { // load file respecting sandbox @@ -423,6 +431,8 @@ func (template *Template) ImportFileRefs(options *protocols.ExecutorOptions) err bin, err := io.ReadAll(data) if err == nil { + template.ImportedFiles = append(template.ImportedFiles, source) + template.importedFileContents = append(template.importedFileContents, bytes.Clone(bin)) return string(bin), true } else { errs = append(errs, err) @@ -439,7 +449,6 @@ func (template *Template) ImportFileRefs(options *protocols.ExecutorOptions) err // simple test to check if source is a file or a snippet if !strings.ContainsRune(request.Source, '\n') && fileutil.FileExists(request.Source) { if val, ok := loadFile(request.Source); ok { - template.ImportedFiles = append(template.ImportedFiles, request.Source) request.Source = val } } @@ -450,7 +459,6 @@ func (template *Template) ImportFileRefs(options *protocols.ExecutorOptions) err // simple test to check if source is a file or a snippet if !strings.ContainsRune(request.Code, '\n') && fileutil.FileExists(request.Code) { if val, ok := loadFile(request.Code); ok { - template.ImportedFiles = append(template.ImportedFiles, request.Code) request.Code = val } } @@ -460,7 +468,6 @@ func (template *Template) ImportFileRefs(options *protocols.ExecutorOptions) err if template.IsFlowTemplate() { if filepath.Ext(template.Flow) == ".js" && fileutil.FileExists(template.Flow) { if val, ok := loadFile(template.Flow); ok { - template.ImportedFiles = append(template.ImportedFiles, template.Flow) template.Flow = val } } @@ -478,7 +485,6 @@ func (template *Template) ImportFileRefs(options *protocols.ExecutorOptions) err // simple test to check if source is a file or a snippet if !strings.ContainsRune(request.Source, '\n') && fileutil.FileExists(request.Source) { if val, ok := loadFile(request.Source); ok { - template.ImportedFiles = append(template.ImportedFiles, request.Source) request.Source = val } } @@ -492,7 +498,6 @@ func (template *Template) ImportFileRefs(options *protocols.ExecutorOptions) err // simple test to check if source is a file or a snippet if !strings.ContainsRune(request.Code, '\n') && fileutil.FileExists(request.Code) { if val, ok := loadFile(request.Code); ok { - template.ImportedFiles = append(template.ImportedFiles, request.Code) request.Code = val } } @@ -508,6 +513,20 @@ func (template *Template) GetFileImports() []string { return template.ImportedFiles } +// GetFileImportContents returns a copy of the imported-file content snapshot. +// The second result reports whether ImportFileRefs captured a snapshot. +func (template *Template) GetFileImportContents() ([][]byte, bool) { + if !template.importedFileContentsCaptured { + return nil, false + } + + contents := make([][]byte, len(template.importedFileContents)) + for i, content := range template.importedFileContents { + contents[i] = bytes.Clone(content) + } + return contents, true +} + // addRequestsToQueue adds protocol requests to the queue and preserves order of the protocols and requests func (template *Template) addRequestsToQueue(keys ...string) { for _, key := range keys {