diff --git a/internal/fetch/fetch.go b/internal/fetch/fetch.go new file mode 100644 index 0000000000..b4ce9edd0b --- /dev/null +++ b/internal/fetch/fetch.go @@ -0,0 +1,227 @@ +// Package fetch provides an SSRF-hardened HTTP client for safely retrieving +// content from external URLs. It enforces HTTPS-only access, domain allowlists, +// DNS pre-resolution with IP validation, and DNS-rebinding protection via +// transport-level IP pinning. +package fetch + +import ( + "context" + "crypto/sha256" + "crypto/tls" + "encoding/hex" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/url" + "strings" + "time" + + "github.com/fullsend-ai/fullsend/internal/netutil" +) + +// FetchPolicy controls the security constraints applied when fetching a URL. +type FetchPolicy struct { + // AllowedDomains is the list of permitted hostnames. Entries may use a + // wildcard prefix (e.g. "*.example.com") to match any subdomain, + // including multi-level subdomains (e.g. "deep.sub.example.com"). + AllowedDomains []string + + // AllowedPorts restricts which ports may be used. When empty, only + // port 443 is permitted. + AllowedPorts []string + + // MaxSizeBytes is the maximum response body size in bytes. + MaxSizeBytes int64 + + // Timeout is the overall HTTP request timeout. + Timeout time.Duration + + // Offline, when true, causes FetchURL to reject all requests immediately. + Offline bool + + // tlsConfig is an optional TLS configuration, used in tests to trust + // the self-signed certificates generated by httptest.NewTLSServer. + tlsConfig *tls.Config + + // skipIPCheck disables the internal-IP validation step. This is used in + // tests where the test server necessarily listens on 127.0.0.1. + skipIPCheck bool +} + +// DefaultPolicy is a sensible default policy allowing GitHub content hosts. +var DefaultPolicy = FetchPolicy{ + AllowedDomains: []string{"github.com", "raw.githubusercontent.com"}, + MaxSizeBytes: 10 * 1024 * 1024, // 10 MB + Timeout: 30 * time.Second, +} + +var ( + errOffline = errors.New("fetch: offline mode enabled") + errNotHTTPS = errors.New("fetch: only https URLs are allowed") + errDoubleEncoding = errors.New("fetch: URL contains double-encoded percent (%25)") + errDomainBlocked = errors.New("fetch: domain not in allowlist") + errPortBlocked = errors.New("fetch: port not allowed") + errInternalIP = errors.New("fetch: resolved IP is internal/reserved") + errNoAddrs = errors.New("fetch: DNS resolution returned no addresses") + errNonOK = errors.New("fetch: non-200 status code") + errTooLarge = errors.New("fetch: response body exceeds size limit") +) + +// FetchURL retrieves the content at rawURL subject to the given policy. +// It returns the response body bytes or an error describing why the fetch +// was rejected or failed. +func FetchURL(ctx context.Context, rawURL string, policy FetchPolicy) ([]byte, error) { + // 1. Offline check. + if policy.Offline { + return nil, errOffline + } + + // 2. Parse and validate scheme. + parsed, err := url.Parse(rawURL) + if err != nil { + return nil, fmt.Errorf("fetch: invalid URL: %w", err) + } + if strings.ToLower(parsed.Scheme) != "https" { + return nil, errNotHTTPS + } + + // 3. Reject double-encoded percent signs. + if strings.Contains(rawURL, "%25") { + return nil, errDoubleEncoding + } + + // 4. Domain allowlist. + hostname := strings.ToLower(parsed.Hostname()) + if !isAllowedDomain(hostname, policy.AllowedDomains) { + return nil, errDomainBlocked + } + + // 5. DNS pre-resolution. + addrs, err := net.DefaultResolver.LookupIPAddr(ctx, hostname) + if err != nil { + return nil, fmt.Errorf("fetch: DNS resolution failed for %s: %w", hostname, err) + } + if len(addrs) == 0 { + return nil, errNoAddrs + } + + // 6. Validate ALL resolved IPs. + if !policy.skipIPCheck { + for _, addr := range addrs { + if netutil.IsInternal(addr.IP) { + return nil, fmt.Errorf("%w: %s resolved to %s", errInternalIP, hostname, addr.IP) + } + } + } + + // Determine the port (default 443 for HTTPS). + port := parsed.Port() + if port == "" { + port = "443" + } + + // 6b. Port restriction. + allowedPorts := policy.AllowedPorts + if len(allowedPorts) == 0 { + allowedPorts = []string{"443"} + } + if !portAllowed(port, allowedPorts) { + return nil, fmt.Errorf("%w: port %s", errPortBlocked, port) + } + + // 7. Build transport with DNS-rebinding protection. + // Try each pre-validated IP in order for IPv4/IPv6 fallback. + // Each FetchURL call creates its own transport to pin resolved IPs; + // idle connections are closed after use. + transport := &http.Transport{ + DialContext: func(dialCtx context.Context, _, _ string) (net.Conn, error) { + d := net.Dialer{Timeout: policy.Timeout} + var lastErr error + for _, addr := range addrs { + conn, err := d.DialContext(dialCtx, "tcp", net.JoinHostPort(addr.IP.String(), port)) + if err != nil { + lastErr = err + continue + } + return conn, nil + } + return nil, lastErr + }, + TLSClientConfig: policy.tlsConfig, + } + defer transport.CloseIdleConnections() + + client := &http.Client{ + Transport: transport, + Timeout: policy.Timeout, + // 8. Block redirects. + CheckRedirect: func(_ *http.Request, _ []*http.Request) error { + return http.ErrUseLastResponse + }, + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) + if err != nil { + return nil, fmt.Errorf("fetch: failed to create request: %w", err) + } + + resp, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("fetch: request failed: %w", err) + } + defer resp.Body.Close() + + // 9. Only accept 200 OK. + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("%w: got %d", errNonOK, resp.StatusCode) + } + + // 10. Size limit: read one extra byte to detect overflow. + limited := io.LimitReader(resp.Body, policy.MaxSizeBytes+1) + data, err := io.ReadAll(limited) + if err != nil { + return nil, fmt.Errorf("fetch: reading response body: %w", err) + } + if int64(len(data)) > policy.MaxSizeBytes { + return nil, errTooLarge + } + + return data, nil +} + +// isAllowedDomain checks whether hostname matches any entry in the allowed +// list. Entries may be exact hostnames or wildcard patterns like "*.example.com" +// which match any subdomain of example.com (but not example.com itself). +func isAllowedDomain(hostname string, allowed []string) bool { + for _, pattern := range allowed { + pattern = strings.ToLower(pattern) + if strings.HasPrefix(pattern, "*.") { + // Wildcard: *.example.com matches sub.example.com + // but not example.com itself. + suffix := pattern[1:] // ".example.com" + if strings.HasSuffix(hostname, suffix) && hostname != pattern[2:] { + return true + } + } else if hostname == pattern { + return true + } + } + return false +} + +func portAllowed(port string, allowed []string) bool { + for _, p := range allowed { + if port == p { + return true + } + } + return false +} + +// ComputeSHA256 returns the lowercase hex-encoded SHA-256 digest of data. +func ComputeSHA256(data []byte) string { + h := sha256.Sum256(data) + return hex.EncodeToString(h[:]) +} diff --git a/internal/fetch/fetch_test.go b/internal/fetch/fetch_test.go new file mode 100644 index 0000000000..be13206abd --- /dev/null +++ b/internal/fetch/fetch_test.go @@ -0,0 +1,247 @@ +package fetch + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "net" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/fullsend-ai/fullsend/internal/netutil" +) + +// newTestServer creates an HTTPS test server and returns it along with a +// FetchPolicy configured to trust the server's TLS certificate, allow +// its hostname, and skip internal-IP checks (the server listens on 127.0.0.1). +func newTestServer(t *testing.T, handler http.Handler) (*httptest.Server, FetchPolicy) { + t.Helper() + srv := httptest.NewTLSServer(handler) + t.Cleanup(srv.Close) + + // Extract hostname and port from the test server URL. + // srv.URL looks like "https://127.0.0.1:PORT". + hostPort := strings.TrimPrefix(srv.URL, "https://") + hostname, port, _ := net.SplitHostPort(hostPort) + + // The httptest server listens on 127.0.0.1 which is loopback, so we + // must skip the internal-IP check for integration tests. + policy := FetchPolicy{ + AllowedDomains: []string{hostname}, + AllowedPorts: []string{port}, + MaxSizeBytes: 1024, + Timeout: 5 * time.Second, + tlsConfig: srv.TLS.Clone(), + skipIPCheck: true, + } + // Skip TLS verification — httptest servers use self-signed certificates. + policy.tlsConfig.InsecureSkipVerify = true + + return srv, policy +} + +func TestFetchURL(t *testing.T) { + t.Run("HTTPSOnly", func(t *testing.T) { + policy := FetchPolicy{ + AllowedDomains: []string{"example.com"}, + MaxSizeBytes: 1024, + Timeout: 5 * time.Second, + } + _, err := FetchURL(context.Background(), "http://example.com/file", policy) + if !errors.Is(err, errNotHTTPS) { + t.Fatalf("expected errNotHTTPS, got: %v", err) + } + }) + + t.Run("DomainAllowlist", func(t *testing.T) { + policy := FetchPolicy{ + AllowedDomains: []string{"allowed.com"}, + MaxSizeBytes: 1024, + Timeout: 5 * time.Second, + } + _, err := FetchURL(context.Background(), "https://blocked.com/file", policy) + if !errors.Is(err, errDomainBlocked) { + t.Fatalf("expected errDomainBlocked, got: %v", err) + } + }) + + t.Run("WildcardDomain", func(t *testing.T) { + // Wildcard should match subdomains but not the bare domain. + if !isAllowedDomain("sub.example.com", []string{"*.example.com"}) { + t.Fatal("expected sub.example.com to match *.example.com") + } + if !isAllowedDomain("deep.sub.example.com", []string{"*.example.com"}) { + t.Fatal("expected deep.sub.example.com to match *.example.com") + } + if isAllowedDomain("example.com", []string{"*.example.com"}) { + t.Fatal("expected example.com NOT to match *.example.com") + } + if isAllowedDomain("notexample.com", []string{"*.example.com"}) { + t.Fatal("expected notexample.com NOT to match *.example.com") + } + }) + + t.Run("NoRedirects", func(t *testing.T) { + srv, policy := newTestServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, "/other", http.StatusMovedPermanently) + })) + + _, err := FetchURL(context.Background(), srv.URL+"/start", policy) + if !errors.Is(err, errNonOK) { + t.Fatalf("expected errNonOK for redirect response, got: %v", err) + } + }) + + t.Run("SizeLimit", func(t *testing.T) { + // Write 2048 bytes; policy.MaxSizeBytes is 1024. + srv, policy := newTestServer(t, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + data := make([]byte, 2048) + _, _ = w.Write(data) + })) + + _, err := FetchURL(context.Background(), srv.URL+"/big", policy) + if !errors.Is(err, errTooLarge) { + t.Fatalf("expected errTooLarge, got: %v", err) + } + }) + + t.Run("Timeout", func(t *testing.T) { + srv, policy := newTestServer(t, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + time.Sleep(2 * time.Second) + w.WriteHeader(http.StatusOK) + })) + policy.Timeout = 100 * time.Millisecond + + _, err := FetchURL(context.Background(), srv.URL+"/slow", policy) + if err == nil { + t.Fatal("expected timeout error, got nil") + } + }) + + t.Run("OfflineMode", func(t *testing.T) { + policy := FetchPolicy{ + AllowedDomains: []string{"example.com"}, + MaxSizeBytes: 1024, + Timeout: 5 * time.Second, + Offline: true, + } + _, err := FetchURL(context.Background(), "https://example.com/file", policy) + if !errors.Is(err, errOffline) { + t.Fatalf("expected errOffline, got: %v", err) + } + }) + + t.Run("DoubleEncoding", func(t *testing.T) { + policy := FetchPolicy{ + AllowedDomains: []string{"example.com"}, + MaxSizeBytes: 1024, + Timeout: 5 * time.Second, + } + _, err := FetchURL(context.Background(), "https://example.com/%25252e%25252e", policy) + if !errors.Is(err, errDoubleEncoding) { + t.Fatalf("expected errDoubleEncoding, got: %v", err) + } + }) + + t.Run("NonOKStatus", func(t *testing.T) { + srv, policy := newTestServer(t, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusNotFound) + })) + + _, err := FetchURL(context.Background(), srv.URL+"/missing", policy) + if !errors.Is(err, errNonOK) { + t.Fatalf("expected errNonOK, got: %v", err) + } + }) + + t.Run("Success", func(t *testing.T) { + srv, policy := newTestServer(t, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + fmt.Fprint(w, "hello world") + })) + + data, err := FetchURL(context.Background(), srv.URL+"/ok", policy) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if string(data) != "hello world" { + t.Fatalf("unexpected body: %q", string(data)) + } + }) +} + +func TestIsInternalIPDelegatesToNetutil(t *testing.T) { + // Verify that the fetch package uses netutil.IsInternal (smoke test). + ip := net.ParseIP("127.0.0.1") + if !netutil.IsInternal(ip) { + t.Fatal("expected 127.0.0.1 to be internal") + } + ip = net.ParseIP("8.8.8.8") + if netutil.IsInternal(ip) { + t.Fatal("expected 8.8.8.8 to be public") + } +} + +func TestPortRestriction(t *testing.T) { + t.Run("DefaultRejectsNonStandard", func(t *testing.T) { + policy := FetchPolicy{ + AllowedDomains: []string{"example.com"}, + MaxSizeBytes: 1024, + Timeout: 5 * time.Second, + } + _, err := FetchURL(context.Background(), "https://example.com:8443/file", policy) + if !errors.Is(err, errPortBlocked) { + t.Fatalf("expected errPortBlocked, got: %v", err) + } + }) + + t.Run("DefaultAllows443", func(t *testing.T) { + policy := FetchPolicy{ + AllowedDomains: []string{"example.com"}, + MaxSizeBytes: 1024, + Timeout: 5 * time.Second, + } + // Port 443 is allowed by default; this will fail at DNS, not port check. + _, err := FetchURL(context.Background(), "https://example.com:443/file", policy) + if errors.Is(err, errPortBlocked) { + t.Fatal("port 443 should be allowed by default") + } + }) + + t.Run("ExplicitPortAllowed", func(t *testing.T) { + policy := FetchPolicy{ + AllowedDomains: []string{"example.com"}, + AllowedPorts: []string{"443", "8443"}, + MaxSizeBytes: 1024, + Timeout: 5 * time.Second, + } + // Port 8443 is explicitly allowed; will fail at DNS, not port check. + _, err := FetchURL(context.Background(), "https://example.com:8443/file", policy) + if errors.Is(err, errPortBlocked) { + t.Fatal("port 8443 should be allowed when explicitly configured") + } + }) +} + +func TestComputeSHA256(t *testing.T) { + input := []byte("hello world") + expected := sha256.Sum256(input) + expectedHex := hex.EncodeToString(expected[:]) + + got := ComputeSHA256(input) + if got != expectedHex { + t.Fatalf("ComputeSHA256(%q) = %s, want %s", input, got, expectedHex) + } + + // Verify against a known hash value. + const knownHash = "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9" + if got != knownHash { + t.Fatalf("ComputeSHA256(%q) = %s, want known hash %s", input, got, knownHash) + } +} diff --git a/internal/netutil/ip.go b/internal/netutil/ip.go new file mode 100644 index 0000000000..7e8cc7f73d --- /dev/null +++ b/internal/netutil/ip.go @@ -0,0 +1,72 @@ +// Package netutil provides shared network utilities for IP classification. +package netutil + +import ( + "fmt" + "net" +) + +type reservedNet struct { + network *net.IPNet + reason string +} + +var reservedNets []reservedNet + +func init() { + for _, entry := range []struct { + cidr string + reason string + }{ + {"0.0.0.0/8", "\"this\" network (RFC 1122)"}, + {"100.64.0.0/10", "CGNAT address (RFC 6598)"}, + {"192.0.2.0/24", "documentation address (TEST-NET-1, RFC 5737)"}, + {"198.18.0.0/15", "benchmark testing (RFC 2544)"}, + {"198.51.100.0/24", "documentation address (TEST-NET-2, RFC 5737)"}, + {"203.0.113.0/24", "documentation address (TEST-NET-3, RFC 5737)"}, + } { + _, network, err := net.ParseCIDR(entry.cidr) + if err != nil { + panic(fmt.Sprintf("netutil: bad CIDR %q: %v", entry.cidr, err)) + } + reservedNets = append(reservedNets, reservedNet{network: network, reason: entry.reason}) + } +} + +// CheckIP reports whether ip is a reserved address that should not be +// contacted by an outbound HTTP client. Returns an empty string if +// the IP is safe, or a human-readable reason if it is blocked. +func CheckIP(ip net.IP) string { + if v4 := ip.To4(); v4 != nil { + ip = v4 + } + + if ip.IsLoopback() { + return "loopback address" + } + if ip.IsPrivate() { + return "private address (RFC 1918)" + } + if ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() { + return "link-local address" + } + if ip.IsMulticast() { + return "multicast address" + } + if ip.IsUnspecified() { + return "unspecified address" + } + + for _, r := range reservedNets { + if r.network.Contains(ip) { + return r.reason + } + } + + return "" +} + +// IsInternal is a convenience wrapper that returns true if ip is reserved. +func IsInternal(ip net.IP) bool { + return CheckIP(ip) != "" +} diff --git a/internal/netutil/ip_test.go b/internal/netutil/ip_test.go new file mode 100644 index 0000000000..2a9f4ccd29 --- /dev/null +++ b/internal/netutil/ip_test.go @@ -0,0 +1,81 @@ +package netutil + +import ( + "net" + "testing" +) + +func TestCheckIP(t *testing.T) { + tests := []struct { + name string + ip string + internal bool + }{ + // Loopback + {"loopback_v4", "127.0.0.1", true}, + {"loopback_v6", "::1", true}, + + // RFC 1918 private ranges + {"rfc1918_10", "10.0.0.1", true}, + {"rfc1918_172", "172.16.0.1", true}, + {"rfc1918_192", "192.168.1.1", true}, + + // Link-local + {"link_local_v4", "169.254.1.1", true}, + {"link_local_v6", "fe80::1", true}, + + // CGNAT (RFC 6598) + {"cgnat", "100.64.0.1", true}, + {"cgnat_end", "100.127.255.254", true}, + + // Benchmark (RFC 2544) + {"benchmark", "198.18.0.1", true}, + {"benchmark_end", "198.19.255.254", true}, + + // Unspecified + {"unspecified_v4", "0.0.0.0", true}, + {"unspecified_v6", "::", true}, + + // "This" network + {"this_network", "0.1.2.3", true}, + + // Documentation / TEST-NET (RFC 5737) + {"doc_test_net_1", "192.0.2.1", true}, + {"doc_test_net_2", "198.51.100.1", true}, + {"doc_test_net_3", "203.0.113.1", true}, + + // Multicast + {"multicast_v4", "224.0.0.1", true}, + {"multicast_v6", "ff02::1", true}, + + // IPv4-mapped IPv6 + {"mapped_loopback", "::ffff:127.0.0.1", true}, + {"mapped_private", "::ffff:10.0.0.1", true}, + {"mapped_public", "::ffff:8.8.8.8", false}, + + // Public IPs (should NOT be internal) + {"public_google_dns", "8.8.8.8", false}, + {"public_cloudflare", "1.1.1.1", false}, + {"public_v6", "2001:4860:4860::8888", false}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + ip := net.ParseIP(tc.ip) + if ip == nil { + t.Fatalf("failed to parse IP: %s", tc.ip) + } + got := IsInternal(ip) + if got != tc.internal { + t.Errorf("IsInternal(%s) = %v, want %v", tc.ip, got, tc.internal) + } + reason := CheckIP(ip) + if tc.internal && reason == "" { + t.Errorf("CheckIP(%s) returned empty reason for internal IP", tc.ip) + } + if !tc.internal && reason != "" { + t.Errorf("CheckIP(%s) returned reason %q for public IP", tc.ip, reason) + } + }) + } +} diff --git a/internal/security/ssrf.go b/internal/security/ssrf.go index 745f7bfe55..4904e3626f 100644 --- a/internal/security/ssrf.go +++ b/internal/security/ssrf.go @@ -6,6 +6,8 @@ import ( "net/url" "regexp" "strings" + + "github.com/fullsend-ai/fullsend/internal/netutil" ) // reURLPattern matches HTTP(S) and dangerous-scheme URLs in free text. @@ -110,7 +112,7 @@ func (s *SSRFValidator) ValidateURL(rawURL string, resolveDNS bool) ScanResult { // Check if hostname is a raw IP if ip := net.ParseIP(hostname); ip != nil { - if reason := checkIP(ip); reason != "" { + if reason := netutil.CheckIP(ip); reason != "" { return ScanResult{ Safe: false, Findings: []Finding{{ @@ -139,7 +141,7 @@ func (s *SSRFValidator) ValidateURL(rawURL string, resolveDNS bool) ScanResult { } for _, addr := range addrs { if ip := net.ParseIP(addr); ip != nil { - if reason := checkIP(ip); reason != "" { + if reason := netutil.CheckIP(ip); reason != "" { return ScanResult{ Safe: false, Findings: []Finding{{ @@ -189,38 +191,3 @@ func (s *SSRFValidator) Scan(text string) ScanResult { return result } - -func checkIP(ip net.IP) string { - if ip.IsLoopback() { - return "loopback address" - } - if ip.IsPrivate() { - return "private address (RFC 1918)" - } - if ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() { - return "link-local address" - } - if ip.IsMulticast() { - return "multicast address" - } - if ip.IsUnspecified() { - return "unspecified address" - } - - // CGNAT / shared address space (RFC 6598): 100.64.0.0/10 - _, cgnat, _ := net.ParseCIDR("100.64.0.0/10") - if cgnat.Contains(ip) { - return "CGNAT address (RFC 6598)" - } - - // Documentation ranges - docRanges := []string{"192.0.2.0/24", "198.51.100.0/24", "203.0.113.0/24"} - for _, cidr := range docRanges { - _, network, _ := net.ParseCIDR(cidr) - if network.Contains(ip) { - return "documentation address" - } - } - - return "" -}