feat: add internal/fetch package with SSRF-protected HTTP client

internal/fetch/fetch.go

@@ -0,0 +1,227 @@
+// Package fetch provides an SSRF-hardened HTTP client for safely retrieving
+// content from external URLs. It enforces HTTPS-only access, domain allowlists,
+// DNS pre-resolution with IP validation, and DNS-rebinding protection via
+// transport-level IP pinning.
+package fetch
+
+import (
+	"context"
+	"crypto/sha256"
+	"crypto/tls"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"net/url"
+	"strings"
+	"time"
+
+	"github.com/fullsend-ai/fullsend/internal/netutil"
+)
+
+// FetchPolicy controls the security constraints applied when fetching a URL.
+type FetchPolicy struct {
+	// AllowedDomains is the list of permitted hostnames. Entries may use a
+	// wildcard prefix (e.g. "*.example.com") to match any subdomain,
+	// including multi-level subdomains (e.g. "deep.sub.example.com").
+	AllowedDomains []string
+
+	// AllowedPorts restricts which ports may be used. When empty, only
+	// port 443 is permitted.
+	AllowedPorts []string
+
+	// MaxSizeBytes is the maximum response body size in bytes.
+	MaxSizeBytes int64
+
+	// Timeout is the overall HTTP request timeout.
+	Timeout time.Duration
+
+	// Offline, when true, causes FetchURL to reject all requests immediately.
+	Offline bool
+
+	// tlsConfig is an optional TLS configuration, used in tests to trust
+	// the self-signed certificates generated by httptest.NewTLSServer.
+	tlsConfig *tls.Config
+
+	// skipIPCheck disables the internal-IP validation step. This is used in
+	// tests where the test server necessarily listens on 127.0.0.1.
+	skipIPCheck bool
+}
+
+// DefaultPolicy is a sensible default policy allowing GitHub content hosts.
+var DefaultPolicy = FetchPolicy{
+	AllowedDomains: []string{"github.com", "raw.githubusercontent.com"},
+	MaxSizeBytes:   10 * 1024 * 1024, // 10 MB
+	Timeout:        30 * time.Second,
+}
+
+var (
+	errOffline        = errors.New("fetch: offline mode enabled")
+	errNotHTTPS       = errors.New("fetch: only https URLs are allowed")
+	errDoubleEncoding = errors.New("fetch: URL contains double-encoded percent (%25)")
+	errDomainBlocked  = errors.New("fetch: domain not in allowlist")
+	errPortBlocked    = errors.New("fetch: port not allowed")
+	errInternalIP     = errors.New("fetch: resolved IP is internal/reserved")
+	errNoAddrs        = errors.New("fetch: DNS resolution returned no addresses")
+	errNonOK          = errors.New("fetch: non-200 status code")
+	errTooLarge       = errors.New("fetch: response body exceeds size limit")
+)
+
+// FetchURL retrieves the content at rawURL subject to the given policy.
+// It returns the response body bytes or an error describing why the fetch
+// was rejected or failed.
+func FetchURL(ctx context.Context, rawURL string, policy FetchPolicy) ([]byte, error) {
+	// 1. Offline check.
+	if policy.Offline {
+		return nil, errOffline
+	}
+
+	// 2. Parse and validate scheme.
+	parsed, err := url.Parse(rawURL)
+	if err != nil {
+		return nil, fmt.Errorf("fetch: invalid URL: %w", err)
+	}
+	if strings.ToLower(parsed.Scheme) != "https" {
+		return nil, errNotHTTPS
+	}
+
+	// 3. Reject double-encoded percent signs.
+	if strings.Contains(rawURL, "%25") {
+		return nil, errDoubleEncoding
+	}
+
+	// 4. Domain allowlist.
+	hostname := strings.ToLower(parsed.Hostname())
+	if !isAllowedDomain(hostname, policy.AllowedDomains) {
+		return nil, errDomainBlocked
+	}
+
+	// 5. DNS pre-resolution.
+	addrs, err := net.DefaultResolver.LookupIPAddr(ctx, hostname)
+	if err != nil {
+		return nil, fmt.Errorf("fetch: DNS resolution failed for %s: %w", hostname, err)
+	}
+	if len(addrs) == 0 {
+		return nil, errNoAddrs
+	}
+
+	// 6. Validate ALL resolved IPs.
+	if !policy.skipIPCheck {
+		for _, addr := range addrs {
+			if netutil.IsInternal(addr.IP) {
+				return nil, fmt.Errorf("%w: %s resolved to %s", errInternalIP, hostname, addr.IP)
+			}
+		}
+	}
+
+	// Determine the port (default 443 for HTTPS).
+	port := parsed.Port()
+	if port == "" {
+		port = "443"
+	}
+
+	// 6b. Port restriction.
+	allowedPorts := policy.AllowedPorts
+	if len(allowedPorts) == 0 {
+		allowedPorts = []string{"443"}
+	}
+	if !portAllowed(port, allowedPorts) {
+		return nil, fmt.Errorf("%w: port %s", errPortBlocked, port)
+	}
+
+	// 7. Build transport with DNS-rebinding protection.
+	//    Try each pre-validated IP in order for IPv4/IPv6 fallback.
+	//    Each FetchURL call creates its own transport to pin resolved IPs;
+	//    idle connections are closed after use.
+	transport := &http.Transport{
+		DialContext: func(dialCtx context.Context, _, _ string) (net.Conn, error) {
+			d := net.Dialer{Timeout: policy.Timeout}
+			var lastErr error
+			for _, addr := range addrs {
+				conn, err := d.DialContext(dialCtx, "tcp", net.JoinHostPort(addr.IP.String(), port))
+				if err != nil {
+					lastErr = err
+					continue
+				}
+				return conn, nil
+			}
+			return nil, lastErr
+		},
+		TLSClientConfig: policy.tlsConfig,
+	}
+	defer transport.CloseIdleConnections()
+
+	client := &http.Client{
+		Transport: transport,
+		Timeout:   policy.Timeout,
+		// 8. Block redirects.
+		CheckRedirect: func(_ *http.Request, _ []*http.Request) error {
+			return http.ErrUseLastResponse
+		},
+	}
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)
+	if err != nil {
+		return nil, fmt.Errorf("fetch: failed to create request: %w", err)
+	}
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("fetch: request failed: %w", err)
+	}
+	defer resp.Body.Close()
+
+	// 9. Only accept 200 OK.
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("%w: got %d", errNonOK, resp.StatusCode)
+	}
+
+	// 10. Size limit: read one extra byte to detect overflow.
+	limited := io.LimitReader(resp.Body, policy.MaxSizeBytes+1)
+	data, err := io.ReadAll(limited)
+	if err != nil {
+		return nil, fmt.Errorf("fetch: reading response body: %w", err)
+	}
+	if int64(len(data)) > policy.MaxSizeBytes {
+		return nil, errTooLarge
+	}
+
+	return data, nil
+}
+
+// isAllowedDomain checks whether hostname matches any entry in the allowed
+// list. Entries may be exact hostnames or wildcard patterns like "*.example.com"
+// which match any subdomain of example.com (but not example.com itself).
+func isAllowedDomain(hostname string, allowed []string) bool {
+	for _, pattern := range allowed {
+		pattern = strings.ToLower(pattern)
+		if strings.HasPrefix(pattern, "*.") {
+			// Wildcard: *.example.com matches sub.example.com
+			// but not example.com itself.
+			suffix := pattern[1:] // ".example.com"
+			if strings.HasSuffix(hostname, suffix) && hostname != pattern[2:] {
+				return true
+			}
+		} else if hostname == pattern {
+			return true
+		}
+	}
+	return false
+}
+
+func portAllowed(port string, allowed []string) bool {
+	for _, p := range allowed {
+		if port == p {
+			return true
+		}
+	}
+	return false
+}
+
+// ComputeSHA256 returns the lowercase hex-encoded SHA-256 digest of data.
+func ComputeSHA256(data []byte) string {
+	h := sha256.Sum256(data)
+	return hex.EncodeToString(h[:])
+}