feat: import workflow or JSON workflow from arbitrary HTTP(S) URL

pkg/cli/add_command.go

@@ -62,6 +62,8 @@ Examples:
   ` + string(constants.CLIExtensionPrefix) + ` add githubnext/agentics/ci-doctor@v1.0.0         # Add with version
   ` + string(constants.CLIExtensionPrefix) + ` add githubnext/agentics/workflows/ci-doctor.md@main
   ` + string(constants.CLIExtensionPrefix) + ` add https://github.com/githubnext/agentics/blob/main/workflows/ci-doctor.md
+  ` + string(constants.CLIExtensionPrefix) + ` add https://example.com/my-workflow.md           # Add workflow from any HTTPS URL
+  ` + string(constants.CLIExtensionPrefix) + ` add https://example.com/workflow.json            # Import JSON workflow definition
   ` + string(constants.CLIExtensionPrefix) + ` add githubnext/agentics/ci-doctor --create-pull-request --force
   ` + string(constants.CLIExtensionPrefix) + ` add ./my-workflow.md                             # Add local workflow
   ` + string(constants.CLIExtensionPrefix) + ` add ./*.md                                       # Add all local workflows
@@ -73,6 +75,9 @@ Workflow specifications:
   - Three parts: "owner/repo/workflow-name[@version]" (implicitly looks in workflows/ directory)
   - Four+ parts: "owner/repo/workflows/workflow-name.md[@version]" (requires explicit .md extension)
   - GitHub URL: "https://github.com/owner/repo/blob/branch/path/to/workflow.md"
+  - Arbitrary URL: "https://example.com/workflow.md" (fetches and dispatches on Content-Type)
+    - text/markdown → treated as a gh-aw workflow markdown file
+    - application/json → converted from a JSON workflow definition
   - Local file: "./path/to/workflow.md" (adds a workflow from local filesystem)
   - Local wildcard: "./*.md" or "./dir/*.md" (adds all .md files matching pattern)
   - Version can be tag, branch, or SHA (for remote workflows)
@@ -363,7 +368,9 @@ func addWorkflowWithTracking(ctx context.Context, resolved *ResolvedWorkflow, tr
 	}
 
 	// For remote workflows, fetch and save all dependencies (includes, imports, dispatch workflows, resources)
-	if !isLocalWorkflowPath(workflowSpec.WorkflowPath) {
+	if workflowSpec.RawURL != "" {
+		// Generic URL imports carry no GitHub repo context; dependency fetching is skipped.
+	} else if !isLocalWorkflowPath(workflowSpec.WorkflowPath) {
 		if err := fetchAllRemoteDependencies(ctx, string(sourceContent), workflowSpec, githubWorkflowsDir, opts.Verbose, opts.Force, tracker); err != nil {
 			return err
 		}

pkg/cli/add_wizard_command.go

@@ -33,6 +33,8 @@ Examples:
   ` + string(constants.CLIExtensionPrefix) + ` add-wizard githubnext/agentics/daily-repo-status    # Guided setup
   ` + string(constants.CLIExtensionPrefix) + ` add-wizard githubnext/agentics/ci-doctor@v1.0.0     # Guided setup with version
   ` + string(constants.CLIExtensionPrefix) + ` add-wizard ./my-workflow.md                         # Guided setup for local workflow
+  ` + string(constants.CLIExtensionPrefix) + ` add-wizard https://example.com/my-workflow.md       # Guided setup from any HTTPS URL
+  ` + string(constants.CLIExtensionPrefix) + ` add-wizard https://example.com/workflow.json        # Import JSON workflow definition with guided setup
   ` + string(constants.CLIExtensionPrefix) + ` add-wizard githubnext/agentics/ci-doctor --engine copilot   # Pre-select engine
   ` + string(constants.CLIExtensionPrefix) + ` add-wizard githubnext/agentics/ci-doctor --skip-secret      # Skip secret prompt
 
@@ -42,6 +44,9 @@ Workflow specifications:
   - Three parts: "owner/repo/workflow-name[@version]" (implicitly looks in workflows/ directory)
   - Four+ parts: "owner/repo/workflows/workflow-name.md[@version]" (requires explicit .md extension)
   - GitHub URL: "https://github.com/owner/repo/blob/branch/path/to/workflow.md"
+  - Arbitrary URL: "https://example.com/workflow.md" (fetches and dispatches on Content-Type)
+    - text/markdown → treated as a gh-aw workflow markdown file
+    - application/json → converted from a JSON workflow definition
   - Local file: "./path/to/workflow.md"
   - Version can be tag, branch, or SHA (for remote workflows)
 

pkg/cli/fetch.go

@@ -2,6 +2,8 @@ package cli
 
 import (
 	"context"
+	"encoding/json"
+	"errors"
 	"fmt"
 	"os"
 	"regexp"
@@ -41,6 +43,11 @@ type FetchedWorkflow struct {
 func FetchWorkflowFromSourceWithContext(ctx context.Context, spec *WorkflowSpec, verbose bool) (*FetchedWorkflow, error) {
 	remoteWorkflowLog.Printf("Fetching workflow from source: spec=%s", spec.String())
 
+	// Handle generic HTTP(S) URL imports (non-GitHub hosts).
+	if spec.RawURL != "" {
+		return fetchGenericURLWorkflow(ctx, spec, verbose)
+	}
+
 	// Handle local workflows
 	if isLocalWorkflowPath(spec.WorkflowPath) {
 		return fetchLocalWorkflow(spec, verbose)
@@ -223,3 +230,87 @@ func isTransientSHAResolutionError(err error) bool {
 
 	return transientHTTP5xxPattern.MatchString(errorText)
 }
+
+// fetchGenericURLWorkflow fetches a workflow from an arbitrary HTTP(S) URL and dispatches
+// on the response Content-Type to produce a FetchedWorkflow.
+//
+// Supported content types:
+//   - text/markdown, text/x-markdown → treated as raw gh-aw workflow markdown.
+//   - application/json (or any +json suffix) → treated as a JSON workflow definition
+//     and converted via ConvertJSONWorkflowToMarkdown.
+//
+// Any other content type is an error with an actionable message.
+// Warnings from JSON conversion are printed to stderr when verbose is true.
+func fetchGenericURLWorkflow(ctx context.Context, spec *WorkflowSpec, verbose bool) (*FetchedWorkflow, error) {
+	remoteWorkflowLog.Printf("Fetching generic URL workflow")
+
+	if verbose {
+		fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Fetching workflow from URL..."))
+	}
+
+	resource, err := FetchImportURL(ctx, spec.RawURL, FetchOptions{})
+	if err != nil {
+		return nil, err
+	}
+
+	ct := resource.ContentType
+
+	switch {
+	case ct == "text/markdown" || ct == "text/x-markdown":
+		remoteWorkflowLog.Printf("URL returned markdown content (%d bytes)", len(resource.Body))
+		if verbose {
+			fmt.Fprintln(os.Stderr, console.FormatSuccessMessage(fmt.Sprintf("Downloaded workflow markdown (%d bytes)", len(resource.Body))))
+		}
+		return &FetchedWorkflow{
+			Content:    resource.Body,
+			CommitSHA:  "",
+			IsLocal:    false,
+			SourcePath: spec.RawURL,
+		}, nil
+
+	case ct == "application/json" || strings.HasSuffix(ct, "+json"):
+		remoteWorkflowLog.Printf("URL returned JSON content (%d bytes); converting", len(resource.Body))
+		if verbose {
+			fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Downloaded JSON workflow (%d bytes); converting to markdown...", len(resource.Body))))
+		}
+
+		var wf JSONWorkflow
+		if err := json.Unmarshal(resource.Body, &wf); err != nil {
+			return nil, fmt.Errorf("failed to parse JSON workflow from URL: %w", err)
+		}
+
+		nameOverride := spec.WorkflowName
+		generated, err := ConvertJSONWorkflowToMarkdown(&wf, ConvertOptions{NameOverride: nameOverride})
+		if err != nil {
+			return nil, fmt.Errorf("failed to convert JSON workflow: %w", err)
+		}
+
+		for _, w := range generated.Warnings {
+			fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("JSON workflow import: %s", w)))
+		}
+
+		// Use the generated filename as the WorkflowName on the spec so that
+		// downstream code (e.g. add_command.go) uses the correct file name.
+		spec.WorkflowName = generated.Filename
+
+		if verbose {
+			fmt.Fprintln(os.Stderr, console.FormatSuccessMessage(fmt.Sprintf("Converted JSON workflow to markdown (%d bytes)", len(generated.Markdown))))
+		}
+
+		return &FetchedWorkflow{
+			Content:    []byte(generated.Markdown),
+			CommitSHA:  "",
+			IsLocal:    false,
+			SourcePath: spec.RawURL,
+		}, nil
+
+	default:
+		if ct == "" {
+			return nil, errors.New(console.FormatErrorMessage(
+				"URL did not return a Content-Type header. Expected text/markdown or application/json."))
+		}
+		return nil, errors.New(console.FormatErrorMessage(
+			fmt.Sprintf("unsupported Content-Type %q from URL. Expected text/markdown or application/json.", ct)))
+	}
+}
+

pkg/cli/import_url_fetcher.go

@@ -0,0 +1,200 @@
+package cli
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"mime"
+	"net/http"
+	"net/url"
+	"os"
+	"strings"
+
+	"github.com/github/gh-aw/pkg/console"
+	"github.com/github/gh-aw/pkg/logger"
+)
+
+const importURLMaxBytes = 10 * 1024 * 1024 // 10 MB
+
+var importURLFetcherLog = logger.New("cli:import_url_fetcher")
+
+// FetchOptions configures FetchImportURL.
+type FetchOptions struct {
+	// HTTPClient overrides the default http.Client.  When nil a default client is used.
+	HTTPClient *http.Client
+}
+
+// FetchedResource is the result of fetching a URL for workflow import.
+type FetchedResource struct {
+	URL         string // the original URL
+	ContentType string // canonicalized media type without parameters (e.g. "application/json")
+	Body        []byte
+}
+
+// FetchImportURL fetches rawURL and returns its content and canonicalized Content-Type.
+//
+// Resolution order:
+//  1. HEAD request to detect Content-Type without downloading the body.
+//     If the server returns 405/501 or omits Content-Type, skip to step 2.
+//  2. GET request – response headers are checked before the body is consumed.
+//
+// Authentication is attached only when the request host equals "github.com" or the value
+// of the GH_HOST environment variable.  In that case the value of GH_TOKEN (falling back
+// to GITHUB_TOKEN) is sent as "Authorization: Bearer <token>".  For all other hosts no
+// authentication header is added; TLS verification is always enabled.
+//
+// The body is capped at importURLMaxBytes to prevent runaway downloads.
+func FetchImportURL(ctx context.Context, rawURL string, opts FetchOptions) (*FetchedResource, error) {
+	importURLFetcherLog.Printf("Fetching import URL (redacted for security)")
+
+	client := opts.HTTPClient
+	if client == nil {
+		client = &http.Client{}
+	}
+
+	// Attempt HEAD first to get Content-Type without downloading the body.
+	ct, headOK := tryHead(ctx, client, rawURL)
+
+	// Always perform the GET to retrieve the body.
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)
+	if err != nil {
+		return nil, fmt.Errorf("failed to build GET request: %w", err)
+	}
+	attachImportAuthHeader(req, rawURL)
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("failed to fetch URL: %w", err)
+	}
+	defer resp.Body.Close()
+
+	switch resp.StatusCode {
+	case http.StatusUnauthorized, http.StatusForbidden:
+		return nil, errors.New(console.FormatErrorMessage(
+			fmt.Sprintf("access denied (HTTP %d). Check that the URL is accessible or set an auth token.", resp.StatusCode),
+		))
+	case http.StatusNotFound:
+		return nil, errors.New(console.FormatErrorMessage("URL not found (HTTP 404)"))
+	}
+	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+		return nil, errors.New(console.FormatErrorMessage(
+			fmt.Sprintf("unexpected HTTP %d response from server", resp.StatusCode),
+		))
+	}
+
+	// Prefer Content-Type obtained via HEAD; fall back to GET response headers.
+	if !headOK || ct == "" {
+		ct = canonicalContentType(resp.Header.Get("Content-Type"))
+	}
+
+	// Guard against oversized responses.
+	limited := io.LimitReader(resp.Body, int64(importURLMaxBytes)+1)
+	body, err := io.ReadAll(limited)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read response body: %w", err)
+	}
+	if len(body) > importURLMaxBytes {
+		return nil, errors.New(console.FormatErrorMessage(
+			fmt.Sprintf("response body exceeds size limit (%s)", console.FormatFileSize(importURLMaxBytes)),
+		))
+	}
+
+	importURLFetcherLog.Printf("Fetched import URL: content_type=%s, bytes=%d", ct, len(body))
+
+	return &FetchedResource{
+		URL:         rawURL,
+		ContentType: ct,
+		Body:        body,
+	}, nil
+}
+
+// tryHead issues a HEAD request and returns the canonicalized Content-Type and whether
+// the request succeeded (status 2xx with a Content-Type header).  Any error or non-useful
+// response is silently swallowed – the caller falls back to GET.
+func tryHead(ctx context.Context, client *http.Client, rawURL string) (string, bool) {
+	req, err := http.NewRequestWithContext(ctx, http.MethodHead, rawURL, nil)
+	if err != nil {
+		return "", false
+	}
+	attachImportAuthHeader(req, rawURL)
+
+	resp, err := client.Do(req)
+	if err != nil {
+		importURLFetcherLog.Printf("HEAD request failed (will fallback to GET): %v", err)
+		return "", false
+	}
+	defer resp.Body.Close()
+
+	// 405 Method Not Allowed / 501 Not Implemented → server doesn't support HEAD.
+	if resp.StatusCode == http.StatusMethodNotAllowed || resp.StatusCode == http.StatusNotImplemented {
+		return "", false
+	}
+	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+		return "", false
+	}
+
+	ct := canonicalContentType(resp.Header.Get("Content-Type"))
+	return ct, ct != ""
+}
+
+// canonicalContentType strips parameters (e.g. "; charset=utf-8") from a Content-Type
+// header value and returns the lower-cased media type.  Returns "" on parse failure.
+func canonicalContentType(raw string) string {
+	if raw == "" {
+		return ""
+	}
+	mt, _, err := mime.ParseMediaType(raw)
+	if err != nil {
+		// Best-effort: strip parameters manually.
+		if idx := strings.IndexByte(raw, ';'); idx != -1 {
+			raw = raw[:idx]
+		}
+		return strings.ToLower(strings.TrimSpace(raw))
+	}
+	return strings.ToLower(mt)
+}
+
+// attachImportAuthHeader adds "Authorization: Bearer <token>" to req if and only if
+// the request host equals github.com or the value of GH_HOST.  The token is read
+// from GH_TOKEN, falling back to GITHUB_TOKEN.  Nothing is added when no matching
+// host is found or no token is set.  The token value is never logged.
+func attachImportAuthHeader(req *http.Request, rawURL string) {
+	parsed, err := url.Parse(rawURL)
+	if err != nil || parsed.Host == "" {
+		return
+	}
+	host := strings.ToLower(parsed.Hostname())
+
+	// Authoritative GitHub hosts to which the token may be sent.
+	allowedHosts := []string{"github.com"}
+	if ghHost := os.Getenv("GH_HOST"); ghHost != "" {
+		// GH_HOST may carry a scheme prefix; extract just the hostname.
+		if u, parseErr := url.Parse(ghHost); parseErr == nil && u.Host != "" {
+			allowedHosts = append(allowedHosts, strings.ToLower(u.Hostname()))
+		} else {
+			allowedHosts = append(allowedHosts, strings.ToLower(strings.TrimPrefix(ghHost, "https://")))
+		}
+	}
+
+	isAllowed := false
+	for _, h := range allowedHosts {
+		if host == h {
+			isAllowed = true
+			break
+		}
+	}
+	if !isAllowed {
+		return
+	}
+
+	token := os.Getenv("GH_TOKEN")
+	if token == "" {
+		token = os.Getenv("GITHUB_TOKEN")
+	}
+	if token == "" {
+		return
+	}
+
+	req.Header.Set("Authorization", "Bearer "+token)
+}