From 99a3adac93b8c041580672019a2ce5ac66b1cd3f Mon Sep 17 00:00:00 2001 From: Isma <71719097+Doozers@users.noreply.github.com> Date: Fri, 24 Mar 2023 08:05:06 +0100 Subject: [PATCH] feat: split (#650) --- cmd/depviz/main.go | 147 +++++++++++++------ internal/dvcore/fetch.go | 41 ++++++ internal/dvcore/{run.go => gen.go} | 65 ++++---- internal/dvcore/{run_test.go => gen_test.go} | 2 +- internal/dvmodel/task.go | 22 +++ internal/dvstore/query_test.go | 2 +- 6 files changed, 193 insertions(+), 86 deletions(-) create mode 100644 internal/dvcore/fetch.go rename internal/dvcore/{run.go => gen.go} (86%) rename internal/dvcore/{run_test.go => gen_test.go} (98%) diff --git a/cmd/depviz/main.go b/cmd/depviz/main.go index a7a35c5ec..66240bf9f 100644 --- a/cmd/depviz/main.go +++ b/cmd/depviz/main.go @@ -62,18 +62,18 @@ var ( serverGitHubClientID = serverFlags.String("github-client-id", "", "GitHub client ID") serverGitHubClientSecret = serverFlags.String("github-client-secret", "", "GitHub client secret") - runFlags = flag.NewFlagSet("run", flag.ExitOnError) - runNoPull = runFlags.Bool("no-pull", false, "don't pull providers (graph only)") - runNoGraph = runFlags.Bool("no-graph", false, "don't generate graph (pull only)") - runResync = runFlags.Bool("resync", false, "resync already synced content") - runGitHubToken = runFlags.String("github-token", "", "GitHub token") - runNoPert = runFlags.Bool("no-pert", false, "disable PERT computing") - runFormat = runFlags.String("format", "dot", "output format") - runVertical = runFlags.Bool("vertical", false, "vertical mode") - runHidePRs = runFlags.Bool("hide-prs", false, "hide PRs") - runHideExternalDeps = runFlags.Bool("hide-external-deps", false, "hide dependencies outside of the specified targets") - runHideIsolated = runFlags.Bool("hide-isolated", false, "hide isolated tasks") - runShowClosed = runFlags.Bool("show-closed", false, "show closed tasks") + genFlags = flag.NewFlagSet("gen", flag.ExitOnError) + genNoGraph = genFlags.Bool("no-graph", false, "don't generate graph (pull only)") + genNoPert = genFlags.Bool("no-pert", false, "disable PERT computing") + genVertical = genFlags.Bool("vertical", false, "vertical mode") + genHidePRs = genFlags.Bool("hide-prs", false, "hide PRs") + genHideExternalDeps = genFlags.Bool("hide-external-deps", false, "hide dependencies outside of the specified targets") + genHideIsolated = genFlags.Bool("hide-isolated", false, "hide isolated tasks") + genShowClosed = genFlags.Bool("show-closed", false, "show closed tasks") + + fetchFlags = flag.NewFlagSet("fetch", flag.ExitOnError) + fetchGitHubToken = fetchFlags.String("github-token", "", "GitHub token") + fetchResync = fetchFlags.Bool("resync", false, "resync already synced content") ) func main() { @@ -121,17 +121,25 @@ func Main(args []string) error { // restore-json }, Exec: func(context.Context, []string) error { return flag.ErrHelp }, - }, { - Name: "run", - ShortHelp: "sync target urls and draw a graph", - ShortUsage: "run [flags] [url...]", - Exec: execRun, - FlagSet: runFlags, }, { Name: "server", ShortHelp: "start a depviz server with depviz API", FlagSet: serverFlags, Exec: execServer, + }, { + Name: "gen", + ShortHelp: "use the db to generate outputs, without requiring any fetch", + Subcommands: []*ffcli.Command{ + {Name: "graphviz", Exec: execGenGraphviz, ShortHelp: "generate graphviz output"}, + {Name: "json", Exec: execGenJSON, ShortHelp: "generate JSON output"}, + {Name: "csv", Exec: execGenCSV, ShortHelp: "generate CSV output"}, + }, + FlagSet: genFlags, + }, { + Name: "fetch", + ShortHelp: "fetch data from providers", + Exec: execFetch, + FlagSet: fetchFlags, }, }, Exec: func(context.Context, []string) error { return flag.ErrHelp }, @@ -245,34 +253,6 @@ func execStoreInfo(ctx context.Context, args []string) error { return dvcore.StoreInfo(store) } -func execRun(ctx context.Context, args []string) error { - if err := globalPreRun(); err != nil { - return err - } - - store, err := storeFromArgs() - if err != nil { - return fmt.Errorf("init store: %w", err) - } - - opts := dvcore.RunOpts{ - Logger: logger, - Schema: schemaConfig, - Vertical: *runVertical, - NoPert: *runNoPert, - NoGraph: *runNoGraph, - NoPull: *runNoPull, - Format: *runFormat, - Resync: *runResync, - GitHubToken: *runGitHubToken, - ShowClosed: *runShowClosed, - HideIsolated: *runHideIsolated, - HidePRs: *runHidePRs, - HideExternalDeps: *runHideExternalDeps, - } - return dvcore.Run(store, args, opts) -} - func execServer(ctx context.Context, args []string) error { if err := globalPreRun(); err != nil { return err @@ -365,3 +345,78 @@ func storeFromArgs() (*cayley.Handle, error) { return store, nil } + +func execGenGraphviz(ctx context.Context, args []string) error { + return fmt.Errorf("not implemented yet") +} + +func execGenJSON(ctx context.Context, args []string) error { + if err := globalPreRun(); err != nil { + return err + } + + store, err := storeFromArgs() + if err != nil { + return fmt.Errorf("init store: %w", err) + } + + opts := dvcore.GenOpts{ + Logger: logger, + Schema: schemaConfig, + Vertical: *genVertical, + NoPert: *genNoPert, + NoGraph: *genNoGraph, + ShowClosed: *genShowClosed, + HideIsolated: *genHideIsolated, + HidePRs: *genHidePRs, + HideExternalDeps: *genHideExternalDeps, + Format: "json", + } + + return dvcore.Gen(store, args, opts) +} + +func execGenCSV(ctx context.Context, args []string) error { + if err := globalPreRun(); err != nil { + return err + } + + store, err := storeFromArgs() + if err != nil { + return fmt.Errorf("init store: %w", err) + } + + opts := dvcore.GenOpts{ + Logger: logger, + Schema: schemaConfig, + Vertical: *genVertical, + NoPert: *genNoPert, + NoGraph: *genNoGraph, + ShowClosed: *genShowClosed, + HideIsolated: *genHideIsolated, + HidePRs: *genHidePRs, + HideExternalDeps: *genHideExternalDeps, + Format: "csv", + } + + return dvcore.Gen(store, args, opts) +} + +func execFetch(ctx context.Context, args []string) error { + if err := globalPreRun(); err != nil { + return err + } + + store, err := storeFromArgs() + if err != nil { + return fmt.Errorf("init store: %w", err) + } + + opts := dvcore.FetchOpts{ + Logger: logger, + Schema: schemaConfig, + GitHubToken: *fetchGitHubToken, + Resync: *fetchResync, + } + return dvcore.Fetch(store, args, opts) +} diff --git a/internal/dvcore/fetch.go b/internal/dvcore/fetch.go new file mode 100644 index 000000000..d20c752b4 --- /dev/null +++ b/internal/dvcore/fetch.go @@ -0,0 +1,41 @@ +package dvcore + +import ( + "fmt" + + "github.com/cayleygraph/cayley" + "github.com/cayleygraph/cayley/schema" + "go.uber.org/zap" + "moul.io/depviz/v3/internal/dvparser" + "moul.io/multipmuri" +) + +type FetchOpts struct { + Logger *zap.Logger + Schema *schema.Config + + GitHubToken string + Resync bool +} + +func Fetch(h *cayley.Handle, args []string, opts FetchOpts) error { + targets, err := dvparser.ParseTargets(args) + if err != nil { + return fmt.Errorf("parse targets: %w", err) + } + + _, err = PullAndSave(targets, h, opts.Schema, opts.GitHubToken, opts.Resync, opts.Logger) + return err +} + +func PullAndSave(targets []multipmuri.Entity, h *cayley.Handle, schema *schema.Config, githubToken string, resync bool, logger *zap.Logger) (bool, error) { + batches := pullBatches(targets, h, githubToken, resync, logger) + if len(batches) > 0 { + err := saveBatches(h, schema, batches) + if err != nil { + return false, fmt.Errorf("save batches: %w", err) + } + return true, nil + } + return false, nil +} diff --git a/internal/dvcore/run.go b/internal/dvcore/gen.go similarity index 86% rename from internal/dvcore/run.go rename to internal/dvcore/gen.go index ffe28fcc0..497c04504 100644 --- a/internal/dvcore/run.go +++ b/internal/dvcore/gen.go @@ -2,8 +2,10 @@ package dvcore import ( "context" + "encoding/csv" "encoding/json" "fmt" + "os" "sync" "github.com/cayleygraph/cayley" @@ -20,22 +22,13 @@ import ( "moul.io/multipmuri" ) -type RunOpts struct { +type GenOpts struct { // global - NoPull bool NoGraph bool Logger *zap.Logger Schema *schema.Config - // pull - - GitHubToken string - // GitLabToken string - // TrelloToken string - // JiraToken string - Resync bool - // graph Format string @@ -47,11 +40,11 @@ type RunOpts struct { HideExternalDeps bool } -func Run(h *cayley.Handle, args []string, opts RunOpts) error { +func Gen(h *cayley.Handle, args []string, opts GenOpts) error { if opts.Logger == nil { opts.Logger = zap.NewNop() } - opts.Logger.Debug("Run called", zap.Strings("args", args), zap.Any("opts", opts)) + opts.Logger.Debug("Gen called", zap.Strings("args", args), zap.Any("opts", opts)) // FIXME: support the world @@ -60,13 +53,6 @@ func Run(h *cayley.Handle, args []string, opts RunOpts) error { return fmt.Errorf("parse targets: %w", err) } - if !opts.NoPull { - _, err := PullAndSave(targets, h, opts.Schema, opts.GitHubToken, opts.Resync, opts.Logger) - if err != nil { - return fmt.Errorf("pull: %w", err) - } - } - if !opts.NoGraph { // nolint:nestif // load tasks filters := dvstore.LoadTasksFilters{ @@ -86,12 +72,9 @@ func Run(h *cayley.Handle, args []string, opts RunOpts) error { switch opts.Format { case "json": - out, err := json.MarshalIndent(tasks, "", " ") - if err != nil { - return err - } - fmt.Println(string(out)) - return nil + return genJSON(tasks) + case "csv": + return genCSV(tasks) case "graphman-pert": out, err := yaml.Marshal(pertConfig) if err != nil { @@ -152,18 +135,6 @@ func Run(h *cayley.Handle, args []string, opts RunOpts) error { return nil } -func PullAndSave(targets []multipmuri.Entity, h *cayley.Handle, schema *schema.Config, githubToken string, resync bool, logger *zap.Logger) (bool, error) { - batches := pullBatches(targets, h, githubToken, resync, logger) - if len(batches) > 0 { - err := saveBatches(h, schema, batches) - if err != nil { - return false, fmt.Errorf("save batches: %w", err) - } - return true, nil - } - return false, nil -} - func pullBatches(targets []multipmuri.Entity, h *cayley.Handle, githubToken string, resync bool, logger *zap.Logger) []dvmodel.Batch { // FIXME: handle the special '@me' target var ( @@ -264,7 +235,7 @@ func saveBatches(h *cayley.Handle, schema *schema.Config, batches []dvmodel.Batc return nil } -func graphmanPertConfig(tasks []dvmodel.Task, opts RunOpts) *graphman.PertConfig { +func graphmanPertConfig(tasks []dvmodel.Task, opts GenOpts) *graphman.PertConfig { opts.Logger.Debug("graphTargets", zap.Int("tasks", len(tasks)), zap.Any("opts", opts)) // initialize graph config @@ -315,3 +286,21 @@ func graphmanPertConfig(tasks []dvmodel.Task, opts RunOpts) *graphman.PertConfig return &config } + +func genJSON(tasks []dvmodel.Task) error { + out, err := json.MarshalIndent(tasks, "", " ") + if err != nil { + return err + } + fmt.Println(string(out)) + return nil +} + +func genCSV(tasks []dvmodel.Task) error { + csvTasks := make([][]string, len(tasks)) + for _, task := range tasks { + csvTasks = append(csvTasks, task.MarshalCSV()) + } + w := csv.NewWriter(os.Stdout) + return w.WriteAll(csvTasks) +} diff --git a/internal/dvcore/run_test.go b/internal/dvcore/gen_test.go similarity index 98% rename from internal/dvcore/run_test.go rename to internal/dvcore/gen_test.go index f2d9a5f18..c5f1a165e 100644 --- a/internal/dvcore/run_test.go +++ b/internal/dvcore/gen_test.go @@ -81,7 +81,7 @@ func TestPullAndSave(t *testing.T) { gp := dvstore.TestingGoldenDumpPath(t, test.name) if testutil.UpdateGolden() { t.Logf("update golden file: %s", gp) - err := os.WriteFile(gp, b.Bytes(), 0644) + err := os.WriteFile(gp, b.Bytes(), 0o644) assert.NoError(t, err, test.name) } diff --git a/internal/dvmodel/task.go b/internal/dvmodel/task.go index bc95ed9d1..fcc90541f 100644 --- a/internal/dvmodel/task.go +++ b/internal/dvmodel/task.go @@ -59,3 +59,25 @@ func FilterIsolatedTasks(in []Task, logger *zap.Logger) []Task { return out } + +func (t *Task) MarshalCSV() []string { + if t == nil { + return nil + } + return []string{ + t.ID.String(), + t.CreatedAt.String(), + t.UpdatedAt.String(), + t.LocalID, + t.Kind.String(), + t.Title, + t.Description, + t.Driver.String(), + t.State.String(), + t.EstimatedDuration, + t.HasAuthor.String(), + t.HasOwner.String(), + // t.IsDependingOn, + // t.IsBlocking, + } +} diff --git a/internal/dvstore/query_test.go b/internal/dvstore/query_test.go index 9850251da..f47e2ef97 100644 --- a/internal/dvstore/query_test.go +++ b/internal/dvstore/query_test.go @@ -62,7 +62,7 @@ func TestLoadTasks(t *testing.T) { if testutil.UpdateGolden() { t.Logf("update golden file: %s", gp) - err := os.WriteFile(gp, []byte(actual), 0644) + err := os.WriteFile(gp, []byte(actual), 0o644) assert.NoError(t, err, name) }