another implementation of "fetch-messages"

internal/jsonwriter/jsonwriter.go

@@ -0,0 +1,59 @@
+package jsonwriter
+
+import (
+	"encoding/json"
+	"os"
+)
+
+// WriteCloser writes objects as JSON array. It provides persistent layer for JSON value.
+type WriteCloser interface {
+	Write(interface{}) error
+	Close() error
+}
+
+type fileWriter struct {
+	name    string
+	reverse bool
+
+	// FIXME: いったん全部メモリにためるのであんまよくない
+	buf []interface{}
+}
+
+func (fw *fileWriter) Write(v interface{}) error {
+	// XXX: 排他してないのでgoroutineからは使えない
+	fw.buf = append(fw.buf, v)
+	return nil
+}
+
+func (fw *fileWriter) Close() error {
+	// FIXME: ファイルの作成が Close まで遅延している。本来なら CreateFile のタ
+	// イミングでやるのが好ましいが、いましばらく目を瞑る
+	f, err := os.Create(fw.name)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	if fw.reverse {
+		reverse(fw.buf)
+		fw.reverse = false
+	}
+	err = json.NewEncoder(f).Encode(fw.buf)
+	if err != nil {
+		return err
+	}
+	fw.buf = nil
+	return nil
+}
+
+// CreateFile creates a WriteCloser which implemented by file.
+func CreateFile(name string, reverse bool) (WriteCloser, error) {
+	return &fileWriter{name: name}, nil
+}
+
+func reverse(x []interface{}) {
+	for i, j := 0, len(x)-1; i < j; {
+		x[i], x[j] = x[j], x[i]
+		i++
+		j--
+	}
+}

internal/slackadapter/common.go

@@ -0,0 +1,20 @@
+package slackadapter
+
+// Error represents error response of Slack.
+type Error struct {
+	Ok  bool   `json:"ok"`
+	Err string `json:"error"`
+}
+
+// Error returns error message.
+func (err *Error) Error() string {
+	return err.Err
+}
+
+// NextCursor is cursor for next request.
+type NextCursor struct {
+	NextCursor Cursor `json:"next_cursor"`
+}
+
+// Cursor is type of cursor of Slack API.
+type Cursor string

internal/slackadapter/conversations_history.go

@@ -0,0 +1,33 @@
+package slackadapter
+
+import (
+	"context"
+	"errors"
+	"time"
+
+	"github.com/vim-jp/slacklog-generator/internal/slacklog"
+)
+
+// ConversationsHistoryParams is optional parameters for ConversationsHistory
+type ConversationsHistoryParams struct {
+	Cursor    Cursor     `json:"cursor,omitempty"`
+	Inclusive bool       `json:"inclusive,omitempty"`
+	Latest    *time.Time `json:"latest,omitempty"`
+	Limit     int        `json:"limit,omitempty"`
+	Oldest    *time.Time `json:"oldest,omitempty"`
+}
+
+// ConversationsHistoryReponse is response for ConversationsHistory
+type ConversationsHistoryReponse struct {
+	Ok               bool                `json:"ok"`
+	Messages         []*slacklog.Message `json:"messages,omitempty"`
+	HasMore          bool                `json:"has_more"`
+	PinCount         int                 `json:"pin_count"`
+	ResponseMetadata *NextCursor         `json:"response_metadata"`
+}
+
+// ConversationsHistory gets conversation messages in a channel.
+func ConversationsHistory(ctx context.Context, token, channel string, params ConversationsHistoryParams) (*ConversationsHistoryReponse, error) {
+	// TODO: call Slack's conversations.history
+	return nil, errors.New("not implemented yet")
+}

internal/slackadapter/cursor_iter.go

@@ -0,0 +1,35 @@
+package slackadapter
+
+import "context"
+
+// CursorIterator is requirements of IterateCursor iterates with cursor.
+type CursorIterator interface {
+	Iterate(context.Context, Cursor) (Cursor, error)
+}
+
+// CursorIteratorFunc is a function which implements CursorIterator.
+type CursorIteratorFunc func(context.Context, Cursor) (Cursor, error)
+
+// Iterate is an implementation for CursorIterator.
+func (fn CursorIteratorFunc) Iterate(ctx context.Context, c Cursor) (Cursor, error) {
+	return fn(ctx, c)
+}
+
+// IterateCursor iterates CursorIterator until returning empty cursor.
+func IterateCursor(ctx context.Context, iter CursorIterator) error {
+	var c Cursor
+	for {
+		err := ctx.Err()
+		if err != nil {
+			return err
+		}
+		next, err := iter.Iterate(ctx, c)
+		if err != nil {
+			return err
+		}
+		if next == Cursor("") {
+			return nil
+		}
+		c = next
+	}
+}

main.go

@@ -7,6 +7,7 @@ import (
 	"github.com/joho/godotenv"
 	cli "github.com/urfave/cli/v2"
 	"github.com/vim-jp/slacklog-generator/subcmd"
+	"github.com/vim-jp/slacklog-generator/subcmd/fetchmessages"
 	"github.com/vim-jp/slacklog-generator/subcmd/serve"
 )
 
@@ -27,6 +28,7 @@ func main() {
 		subcmd.DownloadFilesCommand,       // "download-files"
 		subcmd.GenerateHTMLCommand,        // "generate-html"
 		serve.Command,                     // "serve"
+		fetchmessages.NewCLICommand(),     // "fetch-messages"
 	}
 
 	err = app.Run(os.Args)

subcmd/fetchmessages/fetchmessages.go

@@ -0,0 +1,156 @@
+package fetchmessages
+
+import (
+	"context"
+	"errors"
+	"flag"
+	"os"
+	"path/filepath"
+	"time"
+
+	cli "github.com/urfave/cli/v2"
+	"github.com/vim-jp/slacklog-generator/internal/jsonwriter"
+	"github.com/vim-jp/slacklog-generator/internal/slackadapter"
+	"github.com/vim-jp/slacklog-generator/internal/slacklog"
+)
+
+const dateFormat = "2006-01-02"
+
+func toDateString(ti time.Time) string {
+	return ti.Format(dateFormat)
+}
+
+func parseDateString(s string) (time.Time, error) {
+	l, err := time.LoadLocation("Asia/Tokeyo")
+	if err != nil {
+		return time.Time{}, err
+	}
+	ti, err := time.ParseInLocation(dateFormat, s, l)
+	if err != nil {
+		return time.Time{}, err
+	}
+	return ti, nil
+}
+
+// Run runs "fetch-messages" sub-command. It fetch messages of a channel by a
+// day.
+func Run(args []string) error {
+	var (
+		token   string
+		datadir string
+		date    string
+		verbose bool
+	)
+	fs := flag.NewFlagSet("fetch-messages", flag.ExitOnError)
+	fs.StringVar(&token, "token", os.Getenv("SLACK_TOKEN"), `slack token. can be set by SLACK_TOKEN env var`)
+	fs.StringVar(&datadir, "datadir", "_logdata", `directory to load/save data`)
+	fs.StringVar(&date, "date", toDateString(time.Now()), `target date to get`)
+	fs.BoolVar(&verbose, "verbose", false, "verbose log")
+	err := fs.Parse(args)
+	if err != nil {
+		return err
+	}
+	if token == "" {
+		return errors.New("SLACK_TOKEN environment variable requied")
+	}
+	return run(token, datadir, date, verbose)
+}
+
+func run(token, datadir, date string, verbose bool) error {
+	oldest, err := parseDateString(date)
+	if err != nil {
+		return err
+	}
+	latest := oldest.AddDate(0, 0, 1)
+
+	ct, err := slacklog.NewChannelTable(filepath.Join(datadir, "channels.json"), []string{"*"})
+	if err != nil {
+		return err
+	}
+
+	for _, sch := range ct.Channels {
+		outfile := filepath.Join(datadir, sch.ID, toDateString(oldest)+".json")
+		fw, err := jsonwriter.CreateFile(outfile, true)
+		if err != nil {
+			return err
+		}
+		err = slackadapter.IterateCursor(context.Background(),
+			slackadapter.CursorIteratorFunc(func(ctx context.Context, c slackadapter.Cursor) (slackadapter.Cursor, error) {
+				r, err := slackadapter.ConversationsHistory(ctx, token, sch.ID, slackadapter.ConversationsHistoryParams{
+					Cursor: c,
+					Limit:  100,
+					Oldest: &oldest,
+					Latest: &latest,
+				})
+				if err != nil {
+					return "", err
+				}
+				for _, m := range r.Messages {
+					err := fw.Write(m)
+					if err != nil {
+						return "", err
+					}
+				}
+				if m := r.ResponseMetadata; r.HasMore && m != nil {
+					return m.NextCursor, nil
+				}
+				// HasMore && ResponseMetadata == nil は明らかにエラーだがいま
+				// は握りつぶしてる
+				return "", nil
+			}))
+		if err != nil {
+			// ロールバック相当が好ましいが今はまだその時期ではない
+			fw.Close()
+			return err
+		}
+		err = fw.Close()
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// NewCLICommand creates a cli.Command, which provides "fetch-messages"
+// sub-command.
+func NewCLICommand() *cli.Command {
+	var (
+		token   string
+		datadir string
+		date    string
+		verbose bool
+	)
+	return &cli.Command{
+		Name:  "fetch-messages",
+		Usage: "fetch messages of channel by day",
+		Action: func(c *cli.Context) error {
+			return run(token, datadir, date, verbose)
+		},
+		Flags: []cli.Flag{
+			&cli.StringFlag{
+				Name:        "token",
+				Usage:       "slack token",
+				EnvVars:     []string{"SLACK_TOKEN"},
+				Destination: &token,
+			},
+			&cli.StringFlag{
+				Name:        "datadir",
+				Usage:       "directory to load/save data",
+				Value:       "_logdata",
+				Destination: &datadir,
+			},
+			&cli.StringFlag{
+				Name:        "date",
+				Usage:       "target date to get",
+				Value:       toDateString(time.Now()),
+				Destination: &date,
+			},
+			&cli.BoolFlag{
+				Name:        "verbose",
+				Usage:       "verbose log",
+				Destination: &verbose,
+			},
+		},
+	}
+}