Merge pull request #1 from mlhamel/refactor-packages-structure

Refactor packages structure

pkg/app/faktory.go

@@ -6,8 +6,8 @@ import (
 	"github.com/mlhamel/trieugene/pkg/config"
 	"github.com/mlhamel/trieugene/pkg/jobs"
 	"github.com/mlhamel/trieugene/pkg/store"
-	rougecombien "github.com/mlhamel/trieugene/services/rougecombien/pkg/jobs"
-	"github.com/mlhamel/trieugene/services/rougecombien/pkg/scraper"
+	rougecombienJobs "github.com/mlhamel/trieugene/services/rougecombien/pkg/jobs"
+	rougecombienScraper "github.com/mlhamel/trieugene/services/rougecombien/pkg/scraper"
 	"github.com/pior/runnable"
 )
 
@@ -23,10 +23,10 @@ func NewFaktory(cfg *config.Config, store store.Store) runnable.Runnable {
 
 	manager := jobs.NewFaktoryManager(cfg)
 
-	httpScraper := scraper.NewHttpScraper(cfg)
-	parser := scraper.NewParser(cfg, httpScraper)
+	httpScraper := rougecombienScraper.NewHttpScraper(cfg)
+	parser := rougecombienScraper.NewParser(cfg, httpScraper)
 
-	csvJob := rougecombien.NewCsvJob(&rougecombien.CsvJobKwargs{
+	csvJob := rougecombienJobs.NewCsvJob(&rougecombienJobs.CsvJobKwargs{
 		Cfg:      cfg,
 		Manager:  manager,
 		StoreJob: csvStoreJob,
@@ -36,7 +36,7 @@ func NewFaktory(cfg *config.Config, store store.Store) runnable.Runnable {
 
 	manager.Register(jsonStoreJob)
 	manager.Register(csvStoreJob)
-	manager.Register(rougecombien.NewJsonJob(cfg, manager, jsonStoreJob))
+	manager.Register(rougecombienJobs.NewJsonJob(cfg, manager, jsonStoreJob))
 	manager.Register(csvJob)
 
 	return &Faktory{

services/rougecombien/pkg/scraper/interface.go → pkg/scraper/interface.go

@@ -23,12 +23,12 @@ type Consumer func(context.Context, Result) error
 type Result struct {
 	ScrapedAt time.Time
 	TakenAt   time.Time
-	Outflow   float64
+	Outflow   interface{}
 }
 
 func (r *Result) Sha1() string {
 	hasher := sha1.New()
-	_, err := hasher.Write([]byte(fmt.Sprintf("%d:%f", r.TakenAt, r.Outflow)))
+	_, err := hasher.Write([]byte(fmt.Sprintf("%d:%f", r.TakenAt.Unix(), r.Outflow)))
 	if err != nil {
 		return ""
 	}
@@ -38,7 +38,7 @@ func (r *Result) Sha1() string {
 
 func (r *Result) MD5() string {
 	hasher := md5.New()
-	_, err := io.WriteString(hasher, fmt.Sprintf("%d:%f", r.TakenAt, r.Outflow))
+	_, err := io.WriteString(hasher, fmt.Sprintf("%d:%f", r.TakenAt.Unix(), r.Outflow))
 	if err != nil {
 		return ""
 	}

services/rougecombien/pkg/jobs/csv_job.go

@@ -5,15 +5,15 @@ import (
 
 	"github.com/mlhamel/trieugene/pkg/config"
 	trieugene "github.com/mlhamel/trieugene/pkg/jobs"
-	"github.com/mlhamel/trieugene/services/rougecombien/pkg/scraper"
+	base "github.com/mlhamel/trieugene/pkg/scraper"
 )
 
 type CsvJobKwargs struct {
 	Cfg      *config.Config
 	Manager  trieugene.Manager
 	StoreJob trieugene.Job
-	Parser   scraper.Parser
-	Scraper  scraper.Scraper
+	Parser   base.Parser
+	Scraper  base.Scraper
 }
 
 type CsvJob struct {
@@ -29,11 +29,11 @@ func (o *CsvJob) Kind() string {
 }
 
 func (o *CsvJob) Perform(ctx context.Context, args ...interface{}) error {
-	var results = make(map[int][]scraper.Result)
+	var results = make(map[int][]base.Result)
 
-	err := o.kwargs.Parser.Run(ctx, func(ctx context.Context, result scraper.Result) error {
+	err := o.kwargs.Parser.Run(ctx, func(ctx context.Context, result base.Result) error {
 		if len(results[result.TakenAt.Day()]) <= 0 {
-			results[result.TakenAt.Day()] = []scraper.Result{}
+			results[result.TakenAt.Day()] = []base.Result{}
 		}
 		results[result.TakenAt.Day()] = append(results[result.TakenAt.Day()], result)
 		return nil

services/rougecombien/pkg/jobs/csv_job_test.go

@@ -0,0 +1,29 @@
+package jobs
+
+import (
+	"context"
+	"testing"
+
+	"github.com/mlhamel/trieugene/pkg/config"
+	trieugene "github.com/mlhamel/trieugene/pkg/jobs"
+	"github.com/mlhamel/trieugene/services/rougecombien/pkg/scraper"
+)
+
+func Test(t *testing.T) {
+	ctx := context.Background()
+	cfg := config.NewConfig()
+	manager := trieugene.NewDummyManager(cfg)
+	expectedJob := trieugene.ExpectsJob{}
+	dummyScraper := scraper.NewDummyScraper(cfg)
+	parser := scraper.NewParser(cfg, dummyScraper)
+
+	csvJob := NewCsvJob(&CsvJobKwargs{
+		Cfg:      cfg,
+		Manager:  manager,
+		StoreJob: &expectedJob,
+		Parser:   parser,
+		Scraper:  dummyScraper,
+	})
+
+	csvJob.Run(ctx)
+}

services/rougecombien/pkg/jobs/json_job.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/mlhamel/trieugene/pkg/config"
 	trieugene "github.com/mlhamel/trieugene/pkg/jobs"
+	base "github.com/mlhamel/trieugene/pkg/scraper"
 	"github.com/mlhamel/trieugene/services/rougecombien/pkg/scraper"
 )
 
@@ -30,7 +31,7 @@ func (o *JsonJob) Perform(ctx context.Context, args ...interface{}) error {
 	httpScraper := scraper.NewHttpScraper(o.cfg)
 	parser := scraper.NewParser(o.cfg, httpScraper)
 
-	return parser.Run(ctx, func(ctx context.Context, result scraper.Result) error {
+	return parser.Run(ctx, func(ctx context.Context, result base.Result) error {
 		return o.manager.Perform(o.storejob, &trieugene.Message{
 			ID:          result.Sha1(),
 			Kind:        o.storejob.Kind(),

services/rougecombien/pkg/scraper/dummy_scraper.go → services/rougecombien/pkg/scraper/dummy.go

@@ -4,13 +4,14 @@ import (
 	"context"
 
 	"github.com/mlhamel/trieugene/pkg/config"
+	"github.com/mlhamel/trieugene/pkg/scraper"
 )
 
 type DummyScraper struct {
 	cfg *config.Config
 }
 
-func NewDummyScraper(cfg *config.Config) Scraper {
+func NewDummyScraper(cfg *config.Config) scraper.Scraper {
 	return &HttpScraper{
 		cfg: cfg,
 	}

services/rougecombien/pkg/scraper/parser.go

@@ -8,28 +8,29 @@ import (
 	"time"
 
 	"github.com/mlhamel/trieugene/pkg/config"
+	"github.com/mlhamel/trieugene/pkg/scraper"
 )
 
 type parserImpl struct {
 	cfg     *config.Config
-	scraper Scraper
+	scraper scraper.Scraper
 }
 
-func NewParser(cfg *config.Config, scraper Scraper) Parser {
+func NewParser(cfg *config.Config, scraper scraper.Scraper) scraper.Parser {
 	return &parserImpl{
 		cfg:     cfg,
 		scraper: scraper,
 	}
 }
 
-func (p *parserImpl) Run(ctx context.Context, consumer Consumer) error {
+func (p *parserImpl) Run(ctx context.Context, consumer scraper.Consumer) error {
 	data, err := p.scraper.Run(ctx)
 
 	if err != nil {
 		return err
 	}
 
-	var record Result
+	var record scraper.Result
 
 	if err != nil {
 		return fmt.Errorf("Error while parsing: %w", err)
@@ -71,7 +72,7 @@ func (p *parserImpl) Run(ctx context.Context, consumer Consumer) error {
 			Info().
 			Time("ScrapedAt", record.ScrapedAt).
 			Time("TakenAt", record.TakenAt).
-			Float64("Outflow", record.Outflow).
+			Interface("Outflow", record.Outflow).
 			Msg("Record parsed")
 
 		if err = consumer(ctx, record); err != nil {