Moves motives to Badger

See #197

transform/badger.go

@@ -10,12 +10,54 @@ import (
 
 const badgerFilePrefix = "minha-receita-badger-"
 
+func removeLeadingZeros(n string) string {
+	var o string
+	isZero := true
+	for _, ch := range n {
+		if ch != '0' || !isZero {
+			o += string(ch)
+			isZero = false
+		}
+	}
+	if o == "" {
+		o = "0"
+	}
+	return o
+}
+
+func keyForMotives(n string) string  { return fmt.Sprintf("motives%s", removeLeadingZeros(n)) }
 func keyForPartners(n string) string { return fmt.Sprintf("partners%s", n) }
 func keyForBase(n string) string     { return fmt.Sprintf("base%s", n) }
 func keyForTaxes(n string) string    { return fmt.Sprintf("taxes%s", n) }
 
 // functions to read data from Badger
 
+func motivesOf(db *badger.DB, i int) (string, bool, error) {
+	var m string
+	found := false
+	n := fmt.Sprintf("%d", i)
+	err := db.View(func(txn *badger.Txn) error {
+		i, err := txn.Get([]byte(keyForMotives(n)))
+		if errors.Is(err, badger.ErrKeyNotFound) {
+			return nil
+		}
+		if err != nil {
+			return fmt.Errorf("could not get key %s: %w", keyForMotives(n), err)
+		}
+		v, err := i.ValueCopy(nil)
+		if err != nil {
+			return fmt.Errorf("could not read value for key %s: %w", keyForMotives(n), err)
+		}
+		m = string(v)
+		found = true
+		return nil
+	})
+	if err != nil {
+		return m, false, fmt.Errorf("error getting motive for %s: %w", n, err)
+	}
+	return m, found, nil
+}
+
 func partnersOf(db *badger.DB, n string) ([]partnerData, error) {
 	p := []partnerData{}
 	err := db.View(func(txn *badger.Txn) error {

transform/badger_test.go

@@ -200,3 +200,22 @@ func TestSaveAndReadItems(t *testing.T) {
 		})
 	}
 }
+
+func TestRemoveLeadingZeros(t *testing.T) {
+	for _, tc := range []struct {
+		value    string
+		expected string
+	}{
+		{"42", "42"},
+		{"420", "420"},
+		{"042", "42"},
+		{"0042", "42"},
+		{"test", "test"},
+		{"0", "0"},
+	} {
+		got := removeLeadingZeros(tc.value)
+		if got != tc.expected {
+			t.Errorf("expected %s, got %s", tc.expected, got)
+		}
+	}
+}

transform/kv.go

@@ -20,6 +20,11 @@ type item struct {
 func newKVItem(s sourceType, l *lookups, r []string) (i item, err error) {
 	var h func(l *lookups, r []string) ([]byte, error)
 	switch s {
+	case motives:
+		i.key = []byte(keyForMotives(r[0]))
+		h = func(_ *lookups, r []string) ([]byte, error) {
+			return []byte(r[1]), nil
+		}
 	case partners:
 		i.key = []byte(keyForPartners(r[0]))
 		h = loadPartnerRow
@@ -46,7 +51,7 @@ type badgerStorage struct {
 }
 
 func (kv *badgerStorage) load(dir string, l *lookups) error {
-	srcs, err := newSources(dir, []sourceType{base, partners, taxes})
+	srcs, err := newSources(dir, []sourceType{motives, base, partners, taxes})
 	if err != nil {
 		return fmt.Errorf("could not load sources: %w", err)
 	}
@@ -110,33 +115,53 @@ func (kv *badgerStorage) load(dir string, l *lookups) error {
 
 func (kv *badgerStorage) enrichCompany(c *company) error {
 	n := cnpj.Base(c.CNPJ)
+	ms := make(chan string)
 	ps := make(chan []partnerData)
 	bs := make(chan baseData)
 	ts := make(chan taxesData)
 	errs := make(chan error)
+	go func() {
+		if c.MotivoSituacaoCadastral == nil {
+			return
+		}
+		m, ok, err := motivesOf(kv.db, *c.MotivoSituacaoCadastral)
+		if err != nil {
+			errs <- err
+			return
+		}
+		if !ok {
+			return
+		}
+		ms <- m
+	}()
 	go func() {
 		p, err := partnersOf(kv.db, n)
 		if err != nil {
 			errs <- err
+			return
 		}
 		ps <- p
 	}()
 	go func() {
 		v, err := baseOf(kv.db, n)
 		if err != nil {
 			errs <- err
+			return
 		}
 		bs <- v
 	}()
 	go func() {
 		t, err := taxesOf(kv.db, n)
 		if err != nil {
 			errs <- err
+			return
 		}
 		ts <- t
 	}()
 	for i := 0; i < 3; i++ {
 		select {
+		case m := <-ms:
+			c.DescricaoMotivoSituacaoCadastral = &m
 		case p := <-ps:
 			c.QuadroSocietario = p
 		case v := <-bs:

transform/lookups.go

@@ -22,7 +22,6 @@ func newLookup(p string) (lookup, error) {
 }
 
 type lookups struct {
-	motives        lookup
 	cities         lookup
 	countries      lookup
 	cnaes          lookup
@@ -33,7 +32,7 @@ type lookups struct {
 
 func newLookups(d string) (lookups, error) {
 	var ls []lookup
-	srcs := []sourceType{motives, cities, countries, cnaes, qualifications, natures}
+	srcs := []sourceType{cities, countries, cnaes, qualifications, natures}
 	for _, src := range srcs {
 		paths, err := pathsForSource(src, d)
 		if err != nil {
@@ -54,7 +53,7 @@ func newLookups(d string) (lookups, error) {
 	if err != nil {
 		return lookups{}, fmt.Errorf("error creating ibge lookup: %w", err)
 	}
-	return lookups{ls[0], ls[1], ls[2], ls[3], ls[4], ls[5], c}, nil
+	return lookups{ls[0], ls[1], ls[2], ls[3], ls[4], c}, nil
 }
 
 func (c *company) motivoSituacaoCadastral(l *lookups, v string) error {
@@ -65,11 +64,7 @@ func (c *company) motivoSituacaoCadastral(l *lookups, v string) error {
 	if i == nil {
 		return nil
 	}
-	s := l.motives[*i]
 	c.MotivoSituacaoCadastral = i
-	if s != "" {
-		c.DescricaoMotivoSituacaoCadastral = &s
-	}
 	return nil
 }