RFC-0001: Rfc 0001 impl (#1069)

Adds an Exporter interface and a noop exporter implementation with factory methods for construction

exporters/exporter.go

@@ -0,0 +1,72 @@
+package exporters
+
+import (
+	"github.com/algorand/go-algorand/agreement"
+	"github.com/algorand/go-algorand/data/bookkeeping"
+	"github.com/algorand/go-algorand/ledger/ledgercore"
+)
+
+// ExporterConfig is a generic string which can be deserialized by each individual Exporter.
+type ExporterConfig string
+
+// ExportData is the interface which all data types sent to Exporters should implement
+type ExportData interface {
+	Round() uint64
+}
+
+// BlockExportData is provided to the Exporter on each round.
+type BlockExportData struct {
+	// Block is the block data written to the blockchain.
+	Block bookkeeping.Block
+
+	// Delta contains a list of account changes resulting from the block. Processor plugins may have modify this data.
+	Delta ledgercore.StateDelta
+
+	// Certificate contains voting data that certifies the block. The certificate is non deterministic, a node stops collecting votes once the voting threshold is reached.
+	Certificate agreement.Certificate
+}
+
+// Round returns the round to which the BlockExportData corresponds
+func (blkData *BlockExportData) Round() uint64 {
+	return uint64(blkData.Block.Round())
+}
+
+// ExporterMetadata contains fields relevant to identification and description of plugins.
+type ExporterMetadata struct {
+	Name        string
+	Description string
+	Deprecated  bool
+}
+
+// Exporter defines the interface for plugins
+type Exporter interface {
+	// Metadata associated with each Exporter.
+	Metadata() ExporterMetadata
+
+	// Connect will be called during initialization, before block data starts going through the pipeline.
+	// Typically used for things like initializing network connections.
+	// The ExporterConfig passed to Connect will contain the Unmarhsalled config file specific to this plugin.
+	// Should return an error if it fails--this will result in the Indexer process terminating.
+	Connect(cfg ExporterConfig) error
+
+	// Config returns the configuration options used to create an Exporter.
+	// Initialized during Connect, it should return nil until the Exporter has been Connected.
+	Config() ExporterConfig
+
+	// Disconnect will be called during termination of the Indexer process.
+	// There is no guarantee that plugin lifecycle hooks will be invoked in any specific order in relation to one another.
+	// Returns an error if it fails which will be surfaced in the logs, but the process is already terminating.
+	Disconnect() error
+
+	// Receive is called for each block to be processed by the exporter.
+	// Should return an error on failure--retries are configurable.
+	Receive(exportData ExportData) error
+
+	// HandleGenesis is an Exporter's opportunity to do initial validation and handling of the Genesis block.
+	// If validation (such as a check to ensure `genesis` matches a previously stored genesis block) or handling fails,
+	// it returns an error.
+	HandleGenesis(genesis bookkeeping.Genesis) error
+
+	// Round returns the next round not yet processed by the Exporter. Atomically updated when Receive successfully completes.
+	Round() uint64
+}

exporters/exporter_factory.go

@@ -0,0 +1,40 @@
+package exporters
+
+import (
+	"fmt"
+)
+
+// ExporterConstructor must be implemented by each Exporter.
+// It provides a basic no-arg constructor for instances of an ExporterImpl.
+type ExporterConstructor interface {
+	// New should return an instantiation of an Exporter.
+	// Configuration values should be passed and can be processed during `Connect()`.
+	New() Exporter
+}
+
+// exporterImpls is a k/v store from exporter names to their constructor implementations.
+// This layer of indirection allows for different exporter integrations to be compiled in or compiled out by `go build --tags ...`
+var exporterImpls = make(map[string]ExporterConstructor)
+
+// RegisterExporter is used to register ExporterConstructor implementations. This mechanism allows
+// for loose coupling between the configuration and the implementation. It is extremely similar to the way sql.DB
+// driver's are configured and used.
+func RegisterExporter(name string, constructor ExporterConstructor) {
+	exporterImpls[name] = constructor
+}
+
+// ExporterByName is used to construct an Exporter object by name.
+// Returns an Exporter object, an availability channel that closes when the database
+// becomes available, and an error object.
+func ExporterByName(name string, cfg ExporterConfig) (Exporter, error) {
+	var constructor ExporterConstructor
+	var ok bool
+	if constructor, ok = exporterImpls[name]; !ok {
+		return nil, fmt.Errorf("no Exporter Constructor for %s", name)
+	}
+	val := constructor.New()
+	if err := val.Connect(cfg); err != nil {
+		return nil, err
+	}
+	return val, nil
+}

exporters/exporter_factory_test.go

@@ -0,0 +1,51 @@
+package exporters
+
+import (
+	"fmt"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/mock"
+	"testing"
+)
+
+type mockExporter struct {
+	mock.Mock
+	Exporter
+}
+
+func (m *mockExporter) Connect(config ExporterConfig) error {
+	args := m.Called(config)
+	return args.Error(0)
+}
+
+type mockExporterConstructor struct {
+	me *mockExporter
+}
+
+func (c *mockExporterConstructor) New() Exporter {
+	return c.me
+}
+
+func TestExporterByNameSuccess(t *testing.T) {
+	me := mockExporter{}
+	me.On("Connect", mock.Anything).Return(nil)
+	RegisterExporter("foobar", &mockExporterConstructor{&me})
+
+	exp, err := ExporterByName("foobar", "")
+	assert.NoError(t, err)
+	assert.Implements(t, (*Exporter)(nil), exp)
+}
+
+func TestExporterByNameNotFound(t *testing.T) {
+	_, err := ExporterByName("barfoo", "")
+	expectedErr := "no Exporter Constructor for barfoo"
+	assert.EqualError(t, err, expectedErr)
+}
+
+func TestExporterByNameConnectFailure(t *testing.T) {
+	me := mockExporter{}
+	expectedErr := fmt.Errorf("connect failure")
+	me.On("Connect", mock.Anything).Return(expectedErr)
+	RegisterExporter("baz", &mockExporterConstructor{&me})
+	_, err := ExporterByName("baz", "")
+	assert.EqualError(t, err, expectedErr.Error())
+}

exporters/noop/noop_exporter.go

@@ -0,0 +1,65 @@
+package noop
+
+import (
+	"github.com/algorand/go-algorand/data/bookkeeping"
+	"github.com/algorand/indexer/exporters"
+)
+
+// `noopExporter`s will function without ever erroring. This means they will also process out of order blocks
+// which may or may not be desirable for different use cases--it can hide errors in actual exporters expecting in order
+// block processing.
+// The `noopExporter` will maintain `Round` state according to the round of the last block it processed.
+type noopExporter struct {
+	round uint64
+	cfg   exporters.ExporterConfig
+}
+
+var noopExporterMetadata exporters.ExporterMetadata = exporters.ExporterMetadata{
+	Name:        "noop",
+	Description: "noop exporter",
+	Deprecated:  false,
+}
+
+// Constructor is the ExporterConstructor implementation for the "noop" exporter
+type Constructor struct{}
+
+// New initializes a noopExporter
+func (c *Constructor) New() exporters.Exporter {
+	return &noopExporter{
+		round: 0,
+		cfg:   "",
+	}
+}
+
+func (exp *noopExporter) Metadata() exporters.ExporterMetadata {
+	return noopExporterMetadata
+}
+
+func (exp *noopExporter) Connect(_ exporters.ExporterConfig) error {
+	return nil
+}
+
+func (exp *noopExporter) Config() exporters.ExporterConfig {
+	return exp.cfg
+}
+
+func (exp *noopExporter) Disconnect() error {
+	return nil
+}
+
+func (exp *noopExporter) Receive(exportData exporters.ExportData) error {
+	exp.round = exportData.Round() + 1
+	return nil
+}
+
+func (exp *noopExporter) HandleGenesis(_ bookkeeping.Genesis) error {
+	return nil
+}
+
+func (exp *noopExporter) Round() uint64 {
+	return exp.round
+}
+
+func init() {
+	exporters.RegisterExporter(noopExporterMetadata.Name, &Constructor{})
+}

exporters/noop/noop_exporter_test.go

@@ -0,0 +1,55 @@
+package noop
+
+import (
+	"github.com/algorand/go-algorand/data/bookkeeping"
+	"github.com/algorand/indexer/exporters"
+	"github.com/stretchr/testify/assert"
+	"testing"
+)
+
+var nc = &Constructor{}
+
+var ne = nc.New()
+
+func TestExporterByName(t *testing.T) {
+	exporters.RegisterExporter(noopExporterMetadata.Name, nc)
+	ne, err := exporters.ExporterByName(noopExporterMetadata.Name, "")
+	assert.NoError(t, err)
+	assert.Implements(t, (*exporters.Exporter)(nil), ne)
+}
+
+func TestExporterMetadata(t *testing.T) {
+	meta := ne.Metadata()
+	assert.Equal(t, noopExporterMetadata.Name, meta.Name)
+	assert.Equal(t, noopExporterMetadata.Description, meta.Description)
+	assert.Equal(t, noopExporterMetadata.Deprecated, meta.Deprecated)
+}
+
+func TestExporterConnect(t *testing.T) {
+	assert.NoError(t, ne.Connect(""))
+}
+
+func TestExporterConfig(t *testing.T) {
+	assert.Equal(t, exporters.ExporterConfig(""), ne.Config())
+}
+
+func TestExporterDisconnect(t *testing.T) {
+	assert.NoError(t, ne.Disconnect())
+}
+
+func TestExporterHandleGenesis(t *testing.T) {
+	assert.NoError(t, ne.HandleGenesis(bookkeeping.Genesis{}))
+}
+
+func TestExporterRoundReceive(t *testing.T) {
+	eData := &exporters.BlockExportData{
+		Block: bookkeeping.Block{
+			BlockHeader: bookkeeping.BlockHeader{
+				Round: 5,
+			},
+		},
+	}
+	assert.Equal(t, uint64(0), ne.Round())
+	assert.NoError(t, ne.Receive(eData))
+	assert.Equal(t, uint64(6), ne.Round())
+}