From d939d70d89ddd02935aaf94f877ff73725e74694 Mon Sep 17 00:00:00 2001 From: Eric Warehime Date: Thu, 7 Jul 2022 09:21:28 -0700 Subject: [PATCH] RFC-0001: Rfc 0001 impl (#1069) Adds an Exporter interface and a noop exporter implementation with factory methods for construction --- exporters/exporter.go | 72 ++++++++++++++++++++++++++++ exporters/exporter_factory.go | 40 ++++++++++++++++ exporters/exporter_factory_test.go | 51 ++++++++++++++++++++ exporters/noop/noop_exporter.go | 65 +++++++++++++++++++++++++ exporters/noop/noop_exporter_test.go | 55 +++++++++++++++++++++ 5 files changed, 283 insertions(+) create mode 100644 exporters/exporter.go create mode 100644 exporters/exporter_factory.go create mode 100644 exporters/exporter_factory_test.go create mode 100644 exporters/noop/noop_exporter.go create mode 100644 exporters/noop/noop_exporter_test.go diff --git a/exporters/exporter.go b/exporters/exporter.go new file mode 100644 index 000000000..ad2a837e3 --- /dev/null +++ b/exporters/exporter.go @@ -0,0 +1,72 @@ +package exporters + +import ( + "github.com/algorand/go-algorand/agreement" + "github.com/algorand/go-algorand/data/bookkeeping" + "github.com/algorand/go-algorand/ledger/ledgercore" +) + +// ExporterConfig is a generic string which can be deserialized by each individual Exporter. +type ExporterConfig string + +// ExportData is the interface which all data types sent to Exporters should implement +type ExportData interface { + Round() uint64 +} + +// BlockExportData is provided to the Exporter on each round. +type BlockExportData struct { + // Block is the block data written to the blockchain. + Block bookkeeping.Block + + // Delta contains a list of account changes resulting from the block. Processor plugins may have modify this data. + Delta ledgercore.StateDelta + + // Certificate contains voting data that certifies the block. The certificate is non deterministic, a node stops collecting votes once the voting threshold is reached. + Certificate agreement.Certificate +} + +// Round returns the round to which the BlockExportData corresponds +func (blkData *BlockExportData) Round() uint64 { + return uint64(blkData.Block.Round()) +} + +// ExporterMetadata contains fields relevant to identification and description of plugins. +type ExporterMetadata struct { + Name string + Description string + Deprecated bool +} + +// Exporter defines the interface for plugins +type Exporter interface { + // Metadata associated with each Exporter. + Metadata() ExporterMetadata + + // Connect will be called during initialization, before block data starts going through the pipeline. + // Typically used for things like initializing network connections. + // The ExporterConfig passed to Connect will contain the Unmarhsalled config file specific to this plugin. + // Should return an error if it fails--this will result in the Indexer process terminating. + Connect(cfg ExporterConfig) error + + // Config returns the configuration options used to create an Exporter. + // Initialized during Connect, it should return nil until the Exporter has been Connected. + Config() ExporterConfig + + // Disconnect will be called during termination of the Indexer process. + // There is no guarantee that plugin lifecycle hooks will be invoked in any specific order in relation to one another. + // Returns an error if it fails which will be surfaced in the logs, but the process is already terminating. + Disconnect() error + + // Receive is called for each block to be processed by the exporter. + // Should return an error on failure--retries are configurable. + Receive(exportData ExportData) error + + // HandleGenesis is an Exporter's opportunity to do initial validation and handling of the Genesis block. + // If validation (such as a check to ensure `genesis` matches a previously stored genesis block) or handling fails, + // it returns an error. + HandleGenesis(genesis bookkeeping.Genesis) error + + // Round returns the next round not yet processed by the Exporter. Atomically updated when Receive successfully completes. + Round() uint64 +} diff --git a/exporters/exporter_factory.go b/exporters/exporter_factory.go new file mode 100644 index 000000000..5c29b511f --- /dev/null +++ b/exporters/exporter_factory.go @@ -0,0 +1,40 @@ +package exporters + +import ( + "fmt" +) + +// ExporterConstructor must be implemented by each Exporter. +// It provides a basic no-arg constructor for instances of an ExporterImpl. +type ExporterConstructor interface { + // New should return an instantiation of an Exporter. + // Configuration values should be passed and can be processed during `Connect()`. + New() Exporter +} + +// exporterImpls is a k/v store from exporter names to their constructor implementations. +// This layer of indirection allows for different exporter integrations to be compiled in or compiled out by `go build --tags ...` +var exporterImpls = make(map[string]ExporterConstructor) + +// RegisterExporter is used to register ExporterConstructor implementations. This mechanism allows +// for loose coupling between the configuration and the implementation. It is extremely similar to the way sql.DB +// driver's are configured and used. +func RegisterExporter(name string, constructor ExporterConstructor) { + exporterImpls[name] = constructor +} + +// ExporterByName is used to construct an Exporter object by name. +// Returns an Exporter object, an availability channel that closes when the database +// becomes available, and an error object. +func ExporterByName(name string, cfg ExporterConfig) (Exporter, error) { + var constructor ExporterConstructor + var ok bool + if constructor, ok = exporterImpls[name]; !ok { + return nil, fmt.Errorf("no Exporter Constructor for %s", name) + } + val := constructor.New() + if err := val.Connect(cfg); err != nil { + return nil, err + } + return val, nil +} diff --git a/exporters/exporter_factory_test.go b/exporters/exporter_factory_test.go new file mode 100644 index 000000000..0bf71b8ba --- /dev/null +++ b/exporters/exporter_factory_test.go @@ -0,0 +1,51 @@ +package exporters + +import ( + "fmt" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "testing" +) + +type mockExporter struct { + mock.Mock + Exporter +} + +func (m *mockExporter) Connect(config ExporterConfig) error { + args := m.Called(config) + return args.Error(0) +} + +type mockExporterConstructor struct { + me *mockExporter +} + +func (c *mockExporterConstructor) New() Exporter { + return c.me +} + +func TestExporterByNameSuccess(t *testing.T) { + me := mockExporter{} + me.On("Connect", mock.Anything).Return(nil) + RegisterExporter("foobar", &mockExporterConstructor{&me}) + + exp, err := ExporterByName("foobar", "") + assert.NoError(t, err) + assert.Implements(t, (*Exporter)(nil), exp) +} + +func TestExporterByNameNotFound(t *testing.T) { + _, err := ExporterByName("barfoo", "") + expectedErr := "no Exporter Constructor for barfoo" + assert.EqualError(t, err, expectedErr) +} + +func TestExporterByNameConnectFailure(t *testing.T) { + me := mockExporter{} + expectedErr := fmt.Errorf("connect failure") + me.On("Connect", mock.Anything).Return(expectedErr) + RegisterExporter("baz", &mockExporterConstructor{&me}) + _, err := ExporterByName("baz", "") + assert.EqualError(t, err, expectedErr.Error()) +} diff --git a/exporters/noop/noop_exporter.go b/exporters/noop/noop_exporter.go new file mode 100644 index 000000000..651992ce1 --- /dev/null +++ b/exporters/noop/noop_exporter.go @@ -0,0 +1,65 @@ +package noop + +import ( + "github.com/algorand/go-algorand/data/bookkeeping" + "github.com/algorand/indexer/exporters" +) + +// `noopExporter`s will function without ever erroring. This means they will also process out of order blocks +// which may or may not be desirable for different use cases--it can hide errors in actual exporters expecting in order +// block processing. +// The `noopExporter` will maintain `Round` state according to the round of the last block it processed. +type noopExporter struct { + round uint64 + cfg exporters.ExporterConfig +} + +var noopExporterMetadata exporters.ExporterMetadata = exporters.ExporterMetadata{ + Name: "noop", + Description: "noop exporter", + Deprecated: false, +} + +// Constructor is the ExporterConstructor implementation for the "noop" exporter +type Constructor struct{} + +// New initializes a noopExporter +func (c *Constructor) New() exporters.Exporter { + return &noopExporter{ + round: 0, + cfg: "", + } +} + +func (exp *noopExporter) Metadata() exporters.ExporterMetadata { + return noopExporterMetadata +} + +func (exp *noopExporter) Connect(_ exporters.ExporterConfig) error { + return nil +} + +func (exp *noopExporter) Config() exporters.ExporterConfig { + return exp.cfg +} + +func (exp *noopExporter) Disconnect() error { + return nil +} + +func (exp *noopExporter) Receive(exportData exporters.ExportData) error { + exp.round = exportData.Round() + 1 + return nil +} + +func (exp *noopExporter) HandleGenesis(_ bookkeeping.Genesis) error { + return nil +} + +func (exp *noopExporter) Round() uint64 { + return exp.round +} + +func init() { + exporters.RegisterExporter(noopExporterMetadata.Name, &Constructor{}) +} diff --git a/exporters/noop/noop_exporter_test.go b/exporters/noop/noop_exporter_test.go new file mode 100644 index 000000000..438189a84 --- /dev/null +++ b/exporters/noop/noop_exporter_test.go @@ -0,0 +1,55 @@ +package noop + +import ( + "github.com/algorand/go-algorand/data/bookkeeping" + "github.com/algorand/indexer/exporters" + "github.com/stretchr/testify/assert" + "testing" +) + +var nc = &Constructor{} + +var ne = nc.New() + +func TestExporterByName(t *testing.T) { + exporters.RegisterExporter(noopExporterMetadata.Name, nc) + ne, err := exporters.ExporterByName(noopExporterMetadata.Name, "") + assert.NoError(t, err) + assert.Implements(t, (*exporters.Exporter)(nil), ne) +} + +func TestExporterMetadata(t *testing.T) { + meta := ne.Metadata() + assert.Equal(t, noopExporterMetadata.Name, meta.Name) + assert.Equal(t, noopExporterMetadata.Description, meta.Description) + assert.Equal(t, noopExporterMetadata.Deprecated, meta.Deprecated) +} + +func TestExporterConnect(t *testing.T) { + assert.NoError(t, ne.Connect("")) +} + +func TestExporterConfig(t *testing.T) { + assert.Equal(t, exporters.ExporterConfig(""), ne.Config()) +} + +func TestExporterDisconnect(t *testing.T) { + assert.NoError(t, ne.Disconnect()) +} + +func TestExporterHandleGenesis(t *testing.T) { + assert.NoError(t, ne.HandleGenesis(bookkeeping.Genesis{})) +} + +func TestExporterRoundReceive(t *testing.T) { + eData := &exporters.BlockExportData{ + Block: bookkeeping.Block{ + BlockHeader: bookkeeping.BlockHeader{ + Round: 5, + }, + }, + } + assert.Equal(t, uint64(0), ne.Round()) + assert.NoError(t, ne.Receive(eData)) + assert.Equal(t, uint64(6), ne.Round()) +}