refactor: Rework client.DB to ensure interface contains only public t…

…ypes (#277)

* Rename var to collection name

Is clearer

* Rename GetCollection=>GetCollectionByName

Previous fn name was a bit ambigious, particularly due to the existance of GetCollectionBySchemaID, which has the same method signature.

* Remove schemaManager from DB interface

Exposes a large amount of internal generator types and is not nice on a public interface

* Use internal multistore instead of public fn

Pub fn is an oddity that might be removed, this is one of two refs to it

* Use blockstore over dagstore for public interfaces

Removes the defra-only DagStore type/name from the db interface - blockstore concept appears to be larger than defra, and there seems no sensible way to remove it from the interfaces (p2p lib requires one).  If Cid and Block should be considered standard, then blockstore should be too.

* Remove printDebugDb

Is a duplicate of the public PrintDump

* Remove public XYZStore accessors

None of these need to be public.  Leaves us with the (standard-typed, standard-concept) Root and Block stores only.

* Move store and txn interfaces into client

txn is dependent on store, and txn is on the public db+collection interfaces

* Reorder the DB interface

* Remove unused sequence interface

* Remove schema from document

Is not used, and creates extra dependency headaches when moving public items into client package.

* Remove blockstore.GetBlock

* Rename datastores to datastore

* Move store package to datastore dir

* Move store.go into datastore package

* Remove unwanted comment

api/http/api.go

@@ -179,7 +179,7 @@ func (s *Server) getBlock(w http.ResponseWriter, r *http.Request) {
 		c = cid.NewCidV1(cid.Raw, hash)
 	}
 
-	block, err := s.db.GetBlock(ctx, c)
+	block, err := s.db.Blockstore().Get(ctx, c)
 	if err != nil {
 		result.Errors = []interface{}{err.Error()}
 

bench/bench_util.go

@@ -90,7 +90,7 @@ func SetupCollections(b *testing.B, ctx context.Context, db *defradb.DB, fixture
 
 	// loop to get collections
 	for i := 0; i < numTypes; i++ {
-		col, err := db.GetCollection(ctx, fixture.TypeName(i))
+		col, err := db.GetCollectionByName(ctx, fixture.TypeName(i))
 		if err != nil {
 			return nil, fmt.Errorf("Couldn't get the collection %v: %w", fixture.TypeName(i), err)
 		}

cli/defradb/cmd/config.go

@@ -11,7 +11,7 @@
 package cmd
 
 import (
-	badgerds "github.com/sourcenetwork/defradb/datastores/badger/v3"
+	badgerds "github.com/sourcenetwork/defradb/datastore/badger/v3"
 	"github.com/sourcenetwork/defradb/logging"
 )
 

cli/defradb/cmd/serverdump.go

@@ -16,7 +16,7 @@ import (
 	"os/signal"
 
 	ds "github.com/ipfs/go-datastore"
-	badgerds "github.com/sourcenetwork/defradb/datastores/badger/v3"
+	badgerds "github.com/sourcenetwork/defradb/datastore/badger/v3"
 	"github.com/sourcenetwork/defradb/logging"
 	"github.com/spf13/cobra"
 

cli/defradb/cmd/start.go

@@ -20,7 +20,7 @@ import (
 	"time"
 
 	ma "github.com/multiformats/go-multiaddr"
-	badgerds "github.com/sourcenetwork/defradb/datastores/badger/v3"
+	badgerds "github.com/sourcenetwork/defradb/datastore/badger/v3"
 	"github.com/sourcenetwork/defradb/db"
 	netapi "github.com/sourcenetwork/defradb/net/api"
 	netpb "github.com/sourcenetwork/defradb/net/api/pb"

client/core.go

@@ -13,38 +13,32 @@ package client
 import (
 	"context"
 
-	"github.com/sourcenetwork/defradb/core"
+	"github.com/sourcenetwork/defradb/datastore"
 	"github.com/sourcenetwork/defradb/db/base"
 	"github.com/sourcenetwork/defradb/document"
 	"github.com/sourcenetwork/defradb/document/key"
-	"github.com/sourcenetwork/defradb/query/graphql/schema"
 
-	blocks "github.com/ipfs/go-block-format"
-	cid "github.com/ipfs/go-cid"
 	ds "github.com/ipfs/go-datastore"
+	blockstore "github.com/ipfs/go-ipfs-blockstore"
 )
 
 type DB interface {
-	// Collections
+	AddSchema(context.Context, string) error
+
 	CreateCollection(context.Context, base.CollectionDescription) (Collection, error)
-	GetCollection(context.Context, string) (Collection, error)
+	GetCollectionByName(context.Context, string) (Collection, error)
 	GetCollectionBySchemaID(context.Context, string) (Collection, error)
-	ExecQuery(context.Context, string) *QueryResult
-	SchemaManager() *schema.SchemaManager
-	AddSchema(context.Context, string) error
-	PrintDump(ctx context.Context)
-	GetBlock(ctx context.Context, c cid.Cid) (blocks.Block, error)
+	GetAllCollections(ctx context.Context) ([]Collection, error)
+	GetRelationshipIdField(fieldName, targetType, thisType string) (string, error)
+
 	Root() ds.Batching
-	// Rootstore() core.DSReaderWriter
-	// Headstore() core.DSReaderWriter
-	// Datastore() core.DSReaderWriter
-	DAGstore() core.DAGStore
+	Blockstore() blockstore.Blockstore
 
-	NewTxn(context.Context, bool) (core.Txn, error)
-	GetAllCollections(ctx context.Context) ([]Collection, error)
-}
+	NewTxn(context.Context, bool) (datastore.Txn, error)
+	ExecQuery(context.Context, string) *QueryResult
 
-type Sequence interface{}
+	PrintDump(ctx context.Context)
+}
 
 type Collection interface {
 	Description() base.CollectionDescription
@@ -77,7 +71,7 @@ type Collection interface {
 
 	Get(context.Context, key.DocKey) (*document.Document, error)
 
-	WithTxn(core.Txn) Collection
+	WithTxn(datastore.Txn) Collection
 
 	GetAllDocKeys(ctx context.Context) (<-chan DocKeysResult, error)
 }

core/crdt/base.go

@@ -17,19 +17,20 @@ import (
 
 	ds "github.com/ipfs/go-datastore"
 	"github.com/sourcenetwork/defradb/core"
+	"github.com/sourcenetwork/defradb/datastore"
 )
 
 // baseCRDT is embedded as a base layer into all
 // the core CRDT implementations to reduce code
 // duplication, and better manage the overhead
 // tasks that all the CRDTs need to implement anyway
 type baseCRDT struct {
-	store core.DSReaderWriter
+	store datastore.DSReaderWriter
 	key   core.DataStoreKey
 }
 
 // @TODO paramaterize ns/suffix
-func newBaseCRDT(store core.DSReaderWriter, key core.DataStoreKey) baseCRDT {
+func newBaseCRDT(store datastore.DSReaderWriter, key core.DataStoreKey) baseCRDT {
 	return baseCRDT{
 		store: store,
 		key:   key,

core/crdt/base_test.go

@@ -16,14 +16,14 @@ import (
 
 	ds "github.com/ipfs/go-datastore"
 	"github.com/sourcenetwork/defradb/core"
-	"github.com/sourcenetwork/defradb/store"
+	"github.com/sourcenetwork/defradb/datastore"
 )
 
-func newDS() core.DSReaderWriter {
-	return store.AsDSReaderWriter(ds.NewMapDatastore())
+func newDS() datastore.DSReaderWriter {
+	return datastore.AsDSReaderWriter(ds.NewMapDatastore())
 }
 
-func newSeededDS() core.DSReaderWriter {
+func newSeededDS() datastore.DSReaderWriter {
 	return newDS()
 }
 

core/crdt/composite.go

@@ -18,6 +18,7 @@ import (
 	"strings"
 
 	"github.com/sourcenetwork/defradb/core"
+	"github.com/sourcenetwork/defradb/datastore"
 
 	ipld "github.com/ipfs/go-ipld-format"
 	dag "github.com/ipfs/go-merkledag"
@@ -80,7 +81,7 @@ type CompositeDAG struct {
 	schemaID string
 }
 
-func NewCompositeDAG(store core.DSReaderWriter, schemaID string, namespace core.Key, key string) CompositeDAG {
+func NewCompositeDAG(store datastore.DSReaderWriter, schemaID string, namespace core.Key, key string) CompositeDAG {
 	return CompositeDAG{
 		key:      key,
 		schemaID: schemaID,

core/crdt/lwwreg.go

@@ -20,6 +20,7 @@ import (
 	"errors"
 
 	"github.com/sourcenetwork/defradb/core"
+	"github.com/sourcenetwork/defradb/datastore"
 
 	ipld "github.com/ipfs/go-ipld-format"
 	dag "github.com/ipfs/go-merkledag"
@@ -80,7 +81,7 @@ type LWWRegister struct {
 }
 
 // NewLWWRegister returns a new instance of the LWWReg with the given ID
-func NewLWWRegister(store core.DSReaderWriter, key core.DataStoreKey) LWWRegister {
+func NewLWWRegister(store datastore.DSReaderWriter, key core.DataStoreKey) LWWRegister {
 	return LWWRegister{
 		baseCRDT: newBaseCRDT(store, key),
 		// id:    id,

core/crdt/lwwreg_test.go

@@ -18,7 +18,7 @@ import (
 	"github.com/ipfs/go-cid"
 	ds "github.com/ipfs/go-datastore"
 	"github.com/sourcenetwork/defradb/core"
-	"github.com/sourcenetwork/defradb/store"
+	"github.com/sourcenetwork/defradb/datastore"
 
 	"github.com/ugorji/go/codec"
 
@@ -27,8 +27,8 @@ import (
 	mh "github.com/multiformats/go-multihash"
 )
 
-func newMockStore() core.DSReaderWriter {
-	return store.AsDSReaderWriter(ds.NewMapDatastore())
+func newMockStore() datastore.DSReaderWriter {
+	return datastore.AsDSReaderWriter(ds.NewMapDatastore())
 }
 
 func setupLWWRegister() LWWRegister {

datastores/badger/v3/datastore.go → datastore/badger/v3/datastore.go

@@ -18,7 +18,7 @@ import (
 	dsq "github.com/ipfs/go-datastore/query"
 	logger "github.com/ipfs/go-log/v2"
 	goprocess "github.com/jbenet/goprocess"
-	"github.com/sourcenetwork/defradb/datastores/iterable"
+	"github.com/sourcenetwork/defradb/datastore/iterable"
 	"go.uber.org/zap"
 )
 

datastores/badger/v3/iterator.go → datastore/badger/v3/iterator.go

@@ -20,7 +20,7 @@ import (
 	ds "github.com/ipfs/go-datastore"
 	dsq "github.com/ipfs/go-datastore/query"
 	goprocess "github.com/jbenet/goprocess"
-	"github.com/sourcenetwork/defradb/datastores/iterable"
+	"github.com/sourcenetwork/defradb/datastore/iterable"
 )
 
 type BadgerIterator struct {

store/blockstore.go → datastore/blockstore.go

@@ -8,14 +8,12 @@
 // by the Apache License, Version 2.0, included in the file
 // licenses/APL.txt.
 
-package store
+package datastore
 
 import (
 	"context"
 	"errors"
 
-	"github.com/sourcenetwork/defradb/core"
-
 	blocks "github.com/ipfs/go-block-format"
 	"github.com/ipfs/go-cid"
 	ds "github.com/ipfs/go-datastore"
@@ -27,15 +25,15 @@ import (
 // Blockstore implementation taken from https://github.com/ipfs/go-ipfs-blockstore/blob/master/blockstore.go
 // Needed a custom implementation that didn't rely on the ds.Batching interface.
 //
-// All datastore operations in DefraDB are interfaced by core.DSReaderWriter. This simplifies the interface to just
+// All datastore operations in DefraDB are interfaced by DSReaderWriter. This simplifies the interface to just
 // that of read/write operations, leaving the management of the datastore to the parent objects. This also allows
 // us to swap between a regular ds.Datastore, and a ds.Txn which as of https://github.com/ipfs/go-datastore/issues/114
 // no longer implements ds.Datastore.
 //
 // The original blockstore.Blockstore implementation relied on ds.Batching, so it could internally use store.Batch()
 // to optimize the PutMany function. However, in DefraDB, since we rely on a single rootstore for all our various
 // substores (data, heads, blocks), which includes a Txn/Batch system already, our respective substores don't need
-// to optimize or worry about Batching/Txn. Hence the simplified core.DSReaderWriter.
+// to optimize or worry about Batching/Txn. Hence the simplified DSReaderWriter.
 
 // ErrHashMismatch is an error returned when the hash of a block
 // is different than expected.
@@ -48,14 +46,14 @@ var ErrNotFound = errors.New("blockstore: block not found")
 
 // NewBlockstore returns a default Blockstore implementation
 // using the provided datastore.Batching backend.
-func NewBlockstore(store core.DSReaderWriter) blockstore.Blockstore {
+func NewBlockstore(store DSReaderWriter) blockstore.Blockstore {
 	return &bstore{
 		store: store,
 	}
 }
 
 type bstore struct {
-	store core.DSReaderWriter
+	store DSReaderWriter
 
 	rehash bool
 }

store/dag.go → datastore/dag.go

@@ -8,25 +8,23 @@
 // by the Apache License, Version 2.0, included in the file
 // licenses/APL.txt.
 
-package store
+package datastore
 
 import (
-	"github.com/sourcenetwork/defradb/core"
-
 	blockstore "github.com/ipfs/go-ipfs-blockstore"
 )
 
 // DAGStore is the interface to the underlying BlockStore and BlockService
 type dagStore struct {
 	blockstore.Blockstore // become a Blockstore
-	store                 core.DSReaderWriter
+	store                 DSReaderWriter
 	// bstore          blockstore.Blockstore
 	// bserv           blockservice.BlockService
 }
 
 // NewDAGStore creates a new DAGStore with the supplied
 // Batching datastore
-func NewDAGStore(store core.DSReaderWriter) core.DAGStore {
+func NewDAGStore(store DSReaderWriter) DAGStore {
 	dstore := &dagStore{
 		Blockstore: NewBlockstore(store),
 		store:      store,