From efd156fe9e31a5ccd4e6e2da4eff9ec9ddac946b Mon Sep 17 00:00:00 2001 From: Elad Date: Fri, 22 Mar 2019 17:43:17 +0900 Subject: [PATCH 01/31] swarm/shed: added tags as int vector to shed item --- swarm/storage/localstore/localstore.go | 1 + swarm/storage/localstore/localstore_test.go | 14 +++++++------- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/swarm/storage/localstore/localstore.go b/swarm/storage/localstore/localstore.go index e25c65699d..4fc86338ae 100644 --- a/swarm/storage/localstore/localstore.go +++ b/swarm/storage/localstore/localstore.go @@ -274,6 +274,7 @@ func New(path string, baseKey []byte, o *Options) (db *DB, err error) { // create a pull syncing triggers used by SubscribePull function db.pullTriggers = make(map[uint8][]chan struct{}) // push index contains as yet unsynced chunks + // Tags is []uint64 db.pushIndex, err = db.shed.NewIndex("StoreTimestamp|Hash->Tags", shed.IndexFuncs{ EncodeKey: func(fields shed.Item) (key []byte, err error) { key = make([]byte, 40) diff --git a/swarm/storage/localstore/localstore_test.go b/swarm/storage/localstore/localstore_test.go index 2dc5e0d462..f070d7a618 100644 --- a/swarm/storage/localstore/localstore_test.go +++ b/swarm/storage/localstore/localstore_test.go @@ -223,7 +223,7 @@ func newRetrieveIndexesTest(db *DB, chunk chunk.Chunk, storeTimestamp, accessTim if err != nil { t.Fatal(err) } - validateItem(t, item, chunk.Address(), chunk.Data(), storeTimestamp, 0) + validateItem(t, item, chunk.Address(), chunk.Data(), storeTimestamp, 0, []uint64{}) // access index should not be set wantErr := leveldb.ErrNotFound @@ -242,14 +242,14 @@ func newRetrieveIndexesTestWithAccess(db *DB, ch chunk.Chunk, storeTimestamp, ac if err != nil { t.Fatal(err) } - validateItem(t, item, ch.Address(), ch.Data(), storeTimestamp, 0) + validateItem(t, item, ch.Address(), ch.Data(), storeTimestamp, []uint64) if accessTimestamp > 0 { item, err = db.retrievalAccessIndex.Get(addressToItem(ch.Address())) if err != nil { t.Fatal(err) } - validateItem(t, item, ch.Address(), nil, 0, accessTimestamp) + validateItem(t, item, ch.Address(), nil, 0, accessTimestamp, []uint64{}) } } } @@ -266,7 +266,7 @@ func newPullIndexTest(db *DB, ch chunk.Chunk, binID uint64, wantError error) fun t.Errorf("got error %v, want %v", err, wantError) } if err == nil { - validateItem(t, item, ch.Address(), nil, 0, 0) + validateItem(t, item, ch.Address(), nil, 0, []uint64{}) } } } @@ -283,7 +283,7 @@ func newPushIndexTest(db *DB, ch chunk.Chunk, storeTimestamp int64, wantError er t.Errorf("got error %v, want %v", err, wantError) } if err == nil { - validateItem(t, item, ch.Address(), nil, storeTimestamp, 0) + validateItem(t, item, ch.Address(), nil, storeTimestamp, []uint64{}) } } } @@ -300,7 +300,7 @@ func newGCIndexTest(db *DB, chunk chunk.Chunk, storeTimestamp, accessTimestamp i if err != nil { t.Fatal(err) } - validateItem(t, item, chunk.Address(), nil, 0, accessTimestamp) + validateItem(t, item, chunk.Address(), nil, 0, accessTimestamp, []uint64{}) } } @@ -376,7 +376,7 @@ func testItemsOrder(t *testing.T, i shed.Index, chunks []testIndexChunk, sortFun } // validateItem is a helper function that checks Item values. -func validateItem(t *testing.T, item shed.Item, address, data []byte, storeTimestamp, accessTimestamp int64) { +func validateItem(t *testing.T, item shed.Item, address, data []byte, storeTimestamp, accessTimestamp int64, tags []uint64) { t.Helper() if !bytes.Equal(item.Address, address) { From 52406a3771654fae16c7b7c822c1ba8e1282f6dc Mon Sep 17 00:00:00 2001 From: Elad Date: Fri, 22 Mar 2019 18:20:14 +0900 Subject: [PATCH 02/31] swarm/storage: account for random tags added to chunk --- swarm/chunk/chunk.go | 9 ++++++++- swarm/storage/localstore/export.go | 4 ++-- swarm/storage/localstore/localstore_test.go | 16 +++++++++++++++- swarm/storage/localstore/mode_get.go | 2 +- swarm/storage/localstore/subscription_push.go | 2 +- 5 files changed, 27 insertions(+), 6 deletions(-) diff --git a/swarm/chunk/chunk.go b/swarm/chunk/chunk.go index c57d55ecc4..8ecffcfc8c 100644 --- a/swarm/chunk/chunk.go +++ b/swarm/chunk/chunk.go @@ -22,17 +22,20 @@ var ( type Chunk interface { Address() Address Data() []byte + Tags() []uint64 } type chunk struct { addr Address sdata []byte + tags []uint64 } -func NewChunk(addr Address, data []byte) Chunk { +func NewChunk(addr Address, data []byte, tags []uint64) Chunk { return &chunk{ addr: addr, sdata: data, + tags: tags, } } @@ -44,6 +47,10 @@ func (c *chunk) Data() []byte { return c.sdata } +func (c *chunk) Tags() []uint64 { + return c.tags +} + func (self *chunk) String() string { return fmt.Sprintf("Address: %v Chunksize: %v", self.addr.Log(), len(self.sdata)) } diff --git a/swarm/storage/localstore/export.go b/swarm/storage/localstore/export.go index 42585d59df..94db474db3 100644 --- a/swarm/storage/localstore/export.go +++ b/swarm/storage/localstore/export.go @@ -146,9 +146,9 @@ func (db *DB) Import(r io.Reader) (count int64, err error) { // LDBStore Export exported chunk data prefixed with the chunk key. // That is not necessary, as the key is in the chunk filename, // but backward compatibility needs to be preserved. - ch = chunk.NewChunk(key, data[32:]) + ch = chunk.NewChunk(key, data[32:], nil) case currentExportVersion: - ch = chunk.NewChunk(key, data) + ch = chunk.NewChunk(key, data, nil) default: select { case errC <- fmt.Errorf("unsupported export data version %q", version): diff --git a/swarm/storage/localstore/localstore_test.go b/swarm/storage/localstore/localstore_test.go index f070d7a618..371205f902 100644 --- a/swarm/storage/localstore/localstore_test.go +++ b/swarm/storage/localstore/localstore_test.go @@ -182,7 +182,13 @@ func generateTestRandomChunk() chunk.Chunk { rand.Read(data) key := make([]byte, 32) rand.Read(key) - return chunk.NewChunk(key, data) + r := rand.New(rand.NewSource(time.Now().UnixNano())) + n := r.Intn(10) + tags := []uint64{} + for i := 0; i < n; i++ { + tags = append(tags, r.Uint64()) + } + return chunk.NewChunk(key, data, tags) } // TestGenerateTestRandomChunk validates that @@ -213,6 +219,14 @@ func TestGenerateTestRandomChunk(t *testing.T) { if bytes.Equal(c1.Data(), c2.Data()) { t.Error("fake chunks data bytes do not differ") } + for i, _ := range c1.Tags() { + if i > len(c2.Tags())-1 { + break + } + if c1.Tags()[i] == c2.Tags()[i] { + t.Fatal("tags should be different") + } + } } // newRetrieveIndexesTest returns a test function that validates if the right diff --git a/swarm/storage/localstore/mode_get.go b/swarm/storage/localstore/mode_get.go index 0df0e9b7d4..c177860a1c 100644 --- a/swarm/storage/localstore/mode_get.go +++ b/swarm/storage/localstore/mode_get.go @@ -38,7 +38,7 @@ func (db *DB) Get(_ context.Context, mode chunk.ModeGet, addr chunk.Address) (ch } return nil, err } - return chunk.NewChunk(out.Address, out.Data), nil + return chunk.NewChunk(out.Address, out.Data, nil), nil } // get returns Item from the retrieval index diff --git a/swarm/storage/localstore/subscription_push.go b/swarm/storage/localstore/subscription_push.go index 5cbc2eb6ff..6fea59c38d 100644 --- a/swarm/storage/localstore/subscription_push.go +++ b/swarm/storage/localstore/subscription_push.go @@ -65,7 +65,7 @@ func (db *DB) SubscribePush(ctx context.Context) (c <-chan chunk.Chunk, stop fun } select { - case chunks <- chunk.NewChunk(dataItem.Address, dataItem.Data): + case chunks <- chunk.NewChunk(dataItem.Address, dataItem.Data, nil): // set next iteration start item // when its chunk is successfully sent to channel sinceItem = &item From d20050f0a28b9cc5f7a4fe4a400377fe02b56980 Mon Sep 17 00:00:00 2001 From: Elad Date: Mon, 25 Mar 2019 20:40:25 +0900 Subject: [PATCH 03/31] swarm/storage: wip tags --- swarm/storage/localstore/localstore.go | 1 + 1 file changed, 1 insertion(+) diff --git a/swarm/storage/localstore/localstore.go b/swarm/storage/localstore/localstore.go index 4fc86338ae..fa73c69f47 100644 --- a/swarm/storage/localstore/localstore.go +++ b/swarm/storage/localstore/localstore.go @@ -374,6 +374,7 @@ func chunkToItem(ch chunk.Chunk) shed.Item { return shed.Item{ Address: ch.Address(), Data: ch.Data(), + Tags: ch.Tags(), } } From bac0592b9ca48d897db9f3b9f422ba238348aa33 Mon Sep 17 00:00:00 2001 From: Elad Date: Mon, 25 Mar 2019 21:15:09 +0900 Subject: [PATCH 04/31] swarm/storage: fix tags decode, cleanup, fix tests to check the correct tags --- swarm/storage/localstore/localstore_test.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/swarm/storage/localstore/localstore_test.go b/swarm/storage/localstore/localstore_test.go index 371205f902..3e01402e03 100644 --- a/swarm/storage/localstore/localstore_test.go +++ b/swarm/storage/localstore/localstore_test.go @@ -297,7 +297,7 @@ func newPushIndexTest(db *DB, ch chunk.Chunk, storeTimestamp int64, wantError er t.Errorf("got error %v, want %v", err, wantError) } if err == nil { - validateItem(t, item, ch.Address(), nil, storeTimestamp, []uint64{}) + validateItem(t, item, ch.Address(), nil, storeTimestamp, ch.Tags()) } } } @@ -392,7 +392,6 @@ func testItemsOrder(t *testing.T, i shed.Index, chunks []testIndexChunk, sortFun // validateItem is a helper function that checks Item values. func validateItem(t *testing.T, item shed.Item, address, data []byte, storeTimestamp, accessTimestamp int64, tags []uint64) { t.Helper() - if !bytes.Equal(item.Address, address) { t.Errorf("got item address %x, want %x", item.Address, address) } @@ -405,6 +404,16 @@ func validateItem(t *testing.T, item shed.Item, address, data []byte, storeTimes if item.AccessTimestamp != accessTimestamp { t.Errorf("got item access timestamp %v, want %v", item.AccessTimestamp, accessTimestamp) } + count := 0 + for i, v := range tags { + if item.Tags[i] != v { + t.Errorf("expected item %d to equal %d but got %d", i, v, tags[i]) + } + count++ + } + if count != len(tags) { + t.Errorf("did not process enough tags, want %d got %d", len(item.Tags), count) + } } // setNow replaces now function and From 3494723f6a6c94b3c3a8662ec969185a5fc07097 Mon Sep 17 00:00:00 2001 From: Elad Date: Tue, 26 Mar 2019 19:20:17 +0900 Subject: [PATCH 05/31] swarm: build is green pre-hasherstore integration --- swarm/network/stream/delivery.go | 2 +- swarm/network/stream/messages.go | 2 +- swarm/storage/feed/request.go | 2 +- swarm/storage/hasherstore.go | 3 ++- swarm/storage/types.go | 10 +++++++++- 5 files changed, 14 insertions(+), 5 deletions(-) diff --git a/swarm/network/stream/delivery.go b/swarm/network/stream/delivery.go index 6fa9f3ad19..71e7c40017 100644 --- a/swarm/network/stream/delivery.go +++ b/swarm/network/stream/delivery.go @@ -262,7 +262,7 @@ func (d *Delivery) handleChunkDeliveryMsg(ctx context.Context, sp *Peer, req int msg.peer = sp log.Trace("handle.chunk.delivery", "put", msg.Addr) - err := d.chunkStore.Put(ctx, mode, storage.NewChunk(msg.Addr, msg.SData)) + err := d.chunkStore.Put(ctx, mode, storage.NewChunk(msg.Addr, msg.SData, []uint64{})) //TODO: TAGS if err != nil { if err == storage.ErrChunkInvalid { // we removed this log because it spams the logs diff --git a/swarm/network/stream/messages.go b/swarm/network/stream/messages.go index b293724cc7..bf7ac8535a 100644 --- a/swarm/network/stream/messages.go +++ b/swarm/network/stream/messages.go @@ -356,7 +356,7 @@ func (p *Peer) handleWantedHashesMsg(ctx context.Context, req *WantedHashesMsg) if err != nil { return fmt.Errorf("handleWantedHashesMsg get data %x: %v", hash, err) } - chunk := storage.NewChunk(hash, data) + chunk := storage.NewChunk(hash, data, []uint64{}) syncing := true if err := p.Deliver(ctx, chunk, s.priority, syncing); err != nil { return err diff --git a/swarm/storage/feed/request.go b/swarm/storage/feed/request.go index dd91a7cf45..dd1c746c7d 100644 --- a/swarm/storage/feed/request.go +++ b/swarm/storage/feed/request.go @@ -166,7 +166,7 @@ func (r *Request) toChunk() (storage.Chunk, error) { // signature is the last item in the chunk data copy(r.binaryData[updateLength:], r.Signature[:]) - chunk := storage.NewChunk(r.idAddr, r.binaryData) + chunk := storage.NewChunk(r.idAddr, r.binaryData, []uint64{}) //TODO: FIX THIS return chunk, nil } diff --git a/swarm/storage/hasherstore.go b/swarm/storage/hasherstore.go index b7880da2fc..e0d4982df8 100644 --- a/swarm/storage/hasherstore.go +++ b/swarm/storage/hasherstore.go @@ -158,7 +158,8 @@ func (h *hasherStore) createHash(chunkData ChunkData) Address { func (h *hasherStore) createChunk(chunkData ChunkData) Chunk { hash := h.createHash(chunkData) - chunk := NewChunk(hash, chunkData) + //TODO: CHANGE THIS, should come from context + chunk := NewChunk(hash, chunkData, []uint64{}) return chunk } diff --git a/swarm/storage/types.go b/swarm/storage/types.go index 0c7b16a671..8bc779fee5 100644 --- a/swarm/storage/types.go +++ b/swarm/storage/types.go @@ -23,6 +23,8 @@ import ( "crypto/rand" "encoding/binary" "io" + mrand "math/rand" + "time" "github.com/ethereum/go-ethereum/swarm/bmt" "github.com/ethereum/go-ethereum/swarm/chunk" @@ -95,7 +97,13 @@ func GenerateRandomChunk(dataSize int64) Chunk { binary.LittleEndian.PutUint64(sdata[:8], uint64(dataSize)) hasher.ResetWithLength(sdata[:8]) hasher.Write(sdata[8:]) - return NewChunk(hasher.Sum(nil), sdata) + r := mrand.New(mrand.NewSource(time.Now().UnixNano())) + n := r.Intn(10) + tags := []uint64{} + for i := 0; i < n; i++ { + tags = append(tags, r.Uint64()) + } + return NewChunk(hasher.Sum(nil), sdata, tags) } func GenerateRandomChunks(dataSize int64, count int) (chunks []Chunk) { From bf8b0e65e1ecd4e9234c368bbc7d203c5af95924 Mon Sep 17 00:00:00 2001 From: Elad Date: Tue, 26 Mar 2019 20:37:45 +0900 Subject: [PATCH 06/31] swarm/chunk, swarm/sctx, swarm/storage: add hasherstore test to inject tag to chunk from context, add getter mode to get tags from localstore --- swarm/chunk/chunk.go | 2 ++ swarm/sctx/sctx.go | 1 + swarm/storage/hasherstore.go | 13 ++++++++++ swarm/storage/hasherstore_test.go | 37 +++++++++++++++++++++++++--- swarm/storage/localstore/mode_get.go | 6 +++++ swarm/storage/types.go | 11 +++++++++ 6 files changed, 67 insertions(+), 3 deletions(-) diff --git a/swarm/chunk/chunk.go b/swarm/chunk/chunk.go index 8ecffcfc8c..73d0fcf1a0 100644 --- a/swarm/chunk/chunk.go +++ b/swarm/chunk/chunk.go @@ -127,6 +127,8 @@ const ( ModeGetSync // ModeGetLookup: when accessed to lookup a a chunk in feeds or other places ModeGetLookup + + ModeGetTags ) // ModePut enumerates different Putter modes. diff --git a/swarm/sctx/sctx.go b/swarm/sctx/sctx.go index fb7d35b000..f74b1875fb 100644 --- a/swarm/sctx/sctx.go +++ b/swarm/sctx/sctx.go @@ -5,6 +5,7 @@ import "context" type ( HTTPRequestIDKey struct{} requestHostKey struct{} + PushTagKey struct{} ) func SetHost(ctx context.Context, domain string) context.Context { diff --git a/swarm/storage/hasherstore.go b/swarm/storage/hasherstore.go index e0d4982df8..c4536e872d 100644 --- a/swarm/storage/hasherstore.go +++ b/swarm/storage/hasherstore.go @@ -110,6 +110,19 @@ func (h *hasherStore) Get(ctx context.Context, ref Reference) (ChunkData, error) return chunkData, nil } +func (h *hasherStore) GetTags(ctx context.Context, ref Reference) ([]uint64, error) { + addr, _, err := parseReference(ref, h.hashSize) + if err != nil { + return nil, err + } + + chunk, err := h.store.Get(ctx, chunk.ModeGetTags, addr) + if err != nil { + return nil, err + } + return chunk.Tags(), nil +} + // Close indicates that no more chunks will be put with the hasherStore, so the Wait // function can return when all the previously put chunks has been stored. func (h *hasherStore) Close() { diff --git a/swarm/storage/hasherstore_test.go b/swarm/storage/hasherstore_test.go index c95537db73..e1cfe0c88e 100644 --- a/swarm/storage/hasherstore_test.go +++ b/swarm/storage/hasherstore_test.go @@ -19,10 +19,13 @@ package storage import ( "bytes" "context" + mrand "math/rand" "testing" + "time" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/swarm/chunk" + "github.com/ethereum/go-ethereum/swarm/sctx" "github.com/ethereum/go-ethereum/swarm/storage/encryption" ) @@ -45,16 +48,19 @@ func TestHasherStore(t *testing.T) { chunkStore := NewMapChunkStore() hasherStore := NewHasherStore(chunkStore, MakeHashFunc(DefaultHash), tt.toEncrypt) + r := mrand.New(mrand.NewSource(time.Now().UnixNano())) + customTag := r.Uint64() // Put two random chunks into the hasherStore - chunkData1 := GenerateRandomChunk(int64(tt.chunkLength)).Data() - ctx, cancel := context.WithTimeout(context.Background(), getTimeout) + chunkData1 := GenerateRandomChunkWithTag(int64(tt.chunkLength), customTag).Data() + parentCtx := context.WithValue(context.Background(), sctx.PushTagKey{}, customTag) + ctx, cancel := context.WithTimeout(parentCtx, getTimeout) defer cancel() key1, err := hasherStore.Put(ctx, chunkData1) if err != nil { t.Fatalf("Expected no error got \"%v\"", err) } - chunkData2 := GenerateRandomChunk(int64(tt.chunkLength)).Data() + chunkData2 := GenerateRandomChunkWithTag(int64(tt.chunkLength), customTag).Data() key2, err := hasherStore.Put(ctx, chunkData2) if err != nil { t.Fatalf("Expected no error got \"%v\"", err) @@ -79,6 +85,17 @@ func TestHasherStore(t *testing.T) { t.Fatalf("Expected retrieved chunk data %v, got %v", common.Bytes2Hex(chunkData1), common.Bytes2Hex(retrievedChunkData1)) } + tags, err := hasherStore.GetTags(ctx, key1) + if err != nil { + t.Fatalf("had an error fetching tags from hasher store: %v", err) + } + if len(tags) != 1 { + t.Fatalf("tag length mismatch, want %d got %d", 1, len(tags)) + } + if tags[0] != customTag { + t.Fatalf("tag mismatch. want %d, got %d", customTag, tags[0]) + } + // Get the second chunk retrievedChunkData2, err := hasherStore.Get(ctx, key2) if err != nil { @@ -90,6 +107,20 @@ func TestHasherStore(t *testing.T) { t.Fatalf("Expected retrieved chunk data %v, got %v", common.Bytes2Hex(chunkData2), common.Bytes2Hex(retrievedChunkData2)) } + // get the tags from the underlying localstore and assert they are equal to the tag assigned + // in the context above (hasherstore should put the file with the appropriate tags from Context + // rather than from the chunk struct) + tags2, err := hasherStore.GetTags(ctx, key2) + if err != nil { + t.Fatalf("had an error fetching tags from hasher store: %v", err) + } + if len(tags2) != 1 { + t.Fatalf("tag length mismatch, want %d got %d", 1, len(tags2)) + } + if tags2[0] != customTag { + t.Fatalf("tag mismatch. want %d, got %d", customTag, tags2[0]) + } + hash1, encryptionKey1, err := parseReference(key1, hasherStore.hashSize) if err != nil { t.Fatalf("Expected no error, got \"%v\"", err) diff --git a/swarm/storage/localstore/mode_get.go b/swarm/storage/localstore/mode_get.go index c177860a1c..7d8654603c 100644 --- a/swarm/storage/localstore/mode_get.go +++ b/swarm/storage/localstore/mode_get.go @@ -79,6 +79,12 @@ func (db *DB) get(mode chunk.ModeGet, addr chunk.Address) (out shed.Item, err er // no updates to indexes case chunk.ModeGetSync: case chunk.ModeGetLookup: + case chunk.ModeGetTags: + c, err := db.pushIndex.Get(out) + if err != nil { + return out, err + } + return c, nil default: return out, ErrInvalidMode } diff --git a/swarm/storage/types.go b/swarm/storage/types.go index 8bc779fee5..8e2db7ad10 100644 --- a/swarm/storage/types.go +++ b/swarm/storage/types.go @@ -90,6 +90,17 @@ type Chunk = chunk.Chunk // NewChunk is the same as chunk.NewChunk for backward compatibility. var NewChunk = chunk.NewChunk +func GenerateRandomChunkWithTag(dataSize int64, tag uint64) Chunk { + hasher := MakeHashFunc(DefaultHash)() + sdata := make([]byte, dataSize+8) + rand.Read(sdata[8:]) + binary.LittleEndian.PutUint64(sdata[:8], uint64(dataSize)) + hasher.ResetWithLength(sdata[:8]) + hasher.Write(sdata[8:]) + + return NewChunk(hasher.Sum(nil), sdata, []uint64{tag}) +} + func GenerateRandomChunk(dataSize int64) Chunk { hasher := MakeHashFunc(DefaultHash)() sdata := make([]byte, dataSize+8) From f6fb305582337d707fa65d117efef8b274d66b42 Mon Sep 17 00:00:00 2001 From: Elad Date: Tue, 26 Mar 2019 20:48:02 +0900 Subject: [PATCH 07/31] swarm/storage: make hasherstore test pass --- swarm/sctx/sctx.go | 14 +++++++++++++- swarm/storage/hasherstore.go | 13 +++++++++---- swarm/storage/hasherstore_test.go | 3 +-- 3 files changed, 23 insertions(+), 7 deletions(-) diff --git a/swarm/sctx/sctx.go b/swarm/sctx/sctx.go index f74b1875fb..a30e351d92 100644 --- a/swarm/sctx/sctx.go +++ b/swarm/sctx/sctx.go @@ -5,7 +5,7 @@ import "context" type ( HTTPRequestIDKey struct{} requestHostKey struct{} - PushTagKey struct{} + pushTagKey struct{} ) func SetHost(ctx context.Context, domain string) context.Context { @@ -19,3 +19,15 @@ func GetHost(ctx context.Context) string { } return "" } + +func SetPushTag(ctx context.Context, tag uint64) context.Context { + return context.WithValue(ctx, pushTagKey{}, tag) +} + +func GetPushTag(ctx context.Context) uint64 { + v, ok := ctx.Value(pushTagKey{}).(uint64) + if ok { + return v + } + return 0 +} diff --git a/swarm/storage/hasherstore.go b/swarm/storage/hasherstore.go index c4536e872d..a08c797081 100644 --- a/swarm/storage/hasherstore.go +++ b/swarm/storage/hasherstore.go @@ -22,6 +22,7 @@ import ( "sync/atomic" "github.com/ethereum/go-ethereum/swarm/chunk" + "github.com/ethereum/go-ethereum/swarm/sctx" "github.com/ethereum/go-ethereum/swarm/storage/encryption" "golang.org/x/crypto/sha3" ) @@ -78,7 +79,7 @@ func (h *hasherStore) Put(ctx context.Context, chunkData ChunkData) (Reference, return nil, err } } - chunk := h.createChunk(c) + chunk := h.createChunk(ctx, c) h.storeChunk(ctx, chunk) return Reference(append(chunk.Address(), encryptionKey...)), nil @@ -169,10 +170,14 @@ func (h *hasherStore) createHash(chunkData ChunkData) Address { return hasher.Sum(nil) } -func (h *hasherStore) createChunk(chunkData ChunkData) Chunk { +func (h *hasherStore) createChunk(ctx context.Context, chunkData ChunkData) Chunk { hash := h.createHash(chunkData) - //TODO: CHANGE THIS, should come from context - chunk := NewChunk(hash, chunkData, []uint64{}) + tag := sctx.GetPushTag(ctx) + tags := []uint64{} + if tag != 0 { + tags = append(tags, tag) + } + chunk := NewChunk(hash, chunkData, tags) return chunk } diff --git a/swarm/storage/hasherstore_test.go b/swarm/storage/hasherstore_test.go index e1cfe0c88e..af300ae1ea 100644 --- a/swarm/storage/hasherstore_test.go +++ b/swarm/storage/hasherstore_test.go @@ -52,8 +52,7 @@ func TestHasherStore(t *testing.T) { customTag := r.Uint64() // Put two random chunks into the hasherStore chunkData1 := GenerateRandomChunkWithTag(int64(tt.chunkLength), customTag).Data() - parentCtx := context.WithValue(context.Background(), sctx.PushTagKey{}, customTag) - ctx, cancel := context.WithTimeout(parentCtx, getTimeout) + ctx, cancel := context.WithTimeout(sctx.SetPushTag(context.Background(), customTag), getTimeout) defer cancel() key1, err := hasherStore.Put(ctx, chunkData1) if err != nil { From 8f26b956d8c942103a7fa71e678c1694f4c20a1e Mon Sep 17 00:00:00 2001 From: Elad Date: Fri, 29 Mar 2019 17:48:23 +0900 Subject: [PATCH 08/31] swarm/api, swarm/chunk, swarm/storage: start to trickle tags through api package, add mode put tags (wip) --- swarm/api/api.go | 5 +++++ swarm/api/manifest.go | 7 ++++++- swarm/chunk/chunk.go | 4 +++- swarm/storage/filestore.go | 15 +++++++++++++++ swarm/storage/hasherstore.go | 7 ++++++- 5 files changed, 35 insertions(+), 3 deletions(-) diff --git a/swarm/api/api.go b/swarm/api/api.go index 86c1119232..332752c8a8 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -874,6 +874,11 @@ func (a *API) AppendFile(ctx context.Context, mhash, path, fname string, existin return fkey, newMkey.String(), nil } +// CreateTag creates a new push tag and stores it in localstore +func (a *API) CreateTag(filename string, timestamp uint64) (uint64, error) { + return a.fileStore.CreateTag(filename, timestamp) +} + // BuildDirectoryTree used by swarmfs_unix func (a *API) BuildDirectoryTree(ctx context.Context, mhash string, nameresolver bool) (addr storage.Address, manifestEntryMap map[string]*manifestTrieEntry, err error) { diff --git a/swarm/api/manifest.go b/swarm/api/manifest.go index 890ed88bd4..a12888e786 100644 --- a/swarm/api/manifest.go +++ b/swarm/api/manifest.go @@ -27,6 +27,7 @@ import ( "strings" "time" + "github.com/ethereum/go-ethereum/swarm/sctx" "github.com/ethereum/go-ethereum/swarm/storage/feed" "github.com/ethereum/go-ethereum/common" @@ -118,10 +119,14 @@ func (a *API) NewManifestWriter(ctx context.Context, addr storage.Address, quitC // AddEntry stores the given data and adds the resulting address to the manifest func (m *ManifestWriter) AddEntry(ctx context.Context, data io.Reader, e *ManifestEntry) (addr storage.Address, err error) { + + now := time.Now().Unix() + ctxTag := m.api.CreateTag(e.Path, now) + childCtx := sctx.SetPushTag(ctx, ctxTag) entry := newManifestTrieEntry(e, nil) if data != nil { var wait func(context.Context) error - addr, wait, err = m.api.Store(ctx, data, e.Size, m.trie.encrypted) + addr, wait, err = m.api.Store(childCtx, data, e.Size, m.trie.encrypted) if err != nil { return nil, err } diff --git a/swarm/chunk/chunk.go b/swarm/chunk/chunk.go index 73d0fcf1a0..85bbd69eda 100644 --- a/swarm/chunk/chunk.go +++ b/swarm/chunk/chunk.go @@ -127,7 +127,7 @@ const ( ModeGetSync // ModeGetLookup: when accessed to lookup a a chunk in feeds or other places ModeGetLookup - + // ModeGetTags: gets the tags for a given reference ModeGetTags ) @@ -142,6 +142,8 @@ const ( ModePutSync // ModePutUpload: when a chunk is created by local upload ModePutUpload + // ModePutTags + ModePutTags ) // ModeSet enumerates different Setter modes. diff --git a/swarm/storage/filestore.go b/swarm/storage/filestore.go index 2b15f7da68..d15134a09f 100644 --- a/swarm/storage/filestore.go +++ b/swarm/storage/filestore.go @@ -18,10 +18,12 @@ package storage import ( "context" + "encoding/binary" "io" "sort" "sync" + "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/swarm/chunk" "github.com/ethereum/go-ethereum/swarm/storage/localstore" ) @@ -99,6 +101,19 @@ func (f *FileStore) HashSize() int { return f.hashFunc().Size() } +// CreateTag creates a new push tag and stores it in localstore +// it returns the tag as uint64 +func (f *FileStore) CreateTag(filename string, timestamp uint64) (uint64, error) { + intBuf := make([]byte, 8) + binary.BigEndian.PutUint64(intBuf, now) + // Tag is SHA3(filename|storetimestamp)[:8] + tag := make([]byte, 8) + buf := []byte(hdr.Name) + buf = append(buf, intBuf) + tagHash := crypto.Keccak256(buf)[:8] + +} + // GetAllReferences is a public API. This endpoint returns all chunk hashes (only) for a given file func (f *FileStore) GetAllReferences(ctx context.Context, data io.Reader, toEncrypt bool) (addrs AddressCollection, err error) { // create a special kind of putter, which only will store the references diff --git a/swarm/storage/hasherstore.go b/swarm/storage/hasherstore.go index a08c797081..07f8222a5e 100644 --- a/swarm/storage/hasherstore.go +++ b/swarm/storage/hasherstore.go @@ -23,6 +23,7 @@ import ( "github.com/ethereum/go-ethereum/swarm/chunk" "github.com/ethereum/go-ethereum/swarm/sctx" + "github.com/ethereum/go-ethereum/swarm/storage" "github.com/ethereum/go-ethereum/swarm/storage/encryption" "golang.org/x/crypto/sha3" ) @@ -79,6 +80,7 @@ func (h *hasherStore) Put(ctx context.Context, chunkData ChunkData) (Reference, return nil, err } } + h.GetTags() chunk := h.createChunk(ctx, c) h.storeChunk(ctx, chunk) @@ -172,8 +174,11 @@ func (h *hasherStore) createHash(chunkData ChunkData) Address { func (h *hasherStore) createChunk(ctx context.Context, chunkData ChunkData) Chunk { hash := h.createHash(chunkData) + tags, err := h.GetTags(ctx, storage.Address(hash)) // TODO: this is really bad but if we want to persist tags across sessions this would be the way + if err != nil { + panic("wtf") + } tag := sctx.GetPushTag(ctx) - tags := []uint64{} if tag != 0 { tags = append(tags, tag) } From 581cf47c7ab8780fbefa0dad6776b4abce2c0f1c Mon Sep 17 00:00:00 2001 From: Elad Date: Fri, 29 Mar 2019 22:13:04 +0900 Subject: [PATCH 09/31] swarm/storage, swarm/shed: add generic index functionality (wip) --- swarm/chunk/chunk.go | 2 +- swarm/shed/generic_index.go | 344 ++++++++++++++++++++++ swarm/shed/generic_index_test.go | 266 +++++++++++++++++ swarm/shed/index.go | 1 - swarm/storage/localstore/localstore.go | 23 ++ swarm/storage/localstore/mode_put.go | 2 + swarm/storage/localstore/mode_put_test.go | 22 ++ 7 files changed, 658 insertions(+), 2 deletions(-) create mode 100644 swarm/shed/generic_index.go create mode 100644 swarm/shed/generic_index_test.go diff --git a/swarm/chunk/chunk.go b/swarm/chunk/chunk.go index 85bbd69eda..02c6116d25 100644 --- a/swarm/chunk/chunk.go +++ b/swarm/chunk/chunk.go @@ -142,7 +142,7 @@ const ( ModePutSync // ModePutUpload: when a chunk is created by local upload ModePutUpload - // ModePutTags + // ModePutTags: used to store a new tag when a file is uploaded ModePutTags ) diff --git a/swarm/shed/generic_index.go b/swarm/shed/generic_index.go new file mode 100644 index 0000000000..1ffbf478b0 --- /dev/null +++ b/swarm/shed/generic_index.go @@ -0,0 +1,344 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package shed + +import ( + "github.com/syndtr/goleveldb/leveldb" +) + +// Index represents a set of LevelDB key value pairs that have common +// prefix. It holds functions for encoding and decoding keys and values +// to provide transparent actions on saved data which inclide: +// - getting a particular Item +// - saving a particular Item +// - iterating over a sorted LevelDB keys +type GenericIndex struct { + db *DB + prefix []byte + encodeKeyFunc func(item interface{}) (key []byte, err error) + decodeKeyFunc func(key []byte) (item interface{}, err error) + encodeValueFunc func(fields Item) (value []byte, err error) + decodeValueFunc func(keyFields Item, value []byte) (e Item, err error) +} + +// GenericIndexFuncs structure defines functions for encoding and decoding +// LevelDB keys and values for a specific index. +type GenericIndexFuncs struct { + EncodeKey func(fields interface{}) (key []byte, err error) + DecodeKey func(key []byte) (e interface{}, err error) + EncodeValue func(fields interface{}) (value []byte, err error) + DecodeValue func(keyFields Item, value []byte) (e interface{}, err error) +} + +// NewIndex returns a new Index instance with defined name and +// encoding functions. The name must be unique and will be validated +// on database schema for a key prefix byte. +func (db *DB) NewGenericIndex(name string, funcs GenericIndexFuncs) (f Index, err error) { + id, err := db.schemaIndexPrefix(name) + if err != nil { + return f, err + } + prefix := []byte{id} + return Index{ + db: db, + prefix: prefix, + // This function adjusts Index LevelDB key + // by appending the provided index id byte. + // This is needed to avoid collisions between keys of different + // indexes as all index ids are unique. + encodeKeyFunc: func(e interface{}) (key []byte, err error) { + key, err = funcs.EncodeKey(e) + if err != nil { + return nil, err + } + return append(append(make([]byte, 0, len(key)+1), prefix...), key...), nil + }, + // This function reverses the encodeKeyFunc constructed key + // to transparently work with index keys without their index ids. + // It assumes that index keys are prefixed with only one byte. + decodeKeyFunc: func(key []byte) (e interface{}, err error) { + return funcs.DecodeKey(key[1:]) + }, + encodeValueFunc: funcs.EncodeValue, + decodeValueFunc: funcs.DecodeValue, + }, nil +} + +// Get accepts key fields represented as Item to retrieve a +// value from the index and return maximum available information +// from the index represented as another Item. +func (f *GenericIndex) Get(keyFields interface{}) (out interface{}, err error) { + key, err := f.encodeKeyFunc(keyFields) + if err != nil { + return out, err + } + value, err := f.db.Get(key) + if err != nil { + return out, err + } + out, err = f.decodeValueFunc(keyFields, value) + if err != nil { + return out, err + } + return out, nil +} + +// Has accepts key fields represented as Item to check +// if there this Item's encoded key is stored in +// the index. +func (f GenericIndex) Has(keyFields interface{}) (bool, error) { + key, err := f.encodeKeyFunc(keyFields) + if err != nil { + return false, err + } + return f.db.Has(key) +} + +// Put accepts Item to encode information from it +// and save it to the database. +func (f GenericIndex) Put(i interface{}) (err error) { + key, err := f.encodeKeyFunc(i) + if err != nil { + return err + } + value, err := f.encodeValueFunc(i) + if err != nil { + return err + } + return f.db.Put(key, value) +} + +// PutInBatch is the same as Put method, but it just +// saves the key/value pair to the batch instead +// directly to the database. +func (f GenericIndex) PutInBatch(batch *leveldb.Batch, i interface{}) (err error) { + key, err := f.encodeKeyFunc(i) + if err != nil { + return err + } + value, err := f.encodeValueFunc(i) + if err != nil { + return err + } + batch.Put(key, value) + return nil +} + +// Delete accepts Item to remove a key/value pair +// from the database based on its fields. +func (f Index) Delete(keyFields interface{}) (err error) { + key, err := f.encodeKeyFunc(keyFields) + if err != nil { + return err + } + return f.db.Delete(key) +} + +// DeleteInBatch is the same as Delete just the operation +// is performed on the batch instead on the database. +func (f GenericIndex) DeleteInBatch(batch *leveldb.Batch, keyFields interface{}) (err error) { + key, err := f.encodeKeyFunc(keyFields) + if err != nil { + return err + } + batch.Delete(key) + return nil +} + +// IndexIterFunc is a callback on every Item that is decoded +// by iterating on an Index keys. +// By returning a true for stop variable, iteration will +// stop, and by returning the error, that error will be +// propagated to the called iterator method on Index. +/*type IndexIterFunc func(item ) (stop bool, err error) + +// IterateOptions defines optional parameters for Iterate function. +type IterateOptions struct { + // StartFrom is the Item to start the iteration from. + StartFrom *Item + // If SkipStartFromItem is true, StartFrom item will not + // be iterated on. + SkipStartFromItem bool + // Iterate over items which keys have a common prefix. + Prefix []byte +} + +// Iterate function iterates over keys of the Index. +// If IterateOptions is nil, the iterations is over all keys. +func (f GenericIndex) Iterate(fn IndexIterFunc, options *IterateOptions) (err error) { + if options == nil { + options = new(IterateOptions) + } + // construct a prefix with Index prefix and optional common key prefix + prefix := append(f.prefix, options.Prefix...) + // start from the prefix + startKey := prefix + if options.StartFrom != nil { + // start from the provided StartFrom Item key value + startKey, err = f.encodeKeyFunc(*options.StartFrom) + if err != nil { + return err + } + } + it := f.db.NewIterator() + defer it.Release() + + // move the cursor to the start key + ok := it.Seek(startKey) + if !ok { + // stop iterator if seek has failed + return it.Error() + } + if options.SkipStartFromItem && bytes.Equal(startKey, it.Key()) { + // skip the start from Item if it is the first key + // and it is explicitly configured to skip it + ok = it.Next() + } + for ; ok; ok = it.Next() { + item, err := f.itemFromIterator(it, prefix) + if err != nil { + if err == leveldb.ErrNotFound { + break + } + return err + } + stop, err := fn(item) + if err != nil { + return err + } + if stop { + break + } + } + return it.Error() +} + +// First returns the first item in the Index which encoded key starts with a prefix. +// If the prefix is nil, the first element of the whole index is returned. +// If Index has no elements, a leveldb.ErrNotFound error is returned. +func (f GenericIndex) First(prefix []byte) (i Item, err error) { + it := f.db.NewIterator() + defer it.Release() + + totalPrefix := append(f.prefix, prefix...) + it.Seek(totalPrefix) + + return f.itemFromIterator(it, totalPrefix) +} + +// itemFromIterator returns the Item from the current iterator position. +// If the complete encoded key does not start with totalPrefix, +// leveldb.ErrNotFound is returned. Value for totalPrefix must start with +// Index prefix. +func (f GenericIndex) itemFromIterator(it iterator.Iterator, totalPrefix []byte) (i Item, err error) { + key := it.Key() + if !bytes.HasPrefix(key, totalPrefix) { + return i, leveldb.ErrNotFound + } + // create a copy of key byte slice not to share leveldb underlaying slice array + keyItem, err := f.decodeKeyFunc(append([]byte(nil), key...)) + if err != nil { + return i, err + } + // create a copy of value byte slice not to share leveldb underlaying slice array + valueItem, err := f.decodeValueFunc(keyItem, append([]byte(nil), it.Value()...)) + if err != nil { + return i, err + } + return keyItem.Merge(valueItem), it.Error() +} + +// Last returns the last item in the Index which encoded key starts with a prefix. +// If the prefix is nil, the last element of the whole index is returned. +// If Index has no elements, a leveldb.ErrNotFound error is returned. +func (f GenericIndex) Last(prefix []byte) (i Item, err error) { + it := f.db.NewIterator() + defer it.Release() + + // get the next prefix in line + // since leveldb iterator Seek seeks to the + // next key if the key that it seeks to is not found + // and by getting the previous key, the last one for the + // actual prefix is found + nextPrefix := incByteSlice(prefix) + l := len(prefix) + + if l > 0 && nextPrefix != nil { + it.Seek(append(f.prefix, nextPrefix...)) + it.Prev() + } else { + it.Last() + } + + totalPrefix := append(f.prefix, prefix...) + return f.itemFromIterator(it, totalPrefix) +} +*/ +// incByteSlice returns the byte slice of the same size +// of the provided one that is by one incremented in its +// total value. If all bytes in provided slice are equal +// to 255 a nil slice would be returned indicating that +// increment can not happen for the same length. +func incByteSlice(b []byte) (next []byte) { + l := len(b) + next = make([]byte, l) + copy(next, b) + for i := l - 1; i >= 0; i-- { + if b[i] == 255 { + next[i] = 0 + } else { + next[i] = b[i] + 1 + return next + } + } + return nil +} + +// Count returns the number of items in index. +/*func (f GenericIndex) Count() (count int, err error) { + it := f.db.NewIterator() + defer it.Release() + + for ok := it.Seek(f.prefix); ok; ok = it.Next() { + key := it.Key() + if key[0] != f.prefix[0] { + break + } + count++ + } + return count, it.Error() +} + +// CountFrom returns the number of items in index keys +// starting from the key encoded from the provided Item. +func (f GenericIndex) CountFrom(start Item) (count int, err error) { + startKey, err := f.encodeKeyFunc(start) + if err != nil { + return 0, err + } + it := f.db.NewIterator() + defer it.Release() + + for ok := it.Seek(startKey); ok; ok = it.Next() { + key := it.Key() + if key[0] != f.prefix[0] { + break + } + count++ + } + return count, it.Error() +}*/ diff --git a/swarm/shed/generic_index_test.go b/swarm/shed/generic_index_test.go new file mode 100644 index 0000000000..7774828274 --- /dev/null +++ b/swarm/shed/generic_index_test.go @@ -0,0 +1,266 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package shed + +import ( + "testing" + "time" + + "github.com/syndtr/goleveldb/leveldb" +) + +// Index functions for the index that is used in tests in this file. +var retrievalIndexFuncs = GenericIndexFuncs{ + EncodeKey: func(fields interface{}) (key []byte, err error) { + return nil, nil + }, + DecodeKey: func(key []byte) (e interface{}, err error) { + return e, nil + }, + EncodeValue: func(fields interface{}) (value []byte, err error) { + return value, nil + }, + DecodeValue: func(keyItem interface{}, value []byte) (e interface{}, err error) { + return nil, nil + }, +} + +// TestIndex validates put, get, has and delete functions of the Index implementation. +func TestGenericIndex(t *testing.T) { + db, cleanupFunc := newTestDB(t) + defer cleanupFunc() + + index, err := db.NewGenericIndex("retrieval", retrievalIndexFuncs) + if err != nil { + t.Fatal(err) + } + + t.Run("put", func(t *testing.T) { + want := // + err := index.Put(want) + if err != nil { + t.Fatal(err) + } + got, err := index.Get(...) + if err != nil { + t.Fatal(err) + } + //checkItem(t, got, want) + + t.Run("overwrite", func(t *testing.T) { + want := ... + err = index.Put(want) + if err != nil { + t.Fatal(err) + } + got, err := index.Get(...) + if err != nil { + t.Fatal(err) + } + checkItem(t, got, want) + }) + }) + + t.Run("put in batch", func(t *testing.T) { + want :=... + batch := new(leveldb.Batch) + index.PutInBatch(batch, want) + err := db.WriteBatch(batch) + if err != nil { + t.Fatal(err) + } + got, err := index.Get(...) + if err != nil { + t.Fatal(err) + } + checkItem(t, got, want) + + t.Run("overwrite", func(t *testing.T) { + want :=... + batch := new(leveldb.Batch) + index.PutInBatch(batch, want) + db.WriteBatch(batch) + if err != nil { + t.Fatal(err) + } + got, err := index.Get(...) + if err != nil { + t.Fatal(err) + } + checkItem(t, got, want) + }) + }) + + t.Run("put in batch twice", func(t *testing.T) { + // ensure that the last item of items with the same db keys + // is actually saved + batch := new(leveldb.Batch) + address := []byte("put-in-batch-twice-hash") + + // put the first item + index.PutInBatch(batch, Item{ + Address: address, + Data: []byte("DATA"), + StoreTimestamp: time.Now().UTC().UnixNano(), + }) + + want := Item{ + Address: address, + Data: []byte("New DATA"), + StoreTimestamp: time.Now().UTC().UnixNano(), + } + // then put the item that will produce the same key + // but different value in the database + index.PutInBatch(batch, want) + db.WriteBatch(batch) + if err != nil { + t.Fatal(err) + } + got, err := index.Get(Item{ + Address: address, + }) + if err != nil { + t.Fatal(err) + } + checkItem(t, got, want) + }) + + t.Run("has", func(t *testing.T) { + want := Item{ + Address: []byte("has-hash"), + Data: []byte("DATA"), + StoreTimestamp: time.Now().UTC().UnixNano(), + } + + dontWant := Item{ + Address: []byte("do-not-has-hash"), + Data: []byte("DATA"), + StoreTimestamp: time.Now().UTC().UnixNano(), + } + + err := index.Put(want) + if err != nil { + t.Fatal(err) + } + + has, err := index.Has(want) + if err != nil { + t.Fatal(err) + } + if !has { + t.Error("item is not found") + } + + has, err = index.Has(dontWant) + if err != nil { + t.Fatal(err) + } + if has { + t.Error("unwanted item is found") + } + }) + + t.Run("delete", func(t *testing.T) { + want := Item{ + Address: []byte("delete-hash"), + Data: []byte("DATA"), + StoreTimestamp: time.Now().UTC().UnixNano(), + } + + err := index.Put(want) + if err != nil { + t.Fatal(err) + } + got, err := index.Get(Item{ + Address: want.Address, + }) + if err != nil { + t.Fatal(err) + } + checkItem(t, got, want) + + err = index.Delete(Item{ + Address: want.Address, + }) + if err != nil { + t.Fatal(err) + } + + wantErr := leveldb.ErrNotFound + got, err = index.Get(Item{ + Address: want.Address, + }) + if err != wantErr { + t.Fatalf("got error %v, want %v", err, wantErr) + } + }) + + t.Run("delete in batch", func(t *testing.T) { + want := Item{ + Address: []byte("delete-in-batch-hash"), + Data: []byte("DATA"), + StoreTimestamp: time.Now().UTC().UnixNano(), + } + + err := index.Put(want) + if err != nil { + t.Fatal(err) + } + got, err := index.Get(Item{ + Address: want.Address, + }) + if err != nil { + t.Fatal(err) + } + checkItem(t, got, want) + + batch := new(leveldb.Batch) + index.DeleteInBatch(batch, Item{ + Address: want.Address, + }) + err = db.WriteBatch(batch) + if err != nil { + t.Fatal(err) + } + + wantErr := leveldb.ErrNotFound + got, err = index.Get(Item{ + Address: want.Address, + }) + if err != wantErr { + t.Fatalf("got error %v, want %v", err, wantErr) + } + }) +} + +// checkItem is a test helper function that compares if two Index items are the same. +/*func checkItem(t *testing.T, got, want Item) { + t.Helper() + + if !bytes.Equal(got.Address, want.Address) { + t.Errorf("got hash %q, expected %q", string(got.Address), string(want.Address)) + } + if !bytes.Equal(got.Data, want.Data) { + t.Errorf("got data %q, expected %q", string(got.Data), string(want.Data)) + } + if got.StoreTimestamp != want.StoreTimestamp { + t.Errorf("got store timestamp %v, expected %v", got.StoreTimestamp, want.StoreTimestamp) + } + if got.AccessTimestamp != want.AccessTimestamp { + t.Errorf("got access timestamp %v, expected %v", got.AccessTimestamp, want.AccessTimestamp) + } +}*/ diff --git a/swarm/shed/index.go b/swarm/shed/index.go index 815f930670..3d989bbb3f 100644 --- a/swarm/shed/index.go +++ b/swarm/shed/index.go @@ -72,7 +72,6 @@ func (i Item) Merge(i2 Item) (new Item) { // - getting a particular Item // - saving a particular Item // - iterating over a sorted LevelDB keys -// It implements IndexIteratorInterface interface. type Index struct { db *DB prefix []byte diff --git a/swarm/storage/localstore/localstore.go b/swarm/storage/localstore/localstore.go index fa73c69f47..af3b7f860f 100644 --- a/swarm/storage/localstore/localstore.go +++ b/swarm/storage/localstore/localstore.go @@ -76,6 +76,9 @@ type DB struct { // proximity order bin binIDs shed.Uint64Vector + // push syncing tags index + tagIndex shed.Item + // garbage collection index gcIndex shed.Index @@ -318,6 +321,26 @@ func New(path string, baseKey []byte, o *Options) (db *DB, err error) { } // create a push syncing triggers used by SubscribePush function db.pushTriggers = make([]chan struct{}, 0) + + // tag index for push syncing tags + db.pushIndex, err = db.shed.NewIndex("Tag->Filename", shed.IndexFuncs{ //TODO: should this be Tag->Filename|StoreTimestamp? + EncodeKey: func(fields shed.Item) (key []byte, err error) { + return nil, nil + }, + DecodeKey: func(key []byte) (e shed.Item, err error) { + return nil, nil + }, + EncodeValue: func(fields shed.Item) (value []byte, err error) { + return nil, nil + }, + DecodeValue: func(keyItem shed.Item, value []byte) (e shed.Item, err error) { + return nil, nil + }, + }) + if err != nil { + return nil, err + } + // gc index for removable chunk ordered by ascending last access time db.gcIndex, err = db.shed.NewIndex("AccessTimestamp|BinID|Hash->nil", shed.IndexFuncs{ EncodeKey: func(fields shed.Item) (key []byte, err error) { diff --git a/swarm/storage/localstore/mode_put.go b/swarm/storage/localstore/mode_put.go index 5691f92b94..e1d2093f1e 100644 --- a/swarm/storage/localstore/mode_put.go +++ b/swarm/storage/localstore/mode_put.go @@ -137,6 +137,8 @@ func (db *DB) put(mode chunk.ModePut, item shed.Item) (err error) { triggerPullFeed = true } + case ModePutTag: + // put to indexes: tag default: return ErrInvalidMode } diff --git a/swarm/storage/localstore/mode_put_test.go b/swarm/storage/localstore/mode_put_test.go index 225ba8280f..b2f0cbfee3 100644 --- a/swarm/storage/localstore/mode_put_test.go +++ b/swarm/storage/localstore/mode_put_test.go @@ -98,6 +98,28 @@ func TestModePutSync(t *testing.T) { t.Run("pull index", newPullIndexTest(db, ch, 1, nil)) } +func TestModePutTag(t *testing.T) { + db, cleanupFunc := newTestDB(t, nil) + defer cleanupFunc() + + wantTimestamp := time.Now().UTC().UnixNano() + defer setNow(func() (t int64) { + return wantTimestamp + })() + + ch := generateTestRandomChunk() + + err := db.Put(context.Background(), chunk.ModePutTags, ch) + if err != nil { + t.Fatal(err) + } + + t.Run("retrieve indexes", newRetrieveIndexesTest(db, ch, wantTimestamp, 0)) + + t.Run("pull index", newPullIndexTest(db, ch, 1, nil)) + +} + // TestModePutUpload validates ModePutUpload index values on the provided DB. func TestModePutUpload(t *testing.T) { db, cleanupFunc := newTestDB(t, nil) From fb4875403a08d595f3f83ae084b0de81c0f5bb2f Mon Sep 17 00:00:00 2001 From: Elad Date: Tue, 2 Apr 2019 13:08:26 +0900 Subject: [PATCH 10/31] swarm/shed, swarm/storage: add generic shed index --- swarm/shed/example_store_test.go | 2 +- swarm/shed/generic_index.go | 48 +++---- swarm/shed/generic_index_test.go | 175 +++++++++++-------------- swarm/storage/filestore.go | 8 +- swarm/storage/hasherstore.go | 12 +- swarm/storage/localstore/localstore.go | 12 +- swarm/storage/localstore/mode_put.go | 3 +- 7 files changed, 104 insertions(+), 156 deletions(-) diff --git a/swarm/shed/example_store_test.go b/swarm/shed/example_store_test.go index 9a83855e70..4e5f97d7c0 100644 --- a/swarm/shed/example_store_test.go +++ b/swarm/shed/example_store_test.go @@ -229,7 +229,7 @@ func (s *Store) Get(_ context.Context, addr storage.Address) (c storage.Chunk, e } // Return the chunk. - return storage.NewChunk(item.Address, item.Data), nil + return storage.NewChunk(item.Address, item.Data, []uint64{}), nil } // CollectGarbage is an example of index iteration. diff --git a/swarm/shed/generic_index.go b/swarm/shed/generic_index.go index 1ffbf478b0..0bff341bec 100644 --- a/swarm/shed/generic_index.go +++ b/swarm/shed/generic_index.go @@ -31,8 +31,8 @@ type GenericIndex struct { prefix []byte encodeKeyFunc func(item interface{}) (key []byte, err error) decodeKeyFunc func(key []byte) (item interface{}, err error) - encodeValueFunc func(fields Item) (value []byte, err error) - decodeValueFunc func(keyFields Item, value []byte) (e Item, err error) + encodeValueFunc func(fields interface{}) (value []byte, err error) + decodeValueFunc func(keyFields interface{}, value []byte) (e interface{}, err error) } // GenericIndexFuncs structure defines functions for encoding and decoding @@ -41,19 +41,19 @@ type GenericIndexFuncs struct { EncodeKey func(fields interface{}) (key []byte, err error) DecodeKey func(key []byte) (e interface{}, err error) EncodeValue func(fields interface{}) (value []byte, err error) - DecodeValue func(keyFields Item, value []byte) (e interface{}, err error) + DecodeValue func(keyFields interface{}, value []byte) (e interface{}, err error) } // NewIndex returns a new Index instance with defined name and // encoding functions. The name must be unique and will be validated // on database schema for a key prefix byte. -func (db *DB) NewGenericIndex(name string, funcs GenericIndexFuncs) (f Index, err error) { +func (db *DB) NewGenericIndex(name string, funcs GenericIndexFuncs) (f GenericIndex, err error) { id, err := db.schemaIndexPrefix(name) if err != nil { return f, err } prefix := []byte{id} - return Index{ + return GenericIndex{ db: db, prefix: prefix, // This function adjusts Index LevelDB key @@ -110,12 +110,12 @@ func (f GenericIndex) Has(keyFields interface{}) (bool, error) { // Put accepts Item to encode information from it // and save it to the database. -func (f GenericIndex) Put(i interface{}) (err error) { - key, err := f.encodeKeyFunc(i) +func (f GenericIndex) Put(k, v interface{}) (err error) { + key, err := f.encodeKeyFunc(k) if err != nil { return err } - value, err := f.encodeValueFunc(i) + value, err := f.encodeValueFunc(v) if err != nil { return err } @@ -125,12 +125,12 @@ func (f GenericIndex) Put(i interface{}) (err error) { // PutInBatch is the same as Put method, but it just // saves the key/value pair to the batch instead // directly to the database. -func (f GenericIndex) PutInBatch(batch *leveldb.Batch, i interface{}) (err error) { - key, err := f.encodeKeyFunc(i) +func (f GenericIndex) PutInBatch(batch *leveldb.Batch, k, v interface{}) (err error) { + key, err := f.encodeKeyFunc(k) if err != nil { return err } - value, err := f.encodeValueFunc(i) + value, err := f.encodeValueFunc(v) if err != nil { return err } @@ -140,7 +140,7 @@ func (f GenericIndex) PutInBatch(batch *leveldb.Batch, i interface{}) (err error // Delete accepts Item to remove a key/value pair // from the database based on its fields. -func (f Index) Delete(keyFields interface{}) (err error) { +func (f GenericIndex) Delete(keyFields interface{}) (err error) { key, err := f.encodeKeyFunc(keyFields) if err != nil { return err @@ -288,28 +288,10 @@ func (f GenericIndex) Last(prefix []byte) (i Item, err error) { return f.itemFromIterator(it, totalPrefix) } */ -// incByteSlice returns the byte slice of the same size -// of the provided one that is by one incremented in its -// total value. If all bytes in provided slice are equal -// to 255 a nil slice would be returned indicating that -// increment can not happen for the same length. -func incByteSlice(b []byte) (next []byte) { - l := len(b) - next = make([]byte, l) - copy(next, b) - for i := l - 1; i >= 0; i-- { - if b[i] == 255 { - next[i] = 0 - } else { - next[i] = b[i] + 1 - return next - } - } - return nil -} +/* // Count returns the number of items in index. -/*func (f GenericIndex) Count() (count int, err error) { +func (f GenericIndex) Count() (count int, err error) { it := f.db.NewIterator() defer it.Release() @@ -325,7 +307,7 @@ func incByteSlice(b []byte) (next []byte) { // CountFrom returns the number of items in index keys // starting from the key encoded from the provided Item. -func (f GenericIndex) CountFrom(start Item) (count int, err error) { +func (f GenericIndex) CountFrom(start interface{}) (count int, err error) { startKey, err := f.encodeKeyFunc(start) if err != nil { return 0, err diff --git a/swarm/shed/generic_index_test.go b/swarm/shed/generic_index_test.go index 7774828274..ee2d715e4d 100644 --- a/swarm/shed/generic_index_test.go +++ b/swarm/shed/generic_index_test.go @@ -17,25 +17,36 @@ package shed import ( + "errors" "testing" - "time" "github.com/syndtr/goleveldb/leveldb" ) // Index functions for the index that is used in tests in this file. -var retrievalIndexFuncs = GenericIndexFuncs{ +var retrievalGenericIndexFuncs = GenericIndexFuncs{ EncodeKey: func(fields interface{}) (key []byte, err error) { - return nil, nil + //marshal the fields as something, return a byte array + val, ok := fields.(string) + if !ok { + return nil, errors.New("could not unmarshal field") + } + return []byte(val), nil }, DecodeKey: func(key []byte) (e interface{}, err error) { - return e, nil + str := string(key) + return str, nil }, EncodeValue: func(fields interface{}) (value []byte, err error) { - return value, nil + val, ok := fields.(string) + if !ok { + return nil, errors.New("could not unmarshal value") + } + return []byte(val), nil }, DecodeValue: func(keyItem interface{}, value []byte) (e interface{}, err error) { - return nil, nil + str := string(value) + return str, nil }, } @@ -44,64 +55,68 @@ func TestGenericIndex(t *testing.T) { db, cleanupFunc := newTestDB(t) defer cleanupFunc() - index, err := db.NewGenericIndex("retrieval", retrievalIndexFuncs) + index, err := db.NewGenericIndex("retrieval", retrievalGenericIndexFuncs) if err != nil { t.Fatal(err) } t.Run("put", func(t *testing.T) { - want := // - err := index.Put(want) + wantK := "wantKey" + wantV := "wantVal" + err := index.Put(wantK, wantV) if err != nil { t.Fatal(err) } - got, err := index.Get(...) + got, err := index.Get(wantK) if err != nil { t.Fatal(err) } - //checkItem(t, got, want) + checkItemString(t, got, wantV) t.Run("overwrite", func(t *testing.T) { - want := ... - err = index.Put(want) + wantK := "wantKey" + wantV := "wantNewVal" + err = index.Put(wantK, wantV) if err != nil { t.Fatal(err) } - got, err := index.Get(...) + got, err := index.Get(wantK) if err != nil { t.Fatal(err) } - checkItem(t, got, want) + checkItemString(t, got, wantV) }) }) t.Run("put in batch", func(t *testing.T) { - want :=... + wantK := "wantKey" + wantV := "anotherNewVal" batch := new(leveldb.Batch) - index.PutInBatch(batch, want) + index.PutInBatch(batch, wantK, wantV) err := db.WriteBatch(batch) if err != nil { t.Fatal(err) } - got, err := index.Get(...) + got, err := index.Get(wantK) if err != nil { t.Fatal(err) } - checkItem(t, got, want) + checkItemString(t, got, wantV) t.Run("overwrite", func(t *testing.T) { - want :=... + wantK := "wantKey" + wantV := "overrideBatchVal" batch := new(leveldb.Batch) - index.PutInBatch(batch, want) + index.PutInBatch(batch, wantK, wantV) db.WriteBatch(batch) if err != nil { t.Fatal(err) } - got, err := index.Get(...) + got, err := index.Get(wantK) if err != nil { t.Fatal(err) } - checkItem(t, got, want) + checkItemString(t, got, wantV) }) }) @@ -109,55 +124,37 @@ func TestGenericIndex(t *testing.T) { // ensure that the last item of items with the same db keys // is actually saved batch := new(leveldb.Batch) - address := []byte("put-in-batch-twice-hash") + wantK := "doubleWantKey" + firstWantV := "should override" + secondWantV := "should persist" // put the first item - index.PutInBatch(batch, Item{ - Address: address, - Data: []byte("DATA"), - StoreTimestamp: time.Now().UTC().UnixNano(), - }) + index.PutInBatch(batch, wantK, firstWantV) - want := Item{ - Address: address, - Data: []byte("New DATA"), - StoreTimestamp: time.Now().UTC().UnixNano(), - } // then put the item that will produce the same key // but different value in the database - index.PutInBatch(batch, want) + index.PutInBatch(batch, wantK, secondWantV) db.WriteBatch(batch) if err != nil { t.Fatal(err) } - got, err := index.Get(Item{ - Address: address, - }) + got, err := index.Get(wantK) if err != nil { t.Fatal(err) } - checkItem(t, got, want) + checkItemString(t, got, secondWantV) }) t.Run("has", func(t *testing.T) { - want := Item{ - Address: []byte("has-hash"), - Data: []byte("DATA"), - StoreTimestamp: time.Now().UTC().UnixNano(), - } - - dontWant := Item{ - Address: []byte("do-not-has-hash"), - Data: []byte("DATA"), - StoreTimestamp: time.Now().UTC().UnixNano(), - } - - err := index.Put(want) + wantK := "wantHasThis" + wantV := "shouldHaveThis" + dontWantK := "dontWantHasThis" + err := index.Put(wantK, wantV) if err != nil { t.Fatal(err) } - has, err := index.Has(want) + has, err := index.Has(wantK) if err != nil { t.Fatal(err) } @@ -165,7 +162,7 @@ func TestGenericIndex(t *testing.T) { t.Error("item is not found") } - has, err = index.Has(dontWant) + has, err = index.Has(dontWantK) if err != nil { t.Fatal(err) } @@ -175,92 +172,66 @@ func TestGenericIndex(t *testing.T) { }) t.Run("delete", func(t *testing.T) { - want := Item{ - Address: []byte("delete-hash"), - Data: []byte("DATA"), - StoreTimestamp: time.Now().UTC().UnixNano(), - } - - err := index.Put(want) + wantK := "wantDelete" + wantV := "wantDeleteVal" + err := index.Put(wantK, wantV) if err != nil { t.Fatal(err) } - got, err := index.Get(Item{ - Address: want.Address, - }) + got, err := index.Get(wantK) if err != nil { t.Fatal(err) } - checkItem(t, got, want) + checkItemString(t, got, wantV) - err = index.Delete(Item{ - Address: want.Address, - }) + err = index.Delete(wantK) if err != nil { t.Fatal(err) } wantErr := leveldb.ErrNotFound - got, err = index.Get(Item{ - Address: want.Address, - }) + got, err = index.Get(wantK) if err != wantErr { t.Fatalf("got error %v, want %v", err, wantErr) } }) t.Run("delete in batch", func(t *testing.T) { - want := Item{ - Address: []byte("delete-in-batch-hash"), - Data: []byte("DATA"), - StoreTimestamp: time.Now().UTC().UnixNano(), - } - - err := index.Put(want) + wantK := "wantDelInBatch" + wantV := "wantDelInBatchVal" + err := index.Put(wantK, wantV) if err != nil { t.Fatal(err) } - got, err := index.Get(Item{ - Address: want.Address, - }) + got, err := index.Get(wantK) if err != nil { t.Fatal(err) } - checkItem(t, got, want) + checkItemString(t, got, wantV) batch := new(leveldb.Batch) - index.DeleteInBatch(batch, Item{ - Address: want.Address, - }) + index.DeleteInBatch(batch, wantK) err = db.WriteBatch(batch) if err != nil { t.Fatal(err) } wantErr := leveldb.ErrNotFound - got, err = index.Get(Item{ - Address: want.Address, - }) + got, err = index.Get(wantK) if err != wantErr { t.Fatalf("got error %v, want %v", err, wantErr) } }) } -// checkItem is a test helper function that compares if two Index items are the same. -/*func checkItem(t *testing.T, got, want Item) { +// checkItemString is a test helper function that compares if two generic items are the same string. +func checkItemString(t *testing.T, got, want interface{}) { t.Helper() - if !bytes.Equal(got.Address, want.Address) { - t.Errorf("got hash %q, expected %q", string(got.Address), string(want.Address)) - } - if !bytes.Equal(got.Data, want.Data) { - t.Errorf("got data %q, expected %q", string(got.Data), string(want.Data)) - } - if got.StoreTimestamp != want.StoreTimestamp { - t.Errorf("got store timestamp %v, expected %v", got.StoreTimestamp, want.StoreTimestamp) - } - if got.AccessTimestamp != want.AccessTimestamp { - t.Errorf("got access timestamp %v, expected %v", got.AccessTimestamp, want.AccessTimestamp) + g := got.(string) + w := want.(string) + + if g != w { + t.Errorf("got %s, expected %s", g, w) } -}*/ +} diff --git a/swarm/storage/filestore.go b/swarm/storage/filestore.go index d15134a09f..4b35183de7 100644 --- a/swarm/storage/filestore.go +++ b/swarm/storage/filestore.go @@ -105,13 +105,13 @@ func (f *FileStore) HashSize() int { // it returns the tag as uint64 func (f *FileStore) CreateTag(filename string, timestamp uint64) (uint64, error) { intBuf := make([]byte, 8) - binary.BigEndian.PutUint64(intBuf, now) + binary.BigEndian.PutUint64(intBuf, timestamp) // Tag is SHA3(filename|storetimestamp)[:8] - tag := make([]byte, 8) - buf := []byte(hdr.Name) - buf = append(buf, intBuf) + buf := []byte(filename) + buf = append(buf, intBuf...) tagHash := crypto.Keccak256(buf)[:8] + return binary.BigEndian.Uint64(tagHash), nil } // GetAllReferences is a public API. This endpoint returns all chunk hashes (only) for a given file diff --git a/swarm/storage/hasherstore.go b/swarm/storage/hasherstore.go index 07f8222a5e..d5f328aee3 100644 --- a/swarm/storage/hasherstore.go +++ b/swarm/storage/hasherstore.go @@ -23,7 +23,6 @@ import ( "github.com/ethereum/go-ethereum/swarm/chunk" "github.com/ethereum/go-ethereum/swarm/sctx" - "github.com/ethereum/go-ethereum/swarm/storage" "github.com/ethereum/go-ethereum/swarm/storage/encryption" "golang.org/x/crypto/sha3" ) @@ -80,7 +79,7 @@ func (h *hasherStore) Put(ctx context.Context, chunkData ChunkData) (Reference, return nil, err } } - h.GetTags() + //h.GetTags() chunk := h.createChunk(ctx, c) h.storeChunk(ctx, chunk) @@ -113,12 +112,7 @@ func (h *hasherStore) Get(ctx context.Context, ref Reference) (ChunkData, error) return chunkData, nil } -func (h *hasherStore) GetTags(ctx context.Context, ref Reference) ([]uint64, error) { - addr, _, err := parseReference(ref, h.hashSize) - if err != nil { - return nil, err - } - +func (h *hasherStore) GetTags(ctx context.Context, addr chunk.Address) ([]uint64, error) { chunk, err := h.store.Get(ctx, chunk.ModeGetTags, addr) if err != nil { return nil, err @@ -174,7 +168,7 @@ func (h *hasherStore) createHash(chunkData ChunkData) Address { func (h *hasherStore) createChunk(ctx context.Context, chunkData ChunkData) Chunk { hash := h.createHash(chunkData) - tags, err := h.GetTags(ctx, storage.Address(hash)) // TODO: this is really bad but if we want to persist tags across sessions this would be the way + tags, err := h.GetTags(ctx, chunk.Address(hash)) // TODO: this is really bad but if we want to persist tags across sessions this would be the way if err != nil { panic("wtf") } diff --git a/swarm/storage/localstore/localstore.go b/swarm/storage/localstore/localstore.go index af3b7f860f..a6df24a0fa 100644 --- a/swarm/storage/localstore/localstore.go +++ b/swarm/storage/localstore/localstore.go @@ -77,7 +77,7 @@ type DB struct { binIDs shed.Uint64Vector // push syncing tags index - tagIndex shed.Item + tagIndex shed.GenericIndex // garbage collection index gcIndex shed.Index @@ -323,17 +323,17 @@ func New(path string, baseKey []byte, o *Options) (db *DB, err error) { db.pushTriggers = make([]chan struct{}, 0) // tag index for push syncing tags - db.pushIndex, err = db.shed.NewIndex("Tag->Filename", shed.IndexFuncs{ //TODO: should this be Tag->Filename|StoreTimestamp? - EncodeKey: func(fields shed.Item) (key []byte, err error) { + db.tagIndex, err = db.shed.NewGenericIndex("Tag->Filename", shed.GenericIndexFuncs{ //TODO: should this be Tag->Filename|StoreTimestamp? + EncodeKey: func(fields interface{}) (key []byte, err error) { return nil, nil }, - DecodeKey: func(key []byte) (e shed.Item, err error) { + DecodeKey: func(key []byte) (e interface{}, err error) { return nil, nil }, - EncodeValue: func(fields shed.Item) (value []byte, err error) { + EncodeValue: func(fields interface{}) (value []byte, err error) { return nil, nil }, - DecodeValue: func(keyItem shed.Item, value []byte) (e shed.Item, err error) { + DecodeValue: func(keyItem interface{}, value []byte) (e interface{}, err error) { return nil, nil }, }) diff --git a/swarm/storage/localstore/mode_put.go b/swarm/storage/localstore/mode_put.go index e1d2093f1e..6622c80f08 100644 --- a/swarm/storage/localstore/mode_put.go +++ b/swarm/storage/localstore/mode_put.go @@ -137,8 +137,9 @@ func (db *DB) put(mode chunk.ModePut, item shed.Item) (err error) { triggerPullFeed = true } - case ModePutTag: + case chunk.ModePutTags: // put to indexes: tag + default: return ErrInvalidMode } From 0256b0fe5c2a5e5ff2b5ad25c4d0f776192d074e Mon Sep 17 00:00:00 2001 From: Elad Date: Tue, 2 Apr 2019 13:12:06 +0900 Subject: [PATCH 11/31] swarm/shed: remove currently unused functionlity from generic index --- swarm/shed/generic_index.go | 166 ------------------------------------ 1 file changed, 166 deletions(-) diff --git a/swarm/shed/generic_index.go b/swarm/shed/generic_index.go index 0bff341bec..80bd3f4ed3 100644 --- a/swarm/shed/generic_index.go +++ b/swarm/shed/generic_index.go @@ -158,169 +158,3 @@ func (f GenericIndex) DeleteInBatch(batch *leveldb.Batch, keyFields interface{}) batch.Delete(key) return nil } - -// IndexIterFunc is a callback on every Item that is decoded -// by iterating on an Index keys. -// By returning a true for stop variable, iteration will -// stop, and by returning the error, that error will be -// propagated to the called iterator method on Index. -/*type IndexIterFunc func(item ) (stop bool, err error) - -// IterateOptions defines optional parameters for Iterate function. -type IterateOptions struct { - // StartFrom is the Item to start the iteration from. - StartFrom *Item - // If SkipStartFromItem is true, StartFrom item will not - // be iterated on. - SkipStartFromItem bool - // Iterate over items which keys have a common prefix. - Prefix []byte -} - -// Iterate function iterates over keys of the Index. -// If IterateOptions is nil, the iterations is over all keys. -func (f GenericIndex) Iterate(fn IndexIterFunc, options *IterateOptions) (err error) { - if options == nil { - options = new(IterateOptions) - } - // construct a prefix with Index prefix and optional common key prefix - prefix := append(f.prefix, options.Prefix...) - // start from the prefix - startKey := prefix - if options.StartFrom != nil { - // start from the provided StartFrom Item key value - startKey, err = f.encodeKeyFunc(*options.StartFrom) - if err != nil { - return err - } - } - it := f.db.NewIterator() - defer it.Release() - - // move the cursor to the start key - ok := it.Seek(startKey) - if !ok { - // stop iterator if seek has failed - return it.Error() - } - if options.SkipStartFromItem && bytes.Equal(startKey, it.Key()) { - // skip the start from Item if it is the first key - // and it is explicitly configured to skip it - ok = it.Next() - } - for ; ok; ok = it.Next() { - item, err := f.itemFromIterator(it, prefix) - if err != nil { - if err == leveldb.ErrNotFound { - break - } - return err - } - stop, err := fn(item) - if err != nil { - return err - } - if stop { - break - } - } - return it.Error() -} - -// First returns the first item in the Index which encoded key starts with a prefix. -// If the prefix is nil, the first element of the whole index is returned. -// If Index has no elements, a leveldb.ErrNotFound error is returned. -func (f GenericIndex) First(prefix []byte) (i Item, err error) { - it := f.db.NewIterator() - defer it.Release() - - totalPrefix := append(f.prefix, prefix...) - it.Seek(totalPrefix) - - return f.itemFromIterator(it, totalPrefix) -} - -// itemFromIterator returns the Item from the current iterator position. -// If the complete encoded key does not start with totalPrefix, -// leveldb.ErrNotFound is returned. Value for totalPrefix must start with -// Index prefix. -func (f GenericIndex) itemFromIterator(it iterator.Iterator, totalPrefix []byte) (i Item, err error) { - key := it.Key() - if !bytes.HasPrefix(key, totalPrefix) { - return i, leveldb.ErrNotFound - } - // create a copy of key byte slice not to share leveldb underlaying slice array - keyItem, err := f.decodeKeyFunc(append([]byte(nil), key...)) - if err != nil { - return i, err - } - // create a copy of value byte slice not to share leveldb underlaying slice array - valueItem, err := f.decodeValueFunc(keyItem, append([]byte(nil), it.Value()...)) - if err != nil { - return i, err - } - return keyItem.Merge(valueItem), it.Error() -} - -// Last returns the last item in the Index which encoded key starts with a prefix. -// If the prefix is nil, the last element of the whole index is returned. -// If Index has no elements, a leveldb.ErrNotFound error is returned. -func (f GenericIndex) Last(prefix []byte) (i Item, err error) { - it := f.db.NewIterator() - defer it.Release() - - // get the next prefix in line - // since leveldb iterator Seek seeks to the - // next key if the key that it seeks to is not found - // and by getting the previous key, the last one for the - // actual prefix is found - nextPrefix := incByteSlice(prefix) - l := len(prefix) - - if l > 0 && nextPrefix != nil { - it.Seek(append(f.prefix, nextPrefix...)) - it.Prev() - } else { - it.Last() - } - - totalPrefix := append(f.prefix, prefix...) - return f.itemFromIterator(it, totalPrefix) -} -*/ - -/* -// Count returns the number of items in index. -func (f GenericIndex) Count() (count int, err error) { - it := f.db.NewIterator() - defer it.Release() - - for ok := it.Seek(f.prefix); ok; ok = it.Next() { - key := it.Key() - if key[0] != f.prefix[0] { - break - } - count++ - } - return count, it.Error() -} - -// CountFrom returns the number of items in index keys -// starting from the key encoded from the provided Item. -func (f GenericIndex) CountFrom(start interface{}) (count int, err error) { - startKey, err := f.encodeKeyFunc(start) - if err != nil { - return 0, err - } - it := f.db.NewIterator() - defer it.Release() - - for ok := it.Seek(startKey); ok; ok = it.Next() { - key := it.Key() - if key[0] != f.prefix[0] { - break - } - count++ - } - return count, it.Error() -}*/ From 7f793b5f9186c59a5c329e6b904d4a13c210423c Mon Sep 17 00:00:00 2001 From: Elad Date: Tue, 2 Apr 2019 13:47:10 +0900 Subject: [PATCH 12/31] swarm/api, swarm/storage: addput generic functionality --- swarm/api/manifest.go | 3 +-- swarm/storage/filestore.go | 17 ++++++++++++----- swarm/storage/localstore/mode_put.go | 25 +++++++++++++++++++++++++ 3 files changed, 38 insertions(+), 7 deletions(-) diff --git a/swarm/api/manifest.go b/swarm/api/manifest.go index a12888e786..0228d8c8c9 100644 --- a/swarm/api/manifest.go +++ b/swarm/api/manifest.go @@ -119,8 +119,7 @@ func (a *API) NewManifestWriter(ctx context.Context, addr storage.Address, quitC // AddEntry stores the given data and adds the resulting address to the manifest func (m *ManifestWriter) AddEntry(ctx context.Context, data io.Reader, e *ManifestEntry) (addr storage.Address, err error) { - - now := time.Now().Unix() + now := time.Now().Unix() // leaving this as is since we consider deprecating e.ModTime ctxTag := m.api.CreateTag(e.Path, now) childCtx := sctx.SetPushTag(ctx, ctxTag) entry := newManifestTrieEntry(e, nil) diff --git a/swarm/storage/filestore.go b/swarm/storage/filestore.go index 4b35183de7..c107574d72 100644 --- a/swarm/storage/filestore.go +++ b/swarm/storage/filestore.go @@ -48,7 +48,8 @@ const ( type FileStore struct { ChunkStore - hashFunc SwarmHasher + localStore *localstore.DB + hashFunc SwarmHasher } type FileStoreParams struct { @@ -67,12 +68,13 @@ func NewLocalFileStore(datadir string, basekey []byte) (*FileStore, error) { if err != nil { return nil, err } - return NewFileStore(chunk.NewValidatorStore(localStore, NewContentAddressValidator(MakeHashFunc(DefaultHash))), NewFileStoreParams()), nil + return NewFileStore(localStore, chunk.NewValidatorStore(localStore, NewContentAddressValidator(MakeHashFunc(DefaultHash))), NewFileStoreParams()), nil } -func NewFileStore(store ChunkStore, params *FileStoreParams) *FileStore { +func NewFileStore(localstore *localstore.DB, store ChunkStore, params *FileStoreParams) *FileStore { hashFunc := MakeHashFunc(params.Hash) return &FileStore{ + localStore: localstore, ChunkStore: store, hashFunc: hashFunc, } @@ -103,7 +105,7 @@ func (f *FileStore) HashSize() int { // CreateTag creates a new push tag and stores it in localstore // it returns the tag as uint64 -func (f *FileStore) CreateTag(filename string, timestamp uint64) (uint64, error) { +func (f *FileStore) CreateTag(ctx context.Context, filename string, timestamp uint64) (uint64, error) { intBuf := make([]byte, 8) binary.BigEndian.PutUint64(intBuf, timestamp) // Tag is SHA3(filename|storetimestamp)[:8] @@ -111,7 +113,12 @@ func (f *FileStore) CreateTag(filename string, timestamp uint64) (uint64, error) buf = append(buf, intBuf...) tagHash := crypto.Keccak256(buf)[:8] - return binary.BigEndian.Uint64(tagHash), nil + tag := binary.BigEndian.Uint64(tagHash) + err := f.localStore.PutGeneric(ctx, chunk.ModePutTags, interface{}(tag), interface{}(filename)) + if err != nil { + return tag, err + } + return tag, nil } // GetAllReferences is a public API. This endpoint returns all chunk hashes (only) for a given file diff --git a/swarm/storage/localstore/mode_put.go b/swarm/storage/localstore/mode_put.go index 6622c80f08..1147734ad9 100644 --- a/swarm/storage/localstore/mode_put.go +++ b/swarm/storage/localstore/mode_put.go @@ -161,3 +161,28 @@ func (db *DB) put(mode chunk.ModePut, item shed.Item) (err error) { } return nil } + +func (db *DB) PutGeneric(_ context.Context, mode chunk.ModePut, itemK, itemV interface{}) (err error) { + return db.putGeneric(mode, itemK, itemV) +} + +func (db *DB) putGeneric(mode chunk.ModePut, itemK, itemV interface{}) (err error) { + // protect parallel updates + db.batchMu.Lock() + defer db.batchMu.Unlock() + + batch := new(leveldb.Batch) + + switch mode { + case chunk.ModePutTags: + // put to indexes: tag + db.tagIndex.PutInBatch(batch, itemK, itemV) + default: + return ErrInvalidMode + } + err = db.shed.WriteBatch(batch) + if err != nil { + return err + } + return nil +} From 136b544fd23ac1617d98916083754328dfcae2da Mon Sep 17 00:00:00 2001 From: Elad Date: Tue, 2 Apr 2019 13:52:26 +0900 Subject: [PATCH 13/31] swarm/api: create method stubs for tag status checks --- swarm/api/api.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/swarm/api/api.go b/swarm/api/api.go index 332752c8a8..fb823970b3 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -879,6 +879,18 @@ func (a *API) CreateTag(filename string, timestamp uint64) (uint64, error) { return a.fileStore.CreateTag(filename, timestamp) } +func (a *API) GetTagFilename(tag uint64) (string, error) { + panic("implement this") +} + +func (a *API) TotalChunksForTag(tag uint64) (uint64, error) { + panic("implement this") +} + +func (a *API) RemainingChunksForTag(tag uint64) (uint64, error) { + panic("implement this") +} + // BuildDirectoryTree used by swarmfs_unix func (a *API) BuildDirectoryTree(ctx context.Context, mhash string, nameresolver bool) (addr storage.Address, manifestEntryMap map[string]*manifestTrieEntry, err error) { From afabd61b8a82866741ef0747511c3b12de980ac9 Mon Sep 17 00:00:00 2001 From: Elad Date: Tue, 2 Apr 2019 18:07:05 +0900 Subject: [PATCH 14/31] cmd/swarm, swarm/api: wip integration --- cmd/swarm/explore.go | 2 +- cmd/swarm/hash.go | 2 +- swarm/api/api.go | 4 ++-- swarm/api/http/test_server.go | 2 +- swarm/api/manifest.go | 7 +++++-- swarm/swarm.go | 2 +- 6 files changed, 11 insertions(+), 8 deletions(-) diff --git a/cmd/swarm/explore.go b/cmd/swarm/explore.go index 5b5b8bf41f..fe7b1d7fa0 100644 --- a/cmd/swarm/explore.go +++ b/cmd/swarm/explore.go @@ -47,7 +47,7 @@ func hashes(ctx *cli.Context) { } defer f.Close() - fileStore := storage.NewFileStore(&storage.FakeChunkStore{}, storage.NewFileStoreParams()) + fileStore := storage.NewFileStore(nil, &storage.FakeChunkStore{}, storage.NewFileStoreParams()) refs, err := fileStore.GetAllReferences(context.TODO(), f, false) if err != nil { utils.Fatalf("%v\n", err) diff --git a/cmd/swarm/hash.go b/cmd/swarm/hash.go index 2df02c0ed7..3cf5c1699c 100644 --- a/cmd/swarm/hash.go +++ b/cmd/swarm/hash.go @@ -77,7 +77,7 @@ func hash(ctx *cli.Context) { defer f.Close() stat, _ := f.Stat() - fileStore := storage.NewFileStore(&storage.FakeChunkStore{}, storage.NewFileStoreParams()) + fileStore := storage.NewFileStore(nil, &storage.FakeChunkStore{}, storage.NewFileStoreParams()) addr, _, err := fileStore.Store(context.TODO(), f, stat.Size(), false) if err != nil { utils.Fatalf("%v\n", err) diff --git a/swarm/api/api.go b/swarm/api/api.go index fb823970b3..bf5ecf6880 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -875,8 +875,8 @@ func (a *API) AppendFile(ctx context.Context, mhash, path, fname string, existin } // CreateTag creates a new push tag and stores it in localstore -func (a *API) CreateTag(filename string, timestamp uint64) (uint64, error) { - return a.fileStore.CreateTag(filename, timestamp) +func (a *API) CreateTag(ctx context.Context, filename string, timestamp uint64) (uint64, error) { + return a.fileStore.CreateTag(ctx, filename, timestamp) } func (a *API) GetTagFilename(tag uint64) (string, error) { diff --git a/swarm/api/http/test_server.go b/swarm/api/http/test_server.go index 0a4b6258e6..b121348777 100644 --- a/swarm/api/http/test_server.go +++ b/swarm/api/http/test_server.go @@ -44,7 +44,7 @@ func NewTestSwarmServer(t *testing.T, serverFunc func(*api.API) TestServer, reso t.Fatal(err) } - fileStore := storage.NewFileStore(localStore, storage.NewFileStoreParams()) + fileStore := storage.NewFileStore(localStore, localStore, storage.NewFileStoreParams()) // Swarm feeds test setup feedsDir, err := ioutil.TempDir("", "swarm-feeds-test") diff --git a/swarm/api/manifest.go b/swarm/api/manifest.go index 0228d8c8c9..0f875056ea 100644 --- a/swarm/api/manifest.go +++ b/swarm/api/manifest.go @@ -119,8 +119,11 @@ func (a *API) NewManifestWriter(ctx context.Context, addr storage.Address, quitC // AddEntry stores the given data and adds the resulting address to the manifest func (m *ManifestWriter) AddEntry(ctx context.Context, data io.Reader, e *ManifestEntry) (addr storage.Address, err error) { - now := time.Now().Unix() // leaving this as is since we consider deprecating e.ModTime - ctxTag := m.api.CreateTag(e.Path, now) + now := uint64(time.Now().Unix()) // leaving this as is since we consider deprecating e.ModTime + ctxTag, err := m.api.CreateTag(ctx, e.Path, now) + if err != nil { + return nil, err + } childCtx := sctx.SetPushTag(ctx, ctxTag) entry := newManifestTrieEntry(e, nil) if data != nil { diff --git a/swarm/swarm.go b/swarm/swarm.go index 5c6f50ffe0..40053bac49 100644 --- a/swarm/swarm.go +++ b/swarm/swarm.go @@ -211,7 +211,7 @@ func NewSwarm(config *api.Config, mockStore *mock.NodeStore) (self *Swarm, err e self.streamer = stream.NewRegistry(nodeID, delivery, self.netStore, self.stateStore, registryOptions, self.swap) // Swarm Hash Merklised Chunking for Arbitrary-length Document/File storage - self.fileStore = storage.NewFileStore(self.netStore, self.config.FileStoreParams) + self.fileStore = storage.NewFileStore(localStore, self.netStore, self.config.FileStoreParams) log.Debug("Setup local storage") From 3d4b18481da6f415ee71dbbf2b16b5b1e66de643 Mon Sep 17 00:00:00 2001 From: Elad Date: Wed, 3 Apr 2019 22:34:29 +0900 Subject: [PATCH 15/31] swarm/storage: simplify and ungeneralise tag implementation. move to separate files --- swarm/chunk/chunk.go | 4 -- swarm/storage/localstore/localstore.go | 25 +++++++- swarm/storage/localstore/mode_get.go | 6 -- swarm/storage/localstore/mode_put.go | 28 --------- swarm/storage/localstore/tags.go | 81 ++++++++++++++++++++++++++ swarm/storage/localstore/tags_test.go | 17 ++++++ 6 files changed, 121 insertions(+), 40 deletions(-) create mode 100644 swarm/storage/localstore/tags.go create mode 100644 swarm/storage/localstore/tags_test.go diff --git a/swarm/chunk/chunk.go b/swarm/chunk/chunk.go index 02c6116d25..8ecffcfc8c 100644 --- a/swarm/chunk/chunk.go +++ b/swarm/chunk/chunk.go @@ -127,8 +127,6 @@ const ( ModeGetSync // ModeGetLookup: when accessed to lookup a a chunk in feeds or other places ModeGetLookup - // ModeGetTags: gets the tags for a given reference - ModeGetTags ) // ModePut enumerates different Putter modes. @@ -142,8 +140,6 @@ const ( ModePutSync // ModePutUpload: when a chunk is created by local upload ModePutUpload - // ModePutTags: used to store a new tag when a file is uploaded - ModePutTags ) // ModeSet enumerates different Setter modes. diff --git a/swarm/storage/localstore/localstore.go b/swarm/storage/localstore/localstore.go index a6df24a0fa..3de3a3721c 100644 --- a/swarm/storage/localstore/localstore.go +++ b/swarm/storage/localstore/localstore.go @@ -76,7 +76,10 @@ type DB struct { // proximity order bin binIDs shed.Uint64Vector - // push syncing tags index + // uploads index maintains a list of pending uploads + uploadIndex shed.GenericIndex + + // tags index maintains a mapping between tags and pending uploads tagIndex shed.GenericIndex // garbage collection index @@ -322,8 +325,26 @@ func New(path string, baseKey []byte, o *Options) (db *DB, err error) { // create a push syncing triggers used by SubscribePush function db.pushTriggers = make([]chan struct{}, 0) + db.uploadIndex, err = db.shed.NewGenericIndex("UploadID->UploadTime|UploadName", shed.GenericIndexFuncs{ + EncodeKey: func(fields interface{}) (key []byte, err error) { + return nil, nil + }, + DecodeKey: func(key []byte) (e interface{}, err error) { + return nil, nil + }, + EncodeValue: func(fields interface{}) (value []byte, err error) { + return nil, nil + }, + DecodeValue: func(keyItem interface{}, value []byte) (e interface{}, err error) { + return nil, nil + }, + }) + if err != nil { + return nil, err + } + // tag index for push syncing tags - db.tagIndex, err = db.shed.NewGenericIndex("Tag->Filename", shed.GenericIndexFuncs{ //TODO: should this be Tag->Filename|StoreTimestamp? + db.tagIndex, err = db.shed.NewGenericIndex("UploadID|Tag->Filename", shed.GenericIndexFuncs{ EncodeKey: func(fields interface{}) (key []byte, err error) { return nil, nil }, diff --git a/swarm/storage/localstore/mode_get.go b/swarm/storage/localstore/mode_get.go index 7d8654603c..c177860a1c 100644 --- a/swarm/storage/localstore/mode_get.go +++ b/swarm/storage/localstore/mode_get.go @@ -79,12 +79,6 @@ func (db *DB) get(mode chunk.ModeGet, addr chunk.Address) (out shed.Item, err er // no updates to indexes case chunk.ModeGetSync: case chunk.ModeGetLookup: - case chunk.ModeGetTags: - c, err := db.pushIndex.Get(out) - if err != nil { - return out, err - } - return c, nil default: return out, ErrInvalidMode } diff --git a/swarm/storage/localstore/mode_put.go b/swarm/storage/localstore/mode_put.go index 1147734ad9..5691f92b94 100644 --- a/swarm/storage/localstore/mode_put.go +++ b/swarm/storage/localstore/mode_put.go @@ -137,9 +137,6 @@ func (db *DB) put(mode chunk.ModePut, item shed.Item) (err error) { triggerPullFeed = true } - case chunk.ModePutTags: - // put to indexes: tag - default: return ErrInvalidMode } @@ -161,28 +158,3 @@ func (db *DB) put(mode chunk.ModePut, item shed.Item) (err error) { } return nil } - -func (db *DB) PutGeneric(_ context.Context, mode chunk.ModePut, itemK, itemV interface{}) (err error) { - return db.putGeneric(mode, itemK, itemV) -} - -func (db *DB) putGeneric(mode chunk.ModePut, itemK, itemV interface{}) (err error) { - // protect parallel updates - db.batchMu.Lock() - defer db.batchMu.Unlock() - - batch := new(leveldb.Batch) - - switch mode { - case chunk.ModePutTags: - // put to indexes: tag - db.tagIndex.PutInBatch(batch, itemK, itemV) - default: - return ErrInvalidMode - } - err = db.shed.WriteBatch(batch) - if err != nil { - return err - } - return nil -} diff --git a/swarm/storage/localstore/tags.go b/swarm/storage/localstore/tags.go new file mode 100644 index 0000000000..611ac5e4c9 --- /dev/null +++ b/swarm/storage/localstore/tags.go @@ -0,0 +1,81 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package localstore + +import ( + "github.com/ethereum/go-ethereum/swarm/chunk" + "github.com/ethereum/go-ethereum/swarm/shed" + "github.com/syndtr/goleveldb/leveldb" +) + +var _ TagStore = &DB{} + +type TagStore interface { + PutUploadID(uploadId uint64, timestamp int64, uploadName string) error + PutTags(key, value interface{}) error + + GetTags(addr chunk.Address) ([]uint64, error) + PutTags(item shed.Item, tags []uint64) ([]uint64, error) +} + +func (db *DB) PutUploadID(id uint64, uploadTime int64, uploadName string) (err error) { + // protect parallel updates + db.batchMu.Lock() + defer db.batchMu.Unlock() + + batch := new(leveldb.Batch) + + // put to indexes: tag + db.Index.PutInBatch(batch, itemK, itemV) + err = db.shed.WriteBatch(batch) + if err != nil { + return err + } + return nil + +} + +func (db *DB) GetTags(addr chunk.Address) ([]uint64, error) { + item := addressToItem(addr) + + out, err = db.retrievalDataIndex.Get(item) + if err != nil { + return out, err + } + c, err := db.pushIndex.Get(out) + if err != nil { + return out, err + } + + return c.Tags(), nil +} + +func (db *DB) PutTags(uploadId, tag uint64, path string) (err error) { + // protect parallel updates + db.batchMu.Lock() + defer db.batchMu.Unlock() + + batch := new(leveldb.Batch) + + // put to indexes: tag + db.tagIndex.PutInBatch(batch, itemK, itemV) + err = db.shed.WriteBatch(batch) + if err != nil { + return err + } + return nil +} diff --git a/swarm/storage/localstore/tags_test.go b/swarm/storage/localstore/tags_test.go new file mode 100644 index 0000000000..598d29c0b6 --- /dev/null +++ b/swarm/storage/localstore/tags_test.go @@ -0,0 +1,17 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package localstore From 7afdbdc620bc00b437b9ad8c2f8a196781703116 Mon Sep 17 00:00:00 2001 From: Elad Date: Wed, 3 Apr 2019 22:41:03 +0900 Subject: [PATCH 16/31] swarm/storage: change interface method --- swarm/storage/localstore/tags.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/swarm/storage/localstore/tags.go b/swarm/storage/localstore/tags.go index 611ac5e4c9..43385c2124 100644 --- a/swarm/storage/localstore/tags.go +++ b/swarm/storage/localstore/tags.go @@ -18,7 +18,6 @@ package localstore import ( "github.com/ethereum/go-ethereum/swarm/chunk" - "github.com/ethereum/go-ethereum/swarm/shed" "github.com/syndtr/goleveldb/leveldb" ) @@ -28,8 +27,8 @@ type TagStore interface { PutUploadID(uploadId uint64, timestamp int64, uploadName string) error PutTags(key, value interface{}) error - GetTags(addr chunk.Address) ([]uint64, error) - PutTags(item shed.Item, tags []uint64) ([]uint64, error) + GetChunkTags(addr chunk.Address) ([]uint64, error) + PutTag(uploadId, tag uint64, path string) error } func (db *DB) PutUploadID(id uint64, uploadTime int64, uploadName string) (err error) { @@ -49,7 +48,7 @@ func (db *DB) PutUploadID(id uint64, uploadTime int64, uploadName string) (err e } -func (db *DB) GetTags(addr chunk.Address) ([]uint64, error) { +func (db *DB) GetChunkTags(addr chunk.Address) ([]uint64, error) { item := addressToItem(addr) out, err = db.retrievalDataIndex.Get(item) @@ -64,7 +63,7 @@ func (db *DB) GetTags(addr chunk.Address) ([]uint64, error) { return c.Tags(), nil } -func (db *DB) PutTags(uploadId, tag uint64, path string) (err error) { +func (db *DB) PutTag(uploadId, tag uint64, path string) (err error) { // protect parallel updates db.batchMu.Lock() defer db.batchMu.Unlock() From 4f5e8c94706d231329fb6c202bf885b288a638dc Mon Sep 17 00:00:00 2001 From: Elad Date: Thu, 4 Apr 2019 14:02:23 +0900 Subject: [PATCH 17/31] wip --- swarm/storage/localstore/localstore_test.go | 2 +- swarm/storage/localstore/tags.go | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/swarm/storage/localstore/localstore_test.go b/swarm/storage/localstore/localstore_test.go index 3e01402e03..c526226ab3 100644 --- a/swarm/storage/localstore/localstore_test.go +++ b/swarm/storage/localstore/localstore_test.go @@ -256,7 +256,7 @@ func newRetrieveIndexesTestWithAccess(db *DB, ch chunk.Chunk, storeTimestamp, ac if err != nil { t.Fatal(err) } - validateItem(t, item, ch.Address(), ch.Data(), storeTimestamp, []uint64) + validateItem(t, item, ch.Address(), ch.Data(), storeTimestamp, []uint64{}) if accessTimestamp > 0 { item, err = db.retrievalAccessIndex.Get(addressToItem(ch.Address())) diff --git a/swarm/storage/localstore/tags.go b/swarm/storage/localstore/tags.go index 43385c2124..c4cdd1336d 100644 --- a/swarm/storage/localstore/tags.go +++ b/swarm/storage/localstore/tags.go @@ -25,7 +25,6 @@ var _ TagStore = &DB{} type TagStore interface { PutUploadID(uploadId uint64, timestamp int64, uploadName string) error - PutTags(key, value interface{}) error GetChunkTags(addr chunk.Address) ([]uint64, error) PutTag(uploadId, tag uint64, path string) error @@ -37,9 +36,10 @@ func (db *DB) PutUploadID(id uint64, uploadTime int64, uploadName string) (err e defer db.batchMu.Unlock() batch := new(leveldb.Batch) + var k, v interface{} // put to indexes: tag - db.Index.PutInBatch(batch, itemK, itemV) + db.uploadIndex.PutInBatch(batch, k, v) err = db.shed.WriteBatch(batch) if err != nil { return err @@ -51,16 +51,16 @@ func (db *DB) PutUploadID(id uint64, uploadTime int64, uploadName string) (err e func (db *DB) GetChunkTags(addr chunk.Address) ([]uint64, error) { item := addressToItem(addr) - out, err = db.retrievalDataIndex.Get(item) + out, err := db.retrievalDataIndex.Get(item) if err != nil { - return out, err + return nil, err } c, err := db.pushIndex.Get(out) if err != nil { - return out, err + return nil, err } - return c.Tags(), nil + return c.Tags, nil } func (db *DB) PutTag(uploadId, tag uint64, path string) (err error) { @@ -69,9 +69,9 @@ func (db *DB) PutTag(uploadId, tag uint64, path string) (err error) { defer db.batchMu.Unlock() batch := new(leveldb.Batch) - + var k, v interface{} // put to indexes: tag - db.tagIndex.PutInBatch(batch, itemK, itemV) + db.tagIndex.PutInBatch(batch, k, v) err = db.shed.WriteBatch(batch) if err != nil { return err From 257a2885242b1e86e08fa7dcf96e52526e002077 Mon Sep 17 00:00:00 2001 From: Elad Date: Thu, 4 Apr 2019 14:09:03 +0900 Subject: [PATCH 18/31] iwp --- swarm/storage/localstore/localstore_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/swarm/storage/localstore/localstore_test.go b/swarm/storage/localstore/localstore_test.go index c526226ab3..399e5aac23 100644 --- a/swarm/storage/localstore/localstore_test.go +++ b/swarm/storage/localstore/localstore_test.go @@ -256,7 +256,7 @@ func newRetrieveIndexesTestWithAccess(db *DB, ch chunk.Chunk, storeTimestamp, ac if err != nil { t.Fatal(err) } - validateItem(t, item, ch.Address(), ch.Data(), storeTimestamp, []uint64{}) + validateItem(t, item, ch.Address(), ch.Data(), storeTimestamp, accessTimestamp, []uint64{}) if accessTimestamp > 0 { item, err = db.retrievalAccessIndex.Get(addressToItem(ch.Address())) @@ -280,7 +280,7 @@ func newPullIndexTest(db *DB, ch chunk.Chunk, binID uint64, wantError error) fun t.Errorf("got error %v, want %v", err, wantError) } if err == nil { - validateItem(t, item, ch.Address(), nil, 0, []uint64{}) + validateItem(t, item, ch.Address(), nil, 0, 0, []uint64{}) } } } From deccb1e60512f911588a7aa1506743b8e3f46e1a Mon Sep 17 00:00:00 2001 From: Elad Date: Thu, 4 Apr 2019 14:19:59 +0900 Subject: [PATCH 19/31] wip --- swarm/storage/localstore/localstore_test.go | 2 +- swarm/storage/localstore/mode_put_test.go | 22 --------------------- swarm/storage/localstore/tags_test.go | 22 +++++++++++++++++++++ 3 files changed, 23 insertions(+), 23 deletions(-) diff --git a/swarm/storage/localstore/localstore_test.go b/swarm/storage/localstore/localstore_test.go index 399e5aac23..cc78867d4a 100644 --- a/swarm/storage/localstore/localstore_test.go +++ b/swarm/storage/localstore/localstore_test.go @@ -297,7 +297,7 @@ func newPushIndexTest(db *DB, ch chunk.Chunk, storeTimestamp int64, wantError er t.Errorf("got error %v, want %v", err, wantError) } if err == nil { - validateItem(t, item, ch.Address(), nil, storeTimestamp, ch.Tags()) + validateItem(t, item, ch.Address(), nil, storeTimestamp, 0, ch.Tags()) } } } diff --git a/swarm/storage/localstore/mode_put_test.go b/swarm/storage/localstore/mode_put_test.go index b2f0cbfee3..225ba8280f 100644 --- a/swarm/storage/localstore/mode_put_test.go +++ b/swarm/storage/localstore/mode_put_test.go @@ -98,28 +98,6 @@ func TestModePutSync(t *testing.T) { t.Run("pull index", newPullIndexTest(db, ch, 1, nil)) } -func TestModePutTag(t *testing.T) { - db, cleanupFunc := newTestDB(t, nil) - defer cleanupFunc() - - wantTimestamp := time.Now().UTC().UnixNano() - defer setNow(func() (t int64) { - return wantTimestamp - })() - - ch := generateTestRandomChunk() - - err := db.Put(context.Background(), chunk.ModePutTags, ch) - if err != nil { - t.Fatal(err) - } - - t.Run("retrieve indexes", newRetrieveIndexesTest(db, ch, wantTimestamp, 0)) - - t.Run("pull index", newPullIndexTest(db, ch, 1, nil)) - -} - // TestModePutUpload validates ModePutUpload index values on the provided DB. func TestModePutUpload(t *testing.T) { db, cleanupFunc := newTestDB(t, nil) diff --git a/swarm/storage/localstore/tags_test.go b/swarm/storage/localstore/tags_test.go index 598d29c0b6..aee3a4b0b2 100644 --- a/swarm/storage/localstore/tags_test.go +++ b/swarm/storage/localstore/tags_test.go @@ -15,3 +15,25 @@ // along with the go-ethereum library. If not, see . package localstore + +/*func TestPutTag(t *testing.T) { + db, cleanupFunc := newTestDB(t, nil) + defer cleanupFunc() + + wantTimestamp := time.Now().UTC().UnixNano() + defer setNow(func() (t int64) { + return wantTimestamp + })() + + ch := generateTestRandomChunk() + + err := db.Put(context.Background(), chunk.ModePutTags, ch) + if err != nil { + t.Fatal(err) + } + + t.Run("retrieve indexes", newRetrieveIndexesTest(db, ch, wantTimestamp, 0)) + + t.Run("pull index", newPullIndexTest(db, ch, 1, nil)) + +}*/ From 75028f3fcb7fabde94d72a8dd549b955f4817464 Mon Sep 17 00:00:00 2001 From: Elad Date: Thu, 4 Apr 2019 15:35:12 +0900 Subject: [PATCH 20/31] swarm/storage: unbreak storage tests --- swarm/storage/chunker_test.go | 32 +++++++++++---------- swarm/storage/common_test.go | 17 +++++++++++ swarm/storage/feed/handler_test.go | 4 +-- swarm/storage/feed/request_test.go | 2 +- swarm/storage/filestore.go | 19 ++++++------ swarm/storage/filestore_test.go | 6 ++-- swarm/storage/hasherstore.go | 13 +++++---- swarm/storage/hasherstore_test.go | 18 ++++++------ swarm/storage/localstore/localstore_test.go | 2 +- swarm/storage/localstore/tags.go | 7 +++++ swarm/storage/types.go | 11 +++++++ 11 files changed, 87 insertions(+), 44 deletions(-) diff --git a/swarm/storage/chunker_test.go b/swarm/storage/chunker_test.go index 9a12594443..2cb569dae8 100644 --- a/swarm/storage/chunker_test.go +++ b/swarm/storage/chunker_test.go @@ -24,6 +24,7 @@ import ( "io" "testing" + "github.com/ethereum/go-ethereum/swarm/storage/localstore" "github.com/ethereum/go-ethereum/swarm/testutil" "golang.org/x/crypto/sha3" ) @@ -42,8 +43,8 @@ type chunkerTester struct { t test } -func newTestHasherStore(store ChunkStore, hash string) *hasherStore { - return NewHasherStore(store, MakeHashFunc(hash), false) +func newTestHasherStore(store ChunkStore, tagStore localstore.TagStore, hash string) *hasherStore { + return NewHasherStore(store, tagStore, MakeHashFunc(hash), false) } func testRandomBrokenData(n int, tester *chunkerTester) { @@ -58,8 +59,8 @@ func testRandomBrokenData(n int, tester *chunkerTester) { data = testutil.RandomReader(2, n) brokendata = brokenLimitReader(data, n, n/2) - - putGetter := newTestHasherStore(NewMapChunkStore(), SHA3Hash) + cs := NewMapChunkStore() + putGetter := newTestHasherStore(cs, cs, SHA3Hash) expectedError := fmt.Errorf("Broken reader") ctx := context.Background() @@ -83,8 +84,8 @@ func testRandomData(usePyramid bool, hash string, n int, tester *chunkerTester) } else { data = io.LimitReader(bytes.NewReader(input), int64(n)) } - - putGetter := newTestHasherStore(NewMapChunkStore(), hash) + cs := NewMapChunkStore() + putGetter := newTestHasherStore(cs, cs, hash) var addr Address var wait func(context.Context) error @@ -185,7 +186,7 @@ func TestDataAppend(t *testing.T) { } store := NewMapChunkStore() - putGetter := newTestHasherStore(store, SHA3Hash) + putGetter := newTestHasherStore(store, store, SHA3Hash) ctx := context.TODO() addr, wait, err := PyramidSplit(ctx, data, putGetter, putGetter) @@ -207,7 +208,7 @@ func TestDataAppend(t *testing.T) { appendData = io.LimitReader(bytes.NewReader(appendInput), int64(m)) } - putGetter = newTestHasherStore(store, SHA3Hash) + putGetter = newTestHasherStore(store, store, SHA3Hash) newAddr, wait, err := PyramidAppend(ctx, addr, appendData, putGetter, putGetter) if err != nil { tester.t.Fatalf(err.Error()) @@ -276,7 +277,8 @@ func benchmarkSplitJoin(n int, t *testing.B) { for i := 0; i < t.N; i++ { data := testutil.RandomReader(i, n) - putGetter := newTestHasherStore(NewMapChunkStore(), SHA3Hash) + cs := NewMapChunkStore() + putGetter := newTestHasherStore(cs, cs, SHA3Hash) ctx := context.TODO() key, wait, err := PyramidSplit(ctx, data, putGetter, putGetter) if err != nil { @@ -295,7 +297,7 @@ func benchmarkSplitTreeSHA3(n int, t *testing.B) { t.ReportAllocs() for i := 0; i < t.N; i++ { data := testutil.RandomReader(i, n) - putGetter := newTestHasherStore(&FakeChunkStore{}, SHA3Hash) + putGetter := newTestHasherStore(&FakeChunkStore{}, &FakeChunkStore{}, SHA3Hash) ctx := context.Background() _, wait, err := TreeSplit(ctx, data, int64(n), putGetter) @@ -314,7 +316,7 @@ func benchmarkSplitTreeBMT(n int, t *testing.B) { t.ReportAllocs() for i := 0; i < t.N; i++ { data := testutil.RandomReader(i, n) - putGetter := newTestHasherStore(&FakeChunkStore{}, BMTHash) + putGetter := newTestHasherStore(&FakeChunkStore{}, &FakeChunkStore{}, BMTHash) ctx := context.Background() _, wait, err := TreeSplit(ctx, data, int64(n), putGetter) @@ -332,7 +334,7 @@ func benchmarkSplitPyramidBMT(n int, t *testing.B) { t.ReportAllocs() for i := 0; i < t.N; i++ { data := testutil.RandomReader(i, n) - putGetter := newTestHasherStore(&FakeChunkStore{}, BMTHash) + putGetter := newTestHasherStore(&FakeChunkStore{}, &FakeChunkStore{}, BMTHash) ctx := context.Background() _, wait, err := PyramidSplit(ctx, data, putGetter, putGetter) @@ -350,7 +352,7 @@ func benchmarkSplitPyramidSHA3(n int, t *testing.B) { t.ReportAllocs() for i := 0; i < t.N; i++ { data := testutil.RandomReader(i, n) - putGetter := newTestHasherStore(&FakeChunkStore{}, SHA3Hash) + putGetter := newTestHasherStore(&FakeChunkStore{}, &FakeChunkStore{}, SHA3Hash) ctx := context.Background() _, wait, err := PyramidSplit(ctx, data, putGetter, putGetter) @@ -371,7 +373,7 @@ func benchmarkSplitAppendPyramid(n, m int, t *testing.B) { data1 := testutil.RandomReader(t.N+i, m) store := NewMapChunkStore() - putGetter := newTestHasherStore(store, SHA3Hash) + putGetter := newTestHasherStore(store, store, SHA3Hash) ctx := context.Background() key, wait, err := PyramidSplit(ctx, data, putGetter, putGetter) @@ -383,7 +385,7 @@ func benchmarkSplitAppendPyramid(n, m int, t *testing.B) { t.Fatalf(err.Error()) } - putGetter = newTestHasherStore(store, SHA3Hash) + putGetter = newTestHasherStore(store, store, SHA3Hash) _, wait, err = PyramidAppend(ctx, key, data1, putGetter, putGetter) if err != nil { t.Fatalf(err.Error()) diff --git a/swarm/storage/common_test.go b/swarm/storage/common_test.go index a4e7120dfa..df68666a17 100644 --- a/swarm/storage/common_test.go +++ b/swarm/storage/common_test.go @@ -262,6 +262,23 @@ func (m *MapChunkStore) SubscribePull(ctx context.Context, bin uint8, since, unt return nil, nil } +func (m *MapChunkStore) GetChunkTags(addr Address) ([]uint64, error) { + m.mu.RLock() + defer m.mu.RUnlock() + chunk := m.chunks[addr.Hex()] + if chunk == nil { + return []uint64{}, nil + } + return chunk.Tags(), nil +} +func (m *MapChunkStore) PutUploadID(uploadId uint64, timestamp int64, uploadName string) error { + return nil +} + +func (m *MapChunkStore) PutTag(uploadId, tag uint64, path string) error { + return nil +} + func (m *MapChunkStore) Close() error { return nil } diff --git a/swarm/storage/feed/handler_test.go b/swarm/storage/feed/handler_test.go index e8b6c9f916..451ae17009 100644 --- a/swarm/storage/feed/handler_test.go +++ b/swarm/storage/feed/handler_test.go @@ -375,7 +375,7 @@ func TestValidator(t *testing.T) { address[0] = 11 address[15] = 99 - if rh.Validate(storage.NewChunk(address, chunk.Data())) { + if rh.Validate(storage.NewChunk(address, chunk.Data(), []uint64{})) { t.Fatal("Expected Validate to fail with false chunk address") } } @@ -414,7 +414,7 @@ func TestValidatorInStore(t *testing.T) { // create content addressed chunks, one good, one faulty chunks := storage.GenerateRandomChunks(chunk.DefaultSize, 2) goodChunk := chunks[0] - badChunk := storage.NewChunk(chunks[1].Address(), goodChunk.Data()) + badChunk := storage.NewChunk(chunks[1].Address(), goodChunk.Data(), []uint64{}) topic, _ := NewTopic("xyzzy", nil) fd := Feed{ diff --git a/swarm/storage/feed/request_test.go b/swarm/storage/feed/request_test.go index c30158fddf..47f8059e1c 100644 --- a/swarm/storage/feed/request_test.go +++ b/swarm/storage/feed/request_test.go @@ -197,7 +197,7 @@ func TestUpdateChunkSerializationErrorChecking(t *testing.T) { // Test that parseUpdate fails if the chunk is too small var r Request - if err := r.fromChunk(storage.NewChunk(storage.ZeroAddr, make([]byte, minimumUpdateDataLength-1+signatureLength))); err == nil { + if err := r.fromChunk(storage.NewChunk(storage.ZeroAddr, make([]byte, minimumUpdateDataLength-1+signatureLength), []uint64{})); err == nil { t.Fatalf("Expected request.fromChunk to fail when chunkData contains less than %d bytes", minimumUpdateDataLength) } diff --git a/swarm/storage/filestore.go b/swarm/storage/filestore.go index c107574d72..e6d1ab40d0 100644 --- a/swarm/storage/filestore.go +++ b/swarm/storage/filestore.go @@ -48,8 +48,8 @@ const ( type FileStore struct { ChunkStore - localStore *localstore.DB - hashFunc SwarmHasher + tagStore localstore.TagStore + hashFunc SwarmHasher } type FileStoreParams struct { @@ -71,10 +71,10 @@ func NewLocalFileStore(datadir string, basekey []byte) (*FileStore, error) { return NewFileStore(localStore, chunk.NewValidatorStore(localStore, NewContentAddressValidator(MakeHashFunc(DefaultHash))), NewFileStoreParams()), nil } -func NewFileStore(localstore *localstore.DB, store ChunkStore, params *FileStoreParams) *FileStore { +func NewFileStore(tagStore localstore.TagStore, store ChunkStore, params *FileStoreParams) *FileStore { hashFunc := MakeHashFunc(params.Hash) return &FileStore{ - localStore: localstore, + tagStore: tagStore, ChunkStore: store, hashFunc: hashFunc, } @@ -87,7 +87,7 @@ func NewFileStore(localstore *localstore.DB, store ChunkStore, params *FileStore // It returns a reader with the chunk data and whether the content was encrypted func (f *FileStore) Retrieve(ctx context.Context, addr Address) (reader *LazyChunkReader, isEncrypted bool) { isEncrypted = len(addr) > f.hashFunc().Size() - getter := NewHasherStore(f.ChunkStore, f.hashFunc, isEncrypted) + getter := NewHasherStore(f.ChunkStore, f.tagStore, f.hashFunc, isEncrypted) reader = TreeJoin(ctx, addr, getter, 0) return } @@ -95,7 +95,7 @@ func (f *FileStore) Retrieve(ctx context.Context, addr Address) (reader *LazyChu // Store is a public API. Main entry point for document storage directly. Used by the // FS-aware API and httpaccess func (f *FileStore) Store(ctx context.Context, data io.Reader, size int64, toEncrypt bool) (addr Address, wait func(context.Context) error, err error) { - putter := NewHasherStore(f.ChunkStore, f.hashFunc, toEncrypt) + putter := NewHasherStore(f.ChunkStore, f.tagStore, f.hashFunc, toEncrypt) return PyramidSplit(ctx, data, putter, putter) } @@ -106,6 +106,9 @@ func (f *FileStore) HashSize() int { // CreateTag creates a new push tag and stores it in localstore // it returns the tag as uint64 func (f *FileStore) CreateTag(ctx context.Context, filename string, timestamp uint64) (uint64, error) { + // add uploadID + + var uploadId uint64 = 0 intBuf := make([]byte, 8) binary.BigEndian.PutUint64(intBuf, timestamp) // Tag is SHA3(filename|storetimestamp)[:8] @@ -114,7 +117,7 @@ func (f *FileStore) CreateTag(ctx context.Context, filename string, timestamp ui tagHash := crypto.Keccak256(buf)[:8] tag := binary.BigEndian.Uint64(tagHash) - err := f.localStore.PutGeneric(ctx, chunk.ModePutTags, interface{}(tag), interface{}(filename)) + err := f.tagStore.PutTag(uploadId, tag, filename) if err != nil { return tag, err } @@ -125,7 +128,7 @@ func (f *FileStore) CreateTag(ctx context.Context, filename string, timestamp ui func (f *FileStore) GetAllReferences(ctx context.Context, data io.Reader, toEncrypt bool) (addrs AddressCollection, err error) { // create a special kind of putter, which only will store the references putter := &hashExplorer{ - hasherStore: NewHasherStore(f.ChunkStore, f.hashFunc, toEncrypt), + hasherStore: NewHasherStore(f.ChunkStore, f.tagStore, f.hashFunc, toEncrypt), } // do the actual splitting anyway, no way around it _, wait, err := PyramidSplit(ctx, data, putter, putter) diff --git a/swarm/storage/filestore_test.go b/swarm/storage/filestore_test.go index fe01eed9aa..af3deeb81a 100644 --- a/swarm/storage/filestore_test.go +++ b/swarm/storage/filestore_test.go @@ -48,7 +48,7 @@ func testFileStoreRandom(toEncrypt bool, t *testing.T) { } defer localStore.Close() - fileStore := NewFileStore(localStore, NewFileStoreParams()) + fileStore := NewFileStore(localStore, localStore, NewFileStoreParams()) slice := testutil.RandomBytes(1, testDataSize) ctx := context.TODO() @@ -113,7 +113,7 @@ func testFileStoreCapacity(toEncrypt bool, t *testing.T) { } defer localStore.Close() - fileStore := NewFileStore(localStore, NewFileStoreParams()) + fileStore := NewFileStore(localStore, localStore, NewFileStoreParams()) slice := testutil.RandomBytes(1, testDataSize) ctx := context.TODO() key, wait, err := fileStore.Store(ctx, bytes.NewReader(slice), testDataSize, toEncrypt) @@ -182,7 +182,7 @@ func TestGetAllReferences(t *testing.T) { } defer localStore.Close() - fileStore := NewFileStore(localStore, NewFileStoreParams()) + fileStore := NewFileStore(localStore, localStore, NewFileStoreParams()) // testRuns[i] and expectedLen[i] are dataSize and expected length respectively testRuns := []int{1024, 8192, 16000, 30000, 1000000} diff --git a/swarm/storage/hasherstore.go b/swarm/storage/hasherstore.go index d5f328aee3..85d7fba14e 100644 --- a/swarm/storage/hasherstore.go +++ b/swarm/storage/hasherstore.go @@ -24,11 +24,13 @@ import ( "github.com/ethereum/go-ethereum/swarm/chunk" "github.com/ethereum/go-ethereum/swarm/sctx" "github.com/ethereum/go-ethereum/swarm/storage/encryption" + "github.com/ethereum/go-ethereum/swarm/storage/localstore" "golang.org/x/crypto/sha3" ) type hasherStore struct { store ChunkStore + tagStore localstore.TagStore toEncrypt bool hashFunc SwarmHasher hashSize int // content hash size @@ -45,7 +47,7 @@ type hasherStore struct { // NewHasherStore creates a hasherStore object, which implements Putter and Getter interfaces. // With the HasherStore you can put and get chunk data (which is just []byte) into a ChunkStore // and the hasherStore will take core of encryption/decryption of data if necessary -func NewHasherStore(store ChunkStore, hashFunc SwarmHasher, toEncrypt bool) *hasherStore { +func NewHasherStore(store ChunkStore, tagStore localstore.TagStore, hashFunc SwarmHasher, toEncrypt bool) *hasherStore { hashSize := hashFunc().Size() refSize := int64(hashSize) if toEncrypt { @@ -54,6 +56,7 @@ func NewHasherStore(store ChunkStore, hashFunc SwarmHasher, toEncrypt bool) *has h := &hasherStore{ store: store, + tagStore: tagStore, toEncrypt: toEncrypt, hashFunc: hashFunc, hashSize: hashSize, @@ -113,11 +116,11 @@ func (h *hasherStore) Get(ctx context.Context, ref Reference) (ChunkData, error) } func (h *hasherStore) GetTags(ctx context.Context, addr chunk.Address) ([]uint64, error) { - chunk, err := h.store.Get(ctx, chunk.ModeGetTags, addr) + tags, err := h.tagStore.GetChunkTags(addr) if err != nil { return nil, err } - return chunk.Tags(), nil + return tags, nil } // Close indicates that no more chunks will be put with the hasherStore, so the Wait @@ -168,9 +171,9 @@ func (h *hasherStore) createHash(chunkData ChunkData) Address { func (h *hasherStore) createChunk(ctx context.Context, chunkData ChunkData) Chunk { hash := h.createHash(chunkData) - tags, err := h.GetTags(ctx, chunk.Address(hash)) // TODO: this is really bad but if we want to persist tags across sessions this would be the way + tags, err := h.tagStore.GetChunkTags(chunk.Address(hash)) // TODO: this is really bad but if we want to persist tags across sessions this would be the way if err != nil { - panic("wtf") + panic(err) } tag := sctx.GetPushTag(ctx) if tag != 0 { diff --git a/swarm/storage/hasherstore_test.go b/swarm/storage/hasherstore_test.go index af300ae1ea..97b0a444d2 100644 --- a/swarm/storage/hasherstore_test.go +++ b/swarm/storage/hasherstore_test.go @@ -46,7 +46,7 @@ func TestHasherStore(t *testing.T) { for _, tt := range tests { chunkStore := NewMapChunkStore() - hasherStore := NewHasherStore(chunkStore, MakeHashFunc(DefaultHash), tt.toEncrypt) + hasherStore := NewHasherStore(chunkStore, chunkStore, MakeHashFunc(DefaultHash), tt.toEncrypt) r := mrand.New(mrand.NewSource(time.Now().UnixNano())) customTag := r.Uint64() @@ -83,8 +83,11 @@ func TestHasherStore(t *testing.T) { if !bytes.Equal(chunkData1, retrievedChunkData1) { t.Fatalf("Expected retrieved chunk data %v, got %v", common.Bytes2Hex(chunkData1), common.Bytes2Hex(retrievedChunkData1)) } - - tags, err := hasherStore.GetTags(ctx, key1) + hash1, encryptionKey1, err := parseReference(key1, hasherStore.hashSize) + if err != nil { + t.Fatal(err) + } + tags, err := hasherStore.GetTags(ctx, hash1) if err != nil { t.Fatalf("had an error fetching tags from hasher store: %v", err) } @@ -109,7 +112,9 @@ func TestHasherStore(t *testing.T) { // get the tags from the underlying localstore and assert they are equal to the tag assigned // in the context above (hasherstore should put the file with the appropriate tags from Context // rather than from the chunk struct) - tags2, err := hasherStore.GetTags(ctx, key2) + hash2, _, err := parseReference(key1, hasherStore.hashSize) + + tags2, err := hasherStore.GetTags(ctx, hash2) if err != nil { t.Fatalf("had an error fetching tags from hasher store: %v", err) } @@ -120,11 +125,6 @@ func TestHasherStore(t *testing.T) { t.Fatalf("tag mismatch. want %d, got %d", customTag, tags2[0]) } - hash1, encryptionKey1, err := parseReference(key1, hasherStore.hashSize) - if err != nil { - t.Fatalf("Expected no error, got \"%v\"", err) - } - if tt.toEncrypt { if encryptionKey1 == nil { t.Fatal("Expected non-nil encryption key, got nil") diff --git a/swarm/storage/localstore/localstore_test.go b/swarm/storage/localstore/localstore_test.go index cc78867d4a..8a231a0629 100644 --- a/swarm/storage/localstore/localstore_test.go +++ b/swarm/storage/localstore/localstore_test.go @@ -256,7 +256,7 @@ func newRetrieveIndexesTestWithAccess(db *DB, ch chunk.Chunk, storeTimestamp, ac if err != nil { t.Fatal(err) } - validateItem(t, item, ch.Address(), ch.Data(), storeTimestamp, accessTimestamp, []uint64{}) + validateItem(t, item, ch.Address(), ch.Data(), storeTimestamp, 0, []uint64{}) if accessTimestamp > 0 { item, err = db.retrievalAccessIndex.Get(addressToItem(ch.Address())) diff --git a/swarm/storage/localstore/tags.go b/swarm/storage/localstore/tags.go index c4cdd1336d..035827c414 100644 --- a/swarm/storage/localstore/tags.go +++ b/swarm/storage/localstore/tags.go @@ -53,10 +53,17 @@ func (db *DB) GetChunkTags(addr chunk.Address) ([]uint64, error) { out, err := db.retrievalDataIndex.Get(item) if err != nil { + if err == leveldb.ErrNotFound { + return []uint64{}, nil + } + return nil, err } c, err := db.pushIndex.Get(out) if err != nil { + if err == leveldb.ErrNotFound { + return []uint64{}, nil + } return nil, err } diff --git a/swarm/storage/types.go b/swarm/storage/types.go index 8e2db7ad10..680295b02b 100644 --- a/swarm/storage/types.go +++ b/swarm/storage/types.go @@ -263,6 +263,17 @@ func (f *FakeChunkStore) SubscribePull(ctx context.Context, bin uint8, since, un panic("FakeChunkStore doesn't support SubscribePull") } +func (f *FakeChunkStore) GetChunkTags(addr Address) ([]uint64, error) { + panic("FakeChunkStore doesn't support GetChunkTags") +} +func (f *FakeChunkStore) PutUploadID(uploadId uint64, timestamp int64, uploadName string) error { + panic("FakeChunkStore doesn't support PutUploadID") +} + +func (f *FakeChunkStore) PutTag(uploadId, tag uint64, path string) error { + panic("FakeChunkStore doesn't support PutTag") +} + // Close doesn't store anything it is just here to implement ChunkStore func (f *FakeChunkStore) Close() error { return nil From 92da165f5f2f753a6fae0b1507a7e04588d4475f Mon Sep 17 00:00:00 2001 From: Elad Date: Thu, 4 Apr 2019 15:42:07 +0900 Subject: [PATCH 21/31] swarm/storage: change test name --- swarm/storage/filestore_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swarm/storage/filestore_test.go b/swarm/storage/filestore_test.go index af3deeb81a..2a7f9f0547 100644 --- a/swarm/storage/filestore_test.go +++ b/swarm/storage/filestore_test.go @@ -31,7 +31,7 @@ import ( const testDataSize = 0x0001000 -func TestFileStorerandom(t *testing.T) { +func TestFileStoreRandom(t *testing.T) { testFileStoreRandom(false, t) testFileStoreRandom(true, t) } From 0f5a9300efed961178228f641cfe63f0b2130be4 Mon Sep 17 00:00:00 2001 From: Elad Date: Thu, 4 Apr 2019 17:06:21 +0900 Subject: [PATCH 22/31] swarm: move TagStore interface to `chunk` packge --- swarm/chunk/chunk.go | 7 +++++++ swarm/storage/chunker_test.go | 4 ++-- swarm/storage/filestore.go | 4 ++-- swarm/storage/hasherstore.go | 5 ++--- swarm/storage/localstore/tags.go | 9 +-------- 5 files changed, 14 insertions(+), 15 deletions(-) diff --git a/swarm/chunk/chunk.go b/swarm/chunk/chunk.go index 8ecffcfc8c..5626bd44a5 100644 --- a/swarm/chunk/chunk.go +++ b/swarm/chunk/chunk.go @@ -217,3 +217,10 @@ func (s *ValidatorStore) Put(ctx context.Context, mode ModePut, ch Chunk) (err e } return ErrChunkInvalid } + +type TagStore interface { + PutUploadID(uploadId uint64, timestamp int64, uploadName string) error + + GetChunkTags(addr Address) ([]uint64, error) + PutTag(uploadId, tag uint64, path string) error +} diff --git a/swarm/storage/chunker_test.go b/swarm/storage/chunker_test.go index 2cb569dae8..fd9b999e12 100644 --- a/swarm/storage/chunker_test.go +++ b/swarm/storage/chunker_test.go @@ -24,7 +24,7 @@ import ( "io" "testing" - "github.com/ethereum/go-ethereum/swarm/storage/localstore" + "github.com/ethereum/go-ethereum/swarm/chunk" "github.com/ethereum/go-ethereum/swarm/testutil" "golang.org/x/crypto/sha3" ) @@ -43,7 +43,7 @@ type chunkerTester struct { t test } -func newTestHasherStore(store ChunkStore, tagStore localstore.TagStore, hash string) *hasherStore { +func newTestHasherStore(store ChunkStore, tagStore chunk.TagStore, hash string) *hasherStore { return NewHasherStore(store, tagStore, MakeHashFunc(hash), false) } diff --git a/swarm/storage/filestore.go b/swarm/storage/filestore.go index e6d1ab40d0..bc1d305db1 100644 --- a/swarm/storage/filestore.go +++ b/swarm/storage/filestore.go @@ -48,7 +48,7 @@ const ( type FileStore struct { ChunkStore - tagStore localstore.TagStore + tagStore chunk.TagStore hashFunc SwarmHasher } @@ -71,7 +71,7 @@ func NewLocalFileStore(datadir string, basekey []byte) (*FileStore, error) { return NewFileStore(localStore, chunk.NewValidatorStore(localStore, NewContentAddressValidator(MakeHashFunc(DefaultHash))), NewFileStoreParams()), nil } -func NewFileStore(tagStore localstore.TagStore, store ChunkStore, params *FileStoreParams) *FileStore { +func NewFileStore(tagStore chunk.TagStore, store ChunkStore, params *FileStoreParams) *FileStore { hashFunc := MakeHashFunc(params.Hash) return &FileStore{ tagStore: tagStore, diff --git a/swarm/storage/hasherstore.go b/swarm/storage/hasherstore.go index 85d7fba14e..0063a4fcf5 100644 --- a/swarm/storage/hasherstore.go +++ b/swarm/storage/hasherstore.go @@ -24,13 +24,12 @@ import ( "github.com/ethereum/go-ethereum/swarm/chunk" "github.com/ethereum/go-ethereum/swarm/sctx" "github.com/ethereum/go-ethereum/swarm/storage/encryption" - "github.com/ethereum/go-ethereum/swarm/storage/localstore" "golang.org/x/crypto/sha3" ) type hasherStore struct { store ChunkStore - tagStore localstore.TagStore + tagStore chunk.TagStore toEncrypt bool hashFunc SwarmHasher hashSize int // content hash size @@ -47,7 +46,7 @@ type hasherStore struct { // NewHasherStore creates a hasherStore object, which implements Putter and Getter interfaces. // With the HasherStore you can put and get chunk data (which is just []byte) into a ChunkStore // and the hasherStore will take core of encryption/decryption of data if necessary -func NewHasherStore(store ChunkStore, tagStore localstore.TagStore, hashFunc SwarmHasher, toEncrypt bool) *hasherStore { +func NewHasherStore(store ChunkStore, tagStore chunk.TagStore, hashFunc SwarmHasher, toEncrypt bool) *hasherStore { hashSize := hashFunc().Size() refSize := int64(hashSize) if toEncrypt { diff --git a/swarm/storage/localstore/tags.go b/swarm/storage/localstore/tags.go index 035827c414..175b936117 100644 --- a/swarm/storage/localstore/tags.go +++ b/swarm/storage/localstore/tags.go @@ -21,14 +21,7 @@ import ( "github.com/syndtr/goleveldb/leveldb" ) -var _ TagStore = &DB{} - -type TagStore interface { - PutUploadID(uploadId uint64, timestamp int64, uploadName string) error - - GetChunkTags(addr chunk.Address) ([]uint64, error) - PutTag(uploadId, tag uint64, path string) error -} +var _ chunk.TagStore = &DB{} func (db *DB) PutUploadID(id uint64, uploadTime int64, uploadName string) (err error) { // protect parallel updates From be5627ca10b3c90e1cd65298c406d1861c58af0a Mon Sep 17 00:00:00 2001 From: Elad Date: Thu, 4 Apr 2019 19:06:17 +0900 Subject: [PATCH 23/31] swarm/storage: no comment --- swarm/storage/localstore/tags.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/swarm/storage/localstore/tags.go b/swarm/storage/localstore/tags.go index 175b936117..8d5498d0f3 100644 --- a/swarm/storage/localstore/tags.go +++ b/swarm/storage/localstore/tags.go @@ -17,6 +17,8 @@ package localstore import ( + "encoding/binary" + "github.com/ethereum/go-ethereum/swarm/chunk" "github.com/syndtr/goleveldb/leveldb" ) @@ -29,10 +31,12 @@ func (db *DB) PutUploadID(id uint64, uploadTime int64, uploadName string) (err e defer db.batchMu.Unlock() batch := new(leveldb.Batch) - var k, v interface{} + val := make([]byte, 8) + binary.BigEndian.PutInt64(val, uploadTime) + val = append(val, []byte(uploadName)) // put to indexes: tag - db.uploadIndex.PutInBatch(batch, k, v) + db.uploadIndex.PutInBatch(batch, interface{ id }, interface{ val }) err = db.shed.WriteBatch(batch) if err != nil { return err From 129993c74ce1dc269f1fec784a3b11f5ee60ab0e Mon Sep 17 00:00:00 2001 From: Elad Date: Thu, 4 Apr 2019 21:19:04 +0900 Subject: [PATCH 24/31] swarm/chunk: add the only correct code that has been written so far --- swarm/chunk/tags.go | 164 ++++++++++++++++++ swarm/chunk/tags_test.go | 178 ++++++++++++++++++++ swarm/storage/localstore/localstore_test.go | 11 ++ swarm/storage/localstore/tags.go | 39 ++--- swarm/storage/localstore/tags_test.go | 40 +++++ 5 files changed, 413 insertions(+), 19 deletions(-) create mode 100644 swarm/chunk/tags.go create mode 100644 swarm/chunk/tags_test.go diff --git a/swarm/chunk/tags.go b/swarm/chunk/tags.go new file mode 100644 index 0000000000..6780418291 --- /dev/null +++ b/swarm/chunk/tags.go @@ -0,0 +1,164 @@ +package chunk + +import ( + "context" + "errors" + "sync" + "sync/atomic" + "time" + + "github.com/ethereum/go-ethereum/log" +) + +var ( + errExists = errors.New("already exists") + errNoETA = errors.New("unable to calculate ETA") +) + +// State is the enum type for chunk states +type State = uint32 + +const ( + SPLIT State = iota // chunk has been processed by filehasher/swarm safe call + STORED // chunk stored locally + SENT // chunk sent to neighbourhood + SYNCED // proof is received; chunk removed from sync db; chunk is available everywhere +) + +// Tag represents info on the status of new chunks +type Tag struct { + name string + total uint32 // total chunks belonging to a tag + split uint32 // number of chunks already processed by splitter for hashing + stored uint32 // number of chunks already stored locally + sent uint32 // number of chunks sent for push syncing + synced uint32 // number of chunks synced with proof + startedAt time.Time // tag started to calculate ETA + State chan State // channel to signal completion +} + +// tags holds the tag infos indexed by name +type tags struct { + tags *sync.Map +} + +// NewTags creates a tags object +func newTags() *tags { + return &tags{ + &sync.Map{}, + } +} + +// New creates a new tag, stores it by the name and returns it +// it returns an error if the tag with this name already exists +func (ts *tags) New(s string, total int) (*Tag, error) { + t := &Tag{ + name: s, + startedAt: time.Now(), + total: uint32(total), + State: make(chan State, 5), + } + _, loaded := ts.tags.LoadOrStore(s, t) + if loaded { + return nil, errExists + } + return t, nil +} + +// Inc increments the count for a state +func (t *Tag) Inc(state State) { + var v *uint32 + switch state { + case SPLIT: + v = &t.split + case STORED: + v = &t.stored + case SENT: + v = &t.sent + case SYNCED: + v = &t.synced + } + n := atomic.AddUint32(v, 1) + if int(n) == t.GetTotal() { + t.State <- state + } +} + +// Get returns the count for a state on a tag +func (t *Tag) Get(state State) int { + var v *uint32 + switch state { + case SPLIT: + v = &t.split + case STORED: + v = &t.stored + case SENT: + v = &t.sent + case SYNCED: + v = &t.synced + } + return int(atomic.LoadUint32(v)) +} + +// GetTotal returns the total count +func (t *Tag) GetTotal() int { + return int(atomic.LoadUint32(&t.total)) +} + +// SetTotal sets total count to SPLIT count +// is meant to be called when splitter finishes for input streams of unknown size +func (t *Tag) SetTotal() int { + total := atomic.LoadUint32(&t.split) + atomic.StoreUint32(&t.total, total) + return int(total) +} + +// Status returns the value of state and the total count +func (t *Tag) Status(state State) (int, int) { + return t.Get(state), int(atomic.LoadUint32(&t.total)) +} + +// ETA returns the time of completion estimated based on time passed and rate of completion +func (t *Tag) ETA(state State) (time.Time, error) { + cnt := t.Get(state) + total := t.GetTotal() + if cnt == 0 || total == 0 { + return time.Time{}, errNoETA + } + diff := time.Since(t.startedAt) + dur := time.Duration(total) * diff / time.Duration(cnt) + return t.startedAt.Add(dur), nil +} + +// Inc increments the state count for a tag if tag is found +func (ts *tags) Inc(s string, f State) { + t, ok := ts.tags.Load(s) + if !ok { + return + } + t.(*Tag).Inc(f) +} + +// Get returns the state count for a tag +func (ts *tags) Get(s string, f State) int { + t, _ := ts.tags.Load(s) + return t.(*Tag).Get(f) +} + +// WaitTill blocks until count for the State reaches total cnt +func (tg *Tag) WaitTill(ctx context.Context, s State) error { + ticker := time.NewTicker(1 * time.Second) + defer ticker.Stop() + for { + select { + case c := <-tg.State: + if c == s { + return nil + } + case <-ctx.Done(): + return ctx.Err() + case <-ticker.C: + log.Error("Status", "name", tg.name, "SENT", tg.Get(SENT), "SYNCED", tg.Get(SYNCED)) + } + } +} diff --git a/swarm/chunk/tags_test.go b/swarm/chunk/tags_test.go new file mode 100644 index 0000000000..db6747392f --- /dev/null +++ b/swarm/chunk/tags_test.go @@ -0,0 +1,178 @@ +package chunk + +import ( + "context" + "io/ioutil" + "math/rand" + "os" + "sync" + "testing" + "time" + + "github.com/ethereum/go-ethereum/swarm/chunk" + "github.com/ethereum/go-ethereum/swarm/network" +) + +var ( + allStates = []State{SPLIT, STORED, SENT, SYNCED} +) + +// TestTagSingleIncrements tests if Inc increments the tag state value +func TestTagSingleIncrements(t *testing.T) { + tg := &Tag{total: 10} + for _, f := range allStates { + tg.Inc(f) + if tg.Get(f) != 1 { + t.Fatalf("not incremented") + } + cnt, total := tg.Status(f) + if cnt != 1 { + t.Fatalf("expected count 1 for state %v, got %v", f, cnt) + } + if total != 10 { + t.Fatalf("expected total count %v for state %v, got %v", 10, f, cnt) + } + } +} + +// tests ETA is precise +func TestTagETA(t *testing.T) { + now := time.Now() + maxDiff := 100000 // 100 microsecond + tg := &Tag{total: 10, startedAt: now} + time.Sleep(100 * time.Millisecond) + tg.Inc(SPLIT) + eta, err := tg.ETA(SPLIT) + if err != nil { + t.Fatal(err) + } + diff := time.Until(eta) - 9*time.Since(now) + if int(diff) > maxDiff || int(diff) < -maxDiff { + t.Fatalf("ETA is not precise, got diff %v > .1ms", diff) + } +} + +// TestTagConcurrentIncrements tests Inc calls concurrently +func TestTagConcurrentIncrements(t *testing.T) { + tg := &Tag{} + n := 1000 + wg := sync.WaitGroup{} + wg.Add(4 * n) + for _, f := range allStates { + go func(f State) { + for j := 0; j < n; j++ { + go func() { + tg.Inc(f) + wg.Done() + }() + } + }(f) + } + wg.Wait() + for _, f := range allStates { + v := tg.Get(f) + if v != n { + t.Fatalf("expected state %v to be %v, got %v", f, n, v) + } + } +} + +// TestTagsMultipleConcurrentIncrements tests Inc calls concurrently +func TestTagsMultipleConcurrentIncrements(t *testing.T) { + ts := newTags() + n := 100 + wg := sync.WaitGroup{} + wg.Add(10 * 4 * n) + for i := 0; i < 10; i++ { + s := string([]byte{uint8(i)}) + ts.New(s, n) + for _, f := range allStates { + go func(s string, f State) { + for j := 0; j < n; j++ { + go func() { + ts.Inc(s, f) + wg.Done() + }() + } + }(s, f) + } + } + wg.Wait() + for i := 0; i < 10; i++ { + s := string([]byte{uint8(i)}) + for _, f := range allStates { + v := ts.Get(s, f) + if v != n { + t.Fatalf("expected tag %v state %v to be %v, got %v", s, f, n, v) + } + } + } +} + +// tests the correct behaviour of tags while using the DB +func TestDBWithTags(t *testing.T) { + names := []string{"1", "2", "3", "4"} + receiptsC := make(chan chunk.Address) + quit := make(chan struct{}) + defer close(quit) + // sync function is not called concurrently, so max need no lock + // TODO: chunksSentAt array should use lock + sync := func(chunk chunk.Chunk) error { + + // this go routine mimics the chunk sync - poc response roundrtip + // with random delay (uniform within a fixed range) + go func() { + n := rand.Intn(1000) + delay := time.Duration(n+5) * time.Millisecond + ctx, cancel := context.WithTimeout(context.TODO(), delay) + defer cancel() + select { + case <-ctx.Done(): + receiptsC <- chunk.Address() + case <-quit: + } + + }() + return nil + } + // initialise db, it starts all the go routines + dbpath, err := ioutil.TempDir(os.TempDir(), "syncertest") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(dbpath) + db, err := NewDB(dbpath, nil, sync, receiptsC, nil) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + // feed fake chunks into the db, hashes encode the order so that + // it can be traced + for i, name := range names { + total := i*100 + 100 + db.tags.New(name, total) + go func(name string, total int) { + for j := 0; j < total; j++ { + db.Put(&item{Addr: network.RandomAddr().OAddr, Tag: name}) + } + }(name, total) + } + + err = waitTillEmpty(db) + if err != nil { + t.Fatal(err) + } + + states := []State{STORED, SENT, SYNCED} + var cnt int + for i, name := range names { + total := i*100 + 100 + for _, state := range states { + cnt = db.tags.Get(name, state) + if cnt != total { + t.Fatalf("expected tag %v state %v to count %v, got %v", name, state, total, cnt) + } + } + } +} diff --git a/swarm/storage/localstore/localstore_test.go b/swarm/storage/localstore/localstore_test.go index 8a231a0629..25607c7970 100644 --- a/swarm/storage/localstore/localstore_test.go +++ b/swarm/storage/localstore/localstore_test.go @@ -191,6 +191,17 @@ func generateTestRandomChunk() chunk.Chunk { return chunk.NewChunk(key, data, tags) } +// generateTestRandomChunkWithTags does the same at the above but allows a custom tag +func generateTestRandomChunkWithTags(tags []uint64) chunk.Chunk { + data := make([]byte, chunk.DefaultSize) + rand.Read(data) + key := make([]byte, 32) + rand.Read(key) + r := rand.New(rand.NewSource(time.Now().UnixNano())) + n := r.Intn(10) + return chunk.NewChunk(key, data, tags) +} + // TestGenerateTestRandomChunk validates that // generateTestRandomChunk returns random data by comparing // two generated chunks. diff --git a/swarm/storage/localstore/tags.go b/swarm/storage/localstore/tags.go index 8d5498d0f3..09080e7a24 100644 --- a/swarm/storage/localstore/tags.go +++ b/swarm/storage/localstore/tags.go @@ -18,6 +18,8 @@ package localstore import ( "encoding/binary" + "math/rand" + "time" "github.com/ethereum/go-ethereum/swarm/chunk" "github.com/syndtr/goleveldb/leveldb" @@ -25,10 +27,13 @@ import ( var _ chunk.TagStore = &DB{} -func (db *DB) PutUploadID(id uint64, uploadTime int64, uploadName string) (err error) { +func (db *DB) NewTag(uploadTime int64, uploadName string) (tag uint64, err error) { // protect parallel updates db.batchMu.Lock() defer db.batchMu.Unlock() + r := rand.New(rand.NewSource(time.Now().Unix())) + + tag := r.Uint64() batch := new(leveldb.Batch) val := make([]byte, 8) @@ -36,7 +41,7 @@ func (db *DB) PutUploadID(id uint64, uploadTime int64, uploadName string) (err e val = append(val, []byte(uploadName)) // put to indexes: tag - db.uploadIndex.PutInBatch(batch, interface{ id }, interface{ val }) + db.uploadIndex.PutInBatch(batch, interface{ tag }, interface{ val }) err = db.shed.WriteBatch(batch) if err != nil { return err @@ -45,7 +50,19 @@ func (db *DB) PutUploadID(id uint64, uploadTime int64, uploadName string) (err e } -func (db *DB) GetChunkTags(addr chunk.Address) ([]uint64, error) { +func (db *DB) DeleteTag(tag uint64) error { + +} + +func (db *DB) GetTags() ([]chunk.Tag, error) { + +} + +func (db *DB) GetTag(uint64 tag) (chunk.Tag, error) { + +} + +func (db *DB) ChunkTags(addr chunk.Address) ([]uint64, error) { item := addressToItem(addr) out, err := db.retrievalDataIndex.Get(item) @@ -66,19 +83,3 @@ func (db *DB) GetChunkTags(addr chunk.Address) ([]uint64, error) { return c.Tags, nil } - -func (db *DB) PutTag(uploadId, tag uint64, path string) (err error) { - // protect parallel updates - db.batchMu.Lock() - defer db.batchMu.Unlock() - - batch := new(leveldb.Batch) - var k, v interface{} - // put to indexes: tag - db.tagIndex.PutInBatch(batch, k, v) - err = db.shed.WriteBatch(batch) - if err != nil { - return err - } - return nil -} diff --git a/swarm/storage/localstore/tags_test.go b/swarm/storage/localstore/tags_test.go index aee3a4b0b2..c224ff479e 100644 --- a/swarm/storage/localstore/tags_test.go +++ b/swarm/storage/localstore/tags_test.go @@ -16,6 +16,46 @@ package localstore +import ( + "testing" + "time" +) + +// tests that new tag is created, iterated over (one or all) and deleted in the database +func TestTags(t *testing.T) { + db, cleanupFunc := newTestDB(t, nil) + defer cleanupFunc() + + tag := db.NewTag(time.Now().Unix(), "path/to/directory") + + /* c := generateTestRandomChunkWithTags([]uint64{tag}) + + err := db.Put(context.Background(), chunk.ModePutUpload, c) + if err != nil { + t.Fatal(err) + } + */ + existingTags = db.GetTags() + + //expect tag to be in existingTags + + oneTag = db.GetTag(tag) + + // expect to exist + + //delete tag + err = db.DeleteTag(tag) + if err != nil { + t.Fatal(err) + } + + tagShouldNotExist, err := db.GetTag(tag) + if err == nil { + t.Fatal("tag should not exist") + } + +} + /*func TestPutTag(t *testing.T) { db, cleanupFunc := newTestDB(t, nil) defer cleanupFunc() From 98e52174d8236d67f612158ec642af86fbd0ab49 Mon Sep 17 00:00:00 2001 From: Elad Date: Fri, 5 Apr 2019 12:30:14 +0900 Subject: [PATCH 25/31] swarm/chunk: comment db test from tags --- swarm/chunk/tags.go | 12 ++++++------ swarm/chunk/tags_test.go | 15 ++++----------- 2 files changed, 10 insertions(+), 17 deletions(-) diff --git a/swarm/chunk/tags.go b/swarm/chunk/tags.go index 6780418291..663e794117 100644 --- a/swarm/chunk/tags.go +++ b/swarm/chunk/tags.go @@ -38,20 +38,20 @@ type Tag struct { } // tags holds the tag infos indexed by name -type tags struct { +type Tags struct { tags *sync.Map } // NewTags creates a tags object -func newTags() *tags { - return &tags{ +func NewTags() *Tags { + return &Tags{ &sync.Map{}, } } // New creates a new tag, stores it by the name and returns it // it returns an error if the tag with this name already exists -func (ts *tags) New(s string, total int) (*Tag, error) { +func (ts *Tags) New(s string, total int) (*Tag, error) { t := &Tag{ name: s, startedAt: time.Now(), @@ -131,7 +131,7 @@ func (t *Tag) ETA(state State) (time.Time, error) { } // Inc increments the state count for a tag if tag is found -func (ts *tags) Inc(s string, f State) { +func (ts *Tags) Inc(s string, f State) { t, ok := ts.tags.Load(s) if !ok { return @@ -140,7 +140,7 @@ func (ts *tags) Inc(s string, f State) { } // Get returns the state count for a tag -func (ts *tags) Get(s string, f State) int { +func (ts *Tags) Get(s string, f State) int { t, _ := ts.tags.Load(s) return t.(*Tag).Get(f) } diff --git a/swarm/chunk/tags_test.go b/swarm/chunk/tags_test.go index db6747392f..35897e370d 100644 --- a/swarm/chunk/tags_test.go +++ b/swarm/chunk/tags_test.go @@ -1,16 +1,9 @@ package chunk import ( - "context" - "io/ioutil" - "math/rand" - "os" "sync" "testing" "time" - - "github.com/ethereum/go-ethereum/swarm/chunk" - "github.com/ethereum/go-ethereum/swarm/network" ) var ( @@ -79,7 +72,7 @@ func TestTagConcurrentIncrements(t *testing.T) { // TestTagsMultipleConcurrentIncrements tests Inc calls concurrently func TestTagsMultipleConcurrentIncrements(t *testing.T) { - ts := newTags() + ts := NewTags() n := 100 wg := sync.WaitGroup{} wg.Add(10 * 4 * n) @@ -110,7 +103,7 @@ func TestTagsMultipleConcurrentIncrements(t *testing.T) { } // tests the correct behaviour of tags while using the DB -func TestDBWithTags(t *testing.T) { +/*func TestDBWithTags(t *testing.T) { names := []string{"1", "2", "3", "4"} receiptsC := make(chan chunk.Address) quit := make(chan struct{}) @@ -141,7 +134,7 @@ func TestDBWithTags(t *testing.T) { t.Fatal(err) } defer os.RemoveAll(dbpath) - db, err := NewDB(dbpath, nil, sync, receiptsC, nil) + db, err := localstore.New(dbpath, nil, sync, receiptsC, nil) if err != nil { t.Fatal(err) } @@ -175,4 +168,4 @@ func TestDBWithTags(t *testing.T) { } } } -} +}*/ From b6cad27ef812fcaba6e34d574803c02b9881a40f Mon Sep 17 00:00:00 2001 From: Elad Date: Fri, 5 Apr 2019 12:43:12 +0900 Subject: [PATCH 26/31] swarm/shed: add generic index iterator support wip --- swarm/shed/generic_index_test.go | 372 +++++++++++++++++++++++++ swarm/storage/localstore/localstore.go | 7 +- 2 files changed, 374 insertions(+), 5 deletions(-) diff --git a/swarm/shed/generic_index_test.go b/swarm/shed/generic_index_test.go index ee2d715e4d..8ac1847d0f 100644 --- a/swarm/shed/generic_index_test.go +++ b/swarm/shed/generic_index_test.go @@ -17,7 +17,10 @@ package shed import ( + "bytes" "errors" + "fmt" + "sort" "testing" "github.com/syndtr/goleveldb/leveldb" @@ -224,6 +227,375 @@ func TestGenericIndex(t *testing.T) { }) } +// TestGenericIndex_Iterate validates index Iterate +// functions for correctness. +func TestGenericIndex_Iterate(t *testing.T) { + db, cleanupFunc := newTestDB(t) + defer cleanupFunc() + + index, err := db.NewGenericIndex("retrieval", retrievalGenericIndexFuncs) + if err != nil { + t.Fatal(err) + } + + items := []struct { + k string + v string + }{ + { + k: "iterate-hash-01", + v: "data80", + }, + { + k: "iterate-hash-03", + v: "data22", + }, + { + k: "iterate-hash-05", + v: "data41", + }, + { + k: "iterate-hash-02", + v: "data84", + }, + { + k: "iterate-hash-06", + v: "data1", + }, + } + batch := new(leveldb.Batch) + for _, i := range items { + index.PutInBatch(batch, i.k, i.v) + } + err = db.WriteBatch(batch) + if err != nil { + t.Fatal(err) + } + k := "iterate-hash-04" + v := "data0" + err = index.Put(k, v) + if err != nil { + t.Fatal(err) + } + items = append(items, struct { + k string + v string + }{k: k, + v: v, + }) + + sort.SliceStable(items, func(i, j int) bool { + return bytes.Compare(items[i].k, items[j].k) < 0 + }) + ////////// + t.Run("all", func(t *testing.T) { + var i int + err := index.Iterate(func(item Item) (stop bool, err error) { + if i > len(items)-1 { + return true, fmt.Errorf("got unexpected index item: %#v", item) + } + want := items[i] + checkItem(t, item, want) + i++ + return false, nil + }, nil) + if err != nil { + t.Fatal(err) + } + }) + + t.Run("start from", func(t *testing.T) { + startIndex := 2 + i := startIndex + err := index.Iterate(func(item Item) (stop bool, err error) { + if i > len(items)-1 { + return true, fmt.Errorf("got unexpected index item: %#v", item) + } + want := items[i] + checkItem(t, item, want) + i++ + return false, nil + }, &IterateOptions{ + StartFrom: &items[startIndex], + }) + if err != nil { + t.Fatal(err) + } + }) + + t.Run("skip start from", func(t *testing.T) { + startIndex := 2 + i := startIndex + 1 + err := index.Iterate(func(item Item) (stop bool, err error) { + if i > len(items)-1 { + return true, fmt.Errorf("got unexpected index item: %#v", item) + } + want := items[i] + checkItem(t, item, want) + i++ + return false, nil + }, &IterateOptions{ + StartFrom: &items[startIndex], + SkipStartFromItem: true, + }) + if err != nil { + t.Fatal(err) + } + }) + + t.Run("stop", func(t *testing.T) { + var i int + stopIndex := 3 + var count int + err := index.Iterate(func(item Item) (stop bool, err error) { + if i > len(items)-1 { + return true, fmt.Errorf("got unexpected index item: %#v", item) + } + want := items[i] + checkItem(t, item, want) + count++ + if i == stopIndex { + return true, nil + } + i++ + return false, nil + }, nil) + if err != nil { + t.Fatal(err) + } + wantItemsCount := stopIndex + 1 + if count != wantItemsCount { + t.Errorf("got %v items, expected %v", count, wantItemsCount) + } + }) + + t.Run("no overflow", func(t *testing.T) { + secondIndex, err := db.NewIndex("second-index", retrievalIndexFuncs) + if err != nil { + t.Fatal(err) + } + + secondItem := Item{ + Address: []byte("iterate-hash-10"), + Data: []byte("data-second"), + } + err = secondIndex.Put(secondItem) + if err != nil { + t.Fatal(err) + } + + var i int + err = index.Iterate(func(item Item) (stop bool, err error) { + if i > len(items)-1 { + return true, fmt.Errorf("got unexpected index item: %#v", item) + } + want := items[i] + checkItem(t, item, want) + i++ + return false, nil + }, nil) + if err != nil { + t.Fatal(err) + } + + i = 0 + err = secondIndex.Iterate(func(item Item) (stop bool, err error) { + if i > 1 { + return true, fmt.Errorf("got unexpected index item: %#v", item) + } + checkItem(t, item, secondItem) + i++ + return false, nil + }, nil) + if err != nil { + t.Fatal(err) + } + }) +} + +// TestIndex_Iterate_withPrefix validates index Iterate +// function for correctness. +func TestGenericIndex_Iterate_withPrefix(t *testing.T) { + db, cleanupFunc := newTestDB(t) + defer cleanupFunc() + + index, err := db.NewIndex("retrieval", retrievalIndexFuncs) + if err != nil { + t.Fatal(err) + } + + allItems := []Item{ + {Address: []byte("want-hash-00"), Data: []byte("data80")}, + {Address: []byte("skip-hash-01"), Data: []byte("data81")}, + {Address: []byte("skip-hash-02"), Data: []byte("data82")}, + {Address: []byte("skip-hash-03"), Data: []byte("data83")}, + {Address: []byte("want-hash-04"), Data: []byte("data84")}, + {Address: []byte("want-hash-05"), Data: []byte("data85")}, + {Address: []byte("want-hash-06"), Data: []byte("data86")}, + {Address: []byte("want-hash-07"), Data: []byte("data87")}, + {Address: []byte("want-hash-08"), Data: []byte("data88")}, + {Address: []byte("want-hash-09"), Data: []byte("data89")}, + {Address: []byte("skip-hash-10"), Data: []byte("data90")}, + } + batch := new(leveldb.Batch) + for _, i := range allItems { + index.PutInBatch(batch, i) + } + err = db.WriteBatch(batch) + if err != nil { + t.Fatal(err) + } + + prefix := []byte("want") + + items := make([]Item, 0) + for _, item := range allItems { + if bytes.HasPrefix(item.Address, prefix) { + items = append(items, item) + } + } + sort.SliceStable(items, func(i, j int) bool { + return bytes.Compare(items[i].Address, items[j].Address) < 0 + }) + + t.Run("with prefix", func(t *testing.T) { + var i int + err := index.Iterate(func(item Item) (stop bool, err error) { + if i > len(items)-1 { + return true, fmt.Errorf("got unexpected index item: %#v", item) + } + want := items[i] + checkItem(t, item, want) + i++ + return false, nil + }, &IterateOptions{ + Prefix: prefix, + }) + if err != nil { + t.Fatal(err) + } + if i != len(items) { + t.Errorf("got %v items, want %v", i, len(items)) + } + }) + + t.Run("with prefix and start from", func(t *testing.T) { + startIndex := 2 + var count int + i := startIndex + err := index.Iterate(func(item Item) (stop bool, err error) { + if i > len(items)-1 { + return true, fmt.Errorf("got unexpected index item: %#v", item) + } + want := items[i] + checkItem(t, item, want) + i++ + count++ + return false, nil + }, &IterateOptions{ + StartFrom: &items[startIndex], + Prefix: prefix, + }) + if err != nil { + t.Fatal(err) + } + wantCount := len(items) - startIndex + if count != wantCount { + t.Errorf("got %v items, want %v", count, wantCount) + } + }) + + t.Run("with prefix and skip start from", func(t *testing.T) { + startIndex := 2 + var count int + i := startIndex + 1 + err := index.Iterate(func(item Item) (stop bool, err error) { + if i > len(items)-1 { + return true, fmt.Errorf("got unexpected index item: %#v", item) + } + want := items[i] + checkItem(t, item, want) + i++ + count++ + return false, nil + }, &IterateOptions{ + StartFrom: &items[startIndex], + SkipStartFromItem: true, + Prefix: prefix, + }) + if err != nil { + t.Fatal(err) + } + wantCount := len(items) - startIndex - 1 + if count != wantCount { + t.Errorf("got %v items, want %v", count, wantCount) + } + }) + + t.Run("stop", func(t *testing.T) { + var i int + stopIndex := 3 + var count int + err := index.Iterate(func(item Item) (stop bool, err error) { + if i > len(items)-1 { + return true, fmt.Errorf("got unexpected index item: %#v", item) + } + want := items[i] + checkItem(t, item, want) + count++ + if i == stopIndex { + return true, nil + } + i++ + return false, nil + }, &IterateOptions{ + Prefix: prefix, + }) + if err != nil { + t.Fatal(err) + } + wantItemsCount := stopIndex + 1 + if count != wantItemsCount { + t.Errorf("got %v items, expected %v", count, wantItemsCount) + } + }) + + t.Run("no overflow", func(t *testing.T) { + secondIndex, err := db.NewIndex("second-index", retrievalIndexFuncs) + if err != nil { + t.Fatal(err) + } + + secondItem := Item{ + Address: []byte("iterate-hash-10"), + Data: []byte("data-second"), + } + err = secondIndex.Put(secondItem) + if err != nil { + t.Fatal(err) + } + + var i int + err = index.Iterate(func(item Item) (stop bool, err error) { + if i > len(items)-1 { + return true, fmt.Errorf("got unexpected index item: %#v", item) + } + want := items[i] + checkItem(t, item, want) + i++ + return false, nil + }, &IterateOptions{ + Prefix: prefix, + }) + if err != nil { + t.Fatal(err) + } + if i != len(items) { + t.Errorf("got %v items, want %v", i, len(items)) + } + }) +} + // checkItemString is a test helper function that compares if two generic items are the same string. func checkItemString(t *testing.T, got, want interface{}) { t.Helper() diff --git a/swarm/storage/localstore/localstore.go b/swarm/storage/localstore/localstore.go index 3de3a3721c..63533a75c4 100644 --- a/swarm/storage/localstore/localstore.go +++ b/swarm/storage/localstore/localstore.go @@ -76,10 +76,7 @@ type DB struct { // proximity order bin binIDs shed.Uint64Vector - // uploads index maintains a list of pending uploads - uploadIndex shed.GenericIndex - - // tags index maintains a mapping between tags and pending uploads + // tags index maintains a mapping between tags and upload time, chunk status and tag name tagIndex shed.GenericIndex // garbage collection index @@ -325,7 +322,7 @@ func New(path string, baseKey []byte, o *Options) (db *DB, err error) { // create a push syncing triggers used by SubscribePush function db.pushTriggers = make([]chan struct{}, 0) - db.uploadIndex, err = db.shed.NewGenericIndex("UploadID->UploadTime|UploadName", shed.GenericIndexFuncs{ + db.tagIndex, err = db.shed.NewGenericIndex("Tag->UploadTime|UploadName", shed.GenericIndexFuncs{ EncodeKey: func(fields interface{}) (key []byte, err error) { return nil, nil }, From 9c90a6064daefe454ec1b8c9854c4e19bc10894c Mon Sep 17 00:00:00 2001 From: Elad Date: Fri, 5 Apr 2019 15:26:03 +0900 Subject: [PATCH 27/31] swarm/shed: add iterator functionality to generic shed index from shed item index --- swarm/chunk/chunk.go | 7 -- swarm/chunk/tags.go | 5 + swarm/shed/generic_index.go | 104 +++++++++++++++- swarm/shed/generic_index_test.go | 168 +++++++++++++------------- swarm/storage/filestore.go | 26 ++-- swarm/storage/hasherstore.go | 4 +- swarm/storage/localstore/tags.go | 18 +-- swarm/storage/localstore/tags_test.go | 1 + 8 files changed, 216 insertions(+), 117 deletions(-) diff --git a/swarm/chunk/chunk.go b/swarm/chunk/chunk.go index 5626bd44a5..8ecffcfc8c 100644 --- a/swarm/chunk/chunk.go +++ b/swarm/chunk/chunk.go @@ -217,10 +217,3 @@ func (s *ValidatorStore) Put(ctx context.Context, mode ModePut, ch Chunk) (err e } return ErrChunkInvalid } - -type TagStore interface { - PutUploadID(uploadId uint64, timestamp int64, uploadName string) error - - GetChunkTags(addr Address) ([]uint64, error) - PutTag(uploadId, tag uint64, path string) error -} diff --git a/swarm/chunk/tags.go b/swarm/chunk/tags.go index 663e794117..05bbb5a6c3 100644 --- a/swarm/chunk/tags.go +++ b/swarm/chunk/tags.go @@ -15,6 +15,11 @@ var ( errNoETA = errors.New("unable to calculate ETA") ) +type TagStore interface { + ChunkTags(addr Address) ([]uint64, error) + NewTag(uploadTime int64, path string) (tag uint64, err error) +} + // State is the enum type for chunk states type State = uint32 diff --git a/swarm/shed/generic_index.go b/swarm/shed/generic_index.go index 80bd3f4ed3..245d2f8713 100644 --- a/swarm/shed/generic_index.go +++ b/swarm/shed/generic_index.go @@ -17,7 +17,11 @@ package shed import ( + "bytes" + "fmt" + "github.com/syndtr/goleveldb/leveldb" + "github.com/syndtr/goleveldb/leveldb/iterator" ) // Index represents a set of LevelDB key value pairs that have common @@ -29,10 +33,10 @@ import ( type GenericIndex struct { db *DB prefix []byte - encodeKeyFunc func(item interface{}) (key []byte, err error) - decodeKeyFunc func(key []byte) (item interface{}, err error) - encodeValueFunc func(fields interface{}) (value []byte, err error) - decodeValueFunc func(keyFields interface{}, value []byte) (e interface{}, err error) + encodeKeyFunc func(k interface{}) (key []byte, err error) + decodeKeyFunc func(key []byte) (k interface{}, err error) + encodeValueFunc func(v interface{}) (value []byte, err error) + decodeValueFunc func(v interface{}, value []byte) (e interface{}, err error) } // GenericIndexFuncs structure defines functions for encoding and decoding @@ -158,3 +162,95 @@ func (f GenericIndex) DeleteInBatch(batch *leveldb.Batch, keyFields interface{}) batch.Delete(key) return nil } + +// GenericIndexIterFunc is a callback on every Item that is decoded +// by iterating on an Index keys. +// By returning a true for stop variable, iteration will +// stop, and by returning the error, that error will be +// propagated to the called iterator method on Index. +type GenericIndexIterFunc func(k, v interface{}) (stop bool, err error) + +// IterateOptions defines optional parameters for Iterate function. +type GenericIterateOptions struct { + // StartFrom is the Item to start the iteration from. + StartFrom interface{} + // If SkipStartFromItem is true, StartFrom item will not + // be iterated on. + SkipStartFromItem bool + // Iterate over items which keys have a common prefix. + Prefix []byte +} + +// Iterate function iterates over keys of the Index. +// If IterateOptions is nil, the iterations is over all keys. +func (f GenericIndex) Iterate(fn GenericIndexIterFunc, options *GenericIterateOptions) (err error) { + if options == nil { + options = new(GenericIterateOptions) + } + // construct a prefix with Index prefix and optional common key prefix + prefix := append(f.prefix, options.Prefix...) + // start from the prefix + startKey := prefix + if options.StartFrom != nil { + // start from the provided StartFrom Item key value + startKey, err = f.encodeKeyFunc(options.StartFrom) + if err != nil { + return err + } + } + it := f.db.NewIterator() + defer it.Release() + + // move the cursor to the start key + ok := it.Seek(startKey) + if !ok { + // stop iterator if seek has failed + return it.Error() + } + if options.SkipStartFromItem && bytes.Equal(startKey, it.Key()) { + // skip the start from Item if it is the first key + // and it is explicitly configured to skip it + ok = it.Next() + } + for ; ok; ok = it.Next() { + itemK, itemV, err := f.tupleFromIterator(it, prefix) + if err != nil { + if err == leveldb.ErrNotFound { + break + } + return err + } + stop, err := fn(itemK, itemV) + if err != nil { + return err + } + if stop { + break + } + } + return it.Error() +} + +// tupleFromIterator returns the key value tuple from the current iterator position. +// If the complete encoded key does not start with totalPrefix, +// leveldb.ErrNotFound is returned. Value for totalPrefix must start with +// Index prefix. +func (f GenericIndex) tupleFromIterator(it iterator.Iterator, totalPrefix []byte) (k, v interface{}, err error) { + fmt.Println("tupleFrom") + + key := it.Key() + if !bytes.HasPrefix(key, totalPrefix) { + return nil, nil, leveldb.ErrNotFound + } + // create a copy of key byte slice not to share leveldb underlaying slice array + keyItem, err := f.decodeKeyFunc(append([]byte(nil), key...)) + if err != nil { + return nil, nil, err + } + // create a copy of value byte slice not to share leveldb underlaying slice array + valueItem, err := f.decodeValueFunc(keyItem, append([]byte(nil), it.Value()...)) + if err != nil { + return nil, nil, err + } + return keyItem, valueItem, it.Error() +} diff --git a/swarm/shed/generic_index_test.go b/swarm/shed/generic_index_test.go index 8ac1847d0f..c933693d1a 100644 --- a/swarm/shed/generic_index_test.go +++ b/swarm/shed/generic_index_test.go @@ -285,17 +285,16 @@ func TestGenericIndex_Iterate(t *testing.T) { }) sort.SliceStable(items, func(i, j int) bool { - return bytes.Compare(items[i].k, items[j].k) < 0 + return bytes.Compare([]byte(items[i].k), []byte(items[j].k)) < 0 }) - ////////// t.Run("all", func(t *testing.T) { var i int - err := index.Iterate(func(item Item) (stop bool, err error) { + err := index.Iterate(func(k, v interface{}) (stop bool, err error) { if i > len(items)-1 { - return true, fmt.Errorf("got unexpected index item: %#v", item) + return true, fmt.Errorf("got unexpected index item: %#v", v) } - want := items[i] - checkItem(t, item, want) + want := items[i].v + checkItemString(t, v, want) i++ return false, nil }, nil) @@ -307,16 +306,16 @@ func TestGenericIndex_Iterate(t *testing.T) { t.Run("start from", func(t *testing.T) { startIndex := 2 i := startIndex - err := index.Iterate(func(item Item) (stop bool, err error) { + err := index.Iterate(func(k, v interface{}) (stop bool, err error) { if i > len(items)-1 { - return true, fmt.Errorf("got unexpected index item: %#v", item) + return true, fmt.Errorf("got unexpected index item: %#v", v) } - want := items[i] - checkItem(t, item, want) + want := items[i].v + checkItemString(t, v, want) i++ return false, nil - }, &IterateOptions{ - StartFrom: &items[startIndex], + }, &GenericIterateOptions{ + StartFrom: items[startIndex].k, }) if err != nil { t.Fatal(err) @@ -326,16 +325,16 @@ func TestGenericIndex_Iterate(t *testing.T) { t.Run("skip start from", func(t *testing.T) { startIndex := 2 i := startIndex + 1 - err := index.Iterate(func(item Item) (stop bool, err error) { + err := index.Iterate(func(k, v interface{}) (stop bool, err error) { if i > len(items)-1 { - return true, fmt.Errorf("got unexpected index item: %#v", item) + return true, fmt.Errorf("got unexpected index item: %#v", v) } - want := items[i] - checkItem(t, item, want) + want := items[i].v + checkItemString(t, v, want) i++ return false, nil - }, &IterateOptions{ - StartFrom: &items[startIndex], + }, &GenericIterateOptions{ + StartFrom: items[startIndex].k, SkipStartFromItem: true, }) if err != nil { @@ -347,12 +346,12 @@ func TestGenericIndex_Iterate(t *testing.T) { var i int stopIndex := 3 var count int - err := index.Iterate(func(item Item) (stop bool, err error) { + err := index.Iterate(func(k, v interface{}) (stop bool, err error) { if i > len(items)-1 { - return true, fmt.Errorf("got unexpected index item: %#v", item) + return true, fmt.Errorf("got unexpected index item: %#v", v) } - want := items[i] - checkItem(t, item, want) + want := items[i].v + checkItemString(t, v, want) count++ if i == stopIndex { return true, nil @@ -370,27 +369,30 @@ func TestGenericIndex_Iterate(t *testing.T) { }) t.Run("no overflow", func(t *testing.T) { - secondIndex, err := db.NewIndex("second-index", retrievalIndexFuncs) + secondIndex, err := db.NewGenericIndex("second-index", retrievalGenericIndexFuncs) if err != nil { t.Fatal(err) } - secondItem := Item{ - Address: []byte("iterate-hash-10"), - Data: []byte("data-second"), + secondItem := struct { + k string + v string + }{ + k: "iterate-hash-10", + v: "data-second", } - err = secondIndex.Put(secondItem) + err = secondIndex.Put(secondItem.k, secondItem.v) if err != nil { t.Fatal(err) } var i int - err = index.Iterate(func(item Item) (stop bool, err error) { + err = index.Iterate(func(k, v interface{}) (stop bool, err error) { if i > len(items)-1 { - return true, fmt.Errorf("got unexpected index item: %#v", item) + return true, fmt.Errorf("got unexpected index item: %#v", v) } - want := items[i] - checkItem(t, item, want) + want := items[i].v + checkItemString(t, v, want) i++ return false, nil }, nil) @@ -399,11 +401,11 @@ func TestGenericIndex_Iterate(t *testing.T) { } i = 0 - err = secondIndex.Iterate(func(item Item) (stop bool, err error) { + err = secondIndex.Iterate(func(k, v interface{}) (stop bool, err error) { if i > 1 { - return true, fmt.Errorf("got unexpected index item: %#v", item) + return true, fmt.Errorf("got unexpected index item: %#v", v) } - checkItem(t, item, secondItem) + checkItemString(t, v, secondItem.v) i++ return false, nil }, nil) @@ -419,27 +421,27 @@ func TestGenericIndex_Iterate_withPrefix(t *testing.T) { db, cleanupFunc := newTestDB(t) defer cleanupFunc() - index, err := db.NewIndex("retrieval", retrievalIndexFuncs) + index, err := db.NewGenericIndex("retrieval", retrievalGenericIndexFuncs) if err != nil { t.Fatal(err) } - allItems := []Item{ - {Address: []byte("want-hash-00"), Data: []byte("data80")}, - {Address: []byte("skip-hash-01"), Data: []byte("data81")}, - {Address: []byte("skip-hash-02"), Data: []byte("data82")}, - {Address: []byte("skip-hash-03"), Data: []byte("data83")}, - {Address: []byte("want-hash-04"), Data: []byte("data84")}, - {Address: []byte("want-hash-05"), Data: []byte("data85")}, - {Address: []byte("want-hash-06"), Data: []byte("data86")}, - {Address: []byte("want-hash-07"), Data: []byte("data87")}, - {Address: []byte("want-hash-08"), Data: []byte("data88")}, - {Address: []byte("want-hash-09"), Data: []byte("data89")}, - {Address: []byte("skip-hash-10"), Data: []byte("data90")}, + allItems := []struct{ k, v string }{ + {k: "want-hash-00", v: "data80"}, + {k: "skip-hash-01", v: "data81"}, + {k: "skip-hash-02", v: "data82"}, + {k: "skip-hash-03", v: "data83"}, + {k: "want-hash-04", v: "data84"}, + {k: "want-hash-05", v: "data85"}, + {k: "want-hash-06", v: "data86"}, + {k: "want-hash-07", v: "data87"}, + {k: "want-hash-08", v: "data88"}, + {k: "want-hash-09", v: "data89"}, + {k: "skip-hash-10", v: "data90"}, } batch := new(leveldb.Batch) for _, i := range allItems { - index.PutInBatch(batch, i) + index.PutInBatch(batch, i.k, i.v) } err = db.WriteBatch(batch) if err != nil { @@ -448,27 +450,27 @@ func TestGenericIndex_Iterate_withPrefix(t *testing.T) { prefix := []byte("want") - items := make([]Item, 0) + items := make([]struct{ k, v string }, 0) for _, item := range allItems { - if bytes.HasPrefix(item.Address, prefix) { + if bytes.HasPrefix([]byte(item.k), prefix) { items = append(items, item) } } sort.SliceStable(items, func(i, j int) bool { - return bytes.Compare(items[i].Address, items[j].Address) < 0 + return bytes.Compare([]byte(items[i].k), []byte(items[j].k)) < 0 }) t.Run("with prefix", func(t *testing.T) { var i int - err := index.Iterate(func(item Item) (stop bool, err error) { + err := index.Iterate(func(k, v interface{}) (stop bool, err error) { if i > len(items)-1 { - return true, fmt.Errorf("got unexpected index item: %#v", item) + return true, fmt.Errorf("got unexpected index item: %#v", v) } - want := items[i] - checkItem(t, item, want) + want := items[i].v + checkItemString(t, v, want) i++ return false, nil - }, &IterateOptions{ + }, &GenericIterateOptions{ Prefix: prefix, }) if err != nil { @@ -483,17 +485,17 @@ func TestGenericIndex_Iterate_withPrefix(t *testing.T) { startIndex := 2 var count int i := startIndex - err := index.Iterate(func(item Item) (stop bool, err error) { + err := index.Iterate(func(k, v interface{}) (stop bool, err error) { if i > len(items)-1 { - return true, fmt.Errorf("got unexpected index item: %#v", item) + return true, fmt.Errorf("got unexpected index item: %#v", v) } - want := items[i] - checkItem(t, item, want) + want := items[i].v + checkItemString(t, v, want) i++ count++ return false, nil - }, &IterateOptions{ - StartFrom: &items[startIndex], + }, &GenericIterateOptions{ + StartFrom: items[startIndex].k, Prefix: prefix, }) if err != nil { @@ -509,17 +511,17 @@ func TestGenericIndex_Iterate_withPrefix(t *testing.T) { startIndex := 2 var count int i := startIndex + 1 - err := index.Iterate(func(item Item) (stop bool, err error) { + err := index.Iterate(func(k, v interface{}) (stop bool, err error) { if i > len(items)-1 { - return true, fmt.Errorf("got unexpected index item: %#v", item) + return true, fmt.Errorf("got unexpected index item: %#v", v) } - want := items[i] - checkItem(t, item, want) + want := items[i].v + checkItemString(t, v, want) i++ count++ return false, nil - }, &IterateOptions{ - StartFrom: &items[startIndex], + }, &GenericIterateOptions{ + StartFrom: items[startIndex].k, SkipStartFromItem: true, Prefix: prefix, }) @@ -536,19 +538,19 @@ func TestGenericIndex_Iterate_withPrefix(t *testing.T) { var i int stopIndex := 3 var count int - err := index.Iterate(func(item Item) (stop bool, err error) { + err := index.Iterate(func(k, v interface{}) (stop bool, err error) { if i > len(items)-1 { - return true, fmt.Errorf("got unexpected index item: %#v", item) + return true, fmt.Errorf("got unexpected index item: %#v", v) } - want := items[i] - checkItem(t, item, want) + want := items[i].v + checkItemString(t, v, want) count++ if i == stopIndex { return true, nil } i++ return false, nil - }, &IterateOptions{ + }, &GenericIterateOptions{ Prefix: prefix, }) if err != nil { @@ -561,30 +563,30 @@ func TestGenericIndex_Iterate_withPrefix(t *testing.T) { }) t.Run("no overflow", func(t *testing.T) { - secondIndex, err := db.NewIndex("second-index", retrievalIndexFuncs) + secondIndex, err := db.NewGenericIndex("second-index", retrievalGenericIndexFuncs) if err != nil { t.Fatal(err) } - secondItem := Item{ - Address: []byte("iterate-hash-10"), - Data: []byte("data-second"), + secondItem := struct{ k, v string }{ + k: "iterate-hash-10", + v: "data-second", } - err = secondIndex.Put(secondItem) + err = secondIndex.Put(secondItem.k, secondItem.v) if err != nil { t.Fatal(err) } var i int - err = index.Iterate(func(item Item) (stop bool, err error) { + err = index.Iterate(func(k, v interface{}) (stop bool, err error) { if i > len(items)-1 { - return true, fmt.Errorf("got unexpected index item: %#v", item) + return true, fmt.Errorf("got unexpected index item: %#v", v) } - want := items[i] - checkItem(t, item, want) + want := items[i].v + checkItemString(t, v, want) i++ return false, nil - }, &IterateOptions{ + }, &GenericIterateOptions{ Prefix: prefix, }) if err != nil { diff --git a/swarm/storage/filestore.go b/swarm/storage/filestore.go index bc1d305db1..4d698076ea 100644 --- a/swarm/storage/filestore.go +++ b/swarm/storage/filestore.go @@ -18,12 +18,11 @@ package storage import ( "context" - "encoding/binary" "io" "sort" "sync" + "time" - "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/swarm/chunk" "github.com/ethereum/go-ethereum/swarm/storage/localstore" ) @@ -107,17 +106,18 @@ func (f *FileStore) HashSize() int { // it returns the tag as uint64 func (f *FileStore) CreateTag(ctx context.Context, filename string, timestamp uint64) (uint64, error) { // add uploadID - - var uploadId uint64 = 0 - intBuf := make([]byte, 8) - binary.BigEndian.PutUint64(intBuf, timestamp) - // Tag is SHA3(filename|storetimestamp)[:8] - buf := []byte(filename) - buf = append(buf, intBuf...) - tagHash := crypto.Keccak256(buf)[:8] - - tag := binary.BigEndian.Uint64(tagHash) - err := f.tagStore.PutTag(uploadId, tag, filename) + /* + var uploadId uint64 = 0 + intBuf := make([]byte, 8) + binary.BigEndian.PutUint64(intBuf, timestamp) + // Tag is SHA3(filename|storetimestamp)[:8] + buf := []byte(filename) + buf = append(buf, intBuf...) + tagHash := crypto.Keccak256(buf)[:8] + + tag := binary.BigEndian.Uint64(tagHash) + */ + tag, err := f.tagStore.NewTag(time.Now().Unix(), filename) if err != nil { return tag, err } diff --git a/swarm/storage/hasherstore.go b/swarm/storage/hasherstore.go index 0063a4fcf5..5c6e23bda1 100644 --- a/swarm/storage/hasherstore.go +++ b/swarm/storage/hasherstore.go @@ -115,7 +115,7 @@ func (h *hasherStore) Get(ctx context.Context, ref Reference) (ChunkData, error) } func (h *hasherStore) GetTags(ctx context.Context, addr chunk.Address) ([]uint64, error) { - tags, err := h.tagStore.GetChunkTags(addr) + tags, err := h.tagStore.ChunkTags(addr) if err != nil { return nil, err } @@ -170,7 +170,7 @@ func (h *hasherStore) createHash(chunkData ChunkData) Address { func (h *hasherStore) createChunk(ctx context.Context, chunkData ChunkData) Chunk { hash := h.createHash(chunkData) - tags, err := h.tagStore.GetChunkTags(chunk.Address(hash)) // TODO: this is really bad but if we want to persist tags across sessions this would be the way + tags, err := h.tagStore.ChunkTags(chunk.Address(hash)) // TODO: this is really bad but if we want to persist tags across sessions this would be the way if err != nil { panic(err) } diff --git a/swarm/storage/localstore/tags.go b/swarm/storage/localstore/tags.go index 09080e7a24..469ba93f21 100644 --- a/swarm/storage/localstore/tags.go +++ b/swarm/storage/localstore/tags.go @@ -33,33 +33,35 @@ func (db *DB) NewTag(uploadTime int64, uploadName string) (tag uint64, err error defer db.batchMu.Unlock() r := rand.New(rand.NewSource(time.Now().Unix())) - tag := r.Uint64() + tag = r.Uint64() batch := new(leveldb.Batch) val := make([]byte, 8) - binary.BigEndian.PutInt64(val, uploadTime) - val = append(val, []byte(uploadName)) + binary.BigEndian.PutUint64(val, uint64(uploadTime)) + val = append(val, []byte(uploadName)...) // put to indexes: tag - db.uploadIndex.PutInBatch(batch, interface{ tag }, interface{ val }) + db.tagIndex.PutInBatch(batch, tag, val) err = db.shed.WriteBatch(batch) if err != nil { - return err + return tag, err } - return nil + return tag, nil } func (db *DB) DeleteTag(tag uint64) error { - + return nil } func (db *DB) GetTags() ([]chunk.Tag, error) { + return nil, nil } -func (db *DB) GetTag(uint64 tag) (chunk.Tag, error) { +func (db *DB) GetTag(tag uint64) (chunk.Tag, error) { + return chunk.Tag{}, nil } func (db *DB) ChunkTags(addr chunk.Address) ([]uint64, error) { diff --git a/swarm/storage/localstore/tags_test.go b/swarm/storage/localstore/tags_test.go index c224ff479e..0a33fedc2c 100644 --- a/swarm/storage/localstore/tags_test.go +++ b/swarm/storage/localstore/tags_test.go @@ -35,6 +35,7 @@ func TestTags(t *testing.T) { t.Fatal(err) } */ + existingTags = db.GetTags() //expect tag to be in existingTags From d0498cd831ff679e0998eacde2e4147c79a70bce Mon Sep 17 00:00:00 2001 From: Elad Date: Sat, 6 Apr 2019 11:50:54 +0900 Subject: [PATCH 28/31] swarm/storage: move tags to tagstore, localstore tests pass --- swarm/api/api.go | 2 + swarm/chunk/tags.go | 4 + swarm/storage/hasherstore.go | 2 + swarm/storage/localstore/localstore.go | 37 -- swarm/storage/localstore/localstore_test.go | 2 - .../storage/{localstore => tagstore}/tags.go | 22 +- .../{localstore => tagstore}/tags_test.go | 31 +- swarm/storage/tagstore/tagstore.go | 209 +++++++++++ swarm/storage/tagstore/tagstore_test.go | 352 ++++++++++++++++++ 9 files changed, 609 insertions(+), 52 deletions(-) rename swarm/storage/{localstore => tagstore}/tags.go (82%) rename swarm/storage/{localstore => tagstore}/tags_test.go (81%) create mode 100644 swarm/storage/tagstore/tagstore.go create mode 100644 swarm/storage/tagstore/tagstore_test.go diff --git a/swarm/api/api.go b/swarm/api/api.go index bf5ecf6880..35a8629e4f 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -686,6 +686,8 @@ func (a *API) AddFile(ctx context.Context, mhash, path, fname string, content [] func (a *API) UploadTar(ctx context.Context, bodyReader io.ReadCloser, manifestPath, defaultPath string, mw *ManifestWriter) (storage.Address, error) { apiUploadTarCount.Inc(1) + + panic("create tag here and inject to context") var contentKey storage.Address tr := tar.NewReader(bodyReader) defer bodyReader.Close() diff --git a/swarm/chunk/tags.go b/swarm/chunk/tags.go index 05bbb5a6c3..49a550a9af 100644 --- a/swarm/chunk/tags.go +++ b/swarm/chunk/tags.go @@ -150,6 +150,10 @@ func (ts *Tags) Get(s string, f State) int { return t.(*Tag).Get(f) } +func (ts *Tags) Range(f func(key, value interface{}) bool) { + ts.tags.Range(f) +} + // WaitTill blocks until count for the State reaches total cnt func (tg *Tag) WaitTill(ctx context.Context, s State) error { ticker := time.NewTicker(1 * time.Second) diff --git a/swarm/storage/hasherstore.go b/swarm/storage/hasherstore.go index 5c6e23bda1..abcadb3a94 100644 --- a/swarm/storage/hasherstore.go +++ b/swarm/storage/hasherstore.go @@ -264,6 +264,8 @@ func (h *hasherStore) storeChunk(ctx context.Context, ch Chunk) { go func() { select { case h.errC <- h.store.Put(ctx, chunk.ModePutUpload, ch): + //if its a new chunk -> increment some counter and store in tag store + case <-h.quitC: } }() diff --git a/swarm/storage/localstore/localstore.go b/swarm/storage/localstore/localstore.go index 63533a75c4..0c02719ef4 100644 --- a/swarm/storage/localstore/localstore.go +++ b/swarm/storage/localstore/localstore.go @@ -322,43 +322,6 @@ func New(path string, baseKey []byte, o *Options) (db *DB, err error) { // create a push syncing triggers used by SubscribePush function db.pushTriggers = make([]chan struct{}, 0) - db.tagIndex, err = db.shed.NewGenericIndex("Tag->UploadTime|UploadName", shed.GenericIndexFuncs{ - EncodeKey: func(fields interface{}) (key []byte, err error) { - return nil, nil - }, - DecodeKey: func(key []byte) (e interface{}, err error) { - return nil, nil - }, - EncodeValue: func(fields interface{}) (value []byte, err error) { - return nil, nil - }, - DecodeValue: func(keyItem interface{}, value []byte) (e interface{}, err error) { - return nil, nil - }, - }) - if err != nil { - return nil, err - } - - // tag index for push syncing tags - db.tagIndex, err = db.shed.NewGenericIndex("UploadID|Tag->Filename", shed.GenericIndexFuncs{ - EncodeKey: func(fields interface{}) (key []byte, err error) { - return nil, nil - }, - DecodeKey: func(key []byte) (e interface{}, err error) { - return nil, nil - }, - EncodeValue: func(fields interface{}) (value []byte, err error) { - return nil, nil - }, - DecodeValue: func(keyItem interface{}, value []byte) (e interface{}, err error) { - return nil, nil - }, - }) - if err != nil { - return nil, err - } - // gc index for removable chunk ordered by ascending last access time db.gcIndex, err = db.shed.NewIndex("AccessTimestamp|BinID|Hash->nil", shed.IndexFuncs{ EncodeKey: func(fields shed.Item) (key []byte, err error) { diff --git a/swarm/storage/localstore/localstore_test.go b/swarm/storage/localstore/localstore_test.go index 25607c7970..1df915b4e7 100644 --- a/swarm/storage/localstore/localstore_test.go +++ b/swarm/storage/localstore/localstore_test.go @@ -197,8 +197,6 @@ func generateTestRandomChunkWithTags(tags []uint64) chunk.Chunk { rand.Read(data) key := make([]byte, 32) rand.Read(key) - r := rand.New(rand.NewSource(time.Now().UnixNano())) - n := r.Intn(10) return chunk.NewChunk(key, data, tags) } diff --git a/swarm/storage/localstore/tags.go b/swarm/storage/tagstore/tags.go similarity index 82% rename from swarm/storage/localstore/tags.go rename to swarm/storage/tagstore/tags.go index 469ba93f21..6fca5f2410 100644 --- a/swarm/storage/localstore/tags.go +++ b/swarm/storage/tagstore/tags.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -package localstore +package tagstore import ( "encoding/binary" @@ -54,9 +54,20 @@ func (db *DB) DeleteTag(tag uint64) error { return nil } -func (db *DB) GetTags() ([]chunk.Tag, error) { +func (db *DB) GetTags() (*chunk.Tags, error) { + //tags := make([]chunk.Tag, 0) + t := chunk.NewTags() + err := db.tagIndex.Iterate(func(k, v interface{}) (bool, error) { + _ = k.(uint64) - return nil, nil + _, err := t.New("tag", 0) + if err != nil { + return true, err + } + //todo append + return false, nil + }, nil) + return t, err } func (db *DB) GetTag(tag uint64) (chunk.Tag, error) { @@ -65,7 +76,7 @@ func (db *DB) GetTag(tag uint64) (chunk.Tag, error) { } func (db *DB) ChunkTags(addr chunk.Address) ([]uint64, error) { - item := addressToItem(addr) + /*item := addressToItem(addr) out, err := db.retrievalDataIndex.Get(item) if err != nil { @@ -83,5 +94,6 @@ func (db *DB) ChunkTags(addr chunk.Address) ([]uint64, error) { return nil, err } - return c.Tags, nil + return c.Tags, nil*/ + return []uint64{}, nil } diff --git a/swarm/storage/localstore/tags_test.go b/swarm/storage/tagstore/tags_test.go similarity index 81% rename from swarm/storage/localstore/tags_test.go rename to swarm/storage/tagstore/tags_test.go index 0a33fedc2c..41de2a0c29 100644 --- a/swarm/storage/localstore/tags_test.go +++ b/swarm/storage/tagstore/tags_test.go @@ -14,9 +14,10 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -package localstore +package tagstore import ( + "fmt" "testing" "time" ) @@ -25,8 +26,11 @@ import ( func TestTags(t *testing.T) { db, cleanupFunc := newTestDB(t, nil) defer cleanupFunc() - - tag := db.NewTag(time.Now().Unix(), "path/to/directory") + timeNow := time.Now().Unix() + tag, err := db.NewTag(timeNow, "path/to/directory") + if err != nil { + t.Fatal(err) + } /* c := generateTestRandomChunkWithTags([]uint64{tag}) @@ -36,16 +40,27 @@ func TestTags(t *testing.T) { } */ - existingTags = db.GetTags() + existingTags, err := db.GetTags() + if err != nil { + t.Fatal(err) + } + tag++ + existingTags.Range(func(k, v interface{}) bool { + fmt.Println(k) + fmt.Println(v) + return true + }) //expect tag to be in existingTags - oneTag = db.GetTag(tag) - + //oneTag, err := db.GetTag(tag) + if err != nil { + t.Fatal(err) + } // expect to exist //delete tag - err = db.DeleteTag(tag) + /*err = db.DeleteTag(tag) if err != nil { t.Fatal(err) } @@ -54,7 +69,7 @@ func TestTags(t *testing.T) { if err == nil { t.Fatal("tag should not exist") } - + */ } /*func TestPutTag(t *testing.T) { diff --git a/swarm/storage/tagstore/tagstore.go b/swarm/storage/tagstore/tagstore.go new file mode 100644 index 0000000000..cae39d6ec7 --- /dev/null +++ b/swarm/storage/tagstore/tagstore.go @@ -0,0 +1,209 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package tagstore + +import ( + "sync" + "time" + + "github.com/ethereum/go-ethereum/swarm/chunk" + "github.com/ethereum/go-ethereum/swarm/shed" + "github.com/ethereum/go-ethereum/swarm/storage/mock" +) + +// DB implements chunk.Store. +var _ chunk.TagStore = &DB{} + +var ( + // Default value for Capacity DB option. + defaultCapacity uint64 = 5000000 +) + +// DB is the tag store implementation and holds +// database related objects. +type DB struct { + shed *shed.DB + + // schema name of loaded data + schemaName shed.StringField + + // capacity + capacity uint64 + + // retrieval indexes + // tags index maintains a mapping between tags and upload time, chunk status and tag name + tagIndex shed.GenericIndex + + baseKey []byte + + batchMu sync.Mutex + + // this channel is closed when close function is called + // to terminate other goroutines + close chan struct{} +} + +// Options struct holds optional parameters for configuring DB. +type Options struct { + // MockStore is a mock node store that is used to store + // chunk data in a central store. It can be used to reduce + // total storage space requirements in testing large number + // of swarm nodes with chunk data deduplication provided by + // the mock global store. + MockStore *mock.NodeStore + // Capacity is a limit that triggers garbage collection when + // number of items in gcIndex equals or exceeds it. + Capacity uint64 + // MetricsPrefix defines a prefix for metrics names. + MetricsPrefix string +} + +// New returns a new DB. All fields and indexes are initialized +// and possible conflicts with schema from existing database is checked. +// One goroutine for writing batches is created. +func New(path string, o *Options) (db *DB, err error) { + if o == nil { + // default options + o = &Options{ + Capacity: 5000000, + } + } + db = &DB{ + capacity: o.Capacity, + // channel collectGarbageTrigger + // needs to be buffered with the size of 1 + // to signal another event if it + // is triggered during already running function + close: make(chan struct{}), + } + if db.capacity <= 0 { + db.capacity = defaultCapacity + } + + db.shed, err = shed.NewDB(path, o.MetricsPrefix) + if err != nil { + return nil, err + } + // Identify current storage schema by arbitrary name. + db.schemaName, err = db.shed.NewStringField("schema-name") + if err != nil { + return nil, err + } + + // Functions for retrieval data index. + /* var ( + encodeValueFunc func(fields shed.Item) (value []byte, err error) + decodeValueFunc func(keyItem shed.Item, value []byte) (e shed.Item, err error) + )*/ + + /*db.pushIndex, err = db.shed.NewIndex("StoreTimestamp|Hash->Tags", shed.IndexFuncs{ + EncodeKey: func(fields shed.Item) (key []byte, err error) { + key = make([]byte, 40) + binary.BigEndian.PutUint64(key[:8], uint64(fields.StoreTimestamp)) + copy(key[8:], fields.Address[:]) + return key, nil + }, + DecodeKey: func(key []byte) (e shed.Item, err error) { + e.Address = key[8:] + e.StoreTimestamp = int64(binary.BigEndian.Uint64(key[:8])) + return e, nil + }, + EncodeValue: func(fields shed.Item) (value []byte, err error) { + valueBuffer := []byte{} + buf := make([]byte, binary.MaxVarintLen64) + + for _, v := range fields.Tags { + n := binary.PutUvarint(buf, v) + if n > 0 { + valueBuffer = append(valueBuffer, buf[:n]...) + } + } + + return valueBuffer, nil + }, + DecodeValue: func(keyItem shed.Item, value []byte) (e shed.Item, err error) { + for { + v, n := binary.Uvarint(value) + if n > 0 { + e.Tags = append(e.Tags, v) + } else { + break + } + value = value[n:] + } + return e, nil + }, + }) + if err != nil { + return nil, err + }*/ + + db.tagIndex, err = db.shed.NewGenericIndex("Tag->UploadTime|UploadName", shed.GenericIndexFuncs{ + EncodeKey: func(fields interface{}) (key []byte, err error) { + return nil, nil + }, + DecodeKey: func(key []byte) (e interface{}, err error) { + return nil, nil + }, + EncodeValue: func(fields interface{}) (value []byte, err error) { + return nil, nil + }, + DecodeValue: func(keyItem interface{}, value []byte) (e interface{}, err error) { + return nil, nil + }, + }) + if err != nil { + return nil, err + } + return db, err +} + +// Close closes the underlying database. +func (db *DB) Close() (err error) { + close(db.close) + + return db.shed.Close() +} + +// chunkToItem creates new Item with data provided by the Chunk. +func chunkToItem(ch chunk.Chunk) shed.Item { + return shed.Item{ + Address: ch.Address(), + Data: ch.Data(), + Tags: ch.Tags(), + } +} + +// addressToItem creates new Item with a provided address. +func addressToItem(addr chunk.Address) shed.Item { + return shed.Item{ + Address: addr, + } +} + +// now is a helper function that returns a current unix timestamp +// in UTC timezone. +// It is set in the init function for usage in production, and +// optionally overridden in tests for data validation. +var now func() int64 + +func init() { + // set the now function + now = func() (t int64) { + return time.Now().UTC().UnixNano() + } +} diff --git a/swarm/storage/tagstore/tagstore_test.go b/swarm/storage/tagstore/tagstore_test.go new file mode 100644 index 0000000000..b8d2b6b20b --- /dev/null +++ b/swarm/storage/tagstore/tagstore_test.go @@ -0,0 +1,352 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package tagstore + +import ( + "bytes" + "io/ioutil" + "math/rand" + "os" + "runtime" + "testing" + "time" + + "github.com/ethereum/go-ethereum/swarm/chunk" +) + +// TestDB validates if the chunk can be uploaded and +// correctly retrieved. +func TestDB(t *testing.T) { + // db, cleanupFunc := newTestDB(t, nil) + // defer cleanupFunc() + +} + +// generateTestRandomChunk generates a Chunk that is not +// valid, but it contains a random key and a random value. +// This function is faster then storage.generateTestRandomChunk +// which generates a valid chunk. +// Some tests in this package do not need valid chunks, just +// random data, and their execution time can be decreased +// using this function. +func generateTestRandomChunk() chunk.Chunk { + data := make([]byte, chunk.DefaultSize) + rand.Read(data) + key := make([]byte, 32) + rand.Read(key) + r := rand.New(rand.NewSource(time.Now().UnixNano())) + n := r.Intn(10) + tags := []uint64{} + for i := 0; i < n; i++ { + tags = append(tags, r.Uint64()) + } + return chunk.NewChunk(key, data, tags) +} + +// generateTestRandomChunkWithTags does the same at the above but allows a custom tag +func generateTestRandomChunkWithTags(tags []uint64) chunk.Chunk { + data := make([]byte, chunk.DefaultSize) + rand.Read(data) + key := make([]byte, 32) + rand.Read(key) + return chunk.NewChunk(key, data, tags) +} + +// TestGenerateTestRandomChunk validates that +// generateTestRandomChunk returns random data by comparing +// two generated chunks. +func TestGenerateTestRandomChunk(t *testing.T) { + c1 := generateTestRandomChunk() + c2 := generateTestRandomChunk() + addrLen := len(c1.Address()) + if addrLen != 32 { + t.Errorf("first chunk address length %v, want %v", addrLen, 32) + } + dataLen := len(c1.Data()) + if dataLen != chunk.DefaultSize { + t.Errorf("first chunk data length %v, want %v", dataLen, chunk.DefaultSize) + } + addrLen = len(c2.Address()) + if addrLen != 32 { + t.Errorf("second chunk address length %v, want %v", addrLen, 32) + } + dataLen = len(c2.Data()) + if dataLen != chunk.DefaultSize { + t.Errorf("second chunk data length %v, want %v", dataLen, chunk.DefaultSize) + } + if bytes.Equal(c1.Address(), c2.Address()) { + t.Error("fake chunks addresses do not differ") + } + if bytes.Equal(c1.Data(), c2.Data()) { + t.Error("fake chunks data bytes do not differ") + } + for i, _ := range c1.Tags() { + if i > len(c2.Tags())-1 { + break + } + if c1.Tags()[i] == c2.Tags()[i] { + t.Fatal("tags should be different") + } + } +} + +/* +// newRetrieveIndexesTest returns a test function that validates if the right +// chunk values are in the retrieval indexes. +func newRetrieveIndexesTest(db *DB, chunk chunk.Chunk, storeTimestamp, accessTimestamp int64) func(t *testing.T) { + return func(t *testing.T) { + item, err := db.retrievalDataIndex.Get(addressToItem(chunk.Address())) + if err != nil { + t.Fatal(err) + } + validateItem(t, item, chunk.Address(), chunk.Data(), storeTimestamp, 0, []uint64{}) + + // access index should not be set + wantErr := leveldb.ErrNotFound + item, err = db.retrievalAccessIndex.Get(addressToItem(chunk.Address())) + if err != wantErr { + t.Errorf("got error %v, want %v", err, wantErr) + } + } +}*/ +/* +// newRetrieveIndexesTestWithAccess returns a test function that validates if the right +// chunk values are in the retrieval indexes when access time must be stored. +func newRetrieveIndexesTestWithAccess(db *DB, ch chunk.Chunk, storeTimestamp, accessTimestamp int64) func(t *testing.T) { + return func(t *testing.T) { + item, err := db.retrievalDataIndex.Get(addressToItem(ch.Address())) + if err != nil { + t.Fatal(err) + } + validateItem(t, item, ch.Address(), ch.Data(), storeTimestamp, 0, []uint64{}) + + if accessTimestamp > 0 { + item, err = db.retrievalAccessIndex.Get(addressToItem(ch.Address())) + if err != nil { + t.Fatal(err) + } + validateItem(t, item, ch.Address(), nil, 0, accessTimestamp, []uint64{}) + } + } +} + +// newPullIndexTest returns a test function that validates if the right +// chunk values are in the pull index. +func newPullIndexTest(db *DB, ch chunk.Chunk, binID uint64, wantError error) func(t *testing.T) { + return func(t *testing.T) { + item, err := db.pullIndex.Get(shed.Item{ + Address: ch.Address(), + BinID: binID, + }) + if err != wantError { + t.Errorf("got error %v, want %v", err, wantError) + } + if err == nil { + validateItem(t, item, ch.Address(), nil, 0, 0, []uint64{}) + } + } +} + +// newPushIndexTest returns a test function that validates if the right +// chunk values are in the push index. +func newPushIndexTest(db *DB, ch chunk.Chunk, storeTimestamp int64, wantError error) func(t *testing.T) { + return func(t *testing.T) { + item, err := db.pushIndex.Get(shed.Item{ + Address: ch.Address(), + StoreTimestamp: storeTimestamp, + }) + if err != wantError { + t.Errorf("got error %v, want %v", err, wantError) + } + if err == nil { + validateItem(t, item, ch.Address(), nil, storeTimestamp, 0, ch.Tags()) + } + } +} + +// newGCIndexTest returns a test function that validates if the right +// chunk values are in the push index. +func newGCIndexTest(db *DB, chunk chunk.Chunk, storeTimestamp, accessTimestamp int64, binID uint64) func(t *testing.T) { + return func(t *testing.T) { + item, err := db.gcIndex.Get(shed.Item{ + Address: chunk.Address(), + BinID: binID, + AccessTimestamp: accessTimestamp, + }) + if err != nil { + t.Fatal(err) + } + validateItem(t, item, chunk.Address(), nil, 0, accessTimestamp, []uint64{}) + } +}*/ +/* +// newItemsCountTest returns a test function that validates if +// an index contains expected number of key/value pairs. +func newItemsCountTest(i shed.Index, want int) func(t *testing.T) { + return func(t *testing.T) { + var c int + err := i.Iterate(func(item shed.Item) (stop bool, err error) { + c++ + return + }, nil) + if err != nil { + t.Fatal(err) + } + if c != want { + t.Errorf("got %v items in index, want %v", c, want) + } + } +} +*/ + +// testItemsOrder tests the order of chunks in the index. If sortFunc is not nil, +// chunks will be sorted with it before validation. +/*func testItemsOrder(t *testing.T, i shed.Index, chunks []testIndexChunk, sortFunc func(i, j int) (less bool)) { + newItemsCountTest(i, len(chunks))(t) + + if sortFunc != nil { + sort.Slice(chunks, sortFunc) + } + + var cursor int + err := i.Iterate(func(item shed.Item) (stop bool, err error) { + want := chunks[cursor].Address() + got := item.Address + if !bytes.Equal(got, want) { + return true, fmt.Errorf("got address %x at position %v, want %x", got, cursor, want) + } + cursor++ + return false, nil + }, nil) + if err != nil { + t.Fatal(err) + } +}*/ + +// setNow replaces now function and +// returns a function that will reset it to the +// value before the change. +func setNow(f func() int64) (reset func()) { + current := now + reset = func() { now = current } + now = f + return reset +} + +// TestSetNow tests if setNow function changes now function +// correctly and if its reset function resets the original function. +func TestSetNow(t *testing.T) { + // set the current function after the test finishes + defer func(f func() int64) { now = f }(now) + + // expected value for the unchanged function + var original int64 = 1 + // expected value for the changed function + var changed int64 = 2 + + // define the original (unchanged) functions + now = func() int64 { + return original + } + + // get the time + got := now() + + // test if got variable is set correctly + if got != original { + t.Errorf("got now value %v, want %v", got, original) + } + + // set the new function + reset := setNow(func() int64 { + return changed + }) + + // get the time + got = now() + + // test if got variable is set correctly to changed value + if got != changed { + t.Errorf("got hook value %v, want %v", got, changed) + } + + // set the function to the original one + reset() + + // get the time + got = now() + + // test if got variable is set correctly to original value + if got != original { + t.Errorf("got hook value %v, want %v", got, original) + } +} + +// newTestDB is a helper function that constructs a +// temporary database and returns a cleanup function that must +// be called to remove the data. +func newTestDB(t testing.TB, o *Options) (db *DB, cleanupFunc func()) { + t.Helper() + + dir, err := ioutil.TempDir("", "tagstore-test") + if err != nil { + t.Fatal(err) + } + cleanupFunc = func() { os.RemoveAll(dir) } + baseKey := make([]byte, 32) + if _, err := rand.Read(baseKey); err != nil { + t.Fatal(err) + } + db, err = New(dir, o) + if err != nil { + cleanupFunc() + t.Fatal(err) + } + cleanupFunc = func() { + err := db.Close() + if err != nil { + t.Error(err) + } + os.RemoveAll(dir) + } + return db, cleanupFunc +} + +func init() { + // needed for generateTestRandomChunk + rand.Seed(time.Now().UnixNano()) +} + +func init() { + // Some of the tests in localstore package rely on the same ordering of + // items uploaded or accessed compared to the ordering of items in indexes + // that contain StoreTimestamp or AccessTimestamp in keys. In tests + // where the same order is required from the database as the order + // in which chunks are put or accessed, if the StoreTimestamp or + // AccessTimestamp are the same for two or more sequential items + // their order in database will be based on the chunk address value, + // in which case the ordering of items/chunks stored in a test slice + // will not be the same. To ensure the same ordering in database on such + // indexes on windows systems, an additional short sleep is added to + // the now function. + if runtime.GOOS == "windows" { + setNow(func() int64 { + time.Sleep(time.Microsecond) + return time.Now().UTC().UnixNano() + }) + } +} From 7850763fcc27bd83d985b77ce27c116e01d91dad Mon Sep 17 00:00:00 2001 From: Elad Date: Sat, 6 Apr 2019 13:33:15 +0900 Subject: [PATCH 29/31] swarm: wip tags test --- swarm/chunk/tags.go | 15 ++++++- swarm/shed/generic_index.go | 3 -- swarm/storage/tagstore/tags.go | 26 +++++------ swarm/storage/tagstore/tags_test.go | 30 ++++++++----- swarm/storage/tagstore/tagstore.go | 70 ++++++++--------------------- 5 files changed, 64 insertions(+), 80 deletions(-) diff --git a/swarm/chunk/tags.go b/swarm/chunk/tags.go index 49a550a9af..fbfd215d78 100644 --- a/swarm/chunk/tags.go +++ b/swarm/chunk/tags.go @@ -32,6 +32,7 @@ const ( // Tag represents info on the status of new chunks type Tag struct { + uid uint64 //a unique identifier for this tag name string total uint32 // total chunks belonging to a tag split uint32 // number of chunks already processed by splitter for hashing @@ -56,14 +57,15 @@ func NewTags() *Tags { // New creates a new tag, stores it by the name and returns it // it returns an error if the tag with this name already exists -func (ts *Tags) New(s string, total int) (*Tag, error) { +func (ts *Tags) New(uid uint64, s string, total int) (*Tag, error) { t := &Tag{ + uid: uid, name: s, startedAt: time.Now(), total: uint32(total), State: make(chan State, 5), } - _, loaded := ts.tags.LoadOrStore(s, t) + _, loaded := ts.tags.LoadOrStore(uid, t) if loaded { return nil, errExists } @@ -105,6 +107,15 @@ func (t *Tag) Get(state State) int { return int(atomic.LoadUint32(v)) } +// GetUid returns the unique identifier +func (t Tag) GetUid() uint64 { + return t.uid +} + +func (t Tag) GetName() string { + return t.name +} + // GetTotal returns the total count func (t *Tag) GetTotal() int { return int(atomic.LoadUint32(&t.total)) diff --git a/swarm/shed/generic_index.go b/swarm/shed/generic_index.go index 245d2f8713..defc1cc35a 100644 --- a/swarm/shed/generic_index.go +++ b/swarm/shed/generic_index.go @@ -18,7 +18,6 @@ package shed import ( "bytes" - "fmt" "github.com/syndtr/goleveldb/leveldb" "github.com/syndtr/goleveldb/leveldb/iterator" @@ -236,8 +235,6 @@ func (f GenericIndex) Iterate(fn GenericIndexIterFunc, options *GenericIterateOp // leveldb.ErrNotFound is returned. Value for totalPrefix must start with // Index prefix. func (f GenericIndex) tupleFromIterator(it iterator.Iterator, totalPrefix []byte) (k, v interface{}, err error) { - fmt.Println("tupleFrom") - key := it.Key() if !bytes.HasPrefix(key, totalPrefix) { return nil, nil, leveldb.ErrNotFound diff --git a/swarm/storage/tagstore/tags.go b/swarm/storage/tagstore/tags.go index 6fca5f2410..206d47a847 100644 --- a/swarm/storage/tagstore/tags.go +++ b/swarm/storage/tagstore/tags.go @@ -18,8 +18,6 @@ package tagstore import ( "encoding/binary" - "math/rand" - "time" "github.com/ethereum/go-ethereum/swarm/chunk" "github.com/syndtr/goleveldb/leveldb" @@ -31,23 +29,24 @@ func (db *DB) NewTag(uploadTime int64, uploadName string) (tag uint64, err error // protect parallel updates db.batchMu.Lock() defer db.batchMu.Unlock() - r := rand.New(rand.NewSource(time.Now().Unix())) - - tag = r.Uint64() - + tag = db.rng.Uint64() batch := new(leveldb.Batch) val := make([]byte, 8) binary.BigEndian.PutUint64(val, uint64(uploadTime)) val = append(val, []byte(uploadName)...) - + //check that it doesnt exist // put to indexes: tag - db.tagIndex.PutInBatch(batch, tag, val) + err = db.tagIndex.PutInBatch(batch, tag, val) + if err != nil { + return tag, err + } + err = db.shed.WriteBatch(batch) if err != nil { return tag, err } - return tag, nil + return tag, nil } func (db *DB) DeleteTag(tag uint64) error { @@ -55,16 +54,17 @@ func (db *DB) DeleteTag(tag uint64) error { } func (db *DB) GetTags() (*chunk.Tags, error) { - //tags := make([]chunk.Tag, 0) t := chunk.NewTags() err := db.tagIndex.Iterate(func(k, v interface{}) (bool, error) { - _ = k.(uint64) + keyVal := k.(uint64) + valBytes := v.([]byte) + _ = binary.BigEndian.Uint64(valBytes) - _, err := t.New("tag", 0) + tagName := string(valBytes[8:]) + _, err := t.New(keyVal, tagName, 0) if err != nil { return true, err } - //todo append return false, nil }, nil) return t, err diff --git a/swarm/storage/tagstore/tags_test.go b/swarm/storage/tagstore/tags_test.go index 41de2a0c29..3801effdea 100644 --- a/swarm/storage/tagstore/tags_test.go +++ b/swarm/storage/tagstore/tags_test.go @@ -17,9 +17,10 @@ package tagstore import ( - "fmt" "testing" "time" + + "github.com/ethereum/go-ethereum/swarm/chunk" ) // tests that new tag is created, iterated over (one or all) and deleted in the database @@ -27,27 +28,36 @@ func TestTags(t *testing.T) { db, cleanupFunc := newTestDB(t, nil) defer cleanupFunc() timeNow := time.Now().Unix() - tag, err := db.NewTag(timeNow, "path/to/directory") - if err != nil { - t.Fatal(err) + testTags := []struct { + tag uint64 + path string + }{ + {path: "path/to/dir1"}, + {path: "path/to/dir2"}, + {path: "another/path"}, } - /* c := generateTestRandomChunkWithTags([]uint64{tag}) + tagMap := make(map[uint64]string) - err := db.Put(context.Background(), chunk.ModePutUpload, c) + for _, v := range testTags { + localTag, err := db.NewTag(timeNow, v.path) if err != nil { t.Fatal(err) } - */ + tagMap[localTag] = v.path + } existingTags, err := db.GetTags() if err != nil { t.Fatal(err) } - tag++ + existingTags.Range(func(k, v interface{}) bool { - fmt.Println(k) - fmt.Println(v) + keyVal := k.(uint64) + vv := v.(*chunk.Tag) + if vv.GetName() != tagMap[keyVal] { + t.Fatal("tag not equal") + } return true }) diff --git a/swarm/storage/tagstore/tagstore.go b/swarm/storage/tagstore/tagstore.go index cae39d6ec7..22b1959721 100644 --- a/swarm/storage/tagstore/tagstore.go +++ b/swarm/storage/tagstore/tagstore.go @@ -17,6 +17,8 @@ package tagstore import ( + "encoding/binary" + "math/rand" "sync" "time" @@ -44,6 +46,9 @@ type DB struct { // capacity capacity uint64 + // random number generator + rng *rand.Rand + // retrieval indexes // tags index maintains a mapping between tags and upload time, chunk status and tag name tagIndex shed.GenericIndex @@ -104,66 +109,27 @@ func New(path string, o *Options) (db *DB, err error) { return nil, err } - // Functions for retrieval data index. - /* var ( - encodeValueFunc func(fields shed.Item) (value []byte, err error) - decodeValueFunc func(keyItem shed.Item, value []byte) (e shed.Item, err error) - )*/ - - /*db.pushIndex, err = db.shed.NewIndex("StoreTimestamp|Hash->Tags", shed.IndexFuncs{ - EncodeKey: func(fields shed.Item) (key []byte, err error) { - key = make([]byte, 40) - binary.BigEndian.PutUint64(key[:8], uint64(fields.StoreTimestamp)) - copy(key[8:], fields.Address[:]) - return key, nil - }, - DecodeKey: func(key []byte) (e shed.Item, err error) { - e.Address = key[8:] - e.StoreTimestamp = int64(binary.BigEndian.Uint64(key[:8])) - return e, nil - }, - EncodeValue: func(fields shed.Item) (value []byte, err error) { - valueBuffer := []byte{} - buf := make([]byte, binary.MaxVarintLen64) - - for _, v := range fields.Tags { - n := binary.PutUvarint(buf, v) - if n > 0 { - valueBuffer = append(valueBuffer, buf[:n]...) - } - } - - return valueBuffer, nil - }, - DecodeValue: func(keyItem shed.Item, value []byte) (e shed.Item, err error) { - for { - v, n := binary.Uvarint(value) - if n > 0 { - e.Tags = append(e.Tags, v) - } else { - break - } - value = value[n:] - } - return e, nil - }, - }) - if err != nil { - return nil, err - }*/ + // initialise the random number generator + db.rng = rand.New(rand.NewSource(time.Now().Unix())) db.tagIndex, err = db.shed.NewGenericIndex("Tag->UploadTime|UploadName", shed.GenericIndexFuncs{ - EncodeKey: func(fields interface{}) (key []byte, err error) { - return nil, nil + EncodeKey: func(tag interface{}) (key []byte, err error) { + // key is uint64 + key = make([]byte, 8) + tagUint := tag.(uint64) + binary.BigEndian.PutUint64(key, tagUint) + return key, nil }, DecodeKey: func(key []byte) (e interface{}, err error) { - return nil, nil + tag := binary.BigEndian.Uint64(key) + return tag, nil }, EncodeValue: func(fields interface{}) (value []byte, err error) { - return nil, nil + b := fields.([]byte) + return b, nil }, DecodeValue: func(keyItem interface{}, value []byte) (e interface{}, err error) { - return nil, nil + return value, nil }, }) if err != nil { From 314eadfccbe6774865b20cb88b0258c01cdf2753 Mon Sep 17 00:00:00 2001 From: Elad Date: Sun, 7 Apr 2019 08:15:44 +0900 Subject: [PATCH 30/31] swarm: change uid to uint32 --- swarm/chunk/tags.go | 10 ++++---- swarm/storage/tagstore/tags.go | 37 +++++++++++++++++++++-------- swarm/storage/tagstore/tags_test.go | 6 ++--- swarm/storage/tagstore/tagstore.go | 25 ++++++++++++++----- 4 files changed, 54 insertions(+), 24 deletions(-) diff --git a/swarm/chunk/tags.go b/swarm/chunk/tags.go index fbfd215d78..b219969d77 100644 --- a/swarm/chunk/tags.go +++ b/swarm/chunk/tags.go @@ -16,8 +16,8 @@ var ( ) type TagStore interface { - ChunkTags(addr Address) ([]uint64, error) - NewTag(uploadTime int64, path string) (tag uint64, err error) + ChunkTags(addr Address) ([]uint32, error) + NewTag(uploadTime int64, path string) (tag uint32, err error) } // State is the enum type for chunk states @@ -32,7 +32,7 @@ const ( // Tag represents info on the status of new chunks type Tag struct { - uid uint64 //a unique identifier for this tag + uid uint32 //a unique identifier for this tag name string total uint32 // total chunks belonging to a tag split uint32 // number of chunks already processed by splitter for hashing @@ -57,7 +57,7 @@ func NewTags() *Tags { // New creates a new tag, stores it by the name and returns it // it returns an error if the tag with this name already exists -func (ts *Tags) New(uid uint64, s string, total int) (*Tag, error) { +func (ts *Tags) New(uid uint32, s string, total int) (*Tag, error) { t := &Tag{ uid: uid, name: s, @@ -108,7 +108,7 @@ func (t *Tag) Get(state State) int { } // GetUid returns the unique identifier -func (t Tag) GetUid() uint64 { +func (t Tag) GetUid() uint32 { return t.uid } diff --git a/swarm/storage/tagstore/tags.go b/swarm/storage/tagstore/tags.go index 206d47a847..8686eaede4 100644 --- a/swarm/storage/tagstore/tags.go +++ b/swarm/storage/tagstore/tags.go @@ -25,11 +25,28 @@ import ( var _ chunk.TagStore = &DB{} -func (db *DB) NewTag(uploadTime int64, uploadName string) (tag uint64, err error) { +/* +type Tag struct { + uid uint64 //a unique identifier for this tag + name string + total uint32 // total chunks belonging to a tag + split uint32 // number of chunks already processed by splitter for hashing + stored uint32 // number of chunks already stored locally + sent uint32 // number of chunks sent for push syncing + synced uint32 // number of chunks synced with proof + startedAt time.Time // tag started to calculate ETA + State chan State // channel to signal completion +} +*/ +func (db *DB) SaveTag(tag *chunk.Tag) error { + return nil +} + +func (db *DB) NewTag(uploadTime int64, uploadName string) (tag uint32, err error) { // protect parallel updates db.batchMu.Lock() defer db.batchMu.Unlock() - tag = db.rng.Uint64() + tag = db.rng.Uint32() batch := new(leveldb.Batch) val := make([]byte, 8) binary.BigEndian.PutUint64(val, uint64(uploadTime)) @@ -49,16 +66,16 @@ func (db *DB) NewTag(uploadTime int64, uploadName string) (tag uint64, err error return tag, nil } -func (db *DB) DeleteTag(tag uint64) error { +func (db *DB) DeleteTag(tag uint32) error { return nil } func (db *DB) GetTags() (*chunk.Tags, error) { t := chunk.NewTags() err := db.tagIndex.Iterate(func(k, v interface{}) (bool, error) { - keyVal := k.(uint64) + keyVal := k.(uint32) valBytes := v.([]byte) - _ = binary.BigEndian.Uint64(valBytes) + _ = binary.BigEndian.Uint32(valBytes) tagName := string(valBytes[8:]) _, err := t.New(keyVal, tagName, 0) @@ -70,18 +87,18 @@ func (db *DB) GetTags() (*chunk.Tags, error) { return t, err } -func (db *DB) GetTag(tag uint64) (chunk.Tag, error) { +func (db *DB) GetTag(tag uint32) (chunk.Tag, error) { return chunk.Tag{}, nil } -func (db *DB) ChunkTags(addr chunk.Address) ([]uint64, error) { +func (db *DB) ChunkTags(addr chunk.Address) ([]uint32, error) { /*item := addressToItem(addr) out, err := db.retrievalDataIndex.Get(item) if err != nil { if err == leveldb.ErrNotFound { - return []uint64{}, nil + return []uint32{}, nil } return nil, err @@ -89,11 +106,11 @@ func (db *DB) ChunkTags(addr chunk.Address) ([]uint64, error) { c, err := db.pushIndex.Get(out) if err != nil { if err == leveldb.ErrNotFound { - return []uint64{}, nil + return []uint32{}, nil } return nil, err } return c.Tags, nil*/ - return []uint64{}, nil + return []uint32{}, nil } diff --git a/swarm/storage/tagstore/tags_test.go b/swarm/storage/tagstore/tags_test.go index 3801effdea..30cd0207fd 100644 --- a/swarm/storage/tagstore/tags_test.go +++ b/swarm/storage/tagstore/tags_test.go @@ -29,7 +29,7 @@ func TestTags(t *testing.T) { defer cleanupFunc() timeNow := time.Now().Unix() testTags := []struct { - tag uint64 + tag uint32 path string }{ {path: "path/to/dir1"}, @@ -37,7 +37,7 @@ func TestTags(t *testing.T) { {path: "another/path"}, } - tagMap := make(map[uint64]string) + tagMap := make(map[uint32]string) for _, v := range testTags { localTag, err := db.NewTag(timeNow, v.path) @@ -53,7 +53,7 @@ func TestTags(t *testing.T) { } existingTags.Range(func(k, v interface{}) bool { - keyVal := k.(uint64) + keyVal := k.(uint32) vv := v.(*chunk.Tag) if vv.GetName() != tagMap[keyVal] { t.Fatal("tag not equal") diff --git a/swarm/storage/tagstore/tagstore.go b/swarm/storage/tagstore/tagstore.go index 22b1959721..4ac6bc3de3 100644 --- a/swarm/storage/tagstore/tagstore.go +++ b/swarm/storage/tagstore/tagstore.go @@ -111,17 +111,30 @@ func New(path string, o *Options) (db *DB, err error) { // initialise the random number generator db.rng = rand.New(rand.NewSource(time.Now().Unix())) - - db.tagIndex, err = db.shed.NewGenericIndex("Tag->UploadTime|UploadName", shed.GenericIndexFuncs{ + /* + type Tag struct { + uid uint64 //a unique identifier for this tag + name string + total uint32 // total chunks belonging to a tag + split uint32 // number of chunks already processed by splitter for hashing + stored uint32 // number of chunks already stored locally + sent uint32 // number of chunks sent for push syncing + synced uint32 // number of chunks synced with proof + startedAt time.Time // tag started to calculate ETA + State chan State // channel to signal completion + } + */ + + db.tagIndex, err = db.shed.NewGenericIndex("Tag->TotalChunks|SplitChunks|StoredChunks|SentChunks|SyncedChunks|UploadTime|UploadName", shed.GenericIndexFuncs{ EncodeKey: func(tag interface{}) (key []byte, err error) { - // key is uint64 + // Tag is uint64 key = make([]byte, 8) - tagUint := tag.(uint64) - binary.BigEndian.PutUint64(key, tagUint) + tagUint := tag.(uint32) + binary.BigEndian.PutUint32(key, tagUint) return key, nil }, DecodeKey: func(key []byte) (e interface{}, err error) { - tag := binary.BigEndian.Uint64(key) + tag := binary.BigEndian.Uint32(key) return tag, nil }, EncodeValue: func(fields interface{}) (value []byte, err error) { From 5b43db21685364b52fcbb64e6add9cfc734f0f70 Mon Sep 17 00:00:00 2001 From: Elad Date: Sun, 7 Apr 2019 10:06:42 +0900 Subject: [PATCH 31/31] swarm/storage: wip tag creation --- swarm/chunk/tags.go | 4 +++ swarm/storage/tagstore/tags.go | 48 +++++++++++++++++++++++++--------- 2 files changed, 40 insertions(+), 12 deletions(-) diff --git a/swarm/chunk/tags.go b/swarm/chunk/tags.go index b219969d77..6ce4810bfc 100644 --- a/swarm/chunk/tags.go +++ b/swarm/chunk/tags.go @@ -72,6 +72,10 @@ func (ts *Tags) New(uid uint32, s string, total int) (*Tag, error) { return t, nil } +func (ts *Tags) LoadOrStore(k, v interface{}) (actual interface{}, loaded bool) { + return ts.tags.LoadOrStore(k, v) +} + // Inc increments the count for a state func (t *Tag) Inc(state State) { var v *uint32 diff --git a/swarm/storage/tagstore/tags.go b/swarm/storage/tagstore/tags.go index 8686eaede4..c0ea8b82bb 100644 --- a/swarm/storage/tagstore/tags.go +++ b/swarm/storage/tagstore/tags.go @@ -18,6 +18,8 @@ package tagstore import ( "encoding/binary" + "fmt" + "time" "github.com/ethereum/go-ethereum/swarm/chunk" "github.com/syndtr/goleveldb/leveldb" @@ -38,7 +40,7 @@ type Tag struct { State chan State // channel to signal completion } */ -func (db *DB) SaveTag(tag *chunk.Tag) error { +func (db *DB) Write(tag *chunk.Tag) error { return nil } @@ -67,29 +69,32 @@ func (db *DB) NewTag(uploadTime int64, uploadName string) (tag uint32, err error } func (db *DB) DeleteTag(tag uint32) error { - return nil + return db.tagIndex.Delete(tag) } func (db *DB) GetTags() (*chunk.Tags, error) { t := chunk.NewTags() err := db.tagIndex.Iterate(func(k, v interface{}) (bool, error) { - keyVal := k.(uint32) - valBytes := v.([]byte) - _ = binary.BigEndian.Uint32(valBytes) - - tagName := string(valBytes[8:]) - _, err := t.New(keyVal, tagName, 0) - if err != nil { - return true, err + tag := tagFromInterface(k, v) + + _, loaded := t.LoadOrStore(tag.GetUid(), tag) + if loaded { + return true, fmt.Errorf("tag uid %d already exists", tag.GetUid()) } return false, nil }, nil) return t, err } -func (db *DB) GetTag(tag uint32) (chunk.Tag, error) { +func (db *DB) GetTag(tag uint32) (*chunk.Tag, error) { + out, err := db.tagIndex.Get(tag) + if err != nil { + return nil, err + } + + t := tagFromInterface(tag, out) - return chunk.Tag{}, nil + return t, nil } func (db *DB) ChunkTags(addr chunk.Address) ([]uint32, error) { @@ -114,3 +119,22 @@ func (db *DB) ChunkTags(addr chunk.Address) ([]uint32, error) { return c.Tags, nil*/ return []uint32{}, nil } + +func tagFromInterface(k, v interface{}) (*chunk.Tag, error) { + uid := k.(uint32) + valBytes := v.([]byte) + _ = binary.BigEndian.Uint32(valBytes) + + tagName := string(valBytes[8:]) + + t := &chunk.Tag{ + uid: uid, + name: s, + startedAt: time.Now(), + total: uint32(total), + State: make(chan State, 5), + } + + return t + +}