Skip to content
This repository was archived by the owner on Aug 2, 2021. It is now read-only.
Closed
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
81 commits
Select commit Hold shift + click to select a range
d8acb12
swarm/storage/localstore: most basic database
janos Nov 29, 2018
9ec535a
swarm/storage/localstore: fix typos and comments
janos Dec 3, 2018
37205de
swarm/shed: add uint64 field Dec and DecInBatch methods
janos Dec 3, 2018
b1ded5a
swarm/storage/localstore: decrement size counter on ModeRemoval update
janos Dec 3, 2018
572f3cb
swarm/storage/localstore: unexport modeAccess and modeRemoval
janos Dec 3, 2018
cbb510b
swarm/storage/localstore: add WithRetrievalCompositeIndex
janos Dec 3, 2018
c7beb22
swarm/storage/localstore: add TestModeSyncing
janos Dec 3, 2018
391faa7
swarm/storage/localstore: fix test name
janos Dec 3, 2018
58f3f86
swarm/storage/localstore: add TestModeUpload
janos Dec 3, 2018
2d928bf
swarm/storage/localstore: add TestModeRequest
janos Dec 3, 2018
4d58a6f
swarm/storage/localstore: add TestModeSynced
janos Dec 4, 2018
af1b137
swarm/storage/localstore: add TestModeAccess
janos Dec 4, 2018
96409ff
swarm/storage/localstore: add TestModeRemoval
janos Dec 4, 2018
b782bfe
swarm/storage/localstore: add mock store option for chunk data
janos Dec 5, 2018
35376d8
swarm/storage/localstore: add TestDB_pullIndex
janos Dec 5, 2018
e6a7196
swarm/storage/localstore: add TestDB_gcIndex
janos Dec 6, 2018
58c7f11
swarm/storage/localstore: change how batches are written
janos Dec 6, 2018
6e8b2ad
swarm/storage/localstore: add updateOnAccess function
janos Dec 6, 2018
7b8510e
swarm/storage/localhost: add DB.gcSize
janos Dec 6, 2018
cf3ec30
swarm/storage/localstore: update comments
janos Dec 7, 2018
d58e1ee
swarm/storage/localstore: add BenchmarkNew
janos Dec 7, 2018
f2299f4
swarm/storage/localstore: add retrieval tests benchmarks
janos Dec 7, 2018
e6bdda7
swarm/storage/localstore: accessors redesign
janos Dec 13, 2018
b6f5b7a
swarm/storage/localstore: add semaphore for updateGC goroutine
janos Dec 13, 2018
5732c38
swarm/storage/localstore: implement basic garbage collection
janos Dec 13, 2018
ac9d153
swarm/storage/localstore: optimize collectGarbage
janos Dec 14, 2018
e6e29f5
swarm/storage/localstore: add more garbage collection tests cases
janos Dec 14, 2018
750268d
swarm/shed, swarm/storage/localstore: rename IndexItem to Item
janos Dec 17, 2018
a486a09
swarm/shed: add Index.CountFrom
janos Dec 18, 2018
e17fec2
swarm/storage/localstore: persist gcSize
janos Dec 18, 2018
5605323
swarm/storage/localstore: remove composite retrieval index
janos Dec 18, 2018
5ebbcb5
swarm/shed: IterateWithPrefix and IterateWithPrefixFrom Index functions
janos Dec 19, 2018
80c269b
swarm/storage/localstore: writeGCSize function with leveldb batch
janos Dec 19, 2018
bca85ef
swarm/storage/localstore: unexport modeSetRemove
janos Dec 19, 2018
cee4004
swarm/storage/localstore: update writeGCSizeWorker comment
janos Dec 19, 2018
b5a0fd2
swarm/storage/localstore: add triggerGarbageCollection function
janos Dec 19, 2018
4fca004
swarm/storage/localstore: call writeGCSize on DB Close
janos Dec 19, 2018
5ad0c8d
swarm/storage/localstore: additional comment in writeGCSizeWorker
janos Dec 19, 2018
cb8078e
Merge branch 'master' into localstore
janos Dec 19, 2018
c9f5130
swarm/storage/localstore: add MetricsPrefix option
janos Dec 19, 2018
da9bab4
swarm/storage/localstore: fix a typo
janos Dec 19, 2018
d54d7ae
swamr/shed: only one Index Iterate function
janos Dec 19, 2018
67473be
swarm/storage/localstore: use shed Iterate function
janos Dec 19, 2018
2299147
swarm/shed: pass a new byte slice copy to index decode functions
janos Dec 20, 2018
5488a2b
swarm/storage/localstore: implement feed subscriptions
janos Dec 20, 2018
38bdf7f
swarm/storage/localstore: add more subscriptions tests
janos Dec 21, 2018
a74eae2
swarm/storage/localsore: add parallel upload test
janos Dec 21, 2018
534009f
swarm/storage/localstore: use storage.MaxPO in subscription tests
janos Dec 21, 2018
5edd22d
swarm/storage/localstore: subscription of addresses instead chunks
janos Dec 21, 2018
95726d7
swarm/storage/localstore: lock item address in collectGarbage iterator
janos Dec 21, 2018
c5dcae3
swarm/storage/localstore: fix TestSubscribePull to include MaxPO
janos Dec 22, 2018
f3380ea
swarm/storage/localstore: improve subscriptions
janos Jan 7, 2019
1d2d470
swarm/storage/localstore: add TestDB_SubscribePull_sinceAndUntil test
janos Jan 7, 2019
c26c979
swarm/storage/localstore: adjust pull sync tests
janos Jan 7, 2019
c188281
Merge branch 'master' into localstore
janos Jan 7, 2019
c5e4c61
swarm/storage/localstore: remove writeGCSizeDelay and use literal
janos Jan 8, 2019
015d977
swarm/storage/localstore: adjust subscriptions tests delays and comments
janos Jan 8, 2019
abbb4a6
swarm/storage/localstore: add godoc package overview
janos Jan 10, 2019
fb0a822
swarm/storage/localstore: fix a typo
janos Jan 10, 2019
c423060
swarm/storage/localstore: update package overview
janos Jan 10, 2019
c5a6456
swarm/storage/localstore: remove repeated index change
janos Jan 11, 2019
33726a4
swarm/storage/localstore: rename ChunkInfo to ChunkDescriptor
janos Jan 11, 2019
25a068a
swarm/storage/localstore: add comment in collectGarbageWorker
janos Jan 11, 2019
ca1e24f
swarm/storage/localstore: replace atomics with mutexes for gcSize and…
janos Jan 11, 2019
c62fee3
Merge branch 'master' into localstore
janos Jan 14, 2019
5ddc75f
swarm/storage/localstore: protect addrs map in pull subs tests
janos Jan 14, 2019
87bbd61
swarm/storage/localstore: protect slices in push subs test
janos Jan 14, 2019
6ad67d7
swarm/storage/localstore: protect chunks in TestModePutUpload_parallel
janos Jan 14, 2019
a550388
swarm/storage/localstore: fix a race in TestDB_updateGCSem defers
janos Jan 14, 2019
1dae999
swarm/storage/localstore: remove parallel flag from tests
janos Jan 14, 2019
eda338a
swarm/storage/localstore: fix a race in testDB_collectGarbageWorker
janos Jan 14, 2019
ad5b329
swarm/storage/localstore: remove unused code
janos Jan 22, 2019
8d15e82
swarm/storage/localstore: add more context to pull sub log messages
janos Jan 25, 2019
3948044
swarm/storage/localstore: merge branch 'master' into localstore
janos Jan 25, 2019
6c8208a
swarm/storage/localstore: BenchmarkPutUpload and global lock option
janos Jan 26, 2019
6accc6b
swarm/storage/localstore: pre-generate chunks in BenchmarkPutUpload
janos Jan 28, 2019
85cd349
swarm/storage/localstore: correct useGlobalLock in collectGarbage
janos Jan 28, 2019
7fa1ba9
swarm/storage/localstore: fix typos and update comments
janos Jan 29, 2019
ebecd05
swarm/storage/localstore: update writeGCSize comment
janos Jan 29, 2019
f056e86
swarm/storage/localstore: remove global lock option
janos Feb 4, 2019
40432d9
swarm/storage/localstore: add description for gc size counting
janos Feb 5, 2019
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
74 changes: 74 additions & 0 deletions swarm/storage/localstore/accessor.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
// Copyright 2018 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.

package localstore

import (
"context"

"github.com/syndtr/goleveldb/leveldb"

"github.com/ethereum/go-ethereum/swarm/shed"
"github.com/ethereum/go-ethereum/swarm/storage"
)

// Accessor implements ChunkStore to manage data
// in DB with different modes of access and update.
type Accessor struct {
db *DB
mode Mode
}

// Accessor returns a new Accessor with a specified Mode.
func (db *DB) Accessor(mode Mode) *Accessor {
return &Accessor{
mode: mode,
db: db,
}
}

// Put overwrites the underlying DB Put method for the specific mode of update.
func (u *Accessor) Put(ctx context.Context, ch storage.Chunk) error {
Comment thread
zelig marked this conversation as resolved.
Outdated
return u.db.update(ctx, u.mode, chunkToItem(ch))
}

// Get overwrites the underlying DB Get method for the specific mode of access.
Comment thread
zelig marked this conversation as resolved.
Outdated
func (u *Accessor) Get(_ context.Context, addr storage.Address) (chunk storage.Chunk, err error) {
item := addressToItem(addr)
out, err := u.db.access(u.mode, item)
if err != nil {
if err == leveldb.ErrNotFound {
return nil, storage.ErrChunkNotFound
}
return nil, err
}
return storage.NewChunk(out.Address, out.Data), nil
}

// chunkToItem creates new IndexItem with data provided by the Chunk.
func chunkToItem(ch storage.Chunk) shed.IndexItem {
return shed.IndexItem{
Address: ch.Address(),
Data: ch.Data(),
}
}

// addressToItem creates new IndexItem with a provided address.
func addressToItem(addr storage.Address) shed.IndexItem {
return shed.IndexItem{
Address: addr,
}
}
98 changes: 98 additions & 0 deletions swarm/storage/localstore/accessor_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
// Copyright 2018 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.

package localstore

import (
"bytes"
"context"
"testing"

"github.com/ethereum/go-ethereum/swarm/storage"
)

// TestAccessors tests most basic Put and Get functionalities
// for different accessors. This test validates that the chunk
// is retrievable from the database, not if all indexes are set
// correctly.
Comment thread
zelig marked this conversation as resolved.
Outdated
func TestAccessors(t *testing.T) {
db, cleanupFunc := newTestDB(t)
defer cleanupFunc()

for _, m := range []Mode{
ModeSyncing,
ModeUpload,
ModeRequest,
ModeSynced,
ModeAccess,
} {
t.Run(ModeName(m), func(t *testing.T) {
a := db.Accessor(m)

want := generateRandomChunk()

err := a.Put(context.Background(), want)
if err != nil {
t.Fatal(err)
}

got, err := a.Get(context.Background(), want.Address())
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(got.Data(), want.Data()) {
t.Errorf("got chunk data %x, want %x", got.Data(), want.Data())
}
})
}

// Removal mode is a special case as it removes the chunk
// from the database.
t.Run(ModeName(ModeRemoval), func(t *testing.T) {
a := db.Accessor(ModeUpload)

want := generateRandomChunk()

// first put a random chunk to the database
err := a.Put(context.Background(), want)
if err != nil {
t.Fatal(err)
}

got, err := a.Get(context.Background(), want.Address())
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(got.Data(), want.Data()) {
t.Errorf("got chunk data %x, want %x", got.Data(), want.Data())
}

a = db.Accessor(ModeRemoval)

// removal accessor actually removes the chunk on Put
err = a.Put(context.Background(), want)
if err != nil {
t.Fatal(err)
}

// chunk should not be found
wantErr := storage.ErrChunkNotFound
_, err = a.Get(context.Background(), want.Address())
if err != wantErr {
t.Errorf("got error %v, expected %v", err, wantErr)
}
})
}
Loading