roysc/plugeth
1
0
Fork 0
forked from cerc-io/plugeth
Code Pull Requests Activity

swarm/chunk: move chunk related declarations to chunk package (#19170)

Browse Source
This commit is contained in:
Janoš Guljaš 2019-02-26 16:09:32 +01:00 committed by Anton Evangelatov
parent b7e0dec6bd
commit f0233948d2
28 changed files with 325 additions and 286 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

104
swarm/chunk/chunk.go
View File

@ -1,5 +1,109 @@
package chunk
import (
"errors"
"fmt"
"github.com/ethereum/go-ethereum/common"
)
const (
DefaultSize = 4096
MaxPO = 16
AddressLength = 32
)
var (
ErrChunkNotFound = errors.New("chunk not found")
ErrChunkInvalid = errors.New("invalid chunk")
)
type Chunk interface {
Address() Address
Data() []byte
}
type chunk struct {
addr Address
sdata []byte
}
func NewChunk(addr Address, data []byte) *chunk {
return &chunk{
addr: addr,
sdata: data,
}
}
func (c *chunk) Address() Address {
return c.addr
}
func (c *chunk) Data() []byte {
return c.sdata
}
func (self *chunk) String() string {
return fmt.Sprintf("Address: %v Chunksize: %v", self.addr.Log(), len(self.sdata))
}
type Address []byte
var ZeroAddr = Address(common.Hash{}.Bytes())
func (a Address) Hex() string {
return fmt.Sprintf("%064x", []byte(a[:]))
}
func (a Address) Log() string {
if len(a[:]) < 8 {
return fmt.Sprintf("%x", []byte(a[:]))
}
return fmt.Sprintf("%016x", []byte(a[:8]))
}
func (a Address) String() string {
return fmt.Sprintf("%064x", []byte(a))
}
func (a Address) MarshalJSON() (out []byte, err error) {
return []byte(`"` + a.String() + `"`), nil
}
func (a *Address) UnmarshalJSON(value []byte) error {
s := string(value)
*a = make([]byte, 32)
h := common.Hex2Bytes(s[1 : len(s)-1])
copy(*a, h)
return nil
}
// Proximity returns the proximity order of the MSB distance between x and y
//
// The distance metric MSB(x, y) of two equal length byte sequences x an y is the
// value of the binary integer cast of the x^y, ie., x and y bitwise xor-ed.
// the binary cast is big endian: most significant bit first (=MSB).
//
// Proximity(x, y) is a discrete logarithmic scaling of the MSB distance.
// It is defined as the reverse rank of the integer part of the base 2
// logarithm of the distance.
// It is calculated by counting the number of common leading zeros in the (MSB)
// binary representation of the x^y.
//
// (0 farthest, 255 closest, 256 self)
func Proximity(one, other []byte) (ret int) {
b := (MaxPO-1)/8 + 1
if b > len(one) {
b = len(one)
}
m := 8
for i := 0; i < b; i++ {
oxo := one[i] ^ other[i]
for j := 0; j < m; j++ {
if (oxo>>uint8(7-j))&0x01 != 0 {
return i*8 + j
}
}
}
return MaxPO
}

2
swarm/storage/types_test.go → swarm/chunk/proximity_test.go
View File

@ -14,7 +14,7 @@
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package storage
package chunk
import (
"strconv"

6
swarm/storage/chunker.go
View File

@ -25,7 +25,7 @@ import (
"time"
"github.com/ethereum/go-ethereum/metrics"
ch "github.com/ethereum/go-ethereum/swarm/chunk"
"github.com/ethereum/go-ethereum/swarm/chunk"
"github.com/ethereum/go-ethereum/swarm/log"
"github.com/ethereum/go-ethereum/swarm/spancontext"
opentracing "github.com/opentracing/opentracing-go"
@ -127,7 +127,7 @@ type TreeChunker struct {
func TreeJoin(ctx context.Context, addr Address, getter Getter, depth int) *LazyChunkReader {
jp := &JoinerParams{
ChunkerParams: ChunkerParams{
chunkSize: ch.DefaultSize,
chunkSize: chunk.DefaultSize,
hashSize: int64(len(addr)),
},
addr: addr,
@ -147,7 +147,7 @@ func TreeSplit(ctx context.Context, data io.Reader, size int64, putter Putter) (
tsp := &TreeSplitterParams{
SplitterParams: SplitterParams{
ChunkerParams: ChunkerParams{
chunkSize: ch.DefaultSize,
chunkSize: chunk.DefaultSize,
hashSize: putter.RefSize(),
},
reader: data,

4
swarm/storage/common_test.go
View File

@ -29,7 +29,7 @@ import (
"time"
"github.com/ethereum/go-ethereum/log"
ch "github.com/ethereum/go-ethereum/swarm/chunk"
"github.com/ethereum/go-ethereum/swarm/chunk"
"github.com/mattn/go-colorable"
)
@ -94,7 +94,7 @@ func mput(store ChunkStore, n int, f func(i int64) Chunk) (hs []Chunk, err error
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)
defer cancel()
for i := int64(0); i < int64(n); i++ {
chunk := f(ch.DefaultSize)
chunk := f(chunk.DefaultSize)
go func() {
select {
case errc <- store.Put(ctx, chunk):

9
swarm/storage/error.go
View File

@ -16,9 +16,7 @@
package storage
import (
"errors"
)
import "github.com/ethereum/go-ethereum/swarm/chunk"
const (
ErrInit = iota
@ -31,7 +29,8 @@ const (
ErrNotSynced
)
// Errors are the same as the ones in chunk package for backward compatibility.
var (
ErrChunkNotFound = errors.New("chunk not found")
ErrChunkInvalid = errors.New("invalid chunk")
ErrChunkNotFound = chunk.ErrChunkNotFound
ErrChunkInvalid = chunk.ErrChunkNotFound
)

16
swarm/storage/hasherstore.go
View File

@ -21,7 +21,7 @@ import (
"fmt"
"sync/atomic"
ch "github.com/ethereum/go-ethereum/swarm/chunk"
"github.com/ethereum/go-ethereum/swarm/chunk"
"github.com/ethereum/go-ethereum/swarm/storage/encryption"
"golang.org/x/crypto/sha3"
)
@ -156,7 +156,7 @@ func (h *hasherStore) createHash(chunkData ChunkData) Address {
return hasher.Sum(nil)
}
func (h *hasherStore) createChunk(chunkData ChunkData) *chunk {
func (h *hasherStore) createChunk(chunkData ChunkData) Chunk {
hash := h.createHash(chunkData)
chunk := NewChunk(hash, chunkData)
return chunk
@ -189,9 +189,9 @@ func (h *hasherStore) decryptChunkData(chunkData ChunkData, encryptionKey encryp
// removing extra bytes which were just added for padding
length := ChunkData(decryptedSpan).Size()
for length > ch.DefaultSize {
length = length + (ch.DefaultSize - 1)
length = length / ch.DefaultSize
for length > chunk.DefaultSize {
length = length + (chunk.DefaultSize - 1)
length = length / chunk.DefaultSize
length *= uint64(h.refSize)
}
@ -232,14 +232,14 @@ func (h *hasherStore) decrypt(chunkData ChunkData, key encryption.Key) ([]byte,
}
func (h *hasherStore) newSpanEncryption(key encryption.Key) encryption.Encryption {
return encryption.New(key, 0, uint32(ch.DefaultSize/h.refSize), sha3.NewLegacyKeccak256)
return encryption.New(key, 0, uint32(chunk.DefaultSize/h.refSize), sha3.NewLegacyKeccak256)
}
func (h *hasherStore) newDataEncryption(key encryption.Key) encryption.Encryption {
return encryption.New(key, int(ch.DefaultSize), 0, sha3.NewLegacyKeccak256)
return encryption.New(key, int(chunk.DefaultSize), 0, sha3.NewLegacyKeccak256)
}
func (h *hasherStore) storeChunk(ctx context.Context, chunk *chunk) {
func (h *hasherStore) storeChunk(ctx context.Context, chunk Chunk) {
atomic.AddUint64(&h.nrChunks, 1)
go func() {
select {

14
swarm/storage/ldbstore.go
View File

@ -312,7 +312,7 @@ func decodeIndex(data []byte, index *dpaDBIndex) error {
return dec.Decode(index)
}
func decodeData(addr Address, data []byte) (*chunk, error) {
func decodeData(addr Address, data []byte) (Chunk, error) {
return NewChunk(addr, data[32:]), nil
}
@ -502,7 +502,7 @@ func (s *LDBStore) Import(in io.Reader) (int64, error) {
}
// Cleanup iterates over the database and deletes chunks if they pass the `f` condition
func (s *LDBStore) Cleanup(f func(*chunk) bool) {
func (s *LDBStore) Cleanup(f func(Chunk) bool) {
var errorsFound, removed, total int
it := s.db.NewIterator()
@ -551,12 +551,14 @@ func (s *LDBStore) Cleanup(f func(*chunk) bool) {
continue
}
cs := int64(binary.LittleEndian.Uint64(c.sdata[:8]))
log.Trace("chunk", "key", fmt.Sprintf("%x", key), "ck", fmt.Sprintf("%x", ck), "dkey", fmt.Sprintf("%x", datakey), "dataidx", index.Idx, "po", po, "len data", len(data), "len sdata", len(c.sdata), "size", cs)
sdata := c.Data()
cs := int64(binary.LittleEndian.Uint64(sdata[:8]))
log.Trace("chunk", "key", fmt.Sprintf("%x", key), "ck", fmt.Sprintf("%x", ck), "dkey", fmt.Sprintf("%x", datakey), "dataidx", index.Idx, "po", po, "len data", len(data), "len sdata", len(sdata), "size", cs)
// if chunk is to be removed
if f(c) {
log.Warn("chunk for cleanup", "key", fmt.Sprintf("%x", key), "ck", fmt.Sprintf("%x", ck), "dkey", fmt.Sprintf("%x", datakey), "dataidx", index.Idx, "po", po, "len data", len(data), "len sdata", len(c.sdata), "size", cs)
log.Warn("chunk for cleanup", "key", fmt.Sprintf("%x", key), "ck", fmt.Sprintf("%x", ck), "dkey", fmt.Sprintf("%x", datakey), "dataidx", index.Idx, "po", po, "len data", len(data), "len sdata", len(sdata), "size", cs)
s.deleteNow(&index, getIndexKey(key[1:]), po)
removed++
errorsFound++
@ -980,7 +982,7 @@ func (s *LDBStore) Has(_ context.Context, addr Address) bool {
}
// TODO: To conform with other private methods of this object indices should not be updated
func (s *LDBStore) get(addr Address) (chunk *chunk, err error) {
func (s *LDBStore) get(addr Address) (chunk Chunk, err error) {
if s.closed {
return nil, ErrDBClosed
}

8
swarm/storage/ldbstore_test.go
View File

@ -28,7 +28,7 @@ import (
"testing"
"github.com/ethereum/go-ethereum/common"
ch "github.com/ethereum/go-ethereum/swarm/chunk"
"github.com/ethereum/go-ethereum/swarm/chunk"
"github.com/ethereum/go-ethereum/swarm/log"
"github.com/ethereum/go-ethereum/swarm/storage/mock/mem"
ldberrors "github.com/syndtr/goleveldb/leveldb/errors"
@ -103,7 +103,7 @@ func TestMarkAccessed(t *testing.T) {
t.Fatalf("init dbStore failed: %v", err)
}
h := GenerateRandomChunk(ch.DefaultSize)
h := GenerateRandomChunk(chunk.DefaultSize)
db.Put(context.Background(), h)
@ -201,7 +201,7 @@ func testIterator(t *testing.T, mock bool) {
t.Fatalf("init dbStore failed: %v", err)
}
chunks := GenerateRandomChunks(ch.DefaultSize, chunkcount)
chunks := GenerateRandomChunks(chunk.DefaultSize, chunkcount)
for i = 0; i < len(chunks); i++ {
chunkkeys[i] = chunks[i].Address()
@ -468,7 +468,7 @@ func testLDBStoreRemoveThenCollectGarbage(t *testing.T) {
// put capacity count number of chunks
chunks := make([]Chunk, n)
for i := 0; i < n; i++ {
c := GenerateRandomChunk(ch.DefaultSize)
c := GenerateRandomChunk(chunk.DefaultSize)
chunks[i] = c
log.Trace("generate random chunk", "idx", i, "chunk", c)
}

2
swarm/storage/localstore.go
View File

@ -241,7 +241,7 @@ func (ls *LocalStore) Migrate() error {
func (ls *LocalStore) migrateFromNoneToPurity() {
// delete chunks that are not valid, i.e. chunks that do not pass
// any of the ls.Validators
ls.DbStore.Cleanup(func(c *chunk) bool {
ls.DbStore.Cleanup(func(c Chunk) bool {
return !ls.isValid(c)
})
}

47
swarm/storage/localstore/gc_test.go
View File

@ -23,7 +23,7 @@ import (
"testing"
"time"
"github.com/ethereum/go-ethereum/swarm/storage"
"github.com/ethereum/go-ethereum/swarm/chunk"
)
// TestDB_collectGarbageWorker tests garbage collection runs
@ -64,11 +64,11 @@ func testDB_collectGarbageWorker(t *testing.T) {
uploader := db.NewPutter(ModePutUpload)
syncer := db.NewSetter(ModeSetSync)
addrs := make([]storage.Address, 0)
addrs := make([]chunk.Address, 0)
// upload random chunks
for i := 0; i < chunkCount; i++ {
chunk := generateRandomChunk()
chunk := generateTestRandomChunk()
err := uploader.Put(chunk)
if err != nil {
@ -106,8 +106,8 @@ func testDB_collectGarbageWorker(t *testing.T) {
// the first synced chunk should be removed
t.Run("get the first synced chunk", func(t *testing.T) {
_, err := db.NewGetter(ModeGetRequest).Get(addrs[0])
if err != storage.ErrChunkNotFound {
t.Errorf("got error %v, want %v", err, storage.ErrChunkNotFound)
if err != chunk.ErrChunkNotFound {
t.Errorf("got error %v, want %v", err, chunk.ErrChunkNotFound)
}
})
@ -137,11 +137,11 @@ func TestDB_collectGarbageWorker_withRequests(t *testing.T) {
testHookCollectGarbageChan <- collectedCount
})()
addrs := make([]storage.Address, 0)
addrs := make([]chunk.Address, 0)
// upload random chunks just up to the capacity
for i := 0; i < int(db.capacity)-1; i++ {
chunk := generateRandomChunk()
chunk := generateTestRandomChunk()
err := uploader.Put(chunk)
if err != nil {
@ -156,6 +156,14 @@ func TestDB_collectGarbageWorker_withRequests(t *testing.T) {
addrs = append(addrs, chunk.Address())
}
// set update gc test hook to signal when
// update gc goroutine is done by closing
// testHookUpdateGCChan channel
testHookUpdateGCChan := make(chan struct{})
resetTestHookUpdateGC := setTestHookUpdateGC(func() {
close(testHookUpdateGCChan)
})
// request the latest synced chunk
// to prioritize it in the gc index
// not to be collected
@ -164,18 +172,29 @@ func TestDB_collectGarbageWorker_withRequests(t *testing.T) {
t.Fatal(err)
}
// wait for update gc goroutine to finish for garbage
// collector to be correctly triggered after the last upload
select {
case <-testHookUpdateGCChan:
case <-time.After(10 * time.Second):
t.Fatal("updateGC was not called after getting chunk with ModeGetRequest")
}
// no need to wait for update gc hook anymore
resetTestHookUpdateGC()
// upload and sync another chunk to trigger
// garbage collection
chunk := generateRandomChunk()
err = uploader.Put(chunk)
ch := generateTestRandomChunk()
err = uploader.Put(ch)
if err != nil {
t.Fatal(err)
}
err = syncer.Set(chunk.Address())
err = syncer.Set(ch.Address())
if err != nil {
t.Fatal(err)
}
addrs = append(addrs, chunk.Address())
addrs = append(addrs, ch.Address())
// wait for garbage collection
@ -217,8 +236,8 @@ func TestDB_collectGarbageWorker_withRequests(t *testing.T) {
// the second synced chunk should be removed
t.Run("get gc-ed chunk", func(t *testing.T) {
_, err := db.NewGetter(ModeGetRequest).Get(addrs[1])
if err != storage.ErrChunkNotFound {
t.Errorf("got error %v, want %v", err, storage.ErrChunkNotFound)
if err != chunk.ErrChunkNotFound {
t.Errorf("got error %v, want %v", err, chunk.ErrChunkNotFound)
}
})
@ -254,7 +273,7 @@ func TestDB_gcSize(t *testing.T) {
count := 100
for i := 0; i < count; i++ {
chunk := generateRandomChunk()
chunk := generateTestRandomChunk()
err := uploader.Put(chunk)
if err != nil {

10
swarm/storage/localstore/index_test.go
View File

@ -21,7 +21,7 @@ import (
"math/rand"
"testing"
"github.com/ethereum/go-ethereum/swarm/storage"
"github.com/ethereum/go-ethereum/swarm/chunk"
)
// TestDB_pullIndex validates the ordering of keys in pull index.
@ -43,7 +43,7 @@ func TestDB_pullIndex(t *testing.T) {
// upload random chunks
for i := 0; i < chunkCount; i++ {
chunk := generateRandomChunk()
chunk := generateTestRandomChunk()
err := uploader.Put(chunk)
if err != nil {
@ -62,8 +62,8 @@ func TestDB_pullIndex(t *testing.T) {
}
testItemsOrder(t, db.pullIndex, chunks, func(i, j int) (less bool) {
poi := storage.Proximity(db.baseKey, chunks[i].Address())
poj := storage.Proximity(db.baseKey, chunks[j].Address())
poi := chunk.Proximity(db.baseKey, chunks[i].Address())
poj := chunk.Proximity(db.baseKey, chunks[j].Address())
if poi < poj {
return true
}
@ -95,7 +95,7 @@ func TestDB_gcIndex(t *testing.T) {
// upload random chunks
for i := 0; i < chunkCount; i++ {
chunk := generateRandomChunk()
chunk := generateTestRandomChunk()
err := uploader.Put(chunk)
if err != nil {

12
swarm/storage/localstore/localstore.go
View File

@ -24,8 +24,8 @@ import (
"time"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/swarm/chunk"
"github.com/ethereum/go-ethereum/swarm/shed"
"github.com/ethereum/go-ethereum/swarm/storage"
"github.com/ethereum/go-ethereum/swarm/storage/mock"
)
@ -392,8 +392,8 @@ func (db *DB) Close() (err error) {
// po computes the proximity order between the address
// and database base key.
func (db *DB) po(addr storage.Address) (bin uint8) {
return uint8(storage.Proximity(db.baseKey, addr))
func (db *DB) po(addr chunk.Address) (bin uint8) {
return uint8(chunk.Proximity(db.baseKey, addr))
}
var (
@ -409,7 +409,7 @@ var (
// If the address is locked this function will check it
// in a for loop for addressLockTimeout time, after which
// it will return ErrAddressLockTimeout error.
func (db *DB) lockAddr(addr storage.Address) (unlock func(), err error) {
func (db *DB) lockAddr(addr chunk.Address) (unlock func(), err error) {
start := time.Now()
lockKey := hex.EncodeToString(addr)
for {
@ -426,7 +426,7 @@ func (db *DB) lockAddr(addr storage.Address) (unlock func(), err error) {
}
// chunkToItem creates new Item with data provided by the Chunk.
func chunkToItem(ch storage.Chunk) shed.Item {
func chunkToItem(ch chunk.Chunk) shed.Item {
return shed.Item{
Address: ch.Address(),
Data: ch.Data(),
@ -434,7 +434,7 @@ func chunkToItem(ch storage.Chunk) shed.Item {
}
// addressToItem creates new Item with a provided address.
func addressToItem(addr storage.Address) shed.Item {
func addressToItem(addr chunk.Address) shed.Item {
return shed.Item{
Address: addr,
}

57
swarm/storage/localstore/localstore_test.go
View File

@ -29,9 +29,8 @@ import (
"testing"
"time"
ch "github.com/ethereum/go-ethereum/swarm/chunk"
"github.com/ethereum/go-ethereum/swarm/chunk"
"github.com/ethereum/go-ethereum/swarm/shed"
"github.com/ethereum/go-ethereum/swarm/storage"
"github.com/syndtr/goleveldb/leveldb"
)
@ -61,7 +60,7 @@ func TestDB(t *testing.T) {
db, cleanupFunc := newTestDB(t, nil)
defer cleanupFunc()
chunk := generateRandomChunk()
chunk := generateTestRandomChunk()
err := db.NewPutter(ModePutUpload).Put(chunk)
if err != nil {
@ -115,7 +114,7 @@ func TestDB_updateGCSem(t *testing.T) {
db, cleanupFunc := newTestDB(t, nil)
defer cleanupFunc()
chunk := generateRandomChunk()
chunk := generateTestRandomChunk()
err := db.NewPutter(ModePutUpload).Put(chunk)
if err != nil {
@ -188,7 +187,7 @@ func BenchmarkNew(b *testing.B) {
uploader := db.NewPutter(ModePutUpload)
syncer := db.NewSetter(ModeSetSync)
for i := 0; i < count; i++ {
chunk := generateFakeRandomChunk()
chunk := generateTestRandomChunk()
err := uploader.Put(chunk)
if err != nil {
b.Fatal(err)
@ -251,53 +250,47 @@ func newTestDB(t testing.TB, o *Options) (db *DB, cleanupFunc func()) {
return db, cleanupFunc
}
// generateRandomChunk generates a valid Chunk with
// data size of default chunk size.
func generateRandomChunk() storage.Chunk {
return storage.GenerateRandomChunk(ch.DefaultSize)
}
func init() {
// needed for generateFakeRandomChunk
// needed for generateTestRandomChunk
rand.Seed(time.Now().UnixNano())
}
// generateFakeRandomChunk generates a Chunk that is not
// generateTestRandomChunk generates a Chunk that is not
// valid, but it contains a random key and a random value.
// This function is faster then storage.GenerateRandomChunk
// This function is faster then storage.generateTestRandomChunk
// which generates a valid chunk.
// Some tests in this package do not need valid chunks, just
// random data, and their execution time can be decreased
// using this function.
func generateFakeRandomChunk() storage.Chunk {
data := make([]byte, ch.DefaultSize)
func generateTestRandomChunk() chunk.Chunk {
data := make([]byte, chunk.DefaultSize)
rand.Read(data)
key := make([]byte, 32)
rand.Read(key)
return storage.NewChunk(key, data)
return chunk.NewChunk(key, data)
}
// TestGenerateFakeRandomChunk validates that
// generateFakeRandomChunk returns random data by comparing
// TestGenerateTestRandomChunk validates that
// generateTestRandomChunk returns random data by comparing
// two generated chunks.
func TestGenerateFakeRandomChunk(t *testing.T) {
c1 := generateFakeRandomChunk()
c2 := generateFakeRandomChunk()
func TestGenerateTestRandomChunk(t *testing.T) {
c1 := generateTestRandomChunk()
c2 := generateTestRandomChunk()
addrLen := len(c1.Address())
if addrLen != 32 {
t.Errorf("first chunk address length %v, want %v", addrLen, 32)
}
dataLen := len(c1.Data())
if dataLen != ch.DefaultSize {
t.Errorf("first chunk data length %v, want %v", dataLen, ch.DefaultSize)
if dataLen != chunk.DefaultSize {
t.Errorf("first chunk data length %v, want %v", dataLen, chunk.DefaultSize)
}
addrLen = len(c2.Address())
if addrLen != 32 {
t.Errorf("second chunk address length %v, want %v", addrLen, 32)
}
dataLen = len(c2.Data())
if dataLen != ch.DefaultSize {
t.Errorf("second chunk data length %v, want %v", dataLen, ch.DefaultSize)
if dataLen != chunk.DefaultSize {
t.Errorf("second chunk data length %v, want %v", dataLen, chunk.DefaultSize)
}
if bytes.Equal(c1.Address(), c2.Address()) {
t.Error("fake chunks addresses do not differ")
@ -309,7 +302,7 @@ func TestGenerateFakeRandomChunk(t *testing.T) {
// newRetrieveIndexesTest returns a test function that validates if the right
// chunk values are in the retrieval indexes.
func newRetrieveIndexesTest(db *DB, chunk storage.Chunk, storeTimestamp, accessTimestamp int64) func(t *testing.T) {
func newRetrieveIndexesTest(db *DB, chunk chunk.Chunk, storeTimestamp, accessTimestamp int64) func(t *testing.T) {
return func(t *testing.T) {
item, err := db.retrievalDataIndex.Get(addressToItem(chunk.Address()))
if err != nil {
@ -328,7 +321,7 @@ func newRetrieveIndexesTest(db *DB, chunk storage.Chunk, storeTimestamp, accessT
// newRetrieveIndexesTestWithAccess returns a test function that validates if the right
// chunk values are in the retrieval indexes when access time must be stored.
func newRetrieveIndexesTestWithAccess(db *DB, chunk storage.Chunk, storeTimestamp, accessTimestamp int64) func(t *testing.T) {
func newRetrieveIndexesTestWithAccess(db *DB, chunk chunk.Chunk, storeTimestamp, accessTimestamp int64) func(t *testing.T) {
return func(t *testing.T) {
item, err := db.retrievalDataIndex.Get(addressToItem(chunk.Address()))
if err != nil {
@ -348,7 +341,7 @@ func newRetrieveIndexesTestWithAccess(db *DB, chunk storage.Chunk, storeTimestam
// newPullIndexTest returns a test function that validates if the right
// chunk values are in the pull index.
func newPullIndexTest(db *DB, chunk storage.Chunk, storeTimestamp int64, wantError error) func(t *testing.T) {
func newPullIndexTest(db *DB, chunk chunk.Chunk, storeTimestamp int64, wantError error) func(t *testing.T) {
return func(t *testing.T) {
item, err := db.pullIndex.Get(shed.Item{
Address: chunk.Address(),
@ -365,7 +358,7 @@ func newPullIndexTest(db *DB, chunk storage.Chunk, storeTimestamp int64, wantErr
// newPushIndexTest returns a test function that validates if the right
// chunk values are in the push index.
func newPushIndexTest(db *DB, chunk storage.Chunk, storeTimestamp int64, wantError error) func(t *testing.T) {
func newPushIndexTest(db *DB, chunk chunk.Chunk, storeTimestamp int64, wantError error) func(t *testing.T) {
return func(t *testing.T) {
item, err := db.pushIndex.Get(shed.Item{
Address: chunk.Address(),
@ -382,7 +375,7 @@ func newPushIndexTest(db *DB, chunk storage.Chunk, storeTimestamp int64, wantErr
// newGCIndexTest returns a test function that validates if the right
// chunk values are in the push index.
func newGCIndexTest(db *DB, chunk storage.Chunk, storeTimestamp, accessTimestamp int64) func(t *testing.T) {
func newGCIndexTest(db *DB, chunk chunk.Chunk, storeTimestamp, accessTimestamp int64) func(t *testing.T) {
return func(t *testing.T) {
item, err := db.gcIndex.Get(shed.Item{
Address: chunk.Address(),
@ -436,7 +429,7 @@ func newIndexGCSizeTest(db *DB) func(t *testing.T) {
// testIndexChunk embeds storageChunk with additional data that is stored
// in database. It is used for index values validations.
type testIndexChunk struct {
storage.Chunk
chunk.Chunk
storeTimestamp int64
}

12
swarm/storage/localstore/mode_get.go
View File

@ -18,8 +18,8 @@ package localstore
import (
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/swarm/chunk"
"github.com/ethereum/go-ethereum/swarm/shed"
"github.com/ethereum/go-ethereum/swarm/storage"
"github.com/syndtr/goleveldb/leveldb"
)
@ -51,23 +51,23 @@ func (db *DB) NewGetter(mode ModeGet) *Getter {
}
// Get returns a chunk from the database. If the chunk is
// not found storage.ErrChunkNotFound will be returned.
// not found chunk.ErrChunkNotFound will be returned.
// All required indexes will be updated required by the
// Getter Mode.
func (g *Getter) Get(addr storage.Address) (chunk storage.Chunk, err error) {
func (g *Getter) Get(addr chunk.Address) (ch chunk.Chunk, err error) {
out, err := g.db.get(g.mode, addr)
if err != nil {
if err == leveldb.ErrNotFound {
return nil, storage.ErrChunkNotFound
return nil, chunk.ErrChunkNotFound
}
return nil, err
}
return storage.NewChunk(out.Address, out.Data), nil
return chunk.NewChunk(out.Address, out.Data), nil
}
// get returns Item from the retrieval index
// and updates other indexes.
func (db *DB) get(mode ModeGet, addr storage.Address) (out shed.Item, err error) {
func (db *DB) get(mode ModeGet, addr chunk.Address) (out shed.Item, err error) {
item := addressToItem(addr)
out, err = db.retrievalDataIndex.Get(item)

4
swarm/storage/localstore/mode_get_test.go
View File

@ -32,7 +32,7 @@ func TestModeGetRequest(t *testing.T) {
return uploadTimestamp
})()
chunk := generateRandomChunk()
chunk := generateTestRandomChunk()
err := db.NewPutter(ModePutUpload).Put(chunk)
if err != nil {
@ -146,7 +146,7 @@ func TestModeGetSync(t *testing.T) {
return uploadTimestamp
})()
chunk := generateRandomChunk()
chunk := generateTestRandomChunk()
err := db.NewPutter(ModePutUpload).Put(chunk)
if err != nil {

4
swarm/storage/localstore/mode_put.go
View File

@ -17,8 +17,8 @@
package localstore
import (
"github.com/ethereum/go-ethereum/swarm/chunk"
"github.com/ethereum/go-ethereum/swarm/shed"
"github.com/ethereum/go-ethereum/swarm/storage"
"github.com/syndtr/goleveldb/leveldb"
)
@ -53,7 +53,7 @@ func (db *DB) NewPutter(mode ModePut) *Putter {
// Put stores the Chunk to database and depending
// on the Putter mode, it updates required indexes.
func (p *Putter) Put(ch storage.Chunk) (err error) {
func (p *Putter) Put(ch chunk.Chunk) (err error) {
return p.db.put(p.mode, chunkToItem(ch))
}

18
swarm/storage/localstore/mode_put_test.go
View File

@ -23,7 +23,7 @@ import (
"testing"
"time"
"github.com/ethereum/go-ethereum/swarm/storage"
"github.com/ethereum/go-ethereum/swarm/chunk"
)
// TestModePutRequest validates ModePutRequest index values on the provided DB.
@ -33,7 +33,7 @@ func TestModePutRequest(t *testing.T) {
putter := db.NewPutter(ModePutRequest)
chunk := generateRandomChunk()
chunk := generateTestRandomChunk()
// keep the record when the chunk is stored
var storeTimestamp int64
@ -87,7 +87,7 @@ func TestModePutSync(t *testing.T) {
return wantTimestamp
})()
chunk := generateRandomChunk()
chunk := generateTestRandomChunk()
err := db.NewPutter(ModePutSync).Put(chunk)
if err != nil {
@ -109,7 +109,7 @@ func TestModePutUpload(t *testing.T) {
return wantTimestamp
})()
chunk := generateRandomChunk()
chunk := generateTestRandomChunk()
err := db.NewPutter(ModePutUpload).Put(chunk)
if err != nil {
@ -132,7 +132,7 @@ func TestModePutUpload_parallel(t *testing.T) {
chunkCount := 1000
workerCount := 100
chunkChan := make(chan storage.Chunk)
chunkChan := make(chan chunk.Chunk)
errChan := make(chan error)
doneChan := make(chan struct{})
defer close(doneChan)
@ -159,13 +159,13 @@ func TestModePutUpload_parallel(t *testing.T) {
}(i)
}
chunks := make([]storage.Chunk, 0)
chunks := make([]chunk.Chunk, 0)
var chunksMu sync.Mutex
// send chunks to workers
go func() {
for i := 0; i < chunkCount; i++ {
chunk := generateRandomChunk()
chunk := generateTestRandomChunk()
select {
case chunkChan <- chunk:
case <-doneChan:
@ -271,9 +271,9 @@ func benchmarkPutUpload(b *testing.B, o *Options, count, maxParallelUploads int)
defer cleanupFunc()
uploader := db.NewPutter(ModePutUpload)
chunks := make([]storage.Chunk, count)
chunks := make([]chunk.Chunk, count)
for i := 0; i < count; i++ {
chunks[i] = generateFakeRandomChunk()
chunks[i] = generateTestRandomChunk()
}
errs := make(chan error)
b.StartTimer()

6
swarm/storage/localstore/mode_set.go
View File

@ -17,7 +17,7 @@
package localstore
import (
"github.com/ethereum/go-ethereum/swarm/storage"
"github.com/ethereum/go-ethereum/swarm/chunk"
"github.com/syndtr/goleveldb/leveldb"
)
@ -53,7 +53,7 @@ func (db *DB) NewSetter(mode ModeSet) *Setter {
// Set updates database indexes for a specific
// chunk represented by the address.
func (s *Setter) Set(addr storage.Address) (err error) {
func (s *Setter) Set(addr chunk.Address) (err error) {
return s.db.set(s.mode, addr)
}
@ -61,7 +61,7 @@ func (s *Setter) Set(addr storage.Address) (err error) {
// chunk represented by the address.
// It acquires lockAddr to protect two calls
// of this function for the same address in parallel.
func (db *DB) set(mode ModeSet, addr storage.Address) (err error) {
func (db *DB) set(mode ModeSet, addr chunk.Address) (err error) {
// protect parallel updates
unlock, err := db.lockAddr(addr)
if err != nil {

6
swarm/storage/localstore/mode_set_test.go
View File

@ -28,7 +28,7 @@ func TestModeSetAccess(t *testing.T) {
db, cleanupFunc := newTestDB(t, nil)
defer cleanupFunc()
chunk := generateRandomChunk()
chunk := generateTestRandomChunk()
wantTimestamp := time.Now().UTC().UnixNano()
defer setNow(func() (t int64) {
@ -56,7 +56,7 @@ func TestModeSetSync(t *testing.T) {
db, cleanupFunc := newTestDB(t, nil)
defer cleanupFunc()
chunk := generateRandomChunk()
chunk := generateTestRandomChunk()
wantTimestamp := time.Now().UTC().UnixNano()
defer setNow(func() (t int64) {
@ -89,7 +89,7 @@ func TestModeSetRemove(t *testing.T) {
db, cleanupFunc := newTestDB(t, nil)
defer cleanupFunc()
chunk := generateRandomChunk()
chunk := generateTestRandomChunk()
err := db.NewPutter(ModePutUpload).Put(chunk)
if err != nil {

10
swarm/storage/localstore/retrieval_index_test.go
View File

@ -20,7 +20,7 @@ import (
"strconv"
"testing"
"github.com/ethereum/go-ethereum/swarm/storage"
"github.com/ethereum/go-ethereum/swarm/chunk"
)
// BenchmarkRetrievalIndexes uploads a number of chunks in order to measure
@ -64,9 +64,9 @@ func benchmarkRetrievalIndexes(b *testing.B, o *Options, count int) {
uploader := db.NewPutter(ModePutUpload)
syncer := db.NewSetter(ModeSetSync)
requester := db.NewGetter(ModeGetRequest)
addrs := make([]storage.Address, count)
addrs := make([]chunk.Address, count)
for i := 0; i < count; i++ {
chunk := generateFakeRandomChunk()
chunk := generateTestRandomChunk()
err := uploader.Put(chunk)
if err != nil {
b.Fatal(err)
@ -134,9 +134,9 @@ func benchmarkUpload(b *testing.B, o *Options, count int) {
db, cleanupFunc := newTestDB(b, o)
defer cleanupFunc()
uploader := db.NewPutter(ModePutUpload)
chunks := make([]storage.Chunk, count)
chunks := make([]chunk.Chunk, count)
for i := 0; i < count; i++ {
chunk := generateFakeRandomChunk()
chunk := generateTestRandomChunk()
chunks[i] = chunk
}
b.StartTimer()

4
swarm/storage/localstore/subscription_pull.go
View File

@ -24,8 +24,8 @@ import (
"sync"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/swarm/chunk"
"github.com/ethereum/go-ethereum/swarm/shed"
"github.com/ethereum/go-ethereum/swarm/storage"
)
// SubscribePull returns a channel that provides chunk addresses and stored times from pull syncing index.
@ -161,7 +161,7 @@ func (db *DB) SubscribePull(ctx context.Context, bin uint8, since, until *ChunkD
// ChunkDescriptor holds information required for Pull syncing. This struct
// is provided by subscribing to pull index.
type ChunkDescriptor struct {
Address storage.Address
Address chunk.Address
StoreTimestamp int64
}

72
swarm/storage/localstore/subscription_pull_test.go
View File

@ -24,7 +24,7 @@ import (
"testing"
"time"
"github.com/ethereum/go-ethereum/swarm/storage"
"github.com/ethereum/go-ethereum/swarm/chunk"
)
// TestDB_SubscribePull uploads some chunks before and after
@ -37,7 +37,7 @@ func TestDB_SubscribePull(t *testing.T) {
uploader := db.NewPutter(ModePutUpload)
addrs := make(map[uint8][]storage.Address)
addrs := make(map[uint8][]chunk.Address)
var addrsMu sync.Mutex
var wantedChunksCount int
@ -53,7 +53,7 @@ func TestDB_SubscribePull(t *testing.T) {
// to validate the number of addresses received by the subscription
errChan := make(chan error)
for bin := uint8(0); bin <= uint8(storage.MaxPO); bin++ {
for bin := uint8(0); bin <= uint8(chunk.MaxPO); bin++ {
ch, stop := db.SubscribePull(ctx, bin, nil, nil)
defer stop()
@ -84,7 +84,7 @@ func TestDB_SubscribePull_multiple(t *testing.T) {
uploader := db.NewPutter(ModePutUpload)
addrs := make(map[uint8][]storage.Address)
addrs := make(map[uint8][]chunk.Address)
var addrsMu sync.Mutex
var wantedChunksCount int
@ -105,7 +105,7 @@ func TestDB_SubscribePull_multiple(t *testing.T) {
// start a number of subscriptions
// that all of them will write every address error to errChan
for j := 0; j < subsCount; j++ {
for bin := uint8(0); bin <= uint8(storage.MaxPO); bin++ {
for bin := uint8(0); bin <= uint8(chunk.MaxPO); bin++ {
ch, stop := db.SubscribePull(ctx, bin, nil, nil)
defer stop()
@ -137,7 +137,7 @@ func TestDB_SubscribePull_since(t *testing.T) {
uploader := db.NewPutter(ModePutUpload)
addrs := make(map[uint8][]storage.Address)
addrs := make(map[uint8][]chunk.Address)
var addrsMu sync.Mutex
var wantedChunksCount int
@ -156,20 +156,20 @@ func TestDB_SubscribePull_since(t *testing.T) {
last = make(map[uint8]ChunkDescriptor)
for i := 0; i < count; i++ {
chunk := generateRandomChunk()
ch := generateTestRandomChunk()
err := uploader.Put(chunk)
err := uploader.Put(ch)
if err != nil {
t.Fatal(err)
}
bin := db.po(chunk.Address())
bin := db.po(ch.Address())
if _, ok := addrs[bin]; !ok {
addrs[bin] = make([]storage.Address, 0)
addrs[bin] = make([]chunk.Address, 0)
}
if wanted {
addrs[bin] = append(addrs[bin], chunk.Address())
addrs[bin] = append(addrs[bin], ch.Address())
wantedChunksCount++
}
@ -178,7 +178,7 @@ func TestDB_SubscribePull_since(t *testing.T) {
lastTimestampMu.RUnlock()
last[bin] = ChunkDescriptor{
Address: chunk.Address(),
Address: ch.Address(),
StoreTimestamp: storeTimestamp,
}
}
@ -199,7 +199,7 @@ func TestDB_SubscribePull_since(t *testing.T) {
// to validate the number of addresses received by the subscription
errChan := make(chan error)
for bin := uint8(0); bin <= uint8(storage.MaxPO); bin++ {
for bin := uint8(0); bin <= uint8(chunk.MaxPO); bin++ {
var since *ChunkDescriptor
if c, ok := last[bin]; ok {
since = &c
@ -228,7 +228,7 @@ func TestDB_SubscribePull_until(t *testing.T) {
uploader := db.NewPutter(ModePutUpload)
addrs := make(map[uint8][]storage.Address)
addrs := make(map[uint8][]chunk.Address)
var addrsMu sync.Mutex
var wantedChunksCount int
@ -247,20 +247,20 @@ func TestDB_SubscribePull_until(t *testing.T) {
last = make(map[uint8]ChunkDescriptor)
for i := 0; i < count; i++ {
chunk := generateRandomChunk()
ch := generateTestRandomChunk()
err := uploader.Put(chunk)
err := uploader.Put(ch)
if err != nil {
t.Fatal(err)
}
bin := db.po(chunk.Address())
bin := db.po(ch.Address())
if _, ok := addrs[bin]; !ok {
addrs[bin] = make([]storage.Address, 0)
addrs[bin] = make([]chunk.Address, 0)
}
if wanted {
addrs[bin] = append(addrs[bin], chunk.Address())
addrs[bin] = append(addrs[bin], ch.Address())
wantedChunksCount++
}
@ -269,7 +269,7 @@ func TestDB_SubscribePull_until(t *testing.T) {
lastTimestampMu.RUnlock()
last[bin] = ChunkDescriptor{
Address: chunk.Address(),
Address: ch.Address(),
StoreTimestamp: storeTimestamp,
}
}
@ -290,7 +290,7 @@ func TestDB_SubscribePull_until(t *testing.T) {
// to validate the number of addresses received by the subscription
errChan := make(chan error)
for bin := uint8(0); bin <= uint8(storage.MaxPO); bin++ {
for bin := uint8(0); bin <= uint8(chunk.MaxPO); bin++ {
until, ok := last[bin]
if !ok {
continue
@ -318,7 +318,7 @@ func TestDB_SubscribePull_sinceAndUntil(t *testing.T) {
uploader := db.NewPutter(ModePutUpload)
addrs := make(map[uint8][]storage.Address)
addrs := make(map[uint8][]chunk.Address)
var addrsMu sync.Mutex
var wantedChunksCount int
@ -337,20 +337,20 @@ func TestDB_SubscribePull_sinceAndUntil(t *testing.T) {
last = make(map[uint8]ChunkDescriptor)
for i := 0; i < count; i++ {
chunk := generateRandomChunk()
ch := generateTestRandomChunk()
err := uploader.Put(chunk)
err := uploader.Put(ch)
if err != nil {
t.Fatal(err)
}
bin := db.po(chunk.Address())
bin := db.po(ch.Address())
if _, ok := addrs[bin]; !ok {
addrs[bin] = make([]storage.Address, 0)
addrs[bin] = make([]chunk.Address, 0)
}
if wanted {
addrs[bin] = append(addrs[bin], chunk.Address())
addrs[bin] = append(addrs[bin], ch.Address())
wantedChunksCount++
}
@ -359,7 +359,7 @@ func TestDB_SubscribePull_sinceAndUntil(t *testing.T) {
lastTimestampMu.RUnlock()
last[bin] = ChunkDescriptor{
Address: chunk.Address(),
Address: ch.Address(),
StoreTimestamp: storeTimestamp,
}
}
@ -386,7 +386,7 @@ func TestDB_SubscribePull_sinceAndUntil(t *testing.T) {
// to validate the number of addresses received by the subscription
errChan := make(chan error)
for bin := uint8(0); bin <= uint8(storage.MaxPO); bin++ {
for bin := uint8(0); bin <= uint8(chunk.MaxPO); bin++ {
var since *ChunkDescriptor
if c, ok := upload1[bin]; ok {
since = &c
@ -412,23 +412,23 @@ func TestDB_SubscribePull_sinceAndUntil(t *testing.T) {
// uploadRandomChunksBin uploads random chunks to database and adds them to
// the map of addresses ber bin.
func uploadRandomChunksBin(t *testing.T, db *DB, uploader *Putter, addrs map[uint8][]storage.Address, addrsMu *sync.Mutex, wantedChunksCount *int, count int) {
func uploadRandomChunksBin(t *testing.T, db *DB, uploader *Putter, addrs map[uint8][]chunk.Address, addrsMu *sync.Mutex, wantedChunksCount *int, count int) {
addrsMu.Lock()
defer addrsMu.Unlock()
for i := 0; i < count; i++ {
chunk := generateRandomChunk()
ch := generateTestRandomChunk()
err := uploader.Put(chunk)
err := uploader.Put(ch)
if err != nil {
t.Fatal(err)
}
bin := db.po(chunk.Address())
bin := db.po(ch.Address())
if _, ok := addrs[bin]; !ok {
addrs[bin] = make([]storage.Address, 0)
addrs[bin] = make([]chunk.Address, 0)
}
addrs[bin] = append(addrs[bin], chunk.Address())
addrs[bin] = append(addrs[bin], ch.Address())
*wantedChunksCount++
}
@ -437,7 +437,7 @@ func uploadRandomChunksBin(t *testing.T, db *DB, uploader *Putter, addrs map[uin
// readPullSubscriptionBin is a helper function that reads all ChunkDescriptors from a channel and
// sends error to errChan, even if it is nil, to count the number of ChunkDescriptors
// returned by the channel.
func readPullSubscriptionBin(ctx context.Context, bin uint8, ch <-chan ChunkDescriptor, addrs map[uint8][]storage.Address, addrsMu *sync.Mutex, errChan chan error) {
func readPullSubscriptionBin(ctx context.Context, bin uint8, ch <-chan ChunkDescriptor, addrs map[uint8][]chunk.Address, addrsMu *sync.Mutex, errChan chan error) {
var i int // address index
for {
select {

8
swarm/storage/localstore/subscription_push.go
View File

@ -21,16 +21,16 @@ import (
"sync"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/swarm/chunk"
"github.com/ethereum/go-ethereum/swarm/shed"
"github.com/ethereum/go-ethereum/swarm/storage"
)
// SubscribePush returns a channel that provides storage chunks with ordering from push syncing index.
// Returned stop function will terminate current and further iterations, and also it will close
// the returned channel without any errors. Make sure that you check the second returned parameter
// from the channel to stop iteration when its value is false.
func (db *DB) SubscribePush(ctx context.Context) (c <-chan storage.Chunk, stop func()) {
chunks := make(chan storage.Chunk)
func (db *DB) SubscribePush(ctx context.Context) (c <-chan chunk.Chunk, stop func()) {
chunks := make(chan chunk.Chunk)
trigger := make(chan struct{}, 1)
db.pushTriggersMu.Lock()
@ -65,7 +65,7 @@ func (db *DB) SubscribePush(ctx context.Context) (c <-chan storage.Chunk, stop f
}
select {
case chunks <- storage.NewChunk(dataItem.Address, dataItem.Data):
case chunks <- chunk.NewChunk(dataItem.Address, dataItem.Data):
// set next iteration start item
// when its chunk is successfully sent to channel
sinceItem = &item

10
swarm/storage/localstore/subscription_push_test.go
View File

@ -24,7 +24,7 @@ import (
"testing"
"time"
"github.com/ethereum/go-ethereum/swarm/storage"
"github.com/ethereum/go-ethereum/swarm/chunk"
)
// TestDB_SubscribePush uploads some chunks before and after
@ -36,7 +36,7 @@ func TestDB_SubscribePush(t *testing.T) {
uploader := db.NewPutter(ModePutUpload)
chunks := make([]storage.Chunk, 0)
chunks := make([]chunk.Chunk, 0)
var chunksMu sync.Mutex
uploadRandomChunks := func(count int) {
@ -44,7 +44,7 @@ func TestDB_SubscribePush(t *testing.T) {
defer chunksMu.Unlock()
for i := 0; i < count; i++ {
chunk := generateRandomChunk()
chunk := generateTestRandomChunk()
err := uploader.Put(chunk)
if err != nil {
@ -124,7 +124,7 @@ func TestDB_SubscribePush_multiple(t *testing.T) {
uploader := db.NewPutter(ModePutUpload)
addrs := make([]storage.Address, 0)
addrs := make([]chunk.Address, 0)
var addrsMu sync.Mutex
uploadRandomChunks := func(count int) {
@ -132,7 +132,7 @@ func TestDB_SubscribePush_multiple(t *testing.T) {
defer addrsMu.Unlock()
for i := 0; i < count; i++ {
chunk := generateRandomChunk()
chunk := generateTestRandomChunk()
err := uploader.Put(chunk)
if err != nil {

12
swarm/storage/localstore_test.go
View File

@ -23,7 +23,7 @@ import (
"testing"
"time"
ch "github.com/ethereum/go-ethereum/swarm/chunk"
"github.com/ethereum/go-ethereum/swarm/chunk"
)
var (
@ -65,7 +65,7 @@ func TestValidator(t *testing.T) {
// add content address validator and check puts
// bad should fail, good should pass
store.Validators = append(store.Validators, NewContentAddressValidator(hashfunc))
chunks = GenerateRandomChunks(ch.DefaultSize, 2)
chunks = GenerateRandomChunks(chunk.DefaultSize, 2)
goodChunk = chunks[0]
badChunk = chunks[1]
copy(badChunk.Data(), goodChunk.Data())
@ -83,7 +83,7 @@ func TestValidator(t *testing.T) {
var negV boolTestValidator
store.Validators = append(store.Validators, negV)
chunks = GenerateRandomChunks(ch.DefaultSize, 2)
chunks = GenerateRandomChunks(chunk.DefaultSize, 2)
goodChunk = chunks[0]
badChunk = chunks[1]
copy(badChunk.Data(), goodChunk.Data())
@ -101,7 +101,7 @@ func TestValidator(t *testing.T) {
var posV boolTestValidator = true
store.Validators = append(store.Validators, posV)
chunks = GenerateRandomChunks(ch.DefaultSize, 2)
chunks = GenerateRandomChunks(chunk.DefaultSize, 2)
goodChunk = chunks[0]
badChunk = chunks[1]
copy(badChunk.Data(), goodChunk.Data())
@ -138,7 +138,7 @@ func putChunks(store *LocalStore, chunks ...Chunk) []error {
func put(store *LocalStore, n int, f func(i int64) Chunk) (hs []Address, errs []error) {
for i := int64(0); i < int64(n); i++ {
chunk := f(ch.DefaultSize)
chunk := f(chunk.DefaultSize)
err := store.Put(context.TODO(), chunk)
errs = append(errs, err)
hs = append(hs, chunk.Address())
@ -158,7 +158,7 @@ func TestGetFrequentlyAccessedChunkWontGetGarbageCollected(t *testing.T) {
var chunks []Chunk
for i := 0; i < ldbCap; i++ {
chunks = append(chunks, GenerateRandomChunk(ch.DefaultSize))
chunks = append(chunks, GenerateRandomChunk(chunk.DefaultSize))
}
mostAccessed := chunks[0].Address()

24
swarm/storage/netstore_test.go
View File

@ -29,7 +29,7 @@ import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/p2p/enode"
ch "github.com/ethereum/go-ethereum/swarm/chunk"
"github.com/ethereum/go-ethereum/swarm/chunk"
)
var sourcePeerID = enode.HexID("99d8594b52298567d2ca3f4c441a5ba0140ee9245e26460d01102a52773c73b9")
@ -114,7 +114,7 @@ func mustNewNetStoreWithFetcher(t *testing.T) (*NetStore, *mockNetFetcher) {
func TestNetStoreGetAndPut(t *testing.T) {
netStore, fetcher := mustNewNetStoreWithFetcher(t)
chunk := GenerateRandomChunk(ch.DefaultSize)
chunk := GenerateRandomChunk(chunk.DefaultSize)
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
@ -174,7 +174,7 @@ func TestNetStoreGetAndPut(t *testing.T) {
func TestNetStoreGetAfterPut(t *testing.T) {
netStore, fetcher := mustNewNetStoreWithFetcher(t)
chunk := GenerateRandomChunk(ch.DefaultSize)
chunk := GenerateRandomChunk(chunk.DefaultSize)
ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond)
defer cancel()
@ -209,7 +209,7 @@ func TestNetStoreGetAfterPut(t *testing.T) {
func TestNetStoreGetTimeout(t *testing.T) {
netStore, fetcher := mustNewNetStoreWithFetcher(t)
chunk := GenerateRandomChunk(ch.DefaultSize)
chunk := GenerateRandomChunk(chunk.DefaultSize)
ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond)
defer cancel()
@ -261,7 +261,7 @@ func TestNetStoreGetTimeout(t *testing.T) {
func TestNetStoreGetCancel(t *testing.T) {
netStore, fetcher := mustNewNetStoreWithFetcher(t)
chunk := GenerateRandomChunk(ch.DefaultSize)
chunk := GenerateRandomChunk(chunk.DefaultSize)
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second)
@ -313,7 +313,7 @@ func TestNetStoreGetCancel(t *testing.T) {
func TestNetStoreMultipleGetAndPut(t *testing.T) {
netStore, fetcher := mustNewNetStoreWithFetcher(t)
chunk := GenerateRandomChunk(ch.DefaultSize)
chunk := GenerateRandomChunk(chunk.DefaultSize)
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
@ -387,7 +387,7 @@ func TestNetStoreMultipleGetAndPut(t *testing.T) {
func TestNetStoreFetchFuncTimeout(t *testing.T) {
netStore, fetcher := mustNewNetStoreWithFetcher(t)
chunk := GenerateRandomChunk(ch.DefaultSize)
chunk := GenerateRandomChunk(chunk.DefaultSize)
ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond)
defer cancel()
@ -426,7 +426,7 @@ func TestNetStoreFetchFuncTimeout(t *testing.T) {
func TestNetStoreFetchFuncAfterPut(t *testing.T) {
netStore := mustNewNetStore(t)
chunk := GenerateRandomChunk(ch.DefaultSize)
chunk := GenerateRandomChunk(chunk.DefaultSize)
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second)
defer cancel()
@ -453,7 +453,7 @@ func TestNetStoreFetchFuncAfterPut(t *testing.T) {
func TestNetStoreGetCallsRequest(t *testing.T) {
netStore, fetcher := mustNewNetStoreWithFetcher(t)
chunk := GenerateRandomChunk(ch.DefaultSize)
chunk := GenerateRandomChunk(chunk.DefaultSize)
ctx := context.WithValue(context.Background(), "hopcount", uint8(5))
ctx, cancel := context.WithTimeout(ctx, 200*time.Millisecond)
@ -481,7 +481,7 @@ func TestNetStoreGetCallsRequest(t *testing.T) {
func TestNetStoreGetCallsOffer(t *testing.T) {
netStore, fetcher := mustNewNetStoreWithFetcher(t)
chunk := GenerateRandomChunk(ch.DefaultSize)
chunk := GenerateRandomChunk(chunk.DefaultSize)
// If a source peer is added to the context, NetStore will handle it as an offer
ctx := context.WithValue(context.Background(), "source", sourcePeerID.String())
@ -567,7 +567,7 @@ func TestNetStoreFetcherCountPeers(t *testing.T) {
func TestNetStoreFetchFuncCalledMultipleTimes(t *testing.T) {
netStore, fetcher := mustNewNetStoreWithFetcher(t)
chunk := GenerateRandomChunk(ch.DefaultSize)
chunk := GenerateRandomChunk(chunk.DefaultSize)
ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond)
defer cancel()
@ -632,7 +632,7 @@ func TestNetStoreFetchFuncCalledMultipleTimes(t *testing.T) {
func TestNetStoreFetcherLifeCycleWithTimeout(t *testing.T) {
netStore, fetcher := mustNewNetStoreWithFetcher(t)
chunk := GenerateRandomChunk(ch.DefaultSize)
chunk := GenerateRandomChunk(chunk.DefaultSize)
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second)
defer cancel()

6
swarm/storage/pyramid.go
View File

@ -25,7 +25,7 @@ import (
"sync"
"time"
ch "github.com/ethereum/go-ethereum/swarm/chunk"
"github.com/ethereum/go-ethereum/swarm/chunk"
"github.com/ethereum/go-ethereum/swarm/log"
)
@ -97,11 +97,11 @@ func NewPyramidSplitterParams(addr Address, reader io.Reader, putter Putter, get
New chunks to store are store using the putter which the caller provides.
*/
func PyramidSplit(ctx context.Context, reader io.Reader, putter Putter, getter Getter) (Address, func(context.Context) error, error) {
return NewPyramidSplitter(NewPyramidSplitterParams(nil, reader, putter, getter, ch.DefaultSize)).Split(ctx)
return NewPyramidSplitter(NewPyramidSplitterParams(nil, reader, putter, getter, chunk.DefaultSize)).Split(ctx)
}
func PyramidAppend(ctx context.Context, addr Address, reader io.Reader, putter Putter, getter Getter) (Address, func(context.Context) error, error) {
return NewPyramidSplitter(NewPyramidSplitterParams(addr, reader, putter, getter, ch.DefaultSize)).Append(ctx)
return NewPyramidSplitter(NewPyramidSplitterParams(addr, reader, putter, getter, chunk.DefaultSize)).Append(ctx)
}
// Entry to create a tree node

120
swarm/storage/types.go
View File

@ -22,53 +22,29 @@ import (
"crypto"
"crypto/rand"
"encoding/binary"
"fmt"
"io"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/swarm/bmt"
ch "github.com/ethereum/go-ethereum/swarm/chunk"
"github.com/ethereum/go-ethereum/swarm/chunk"
"golang.org/x/crypto/sha3"
)
const MaxPO = 16
const AddressLength = 32
// MaxPO is the same as chunk.MaxPO for backward compatibility.
const MaxPO = chunk.MaxPO
// AddressLength is the same as chunk.AddressLength for backward compatibility.
const AddressLength = chunk.AddressLength
type SwarmHasher func() SwarmHash
type Address []byte
// Address is an alias for chunk.Address for backward compatibility.
type Address = chunk.Address
// Proximity(x, y) returns the proximity order of the MSB distance between x and y
//
// The distance metric MSB(x, y) of two equal length byte sequences x an y is the
// value of the binary integer cast of the x^y, ie., x and y bitwise xor-ed.
// the binary cast is big endian: most significant bit first (=MSB).
//
// Proximity(x, y) is a discrete logarithmic scaling of the MSB distance.
// It is defined as the reverse rank of the integer part of the base 2
// logarithm of the distance.
// It is calculated by counting the number of common leading zeros in the (MSB)
// binary representation of the x^y.
//
// (0 farthest, 255 closest, 256 self)
func Proximity(one, other []byte) (ret int) {
b := (MaxPO-1)/8 + 1
if b > len(one) {
b = len(one)
}
m := 8
for i := 0; i < b; i++ {
oxo := one[i] ^ other[i]
for j := 0; j < m; j++ {
if (oxo>>uint8(7-j))&0x01 != 0 {
return i*8 + j
}
}
}
return MaxPO
}
// Proximity is the same as chunk.Proximity for backward compatibility.
var Proximity = chunk.Proximity
var ZeroAddr = Address(common.Hash{}.Bytes())
// ZeroAddr is the same as chunk.ZeroAddr for backward compatibility.
var ZeroAddr = chunk.ZeroAddr
func MakeHashFunc(hash string) SwarmHasher {
switch hash {
@ -80,7 +56,7 @@ func MakeHashFunc(hash string) SwarmHasher {
return func() SwarmHash {
hasher := sha3.NewLegacyKeccak256
hasherSize := hasher().Size()
segmentCount := ch.DefaultSize / hasherSize
segmentCount := chunk.DefaultSize / hasherSize
pool := bmt.NewTreePool(hasher, segmentCount, bmt.PoolSize)
return bmt.New(pool)
}
@ -88,33 +64,6 @@ func MakeHashFunc(hash string) SwarmHasher {
return nil
}
func (a Address) Hex() string {
return fmt.Sprintf("%064x", []byte(a[:]))
}
func (a Address) Log() string {
if len(a[:]) < 8 {
return fmt.Sprintf("%x", []byte(a[:]))
}
return fmt.Sprintf("%016x", []byte(a[:8]))
}
func (a Address) String() string {
return fmt.Sprintf("%064x", []byte(a))
}
func (a Address) MarshalJSON() (out []byte, err error) {
return []byte(`"` + a.String() + `"`), nil
}
func (a *Address) UnmarshalJSON(value []byte) error {
s := string(value)
*a = make([]byte, 32)
h := common.Hex2Bytes(s[1 : len(s)-1])
copy(*a, h)
return nil
}
type AddressCollection []Address
func NewAddressCollection(l int) AddressCollection {
@ -133,38 +82,11 @@ func (c AddressCollection) Swap(i, j int) {
c[i], c[j] = c[j], c[i]
}
// Chunk interface implemented by context.Contexts and data chunks
type Chunk interface {
Address() Address
Data() []byte
}
// Chunk is an alias for chunk.Chunk for backward compatibility.
type Chunk = chunk.Chunk
type chunk struct {
addr Address
sdata []byte
span int64
}
func NewChunk(addr Address, data []byte) *chunk {
return &chunk{
addr: addr,
sdata: data,
span: -1,
}
}
func (c *chunk) Address() Address {
return c.addr
}
func (c *chunk) Data() []byte {
return c.sdata
}
// String() for pretty printing
func (self *chunk) String() string {
return fmt.Sprintf("Address: %v TreeSize: %v Chunksize: %v", self.addr.Log(), self.span, len(self.sdata))
}
// NewChunk is the same as chunk.NewChunk for backward compatibility.
var NewChunk = chunk.NewChunk
func GenerateRandomChunk(dataSize int64) Chunk {
hasher := MakeHashFunc(DefaultHash)()
@ -274,9 +196,9 @@ func NewContentAddressValidator(hasher SwarmHasher) *ContentAddressValidator {
}
// Validate that the given key is a valid content address for the given data
func (v *ContentAddressValidator) Validate(chunk Chunk) bool {
data := chunk.Data()
if l := len(data); l < 9 || l > ch.DefaultSize+8 {
func (v *ContentAddressValidator) Validate(ch Chunk) bool {
data := ch.Data()
if l := len(data); l < 9 || l > chunk.DefaultSize+8 {
// log.Error("invalid chunk size", "chunk", addr.Hex(), "size", l)
return false
}
@ -286,7 +208,7 @@ func (v *ContentAddressValidator) Validate(chunk Chunk) bool {
hasher.Write(data[8:])
hash := hasher.Sum(nil)
return bytes.Equal(hash, chunk.Address())
return bytes.Equal(hash, ch.Address())
}
type ChunkStore interface {