Update Geth to 1.9.0

vendor/github.com/allegro/bigcache/.gitignore

@@ -2,4 +2,9 @@
 .DS_Store
 /server/server.exe
 /server/server
+/server/server_dar*
+/server/server_fre*
+/server/server_win*
+/server/server_net*
+/server/server_ope*
 CHANGELOG.md

vendor/github.com/allegro/bigcache/.travis.yml

@@ -14,7 +14,7 @@ before_install:
   - go get github.com/mattn/goveralls
   - go get golang.org/x/tools/cmd/cover
   - go get golang.org/x/tools/cmd/goimports
-  - go get github.com/golang/lint/golint
+  - go get golang.org/x/lint/golint
   - go get github.com/stretchr/testify/assert
   - go get github.com/gordonklaus/ineffassign
   

vendor/github.com/allegro/bigcache/README.md

@@ -46,10 +46,15 @@ config := bigcache.Config {
 		// if value is reached then the oldest entries can be overridden for the new ones
 		// 0 value means no size limit
 		HardMaxCacheSize: 8192,
-		// callback fired when the oldest entry is removed because of its
-		// expiration time or no space left for the new entry. Default value is nil which
-		// means no callback and it prevents from unwrapping the oldest entry.
+		// callback fired when the oldest entry is removed because of its expiration time or no space left
+		// for the new entry, or because delete was called. A bitmask representing the reason will be returned.
+		// Default value is nil which means no callback and it prevents from unwrapping the oldest entry.
 		OnRemove: nil,
+		// OnRemoveWithReason is a callback fired when the oldest entry is removed because of its expiration time or no space left
+		// for the new entry, or because delete was called. A constant representing the reason will be passed through.
+		// Default value is nil which means no callback and it prevents from unwrapping the oldest entry.
+		// Ignored if OnRemove is specified.
+		OnRemoveWithReason: nil,
 	}
 
 cache, initErr := bigcache.NewBigCache(config)
@@ -74,20 +79,20 @@ Benchmark tests were made using an i7-6700K with 32GB of RAM on Windows 10.
 ```bash
 cd caches_bench; go test -bench=. -benchtime=10s ./... -timeout 30m
 
-BenchmarkMapSet-8                     	 2000000	       716 ns/op	     336 B/op	       3 allocs/op
-BenchmarkConcurrentMapSet-8           	 1000000	      1292 ns/op	     347 B/op	       8 allocs/op
-BenchmarkFreeCacheSet-8               	 3000000	       501 ns/op	     371 B/op	       3 allocs/op
-BenchmarkBigCacheSet-8                	 3000000	       482 ns/op	     303 B/op	       2 allocs/op
-BenchmarkMapGet-8                     	 5000000	       309 ns/op	      24 B/op	       1 allocs/op
-BenchmarkConcurrentMapGet-8           	 2000000	       659 ns/op	      24 B/op	       2 allocs/op
-BenchmarkFreeCacheGet-8               	 3000000	       541 ns/op	     152 B/op	       3 allocs/op
-BenchmarkBigCacheGet-8                	 3000000	       420 ns/op	     152 B/op	       3 allocs/op
-BenchmarkBigCacheSetParallel-8        	10000000	       184 ns/op	     313 B/op	       3 allocs/op
-BenchmarkFreeCacheSetParallel-8       	10000000	       195 ns/op	     357 B/op	       4 allocs/op
-BenchmarkConcurrentMapSetParallel-8   	 5000000	       242 ns/op	     200 B/op	       6 allocs/op
-BenchmarkBigCacheGetParallel-8        	20000000	       100 ns/op	     152 B/op	       4 allocs/op
-BenchmarkFreeCacheGetParallel-8       	10000000	       133 ns/op	     152 B/op	       4 allocs/op
-BenchmarkConcurrentMapGetParallel-8   	10000000	       202 ns/op	      24 B/op	       2 allocs/op
+BenchmarkMapSet-8                        3000000               569 ns/op             202 B/op          3 allocs/op
+BenchmarkConcurrentMapSet-8              1000000              1592 ns/op             347 B/op          8 allocs/op
+BenchmarkFreeCacheSet-8                  3000000               775 ns/op             355 B/op          2 allocs/op
+BenchmarkBigCacheSet-8                   3000000               640 ns/op             303 B/op          2 allocs/op
+BenchmarkMapGet-8                        5000000               407 ns/op              24 B/op          1 allocs/op
+BenchmarkConcurrentMapGet-8              3000000               558 ns/op              24 B/op          2 allocs/op
+BenchmarkFreeCacheGet-8                  2000000               682 ns/op             136 B/op          2 allocs/op
+BenchmarkBigCacheGet-8                   3000000               512 ns/op             152 B/op          4 allocs/op
+BenchmarkBigCacheSetParallel-8          10000000               225 ns/op             313 B/op          3 allocs/op
+BenchmarkFreeCacheSetParallel-8         10000000               218 ns/op             341 B/op          3 allocs/op
+BenchmarkConcurrentMapSetParallel-8      5000000               318 ns/op             200 B/op          6 allocs/op
+BenchmarkBigCacheGetParallel-8          20000000               178 ns/op             152 B/op          4 allocs/op
+BenchmarkFreeCacheGetParallel-8         20000000               295 ns/op             136 B/op          3 allocs/op
+BenchmarkConcurrentMapGetParallel-8     10000000               237 ns/op              24 B/op          2 allocs/op
 ```
 
 Writes and reads in bigcache are faster than in freecache.

vendor/github.com/allegro/bigcache/bigcache.go

@@ -10,7 +10,7 @@ const (
 )
 
 // BigCache is fast, concurrent, evicting cache created to keep big number of entries without impact on performance.
-// It keeps entries on heap but omits GC for them. To achieve that operations on bytes arrays take place,
+// It keeps entries on heap but omits GC for them. To achieve that, operations take place on byte arrays,
 // therefore entries (de)serialization in front of the cache will be needed in most use cases.
 type BigCache struct {
 	shards       []*cacheShard
@@ -20,8 +20,22 @@ type BigCache struct {
 	config       Config
 	shardMask    uint64
 	maxShardSize uint32
+	close        chan struct{}
 }
 
+// RemoveReason is a value used to signal to the user why a particular key was removed in the OnRemove callback.
+type RemoveReason uint32
+
+const (
+	// Expired means the key is past its LifeWindow.
+	Expired RemoveReason = iota
+	// NoSpace means the key is the oldest and the cache size was at its maximum when Set was called, or the
+	// entry exceeded the maximum shard size.
+	NoSpace
+	// Deleted means Delete was called and this key was removed as a result.
+	Deleted
+)
+
 // NewBigCache initialize new instance of BigCache
 func NewBigCache(config Config) (*BigCache, error) {
 	return newBigCache(config, &systemClock{})
@@ -45,13 +59,16 @@ func newBigCache(config Config, clock clock) (*BigCache, error) {
 		config:       config,
 		shardMask:    uint64(config.Shards - 1),
 		maxShardSize: uint32(config.maximumShardSize()),
+		close:        make(chan struct{}),
 	}
 
-	var onRemove func(wrappedEntry []byte)
-	if config.OnRemove == nil {
-		onRemove = cache.notProvidedOnRemove
-	} else {
+	var onRemove func(wrappedEntry []byte, reason RemoveReason)
+	if config.OnRemove != nil {
 		onRemove = cache.providedOnRemove
+	} else if config.OnRemoveWithReason != nil {
+		onRemove = cache.providedOnRemoveWithReason
+	} else {
+		onRemove = cache.notProvidedOnRemove
 	}
 
 	for i := 0; i < config.Shards; i++ {
@@ -60,8 +77,15 @@ func newBigCache(config Config, clock clock) (*BigCache, error) {
 
 	if config.CleanWindow > 0 {
 		go func() {
-			for t := range time.Tick(config.CleanWindow) {
-				cache.cleanUp(uint64(t.Unix()))
+			ticker := time.NewTicker(config.CleanWindow)
+			defer ticker.Stop()
+			for {
+				select {
+				case t := <-ticker.C:
+					cache.cleanUp(uint64(t.Unix()))
+				case <-cache.close:
+					return
+				}
 			}
 		}()
 	}
@@ -69,8 +93,16 @@ func newBigCache(config Config, clock clock) (*BigCache, error) {
 	return cache, nil
 }
 
+// Close is used to signal a shutdown of the cache when you are done with it.
+// This allows the cleaning goroutines to exit and ensures references are not
+// kept to the cache preventing GC of the entire cache.
+func (c *BigCache) Close() error {
+	close(c.close)
+	return nil
+}
+
 // Get reads entry for the key.
-// It returns an EntryNotFoundError when
+// It returns an ErrEntryNotFound when
 // no entry exists for the given key.
 func (c *BigCache) Get(key string) ([]byte, error) {
 	hashedKey := c.hash.Sum64(key)
@@ -109,6 +141,15 @@ func (c *BigCache) Len() int {
 	return len
 }
 
+// Capacity returns amount of bytes store in the cache.
+func (c *BigCache) Capacity() int {
+	var len int
+	for _, shard := range c.shards {
+		len += shard.capacity()
+	}
+	return len
+}
+
 // Stats returns cache's statistics
 func (c *BigCache) Stats() Stats {
 	var s Stats
@@ -128,10 +169,10 @@ func (c *BigCache) Iterator() *EntryInfoIterator {
 	return newIterator(c)
 }
 
-func (c *BigCache) onEvict(oldestEntry []byte, currentTimestamp uint64, evict func() error) bool {
+func (c *BigCache) onEvict(oldestEntry []byte, currentTimestamp uint64, evict func(reason RemoveReason) error) bool {
 	oldestTimestamp := readTimestampFromEntry(oldestEntry)
 	if currentTimestamp-oldestTimestamp > c.lifeWindow {
-		evict()
+		evict(Expired)
 		return true
 	}
 	return false
@@ -147,9 +188,15 @@ func (c *BigCache) getShard(hashedKey uint64) (shard *cacheShard) {
 	return c.shards[hashedKey&c.shardMask]
 }
 
-func (c *BigCache) providedOnRemove(wrappedEntry []byte) {
+func (c *BigCache) providedOnRemove(wrappedEntry []byte, reason RemoveReason) {
 	c.config.OnRemove(readKeyFromEntry(wrappedEntry), readEntry(wrappedEntry))
 }
 
-func (c *BigCache) notProvidedOnRemove(wrappedEntry []byte) {
+func (c *BigCache) providedOnRemoveWithReason(wrappedEntry []byte, reason RemoveReason) {
+	if c.config.onRemoveFilter == 0 || (1<<uint(reason))&c.config.onRemoveFilter > 0 {
+		c.config.OnRemoveWithReason(readKeyFromEntry(wrappedEntry), readEntry(wrappedEntry), reason)
+	}
+}
+
+func (c *BigCache) notProvidedOnRemove(wrappedEntry []byte, reason RemoveReason) {
 }

vendor/github.com/allegro/bigcache/bigcache_bench_test.go

@@ -98,6 +98,14 @@ func BenchmarkWriteToCacheWith1024ShardsAndSmallShardInitSize(b *testing.B) {
 	writeToCache(b, 1024, 100*time.Second, 100)
 }
 
+func BenchmarkReadFromCacheNonExistentKeys(b *testing.B) {
+	for _, shards := range []int{1, 512, 1024, 8192} {
+		b.Run(fmt.Sprintf("%d-shards", shards), func(b *testing.B) {
+			readFromCacheNonExistentKeys(b, 1024)
+		})
+	}
+}
+
 func writeToCache(b *testing.B, shards int, lifeWindow time.Duration, requestsInLifeWindow int) {
 	cache, _ := NewBigCache(Config{
 		Shards:             shards,
@@ -139,3 +147,21 @@ func readFromCache(b *testing.B, shards int) {
 		}
 	})
 }
+
+func readFromCacheNonExistentKeys(b *testing.B, shards int) {
+	cache, _ := NewBigCache(Config{
+		Shards:             shards,
+		LifeWindow:         1000 * time.Second,
+		MaxEntriesInWindow: max(b.N, 100),
+		MaxEntrySize:       500,
+	})
+	b.ResetTimer()
+
+	b.RunParallel(func(pb *testing.PB) {
+		b.ReportAllocs()
+
+		for pb.Next() {
+			cache.Get(strconv.Itoa(rand.Intn(b.N)))
+		}
+	})
+}

vendor/github.com/allegro/bigcache/bigcache_test.go

@@ -1,7 +1,10 @@
 package bigcache
 
 import (
+	"bytes"
 	"fmt"
+	"math/rand"
+	"runtime"
 	"sync"
 	"testing"
 	"time"
@@ -105,7 +108,7 @@ func TestEntryNotFound(t *testing.T) {
 	_, err := cache.Get("nonExistingKey")
 
 	// then
-	assert.EqualError(t, err, "Entry \"nonExistingKey\" not found")
+	assert.EqualError(t, err, ErrEntryNotFound.Error())
 }
 
 func TestTimingEviction(t *testing.T) {
@@ -127,7 +130,7 @@ func TestTimingEviction(t *testing.T) {
 	_, err := cache.Get("key")
 
 	// then
-	assert.EqualError(t, err, "Entry \"key\" not found")
+	assert.EqualError(t, err, ErrEntryNotFound.Error())
 }
 
 func TestTimingEvictionShouldEvictOnlyFromUpdatedShard(t *testing.T) {
@@ -149,7 +152,7 @@ func TestTimingEvictionShouldEvictOnlyFromUpdatedShard(t *testing.T) {
 	value, err := cache.Get("key")
 
 	// then
-	assert.NoError(t, err, "Entry \"key\" not found")
+	assert.NoError(t, err, ErrEntryNotFound.Error())
 	assert.Equal(t, []byte("value"), value)
 }
 
@@ -171,7 +174,7 @@ func TestCleanShouldEvictAll(t *testing.T) {
 	value, err := cache.Get("key")
 
 	// then
-	assert.EqualError(t, err, "Entry \"key\" not found")
+	assert.EqualError(t, err, ErrEntryNotFound.Error())
 	assert.Equal(t, value, []byte(nil))
 }
 
@@ -181,17 +184,22 @@ func TestOnRemoveCallback(t *testing.T) {
 	// given
 	clock := mockedClock{value: 0}
 	onRemoveInvoked := false
+	onRemoveExtInvoked := false
 	onRemove := func(key string, entry []byte) {
 		onRemoveInvoked = true
 		assert.Equal(t, "key", key)
 		assert.Equal(t, []byte("value"), entry)
 	}
+	onRemoveExt := func(key string, entry []byte, reason RemoveReason) {
+		onRemoveExtInvoked = true
+	}
 	cache, _ := newBigCache(Config{
 		Shards:             1,
 		LifeWindow:         time.Second,
 		MaxEntriesInWindow: 1,
 		MaxEntrySize:       256,
 		OnRemove:           onRemove,
+		OnRemoveWithReason: onRemoveExt,
 	}, &clock)
 
 	// when
@@ -199,6 +207,70 @@ func TestOnRemoveCallback(t *testing.T) {
 	clock.set(5)
 	cache.Set("key2", []byte("value2"))
 
+	// then
+	assert.True(t, onRemoveInvoked)
+	assert.False(t, onRemoveExtInvoked)
+}
+
+func TestOnRemoveWithReasonCallback(t *testing.T) {
+	t.Parallel()
+
+	// given
+	clock := mockedClock{value: 0}
+	onRemoveInvoked := false
+	onRemove := func(key string, entry []byte, reason RemoveReason) {
+		onRemoveInvoked = true
+		assert.Equal(t, "key", key)
+		assert.Equal(t, []byte("value"), entry)
+		assert.Equal(t, reason, RemoveReason(Expired))
+	}
+	cache, _ := newBigCache(Config{
+		Shards:             1,
+		LifeWindow:         time.Second,
+		MaxEntriesInWindow: 1,
+		MaxEntrySize:       256,
+		OnRemoveWithReason: onRemove,
+	}, &clock)
+
+	// when
+	cache.Set("key", []byte("value"))
+	clock.set(5)
+	cache.Set("key2", []byte("value2"))
+
+	// then
+	assert.True(t, onRemoveInvoked)
+}
+
+func TestOnRemoveFilter(t *testing.T) {
+	t.Parallel()
+
+	// given
+	clock := mockedClock{value: 0}
+	onRemoveInvoked := false
+	onRemove := func(key string, entry []byte, reason RemoveReason) {
+		onRemoveInvoked = true
+	}
+	c := Config{
+		Shards:             1,
+		LifeWindow:         time.Second,
+		MaxEntriesInWindow: 1,
+		MaxEntrySize:       256,
+		OnRemoveWithReason: onRemove,
+	}.OnRemoveFilterSet(Deleted, NoSpace)
+
+	cache, _ := newBigCache(c, &clock)
+
+	// when
+	cache.Set("key", []byte("value"))
+	clock.set(5)
+	cache.Set("key2", []byte("value2"))
+
+	// then
+	assert.False(t, onRemoveInvoked)
+
+	// and when
+	cache.Delete("key2")
+
 	// then
 	assert.True(t, onRemoveInvoked)
 }
@@ -276,7 +348,7 @@ func TestCacheDel(t *testing.T) {
 	err := cache.Delete("nonExistingKey")
 
 	// then
-	assert.Equal(t, err.Error(), "Entry \"nonExistingKey\" not found")
+	assert.Equal(t, err.Error(), ErrEntryNotFound.Error())
 
 	// and when
 	cache.Set("existingKey", nil)
@@ -288,6 +360,67 @@ func TestCacheDel(t *testing.T) {
 	assert.Len(t, cachedValue, 0)
 }
 
+// TestCacheDelRandomly does simultaneous deletes, puts and gets, to check for corruption errors.
+func TestCacheDelRandomly(t *testing.T) {
+	t.Parallel()
+	c := Config{
+		Shards:             1,
+		LifeWindow:         time.Second,
+		CleanWindow:        0,
+		MaxEntriesInWindow: 10,
+		MaxEntrySize:       10,
+		Verbose:            true,
+		Hasher:             newDefaultHasher(),
+		HardMaxCacheSize:   1,
+		Logger:             DefaultLogger(),
+	}
+	//c.Hasher = hashStub(5)
+	cache, _ := NewBigCache(c)
+	var wg sync.WaitGroup
+	var ntest = 800000
+	wg.Add(1)
+	go func() {
+		for i := 0; i < ntest; i++ {
+			r := uint8(rand.Int())
+			key := fmt.Sprintf("thekey%d", r)
+
+			cache.Delete(key)
+		}
+		wg.Done()
+	}()
+	wg.Add(1)
+	go func() {
+		val := make([]byte, 1024)
+		for i := 0; i < ntest; i++ {
+			r := byte(rand.Int())
+			key := fmt.Sprintf("thekey%d", r)
+
+			for j := 0; j < len(val); j++ {
+				val[j] = r
+			}
+			cache.Set(key, val)
+		}
+		wg.Done()
+	}()
+	wg.Add(1)
+	go func() {
+		val := make([]byte, 1024)
+		for i := 0; i < ntest; i++ {
+			r := byte(rand.Int())
+			key := fmt.Sprintf("thekey%d", r)
+
+			for j := 0; j < len(val); j++ {
+				val[j] = r
+			}
+			if got, err := cache.Get(key); err == nil && !bytes.Equal(got, val) {
+				t.Errorf("got %s ->\n %x\n expected:\n %x\n ", key, got, val)
+			}
+		}
+		wg.Done()
+	}()
+	wg.Wait()
+}
+
 func TestCacheReset(t *testing.T) {
 	t.Parallel()
 
@@ -369,7 +502,7 @@ func TestGetOnResetCache(t *testing.T) {
 	// then
 	value, err := cache.Get("key1")
 
-	assert.Equal(t, err.Error(), "Entry \"key1\" not found")
+	assert.Equal(t, err.Error(), ErrEntryNotFound.Error())
 	assert.Equal(t, value, []byte(nil))
 }
 
@@ -419,8 +552,8 @@ func TestOldestEntryDeletionWhenMaxCacheSizeIsReached(t *testing.T) {
 	entry3, _ := cache.Get("key3")
 
 	// then
-	assert.EqualError(t, key1Err, "Entry \"key1\" not found")
-	assert.EqualError(t, key2Err, "Entry \"key2\" not found")
+	assert.EqualError(t, key1Err, ErrEntryNotFound.Error())
+	assert.EqualError(t, key2Err, ErrEntryNotFound.Error())
 	assert.Equal(t, blob('c', 1024*800), entry3)
 }
 
@@ -548,6 +681,28 @@ func TestNilValueCaching(t *testing.T) {
 	assert.Equal(t, []byte{}, cachedValue)
 }
 
+func TestClosing(t *testing.T) {
+	// given
+	config := Config{
+		CleanWindow: time.Minute,
+	}
+	startGR := runtime.NumGoroutine()
+
+	// when
+	for i := 0; i < 100; i++ {
+		cache, _ := NewBigCache(config)
+		cache.Close()
+	}
+
+	// wait till all goroutines are stopped.
+	time.Sleep(200 * time.Millisecond)
+
+	// then
+	endGR := runtime.NumGoroutine()
+	assert.True(t, endGR >= startGR)
+	assert.InDelta(t, endGR, startGR, 25)
+}
+
 type mockedLogger struct {
 	lastFormat string
 	lastArgs   []interface{}

vendor/github.com/allegro/bigcache/bytes.go

@@ -0,0 +1,14 @@
+// +build !appengine
+
+package bigcache
+
+import (
+	"reflect"
+	"unsafe"
+)
+
+func bytesToString(b []byte) string {
+	bytesHeader := (*reflect.SliceHeader)(unsafe.Pointer(&b))
+	strHeader := reflect.StringHeader{Data: bytesHeader.Data, Len: bytesHeader.Len}
+	return *(*string)(unsafe.Pointer(&strHeader))
+}

vendor/github.com/allegro/bigcache/bytes_appengine.go

@@ -0,0 +1,7 @@
+// +build appengine
+
+package bigcache
+
+func bytesToString(b []byte) string {
+	return string(b)
+}

vendor/github.com/allegro/bigcache/caches_bench/caches_bench_test.go

@@ -14,7 +14,7 @@ import (
 const maxEntrySize = 256
 
 func BenchmarkMapSet(b *testing.B) {
-	m := make(map[string][]byte)
+	m := make(map[string][]byte, b.N)
 	for i := 0; i < b.N; i++ {
 		m[key(i)] = value()
 	}

vendor/github.com/allegro/bigcache/config.go

@@ -26,8 +26,16 @@ type Config struct {
 	// the oldest entries are overridden for the new ones.
 	HardMaxCacheSize int
 	// OnRemove is a callback fired when the oldest entry is removed because of its expiration time or no space left
-	// for the new entry. Default value is nil which means no callback and it prevents from unwrapping the oldest entry.
+	// for the new entry, or because delete was called.
+	// Default value is nil which means no callback and it prevents from unwrapping the oldest entry.
 	OnRemove func(key string, entry []byte)
+	// OnRemoveWithReason is a callback fired when the oldest entry is removed because of its expiration time or no space left
+	// for the new entry, or because delete was called. A constant representing the reason will be passed through.
+	// Default value is nil which means no callback and it prevents from unwrapping the oldest entry.
+	// Ignored if OnRemove is specified.
+	OnRemoveWithReason func(key string, entry []byte, reason RemoveReason)
+
+	onRemoveFilter int
 
 	// Logger is a logging interface and used in combination with `Verbose`
 	// Defaults to `DefaultLogger()`
@@ -65,3 +73,14 @@ func (c Config) maximumShardSize() int {
 
 	return maxShardSize
 }
+
+// OnRemoveFilterSet sets which remove reasons will trigger a call to OnRemoveWithReason.
+// Filtering out reasons prevents bigcache from unwrapping them, which saves cpu.
+func (c Config) OnRemoveFilterSet(reasons ...RemoveReason) Config {
+	c.onRemoveFilter = 0
+	for i := range reasons {
+		c.onRemoveFilter |= 1 << uint(reasons[i])
+	}
+
+	return c
+}

vendor/github.com/allegro/bigcache/encoding.go

@@ -2,8 +2,6 @@ package bigcache
 
 import (
 	"encoding/binary"
-	"reflect"
-	"unsafe"
 )
 
 const (
@@ -55,12 +53,6 @@ func readKeyFromEntry(data []byte) string {
 	return bytesToString(dst)
 }
 
-func bytesToString(b []byte) string {
-	bytesHeader := (*reflect.SliceHeader)(unsafe.Pointer(&b))
-	strHeader := reflect.StringHeader{Data: bytesHeader.Data, Len: bytesHeader.Len}
-	return *(*string)(unsafe.Pointer(&strHeader))
-}
-
 func readHashFromEntry(data []byte) uint64 {
 	return binary.LittleEndian.Uint64(data[timestampSizeInBytes:])
 }

vendor/github.com/allegro/bigcache/entry_not_found_error.go

@@ -1,17 +1,6 @@
 package bigcache
 
-import "fmt"
+import "errors"
 
-// EntryNotFoundError is an error type struct which is returned when entry was not found for provided key
-type EntryNotFoundError struct {
-	message string
-}
-
-func notFound(key string) error {
-	return &EntryNotFoundError{fmt.Sprintf("Entry %q not found", key)}
-}
-
-// Error returned when entry does not exist.
-func (e EntryNotFoundError) Error() string {
-	return e.message
-}
+// ErrEntryNotFound is an error type struct which is returned when entry was not found for provided key
+var ErrEntryNotFound = errors.New("Entry not found")

vendor/github.com/allegro/bigcache/go.mod

@@ -0,0 +1,9 @@
+module github.com/allegro/bigcache
+
+go 1.12
+
+require (
+	github.com/cespare/xxhash v1.1.0 // indirect
+	github.com/coocood/freecache v1.1.0
+	github.com/stretchr/testify v1.3.0
+)

vendor/github.com/allegro/bigcache/go.sum

@@ -0,0 +1,13 @@
+github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
+github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
+github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
+github.com/coocood/freecache v1.1.0 h1:ENiHOsWdj1BrrlPwblhbn4GdAsMymK3pZORJ+bJGAjA=
+github.com/coocood/freecache v1.1.0/go.mod h1:ePwxCDzOYvARfHdr1pByNct1at3CoKnsipOHwKlNbzI=
+github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=

vendor/github.com/allegro/bigcache/iterator_test.go

@@ -2,6 +2,9 @@ package bigcache
 
 import (
 	"fmt"
+	"runtime"
+	"strconv"
+	"sync"
 	"testing"
 	"time"
 
@@ -148,3 +151,32 @@ func TestEntriesIteratorInInvalidState(t *testing.T) {
 	assert.Equal(t, ErrInvalidIteratorState, err)
 	assert.Equal(t, "Iterator is in invalid state. Use SetNext() to move to next position", err.Error())
 }
+
+func TestEntriesIteratorParallelAdd(t *testing.T) {
+	bc, err := NewBigCache(DefaultConfig(1 * time.Minute))
+	if err != nil {
+		panic(err)
+	}
+
+	wg := sync.WaitGroup{}
+	wg.Add(1)
+	go func() {
+		for i := 0; i < 10000; i++ {
+			err := bc.Set(strconv.Itoa(i), []byte("aaaaaaa"))
+			if err != nil {
+				panic(err)
+			}
+
+			runtime.Gosched()
+		}
+		wg.Done()
+	}()
+
+	for i := 0; i < 100; i++ {
+		iter := bc.Iterator()
+		for iter.SetNext() {
+			_, _ = iter.Value()
+		}
+	}
+	wg.Wait()
+}

vendor/github.com/allegro/bigcache/queue/bytes_queue.go

@@ -16,6 +16,12 @@ const (
 	minimumEmptyBlobSize = 32 + headerEntrySize
 )
 
+var (
+	errEmptyQueue       = &queueError{"Empty queue"}
+	errInvalidIndex     = &queueError{"Index must be greater than zero. Invalid index."}
+	errIndexOutOfBounds = &queueError{"Index out of range"}
+)
+
 // BytesQueue is a non-thread safe queue type of fifo based on bytes array.
 // For every push operation index of entry is returned. It can be used to read the entry later
 type BytesQueue struct {
@@ -162,6 +168,11 @@ func (q *BytesQueue) Get(index int) ([]byte, error) {
 	return data, err
 }
 
+// CheckGet checks if an entry can be read from index
+func (q *BytesQueue) CheckGet(index int) error {
+	return q.peekCheckErr(index)
+}
+
 // Capacity returns number of allocated bytes for queue
 func (q *BytesQueue) Capacity() int {
 	return q.capacity
@@ -177,18 +188,35 @@ func (e *queueError) Error() string {
 	return e.message
 }
 
-func (q *BytesQueue) peek(index int) ([]byte, int, error) {
+// peekCheckErr is identical to peek, but does not actually return any data
+func (q *BytesQueue) peekCheckErr(index int) error {
 
 	if q.count == 0 {
-		return nil, 0, &queueError{"Empty queue"}
+		return errEmptyQueue
 	}
 
 	if index <= 0 {
-		return nil, 0, &queueError{"Index must be grater than zero. Invalid index."}
+		return errInvalidIndex
 	}
 
 	if index+headerEntrySize >= len(q.array) {
-		return nil, 0, &queueError{"Index out of range"}
+		return errIndexOutOfBounds
+	}
+	return nil
+}
+
+func (q *BytesQueue) peek(index int) ([]byte, int, error) {
+
+	if q.count == 0 {
+		return nil, 0, errEmptyQueue
+	}
+
+	if index <= 0 {
+		return nil, 0, errInvalidIndex
+	}
+
+	if index+headerEntrySize >= len(q.array) {
+		return nil, 0, errIndexOutOfBounds
 	}
 
 	blockSize := int(binary.LittleEndian.Uint32(q.array[index : index+headerEntrySize]))

vendor/github.com/allegro/bigcache/queue/bytes_queue_test.go

@@ -50,16 +50,19 @@ func TestPeek(t *testing.T) {
 
 	// when
 	read, err := queue.Peek()
-
+	err2 := queue.peekCheckErr(queue.head)
 	// then
+	assert.Equal(t, err, err2)
 	assert.EqualError(t, err, "Empty queue")
 	assert.Nil(t, read)
 
 	// when
 	queue.Push(entry)
 	read, err = queue.Peek()
+	err2 = queue.peekCheckErr(queue.head)
 
 	// then
+	assert.Equal(t, err, err2)
 	assert.NoError(t, err)
 	assert.Equal(t, pop(queue), read)
 	assert.Equal(t, entry, read)
@@ -286,10 +289,12 @@ func TestGetEntryFromInvalidIndex(t *testing.T) {
 
 	// when
 	result, err := queue.Get(0)
+	err2 := queue.CheckGet(0)
 
 	// then
+	assert.Equal(t, err, err2)
 	assert.Nil(t, result)
-	assert.EqualError(t, err, "Index must be grater than zero. Invalid index.")
+	assert.EqualError(t, err, "Index must be greater than zero. Invalid index.")
 }
 
 func TestGetEntryFromIndexOutOfRange(t *testing.T) {
@@ -301,8 +306,10 @@ func TestGetEntryFromIndexOutOfRange(t *testing.T) {
 
 	// when
 	result, err := queue.Get(42)
+	err2 := queue.CheckGet(42)
 
 	// then
+	assert.Equal(t, err, err2)
 	assert.Nil(t, result)
 	assert.EqualError(t, err, "Index out of range")
 }
@@ -315,8 +322,10 @@ func TestGetEntryFromEmptyQueue(t *testing.T) {
 
 	// when
 	result, err := queue.Get(1)
+	err2 := queue.CheckGet(1)
 
 	// then
+	assert.Equal(t, err, err2)
 	assert.Nil(t, result)
 	assert.EqualError(t, err, "Empty queue")
 }

vendor/github.com/allegro/bigcache/server/middleware_test.go

@@ -0,0 +1,47 @@
+package main
+
+import (
+	"bytes"
+	"log"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+)
+
+func emptyTestHandler() service {
+	return func(h http.Handler) http.Handler {
+		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			w.WriteHeader(http.StatusAccepted)
+		})
+	}
+}
+
+func TestServiceLoader(t *testing.T) {
+	req, err := http.NewRequest("GET", "/api/v1/stats", nil)
+	if err != nil {
+		t.Error(err)
+	}
+	rr := httptest.NewRecorder()
+	testHandlers := serviceLoader(cacheIndexHandler(), emptyTestHandler())
+	testHandlers.ServeHTTP(rr, req)
+	if status := rr.Code; status != http.StatusAccepted {
+		t.Errorf("handlers not loading properly. want: 202, got: %d", rr.Code)
+	}
+}
+
+func TestRequestMetrics(t *testing.T) {
+	var b bytes.Buffer
+	logger := log.New(&b, "", log.LstdFlags)
+	req, err := http.NewRequest("GET", "/api/v1/cache/empty", nil)
+	if err != nil {
+		t.Error(err)
+	}
+	rr := httptest.NewRecorder()
+	testHandlers := serviceLoader(cacheIndexHandler(), requestMetrics(logger))
+	testHandlers.ServeHTTP(rr, req)
+	targetTestString := b.String()
+	if len(targetTestString) == 0 {
+		t.Errorf("we are not logging request length strings.")
+	}
+	t.Log(targetTestString)
+}

vendor/github.com/allegro/bigcache/server/server_test.go

@@ -3,6 +3,7 @@ package main
 import (
 	"bytes"
 	"encoding/json"
+	"errors"
 	"io/ioutil"
 	"net/http/httptest"
 	"testing"
@@ -183,3 +184,100 @@ func TestGetStats(t *testing.T) {
 		t.Errorf("want: > 0; got: 0.\n\thandler not properly returning stats info.")
 	}
 }
+
+func TestGetStatsIndex(t *testing.T) {
+	t.Parallel()
+	var testStats bigcache.Stats
+
+	getreq := httptest.NewRequest("GET", testBaseString+"/api/v1/stats", nil)
+	putreq := httptest.NewRequest("PUT", testBaseString+"/api/v1/stats", nil)
+	rr := httptest.NewRecorder()
+
+	// manually enter a key so there are some stats. get it so there's at least 1 hit.
+	if err := cache.Set("incrementStats", []byte("123")); err != nil {
+		t.Errorf("error setting cache value. error %s", err)
+	}
+	// it's okay if this fails, since we'll catch it downstream.
+	if _, err := cache.Get("incrementStats"); err != nil {
+		t.Errorf("can't find incrementStats. error: %s", err)
+	}
+
+	testHandlers := statsIndexHandler()
+	testHandlers.ServeHTTP(rr, getreq)
+	resp := rr.Result()
+
+	if err := json.NewDecoder(resp.Body).Decode(&testStats); err != nil {
+		t.Errorf("error decoding cache stats. error: %s", err)
+	}
+
+	if testStats.Hits == 0 {
+		t.Errorf("want: > 0; got: 0.\n\thandler not properly returning stats info.")
+	}
+
+	testHandlers = statsIndexHandler()
+	testHandlers.ServeHTTP(rr, putreq)
+	resp = rr.Result()
+	_, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		t.Errorf("cannot deserialise test response: %s", err)
+	}
+}
+
+func TestCacheIndexHandler(t *testing.T) {
+	getreq := httptest.NewRequest("GET", testBaseString+"/api/v1/cache/testkey", nil)
+	putreq := httptest.NewRequest("PUT", testBaseString+"/api/v1/cache/testkey", bytes.NewBuffer([]byte("123")))
+	delreq := httptest.NewRequest("DELETE", testBaseString+"/api/v1/cache/testkey", bytes.NewBuffer([]byte("123")))
+
+	getrr := httptest.NewRecorder()
+	putrr := httptest.NewRecorder()
+	delrr := httptest.NewRecorder()
+	testHandlers := cacheIndexHandler()
+
+	testHandlers.ServeHTTP(putrr, putreq)
+	resp := putrr.Result()
+	if resp.StatusCode != 201 {
+		t.Errorf("want: 201; got: %d.\n\tcan't put keys.", resp.StatusCode)
+	}
+	testHandlers.ServeHTTP(getrr, getreq)
+	resp = getrr.Result()
+	if resp.StatusCode != 200 {
+		t.Errorf("want: 200; got: %d.\n\tcan't get keys.", resp.StatusCode)
+	}
+	testHandlers.ServeHTTP(delrr, delreq)
+	resp = delrr.Result()
+	if resp.StatusCode != 200 {
+		t.Errorf("want: 200; got: %d.\n\tcan't delete keys.", resp.StatusCode)
+	}
+}
+
+func TestInvalidPutWhenExceedShardCap(t *testing.T) {
+	t.Parallel()
+	req := httptest.NewRequest("PUT", testBaseString+"/api/v1/cache/putKey", bytes.NewBuffer(bytes.Repeat([]byte("a"), 8*1024*1024)))
+	rr := httptest.NewRecorder()
+
+	putCacheHandler(rr, req)
+	resp := rr.Result()
+
+	if resp.StatusCode != 500 {
+		t.Errorf("want: 500; got: %d", resp.StatusCode)
+	}
+}
+
+func TestInvalidPutWhenReading(t *testing.T) {
+	t.Parallel()
+	req := httptest.NewRequest("PUT", testBaseString+"/api/v1/cache/putKey", errReader(0))
+	rr := httptest.NewRecorder()
+
+	putCacheHandler(rr, req)
+	resp := rr.Result()
+
+	if resp.StatusCode != 500 {
+		t.Errorf("want: 500; got: %d", resp.StatusCode)
+	}
+}
+
+type errReader int
+
+func (errReader) Read([]byte) (int, error) {
+	return 0, errors.New("test read error")
+}

vendor/github.com/allegro/bigcache/shard.go

@@ -8,12 +8,14 @@ import (
 	"github.com/allegro/bigcache/queue"
 )
 
+type onRemoveCallback func(wrappedEntry []byte, reason RemoveReason)
+
 type cacheShard struct {
 	hashmap     map[uint64]uint32
 	entries     queue.BytesQueue
 	lock        sync.RWMutex
 	entryBuffer []byte
-	onRemove    func(wrappedEntry []byte)
+	onRemove    onRemoveCallback
 
 	isVerbose  bool
 	logger     Logger
@@ -23,8 +25,6 @@ type cacheShard struct {
 	stats Stats
 }
 
-type onRemoveCallback func(wrappedEntry []byte)
-
 func (s *cacheShard) get(key string, hashedKey uint64) ([]byte, error) {
 	s.lock.RLock()
 	itemIndex := s.hashmap[hashedKey]
@@ -32,7 +32,7 @@ func (s *cacheShard) get(key string, hashedKey uint64) ([]byte, error) {
 	if itemIndex == 0 {
 		s.lock.RUnlock()
 		s.miss()
-		return nil, notFound(key)
+		return nil, ErrEntryNotFound
 	}
 
 	wrappedEntry, err := s.entries.Get(int(itemIndex))
@@ -47,11 +47,12 @@ func (s *cacheShard) get(key string, hashedKey uint64) ([]byte, error) {
 		}
 		s.lock.RUnlock()
 		s.collision()
-		return nil, notFound(key)
+		return nil, ErrEntryNotFound
 	}
+	entry := readEntry(wrappedEntry)
 	s.lock.RUnlock()
 	s.hit()
-	return readEntry(wrappedEntry), nil
+	return entry, nil
 }
 
 func (s *cacheShard) set(key string, hashedKey uint64, entry []byte) error {
@@ -77,7 +78,7 @@ func (s *cacheShard) set(key string, hashedKey uint64, entry []byte) error {
 			s.lock.Unlock()
 			return nil
 		}
-		if s.removeOldestEntry() != nil {
+		if s.removeOldestEntry(NoSpace) != nil {
 			s.lock.Unlock()
 			return fmt.Errorf("entry is bigger than max shard size")
 		}
@@ -85,17 +86,17 @@ func (s *cacheShard) set(key string, hashedKey uint64, entry []byte) error {
 }
 
 func (s *cacheShard) del(key string, hashedKey uint64) error {
+	// Optimistic pre-check using only readlock
 	s.lock.RLock()
 	itemIndex := s.hashmap[hashedKey]
 
 	if itemIndex == 0 {
 		s.lock.RUnlock()
 		s.delmiss()
-		return notFound(key)
+		return ErrEntryNotFound
 	}
 
-	wrappedEntry, err := s.entries.Get(int(itemIndex))
-	if err != nil {
+	if err := s.entries.CheckGet(int(itemIndex)); err != nil {
 		s.lock.RUnlock()
 		s.delmiss()
 		return err
@@ -104,8 +105,25 @@ func (s *cacheShard) del(key string, hashedKey uint64) error {
 
 	s.lock.Lock()
 	{
+		// After obtaining the writelock, we need to read the same again,
+		// since the data delivered earlier may be stale now
+		itemIndex = s.hashmap[hashedKey]
+
+		if itemIndex == 0 {
+			s.lock.Unlock()
+			s.delmiss()
+			return ErrEntryNotFound
+		}
+
+		wrappedEntry, err := s.entries.Get(int(itemIndex))
+		if err != nil {
+			s.lock.Unlock()
+			s.delmiss()
+			return err
+		}
+
 		delete(s.hashmap, hashedKey)
-		s.onRemove(wrappedEntry)
+		s.onRemove(wrappedEntry, Deleted)
 		resetKeyFromEntry(wrappedEntry)
 	}
 	s.lock.Unlock()
@@ -114,10 +132,10 @@ func (s *cacheShard) del(key string, hashedKey uint64) error {
 	return nil
 }
 
-func (s *cacheShard) onEvict(oldestEntry []byte, currentTimestamp uint64, evict func() error) bool {
+func (s *cacheShard) onEvict(oldestEntry []byte, currentTimestamp uint64, evict func(reason RemoveReason) error) bool {
 	oldestTimestamp := readTimestampFromEntry(oldestEntry)
 	if currentTimestamp-oldestTimestamp > s.lifeWindow {
-		evict()
+		evict(Expired)
 		return true
 	}
 	return false
@@ -136,17 +154,22 @@ func (s *cacheShard) cleanUp(currentTimestamp uint64) {
 }
 
 func (s *cacheShard) getOldestEntry() ([]byte, error) {
+	s.lock.RLock()
+	defer s.lock.RUnlock()
 	return s.entries.Peek()
 }
 
 func (s *cacheShard) getEntry(index int) ([]byte, error) {
-	return s.entries.Get(index)
+	s.lock.RLock()
+	entry, err := s.entries.Get(index)
+	s.lock.RUnlock()
+
+	return entry, err
 }
 
 func (s *cacheShard) copyKeys() (keys []uint32, next int) {
-	keys = make([]uint32, len(s.hashmap))
-
 	s.lock.RLock()
+	keys = make([]uint32, len(s.hashmap))
 
 	for _, index := range s.hashmap {
 		keys[next] = index
@@ -157,12 +180,12 @@ func (s *cacheShard) copyKeys() (keys []uint32, next int) {
 	return keys, next
 }
 
-func (s *cacheShard) removeOldestEntry() error {
+func (s *cacheShard) removeOldestEntry(reason RemoveReason) error {
 	oldest, err := s.entries.Pop()
 	if err == nil {
 		hash := readHashFromEntry(oldest)
 		delete(s.hashmap, hash)
-		s.onRemove(oldest)
+		s.onRemove(oldest, reason)
 		return nil
 	}
 	return err
@@ -183,6 +206,13 @@ func (s *cacheShard) len() int {
 	return res
 }
 
+func (s *cacheShard) capacity() int {
+	s.lock.RLock()
+	res := s.entries.Capacity()
+	s.lock.RUnlock()
+	return res
+}
+
 func (s *cacheShard) getStats() Stats {
 	var stats = Stats{
 		Hits:       atomic.LoadInt64(&s.stats.Hits),