forked from cerc-io/ipfs-ethdb
Compare commits
18 Commits
v5
...
final_twea
| Author | SHA1 | Date | |
|---|---|---|---|
|
0ccc53b4bb | ||
|
|
1574d1a968 | ||
|
|
2c4e718a67 | ||
|
|
961e10bfe8 | ||
|
|
95178107c0 | ||
|
|
531bc62eff | ||
|
|
07a7a534da | ||
|
|
400f64e7f9 | ||
|
|
6358511b19 | ||
|
|
147186e529 | ||
|
|
f5a12a8d3f | ||
|
|
470bdc5ed4 | ||
|
|
7c6660a5a6 | ||
|
|
05648112e9 | ||
|
|
dece84dc59 | ||
|
|
520d6b9b90 | ||
|
|
f3751d2b3c | ||
|
|
0b26f2cc8b |
@ -71,7 +71,7 @@ func main() {
|
||||
|
||||
database := ipfsethdb.NewDatabase(ipfsNode.Blocks)
|
||||
stateDatabase := state.NewDatabase(database)
|
||||
stateDB, _ := state.New(common.Hash{}, stateDatabase, nil)
|
||||
stateDB, _ := state.New(common.Hash{}, stateDatabase)
|
||||
stateDBNodeIterator := state.NewNodeIterator(stateDB)
|
||||
// do stuff with the statedb node iterator
|
||||
}
|
||||
|
||||
26
database.go
26
database.go
@ -154,15 +154,23 @@ func (d *Database) NewBatch() ethdb.Batch {
|
||||
return b
|
||||
}
|
||||
|
||||
// NewIterator satisfies the ethdb.Iteratee interface
|
||||
// it creates a binary-alphabetical iterator over a subset
|
||||
// of database content with a particular key prefix, starting at a particular
|
||||
// initial key (or after, if it does not exist).
|
||||
//
|
||||
// Note: This method assumes that the prefix is NOT part of the start, so there's
|
||||
// no need for the caller to prepend the prefix to the start
|
||||
func (d *Database) NewIterator(prefix []byte, start []byte) ethdb.Iterator {
|
||||
return NewIterator(start, prefix, d.blockService)
|
||||
// NewIterator creates a binary-alphabetical iterator over the entire keyspace
|
||||
// contained within the key-value database.
|
||||
func (d *Database) NewIterator() ethdb.Iterator {
|
||||
return NewIterator([]byte{}, []byte{}, d.blockService)
|
||||
}
|
||||
|
||||
// NewIteratorWithStart creates a binary-alphabetical iterator over a subset of
|
||||
// database content starting at a particular initial key (or after, if it does
|
||||
// not exist).
|
||||
func (d *Database) NewIteratorWithStart(start []byte) ethdb.Iterator {
|
||||
return NewIterator(start, []byte{}, d.blockService)
|
||||
}
|
||||
|
||||
// NewIteratorWithPrefix creates a binary-alphabetical iterator over a subset
|
||||
// of database content with a particular key prefix.
|
||||
func (d *Database) NewIteratorWithPrefix(prefix []byte) ethdb.Iterator {
|
||||
return NewIterator([]byte{}, prefix, d.blockService)
|
||||
}
|
||||
|
||||
// Close satisfies the io.Closer interface
|
||||
|
||||
3
go.mod
3
go.mod
@ -3,7 +3,7 @@ module github.com/vulcanize/ipfs-ethdb
|
||||
go 1.13
|
||||
|
||||
require (
|
||||
github.com/ethereum/go-ethereum v1.9.15
|
||||
github.com/ethereum/go-ethereum v1.9.10
|
||||
github.com/hashicorp/golang-lru v0.5.4
|
||||
github.com/ipfs/go-block-format v0.0.2
|
||||
github.com/ipfs/go-blockservice v0.1.3
|
||||
@ -16,4 +16,5 @@ require (
|
||||
github.com/multiformats/go-multihash v0.0.13
|
||||
github.com/onsi/ginkgo v1.8.0
|
||||
github.com/onsi/gomega v1.5.0
|
||||
github.com/sirupsen/logrus v1.6.0
|
||||
)
|
||||
|
||||
26
go.sum
26
go.sum
@ -15,7 +15,10 @@ github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbt
|
||||
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||
github.com/Kubuxu/go-os-helper v0.0.1/go.mod h1:N8B+I7vPCT80IcP58r50u4+gEEcsZETFUpAzWW2ep1Y=
|
||||
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
|
||||
github.com/OneOfOne/xxhash v1.2.5/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q=
|
||||
github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
|
||||
github.com/VictoriaMetrics/fastcache v1.5.3 h1:2odJnXLbFZcoV9KYtQ+7TH1UOq3dn3AssMgieaezkR4=
|
||||
github.com/VictoriaMetrics/fastcache v1.5.3/go.mod h1:+jv9Ckb+za/P1ZRg/sulP5Ni1v49daAVERr0H3CuscE=
|
||||
github.com/VictoriaMetrics/fastcache v1.5.7 h1:4y6y0G8PRzszQUYIQHHssv/jgPHAb5qQuuDNdCbyAgw=
|
||||
github.com/VictoriaMetrics/fastcache v1.5.7/go.mod h1:ptDBkNMQI4RtmVo8VS/XwRY6RoTu1dAWCbrk+6WsEM8=
|
||||
github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII=
|
||||
@ -41,6 +44,7 @@ github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46f
|
||||
github.com/cespare/cp v0.1.0/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW+s=
|
||||
github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
|
||||
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
|
||||
github.com/cespare/xxhash/v2 v2.0.1-0.20190104013014-3767db7a7e18/go.mod h1:HD5P3vAIAh+Y2GAxg0PrPN1P8WkepXGpjbUPDHJqqKM=
|
||||
github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
|
||||
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/cloudflare/cloudflare-go v0.10.2-0.20190916151808-a80f83b9add9/go.mod h1:1MxXX1Ux4x6mqPmjkUgTP1CdXIBXKX7T+Jk9Gxrmx+U=
|
||||
@ -57,9 +61,16 @@ github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f/go.mod h1:SqUrOPUn
|
||||
github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
|
||||
github.com/dlclark/regexp2 v1.2.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc=
|
||||
github.com/docker/docker v1.4.2-0.20180625184442-8e610b2b55bf/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
|
||||
github.com/dop251/goja v0.0.0-20200106141417-aaec0e7bde29/go.mod h1:Mw6PkjjMXWbTj+nnj4s3QPXq1jaT0s5pC0iFD4+BOAA=
|
||||
github.com/dop251/goja v0.0.0-20200219165308-d1232e640a87/go.mod h1:Mw6PkjjMXWbTj+nnj4s3QPXq1jaT0s5pC0iFD4+BOAA=
|
||||
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
|
||||
github.com/edsrzf/mmap-go v0.0.0-20160512033002-935e0e8a636c/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M=
|
||||
github.com/elastic/gosigar v0.8.1-0.20180330100440-37f05ff46ffa h1:XKAhUk/dtp+CV0VO6mhG2V7jA9vbcGcnYF/Ay9NjZrY=
|
||||
github.com/elastic/gosigar v0.8.1-0.20180330100440-37f05ff46ffa/go.mod h1:cdorVVzy1fhmEqmtgqkoE3bYtCfSCkVyjTyCIo22xvs=
|
||||
github.com/ethereum/go-ethereum v1.9.10 h1:jooX7tWcscpC7ytufk73t9JMCeJQ7aJF2YmZJQEuvFo=
|
||||
github.com/ethereum/go-ethereum v1.9.10/go.mod h1:lXHkVo/MTvsEXfYsmNzelZ8R1e0DTvdk/wMZJIRpaRw=
|
||||
github.com/ethereum/go-ethereum v1.9.11 h1:Z0jugPDfuI5qsPY1XgBGVwikpdFK/ANqP7MrYvkmk+A=
|
||||
github.com/ethereum/go-ethereum v1.9.11/go.mod h1:7oC0Ni6dosMv5pxMigm6s0hN8g4haJMBnqmmo0D9YfQ=
|
||||
github.com/ethereum/go-ethereum v1.9.15 h1:wrWl+QrtutRUJ9LZXdUqBoGoo2b1tOCYRDrAOQhCY3A=
|
||||
github.com/ethereum/go-ethereum v1.9.15/go.mod h1:slT8bPPRhXsyNTwHQxrOnjuTZ1sDXRajW11EkJ84QJ0=
|
||||
github.com/fatih/color v1.3.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
|
||||
@ -92,11 +103,13 @@ github.com/gorilla/websocket v1.4.1-0.20190629185528-ae1634f6a989/go.mod h1:E7qH
|
||||
github.com/graph-gophers/graphql-go v0.0.0-20191115155744-f33e81362277/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc=
|
||||
github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU=
|
||||
github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48=
|
||||
github.com/hashicorp/golang-lru v0.0.0-20160813221303-0a025b7e63ad/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
|
||||
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
|
||||
github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc=
|
||||
github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
|
||||
github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
|
||||
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
|
||||
github.com/huin/goupnp v0.0.0-20161224104101-679507af18f3/go.mod h1:MZ2ZmwcBpvOoJ22IJsc7va19ZwoheaBk43rKg12SKag=
|
||||
github.com/huin/goupnp v1.0.0/go.mod h1:n9v9KO1tAxYH82qOn+UTIFQDmx5n1Zxd/ClZDMX7Bnc=
|
||||
github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o=
|
||||
github.com/influxdata/influxdb v1.2.3-0.20180221223340-01288bdb0883/go.mod h1:qZna6X/4elxqT3yI9iZYdZrWWdeFOOprn86kgg4+IzY=
|
||||
@ -167,6 +180,7 @@ github.com/karalabe/usb v0.0.0-20190919080040-51dc0efba356/go.mod h1:Od972xHfMJo
|
||||
github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
|
||||
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
||||
github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
github.com/koron/go-ssdp v0.0.0-20180514024734-4a0ed625a78b/go.mod h1:5Ky9EC2xfoUKUor0Hjgi2BJhCSXJfMOFlmyYrVKGQMk=
|
||||
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
|
||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
@ -291,15 +305,19 @@ github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7q
|
||||
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
|
||||
github.com/prometheus/tsdb v0.6.2-0.20190402121629-4f204dcbc150/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
|
||||
github.com/rjeczalik/notify v0.9.1/go.mod h1:rKwnCoCGeuQnwBtTSPL9Dad03Vh2n40ePRrjvIXnJho=
|
||||
github.com/robertkrimen/otto v0.0.0-20170205013659-6a77b7cbc37d/go.mod h1:xvqspoSXJTIpemEonrMDFq6XzwHYYgToXWj5eRX1OtY=
|
||||
github.com/rs/cors v0.0.0-20160617231935-a62a804a8a00/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
|
||||
github.com/rs/xhandler v0.0.0-20160618193221-ed27b6fd6521/go.mod h1:RvLn4FgxWubrpZHtQLnOf6EwhN2hEMusxZOhcW9H3UQ=
|
||||
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||
github.com/shirou/gopsutil v2.20.5-0.20200531151128-663af789c085+incompatible h1:+gAR1bMhuoQnZMTWFIvp7ukynULPsteLzG+siZKLtD8=
|
||||
github.com/shirou/gopsutil v2.20.5-0.20200531151128-663af789c085+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
|
||||
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
|
||||
github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I=
|
||||
github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
|
||||
github.com/spacemonkeygo/openssl v0.0.0-20181017203307-c2dcc5cca94a/go.mod h1:7AyxJNCJ7SBZ1MfVQCWD6Uqo2oubI2Eq2y2eqf+A5r0=
|
||||
github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572/go.mod h1:w0SWMsp6j9O/dk4/ZpIhL+3CkG8ofA2vuv7k+ltqUMc=
|
||||
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
|
||||
github.com/spaolacci/murmur3 v1.0.1-0.20190317074736-539464a789e9/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
|
||||
github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
|
||||
github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
|
||||
github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4/go.mod h1:RZLeN1LMWmRsyYjvAu+I6Dm9QmlDaIIt+Y+4Kd7Tp+Q=
|
||||
@ -330,6 +348,7 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk
|
||||
golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8 h1:1wopBVtVdWnn03fZelqdXTqk7U7zPQCb+T4rbU9ZEoU=
|
||||
golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20200311171314-f7b00557c8c4 h1:QmwruyY+bKbDDL0BaglrbZABEali68eoMFhTZpCjYVA=
|
||||
golang.org/x/crypto v0.0.0-20200311171314-f7b00557c8c4/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
@ -338,6 +357,8 @@ golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73r
|
||||
golang.org/x/net v0.0.0-20190227160552-c95aed5357e7/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190611141213-3f473d35a33a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20190628185345-da137c7871d7 h1:rTIdg5QFRR7XCaK4LCjBiPbx8j4DQRpdYMnGn/bJUEU=
|
||||
golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200425230154-ff2c4b7c35a0 h1:Jcxah/M+oLZ/R4/z5RzfPzGbPXnVDPkEDtf2JnuxN+U=
|
||||
golang.org/x/net v0.0.0-20200425230154-ff2c4b7c35a0/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
|
||||
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
@ -349,7 +370,10 @@ golang.org/x/sys v0.0.0-20190219092855-153ac476189d/go.mod h1:STP8DvDyc/dI5b8T5h
|
||||
golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190228124157-a34e9553db1e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7 h1:LepdCS8Gf/MVejFIt8lsiexZATdoGVyp5bcyS+rYoUI=
|
||||
golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884=
|
||||
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
@ -367,7 +391,9 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8
|
||||
gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
|
||||
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
|
||||
gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce/go.mod h1:5AcXVHNjg+BDxry382+8OKon8SEWiKktQR07RKPsv1c=
|
||||
gopkg.in/olebedev/go-duktape.v3 v3.0.0-20190213234257-ec84240a7772/go.mod h1:uAJfkITjFhyEEuUfm7bsmCZRbW5WRq8s9EY8HZ6hCns=
|
||||
gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200603215123-a4a8cb9d2cbc/go.mod h1:uAJfkITjFhyEEuUfm7bsmCZRbW5WRq8s9EY8HZ6hCns=
|
||||
gopkg.in/sourcemap.v1 v1.0.5/go.mod h1:2RlvNNSMglmRrcvhfuzp4hQHwOtjxlbjX7UPY/GXb78=
|
||||
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
|
||||
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
|
||||
gopkg.in/urfave/cli.v1 v1.20.0/go.mod h1:vuBzUtMdQeixQj8LVd+/98pzhxNGQoyuPBlsXHOQNO0=
|
||||
|
||||
238
postgres/ancient.go
Normal file
238
postgres/ancient.go
Normal file
@ -0,0 +1,238 @@
|
||||
// VulcanizeDB
|
||||
// Copyright © 2020 Vulcanize
|
||||
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package pgipfsethdb
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
const (
|
||||
// FreezerHeaderTable indicates the name of the freezer header table.
|
||||
FreezerHeaderTable = "headers"
|
||||
|
||||
// FreezerHashTable indicates the name of the freezer canonical hash table.
|
||||
FreezerHashTable = "hashes"
|
||||
|
||||
// FreezerBodiesTable indicates the name of the freezer block body table.
|
||||
FreezerBodiesTable = "bodies"
|
||||
|
||||
// FreezerReceiptTable indicates the name of the freezer receipts table.
|
||||
FreezerReceiptTable = "receipts"
|
||||
|
||||
// FreezerDifficultyTable indicates the name of the freezer total difficulty table.
|
||||
FreezerDifficultyTable = "diffs"
|
||||
|
||||
// ancient append Postgres statements
|
||||
appendAncientHeaderPgStr = "INSERT INTO eth.ancient_headers (block_number, header) VALUES ($1, $2) ON CONFLICT (block_number) DO UPDATE SET header = $2"
|
||||
appendAncientHashPgStr = "INSERT INTO eth.ancient_hashes (block_number, hash) VALUES ($1, $2) ON CONFLICT (block_number) DO UPDATE SET hash = $2"
|
||||
appendAncientBodyPgStr = "INSERT INTO eth.ancient_bodies (block_number, body) VALUES ($1, $2) ON CONFLICT (block_number) DO UPDATE SET body = $2"
|
||||
appendAncientReceiptsPgStr = "INSERT INTO eth.ancient_receipts (block_number, receipts) VALUES ($1, $2) ON CONFLICT (block_number) DO UPDATE SET receipts = $2"
|
||||
appendAncientTDPgStr = "INSERT INTO eth.ancient_tds (block_number, td) VALUES ($1, $2) ON CONFLICT (block_number) DO UPDATE SET td = $2"
|
||||
|
||||
// ancient truncate Postgres statements
|
||||
truncateAncientHeaderPgStr = "DELETE FROM eth.ancient_headers WHERE block_number > $1"
|
||||
truncateAncientHashPgStr = "DELETE FROM eth.ancient_hashes WHERE block_number > $1"
|
||||
truncateAncientBodiesPgStr = "DELETE FROM eth.ancient_bodies WHERE block_number > $1"
|
||||
truncateAncientReceiptsPgStr = "DELETE FROM eth.ancient_receipts WHERE block_number > $1"
|
||||
truncateAncientTDPgStr = "DELETE FROM eth.ancient_tds WHERE block_number > $1"
|
||||
|
||||
// ancient size Postgres statement
|
||||
ancientSizePgStr = "SELECT pg_total_relation_size($1)"
|
||||
|
||||
// ancients Postgres statement
|
||||
ancientsPgStr = "SELECT block_number FROM eth.ancient_headers ORDER BY block_number DESC LIMIT 1"
|
||||
|
||||
// ancient has Postgres statements
|
||||
hasAncientHeaderPgStr = "SELECT exists(SELECT 1 FROM eth.ancient_headers WHERE block_number = $1)"
|
||||
hasAncientHashPgStr = "SELECT exists(SELECT 1 FROM eth.ancient_hashes WHERE block_number = $1)"
|
||||
hasAncientBodyPgStr = "SELECT exists(SELECT 1 FROM eth.ancient_bodies WHERE block_number = $1)"
|
||||
hasAncientReceiptsPgStr = "SELECT exists(SELECT 1 FROM eth.ancient_receipts WHERE block_number = $1)"
|
||||
hasAncientTDPgStr = "SELECT exists(SELECT 1 FROM eth.ancient_tds WHERE block_number = $1)"
|
||||
|
||||
// ancient get Postgres statements
|
||||
getAncientHeaderPgStr = "SELECT header FROM eth.ancient_headers WHERE block_number = $1"
|
||||
getAncientHashPgStr = "SELECT hash FROM eth.ancient_hashes WHERE block_number = $1"
|
||||
getAncientBodyPgStr = "SELECT body FROM eth.ancient_bodies WHERE block_number = $1"
|
||||
getAncientReceiptsPgStr = "SELECT receipts FROM eth.ancient_receipts WHERE block_number = $1"
|
||||
getAncientTDPgStr = "SELECT td FROM eth.ancient_tds WHERE block_number = $1"
|
||||
)
|
||||
|
||||
// HasAncient satisfies the ethdb.AncientReader interface
|
||||
// HasAncient returns an indicator whether the specified data exists in the ancient store
|
||||
func (d *Database) HasAncient(kind string, number uint64) (bool, error) {
|
||||
var pgStr string
|
||||
switch kind {
|
||||
case FreezerHeaderTable:
|
||||
pgStr = hasAncientHeaderPgStr
|
||||
case FreezerHashTable:
|
||||
pgStr = hasAncientHashPgStr
|
||||
case FreezerBodiesTable:
|
||||
pgStr = hasAncientBodyPgStr
|
||||
case FreezerReceiptTable:
|
||||
pgStr = hasAncientReceiptsPgStr
|
||||
case FreezerDifficultyTable:
|
||||
pgStr = hasAncientTDPgStr
|
||||
default:
|
||||
return false, fmt.Errorf("unexpected ancient kind: %s", kind)
|
||||
}
|
||||
has := new(bool)
|
||||
return *has, d.db.Get(has, pgStr, number)
|
||||
}
|
||||
|
||||
// Ancient satisfies the ethdb.AncientReader interface
|
||||
// Ancient retrieves an ancient binary blob from the append-only immutable files
|
||||
func (d *Database) Ancient(kind string, number uint64) ([]byte, error) {
|
||||
var pgStr string
|
||||
switch kind {
|
||||
case FreezerHeaderTable:
|
||||
pgStr = getAncientHeaderPgStr
|
||||
case FreezerHashTable:
|
||||
pgStr = getAncientHashPgStr
|
||||
case FreezerBodiesTable:
|
||||
pgStr = getAncientBodyPgStr
|
||||
case FreezerReceiptTable:
|
||||
pgStr = getAncientReceiptsPgStr
|
||||
case FreezerDifficultyTable:
|
||||
pgStr = getAncientTDPgStr
|
||||
default:
|
||||
return nil, fmt.Errorf("unexpected ancient kind: %s", kind)
|
||||
}
|
||||
data := new([]byte)
|
||||
return *data, d.db.Get(data, pgStr, number)
|
||||
}
|
||||
|
||||
// Ancients satisfies the ethdb.AncientReader interface
|
||||
// Ancients returns the ancient item numbers in the ancient store
|
||||
func (d *Database) Ancients() (uint64, error) {
|
||||
num := new(uint64)
|
||||
if err := d.db.Get(num, ancientsPgStr); err != nil {
|
||||
if err == sql.ErrNoRows {
|
||||
return 0, nil
|
||||
}
|
||||
return 0, err
|
||||
}
|
||||
return *num, nil
|
||||
}
|
||||
|
||||
// AncientSize satisfies the ethdb.AncientReader interface
|
||||
// AncientSize returns the ancient size of the specified category
|
||||
func (d *Database) AncientSize(kind string) (uint64, error) {
|
||||
var tableName string
|
||||
switch kind {
|
||||
case FreezerHeaderTable:
|
||||
tableName = "eth.ancient_headers"
|
||||
case FreezerHashTable:
|
||||
tableName = "eth.ancient_hashes"
|
||||
case FreezerBodiesTable:
|
||||
tableName = "eth.ancient_bodies"
|
||||
case FreezerReceiptTable:
|
||||
tableName = "eth.ancient_receipts"
|
||||
case FreezerDifficultyTable:
|
||||
tableName = "eth.ancient_tds"
|
||||
default:
|
||||
return 0, fmt.Errorf("unexpected ancient kind: %s", kind)
|
||||
}
|
||||
size := new(uint64)
|
||||
return *size, d.db.Get(size, ancientSizePgStr, tableName)
|
||||
}
|
||||
|
||||
// AppendAncient satisfies the ethdb.AncientWriter interface
|
||||
// AppendAncient injects all binary blobs belong to block at the end of the append-only immutable table files
|
||||
func (d *Database) AppendAncient(number uint64, hash, header, body, receipts, td []byte) error {
|
||||
// append in batch
|
||||
var err error
|
||||
if d.ancientTx == nil {
|
||||
d.ancientTx, err = d.db.Beginx()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
defer func() {
|
||||
if err != nil {
|
||||
if err := d.ancientTx.Rollback(); err != nil {
|
||||
logrus.Error(err)
|
||||
d.ancientTx = nil
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
if _, err := d.ancientTx.Exec(appendAncientHashPgStr, number, hash); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := d.ancientTx.Exec(appendAncientHeaderPgStr, number, header); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := d.ancientTx.Exec(appendAncientBodyPgStr, number, body); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := d.ancientTx.Exec(appendAncientReceiptsPgStr, number, receipts); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = d.ancientTx.Exec(appendAncientTDPgStr, number, td)
|
||||
return err
|
||||
}
|
||||
|
||||
// TruncateAncients satisfies the ethdb.AncientWriter interface
|
||||
// TruncateAncients discards all but the first n ancient data from the ancient store
|
||||
func (d *Database) TruncateAncients(n uint64) error {
|
||||
// truncate in batch
|
||||
var err error
|
||||
if d.ancientTx == nil {
|
||||
d.ancientTx, err = d.db.Beginx()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
defer func() {
|
||||
if err != nil {
|
||||
if err := d.ancientTx.Rollback(); err != nil {
|
||||
logrus.Error(err)
|
||||
d.ancientTx = nil
|
||||
}
|
||||
}
|
||||
}()
|
||||
if _, err := d.ancientTx.Exec(truncateAncientHeaderPgStr, n); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := d.ancientTx.Exec(truncateAncientHashPgStr, n); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := d.ancientTx.Exec(truncateAncientBodiesPgStr, n); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := d.ancientTx.Exec(truncateAncientReceiptsPgStr, n); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = d.ancientTx.Exec(truncateAncientTDPgStr, n)
|
||||
return err
|
||||
}
|
||||
|
||||
// Sync satisfies the ethdb.AncientWriter interface
|
||||
// Sync flushes all in-memory ancient store data to disk
|
||||
func (d *Database) Sync() error {
|
||||
if d.ancientTx == nil {
|
||||
return nil
|
||||
}
|
||||
if err := d.ancientTx.Commit(); err != nil {
|
||||
return err
|
||||
}
|
||||
d.ancientTx = nil
|
||||
return nil
|
||||
}
|
||||
232
postgres/ancient_test.go
Normal file
232
postgres/ancient_test.go
Normal file
@ -0,0 +1,232 @@
|
||||
// VulcanizeDB
|
||||
// Copyright © 2019 Vulcanize
|
||||
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package pgipfsethdb_test
|
||||
|
||||
import (
|
||||
"math/big"
|
||||
|
||||
"github.com/ethereum/go-ethereum/core/types"
|
||||
"github.com/ethereum/go-ethereum/ethdb"
|
||||
"github.com/ethereum/go-ethereum/rlp"
|
||||
. "github.com/onsi/ginkgo"
|
||||
. "github.com/onsi/gomega"
|
||||
|
||||
"github.com/vulcanize/ipfs-ethdb/postgres"
|
||||
)
|
||||
|
||||
var (
|
||||
ancientDB ethdb.Database
|
||||
testBlockNumber uint64 = 1
|
||||
testAncientHeader = types.Header{Number: big.NewInt(2)}
|
||||
testAncientHeaderRLP, _ = rlp.EncodeToBytes(testHeader2)
|
||||
testAncientHash = testAncientHeader.Hash().Bytes()
|
||||
testAncientBodyBytes = make([]byte, 10000)
|
||||
testAncientReceiptsBytes = make([]byte, 5000)
|
||||
testAncientTD, _ = new(big.Int).SetString("1000000000000000000000", 10)
|
||||
testAncientTDBytes = testAncientTD.Bytes()
|
||||
)
|
||||
|
||||
var _ = Describe("Ancient", func() {
|
||||
BeforeEach(func() {
|
||||
db, err = pgipfsethdb.TestDB()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
ancientDB = pgipfsethdb.NewDatabase(db)
|
||||
|
||||
})
|
||||
AfterEach(func() {
|
||||
err = pgipfsethdb.ResetTestDB(db)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
})
|
||||
|
||||
Describe("AppendAncient/Sync/Has", func() {
|
||||
It("adds eth objects to the Ancient database and returns whether or not an ancient record exists", func() {
|
||||
hasAncient(testBlockNumber, false)
|
||||
|
||||
err = ancientDB.AppendAncient(testBlockNumber, testAncientHash, testAncientHeaderRLP, testAncientBodyBytes, testAncientReceiptsBytes, testAncientTDBytes)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
hasAncient(testBlockNumber, false)
|
||||
|
||||
err = ancientDB.Sync()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
hasAncient(testBlockNumber, true)
|
||||
})
|
||||
})
|
||||
|
||||
Describe("AppendAncient/Sync/Ancient", func() {
|
||||
It("adds the eth objects to the Ancient database and returns the ancient objects on request", func() {
|
||||
hasAncient(testBlockNumber, false)
|
||||
|
||||
_, err := ancientDB.Ancient(pgipfsethdb.FreezerHeaderTable, testBlockNumber)
|
||||
Expect(err).To(HaveOccurred())
|
||||
_, err = ancientDB.Ancient(pgipfsethdb.FreezerHashTable, testBlockNumber)
|
||||
Expect(err).To(HaveOccurred())
|
||||
_, err = ancientDB.Ancient(pgipfsethdb.FreezerBodiesTable, testBlockNumber)
|
||||
Expect(err).To(HaveOccurred())
|
||||
_, err = ancientDB.Ancient(pgipfsethdb.FreezerReceiptTable, testBlockNumber)
|
||||
Expect(err).To(HaveOccurred())
|
||||
_, err = ancientDB.Ancient(pgipfsethdb.FreezerDifficultyTable, testBlockNumber)
|
||||
Expect(err).To(HaveOccurred())
|
||||
|
||||
err = ancientDB.AppendAncient(testBlockNumber, testAncientHash, testAncientHeaderRLP, testAncientBodyBytes, testAncientReceiptsBytes, testAncientTDBytes)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
err = ancientDB.Sync()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
hasAncient(testBlockNumber, true)
|
||||
|
||||
ancientHeader, err := ancientDB.Ancient(pgipfsethdb.FreezerHeaderTable, testBlockNumber)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(ancientHeader).To(Equal(testAncientHeaderRLP))
|
||||
|
||||
ancientHash, err := ancientDB.Ancient(pgipfsethdb.FreezerHashTable, testBlockNumber)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(ancientHash).To(Equal(testAncientHash))
|
||||
|
||||
ancientBody, err := ancientDB.Ancient(pgipfsethdb.FreezerBodiesTable, testBlockNumber)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(ancientBody).To(Equal(testAncientBodyBytes))
|
||||
|
||||
ancientReceipts, err := ancientDB.Ancient(pgipfsethdb.FreezerReceiptTable, testBlockNumber)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(ancientReceipts).To(Equal(testAncientReceiptsBytes))
|
||||
|
||||
ancientTD, err := ancientDB.Ancient(pgipfsethdb.FreezerDifficultyTable, testBlockNumber)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(ancientTD).To(Equal(testAncientTDBytes))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("AppendAncient/Sync/Ancients", func() {
|
||||
It("returns the height of the ancient database", func() {
|
||||
ancients, err := ancientDB.Ancients()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(ancients).To(Equal(uint64(0)))
|
||||
|
||||
for i := uint64(0); i <= 100; i++ {
|
||||
hasAncient(i, false)
|
||||
err = ancientDB.AppendAncient(i, testAncientHash, testAncientHeaderRLP, testAncientBodyBytes, testAncientReceiptsBytes, testAncientTDBytes)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
}
|
||||
|
||||
err = ancientDB.Sync()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
for i := uint64(0); i <= 100; i++ {
|
||||
hasAncient(i, true)
|
||||
}
|
||||
ancients, err = ancientDB.Ancients()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(ancients).To(Equal(uint64(100)))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("AppendAncient/Truncate/Sync", func() {
|
||||
It("truncates the ancient database to the provided height", func() {
|
||||
for i := uint64(0); i <= 100; i++ {
|
||||
hasAncient(i, false)
|
||||
err = ancientDB.AppendAncient(i, testAncientHash, testAncientHeaderRLP, testAncientBodyBytes, testAncientReceiptsBytes, testAncientTDBytes)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
}
|
||||
|
||||
err = ancientDB.Sync()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
err = ancientDB.TruncateAncients(50)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
for i := uint64(0); i <= 100; i++ {
|
||||
hasAncient(i, true)
|
||||
}
|
||||
|
||||
ancients, err := ancientDB.Ancients()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(ancients).To(Equal(uint64(100)))
|
||||
|
||||
err = ancientDB.Sync()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
for i := uint64(0); i <= 100; i++ {
|
||||
if i <= 50 {
|
||||
hasAncient(i, true)
|
||||
} else {
|
||||
hasAncient(i, false)
|
||||
}
|
||||
}
|
||||
|
||||
ancients, err = ancientDB.Ancients()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(ancients).To(Equal(uint64(50)))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("AppendAncient/Sync/AncientSize", func() {
|
||||
It("adds the eth objects to the Ancient database and returns the ancient objects on request", func() {
|
||||
for i := uint64(0); i <= 100; i++ {
|
||||
hasAncient(i, false)
|
||||
err = ancientDB.AppendAncient(i, testAncientHash, testAncientHeaderRLP, testAncientBodyBytes, testAncientReceiptsBytes, testAncientTDBytes)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
}
|
||||
|
||||
err = ancientDB.Sync()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
for i := uint64(0); i <= 100; i++ {
|
||||
hasAncient(i, true)
|
||||
}
|
||||
|
||||
ancientHeaderSize, err := ancientDB.AncientSize(pgipfsethdb.FreezerHeaderTable)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(ancientHeaderSize).To(Equal(uint64(106496)))
|
||||
|
||||
ancientHashSize, err := ancientDB.AncientSize(pgipfsethdb.FreezerHashTable)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(ancientHashSize).To(Equal(uint64(32768)))
|
||||
|
||||
ancientBodySize, err := ancientDB.AncientSize(pgipfsethdb.FreezerBodiesTable)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(ancientBodySize).To(Equal(uint64(73728)))
|
||||
|
||||
ancientReceiptsSize, err := ancientDB.AncientSize(pgipfsethdb.FreezerReceiptTable)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(ancientReceiptsSize).To(Equal(uint64(65536)))
|
||||
|
||||
ancientTDSize, err := ancientDB.AncientSize(pgipfsethdb.FreezerDifficultyTable)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(ancientTDSize).To(Equal(uint64(32768)))
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
func hasAncient(blockNumber uint64, shouldHave bool) {
|
||||
has, err := ancientDB.HasAncient(pgipfsethdb.FreezerHeaderTable, blockNumber)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(has).To(Equal(shouldHave))
|
||||
has, err = ancientDB.HasAncient(pgipfsethdb.FreezerHashTable, blockNumber)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(has).To(Equal(shouldHave))
|
||||
has, err = ancientDB.HasAncient(pgipfsethdb.FreezerBodiesTable, blockNumber)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(has).To(Equal(shouldHave))
|
||||
has, err = ancientDB.HasAncient(pgipfsethdb.FreezerReceiptTable, blockNumber)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(has).To(Equal(shouldHave))
|
||||
has, err = ancientDB.HasAncient(pgipfsethdb.FreezerDifficultyTable, blockNumber)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(has).To(Equal(shouldHave))
|
||||
}
|
||||
@ -17,22 +17,25 @@
|
||||
package pgipfsethdb
|
||||
|
||||
import (
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
"github.com/ethereum/go-ethereum/ethdb"
|
||||
"github.com/jmoiron/sqlx"
|
||||
)
|
||||
|
||||
// Batch is the type that satisfies the ethdb.Batch interface for PG-IPFS Ethereum data using a direct Postgres connection
|
||||
type Batch struct {
|
||||
db *sqlx.DB
|
||||
tx *sqlx.Tx
|
||||
valueSize int
|
||||
db *sqlx.DB
|
||||
tx *sqlx.Tx
|
||||
valueSize int
|
||||
replayCache map[string][]byte
|
||||
}
|
||||
|
||||
// NewBatch returns a ethdb.Batch interface for PG-IPFS
|
||||
func NewBatch(db *sqlx.DB, tx *sqlx.Tx) ethdb.Batch {
|
||||
b := &Batch{
|
||||
db: db,
|
||||
tx: tx,
|
||||
db: db,
|
||||
tx: tx,
|
||||
replayCache: make(map[string][]byte),
|
||||
}
|
||||
if tx == nil {
|
||||
b.Reset()
|
||||
@ -44,26 +47,30 @@ func NewBatch(db *sqlx.DB, tx *sqlx.Tx) ethdb.Batch {
|
||||
// Put inserts the given value into the key-value data store
|
||||
// Key is expected to be the keccak256 hash of value
|
||||
func (b *Batch) Put(key []byte, value []byte) (err error) {
|
||||
mhKey, err := MultihashKeyFromKeccak256(key)
|
||||
dsKey, prefix, err := DatastoreKeyFromGethKey(key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err = b.tx.Exec(putPgStr, mhKey, value); err != nil {
|
||||
if _, err = b.tx.Exec(putPgStr, dsKey, value); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err = b.tx.Exec(putPreimagePgStr, key, dsKey, prefix); err != nil {
|
||||
return err
|
||||
}
|
||||
b.valueSize += len(value)
|
||||
b.replayCache[common.Bytes2Hex(key)] = value
|
||||
return nil
|
||||
}
|
||||
|
||||
// Delete satisfies the ethdb.Batch interface
|
||||
// Delete removes the key from the key-value data store
|
||||
func (b *Batch) Delete(key []byte) (err error) {
|
||||
mhKey, err := MultihashKeyFromKeccak256(key)
|
||||
_, err = b.tx.Exec(deletePgStr, key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = b.tx.Exec(deletePgStr, mhKey)
|
||||
return err
|
||||
delete(b.replayCache, common.Bytes2Hex(key))
|
||||
return nil
|
||||
}
|
||||
|
||||
// ValueSize satisfies the ethdb.Batch interface
|
||||
@ -75,17 +82,32 @@ func (b *Batch) ValueSize() int {
|
||||
|
||||
// Write satisfies the ethdb.Batch interface
|
||||
// Write flushes any accumulated data to disk
|
||||
// Reset should be called after every write
|
||||
func (b *Batch) Write() error {
|
||||
if b.tx == nil {
|
||||
return nil
|
||||
}
|
||||
return b.tx.Commit()
|
||||
if err := b.tx.Commit(); err != nil {
|
||||
return err
|
||||
}
|
||||
b.replayCache = nil
|
||||
return nil
|
||||
}
|
||||
|
||||
// Replay satisfies the ethdb.Batch interface
|
||||
// Replay replays the batch contents
|
||||
func (b *Batch) Replay(w ethdb.KeyValueWriter) error {
|
||||
return errNotSupported
|
||||
if b.tx != nil {
|
||||
b.tx.Rollback()
|
||||
b.tx = nil
|
||||
}
|
||||
for key, value := range b.replayCache {
|
||||
if err := w.Put(common.Hex2Bytes(key), value); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
b.replayCache = nil
|
||||
return nil
|
||||
}
|
||||
|
||||
// Reset satisfies the ethdb.Batch interface
|
||||
@ -97,5 +119,6 @@ func (b *Batch) Reset() {
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
b.replayCache = make(map[string][]byte)
|
||||
b.valueSize = 0
|
||||
}
|
||||
|
||||
@ -29,10 +29,10 @@ import (
|
||||
)
|
||||
|
||||
var (
|
||||
batch ethdb.Batch
|
||||
testHeader2 = types.Header{Number: big.NewInt(2)}
|
||||
testValue2, _ = rlp.EncodeToBytes(testHeader2)
|
||||
testEthKey2 = testHeader2.Hash().Bytes()
|
||||
batch ethdb.Batch
|
||||
testHeader2 = types.Header{Number: big.NewInt(2)}
|
||||
testValue2, _ = rlp.EncodeToBytes(testHeader2)
|
||||
testKeccakEthKey2 = testHeader2.Hash().Bytes()
|
||||
)
|
||||
|
||||
var _ = Describe("Batch", func() {
|
||||
@ -49,24 +49,24 @@ var _ = Describe("Batch", func() {
|
||||
|
||||
Describe("Put/Write", func() {
|
||||
It("adds the key-value pair to the batch", func() {
|
||||
_, err = database.Get(testEthKey)
|
||||
_, err = database.Get(testKeccakEthKey)
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
|
||||
_, err = database.Get(testEthKey2)
|
||||
_, err = database.Get(testKeccakEthKey2)
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
|
||||
|
||||
err = batch.Put(testEthKey, testValue)
|
||||
err = batch.Put(testKeccakEthKey, testValue)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
err = batch.Put(testEthKey2, testValue2)
|
||||
err = batch.Put(testKeccakEthKey2, testValue2)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
err = batch.Write()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
val, err := database.Get(testEthKey)
|
||||
val, err := database.Get(testKeccakEthKey)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(val).To(Equal(testValue))
|
||||
val2, err := database.Get(testEthKey2)
|
||||
val2, err := database.Get(testKeccakEthKey2)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(val2).To(Equal(testValue2))
|
||||
})
|
||||
@ -74,25 +74,25 @@ var _ = Describe("Batch", func() {
|
||||
|
||||
Describe("Delete/Reset/Write", func() {
|
||||
It("deletes the key-value pair in the batch", func() {
|
||||
err = batch.Put(testEthKey, testValue)
|
||||
err = batch.Put(testKeccakEthKey, testValue)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
err = batch.Put(testEthKey2, testValue2)
|
||||
err = batch.Put(testKeccakEthKey2, testValue2)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
err = batch.Write()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
batch.Reset()
|
||||
err = batch.Delete(testEthKey)
|
||||
err = batch.Delete(testKeccakEthKey)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
err = batch.Delete(testEthKey2)
|
||||
err = batch.Delete(testKeccakEthKey2)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
err = batch.Write()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
_, err = database.Get(testEthKey)
|
||||
_, err = database.Get(testKeccakEthKey)
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
|
||||
_, err = database.Get(testEthKey2)
|
||||
_, err = database.Get(testKeccakEthKey2)
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
|
||||
})
|
||||
@ -100,9 +100,9 @@ var _ = Describe("Batch", func() {
|
||||
|
||||
Describe("ValueSize/Reset", func() {
|
||||
It("returns the size of data in the batch queued for write", func() {
|
||||
err = batch.Put(testEthKey, testValue)
|
||||
err = batch.Put(testKeccakEthKey, testValue)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
err = batch.Put(testEthKey2, testValue2)
|
||||
err = batch.Put(testKeccakEthKey2, testValue2)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
err = batch.Write()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
@ -115,4 +115,30 @@ var _ = Describe("Batch", func() {
|
||||
Expect(size).To(Equal(0))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("Replay", func() {
|
||||
It("returns the size of data in the batch queued for write", func() {
|
||||
err = batch.Put(testKeccakEthKey, testValue)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
err = batch.Put(testKeccakEthKey2, testValue2)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
_, err = database.Get(testKeccakEthKey)
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
|
||||
_, err = database.Get(testKeccakEthKey2)
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
|
||||
|
||||
err = batch.Replay(database)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
val, err := database.Get(testKeccakEthKey)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(val).To(Equal(testValue))
|
||||
val2, err := database.Get(testKeccakEthKey2)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(val2).To(Equal(testValue2))
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@ -17,27 +17,28 @@
|
||||
package pgipfsethdb
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/sirupsen/logrus"
|
||||
|
||||
"github.com/ethereum/go-ethereum/ethdb"
|
||||
"github.com/jmoiron/sqlx"
|
||||
)
|
||||
|
||||
var errNotSupported = errors.New("this operation is not supported")
|
||||
|
||||
var (
|
||||
hasPgStr = "SELECT exists(select 1 from public.blocks WHERE key = $1)"
|
||||
getPgStr = "SELECT data FROM public.blocks WHERE key = $1"
|
||||
putPgStr = "INSERT INTO public.blocks (key, data) VALUES ($1, $2) ON CONFLICT (key) DO NOTHING"
|
||||
deletePgStr = "DELETE FROM public.blocks WHERE key = $1"
|
||||
dbSizePgStr = "SELECT pg_database_size(current_database())"
|
||||
const (
|
||||
hasPgStr = "SELECT exists(SELECT 1 FROM eth.key_preimages WHERE eth_key = $1)"
|
||||
getPgStr = "SELECT data FROM public.blocks INNER JOIN eth.key_preimages ON (ipfs_key = blocks.key) WHERE eth_key = $1"
|
||||
putPgStr = "INSERT INTO public.blocks (key, data) VALUES ($1, $2) ON CONFLICT (key) DO NOTHING"
|
||||
putPreimagePgStr = "INSERT INTO eth.key_preimages (eth_key, ipfs_key, prefix) VALUES ($1, $2, $3) ON CONFLICT (eth_key) DO UPDATE SET (ipfs_key, prefix) = ($2, $3)"
|
||||
deletePgStr = "DELETE FROM public.blocks USING eth.key_preimages WHERE ipfs_key = blocks.key AND eth_key = $1"
|
||||
dbSizePgStr = "SELECT pg_database_size(current_database())"
|
||||
)
|
||||
|
||||
// Database is the type that satisfies the ethdb.Database and ethdb.KeyValueStore interfaces for PG-IPFS Ethereum data using a direct Postgres connection
|
||||
type Database struct {
|
||||
db *sqlx.DB
|
||||
db *sqlx.DB
|
||||
ancientTx *sqlx.Tx
|
||||
}
|
||||
|
||||
// NewKeyValueStore returns a ethdb.KeyValueStore interface for PG-IPFS
|
||||
@ -56,46 +57,54 @@ func NewDatabase(db *sqlx.DB) ethdb.Database {
|
||||
|
||||
// Has satisfies the ethdb.KeyValueReader interface
|
||||
// Has retrieves if a key is present in the key-value data store
|
||||
// Has uses the eth.key_preimages table
|
||||
func (d *Database) Has(key []byte) (bool, error) {
|
||||
mhKey, err := MultihashKeyFromKeccak256(key)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
var exists bool
|
||||
return exists, d.db.Get(&exists, hasPgStr, mhKey)
|
||||
return exists, d.db.Get(&exists, hasPgStr, key)
|
||||
}
|
||||
|
||||
// Get satisfies the ethdb.KeyValueReader interface
|
||||
// Get retrieves the given key if it's present in the key-value data store
|
||||
// Get uses the eth.key_preimages table
|
||||
func (d *Database) Get(key []byte) ([]byte, error) {
|
||||
mhKey, err := MultihashKeyFromKeccak256(key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var data []byte
|
||||
return data, d.db.Get(&data, getPgStr, mhKey)
|
||||
return data, d.db.Get(&data, getPgStr, key)
|
||||
}
|
||||
|
||||
// Put satisfies the ethdb.KeyValueWriter interface
|
||||
// Put inserts the given value into the key-value data store
|
||||
// Key is expected to be the keccak256 hash of value
|
||||
// Put inserts the keccak256 key into the eth.key_preimages table
|
||||
func (d *Database) Put(key []byte, value []byte) error {
|
||||
mhKey, err := MultihashKeyFromKeccak256(key)
|
||||
dsKey, prefix, err := DatastoreKeyFromGethKey(key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = d.db.Exec(putPgStr, mhKey, value)
|
||||
tx, err := d.db.Beginx()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
if err != nil {
|
||||
if err := tx.Rollback(); err != nil {
|
||||
logrus.Error(err)
|
||||
}
|
||||
} else {
|
||||
err = tx.Commit()
|
||||
}
|
||||
}()
|
||||
if _, err = tx.Exec(putPgStr, dsKey, value); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = tx.Exec(putPreimagePgStr, key, dsKey, prefix)
|
||||
return err
|
||||
}
|
||||
|
||||
// Delete satisfies the ethdb.KeyValueWriter interface
|
||||
// Delete removes the key from the key-value data store
|
||||
// Delete uses the eth.key_preimages table
|
||||
func (d *Database) Delete(key []byte) error {
|
||||
mhKey, err := MultihashKeyFromKeccak256(key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = d.db.Exec(deletePgStr, mhKey)
|
||||
_, err := d.db.Exec(deletePgStr, key)
|
||||
return err
|
||||
}
|
||||
|
||||
@ -176,7 +185,8 @@ func (d *Database) Stat(property string) (string, error) {
|
||||
// Compact satisfies the ethdb.Compacter interface
|
||||
// Compact flattens the underlying data store for the given key range
|
||||
func (d *Database) Compact(start []byte, limit []byte) error {
|
||||
return errNotSupported
|
||||
// this leveldb functionality doesn't translate to Postgres
|
||||
return nil
|
||||
}
|
||||
|
||||
// NewBatch satisfies the ethdb.Batcher interface
|
||||
@ -186,15 +196,23 @@ func (d *Database) NewBatch() ethdb.Batch {
|
||||
return NewBatch(d.db, nil)
|
||||
}
|
||||
|
||||
// NewIterator satisfies the ethdb.Iteratee interface
|
||||
// it creates a binary-alphabetical iterator over a subset
|
||||
// of database content with a particular key prefix, starting at a particular
|
||||
// initial key (or after, if it does not exist).
|
||||
//
|
||||
// Note: This method assumes that the prefix is NOT part of the start, so there's
|
||||
// no need for the caller to prepend the prefix to the start
|
||||
func (d *Database) NewIterator(prefix []byte, start []byte) ethdb.Iterator {
|
||||
return NewIterator(start, prefix, d.db)
|
||||
// NewIterator creates a binary-alphabetical iterator over the entire keyspace
|
||||
// contained within the key-value database.
|
||||
func (d *Database) NewIterator() ethdb.Iterator {
|
||||
return NewIterator(nil, nil, d.db)
|
||||
}
|
||||
|
||||
// NewIteratorWithStart creates a binary-alphabetical iterator over a subset of
|
||||
// database content starting at a particular initial key (or after, if it does
|
||||
// not exist).
|
||||
func (d *Database) NewIteratorWithStart(start []byte) ethdb.Iterator {
|
||||
return NewIterator(start, nil, d.db)
|
||||
}
|
||||
|
||||
// NewIteratorWithPrefix creates a binary-alphabetical iterator over a subset
|
||||
// of database content with a particular key prefix.
|
||||
func (d *Database) NewIteratorWithPrefix(prefix []byte) ethdb.Iterator {
|
||||
return NewIterator(nil, prefix, d.db)
|
||||
}
|
||||
|
||||
// Close satisfies the io.Closer interface
|
||||
@ -202,45 +220,3 @@ func (d *Database) NewIterator(prefix []byte, start []byte) ethdb.Iterator {
|
||||
func (d *Database) Close() error {
|
||||
return d.db.DB.Close()
|
||||
}
|
||||
|
||||
// HasAncient satisfies the ethdb.AncientReader interface
|
||||
// HasAncient returns an indicator whether the specified data exists in the ancient store
|
||||
func (d *Database) HasAncient(kind string, number uint64) (bool, error) {
|
||||
return false, errNotSupported
|
||||
}
|
||||
|
||||
// Ancient satisfies the ethdb.AncientReader interface
|
||||
// Ancient retrieves an ancient binary blob from the append-only immutable files
|
||||
func (d *Database) Ancient(kind string, number uint64) ([]byte, error) {
|
||||
return nil, errNotSupported
|
||||
}
|
||||
|
||||
// Ancients satisfies the ethdb.AncientReader interface
|
||||
// Ancients returns the ancient item numbers in the ancient store
|
||||
func (d *Database) Ancients() (uint64, error) {
|
||||
return 0, errNotSupported
|
||||
}
|
||||
|
||||
// AncientSize satisfies the ethdb.AncientReader interface
|
||||
// AncientSize returns the ancient size of the specified category
|
||||
func (d *Database) AncientSize(kind string) (uint64, error) {
|
||||
return 0, errNotSupported
|
||||
}
|
||||
|
||||
// AppendAncient satisfies the ethdb.AncientWriter interface
|
||||
// AppendAncient injects all binary blobs belong to block at the end of the append-only immutable table files
|
||||
func (d *Database) AppendAncient(number uint64, hash, header, body, receipt, td []byte) error {
|
||||
return errNotSupported
|
||||
}
|
||||
|
||||
// TruncateAncients satisfies the ethdb.AncientWriter interface
|
||||
// TruncateAncients discards all but the first n ancient data from the ancient store
|
||||
func (d *Database) TruncateAncients(n uint64) error {
|
||||
return errNotSupported
|
||||
}
|
||||
|
||||
// Sync satisfies the ethdb.AncientWriter interface
|
||||
// Sync flushes all in-memory ancient store data to disk
|
||||
func (d *Database) Sync() error {
|
||||
return errNotSupported
|
||||
}
|
||||
|
||||
@ -19,6 +19,8 @@ package pgipfsethdb_test
|
||||
import (
|
||||
"math/big"
|
||||
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
|
||||
"github.com/ethereum/go-ethereum/core/types"
|
||||
"github.com/ethereum/go-ethereum/ethdb"
|
||||
"github.com/ethereum/go-ethereum/rlp"
|
||||
@ -30,13 +32,26 @@ import (
|
||||
)
|
||||
|
||||
var (
|
||||
database ethdb.Database
|
||||
db *sqlx.DB
|
||||
err error
|
||||
testHeader = types.Header{Number: big.NewInt(1337)}
|
||||
testValue, _ = rlp.EncodeToBytes(testHeader)
|
||||
testEthKey = testHeader.Hash().Bytes()
|
||||
testMhKey, _ = pgipfsethdb.MultihashKeyFromKeccak256(testEthKey)
|
||||
database ethdb.Database
|
||||
db *sqlx.DB
|
||||
err error
|
||||
testHeader = types.Header{Number: big.NewInt(1337)}
|
||||
testValue, _ = rlp.EncodeToBytes(testHeader)
|
||||
testKeccakEthKey = testHeader.Hash().Bytes()
|
||||
testMhKey, _ = pgipfsethdb.MultihashKeyFromKeccak256(testKeccakEthKey)
|
||||
|
||||
testPrefixedEthKey = append(append([]byte("prefix"), pgipfsethdb.KeyDelineation...), testKeccakEthKey...)
|
||||
testPrefixedDsKey = common.Bytes2Hex(testPrefixedEthKey)
|
||||
|
||||
testSuffixedEthKey = append(append(testPrefixedEthKey, pgipfsethdb.KeyDelineation...), []byte("suffix")...)
|
||||
testSuffixedDsKey = common.Bytes2Hex(testSuffixedEthKey)
|
||||
|
||||
testHeaderEthKey = append(append(append(append(pgipfsethdb.HeaderPrefix, pgipfsethdb.KeyDelineation...),
|
||||
[]byte("number")...), pgipfsethdb.NumberDelineation...), testKeccakEthKey...)
|
||||
testHeaderDsKey = testMhKey
|
||||
|
||||
testPreimageEthKey = append(append(pgipfsethdb.PreimagePrefix, pgipfsethdb.KeyDelineation...), testKeccakEthKey...)
|
||||
testPreimageDsKey = testMhKey
|
||||
)
|
||||
|
||||
var _ = Describe("Database", func() {
|
||||
@ -50,61 +65,321 @@ var _ = Describe("Database", func() {
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
})
|
||||
|
||||
Describe("Has", func() {
|
||||
Describe("Has - Keccak keys", func() {
|
||||
It("returns false if a key-pair doesn't exist in the db", func() {
|
||||
has, err := database.Has(testEthKey)
|
||||
has, err := database.Has(testKeccakEthKey)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(has).ToNot(BeTrue())
|
||||
})
|
||||
It("returns true if a key-pair exists in the db", func() {
|
||||
_, err = db.Exec("INSERT into public.blocks (key, data) VALUES ($1, $2)", testMhKey, testValue)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
has, err := database.Has(testEthKey)
|
||||
_, err = db.Exec("INSERT into eth.key_preimages (eth_key, ipfs_key) VALUES ($1, $2)", testKeccakEthKey, testMhKey)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
has, err := database.Has(testKeccakEthKey)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(has).To(BeTrue())
|
||||
})
|
||||
})
|
||||
|
||||
Describe("Get", func() {
|
||||
Describe("Has - Prefixed keys", func() {
|
||||
It("returns false if a key-pair doesn't exist in the db", func() {
|
||||
has, err := database.Has(testPrefixedEthKey)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(has).ToNot(BeTrue())
|
||||
})
|
||||
It("returns true if a key-pair exists in the db", func() {
|
||||
_, err = db.Exec("INSERT into public.blocks (key, data) VALUES ($1, $2)", testPrefixedDsKey, testValue)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
_, err = db.Exec("INSERT into eth.key_preimages (eth_key, ipfs_key) VALUES ($1, $2)", testPrefixedEthKey, testPrefixedDsKey)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
has, err := database.Has(testPrefixedEthKey)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(has).To(BeTrue())
|
||||
})
|
||||
})
|
||||
|
||||
Describe("Has - Suffixed keys", func() {
|
||||
It("returns false if a key-pair doesn't exist in the db", func() {
|
||||
has, err := database.Has(testSuffixedEthKey)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(has).ToNot(BeTrue())
|
||||
})
|
||||
It("returns true if a key-pair exists in the db", func() {
|
||||
_, err = db.Exec("INSERT into public.blocks (key, data) VALUES ($1, $2)", testSuffixedDsKey, testValue)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
_, err = db.Exec("INSERT into eth.key_preimages (eth_key, ipfs_key) VALUES ($1, $2)", testSuffixedEthKey, testSuffixedDsKey)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
has, err := database.Has(testSuffixedEthKey)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(has).To(BeTrue())
|
||||
})
|
||||
})
|
||||
|
||||
Describe("Has - Header keys", func() {
|
||||
It("returns false if a key-pair doesn't exist in the db", func() {
|
||||
has, err := database.Has(testHeaderEthKey)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(has).ToNot(BeTrue())
|
||||
})
|
||||
It("returns true if a key-pair exists in the db", func() {
|
||||
_, err = db.Exec("INSERT into public.blocks (key, data) VALUES ($1, $2)", testHeaderDsKey, testValue)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
_, err = db.Exec("INSERT into eth.key_preimages (eth_key, ipfs_key) VALUES ($1, $2)", testHeaderEthKey, testHeaderDsKey)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
has, err := database.Has(testHeaderEthKey)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(has).To(BeTrue())
|
||||
})
|
||||
})
|
||||
|
||||
Describe("Has - Preimage keys", func() {
|
||||
It("returns false if a key-pair doesn't exist in the db", func() {
|
||||
has, err := database.Has(testPreimageEthKey)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(has).ToNot(BeTrue())
|
||||
})
|
||||
It("returns true if a key-pair exists in the db", func() {
|
||||
_, err = db.Exec("INSERT into public.blocks (key, data) VALUES ($1, $2)", testPreimageDsKey, testValue)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
_, err = db.Exec("INSERT into eth.key_preimages (eth_key, ipfs_key) VALUES ($1, $2)", testPreimageEthKey, testPreimageDsKey)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
has, err := database.Has(testPreimageEthKey)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(has).To(BeTrue())
|
||||
})
|
||||
})
|
||||
|
||||
Describe("Get - Keccak keys", func() {
|
||||
It("throws an err if the key-pair doesn't exist in the db", func() {
|
||||
_, err = database.Get(testEthKey)
|
||||
_, err = database.Get(testKeccakEthKey)
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
|
||||
})
|
||||
It("returns the value associated with the key, if the pair exists", func() {
|
||||
_, err = db.Exec("INSERT into public.blocks (key, data) VALUES ($1, $2)", testMhKey, testValue)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
val, err := database.Get(testEthKey)
|
||||
_, err = db.Exec("INSERT into eth.key_preimages (eth_key, ipfs_key) VALUES ($1, $2)", testKeccakEthKey, testMhKey)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
val, err := database.Get(testKeccakEthKey)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(val).To(Equal(testValue))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("Put", func() {
|
||||
Describe("Get - Prefixed keys", func() {
|
||||
It("throws an err if the key-pair doesn't exist in the db", func() {
|
||||
_, err = database.Get(testPrefixedEthKey)
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
|
||||
})
|
||||
It("returns the value associated with the key, if the pair exists", func() {
|
||||
_, err = db.Exec("INSERT into public.blocks (key, data) VALUES ($1, $2)", testPrefixedDsKey, testValue)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
_, err = db.Exec("INSERT into eth.key_preimages (eth_key, ipfs_key) VALUES ($1, $2)", testPrefixedEthKey, testPrefixedDsKey)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
val, err := database.Get(testPrefixedEthKey)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(val).To(Equal(testValue))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("Get - Suffixed keys", func() {
|
||||
It("throws an err if the key-pair doesn't exist in the db", func() {
|
||||
_, err = database.Get(testSuffixedEthKey)
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
|
||||
})
|
||||
It("returns the value associated with the key, if the pair exists", func() {
|
||||
_, err = db.Exec("INSERT into public.blocks (key, data) VALUES ($1, $2)", testSuffixedDsKey, testValue)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
_, err = db.Exec("INSERT into eth.key_preimages (eth_key, ipfs_key) VALUES ($1, $2)", testSuffixedEthKey, testSuffixedDsKey)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
val, err := database.Get(testSuffixedEthKey)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(val).To(Equal(testValue))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("Get - Header keys", func() {
|
||||
It("throws an err if the key-pair doesn't exist in the db", func() {
|
||||
_, err = database.Get(testHeaderEthKey)
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
|
||||
})
|
||||
It("returns the value associated with the key, if the pair exists", func() {
|
||||
_, err = db.Exec("INSERT into public.blocks (key, data) VALUES ($1, $2)", testHeaderDsKey, testValue)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
_, err = db.Exec("INSERT into eth.key_preimages (eth_key, ipfs_key) VALUES ($1, $2)", testHeaderEthKey, testHeaderDsKey)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
val, err := database.Get(testHeaderEthKey)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(val).To(Equal(testValue))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("Get - Preimage keys", func() {
|
||||
It("throws an err if the key-pair doesn't exist in the db", func() {
|
||||
_, err = database.Get(testPreimageEthKey)
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
|
||||
})
|
||||
It("returns the value associated with the key, if the pair exists", func() {
|
||||
_, err = db.Exec("INSERT into public.blocks (key, data) VALUES ($1, $2)", testPreimageDsKey, testValue)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
_, err = db.Exec("INSERT into eth.key_preimages (eth_key, ipfs_key) VALUES ($1, $2)", testPreimageEthKey, testPreimageDsKey)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
val, err := database.Get(testPreimageEthKey)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(val).To(Equal(testValue))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("Put - Keccak keys", func() {
|
||||
It("persists the key-value pair in the database", func() {
|
||||
_, err = database.Get(testEthKey)
|
||||
_, err = database.Get(testKeccakEthKey)
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
|
||||
|
||||
err = database.Put(testEthKey, testValue)
|
||||
err = database.Put(testKeccakEthKey, testValue)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
val, err := database.Get(testEthKey)
|
||||
val, err := database.Get(testKeccakEthKey)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(val).To(Equal(testValue))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("Delete", func() {
|
||||
It("removes the key-value pair from the database", func() {
|
||||
err = database.Put(testEthKey, testValue)
|
||||
Describe("Put - Prefixed keys", func() {
|
||||
It("persists the key-value pair in the database", func() {
|
||||
_, err = database.Get(testPrefixedEthKey)
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
|
||||
|
||||
err = database.Put(testPrefixedEthKey, testValue)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
val, err := database.Get(testEthKey)
|
||||
val, err := database.Get(testPrefixedEthKey)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(val).To(Equal(testValue))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("Put - Suffixed keys", func() {
|
||||
It("persists the key-value pair in the database", func() {
|
||||
_, err = database.Get(testSuffixedEthKey)
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
|
||||
|
||||
err = database.Put(testSuffixedEthKey, testValue)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
val, err := database.Get(testSuffixedEthKey)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(val).To(Equal(testValue))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("Put - Header keys", func() {
|
||||
It("persists the key-value pair in the database", func() {
|
||||
_, err = database.Get(testHeaderEthKey)
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
|
||||
|
||||
err = database.Put(testHeaderEthKey, testValue)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
val, err := database.Get(testHeaderEthKey)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(val).To(Equal(testValue))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("Put - Preimage keys", func() {
|
||||
It("persists the key-value pair in the database", func() {
|
||||
_, err = database.Get(testPreimageEthKey)
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
|
||||
|
||||
err = database.Put(testPreimageEthKey, testValue)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
val, err := database.Get(testPreimageEthKey)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(val).To(Equal(testValue))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("Delete - Keccak keys", func() {
|
||||
It("removes the key-value pair from the database", func() {
|
||||
err = database.Put(testKeccakEthKey, testValue)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
val, err := database.Get(testKeccakEthKey)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(val).To(Equal(testValue))
|
||||
|
||||
err = database.Delete(testEthKey)
|
||||
err = database.Delete(testKeccakEthKey)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
_, err = database.Get(testEthKey)
|
||||
_, err = database.Get(testKeccakEthKey)
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("Delete - Prefixed keys", func() {
|
||||
It("removes the key-value pair from the database", func() {
|
||||
err = database.Put(testPrefixedEthKey, testValue)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
val, err := database.Get(testPrefixedEthKey)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(val).To(Equal(testValue))
|
||||
|
||||
err = database.Delete(testPrefixedEthKey)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
_, err = database.Get(testPrefixedEthKey)
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("Delete - Suffixed keys", func() {
|
||||
It("removes the key-value pair from the database", func() {
|
||||
err = database.Put(testSuffixedEthKey, testValue)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
val, err := database.Get(testSuffixedEthKey)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(val).To(Equal(testValue))
|
||||
|
||||
err = database.Delete(testSuffixedEthKey)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
_, err = database.Get(testSuffixedEthKey)
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("Delete - Header keys", func() {
|
||||
It("removes the key-value pair from the database", func() {
|
||||
err = database.Put(testHeaderEthKey, testValue)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
val, err := database.Get(testHeaderEthKey)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(val).To(Equal(testValue))
|
||||
|
||||
err = database.Delete(testHeaderEthKey)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
_, err = database.Get(testHeaderEthKey)
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("Delete - Preimage keys", func() {
|
||||
It("removes the key-value pair from the database", func() {
|
||||
err = database.Put(testPreimageEthKey, testValue)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
val, err := database.Get(testPreimageEthKey)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(val).To(Equal(testValue))
|
||||
|
||||
err = database.Delete(testPreimageEthKey)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
_, err = database.Get(testPreimageEthKey)
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
|
||||
})
|
||||
|
||||
@ -0,0 +1,8 @@
|
||||
-- +goose Up
|
||||
CREATE TABLE IF NOT EXISTS public.blocks (
|
||||
key TEXT UNIQUE NOT NULL,
|
||||
data BYTEA NOT NULL
|
||||
);
|
||||
|
||||
-- +goose Down
|
||||
DROP TABLE public.blocks;
|
||||
5
postgres/db/migrations/00002_create_eth_schema.sql
Normal file
5
postgres/db/migrations/00002_create_eth_schema.sql
Normal file
@ -0,0 +1,5 @@
|
||||
-- +goose Up
|
||||
CREATE SCHEMA eth;
|
||||
|
||||
-- +goose Down
|
||||
DROP SCHEMA eth;
|
||||
@ -0,0 +1,9 @@
|
||||
-- +goose Up
|
||||
CREATE TABLE IF NOT EXISTS eth.key_preimages (
|
||||
eth_key BYTEA UNIQUE NOT NULL,
|
||||
prefix BYTEA,
|
||||
ipfs_key TEXT NOT NULL REFERENCES public.blocks (key) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED
|
||||
);
|
||||
|
||||
-- +goose Down
|
||||
DROP TABLE eth.key_preimages;
|
||||
@ -0,0 +1,8 @@
|
||||
-- +goose Up
|
||||
CREATE TABLE IF NOT EXISTS eth.ancient_headers (
|
||||
block_number BIGINT UNIQUE NOT NULL,
|
||||
header BYTEA NOT NULL
|
||||
);
|
||||
|
||||
-- +goose Down
|
||||
DROP TABLE eth.ancient_headers;
|
||||
@ -0,0 +1,8 @@
|
||||
-- +goose Up
|
||||
CREATE TABLE IF NOT EXISTS eth.ancient_hashes (
|
||||
block_number BIGINT UNIQUE NOT NULL,
|
||||
hash BYTEA NOT NULL
|
||||
);
|
||||
|
||||
-- +goose Down
|
||||
DROP TABLE eth.ancient_hashes;
|
||||
@ -0,0 +1,8 @@
|
||||
-- +goose Up
|
||||
CREATE TABLE IF NOT EXISTS eth.ancient_bodies (
|
||||
block_number BIGINT UNIQUE NOT NULL,
|
||||
body BYTEA NOT NULL
|
||||
);
|
||||
|
||||
-- +goose Down
|
||||
DROP TABLE eth.ancient_bodies;
|
||||
@ -0,0 +1,8 @@
|
||||
-- +goose Up
|
||||
CREATE TABLE IF NOT EXISTS eth.ancient_receipts (
|
||||
block_number BIGINT UNIQUE NOT NULL,
|
||||
receipts BYTEA NOT NULL
|
||||
);
|
||||
|
||||
-- +goose Down
|
||||
DROP TABLE eth.ancient_receipts;
|
||||
@ -0,0 +1,8 @@
|
||||
-- +goose Up
|
||||
CREATE TABLE IF NOT EXISTS eth.ancient_tds (
|
||||
block_number BIGINT UNIQUE NOT NULL,
|
||||
td BYTEA NOT NULL
|
||||
);
|
||||
|
||||
-- +goose Down
|
||||
DROP TABLE eth.ancient_tds;
|
||||
@ -33,7 +33,7 @@ func main() {
|
||||
|
||||
database := pgipfsethdb.NewDatabase(db)
|
||||
stateDatabase := state.NewDatabase(database)
|
||||
stateDB, _ := state.New(common.Hash{}, stateDatabase, nil)
|
||||
stateDB, _ := state.New(common.Hash{}, stateDatabase)
|
||||
stateDBNodeIterator := state.NewNodeIterator(stateDB)
|
||||
// do stuff with the statedb node iterator
|
||||
}
|
||||
|
||||
@ -21,15 +21,31 @@ import (
|
||||
"github.com/jmoiron/sqlx"
|
||||
)
|
||||
|
||||
const (
|
||||
initPgStr = `SELECT eth_key, data FROM public.blocks
|
||||
INNER JOIN eth.key_preimages ON (ipfs_key = key)
|
||||
WHERE eth_key = $1`
|
||||
nextPgStr = `SELECT eth_key, data FROM public.blocks
|
||||
INNER JOIN eth.key_preimages ON (ipfs_key = key)
|
||||
WHERE eth_key > $1
|
||||
ORDER BY eth_key LIMIT 1`
|
||||
nextPgStrWithPrefix = `SELECT eth_key, data FROM public.blocks
|
||||
INNER JOIN eth.key_preimages ON (ipfs_key = key)
|
||||
WHERE eth_key > $1 AND prefix = $2
|
||||
ORDER BY eth_key LIMIT 1`
|
||||
)
|
||||
|
||||
type nextModel struct {
|
||||
Key []byte `db:"eth_key"`
|
||||
Value []byte `db:"data"`
|
||||
}
|
||||
|
||||
// Iterator is the type that satisfies the ethdb.Iterator interface for PG-IPFS Ethereum data using a direct Postgres connection
|
||||
// Iteratee interface is used in Geth for various tests, trie/sync_bloom.go (for fast sync),
|
||||
// rawdb.InspectDatabase, and the new core/state/snapshot features.
|
||||
// This should not be confused with trie.NodeIterator or state.NodeIteraor (which can be constructed
|
||||
// from the ethdb.KeyValueStoreand ethdb.Database interfaces)
|
||||
type Iterator struct {
|
||||
db *sqlx.DB
|
||||
currentKey, prefix []byte
|
||||
err error
|
||||
db *sqlx.DB
|
||||
currentKey, prefix, currentValue []byte
|
||||
err error
|
||||
init bool
|
||||
}
|
||||
|
||||
// NewIterator returns an ethdb.Iterator interface for PG-IPFS
|
||||
@ -38,6 +54,7 @@ func NewIterator(start, prefix []byte, db *sqlx.DB) ethdb.Iterator {
|
||||
db: db,
|
||||
prefix: prefix,
|
||||
currentKey: start,
|
||||
init: start != nil,
|
||||
}
|
||||
}
|
||||
|
||||
@ -45,9 +62,26 @@ func NewIterator(start, prefix []byte, db *sqlx.DB) ethdb.Iterator {
|
||||
// Next moves the iterator to the next key/value pair
|
||||
// It returns whether the iterator is exhausted
|
||||
func (i *Iterator) Next() bool {
|
||||
// this is complicated by the ipfs db keys not being the keccak256 hashes
|
||||
// go-ethereum usage of this method expects the iteration to occur over keccak256 keys
|
||||
panic("implement me: Next")
|
||||
next := new(nextModel)
|
||||
if i.init {
|
||||
i.init = false
|
||||
if err := i.db.Get(next, initPgStr, i.currentKey); err != nil {
|
||||
i.currentKey, i.currentValue, i.err = nil, nil, err
|
||||
return false
|
||||
}
|
||||
} else if i.prefix != nil {
|
||||
if err := i.db.Get(next, nextPgStrWithPrefix, i.currentKey, i.prefix); err != nil {
|
||||
i.currentKey, i.currentValue, i.err = nil, nil, err
|
||||
return false
|
||||
}
|
||||
} else {
|
||||
if err := i.db.Get(next, nextPgStr, i.currentKey); err != nil {
|
||||
i.currentKey, i.currentValue, i.err = nil, nil, err
|
||||
return false
|
||||
}
|
||||
}
|
||||
i.currentKey, i.currentValue, i.err = next.Key, next.Value, nil
|
||||
return true
|
||||
}
|
||||
|
||||
// Error satisfies the ethdb.Iterator interface
|
||||
@ -70,19 +104,12 @@ func (i *Iterator) Key() []byte {
|
||||
// The caller should not modify the contents of the returned slice
|
||||
// and its contents may change on the next call to Next
|
||||
func (i *Iterator) Value() []byte {
|
||||
mhKey, err := MultihashKeyFromKeccak256(i.currentKey)
|
||||
if err != nil {
|
||||
i.err = err
|
||||
return nil
|
||||
}
|
||||
var data []byte
|
||||
i.err = i.db.Get(&data, getPgStr, mhKey)
|
||||
return data
|
||||
return i.currentValue
|
||||
}
|
||||
|
||||
// Release satisfies the ethdb.Iterator interface
|
||||
// Release releases associated resources
|
||||
// Release should always succeed and can be called multiple times without causing error
|
||||
func (i *Iterator) Release() {
|
||||
i.db.Close()
|
||||
i.db, i.currentKey, i.currentValue, i.err, i.prefix = nil, nil, nil, nil, nil
|
||||
}
|
||||
|
||||
513
postgres/iterator_test.go
Normal file
513
postgres/iterator_test.go
Normal file
@ -0,0 +1,513 @@
|
||||
// VulcanizeDB
|
||||
// Copyright © 2020 Vulcanize
|
||||
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package pgipfsethdb_test
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
|
||||
"github.com/ethereum/go-ethereum/ethdb"
|
||||
. "github.com/onsi/ginkgo"
|
||||
. "github.com/onsi/gomega"
|
||||
|
||||
"github.com/vulcanize/ipfs-ethdb/postgres"
|
||||
)
|
||||
|
||||
var (
|
||||
iterator ethdb.Iterator
|
||||
testPrefix = []byte("testPrefix")
|
||||
testEthKey1 = []byte{'\x01'}
|
||||
testEthKey2 = []byte{'\x01', '\x01'}
|
||||
testEthKey3 = []byte{'\x01', '\x02'}
|
||||
testEthKey4 = []byte{'\x01', '\x0e'}
|
||||
testEthKey5 = []byte{'\x01', '\x02', '\x01'}
|
||||
testEthKey6 = []byte{'\x01', '\x0e', '\x01'}
|
||||
prefixedTestEthKey1 = append(append(testPrefix, pgipfsethdb.KeyDelineation...), testEthKey1...)
|
||||
prefixedTestEthKey2 = append(append(testPrefix, pgipfsethdb.KeyDelineation...), testEthKey2...)
|
||||
prefixedTestEthKey3 = append(append(testPrefix, pgipfsethdb.KeyDelineation...), testEthKey3...)
|
||||
prefixedTestEthKey4 = append(append(testPrefix, pgipfsethdb.KeyDelineation...), testEthKey4...)
|
||||
prefixedTestEthKey5 = append(append(testPrefix, pgipfsethdb.KeyDelineation...), testEthKey5...)
|
||||
prefixedTestEthKey6 = append(append(testPrefix, pgipfsethdb.KeyDelineation...), testEthKey6...)
|
||||
mockValue1 = []byte{1}
|
||||
mockValue2 = []byte{2}
|
||||
mockValue3 = []byte{3}
|
||||
mockValue4 = []byte{4}
|
||||
mockValue5 = []byte{5}
|
||||
mockValue6 = []byte{6}
|
||||
)
|
||||
|
||||
var _ = Describe("Iterator", func() {
|
||||
BeforeEach(func() {
|
||||
db, err = pgipfsethdb.TestDB()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
database = pgipfsethdb.NewDatabase(db)
|
||||
// non-prefixed entries
|
||||
err = database.Put(testEthKey1, mockValue1)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
err = database.Put(testEthKey2, mockValue2)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
err = database.Put(testEthKey3, mockValue3)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
err = database.Put(testEthKey4, mockValue4)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
err = database.Put(testEthKey5, mockValue5)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
err = database.Put(testEthKey6, mockValue6)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
// prefixed entries
|
||||
err = database.Put(prefixedTestEthKey1, mockValue1)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
err = database.Put(prefixedTestEthKey2, mockValue2)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
err = database.Put(prefixedTestEthKey3, mockValue3)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
err = database.Put(prefixedTestEthKey4, mockValue4)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
err = database.Put(prefixedTestEthKey5, mockValue5)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
err = database.Put(prefixedTestEthKey6, mockValue6)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
})
|
||||
AfterEach(func() {
|
||||
err = pgipfsethdb.ResetTestDB(db)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
})
|
||||
|
||||
Describe("NewIterator", func() {
|
||||
It("iterates over the entire key-set (prefixed or not)", func() {
|
||||
iterator = database.NewIterator()
|
||||
Expect(iterator.Value()).To(BeNil())
|
||||
Expect(iterator.Key()).To(BeNil())
|
||||
Expect(iterator.Error()).To(BeNil())
|
||||
|
||||
more := iterator.Next()
|
||||
Expect(more).To(BeTrue())
|
||||
Expect(iterator.Key()).To(Equal(testEthKey1))
|
||||
Expect(iterator.Value()).To(Equal(mockValue1))
|
||||
Expect(iterator.Error()).To(BeNil())
|
||||
|
||||
more = iterator.Next()
|
||||
Expect(more).To(BeTrue())
|
||||
Expect(iterator.Key()).To(Equal(testEthKey2))
|
||||
Expect(iterator.Value()).To(Equal(mockValue2))
|
||||
Expect(iterator.Error()).To(BeNil())
|
||||
|
||||
more = iterator.Next()
|
||||
Expect(more).To(BeTrue())
|
||||
Expect(iterator.Key()).To(Equal(testEthKey3))
|
||||
Expect(iterator.Value()).To(Equal(mockValue3))
|
||||
Expect(iterator.Error()).To(BeNil())
|
||||
|
||||
more = iterator.Next()
|
||||
Expect(more).To(BeTrue())
|
||||
Expect(iterator.Key()).To(Equal(testEthKey5))
|
||||
Expect(iterator.Value()).To(Equal(mockValue5))
|
||||
Expect(iterator.Error()).To(BeNil())
|
||||
|
||||
more = iterator.Next()
|
||||
Expect(more).To(BeTrue())
|
||||
Expect(iterator.Key()).To(Equal(testEthKey4))
|
||||
Expect(iterator.Value()).To(Equal(mockValue4))
|
||||
Expect(iterator.Error()).To(BeNil())
|
||||
|
||||
more = iterator.Next()
|
||||
Expect(more).To(BeTrue())
|
||||
Expect(iterator.Key()).To(Equal(testEthKey6))
|
||||
Expect(iterator.Value()).To(Equal(mockValue6))
|
||||
Expect(iterator.Error()).To(BeNil())
|
||||
|
||||
more = iterator.Next()
|
||||
Expect(more).To(BeTrue())
|
||||
Expect(iterator.Key()).To(Equal(prefixedTestEthKey1))
|
||||
Expect(iterator.Value()).To(Equal(mockValue1))
|
||||
Expect(iterator.Error()).To(BeNil())
|
||||
|
||||
more = iterator.Next()
|
||||
Expect(more).To(BeTrue())
|
||||
Expect(iterator.Key()).To(Equal(prefixedTestEthKey2))
|
||||
Expect(iterator.Value()).To(Equal(mockValue2))
|
||||
Expect(iterator.Error()).To(BeNil())
|
||||
|
||||
more = iterator.Next()
|
||||
Expect(more).To(BeTrue())
|
||||
Expect(iterator.Key()).To(Equal(prefixedTestEthKey3))
|
||||
Expect(iterator.Value()).To(Equal(mockValue3))
|
||||
Expect(iterator.Error()).To(BeNil())
|
||||
|
||||
more = iterator.Next()
|
||||
Expect(more).To(BeTrue())
|
||||
Expect(iterator.Key()).To(Equal(prefixedTestEthKey5))
|
||||
Expect(iterator.Value()).To(Equal(mockValue5))
|
||||
Expect(iterator.Error()).To(BeNil())
|
||||
|
||||
more = iterator.Next()
|
||||
Expect(more).To(BeTrue())
|
||||
Expect(iterator.Key()).To(Equal(prefixedTestEthKey4))
|
||||
Expect(iterator.Value()).To(Equal(mockValue4))
|
||||
Expect(iterator.Error()).To(BeNil())
|
||||
|
||||
more = iterator.Next()
|
||||
Expect(more).To(BeTrue())
|
||||
Expect(iterator.Key()).To(Equal(prefixedTestEthKey6))
|
||||
Expect(iterator.Value()).To(Equal(mockValue6))
|
||||
Expect(iterator.Error()).To(BeNil())
|
||||
|
||||
more = iterator.Next()
|
||||
Expect(more).ToNot(BeTrue())
|
||||
Expect(iterator.Value()).To(BeNil())
|
||||
Expect(iterator.Key()).To(BeNil())
|
||||
Expect(iterator.Error()).To(Equal(sql.ErrNoRows))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("NewIteratorWithPrefix", func() {
|
||||
It("iterates over all db entries that have the provided prefix", func() {
|
||||
iterator = database.NewIteratorWithPrefix(testPrefix)
|
||||
Expect(iterator.Value()).To(BeNil())
|
||||
Expect(iterator.Key()).To(BeNil())
|
||||
Expect(iterator.Error()).To(BeNil())
|
||||
|
||||
more := iterator.Next()
|
||||
Expect(more).To(BeTrue())
|
||||
Expect(iterator.Key()).To(Equal(prefixedTestEthKey1))
|
||||
Expect(iterator.Value()).To(Equal(mockValue1))
|
||||
Expect(iterator.Error()).To(BeNil())
|
||||
|
||||
more = iterator.Next()
|
||||
Expect(more).To(BeTrue())
|
||||
Expect(iterator.Key()).To(Equal(prefixedTestEthKey2))
|
||||
Expect(iterator.Value()).To(Equal(mockValue2))
|
||||
Expect(iterator.Error()).To(BeNil())
|
||||
|
||||
more = iterator.Next()
|
||||
Expect(more).To(BeTrue())
|
||||
Expect(iterator.Key()).To(Equal(prefixedTestEthKey3))
|
||||
Expect(iterator.Value()).To(Equal(mockValue3))
|
||||
Expect(iterator.Error()).To(BeNil())
|
||||
|
||||
more = iterator.Next()
|
||||
Expect(more).To(BeTrue())
|
||||
Expect(iterator.Key()).To(Equal(prefixedTestEthKey5))
|
||||
Expect(iterator.Value()).To(Equal(mockValue5))
|
||||
Expect(iterator.Error()).To(BeNil())
|
||||
|
||||
more = iterator.Next()
|
||||
Expect(more).To(BeTrue())
|
||||
Expect(iterator.Key()).To(Equal(prefixedTestEthKey4))
|
||||
Expect(iterator.Value()).To(Equal(mockValue4))
|
||||
Expect(iterator.Error()).To(BeNil())
|
||||
|
||||
more = iterator.Next()
|
||||
Expect(more).To(BeTrue())
|
||||
Expect(iterator.Key()).To(Equal(prefixedTestEthKey6))
|
||||
Expect(iterator.Value()).To(Equal(mockValue6))
|
||||
Expect(iterator.Error()).To(BeNil())
|
||||
|
||||
more = iterator.Next()
|
||||
Expect(more).ToNot(BeTrue())
|
||||
Expect(iterator.Value()).To(BeNil())
|
||||
Expect(iterator.Key()).To(BeNil())
|
||||
Expect(iterator.Error()).To(Equal(sql.ErrNoRows))
|
||||
})
|
||||
|
||||
It("behaves as no prefix is provided if prefix is nil", func() {
|
||||
iterator = database.NewIteratorWithPrefix(nil)
|
||||
Expect(iterator.Value()).To(BeNil())
|
||||
Expect(iterator.Key()).To(BeNil())
|
||||
Expect(iterator.Error()).To(BeNil())
|
||||
|
||||
more := iterator.Next()
|
||||
Expect(more).To(BeTrue())
|
||||
Expect(iterator.Key()).To(Equal(testEthKey1))
|
||||
Expect(iterator.Value()).To(Equal(mockValue1))
|
||||
Expect(iterator.Error()).To(BeNil())
|
||||
|
||||
more = iterator.Next()
|
||||
Expect(more).To(BeTrue())
|
||||
Expect(iterator.Key()).To(Equal(testEthKey2))
|
||||
Expect(iterator.Value()).To(Equal(mockValue2))
|
||||
Expect(iterator.Error()).To(BeNil())
|
||||
|
||||
more = iterator.Next()
|
||||
Expect(more).To(BeTrue())
|
||||
Expect(iterator.Key()).To(Equal(testEthKey3))
|
||||
Expect(iterator.Value()).To(Equal(mockValue3))
|
||||
Expect(iterator.Error()).To(BeNil())
|
||||
|
||||
more = iterator.Next()
|
||||
Expect(more).To(BeTrue())
|
||||
Expect(iterator.Key()).To(Equal(testEthKey5))
|
||||
Expect(iterator.Value()).To(Equal(mockValue5))
|
||||
Expect(iterator.Error()).To(BeNil())
|
||||
|
||||
more = iterator.Next()
|
||||
Expect(more).To(BeTrue())
|
||||
Expect(iterator.Key()).To(Equal(testEthKey4))
|
||||
Expect(iterator.Value()).To(Equal(mockValue4))
|
||||
Expect(iterator.Error()).To(BeNil())
|
||||
|
||||
more = iterator.Next()
|
||||
Expect(more).To(BeTrue())
|
||||
Expect(iterator.Key()).To(Equal(testEthKey6))
|
||||
Expect(iterator.Value()).To(Equal(mockValue6))
|
||||
Expect(iterator.Error()).To(BeNil())
|
||||
|
||||
more = iterator.Next()
|
||||
Expect(more).To(BeTrue())
|
||||
Expect(iterator.Key()).To(Equal(prefixedTestEthKey1))
|
||||
Expect(iterator.Value()).To(Equal(mockValue1))
|
||||
Expect(iterator.Error()).To(BeNil())
|
||||
|
||||
more = iterator.Next()
|
||||
Expect(more).To(BeTrue())
|
||||
Expect(iterator.Key()).To(Equal(prefixedTestEthKey2))
|
||||
Expect(iterator.Value()).To(Equal(mockValue2))
|
||||
Expect(iterator.Error()).To(BeNil())
|
||||
|
||||
more = iterator.Next()
|
||||
Expect(more).To(BeTrue())
|
||||
Expect(iterator.Key()).To(Equal(prefixedTestEthKey3))
|
||||
Expect(iterator.Value()).To(Equal(mockValue3))
|
||||
Expect(iterator.Error()).To(BeNil())
|
||||
|
||||
more = iterator.Next()
|
||||
Expect(more).To(BeTrue())
|
||||
Expect(iterator.Key()).To(Equal(prefixedTestEthKey5))
|
||||
Expect(iterator.Value()).To(Equal(mockValue5))
|
||||
Expect(iterator.Error()).To(BeNil())
|
||||
|
||||
more = iterator.Next()
|
||||
Expect(more).To(BeTrue())
|
||||
Expect(iterator.Key()).To(Equal(prefixedTestEthKey4))
|
||||
Expect(iterator.Value()).To(Equal(mockValue4))
|
||||
Expect(iterator.Error()).To(BeNil())
|
||||
|
||||
more = iterator.Next()
|
||||
Expect(more).To(BeTrue())
|
||||
Expect(iterator.Key()).To(Equal(prefixedTestEthKey6))
|
||||
Expect(iterator.Value()).To(Equal(mockValue6))
|
||||
Expect(iterator.Error()).To(BeNil())
|
||||
|
||||
more = iterator.Next()
|
||||
Expect(more).ToNot(BeTrue())
|
||||
Expect(iterator.Value()).To(BeNil())
|
||||
Expect(iterator.Key()).To(BeNil())
|
||||
Expect(iterator.Error()).To(Equal(sql.ErrNoRows))
|
||||
})
|
||||
|
||||
It("considers empty but non-nil []byte a valid prefix, which precludes iteration over any other prefixed keys", func() {
|
||||
iterator = database.NewIteratorWithPrefix([]byte{})
|
||||
Expect(iterator.Value()).To(BeNil())
|
||||
Expect(iterator.Key()).To(BeNil())
|
||||
Expect(iterator.Error()).To(BeNil())
|
||||
|
||||
more := iterator.Next()
|
||||
Expect(more).To(BeTrue())
|
||||
Expect(iterator.Key()).To(Equal(testEthKey1))
|
||||
Expect(iterator.Value()).To(Equal(mockValue1))
|
||||
Expect(iterator.Error()).To(BeNil())
|
||||
|
||||
more = iterator.Next()
|
||||
Expect(more).To(BeTrue())
|
||||
Expect(iterator.Key()).To(Equal(testEthKey2))
|
||||
Expect(iterator.Value()).To(Equal(mockValue2))
|
||||
Expect(iterator.Error()).To(BeNil())
|
||||
|
||||
more = iterator.Next()
|
||||
Expect(more).To(BeTrue())
|
||||
Expect(iterator.Key()).To(Equal(testEthKey3))
|
||||
Expect(iterator.Value()).To(Equal(mockValue3))
|
||||
Expect(iterator.Error()).To(BeNil())
|
||||
|
||||
more = iterator.Next()
|
||||
Expect(more).To(BeTrue())
|
||||
Expect(iterator.Key()).To(Equal(testEthKey5))
|
||||
Expect(iterator.Value()).To(Equal(mockValue5))
|
||||
Expect(iterator.Error()).To(BeNil())
|
||||
|
||||
more = iterator.Next()
|
||||
Expect(more).To(BeTrue())
|
||||
Expect(iterator.Key()).To(Equal(testEthKey4))
|
||||
Expect(iterator.Value()).To(Equal(mockValue4))
|
||||
Expect(iterator.Error()).To(BeNil())
|
||||
|
||||
more = iterator.Next()
|
||||
Expect(more).To(BeTrue())
|
||||
Expect(iterator.Key()).To(Equal(testEthKey6))
|
||||
Expect(iterator.Value()).To(Equal(mockValue6))
|
||||
Expect(iterator.Error()).To(BeNil())
|
||||
|
||||
more = iterator.Next()
|
||||
Expect(more).ToNot(BeTrue())
|
||||
Expect(iterator.Value()).To(BeNil())
|
||||
Expect(iterator.Key()).To(BeNil())
|
||||
Expect(iterator.Error()).To(Equal(sql.ErrNoRows))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("NewIteratorWithStart", func() {
|
||||
It("iterates over the entire key-set (prefixed or not) starting with at the provided path", func() {
|
||||
iterator = database.NewIteratorWithStart(testEthKey2)
|
||||
Expect(iterator.Value()).To(BeNil())
|
||||
Expect(iterator.Key()).To(Equal(testEthKey2))
|
||||
Expect(iterator.Error()).To(BeNil())
|
||||
|
||||
more := iterator.Next()
|
||||
Expect(more).To(BeTrue())
|
||||
Expect(iterator.Key()).To(Equal(testEthKey2))
|
||||
Expect(iterator.Value()).To(Equal(mockValue2))
|
||||
Expect(iterator.Error()).To(BeNil())
|
||||
|
||||
more = iterator.Next()
|
||||
Expect(more).To(BeTrue())
|
||||
Expect(iterator.Key()).To(Equal(testEthKey3))
|
||||
Expect(iterator.Value()).To(Equal(mockValue3))
|
||||
Expect(iterator.Error()).To(BeNil())
|
||||
|
||||
more = iterator.Next()
|
||||
Expect(more).To(BeTrue())
|
||||
Expect(iterator.Key()).To(Equal(testEthKey5))
|
||||
Expect(iterator.Value()).To(Equal(mockValue5))
|
||||
Expect(iterator.Error()).To(BeNil())
|
||||
|
||||
more = iterator.Next()
|
||||
Expect(more).To(BeTrue())
|
||||
Expect(iterator.Key()).To(Equal(testEthKey4))
|
||||
Expect(iterator.Value()).To(Equal(mockValue4))
|
||||
Expect(iterator.Error()).To(BeNil())
|
||||
|
||||
more = iterator.Next()
|
||||
Expect(more).To(BeTrue())
|
||||
Expect(iterator.Key()).To(Equal(testEthKey6))
|
||||
Expect(iterator.Value()).To(Equal(mockValue6))
|
||||
Expect(iterator.Error()).To(BeNil())
|
||||
|
||||
more = iterator.Next()
|
||||
Expect(more).To(BeTrue())
|
||||
Expect(iterator.Key()).To(Equal(prefixedTestEthKey1))
|
||||
Expect(iterator.Value()).To(Equal(mockValue1))
|
||||
Expect(iterator.Error()).To(BeNil())
|
||||
|
||||
more = iterator.Next()
|
||||
Expect(more).To(BeTrue())
|
||||
Expect(iterator.Key()).To(Equal(prefixedTestEthKey2))
|
||||
Expect(iterator.Value()).To(Equal(mockValue2))
|
||||
Expect(iterator.Error()).To(BeNil())
|
||||
|
||||
more = iterator.Next()
|
||||
Expect(more).To(BeTrue())
|
||||
Expect(iterator.Key()).To(Equal(prefixedTestEthKey3))
|
||||
Expect(iterator.Value()).To(Equal(mockValue3))
|
||||
Expect(iterator.Error()).To(BeNil())
|
||||
|
||||
more = iterator.Next()
|
||||
Expect(more).To(BeTrue())
|
||||
Expect(iterator.Key()).To(Equal(prefixedTestEthKey5))
|
||||
Expect(iterator.Value()).To(Equal(mockValue5))
|
||||
Expect(iterator.Error()).To(BeNil())
|
||||
|
||||
more = iterator.Next()
|
||||
Expect(more).To(BeTrue())
|
||||
Expect(iterator.Key()).To(Equal(prefixedTestEthKey4))
|
||||
Expect(iterator.Value()).To(Equal(mockValue4))
|
||||
Expect(iterator.Error()).To(BeNil())
|
||||
|
||||
more = iterator.Next()
|
||||
Expect(more).To(BeTrue())
|
||||
Expect(iterator.Key()).To(Equal(prefixedTestEthKey6))
|
||||
Expect(iterator.Value()).To(Equal(mockValue6))
|
||||
Expect(iterator.Error()).To(BeNil())
|
||||
|
||||
more = iterator.Next()
|
||||
Expect(more).ToNot(BeTrue())
|
||||
Expect(iterator.Value()).To(BeNil())
|
||||
Expect(iterator.Key()).To(BeNil())
|
||||
Expect(iterator.Error()).To(Equal(sql.ErrNoRows))
|
||||
})
|
||||
|
||||
It("iterates over the entire key-set (prefixed or not) starting with at the provided path", func() {
|
||||
iterator = database.NewIteratorWithStart(prefixedTestEthKey3)
|
||||
Expect(iterator.Value()).To(BeNil())
|
||||
Expect(iterator.Key()).To(Equal(prefixedTestEthKey3))
|
||||
Expect(iterator.Error()).To(BeNil())
|
||||
|
||||
more := iterator.Next()
|
||||
Expect(more).To(BeTrue())
|
||||
Expect(iterator.Key()).To(Equal(prefixedTestEthKey3))
|
||||
Expect(iterator.Value()).To(Equal(mockValue3))
|
||||
Expect(iterator.Error()).To(BeNil())
|
||||
|
||||
more = iterator.Next()
|
||||
Expect(more).To(BeTrue())
|
||||
Expect(iterator.Key()).To(Equal(prefixedTestEthKey5))
|
||||
Expect(iterator.Value()).To(Equal(mockValue5))
|
||||
Expect(iterator.Error()).To(BeNil())
|
||||
|
||||
more = iterator.Next()
|
||||
Expect(more).To(BeTrue())
|
||||
Expect(iterator.Key()).To(Equal(prefixedTestEthKey4))
|
||||
Expect(iterator.Value()).To(Equal(mockValue4))
|
||||
Expect(iterator.Error()).To(BeNil())
|
||||
|
||||
more = iterator.Next()
|
||||
Expect(more).To(BeTrue())
|
||||
Expect(iterator.Key()).To(Equal(prefixedTestEthKey6))
|
||||
Expect(iterator.Value()).To(Equal(mockValue6))
|
||||
Expect(iterator.Error()).To(BeNil())
|
||||
|
||||
more = iterator.Next()
|
||||
Expect(more).ToNot(BeTrue())
|
||||
Expect(iterator.Value()).To(BeNil())
|
||||
Expect(iterator.Key()).To(BeNil())
|
||||
Expect(iterator.Error()).To(Equal(sql.ErrNoRows))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("Release", func() {
|
||||
It("releases resources associated with the Iterator", func() {
|
||||
iterator = database.NewIteratorWithStart(testEthKey2)
|
||||
Expect(iterator.Value()).To(BeNil())
|
||||
Expect(iterator.Key()).To(Equal(testEthKey2))
|
||||
Expect(iterator.Error()).To(BeNil())
|
||||
|
||||
more := iterator.Next()
|
||||
Expect(more).To(BeTrue())
|
||||
Expect(iterator.Key()).To(Equal(testEthKey2))
|
||||
Expect(iterator.Value()).To(Equal(mockValue2))
|
||||
Expect(iterator.Error()).To(BeNil())
|
||||
|
||||
iterator.Release()
|
||||
iterator.Release() // check that we don't panic if called multiple times
|
||||
|
||||
Expect(iterator.Value()).To(BeNil())
|
||||
Expect(iterator.Key()).To(BeNil())
|
||||
Expect(iterator.Error()).To(BeNil())
|
||||
Expect(func() { iterator.Next() }).To(Panic()) // check that we panic if we try to use released iterator
|
||||
|
||||
// We can still create a new iterator from the same backing db
|
||||
iterator = database.NewIteratorWithStart(testEthKey2)
|
||||
Expect(iterator.Value()).To(BeNil())
|
||||
Expect(iterator.Key()).To(Equal(testEthKey2))
|
||||
Expect(iterator.Error()).To(BeNil())
|
||||
|
||||
more = iterator.Next()
|
||||
Expect(more).To(BeTrue())
|
||||
Expect(iterator.Key()).To(Equal(testEthKey2))
|
||||
Expect(iterator.Value()).To(Equal(mockValue2))
|
||||
Expect(iterator.Error()).To(BeNil())
|
||||
})
|
||||
})
|
||||
})
|
||||
69
postgres/key_type.go
Normal file
69
postgres/key_type.go
Normal file
@ -0,0 +1,69 @@
|
||||
// VulcanizeDB
|
||||
// Copyright © 2020 Vulcanize
|
||||
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package pgipfsethdb
|
||||
|
||||
import "bytes"
|
||||
|
||||
type KeyType uint
|
||||
|
||||
const (
|
||||
Invalid KeyType = iota
|
||||
Static
|
||||
Keccak
|
||||
Prefixed
|
||||
Suffixed
|
||||
Header
|
||||
Preimage
|
||||
)
|
||||
|
||||
var (
|
||||
// KeyDelineation is used to delineate the key prefixes and suffixes
|
||||
KeyDelineation = []byte("-fix-")
|
||||
|
||||
// NumberDelineation is used to delineate the block number encoded in a key
|
||||
NumberDelineation = []byte("-nmb-")
|
||||
|
||||
// Data item prefixes (use single byte to avoid mixing data types, avoid `i`, used for indexes).
|
||||
HeaderPrefix = []byte("h") // headerPrefix + num (uint64 big endian) + hash -> header
|
||||
PreimagePrefix = []byte("secure-key-") // preimagePrefix + hash -> preimage
|
||||
)
|
||||
|
||||
// ResolveKeyType returns the key type based on the prefix
|
||||
func ResolveKeyType(key []byte) (KeyType, [][]byte) {
|
||||
sk := bytes.Split(key, KeyDelineation)
|
||||
|
||||
// these heuristics are reliant on the current db key patterns
|
||||
switch len(sk) {
|
||||
case 1:
|
||||
if len(sk[0]) < 32 {
|
||||
return Static, sk
|
||||
}
|
||||
return Keccak, sk
|
||||
case 2:
|
||||
switch prefix := sk[0]; {
|
||||
case bytes.Equal(prefix, HeaderPrefix):
|
||||
return Header, bytes.Split(sk[1], NumberDelineation)
|
||||
case bytes.Equal(prefix, PreimagePrefix):
|
||||
return Preimage, sk
|
||||
default:
|
||||
return Prefixed, sk
|
||||
}
|
||||
case 3:
|
||||
return Suffixed, sk
|
||||
}
|
||||
return Invalid, sk
|
||||
}
|
||||
67
postgres/test_helpers.go
Normal file
67
postgres/test_helpers.go
Normal file
@ -0,0 +1,67 @@
|
||||
// VulcanizeDB
|
||||
// Copyright © 2020 Vulcanize
|
||||
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package pgipfsethdb
|
||||
|
||||
import (
|
||||
"github.com/jmoiron/sqlx"
|
||||
)
|
||||
|
||||
// TestDB connect to the testing database
|
||||
// it assumes the database has the IPFS public.blocks table present
|
||||
// DO NOT use a production db for the test db, as it will remove all contents of the public.blocks table
|
||||
func TestDB() (*sqlx.DB, error) {
|
||||
connectStr := "postgresql://localhost:5432/vulcanize_testing?sslmode=disable"
|
||||
return sqlx.Connect("postgres", connectStr)
|
||||
}
|
||||
|
||||
// ResetTestDB drops all rows in the test db public.blocks table
|
||||
func ResetTestDB(db *sqlx.DB) error {
|
||||
tx, err := db.Beginx()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
if p := recover(); p != nil {
|
||||
tx.Rollback()
|
||||
panic(p)
|
||||
} else if err != nil {
|
||||
tx.Rollback()
|
||||
} else {
|
||||
err = tx.Commit()
|
||||
}
|
||||
}()
|
||||
if _, err := tx.Exec("TRUNCATE public.blocks CASCADE"); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.Exec("TRUNCATE eth.key_preimages CASCADE"); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.Exec("TRUNCATE eth.ancient_headers CASCADE"); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.Exec("TRUNCATE eth.ancient_hashes CASCADE"); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.Exec("TRUNCATE eth.ancient_bodies CASCADE"); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.Exec("TRUNCATE eth.ancient_receipts CASCADE"); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = tx.Exec("TRUNCATE eth.ancient_tds CASCADE")
|
||||
return err
|
||||
}
|
||||
@ -17,9 +17,11 @@
|
||||
package pgipfsethdb
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
"github.com/ipfs/go-ipfs-blockstore"
|
||||
"github.com/ipfs/go-ipfs-ds-help"
|
||||
"github.com/jmoiron/sqlx"
|
||||
_ "github.com/lib/pq" //postgres driver
|
||||
"github.com/multiformats/go-multihash"
|
||||
)
|
||||
@ -34,16 +36,28 @@ func MultihashKeyFromKeccak256(h []byte) (string, error) {
|
||||
return blockstore.BlockPrefix.String() + dbKey.String(), nil
|
||||
}
|
||||
|
||||
// TestDB connect to the testing database
|
||||
// it assumes the database has the IPFS public.blocks table present
|
||||
// DO NOT use a production db for the test db, as it will remove all contents of the public.blocks table
|
||||
func TestDB() (*sqlx.DB, error) {
|
||||
connectStr := "postgresql://localhost:5432/vulcanize_testing?sslmode=disable"
|
||||
return sqlx.Connect("postgres", connectStr)
|
||||
}
|
||||
|
||||
// ResetTestDB drops all rows in the test db public.blocks table
|
||||
func ResetTestDB(db *sqlx.DB) error {
|
||||
_, err := db.Exec("TRUNCATE public.blocks")
|
||||
return err
|
||||
// DatastoreKeyFromGethKey returns the public.blocks key from the provided geth key
|
||||
// It also returns the key's prefix, if it has one
|
||||
func DatastoreKeyFromGethKey(h []byte) (string, []byte, error) {
|
||||
keyType, keyComponents := ResolveKeyType(h)
|
||||
switch keyType {
|
||||
case Keccak:
|
||||
mhKey, err := MultihashKeyFromKeccak256(h)
|
||||
return mhKey, nil, err
|
||||
case Header:
|
||||
mhKey, err := MultihashKeyFromKeccak256(keyComponents[1])
|
||||
return mhKey, keyComponents[0], err
|
||||
case Preimage:
|
||||
mhKey, err := MultihashKeyFromKeccak256(keyComponents[1])
|
||||
return mhKey, keyComponents[0], err
|
||||
case Prefixed, Suffixed:
|
||||
// This data is not mapped by hash => content by geth, store it using the prefixed/suffixed key directly
|
||||
// I.e. the public.blocks datastore key == the hex representation of the geth key
|
||||
// Alternatively, decompose the data and derive the hash
|
||||
return common.Bytes2Hex(h), keyComponents[0], nil
|
||||
case Static:
|
||||
return common.Bytes2Hex(h), nil, nil
|
||||
default:
|
||||
return "", nil, fmt.Errorf("invalid formatting of database key: %x", h)
|
||||
}
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user