work towards persisting created statediffs in ipfs; based off github.com/vulcanize/eth-block-extractor

core/blockchain.go

@@ -20,7 +20,6 @@ package core
 import (
 	"errors"
 	"fmt"
-	"github.com/ethereum/go-ethereum/statediff"
 	"io"
 	"math/big"
 	mrand "math/rand"
@@ -45,6 +44,7 @@ import (
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/trie"
 	"github.com/hashicorp/golang-lru"
+	"github.com/ethereum/go-ethereum/statediff"
 )
 
 var (
@@ -73,7 +73,7 @@ type CacheConfig struct {
 	TrieCleanLimit int           // Memory allowance (MB) to use for caching trie nodes in memory
 	TrieDirtyLimit int           // Memory limit (MB) at which to start flushing dirty trie nodes to disk
 	TrieTimeLimit  time.Duration // Time limit after which to flush the current in-memory trie to disk
-	StateDiff      bool          // Whether or not to calculate and persist state diffs
+	StateDiff      statediff.Config // Settings for state diff extraction
 }
 
 // BlockChain represents the canonical chain given a database with a genesis
@@ -177,11 +177,14 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par
 	bc.SetValidator(NewBlockValidator(chainConfig, bc, engine))
 	bc.SetProcessor(NewStateProcessor(chainConfig, bc, engine))
 
-	if cacheConfig.StateDiff {
+	var err error
-		bc.diffExtractor = statediff.NewExtractor(db)
+	if cacheConfig.StateDiff.On {
+		bc.diffExtractor, err = statediff.NewExtractor(db, cacheConfig.StateDiff)
+		if err != nil {
+			return nil, err
+		}
 	}
 
-	var err error
 	bc.hc, err = NewHeaderChain(db, chainConfig, engine, bc.getProcInterrupt)
 	if err != nil {
 		return nil, err
@@ -1214,8 +1217,12 @@ func (bc *BlockChain) insertChain(chain types.Blocks) (int, []interface{}, []*ty
 		proctime := time.Since(bstart)
 
 		// If extracting statediffs, do so now
-		if bc.cacheConfig.StateDiff {
+		if bc.cacheConfig.StateDiff.On {
-			bc.diffExtractor.Extract(*parent, *block)
+			// Currently not doing anything with returned cid...
+			_, err = bc.diffExtractor.ExtractStateDiff(*parent, *block)
+			if err != nil {
+				return i, events, coalescedLogs, err
+			}
 		}
 
 		// Write the block to the chain and get the status.

statediff/builder.go

@@ -78,7 +78,7 @@ func (sdb *builder) BuildStateDiff(oldStateRoot, newStateRoot common.Hash, block
 	updatedKeys := findIntersection(createKeys, deleteKeys)
 
 	// Build and return the statediff
-	updatedAccounts, err := sdb.buildDiffIncremental(creations, deletions, &updatedKeys)
+	updatedAccounts, err := sdb.buildDiffIncremental(creations, deletions, updatedKeys)
 	if err != nil {
 		return nil, err
 	}
@@ -133,6 +133,7 @@ func (sdb *builder) collectDiffNodes(a, b trie.NodeIterator) (map[common.Address
 			break
 		}
 	}
+
 	return diffAccounts, nil
 }
 
@@ -140,66 +141,63 @@ func (sdb *builder) buildDiffEventual(accounts map[common.Address]*state.Account
 	accountDiffs := make(map[common.Address]AccountDiffEventual)
 	for addr, val := range accounts {
 		sr := val.Root
-		if storageDiffs, err := sdb.buildStorageDiffsEventual(sr, created); err != nil {
+		storageDiffs, err := sdb.buildStorageDiffsEventual(sr, created)
+		if err != nil {
 			log.Error("Failed building eventual storage diffs", "Address", val, "error", err)
 			return nil, err
-		} else {
+		}
-			code := ""
+
-			codeBytes, err := sdb.chainDB.Get(val.CodeHash)
+		codeBytes, err := sdb.chainDB.Get(val.CodeHash)
-			if err == nil && len(codeBytes) != 0 {
+
-				code = common.ToHex(codeBytes)
+		codeHash := common.ToHex(val.CodeHash)
-			} else {
+		hexRoot := val.Root.Hex()
-				log.Debug("No code field.", "codehash", val.CodeHash, "Address", val, "error", err)
+
+		if created {
+			nonce := diffUint64{
+				NewValue: &val.Nonce,
 			}
-			codeHash := common.ToHex(val.CodeHash)
-			if created {
-				nonce := diffUint64{
-					NewValue: &val.Nonce,
-				}
 
-				balance := diffBigInt{
+			balance := diffBigInt{
-					NewValue: val.Balance,
+				NewValue: val.Balance,
-				}
+			}
 
-				hexRoot := val.Root.Hex()
+			contractRoot := diffString{
-				contractRoot := diffString{
+				NewValue: &hexRoot,
-					NewValue: &hexRoot,
+			}
-				}
+			accountDiffs[addr] = AccountDiffEventual{
-				accountDiffs[addr] = AccountDiffEventual{
+				Nonce:        nonce,
-					Nonce:        nonce,
+				Balance:      balance,
-					Balance:      balance,
+				CodeHash:     codeHash,
-					CodeHash:     codeHash,
+				Code:         codeBytes,
-					Code:         code,
+				ContractRoot: contractRoot,
-					ContractRoot: contractRoot,
+				Storage:      storageDiffs,
-					Storage:      storageDiffs,
+			}
-				}
+		} else {
-			} else {
+			nonce := diffUint64{
-				nonce := diffUint64{
+				OldValue: &val.Nonce,
-					OldValue: &val.Nonce,
+			}
-				}
+			balance := diffBigInt{
-				balance := diffBigInt{
+				OldValue: val.Balance,
-					OldValue: val.Balance,
+			}
-				}
+			contractRoot := diffString{
-				hexRoot := val.Root.Hex()
+				OldValue: &hexRoot,
-				contractRoot := diffString{
+			}
-					OldValue: &hexRoot,
+			accountDiffs[addr] = AccountDiffEventual{
-				}
+				Nonce:        nonce,
-				accountDiffs[addr] = AccountDiffEventual{
+				Balance:      balance,
-					Nonce:        nonce,
+				CodeHash:     codeHash,
-					Balance:      balance,
+				ContractRoot: contractRoot,
-					CodeHash:     codeHash,
+				Storage:      storageDiffs,
-					ContractRoot: contractRoot,
-					Storage:      storageDiffs,
-				}
 			}
 		}
 	}
+
 	return accountDiffs, nil
 }
 
-func (sdb *builder) buildDiffIncremental(creations map[common.Address]*state.Account, deletions map[common.Address]*state.Account, updatedKeys *[]string) (map[common.Address]AccountDiffIncremental, error) {
+func (sdb *builder) buildDiffIncremental(creations map[common.Address]*state.Account, deletions map[common.Address]*state.Account, updatedKeys []string) (map[common.Address]AccountDiffIncremental, error) {
 	updatedAccounts := make(map[common.Address]AccountDiffIncremental)
-	for _, val := range *updatedKeys {
+	for _, val := range updatedKeys {
 		createdAcc := creations[common.HexToAddress(val)]
 		deletedAcc := deletions[common.HexToAddress(val)]
 		oldSR := deletedAcc.Root

statediff/config.go

@@ -22,26 +22,27 @@ package statediff
 import "fmt"
 
 type Config struct {
-	On bool
+	On   bool           // Whether or not to extract state diffs
-	Mode StateDiffMode
+	Mode StateDiffMode  // Mode for storing diffs
+	Path string		    // Path for storing diffs
 }
 
 type StateDiffMode int
 
 const (
-	IPFS  StateDiffMode = iota
+	IPLD  StateDiffMode = iota
 	LDB
 	SQL
 )
 
 func (mode StateDiffMode) IsValid() bool {
-	return mode >= IPFS && mode <= SQL
+	return mode >= IPLD && mode <= SQL
 }
 
 // String implements the stringer interface.
 func (mode StateDiffMode) String() string {
 	switch mode {
-	case IPFS:
+	case IPLD:
 		return "ipfs"
 	case LDB:
 		return "ldb"
@@ -54,7 +55,7 @@ func (mode StateDiffMode) String() string {
 
 func (mode StateDiffMode) MarshalText() ([]byte, error) {
 	switch mode {
-	case IPFS:
+	case IPLD:
 		return []byte("ipfs"), nil
 	case LDB:
 		return []byte("ldb"), nil
@@ -68,7 +69,7 @@ func (mode StateDiffMode) MarshalText() ([]byte, error) {
 func (mode *StateDiffMode) UnmarshalText(text []byte) error {
 	switch string(text) {
 	case "ipfs":
-		*mode = IPFS
+		*mode = IPLD
 	case "ldb":
 		*mode = LDB
 	case "sql":

statediff/extractor.go

@@ -25,26 +25,31 @@ import (
 )
 
 type Extractor interface {
-	ExtractStateDiff(parent, current types.Block) error
+	ExtractStateDiff(parent, current types.Block) (string, error)
 }
 
 type extractor struct {
-	b *builder
+	*builder   // Interface for building state diff objects from two blocks
-	p *persister
+	*publisher // Interface for publishing state diff objects to a datastore (e.g. IPFS)
 }
 
-func NewExtractor(db ethdb.Database) *extractor {
+func NewExtractor(db ethdb.Database, config Config) (*extractor, error) {
-	return &extractor{
+	publisher, err := NewPublisher(config)
-		b: NewBuilder(db),
-		p: NewPersister(),
-	}
-}
-
-func (e *extractor) Extract(parent, current types.Block) error {
-	stateDiff, err := e.b.BuildStateDiff(parent.Root(), current.Root(), current.Number().Int64(), current.Hash())
 	if err != nil {
-		return err
+		return nil, err
 	}
 
-	return e.p.PersistStateDiff(stateDiff)
+	return &extractor{
+		builder: NewBuilder(db),
+		publisher: publisher,
+	}, nil
+}
+
+func (e *extractor) ExtractStateDiff(parent, current types.Block) (string, error) {
+	stateDiff, err := e.BuildStateDiff(parent.Root(), current.Root(), current.Number().Int64(), current.Hash())
+	if err != nil {
+		return "", err
+	}
+
+	return e.PublishStateDiff(stateDiff)
 }

statediff/helpers.go

@@ -112,22 +112,7 @@ func decodeNibbles(nibbles []byte, bytes []byte) {
 	}
 }
 
-// prefixLen returns the length of the common prefix of a and b.
-func prefixLen(a, b []byte) int {
-	var i, length = 0, len(a)
-	if len(b) < length {
-		length = len(b)
-	}
-	for ; i < length; i++ {
-		if a[i] != b[i] {
-			break
-		}
-	}
-
-	return i
-}
-
 // hasTerm returns whether a hex key has the terminator flag.
 func hasTerm(s []byte) bool {
 	return len(s) > 0 && s[len(s)-1] == 16
-}
+}

statediff/ipfs/adder.go

@@ -0,0 +1,58 @@
+// Copyright 2015 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+// Contains a batch of utility type declarations used by the tests. As the node
+// operates on unique types, a lot of them are needed to check various features.
+
+package ipfs
+
+import (
+	"context"
+
+	"github.com/ipfs/go-ipfs/core"
+	"github.com/ipfs/go-ipfs/repo/fsrepo"
+	ipld "gx/ipfs/QmWi2BYBL5gJ3CiAiQchg6rn1A8iBsrWy51EYxvHVjFvLb/go-ipld-format"
+)
+
+type Adder interface {
+	Add(node ipld.Node) error
+}
+
+type adder struct {
+	n   *core.IpfsNode
+	ctx context.Context
+}
+
+func (a adder) Add(node ipld.Node) error {
+	return a.n.DAG.Add(a.n.Context(), node) // For some reason DAG.Add method is not being exposed by the ipld.DAGService
+}
+
+func NewAdder(repoPath string) (*adder, error) {
+	r, err := fsrepo.Open(repoPath)
+	if err != nil {
+		return nil, err
+	}
+	ctx := context.Background()
+	cfg := &core.BuildCfg{
+		Online: false,
+		Repo:   r,
+	}
+	ipfsNode, err := core.NewNode(ctx, cfg)
+	if err != nil {
+		return nil, err
+	}
+	return &adder{n: ipfsNode, ctx: ctx}, nil
+}

statediff/ipfs/dag_putter.go

@@ -0,0 +1,79 @@
+// Copyright 2015 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+// Contains a batch of utility type declarations used by the tests. As the node
+// operates on unique types, a lot of them are needed to check various features.
+
+package ipfs
+
+import (
+	"bytes"
+	"encoding/gob"
+	"github.com/i-norden/go-ethereum/statediff"
+
+	ipld "gx/ipfs/QmWi2BYBL5gJ3CiAiQchg6rn1A8iBsrWy51EYxvHVjFvLb/go-ipld-format"
+)
+
+const (
+	EthStateDiffCode = 0x99 // Register custom codec for state diff?
+)
+
+type DagPutter interface {
+	DagPut(sd *statediff.StateDiff) (string, error)
+}
+
+type dagPutter struct {
+	Adder
+}
+
+func NewDagPutter(adder Adder) *dagPutter {
+	return &dagPutter{Adder: adder}
+}
+
+func (bhdp *dagPutter) DagPut(sd *statediff.StateDiff) (string, error) {
+	nd, err := bhdp.getNode(sd)
+	if err != nil {
+		return "", err
+	}
+	err = bhdp.Add(nd)
+	if err != nil {
+		return "", err
+	}
+	return nd.Cid().String(), nil
+}
+
+func (bhdp *dagPutter) getNode(sd *statediff.StateDiff) (ipld.Node, error) {
+
+	var buff bytes.Buffer
+	enc := gob.NewEncoder(&buff)
+
+	err := enc.Encode(sd)
+	if err != nil {
+		return nil, err
+	}
+
+	raw := buff.Bytes()
+	cid, err := RawToCid(EthStateDiffCode, raw)
+	if err != nil {
+		return nil, err
+	}
+
+	return &StateDiffNode{
+		StateDiff: sd,
+		cid:  cid,
+		rawdata: raw,
+	}, nil
+}

statediff/ipfs/helpers.go

@@ -17,24 +17,22 @@
 // Contains a batch of utility type declarations used by the tests. As the node
 // operates on unique types, a lot of them are needed to check various features.
 
-package statediff
+package ipfs
 
-type Persister interface {
+import (
-	PersistStateDiff(sd *StateDiff) error
+	mh "gx/ipfs/QmZyZDi491cCNTLfAhwcaDii2Kg4pwKRkhqQzURGDvY6ua/go-multihash"
-}
+	"gx/ipfs/QmapdYm1b22Frv3k17fqrBYTFRxwiaVJkB299Mfn33edeB/go-cid"
-
+)
-type persister struct {
-
-}
-
-func NewPersister() *persister {
-	return &persister{
 
+func RawToCid(codec uint64, raw []byte) (*cid.Cid, error) {
+	c, err := cid.Prefix{
+		Codec:    codec,
+		Version:  1,
+		MhType:   mh.KECCAK_256,
+		MhLength: -1,
+	}.Sum(raw)
+	if err != nil {
+		return nil, err
 	}
-}
+	return c, nil
-
-func (p *persister) PersistStateDiff(sd *StateDiff) error {
-	//TODO: Persist state diff in IPFS
-
-	return nil
 }

statediff/ipfs/node.go

@@ -0,0 +1,78 @@
+// Copyright 2015 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+// Contains a batch of utility type declarations used by the tests. As the node
+// operates on unique types, a lot of them are needed to check various features.
+
+package ipfs
+
+import (
+	ipld "gx/ipfs/QmWi2BYBL5gJ3CiAiQchg6rn1A8iBsrWy51EYxvHVjFvLb/go-ipld-format"
+	"gx/ipfs/QmapdYm1b22Frv3k17fqrBYTFRxwiaVJkB299Mfn33edeB/go-cid"
+
+	"github.com/i-norden/go-ethereum/statediff"
+)
+
+type StateDiffNode struct {
+	*statediff.StateDiff
+
+	cid     *cid.Cid
+	rawdata []byte
+}
+
+func (sdn *StateDiffNode) RawData() []byte {
+	return sdn.rawdata
+}
+
+func (sdn *StateDiffNode) Cid() *cid.Cid {
+	return sdn.cid
+}
+
+func (sdn StateDiffNode) String() string {
+	return sdn.cid.String()
+}
+
+func (sdn StateDiffNode) Loggable() map[string]interface{} {
+	return sdn.cid.Loggable()
+}
+
+func (sdn StateDiffNode) Resolve(path []string) (interface{}, []string, error) {
+	panic("implement me")
+}
+
+func (sdn StateDiffNode) Tree(path string, depth int) []string {
+	panic("implement me")
+}
+
+func (sdn StateDiffNode) ResolveLink(path []string) (*ipld.Link, []string, error) {
+	panic("implement me")
+}
+
+func (sdn StateDiffNode) Copy() ipld.Node {
+	panic("implement me")
+}
+
+func (sdn StateDiffNode) Links() []*ipld.Link {
+	panic("implement me")
+}
+
+func (sdn StateDiffNode) Stat() (*ipld.NodeStat, error) {
+	panic("implement me")
+}
+
+func (sdn StateDiffNode) Size() (uint64, error) {
+	panic("implement me")
+}

statediff/publisher.go

@@ -0,0 +1,63 @@
+// Copyright 2015 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+// Contains a batch of utility type declarations used by the tests. As the node
+// operates on unique types, a lot of them are needed to check various features.
+
+package statediff
+
+import (
+	"errors"
+	"github.com/i-norden/go-ethereum/statediff/ipfs"
+)
+
+type Publisher interface {
+	PublishStateDiff(sd *StateDiff) (string, error)
+}
+
+type publisher struct {
+	ipfs.DagPutter
+	Config
+}
+
+func NewPublisher(config Config) (*publisher, error) {
+	adder, err := ipfs.NewAdder(config.Path)
+	if err != nil {
+		return nil, err
+	}
+
+	return &publisher{
+		DagPutter: ipfs.NewDagPutter(adder),
+		Config: config,
+	}, nil
+}
+
+func (p *publisher) PublishStateDiff(sd *StateDiff) (string, error) {
+	switch p.Mode {
+	case IPLD:
+		cidStr, err := p.DagPut(sd)
+		if err != nil {
+			return "", err
+		}
+
+		return cidStr, err
+	case LDB:
+	case SQL:
+	default:
+	}
+
+	return "", errors.New("state diff publisher: unhandled publishing mode")
+}

statediff/struct.go

@@ -58,7 +58,7 @@ func (sd *StateDiff) Encode() ([]byte, error) {
 type AccountDiffEventual struct {
 	Nonce        diffUint64            `json:"nonce"         gencodec:"required"`
 	Balance      diffBigInt            `json:"balance"       gencodec:"required"`
-	Code         string                `json:"code"          gencodec:"required"`
+	Code         []byte                `json:"code"          gencodec:"required"`
 	CodeHash     string                `json:"codeHash"      gencodec:"required"`
 	ContractRoot diffString            `json:"contractRoot"  gencodec:"required"`
 	Storage      map[string]diffString `json:"storage"       gencodec:"required"`