From 869a34cf5a6128ef5573e5fb90701296984be169 Mon Sep 17 00:00:00 2001 From: Elizabeth Engelman Date: Wed, 12 Dec 2018 16:18:24 -0600 Subject: [PATCH] Start to write statediff to a CSV --- statediff/config.go | 9 +- statediff/publisher.go | 164 ++++++++++++++++++++++---- statediff/publisher_test.go | 222 +++++++++++++++++++++++++++++++++--- 3 files changed, 355 insertions(+), 40 deletions(-) diff --git a/statediff/config.go b/statediff/config.go index ac207acd1..fbc258be2 100644 --- a/statediff/config.go +++ b/statediff/config.go @@ -30,7 +30,8 @@ type Config struct { type StateDiffMode int const ( - IPLD StateDiffMode = iota + CSV StateDiffMode = iota + IPLD LDB SQL ) @@ -42,6 +43,8 @@ func (mode StateDiffMode) IsValid() bool { // String implements the stringer interface. func (mode StateDiffMode) String() string { switch mode { + case CSV: + return "csv" case IPLD: return "ipfs" case LDB: @@ -55,6 +58,8 @@ func (mode StateDiffMode) String() string { func (mode StateDiffMode) MarshalText() ([]byte, error) { switch mode { + case CSV: + return []byte("ipfs"), nil case IPLD: return []byte("ipfs"), nil case LDB: @@ -68,6 +73,8 @@ func (mode StateDiffMode) MarshalText() ([]byte, error) { func (mode *StateDiffMode) UnmarshalText(text []byte) error { switch string(text) { + case "csv": + *mode = CSV case "ipfs": *mode = IPLD case "ldb": diff --git a/statediff/publisher.go b/statediff/publisher.go index 48ad7cf64..e6526cee9 100644 --- a/statediff/publisher.go +++ b/statediff/publisher.go @@ -20,8 +20,11 @@ package statediff import ( - "errors" - "github.com/ethereum/go-ethereum/statediff/ipfs" + "os" + "encoding/csv" + "time" + "strconv" + "strings" ) type Publisher interface { @@ -29,35 +32,154 @@ type Publisher interface { } type publisher struct { - ipfs.DagPutter - Config + Config Config } -func NewPublisher(config Config) (*publisher, error) { - adder, err := ipfs.NewAdder(config.Path) - if err != nil { - return nil, err +var ( + Headers = []string{ + "blockNumber", "blockHash", "accountAction", + "code", "codeHash", + "oldNonceValue", "newNonceValue", + "oldBalanceValue", "newBalanceValue", + "oldContractRoot", "newContractRoot", + "storageDiffPaths", } + timeStampFormat = "20060102150405.00000" + deletedAccountAction = "deleted" + createdAccountAction = "created" + updatedAccountAction = "updated" +) + +func NewPublisher(config Config) (*publisher, error) { return &publisher{ - DagPutter: ipfs.NewDagPutter(adder), Config: config, }, nil } func (p *publisher) PublishStateDiff(sd *StateDiff) (string, error) { - switch p.Mode { - case IPLD: - cidStr, err := p.DagPut(sd) - if err != nil { - return "", err - } - - return cidStr, err - case LDB: - case SQL: + switch p.Config.Mode { + case CSV: + return "", p.publishStateDiffToCSV(*sd) default: + return "", p.publishStateDiffToCSV(*sd) + } +} + +func (p *publisher) publishStateDiffToCSV(sd StateDiff) error { + now := time.Now() + timeStamp := now.Format(timeStampFormat) + filePath := p.Config.Path + timeStamp + ".csv" + file, err := os.OpenFile(filePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + return err + } + defer file.Close() + + writer := csv.NewWriter(file) + defer writer.Flush() + + var data [][]string + data = append(data, Headers) + for _, row := range accumulateCreatedAccountRows(sd) { + data = append(data, row) + } + for _, row := range accumulateUpdatedAccountRows(sd) { + data = append(data, row) } - return "", errors.New("state diff publisher: unhandled publishing mode") -} \ No newline at end of file + for _, row := range accumulateDeletedAccountRows(sd) { + data = append(data, row) + } + + for _, value := range data{ + err := writer.Write(value) + if err != nil { + return err + } + } + + return nil +} + +func accumulateUpdatedAccountRows(sd StateDiff) [][]string { + var updatedAccountRows [][]string + for _, accountDiff := range sd.UpdatedAccounts { + formattedAccountData := formatAccountDiffIncremental(accountDiff, sd, updatedAccountAction) + + updatedAccountRows = append(updatedAccountRows, formattedAccountData) + } + + return updatedAccountRows +} + +func accumulateDeletedAccountRows(sd StateDiff) [][]string { + var deletedAccountRows [][]string + for _, accountDiff := range sd.DeletedAccounts { + formattedAccountData := formatAccountDiffEventual(accountDiff, sd, deletedAccountAction) + + deletedAccountRows = append(deletedAccountRows, formattedAccountData) + } + + return deletedAccountRows +} + +func accumulateCreatedAccountRows(sd StateDiff) [][]string { + var createdAccountRows [][]string + for _, accountDiff := range sd.CreatedAccounts { + formattedAccountData := formatAccountDiffEventual(accountDiff, sd, createdAccountAction) + + createdAccountRows = append(createdAccountRows, formattedAccountData) + } + + return createdAccountRows +} + +func formatAccountDiffEventual(accountDiff AccountDiffEventual, sd StateDiff, accountAction string) []string { + oldContractRoot := accountDiff.ContractRoot.OldValue + newContractRoot := accountDiff.ContractRoot.NewValue + var storageDiffPaths []string + for k := range accountDiff.Storage { + storageDiffPaths = append(storageDiffPaths, k) + } + formattedAccountData := []string{ + strconv.FormatInt(sd.BlockNumber, 10), + sd.BlockHash.String(), + accountAction, + string(accountDiff.Code), + accountDiff.CodeHash, + strconv.FormatUint(*accountDiff.Nonce.OldValue, 10), + strconv.FormatUint(*accountDiff.Nonce.NewValue, 10), + accountDiff.Balance.OldValue.String(), + accountDiff.Balance.NewValue.String(), + *oldContractRoot, + *newContractRoot, + strings.Join(storageDiffPaths, ","), + } + return formattedAccountData +} + +func formatAccountDiffIncremental(accountDiff AccountDiffIncremental, sd StateDiff, accountAction string) []string { + oldContractRoot := accountDiff.ContractRoot.OldValue + newContractRoot := accountDiff.ContractRoot.NewValue + var storageDiffPaths []string + for k := range accountDiff.Storage { + storageDiffPaths = append(storageDiffPaths, k) + } + formattedAccountData := []string{ + strconv.FormatInt(sd.BlockNumber, 10), + sd.BlockHash.String(), + accountAction, + "", + accountDiff.CodeHash, + strconv.FormatUint(*accountDiff.Nonce.OldValue, 10), + strconv.FormatUint(*accountDiff.Nonce.NewValue, 10), + accountDiff.Balance.OldValue.String(), + accountDiff.Balance.NewValue.String(), + *oldContractRoot, + *newContractRoot, + strings.Join(storageDiffPaths, ","), + } + return formattedAccountData +} + diff --git a/statediff/publisher_test.go b/statediff/publisher_test.go index de5e17095..9d34c2107 100644 --- a/statediff/publisher_test.go +++ b/statediff/publisher_test.go @@ -1,20 +1,206 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . +package statediff_test -// Contains a batch of utility type declarations used by the tests. As the node -// operates on unique types, a lot of them are needed to check various features. +import ( + "github.com/onsi/ginkgo" + "github.com/ethereum/go-ethereum/statediff" + "github.com/onsi/gomega" + "os" + "encoding/csv" + "github.com/ethereum/go-ethereum/common" + "math/rand" + "math/big" + "path/filepath" + "strings" + "strconv" +) + +var _ = ginkgo.Describe("Publisher", func() { + ginkgo.Context("default CSV publisher", func() { + var ( + publisher statediff.Publisher + err error + config = statediff.Config{ + Path: "./test-", + } + ) + + var ( + blockNumber = rand.Int63() + blockHash = "0xfa40fbe2d98d98b3363a778d52f2bcd29d6790b9b3f3cab2b167fd12d3550f73" + codeHash = "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470" + oldNonceValue = rand.Uint64() + newNonceValue = oldNonceValue + 1 + oldBalanceValue = rand.Int63() + newBalanceValue = oldBalanceValue - 1 + contractRoot = "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421" + storagePath = "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470" + oldStorage = "0x0" + newStorage = "0x03" + storage = map[string]statediff.DiffString{storagePath: { + NewValue: &newStorage, + OldValue: &oldStorage, + }} + address = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476592") + createdAccounts = map[common.Address]statediff.AccountDiffEventual{address: { + Nonce: statediff.DiffUint64{ + NewValue: &newNonceValue, + OldValue: &oldNonceValue, + }, + Balance: statediff.DiffBigInt{ + NewValue: big.NewInt(newBalanceValue), + OldValue: big.NewInt(oldBalanceValue), + }, + ContractRoot: statediff.DiffString{ + NewValue: &contractRoot, + OldValue: &contractRoot, + }, + Code: []byte("created account code"), + CodeHash: codeHash, + Storage: storage, + }} + + updatedAccounts = map[common.Address]statediff.AccountDiffIncremental{address: { + Nonce: statediff.DiffUint64{ + NewValue: &newNonceValue, + OldValue: &oldNonceValue, + }, + Balance: statediff.DiffBigInt{ + NewValue: big.NewInt(newBalanceValue), + OldValue: big.NewInt(oldBalanceValue), + }, + CodeHash: codeHash, + ContractRoot: statediff.DiffString{ + NewValue: &contractRoot, + OldValue: &contractRoot, + }, + Storage: storage, + }} + + deletedAccounts = map[common.Address]statediff.AccountDiffEventual{address: { + Nonce: statediff.DiffUint64{ + NewValue: &newNonceValue, + OldValue: &oldNonceValue, + }, + Balance: statediff.DiffBigInt{ + NewValue: big.NewInt(newBalanceValue), + OldValue: big.NewInt(oldBalanceValue), + }, + ContractRoot: statediff.DiffString{ + NewValue: &contractRoot, + OldValue: &contractRoot, + }, + Code: []byte("deleted account code"), + CodeHash: codeHash, + Storage: storage, + }} + + testStateDiff = statediff.StateDiff{ + BlockNumber: blockNumber, + BlockHash: common.HexToHash(blockHash), + CreatedAccounts: createdAccounts, + DeletedAccounts: deletedAccounts, + UpdatedAccounts: updatedAccounts, + } + ) + + var lines [][]string + var file *os.File + ginkgo.BeforeEach(func() { + publisher, err = statediff.NewPublisher(config) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + _, err := publisher.PublishStateDiff(&testStateDiff) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + filePaths := getTestCSVFiles(".") + file, err = os.Open(filePaths[0]) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + defer file.Close() + + lines, err = csv.NewReader(file).ReadAll() + }) + + ginkgo.AfterEach(func() { + os.Remove(file.Name()) + }) + + ginkgo.It("persists the column headers to a CSV file", func() { + gomega.Expect(len(lines) > 1).To(gomega.BeTrue()) + gomega.Expect(lines[0]).To(gomega.Equal(statediff.Headers)) + }) + + ginkgo.It("persists the created account diffs to a CSV file", func() { + expectedCreatedAccountRow := []string{ + strconv.FormatInt(blockNumber, 10), + blockHash, + "created", + "created account code", + codeHash, + strconv.FormatUint(oldNonceValue, 10), + strconv.FormatUint(newNonceValue, 10), + strconv.FormatInt(oldBalanceValue, 10), + strconv.FormatInt(newBalanceValue, 10), + contractRoot, + contractRoot, + storagePath, + } + + gomega.Expect(len(lines) > 1).To(gomega.BeTrue()) + gomega.Expect(lines[1]).To(gomega.Equal(expectedCreatedAccountRow)) + }) + + ginkgo.It("persists the updated account diffs to a CSV file", func() { + expectedUpdatedAccountRow := []string{ + strconv.FormatInt(blockNumber, 10), + blockHash, + "updated", + "", + codeHash, + strconv.FormatUint(oldNonceValue, 10), + strconv.FormatUint(newNonceValue, 10), + strconv.FormatInt(oldBalanceValue, 10), + strconv.FormatInt(newBalanceValue, 10), + contractRoot, + contractRoot, + storagePath, + } + + gomega.Expect(len(lines) > 2).To(gomega.BeTrue()) + gomega.Expect(lines[2]).To(gomega.Equal(expectedUpdatedAccountRow)) + }) + + ginkgo.It("persists the deleted account diffs to a CSV file", func() { + expectedDeletedAccountRow := []string{ + strconv.FormatInt(blockNumber, 10), + blockHash, + "deleted", + "deleted account code", + codeHash, + strconv.FormatUint(oldNonceValue, 10), + strconv.FormatUint(newNonceValue, 10), + strconv.FormatInt(oldBalanceValue, 10), + strconv.FormatInt(newBalanceValue, 10), + contractRoot, + contractRoot, + storagePath, + } + + gomega.Expect(len(lines) > 3).To(gomega.BeTrue()) + gomega.Expect(lines[3]).To(gomega.Equal(expectedDeletedAccountRow)) + }) + }) +}) + +func getTestCSVFiles(rootPath string) []string{ + var files []string + err := filepath.Walk(rootPath, func(path string, info os.FileInfo, err error) error { + if strings.HasPrefix(path, "test-") { + files = append(files, path) + } + return nil + }) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + return files +} -package statediff_test \ No newline at end of file