go-ethereum/statediff/publisher/csv.go
Elizabeth 52274e48a1
Write state diff to CSV (#2)
* port statediff from 9b7fd9af80/statediff/statediff.go; minor fixes

* integrating state diff extracting, building, and persisting into geth processes

* work towards persisting created statediffs in ipfs; based off github.com/vulcanize/eth-block-extractor

* Add a state diff service

* Remove diff extractor from blockchain

* Update imports

* Move statediff on/off check to geth cmd config

* Update starting state diff service

* Add debugging logs for creating diff

* Add statediff extractor and builder tests and small refactoring

* Start to write statediff to a CSV

* Restructure statediff directory

* Pull CSV publishing methods into their own file

* Reformatting due to go fmt

* Add gomega to vendor dir

* Remove testing focuses

* Update statediff tests to use golang test pkg

instead of ginkgo

- builder_test
- extractor_test
- publisher_test

* Use hexutil.Encode instead of deprecated common.ToHex

* Remove OldValue from DiffBigInt and DiffUint64 fields

* Update builder test

* Remove old storage value from updated accounts

* Remove old values from created/deleted accounts

* Update publisher to account for only storing current account values

* Update service loop and fetching previous block

* Update testing

- remove statediff ginkgo test suite file
- move mocks to their own dir

* Updates per go fmt

* Updates to tests

* Pass statediff mode and path in through cli

* Return filename from publisher

* Remove some duplication in builder

* Remove code field from state diff output

this is the contract byte code, and it can still be obtained by querying
the db by the codeHash

* Consolidate acct diff structs for updated & updated/deleted accts

* Include block number in csv filename

* Clean up error logging

* Cleanup formatting, spelling, etc

* Address PR comments

* Add contract address and storage value to csv

* Refactor accumulating account row in csv publisher

* Add DiffStorage struct

* Add storage key to csv

* Address PR comments

* Fix publisher to include rows for accounts that don't have store updates

* Update builder test after merging in release/1.8

* Update test contract to include storage on contract intialization

- so that we're able to test that storage diffing works for created and
deleted accounts (not just updated accounts).

* Factor out a common trie iterator method in builder
2019-01-28 15:31:01 -06:00

138 lines
3.4 KiB
Go

package publisher
import (
"encoding/csv"
"github.com/ethereum/go-ethereum/statediff/builder"
"os"
"path/filepath"
"strconv"
"time"
"github.com/ethereum/go-ethereum/common"
)
var (
Headers = []string{
"blockNumber", "blockHash", "accountAction", "codeHash",
"nonceValue", "balanceValue", "contractRoot", "storageDiffPaths",
"accountAddress", "storageKey", "storageValue",
}
timeStampFormat = "20060102150405.00000"
deletedAccountAction = "deleted"
createdAccountAction = "created"
updatedAccountAction = "updated"
)
func createCSVFilePath(path, blockNumber string) string {
now := time.Now()
timeStamp := now.Format(timeStampFormat)
suffix := timeStamp + "-" + blockNumber
filePath := filepath.Join(path, suffix)
filePath = filePath + ".csv"
return filePath
}
func (p *publisher) publishStateDiffToCSV(sd builder.StateDiff) (string, error) {
filePath := createCSVFilePath(p.Config.Path, strconv.FormatInt(sd.BlockNumber, 10))
file, err := os.OpenFile(filePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
if err != nil {
return "", err
}
defer file.Close()
writer := csv.NewWriter(file)
defer writer.Flush()
var data [][]string
data = append(data, Headers)
for _, row := range accumulateAccountRows(sd) {
data = append(data, row)
}
for _, value := range data {
err := writer.Write(value)
if err != nil {
return "", err
}
}
return filePath, nil
}
func accumulateAccountRows(sd builder.StateDiff) [][]string {
var accountRows [][]string
for accountAddr, accountDiff := range sd.CreatedAccounts {
formattedAccountData := formatAccountData(accountAddr, accountDiff, sd, createdAccountAction)
for _, accountData := range formattedAccountData {
accountRows = append(accountRows, accountData)
}
}
for accountAddr, accountDiff := range sd.UpdatedAccounts {
formattedAccountData := formatAccountData(accountAddr, accountDiff, sd, updatedAccountAction)
for _, accountData := range formattedAccountData {
accountRows = append(accountRows, accountData)
}
}
for accountAddr, accountDiff := range sd.DeletedAccounts {
formattedAccountData := formatAccountData(accountAddr, accountDiff, sd, deletedAccountAction)
for _, accountData := range formattedAccountData {
accountRows = append(accountRows, accountData)
}
}
return accountRows
}
func formatAccountData(accountAddr common.Address, accountDiff builder.AccountDiff, sd builder.StateDiff, accountAction string) [][]string {
blockNumberString := strconv.FormatInt(sd.BlockNumber, 10)
blockHash := sd.BlockHash.String()
codeHash := accountDiff.CodeHash
nonce := strconv.FormatUint(*accountDiff.Nonce.Value, 10)
balance := accountDiff.Balance.Value.String()
newContractRoot := accountDiff.ContractRoot.Value
address := accountAddr.String()
var result [][]string
if len(accountDiff.Storage) > 0 {
for storagePath, storage := range accountDiff.Storage {
formattedAccountData := []string{
blockNumberString,
blockHash,
accountAction,
codeHash,
nonce,
balance,
*newContractRoot,
storagePath,
address,
*storage.Key,
*storage.Value,
}
result = append(result, formattedAccountData)
}
} else {
formattedAccountData := []string{
blockNumberString,
blockHash,
accountAction,
codeHash,
nonce,
balance,
*newContractRoot,
"",
address,
"",
"",
}
result = append(result, formattedAccountData)
}
return result
}