go-ethereum/statediff/publisher/csv.go

138 lines
3.4 KiB
Go
Raw Normal View History

Write state diff to CSV (#2) * port statediff from https://github.com/jpmorganchase/quorum/blob/9b7fd9af8082795eeeb6863d9746f12b82dd5078/statediff/statediff.go; minor fixes * integrating state diff extracting, building, and persisting into geth processes * work towards persisting created statediffs in ipfs; based off github.com/vulcanize/eth-block-extractor * Add a state diff service * Remove diff extractor from blockchain * Update imports * Move statediff on/off check to geth cmd config * Update starting state diff service * Add debugging logs for creating diff * Add statediff extractor and builder tests and small refactoring * Start to write statediff to a CSV * Restructure statediff directory * Pull CSV publishing methods into their own file * Reformatting due to go fmt * Add gomega to vendor dir * Remove testing focuses * Update statediff tests to use golang test pkg instead of ginkgo - builder_test - extractor_test - publisher_test * Use hexutil.Encode instead of deprecated common.ToHex * Remove OldValue from DiffBigInt and DiffUint64 fields * Update builder test * Remove old storage value from updated accounts * Remove old values from created/deleted accounts * Update publisher to account for only storing current account values * Update service loop and fetching previous block * Update testing - remove statediff ginkgo test suite file - move mocks to their own dir * Updates per go fmt * Updates to tests * Pass statediff mode and path in through cli * Return filename from publisher * Remove some duplication in builder * Remove code field from state diff output this is the contract byte code, and it can still be obtained by querying the db by the codeHash * Consolidate acct diff structs for updated & updated/deleted accts * Include block number in csv filename * Clean up error logging * Cleanup formatting, spelling, etc * Address PR comments * Add contract address and storage value to csv * Refactor accumulating account row in csv publisher * Add DiffStorage struct * Add storage key to csv * Address PR comments * Fix publisher to include rows for accounts that don't have store updates * Update builder test after merging in release/1.8 * Update test contract to include storage on contract intialization - so that we're able to test that storage diffing works for created and deleted accounts (not just updated accounts). * Factor out a common trie iterator method in builder
2019-01-28 21:31:01 +00:00
package publisher
import (
"encoding/csv"
"github.com/ethereum/go-ethereum/statediff/builder"
"os"
"path/filepath"
"strconv"
"time"
"github.com/ethereum/go-ethereum/common"
)
var (
Headers = []string{
"blockNumber", "blockHash", "accountAction", "codeHash",
"nonceValue", "balanceValue", "contractRoot", "storageDiffPaths",
"accountAddress", "storageKey", "storageValue",
}
timeStampFormat = "20060102150405.00000"
deletedAccountAction = "deleted"
createdAccountAction = "created"
updatedAccountAction = "updated"
)
func createCSVFilePath(path, blockNumber string) string {
now := time.Now()
timeStamp := now.Format(timeStampFormat)
suffix := timeStamp + "-" + blockNumber
filePath := filepath.Join(path, suffix)
filePath = filePath + ".csv"
return filePath
}
func (p *publisher) publishStateDiffToCSV(sd builder.StateDiff) (string, error) {
filePath := createCSVFilePath(p.Config.Path, strconv.FormatInt(sd.BlockNumber, 10))
file, err := os.OpenFile(filePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
if err != nil {
return "", err
}
defer file.Close()
writer := csv.NewWriter(file)
defer writer.Flush()
var data [][]string
data = append(data, Headers)
for _, row := range accumulateAccountRows(sd) {
data = append(data, row)
}
for _, value := range data {
err := writer.Write(value)
if err != nil {
return "", err
}
}
return filePath, nil
}
func accumulateAccountRows(sd builder.StateDiff) [][]string {
var accountRows [][]string
for accountAddr, accountDiff := range sd.CreatedAccounts {
formattedAccountData := formatAccountData(accountAddr, accountDiff, sd, createdAccountAction)
for _, accountData := range formattedAccountData {
accountRows = append(accountRows, accountData)
}
}
for accountAddr, accountDiff := range sd.UpdatedAccounts {
formattedAccountData := formatAccountData(accountAddr, accountDiff, sd, updatedAccountAction)
for _, accountData := range formattedAccountData {
accountRows = append(accountRows, accountData)
}
}
for accountAddr, accountDiff := range sd.DeletedAccounts {
formattedAccountData := formatAccountData(accountAddr, accountDiff, sd, deletedAccountAction)
for _, accountData := range formattedAccountData {
accountRows = append(accountRows, accountData)
}
}
return accountRows
}
func formatAccountData(accountAddr common.Address, accountDiff builder.AccountDiff, sd builder.StateDiff, accountAction string) [][]string {
blockNumberString := strconv.FormatInt(sd.BlockNumber, 10)
blockHash := sd.BlockHash.String()
codeHash := accountDiff.CodeHash
nonce := strconv.FormatUint(*accountDiff.Nonce.Value, 10)
balance := accountDiff.Balance.Value.String()
newContractRoot := accountDiff.ContractRoot.Value
address := accountAddr.String()
var result [][]string
if len(accountDiff.Storage) > 0 {
for storagePath, storage := range accountDiff.Storage {
formattedAccountData := []string{
blockNumberString,
blockHash,
accountAction,
codeHash,
nonce,
balance,
*newContractRoot,
storagePath,
address,
*storage.Key,
*storage.Value,
}
result = append(result, formattedAccountData)
}
} else {
formattedAccountData := []string{
blockNumberString,
blockHash,
accountAction,
codeHash,
nonce,
balance,
*newContractRoot,
"",
address,
"",
"",
}
result = append(result, formattedAccountData)
}
return result
}