cerc-io/go-ethereum
18
0
Fork 0
Code Issues 9 Pull Requests Packages Projects Releases 66 Wiki Activity

Write state diff to CSV #2

Merged
elizabethengelman merged 47 commits from ee-state-diff into statediff-for-archive-node 2019-01-28 21:31:02 +00:00
Conversation 0 Commits 47 Files Changed 90 +50 -36
5 changed files with 50 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit ade4a8efdc - Show all commits

2
statediff/config_test.go
View File

@ -9,7 +9,7 @@ import (
func TestNewMode(t *testing.T) {
mode, err := statediff.NewMode("csv")
if err != nil {
t.Errorf(testhelpers.TestFailedFormatString, t.Name(), err)
t.Errorf(testhelpers.ErrorFormatString, t.Name(), err)
}
if mode != statediff.CSV {

8
statediff/publisher/csv.go
View File

@ -31,12 +31,12 @@ func createCSVFilePath(path string) string {
rmulhol commented 2019-01-03 23:00:34 +00:00 (Migrated from github.com)
Review
Copy Link

And then I think the _ here could be captured to publish the address

And then I think the `_` here could be captured to publish the address
rmulhol commented 2019-01-03 23:00:34 +00:00 (Migrated from github.com)
Review
Copy Link

And then I think the _ here could be captured to publish the address

And then I think the `_` here could be captured to publish the address
rmulhol commented 2019-01-03 23:05:31 +00:00 (Migrated from github.com)
Review
Copy Link

I'm a little confused about how this works - isn't accountDiff.Storage a map? Does k capture the key and value? Thinking we probably want both of those so that we can parse the new value and associate it with the corresponding key when consuming the data

I'm a little confused about how this works - isn't `accountDiff.Storage` a map? Does `k` capture the key and value? Thinking we probably want both of those so that we can parse the new value and associate it with the corresponding key when consuming the data
rmulhol commented 2019-01-03 23:05:31 +00:00 (Migrated from github.com)
Review
Copy Link

I'm a little confused about how this works - isn't accountDiff.Storage a map? Does k capture the key and value? Thinking we probably want both of those so that we can parse the new value and associate it with the corresponding key when consuming the data

I'm a little confused about how this works - isn't `accountDiff.Storage` a map? Does `k` capture the key and value? Thinking we probably want both of those so that we can parse the new value and associate it with the corresponding key when consuming the data
elizabethengelman commented 2019-01-10 19:33:36 +00:00 (Migrated from github.com)
Review
Copy Link

Yep, good call - k is just capturing the path to the storage value. I think I'll want both the key and the value when I add the storage value to the csv 👌

Yep, good call - `k` is just capturing the path to the storage value. I think I'll want both the key and the value when I add the storage value to the csv 👌
elizabethengelman commented 2019-01-10 19:33:36 +00:00 (Migrated from github.com)
Review
Copy Link

Yep, good call - k is just capturing the path to the storage value. I think I'll want both the key and the value when I add the storage value to the csv 👌

Yep, good call - `k` is just capturing the path to the storage value. I think I'll want both the key and the value when I add the storage value to the csv 👌
return filePath
}
func (p *publisher) publishStateDiffToCSV(sd builder.StateDiff) error {
rmulhol commented 2019-01-03 23:00:34 +00:00 (Migrated from github.com)
Review
Copy Link

And then I think the _ here could be captured to publish the address

And then I think the `_` here could be captured to publish the address
rmulhol commented 2019-01-03 23:05:31 +00:00 (Migrated from github.com)
Review
Copy Link

I'm a little confused about how this works - isn't accountDiff.Storage a map? Does k capture the key and value? Thinking we probably want both of those so that we can parse the new value and associate it with the corresponding key when consuming the data

I'm a little confused about how this works - isn't `accountDiff.Storage` a map? Does `k` capture the key and value? Thinking we probably want both of those so that we can parse the new value and associate it with the corresponding key when consuming the data
elizabethengelman commented 2019-01-10 19:33:36 +00:00 (Migrated from github.com)
Review
Copy Link

Yep, good call - k is just capturing the path to the storage value. I think I'll want both the key and the value when I add the storage value to the csv 👌

Yep, good call - `k` is just capturing the path to the storage value. I think I'll want both the key and the value when I add the storage value to the csv 👌
func (p *publisher) publishStateDiffToCSV(sd builder.StateDiff) (string, error) {
rmulhol commented 2019-01-03 23:00:34 +00:00 (Migrated from github.com)
Review
Copy Link

And then I think the _ here could be captured to publish the address

And then I think the `_` here could be captured to publish the address
rmulhol commented 2019-01-03 23:05:31 +00:00 (Migrated from github.com)
Review
Copy Link

I'm a little confused about how this works - isn't accountDiff.Storage a map? Does k capture the key and value? Thinking we probably want both of those so that we can parse the new value and associate it with the corresponding key when consuming the data

I'm a little confused about how this works - isn't `accountDiff.Storage` a map? Does `k` capture the key and value? Thinking we probably want both of those so that we can parse the new value and associate it with the corresponding key when consuming the data
elizabethengelman commented 2019-01-10 19:33:36 +00:00 (Migrated from github.com)
Review
Copy Link

Yep, good call - k is just capturing the path to the storage value. I think I'll want both the key and the value when I add the storage value to the csv 👌

Yep, good call - `k` is just capturing the path to the storage value. I think I'll want both the key and the value when I add the storage value to the csv 👌
filePath := createCSVFilePath(p.Config.Path)
file, err := os.OpenFile(filePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
if err != nil {
return err
rmulhol commented 2019-01-03 23:00:34 +00:00 (Migrated from github.com)
Review
Copy Link

And then I think the _ here could be captured to publish the address

And then I think the `_` here could be captured to publish the address
rmulhol commented 2019-01-03 23:05:31 +00:00 (Migrated from github.com)
Review
Copy Link

I'm a little confused about how this works - isn't accountDiff.Storage a map? Does k capture the key and value? Thinking we probably want both of those so that we can parse the new value and associate it with the corresponding key when consuming the data

I'm a little confused about how this works - isn't `accountDiff.Storage` a map? Does `k` capture the key and value? Thinking we probably want both of those so that we can parse the new value and associate it with the corresponding key when consuming the data
elizabethengelman commented 2019-01-10 19:33:36 +00:00 (Migrated from github.com)
Review
Copy Link

Yep, good call - k is just capturing the path to the storage value. I think I'll want both the key and the value when I add the storage value to the csv 👌

Yep, good call - `k` is just capturing the path to the storage value. I think I'll want both the key and the value when I add the storage value to the csv 👌
return "", err
rmulhol commented 2019-01-03 23:00:34 +00:00 (Migrated from github.com)
Review
Copy Link

And then I think the _ here could be captured to publish the address

And then I think the `_` here could be captured to publish the address
rmulhol commented 2019-01-03 23:05:31 +00:00 (Migrated from github.com)
Review
Copy Link

I'm a little confused about how this works - isn't accountDiff.Storage a map? Does k capture the key and value? Thinking we probably want both of those so that we can parse the new value and associate it with the corresponding key when consuming the data

I'm a little confused about how this works - isn't `accountDiff.Storage` a map? Does `k` capture the key and value? Thinking we probably want both of those so that we can parse the new value and associate it with the corresponding key when consuming the data
elizabethengelman commented 2019-01-10 19:33:36 +00:00 (Migrated from github.com)
Review
Copy Link

Yep, good call - k is just capturing the path to the storage value. I think I'll want both the key and the value when I add the storage value to the csv 👌

Yep, good call - `k` is just capturing the path to the storage value. I think I'll want both the key and the value when I add the storage value to the csv 👌
}
defer file.Close()
@ -58,11 +58,11 @@ func (p *publisher) publishStateDiffToCSV(sd builder.StateDiff) error {
rmulhol commented 2019-01-03 23:00:34 +00:00 (Migrated from github.com)
Review
Copy Link

And then I think the _ here could be captured to publish the address

And then I think the `_` here could be captured to publish the address
rmulhol commented 2019-01-03 23:00:34 +00:00 (Migrated from github.com)
Review
Copy Link

And then I think the _ here could be captured to publish the address

And then I think the `_` here could be captured to publish the address
rmulhol commented 2019-01-03 23:05:31 +00:00 (Migrated from github.com)
Review
Copy Link

I'm a little confused about how this works - isn't accountDiff.Storage a map? Does k capture the key and value? Thinking we probably want both of those so that we can parse the new value and associate it with the corresponding key when consuming the data

I'm a little confused about how this works - isn't `accountDiff.Storage` a map? Does `k` capture the key and value? Thinking we probably want both of those so that we can parse the new value and associate it with the corresponding key when consuming the data
rmulhol commented 2019-01-03 23:05:31 +00:00 (Migrated from github.com)
Review
Copy Link

I'm a little confused about how this works - isn't accountDiff.Storage a map? Does k capture the key and value? Thinking we probably want both of those so that we can parse the new value and associate it with the corresponding key when consuming the data

I'm a little confused about how this works - isn't `accountDiff.Storage` a map? Does `k` capture the key and value? Thinking we probably want both of those so that we can parse the new value and associate it with the corresponding key when consuming the data
elizabethengelman commented 2019-01-10 19:33:36 +00:00 (Migrated from github.com)
Review
Copy Link

Yep, good call - k is just capturing the path to the storage value. I think I'll want both the key and the value when I add the storage value to the csv 👌

Yep, good call - `k` is just capturing the path to the storage value. I think I'll want both the key and the value when I add the storage value to the csv 👌
elizabethengelman commented 2019-01-10 19:33:36 +00:00 (Migrated from github.com)
Review
Copy Link

Yep, good call - k is just capturing the path to the storage value. I think I'll want both the key and the value when I add the storage value to the csv 👌

Yep, good call - `k` is just capturing the path to the storage value. I think I'll want both the key and the value when I add the storage value to the csv 👌
for _, value := range data {
err := writer.Write(value)
if err != nil {
return err
rmulhol commented 2019-01-03 23:00:34 +00:00 (Migrated from github.com)
Review
Copy Link

And then I think the _ here could be captured to publish the address

And then I think the `_` here could be captured to publish the address
rmulhol commented 2019-01-03 23:05:31 +00:00 (Migrated from github.com)
Review
Copy Link

I'm a little confused about how this works - isn't accountDiff.Storage a map? Does k capture the key and value? Thinking we probably want both of those so that we can parse the new value and associate it with the corresponding key when consuming the data

I'm a little confused about how this works - isn't `accountDiff.Storage` a map? Does `k` capture the key and value? Thinking we probably want both of those so that we can parse the new value and associate it with the corresponding key when consuming the data
elizabethengelman commented 2019-01-10 19:33:36 +00:00 (Migrated from github.com)
Review
Copy Link

Yep, good call - k is just capturing the path to the storage value. I think I'll want both the key and the value when I add the storage value to the csv 👌

Yep, good call - `k` is just capturing the path to the storage value. I think I'll want both the key and the value when I add the storage value to the csv 👌
return "", err
rmulhol commented 2019-01-03 23:00:34 +00:00 (Migrated from github.com)
Review
Copy Link

And then I think the _ here could be captured to publish the address

And then I think the `_` here could be captured to publish the address
rmulhol commented 2019-01-03 23:05:31 +00:00 (Migrated from github.com)
Review
Copy Link

I'm a little confused about how this works - isn't accountDiff.Storage a map? Does k capture the key and value? Thinking we probably want both of those so that we can parse the new value and associate it with the corresponding key when consuming the data

I'm a little confused about how this works - isn't `accountDiff.Storage` a map? Does `k` capture the key and value? Thinking we probably want both of those so that we can parse the new value and associate it with the corresponding key when consuming the data
elizabethengelman commented 2019-01-10 19:33:36 +00:00 (Migrated from github.com)
Review
Copy Link

Yep, good call - k is just capturing the path to the storage value. I think I'll want both the key and the value when I add the storage value to the csv 👌

Yep, good call - `k` is just capturing the path to the storage value. I think I'll want both the key and the value when I add the storage value to the csv 👌
}
}
return nil
rmulhol commented 2019-01-03 23:00:34 +00:00 (Migrated from github.com)
Review
Copy Link

And then I think the _ here could be captured to publish the address

And then I think the `_` here could be captured to publish the address
rmulhol commented 2019-01-03 23:05:31 +00:00 (Migrated from github.com)
Review
Copy Link

I'm a little confused about how this works - isn't accountDiff.Storage a map? Does k capture the key and value? Thinking we probably want both of those so that we can parse the new value and associate it with the corresponding key when consuming the data

I'm a little confused about how this works - isn't `accountDiff.Storage` a map? Does `k` capture the key and value? Thinking we probably want both of those so that we can parse the new value and associate it with the corresponding key when consuming the data
elizabethengelman commented 2019-01-10 19:33:36 +00:00 (Migrated from github.com)
Review
Copy Link

Yep, good call - k is just capturing the path to the storage value. I think I'll want both the key and the value when I add the storage value to the csv 👌

Yep, good call - `k` is just capturing the path to the storage value. I think I'll want both the key and the value when I add the storage value to the csv 👌
return filePath, nil
rmulhol commented 2019-01-03 23:00:34 +00:00 (Migrated from github.com)
Review
Copy Link

And then I think the _ here could be captured to publish the address

And then I think the `_` here could be captured to publish the address
rmulhol commented 2019-01-03 23:05:31 +00:00 (Migrated from github.com)
Review
Copy Link

I'm a little confused about how this works - isn't accountDiff.Storage a map? Does k capture the key and value? Thinking we probably want both of those so that we can parse the new value and associate it with the corresponding key when consuming the data

I'm a little confused about how this works - isn't `accountDiff.Storage` a map? Does `k` capture the key and value? Thinking we probably want both of those so that we can parse the new value and associate it with the corresponding key when consuming the data
elizabethengelman commented 2019-01-10 19:33:36 +00:00 (Migrated from github.com)
Review
Copy Link

Yep, good call - k is just capturing the path to the storage value. I think I'll want both the key and the value when I add the storage value to the csv 👌

Yep, good call - `k` is just capturing the path to the storage value. I think I'll want both the key and the value when I add the storage value to the csv 👌
}
func accumulateUpdatedAccountRows(sd builder.StateDiff) [][]string {

rmulhol commented 2019-01-03 23:00:34 +00:00 (Migrated from github.com)
Review
Copy Link

And then I think the _ here could be captured to publish the address

And then I think the `_` here could be captured to publish the address
rmulhol commented 2019-01-03 23:00:34 +00:00 (Migrated from github.com)
Review
Copy Link

And then I think the _ here could be captured to publish the address

And then I think the `_` here could be captured to publish the address
rmulhol commented 2019-01-03 23:05:31 +00:00 (Migrated from github.com)
Review
Copy Link

I'm a little confused about how this works - isn't accountDiff.Storage a map? Does k capture the key and value? Thinking we probably want both of those so that we can parse the new value and associate it with the corresponding key when consuming the data

I'm a little confused about how this works - isn't `accountDiff.Storage` a map? Does `k` capture the key and value? Thinking we probably want both of those so that we can parse the new value and associate it with the corresponding key when consuming the data
rmulhol commented 2019-01-03 23:05:31 +00:00 (Migrated from github.com)
Review
Copy Link

I'm a little confused about how this works - isn't accountDiff.Storage a map? Does k capture the key and value? Thinking we probably want both of those so that we can parse the new value and associate it with the corresponding key when consuming the data

I'm a little confused about how this works - isn't `accountDiff.Storage` a map? Does `k` capture the key and value? Thinking we probably want both of those so that we can parse the new value and associate it with the corresponding key when consuming the data
elizabethengelman commented 2019-01-10 19:33:36 +00:00 (Migrated from github.com)
Review
Copy Link

Yep, good call - k is just capturing the path to the storage value. I think I'll want both the key and the value when I add the storage value to the csv 👌

Yep, good call - `k` is just capturing the path to the storage value. I think I'll want both the key and the value when I add the storage value to the csv 👌
elizabethengelman commented 2019-01-10 19:33:36 +00:00 (Migrated from github.com)
Review
Copy Link

Yep, good call - k is just capturing the path to the storage value. I think I'll want both the key and the value when I add the storage value to the csv 👌

Yep, good call - `k` is just capturing the path to the storage value. I think I'll want both the key and the value when I add the storage value to the csv 👌

4
statediff/publisher/publisher.go
View File

@ -41,8 +41,8 @@ func NewPublisher(config statediff.Config) (*publisher, error) {
func (p *publisher) PublishStateDiff(sd *builder.StateDiff) (string, error) {
switch p.Config.Mode {
case statediff.CSV:
return "", p.publishStateDiffToCSV(*sd)
return p.publishStateDiffToCSV(*sd)
default:
return "", p.publishStateDiffToCSV(*sd)
return p.publishStateDiffToCSV(*sd)
}
}

69
statediff/publisher/publisher_test.go
View File

@ -14,6 +14,7 @@ import (
"reflect"
"strconv"
"testing"
"strings"
)
var (
@ -77,6 +78,7 @@ func TestPublisher(t *testing.T) {
type Test func(t *testing.T)
var tests = []Test{
testFileName,
testColumnHeaders,
testAccountDiffs,
testWhenNoDiff,
@ -108,23 +110,34 @@ func removeFilesFromDir(dir string) error {
return nil
}
func testFileName(t *testing.T) {
fileName, err := publisher.PublishStateDiff(&testhelpers.TestStateDiff)
if err != nil {
t.Errorf(testhelpers.ErrorFormatString, t.Name(), err)
}
if !strings.HasPrefix(fileName, dir) {
t.Errorf(testhelpers.TestFailureFormatString, t.Name(), dir, fileName)
}
}
func testColumnHeaders(t *testing.T) {
_, err = publisher.PublishStateDiff(&testhelpers.TestStateDiff)
if err != nil {
t.Errorf(testhelpers.TestFailedFormatString, t.Name(), err)
t.Errorf(testhelpers.ErrorFormatString, t.Name(), err)
}
file, err := getTestDiffFile(dir)
if err != nil {
t.Errorf(testhelpers.TestFailedFormatString, t.Name(), err)
t.Errorf(testhelpers.ErrorFormatString, t.Name(), err)
}
lines, err := csv.NewReader(file).ReadAll()
if err != nil {
t.Errorf(testhelpers.TestFailedFormatString, t.Name(), err)
t.Errorf(testhelpers.ErrorFormatString, t.Name(), err)
}
if len(lines) < 1 {
t.Errorf(testhelpers.TestFailedFormatString, t.Name(), err)
t.Errorf(testhelpers.ErrorFormatString, t.Name(), err)
}
if !equals(lines[0], p.Headers) {
t.Error()
@ -135,29 +148,29 @@ func testAccountDiffs(t *testing.T) {
// it persists the created, updated and deleted account diffs to a CSV file
_, err = publisher.PublishStateDiff(&testhelpers.TestStateDiff)
if err != nil {
t.Errorf(testhelpers.TestFailedFormatString, t.Name(), err)
t.Errorf(testhelpers.ErrorFormatString, t.Name(), err)
}
file, err := getTestDiffFile(dir)
if err != nil {
t.Errorf(testhelpers.TestFailedFormatString, t.Name(), err)
t.Errorf(testhelpers.ErrorFormatString, t.Name(), err)
}
lines, err := csv.NewReader(file).ReadAll()
if err != nil {
t.Errorf(testhelpers.TestFailedFormatString, t.Name(), err)
t.Errorf(testhelpers.ErrorFormatString, t.Name(), err)
}
if len(lines) <= 3 {
t.Errorf(testhelpers.TestFailedFormatString, t.Name(), err)
t.Errorf(testhelpers.ErrorFormatString, t.Name(), err)
}
if !equals(lines[1], expectedCreatedAccountRow) {
t.Errorf(testhelpers.TestFailedFormatString, t.Name(), err)
t.Errorf(testhelpers.ErrorFormatString, t.Name(), err)
}
if !equals(lines[2], expectedUpdatedAccountRow) {
t.Errorf(testhelpers.TestFailedFormatString, t.Name(), err)
t.Errorf(testhelpers.ErrorFormatString, t.Name(), err)
}
if !equals(lines[3], expectedDeletedAccountRow) {
t.Errorf(testhelpers.TestFailedFormatString, t.Name(), err)
t.Errorf(testhelpers.ErrorFormatString, t.Name(), err)
}
}
@ -166,21 +179,21 @@ func testWhenNoDiff(t *testing.T) {
emptyDiff := builder.StateDiff{}
_, err = publisher.PublishStateDiff(&emptyDiff)
if err != nil {
t.Errorf(testhelpers.TestFailedFormatString, t.Name(), err)
t.Errorf(testhelpers.ErrorFormatString, t.Name(), err)
}
file, err := getTestDiffFile(dir)
if err != nil {
t.Errorf(testhelpers.TestFailedFormatString, t.Name(), err)
t.Errorf(testhelpers.ErrorFormatString, t.Name(), err)
}
lines, err := csv.NewReader(file).ReadAll()
if err != nil {
t.Errorf(testhelpers.TestFailedFormatString, t.Name(), err)
t.Errorf(testhelpers.ErrorFormatString, t.Name(), err)
}
if !equals(len(lines), 1) {
t.Errorf(testhelpers.TestFailedFormatString, t.Name(), err)
t.Errorf(testhelpers.ErrorFormatString, t.Name(), err)
}
}
@ -189,28 +202,28 @@ func testDefaultPublisher(t *testing.T) {
config := statediff.Config{Path: dir}
publisher, err = p.NewPublisher(config)
if err != nil {
t.Errorf(testhelpers.TestFailedFormatString, t.Name(), err)
t.Errorf(testhelpers.ErrorFormatString, t.Name(), err)
}
_, err = publisher.PublishStateDiff(&testhelpers.TestStateDiff)
if err != nil {
t.Errorf(testhelpers.TestFailedFormatString, t.Name(), err)
t.Errorf(testhelpers.ErrorFormatString, t.Name(), err)
}
file, err := getTestDiffFile(dir)
if err != nil {
t.Errorf(testhelpers.TestFailedFormatString, t.Name(), err)
t.Errorf(testhelpers.ErrorFormatString, t.Name(), err)
}
lines, err := csv.NewReader(file).ReadAll()
if err != nil {
t.Errorf(testhelpers.TestFailedFormatString, t.Name(), err)
t.Errorf(testhelpers.ErrorFormatString, t.Name(), err)
}
if !equals(len(lines), 4) {
t.Errorf(testhelpers.TestFailedFormatString, t.Name(), err)
t.Errorf(testhelpers.ErrorFormatString, t.Name(), err)
}
if !equals(lines[0], p.Headers) {
t.Errorf(testhelpers.TestFailedFormatString, t.Name(), err)
t.Errorf(testhelpers.ErrorFormatString, t.Name(), err)
}
}
@ -219,33 +232,33 @@ func testDefaultDirectory(t *testing.T) {
config := statediff.Config{}
publisher, err = p.NewPublisher(config)
if err != nil {
t.Errorf(testhelpers.TestFailedFormatString, t.Name(), err)
t.Errorf(testhelpers.ErrorFormatString, t.Name(), err)
}
err := os.Chdir(dir)
if err != nil {
t.Errorf(testhelpers.TestFailedFormatString, t.Name(), err)
t.Errorf(testhelpers.ErrorFormatString, t.Name(), err)
}
_, err = publisher.PublishStateDiff(&testhelpers.TestStateDiff)
if err != nil {
t.Errorf(testhelpers.TestFailedFormatString, t.Name(), err)
t.Errorf(testhelpers.ErrorFormatString, t.Name(), err)
}
file, err := getTestDiffFile(dir)
if err != nil {
t.Errorf(testhelpers.TestFailedFormatString, t.Name(), err)
t.Errorf(testhelpers.ErrorFormatString, t.Name(), err)
}
lines, err := csv.NewReader(file).ReadAll()
if err != nil {
t.Errorf(testhelpers.TestFailedFormatString, t.Name(), err)
t.Errorf(testhelpers.ErrorFormatString, t.Name(), err)
}
if !equals(len(lines), 4) {
t.Errorf(testhelpers.TestFailedFormatString, t.Name(), err)
t.Errorf(testhelpers.ErrorFormatString, t.Name(), err)
}
if !equals(lines[0], p.Headers) {
t.Errorf(testhelpers.TestFailedFormatString, t.Name(), err)
t.Errorf(testhelpers.ErrorFormatString, t.Name(), err)
}
}

3
statediff/testhelpers/helpers.go
View File

@ -1,3 +1,4 @@
package testhelpers
var TestFailedFormatString = "Test failed: %s, %+v"
var ErrorFormatString = "Error: %s, %+v"
var TestFailureFormatString = "Test failed: %s\nexpected %+v, got %+v\n"