From 197f98c93d4ec9e5932e33dff10bf8246c43954c Mon Sep 17 00:00:00 2001 From: Ian Norden Date: Tue, 2 Apr 2019 13:30:15 -0500 Subject: [PATCH] going ahead and indexing the entire uncle blocks (one of the issues open on public); finish tests --- db/migrations/00026_create_uncles_table.sql | 20 ++++++ ...7_update_block_and_uncle_reward_types.sql} | 0 .../00045_create_uncle_rewards_table.sql | 15 ----- pkg/core/block.go | 36 +++++----- pkg/core/uncle.go | 27 ++++++++ .../postgres/repositories/block_repository.go | 30 ++++----- .../repositories/block_repository_test.go | 66 +++++++++++++++++++ pkg/fakes/data.go | 13 +++- pkg/geth/converters/common/block_converter.go | 42 ++++++++++-- .../converters/common/block_converter_test.go | 15 +++-- pkg/geth/converters/common/block_rewards.go | 28 -------- 11 files changed, 201 insertions(+), 91 deletions(-) create mode 100644 db/migrations/00026_create_uncles_table.sql rename db/migrations/{00046_update_block_and_uncle_reward_types.sql => 00027_update_block_and_uncle_reward_types.sql} (100%) delete mode 100644 db/migrations/00045_create_uncle_rewards_table.sql create mode 100644 pkg/core/uncle.go diff --git a/db/migrations/00026_create_uncles_table.sql b/db/migrations/00026_create_uncles_table.sql new file mode 100644 index 00000000..524c1f4b --- /dev/null +++ b/db/migrations/00026_create_uncles_table.sql @@ -0,0 +1,20 @@ +-- +goose Up +CREATE TABLE public.uncles ( + id SERIAL PRIMARY KEY, + hash VARCHAR(66) NOT NULL, + block_id INTEGER NOT NULL REFERENCES blocks (id) ON DELETE CASCADE, + block_hash VARCHAR(66) NOT NULL, + reward NUMERIC NOT NULL, + miner VARCHAR(42) NOT NULL, + raw JSONB, + block_timestamp NUMERIC, + eth_node_id INTEGER, + eth_node_fingerprint VARCHAR(128), + CONSTRAINT eth_nodes_fk FOREIGN KEY (eth_node_id) + REFERENCES eth_nodes (id) + ON DELETE CASCADE, + UNIQUE (block_id, hash) +); + +-- +goose Down +DROP TABLE public.uncles; diff --git a/db/migrations/00046_update_block_and_uncle_reward_types.sql b/db/migrations/00027_update_block_and_uncle_reward_types.sql similarity index 100% rename from db/migrations/00046_update_block_and_uncle_reward_types.sql rename to db/migrations/00027_update_block_and_uncle_reward_types.sql diff --git a/db/migrations/00045_create_uncle_rewards_table.sql b/db/migrations/00045_create_uncle_rewards_table.sql deleted file mode 100644 index 4c40561f..00000000 --- a/db/migrations/00045_create_uncle_rewards_table.sql +++ /dev/null @@ -1,15 +0,0 @@ --- +goose Up -CREATE TABLE public.uncle_rewards ( - id SERIAL PRIMARY KEY, - block_id INTEGER, - block_hash VARCHAR(66) NOT NULL, - uncle_hash VARCHAR(66) NOT NULL, - uncle_reward NUMERIC NOT NULL, - miner_address VARCHAR(66) NOT NULL, - CONSTRAINT block_id_fk FOREIGN KEY (block_id) - REFERENCES blocks (id) - ON DELETE CASCADE -); - --- +goose Down -DROP TABLE public.uncle_rewards; diff --git a/pkg/core/block.go b/pkg/core/block.go index e7546f19..f878df98 100644 --- a/pkg/core/block.go +++ b/pkg/core/block.go @@ -16,24 +16,22 @@ package core -import "math/big" - type Block struct { - Reward string `db:"reward"` - Difficulty int64 `db:"difficulty"` - ExtraData string `db:"extra_data"` - GasLimit uint64 `db:"gaslimit"` - GasUsed uint64 `db:"gasused"` - Hash string `db:"hash"` - IsFinal bool `db:"is_final"` - Miner string `db:"miner"` - Nonce string `db:"nonce"` - Number int64 `db:"number"` - ParentHash string `db:"parenthash"` - Size string `db:"size"` - Time int64 `db:"time"` - Transactions []TransactionModel - UncleHash string `db:"uncle_hash"` - UnclesReward string `db:"uncles_reward"` - MappedUncleRewards map[string]map[string]*big.Int + Reward string `db:"reward"` + Difficulty int64 `db:"difficulty"` + ExtraData string `db:"extra_data"` + GasLimit uint64 `db:"gaslimit"` + GasUsed uint64 `db:"gasused"` + Hash string `db:"hash"` + IsFinal bool `db:"is_final"` + Miner string `db:"miner"` + Nonce string `db:"nonce"` + Number int64 `db:"number"` + ParentHash string `db:"parenthash"` + Size string `db:"size"` + Time int64 `db:"time"` + Transactions []TransactionModel + UncleHash string `db:"uncle_hash"` + UnclesReward string `db:"uncles_reward"` + Uncles []Uncle } diff --git a/pkg/core/uncle.go b/pkg/core/uncle.go new file mode 100644 index 00000000..018d5924 --- /dev/null +++ b/pkg/core/uncle.go @@ -0,0 +1,27 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package core + +type Uncle struct { + Id int64 + Miner string + BlockHash string `db:"block_hash"` + Reward string + Hash string + Timestamp string `db:"block_timestamp"` + Raw []byte +} diff --git a/pkg/datastore/postgres/repositories/block_repository.go b/pkg/datastore/postgres/repositories/block_repository.go index 16e4c5b4..80affa11 100644 --- a/pkg/datastore/postgres/repositories/block_repository.go +++ b/pkg/datastore/postgres/repositories/block_repository.go @@ -19,12 +19,10 @@ package repositories import ( "database/sql" "errors" - "math/big" - "github.com/jmoiron/sqlx" log "github.com/sirupsen/logrus" - "github.com/vulcanize/vulcanizedb/libraries/shared/utilities" + "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" @@ -157,8 +155,8 @@ func (blockRepository BlockRepository) insertBlock(block core.Block) (int64, err } return 0, postgres.ErrDBInsertFailed(insertBlockErr) } - if len(block.MappedUncleRewards) > 0 { - insertUncleErr := blockRepository.createUncleRewards(tx, blockId, block.Hash, block.MappedUncleRewards) + if len(block.Uncles) > 0 { + insertUncleErr := blockRepository.createUncles(tx, blockId, block.Hash, block.Uncles) if insertUncleErr != nil { tx.Rollback() return 0, postgres.ErrDBInsertFailed(insertUncleErr) @@ -185,25 +183,23 @@ func (blockRepository BlockRepository) insertBlock(block core.Block) (int64, err return blockId, nil } -func (blockRepository BlockRepository) createUncleRewards(tx *sqlx.Tx, blockId int64, blockHash string, mappedUncleRewards map[string]map[string]*big.Int) error { - for minerAddr, uncleRewards := range mappedUncleRewards { - for uncleHash, reward := range uncleRewards { - err := blockRepository.createUncleReward(tx, blockId, blockHash, minerAddr, uncleHash, reward.String()) - if err != nil { - return err - } +func (blockRepository BlockRepository) createUncles(tx *sqlx.Tx, blockId int64, blockHash string, uncles []core.Uncle) error { + for _, uncle := range uncles { + err := blockRepository.createUncle(tx, blockId, uncle) + if err != nil { + return err } } return nil } -func (blockRepository BlockRepository) createUncleReward(tx *sqlx.Tx, blockId int64, blockHash, minerAddr, uncleHash, amount string) error { +func (blockRepository BlockRepository) createUncle(tx *sqlx.Tx, blockId int64, uncle core.Uncle) error { _, err := tx.Exec( - `INSERT INTO uncle_rewards - (block_id, block_hash, miner_address, uncle_hash, uncle_reward) - VALUES ($1, $2, $3, $4, $5) + `INSERT INTO uncles + (hash, block_id, block_hash, reward, miner, raw, block_timestamp, eth_node_id, eth_node_fingerprint) + VALUES ($1, $2, $3, $4, $5, $6, $7::NUMERIC, $8, $9) RETURNING id`, - blockId, blockHash, minerAddr, uncleHash, utilities.NullToZero(amount)) + uncle.Hash, blockId, uncle.BlockHash, utilities.NullToZero(uncle.Reward), uncle.Miner, uncle.Raw, uncle.Timestamp, blockRepository.database.NodeID, blockRepository.database.Node.ID) return err } diff --git a/pkg/datastore/postgres/repositories/block_repository_test.go b/pkg/datastore/postgres/repositories/block_repository_test.go index fb126d00..2d355667 100644 --- a/pkg/datastore/postgres/repositories/block_repository_test.go +++ b/pkg/datastore/postgres/repositories/block_repository_test.go @@ -158,6 +158,72 @@ var _ = Describe("Saving blocks", func() { Expect(len(savedBlock.Transactions)).To(Equal(2)) }) + It("saves one uncle associated to the block", func() { + block := core.Block{ + Hash: fakes.FakeHash.String(), + Number: 123, + Transactions: []core.TransactionModel{fakes.FakeTransaction}, + Uncles: []core.Uncle{fakes.GetFakeUncle(common.BytesToHash([]byte{1, 2, 3}).String(), "100000")}, + UnclesReward: "156250000000000000", + } + + id, insertErr := blockRepository.CreateOrUpdateBlock(block) + + Expect(insertErr).NotTo(HaveOccurred()) + savedBlock, getErr := blockRepository.GetBlock(123) + Expect(getErr).NotTo(HaveOccurred()) + Expect(len(savedBlock.Transactions)).To(Equal(1)) + Expect(savedBlock.UnclesReward).To(Equal(big.NewInt(0).Div(big.NewInt(5000000000000000000), big.NewInt(32)).String())) + + var uncleModel core.Uncle + err := db.Get(&uncleModel, `SELECT hash, block_hash, reward, miner, raw, block_timestamp FROM uncles + WHERE block_id = $1 AND hash = $2`, id, common.BytesToHash([]byte{1, 2, 3}).Hex()) + Expect(err).ToNot(HaveOccurred()) + Expect(uncleModel.Hash).To(Equal(common.BytesToHash([]byte{1, 2, 3}).Hex())) + Expect(uncleModel.Reward).To(Equal("100000")) + Expect(uncleModel.Miner).To(Equal(fakes.FakeAddress.Hex())) + Expect(uncleModel.Timestamp).To(Equal("111111111")) + }) + + It("saves one uncle associated to the block", func() { + block := core.Block{ + Hash: fakes.FakeHash.String(), + Number: 123, + Transactions: []core.TransactionModel{fakes.FakeTransaction}, + Uncles: []core.Uncle{ + fakes.GetFakeUncle(common.BytesToHash([]byte{1, 2, 3}).String(), "100000"), + fakes.GetFakeUncle(common.BytesToHash([]byte{3, 2, 1}).String(), "90000")}, + UnclesReward: "312500000000000000", + } + + id, insertErr := blockRepository.CreateOrUpdateBlock(block) + + Expect(insertErr).NotTo(HaveOccurred()) + savedBlock, getErr := blockRepository.GetBlock(123) + Expect(getErr).NotTo(HaveOccurred()) + Expect(len(savedBlock.Transactions)).To(Equal(1)) + b := new(big.Int) + b.SetString("10000000000000000000", 10) + Expect(savedBlock.UnclesReward).To(Equal(big.NewInt(0).Div(b, big.NewInt(32)).String())) + + var uncleModel core.Uncle + err := db.Get(&uncleModel, `SELECT hash, block_hash, reward, miner, raw, block_timestamp FROM uncles + WHERE block_id = $1 AND hash = $2`, id, common.BytesToHash([]byte{1, 2, 3}).Hex()) + Expect(err).ToNot(HaveOccurred()) + Expect(uncleModel.Hash).To(Equal(common.BytesToHash([]byte{1, 2, 3}).Hex())) + Expect(uncleModel.Reward).To(Equal("100000")) + Expect(uncleModel.Miner).To(Equal(fakes.FakeAddress.Hex())) + Expect(uncleModel.Timestamp).To(Equal("111111111")) + + err = db.Get(&uncleModel, `SELECT hash, block_hash, reward, miner, raw, block_timestamp FROM uncles + WHERE block_id = $1 AND hash = $2`, id, common.BytesToHash([]byte{3, 2, 1}).Hex()) + Expect(err).ToNot(HaveOccurred()) + Expect(uncleModel.Hash).To(Equal(common.BytesToHash([]byte{3, 2, 1}).Hex())) + Expect(uncleModel.Reward).To(Equal("90000")) + Expect(uncleModel.Miner).To(Equal(fakes.FakeAddress.Hex())) + Expect(uncleModel.Timestamp).To(Equal("111111111")) + }) + It(`replaces blocks and transactions associated to the block when a more new block is in conflict (same block number + nodeid)`, func() { blockOne := core.Block{ diff --git a/pkg/fakes/data.go b/pkg/fakes/data.go index f8549a0a..6d2829fa 100644 --- a/pkg/fakes/data.go +++ b/pkg/fakes/data.go @@ -73,7 +73,7 @@ func GetFakeTransaction(hash string, receipt core.Receipt) core.TransactionModel var raw bytes.Buffer err := gethTransaction.EncodeRLP(&raw) if err != nil { - panic("failed to marshal transaction creating test fake") + panic("failed to marshal transaction while creating test fake") } return core.TransactionModel{ Data: []byte{}, @@ -89,3 +89,14 @@ func GetFakeTransaction(hash string, receipt core.Receipt) core.TransactionModel Value: "0", } } + +func GetFakeUncle(hash, reward string) core.Uncle { + return core.Uncle{ + Miner: FakeAddress.String(), + Hash: hash, + BlockHash: FakeHash.String(), + Reward: reward, + Raw: rawFakeHeader, + Timestamp: strconv.FormatInt(fakeTimestamp, 10), + } +} diff --git a/pkg/geth/converters/common/block_converter.go b/pkg/geth/converters/common/block_converter.go index 8a503e57..e53d18a2 100644 --- a/pkg/geth/converters/common/block_converter.go +++ b/pkg/geth/converters/common/block_converter.go @@ -17,12 +17,13 @@ package common import ( + "encoding/json" + "math/big" "strings" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" - "github.com/vulcanize/vulcanizedb/libraries/shared/utilities" "github.com/vulcanize/vulcanizedb/pkg/core" ) @@ -56,8 +57,41 @@ func (bc BlockConverter) ToCoreBlock(gethBlock *types.Block) (core.Block, error) UncleHash: gethBlock.UncleHash().Hex(), } coreBlock.Reward = CalcBlockReward(coreBlock, gethBlock.Uncles()).String() - uncleRewards, mappedUncleRewards := CalcUnclesReward(coreBlock, gethBlock.Uncles()) - coreBlock.UnclesReward = utilities.NullToZero(uncleRewards.String()) - coreBlock.MappedUncleRewards = mappedUncleRewards + totalUncleReward, uncles := bc.ToCoreUncle(coreBlock, gethBlock.Uncles()) + + coreBlock.UnclesReward = totalUncleReward.String() + coreBlock.Uncles = uncles return coreBlock, nil } + +// Rewards for the miners of uncles is calculated as (U_n + 8 - B_n) * R / 8 +// Where U_n is the uncle block number, B_n is the parent block number and R is the static block reward at B_n +// https://github.com/ethereum/go-ethereum/issues/1591 +// https://ethereum.stackexchange.com/questions/27172/different-uncles-reward +// https://github.com/ethereum/homestead-guide/issues/399 +// Returns the total uncle reward and the individual processed uncles +func (bc BlockConverter) ToCoreUncle(block core.Block, uncles []*types.Header) (*big.Int, []core.Uncle) { + totalUncleRewards := new(big.Int) + coreUncles := make([]core.Uncle, 0, len(uncles)) + for _, uncle := range uncles { + staticBlockReward := staticRewardByBlockNumber(block.Number) + rewardDiv8 := staticBlockReward.Div(staticBlockReward, big.NewInt(8)) + mainBlock := big.NewInt(block.Number) + uncleBlock := big.NewInt(uncle.Number.Int64()) + uncleBlockPlus8 := uncleBlock.Add(uncleBlock, big.NewInt(8)) + uncleBlockPlus8MinusMainBlock := uncleBlockPlus8.Sub(uncleBlockPlus8, mainBlock) + thisUncleReward := rewardDiv8.Mul(rewardDiv8, uncleBlockPlus8MinusMainBlock) + raw, _ := json.Marshal(uncle) + coreUncle := core.Uncle{ + Miner: uncle.Coinbase.Hex(), + BlockHash: block.Hash, + Hash: uncle.Hash().Hex(), + Raw: raw, + Reward: thisUncleReward.String(), + Timestamp: uncle.Time.String(), + } + coreUncles = append(coreUncles, coreUncle) + totalUncleRewards.Add(totalUncleRewards, thisUncleReward) + } + return totalUncleRewards, coreUncles +} diff --git a/pkg/geth/converters/common/block_converter_test.go b/pkg/geth/converters/common/block_converter_test.go index 7bdf7751..9ecef611 100644 --- a/pkg/geth/converters/common/block_converter_test.go +++ b/pkg/geth/converters/common/block_converter_test.go @@ -159,15 +159,16 @@ var _ = Describe("Conversion of GethBlock to core.Block", func() { expectedTotalReward := new(big.Int) expectedTotalReward.SetString("6875000000000000000", 10) - totalReward, mappedRewards := vulcCommon.CalcUnclesReward(coreBlock, block.Uncles()) + totalReward, coreUncles := blockConverter.ToCoreUncle(coreBlock, block.Uncles()) Expect(totalReward.String()).To(Equal(expectedTotalReward.String())) - Expect(len(mappedRewards)).To(Equal(1)) - Expect(len(mappedRewards["0x0000000000000000000000000000000000000000"])).To(Equal(2)) - Expect(mappedRewards["0x0000000000000000000000000000000000000000"]["0xb629de4014b6e30cf9555ee833f1806fa0d8b8516fde194405f9c98c2deb8772"].String()). - To(Equal(big.NewInt(3125000000000000000).String())) - Expect(mappedRewards["0x0000000000000000000000000000000000000000"]["0x673f5231e4888a951e0bc8a25b5774b982e6e9e258362c21affaff6e02dd5a2b"].String()). - To(Equal(big.NewInt(3750000000000000000).String())) + Expect(len(coreUncles)).To(Equal(2)) + Expect(coreUncles[0].Reward).To(Equal("3125000000000000000")) + Expect(coreUncles[0].Miner).To(Equal("0x0000000000000000000000000000000000000000")) + Expect(coreUncles[0].Hash).To(Equal("0xb629de4014b6e30cf9555ee833f1806fa0d8b8516fde194405f9c98c2deb8772")) + Expect(coreUncles[1].Reward).To(Equal("3750000000000000000")) + Expect(coreUncles[1].Miner).To(Equal("0x0000000000000000000000000000000000000000")) + Expect(coreUncles[1].Hash).To(Equal("0x673f5231e4888a951e0bc8a25b5774b982e6e9e258362c21affaff6e02dd5a2b")) }) It("decreases the static block reward from 5 to 3 for blocks after block 4,269,999", func() { diff --git a/pkg/geth/converters/common/block_rewards.go b/pkg/geth/converters/common/block_rewards.go index fb388910..ce26cadc 100644 --- a/pkg/geth/converters/common/block_rewards.go +++ b/pkg/geth/converters/common/block_rewards.go @@ -23,34 +23,6 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/core" ) -// (U_n + 8 - B_n) * R / 8 -// https://github.com/ethereum/go-ethereum/issues/1591 -// https://ethereum.stackexchange.com/questions/27172/different-uncles-reward -// https://github.com/ethereum/homestead-guide/issues/399 -// Returns a map of miner addresses to a map of the uncles they mined (hashes) to the rewards received for that uncle -func CalcUnclesReward(block core.Block, uncles []*types.Header) (*big.Int, map[string]map[string]*big.Int) { - uncleRewards := new(big.Int) - mappedUncleRewards := make(map[string]map[string]*big.Int) - for _, uncle := range uncles { - staticBlockReward := staticRewardByBlockNumber(block.Number) - rewardDiv8 := staticBlockReward.Div(staticBlockReward, big.NewInt(8)) - uncleBlock := big.NewInt(uncle.Number.Int64()) - uncleBlockPlus8 := uncleBlock.Add(uncleBlock, big.NewInt(8)) - mainBlock := big.NewInt(block.Number) - uncleBlockPlus8MinusMainBlock := uncleBlockPlus8.Sub(uncleBlockPlus8, mainBlock) - thisUncleReward := rewardDiv8.Mul(rewardDiv8, uncleBlockPlus8MinusMainBlock) - uncleRewards = uncleRewards.Add(uncleRewards, thisUncleReward) - if mappedUncleRewards[uncle.Coinbase.Hex()] == nil { - mappedUncleRewards[uncle.Coinbase.Hex()] = make(map[string]*big.Int) - } - if mappedUncleRewards[uncle.Coinbase.Hex()][uncle.Hash().Hex()] == nil { - mappedUncleRewards[uncle.Coinbase.Hex()][uncle.Hash().Hex()] = new(big.Int) - } - mappedUncleRewards[uncle.Coinbase.Hex()][uncle.Hash().Hex()].Add(mappedUncleRewards[uncle.Coinbase.Hex()][uncle.Hash().Hex()], thisUncleReward) - } - return uncleRewards, mappedUncleRewards -} - func CalcBlockReward(block core.Block, uncles []*types.Header) *big.Int { staticBlockReward := staticRewardByBlockNumber(block.Number) transactionFees := calcTransactionFees(block)