going ahead and indexing the entire uncle blocks (one of the issues open on public); finish tests

db/migrations/00026_create_uncles_table.sql

@@ -0,0 +1,20 @@
+-- +goose Up
+CREATE TABLE public.uncles (
+  id                    SERIAL PRIMARY KEY,
+  hash                  VARCHAR(66) NOT NULL,
+  block_id              INTEGER NOT NULL REFERENCES blocks (id) ON DELETE CASCADE,
+  block_hash            VARCHAR(66) NOT NULL,
+  reward                NUMERIC NOT NULL,
+  miner                 VARCHAR(42) NOT NULL,
+  raw                   JSONB,
+  block_timestamp       NUMERIC,
+  eth_node_id           INTEGER,
+  eth_node_fingerprint  VARCHAR(128),
+  CONSTRAINT eth_nodes_fk FOREIGN KEY (eth_node_id)
+  REFERENCES eth_nodes (id)
+  ON DELETE CASCADE,
+  UNIQUE (block_id, hash)
+);
+
+-- +goose Down
+DROP TABLE public.uncles;

db/migrations/00045_create_uncle_rewards_table.sql

@@ -1,15 +0,0 @@
--- +goose Up
-CREATE TABLE public.uncle_rewards (
-  id            SERIAL PRIMARY KEY,
-  block_id      INTEGER,
-  block_hash    VARCHAR(66) NOT NULL,
-  uncle_hash    VARCHAR(66) NOT NULL,
-  uncle_reward  NUMERIC NOT NULL,
-  miner_address VARCHAR(66) NOT NULL,
-  CONSTRAINT block_id_fk FOREIGN KEY (block_id)
-  REFERENCES blocks (id)
-  ON DELETE CASCADE
-);
-
--- +goose Down
-DROP TABLE public.uncle_rewards;

pkg/core/block.go

@@ -16,24 +16,22 @@
 
 package core
 
-import "math/big"
-
 type Block struct {
-	Reward             string `db:"reward"`
+	Reward       string `db:"reward"`
-	Difficulty         int64  `db:"difficulty"`
+	Difficulty   int64  `db:"difficulty"`
-	ExtraData          string `db:"extra_data"`
+	ExtraData    string `db:"extra_data"`
-	GasLimit           uint64 `db:"gaslimit"`
+	GasLimit     uint64 `db:"gaslimit"`
-	GasUsed            uint64 `db:"gasused"`
+	GasUsed      uint64 `db:"gasused"`
-	Hash               string `db:"hash"`
+	Hash         string `db:"hash"`
-	IsFinal            bool   `db:"is_final"`
+	IsFinal      bool   `db:"is_final"`
-	Miner              string `db:"miner"`
+	Miner        string `db:"miner"`
-	Nonce              string `db:"nonce"`
+	Nonce        string `db:"nonce"`
-	Number             int64  `db:"number"`
+	Number       int64  `db:"number"`
-	ParentHash         string `db:"parenthash"`
+	ParentHash   string `db:"parenthash"`
-	Size               string `db:"size"`
+	Size         string `db:"size"`
-	Time               int64  `db:"time"`
+	Time         int64  `db:"time"`
-	Transactions       []TransactionModel
+	Transactions []TransactionModel
-	UncleHash          string `db:"uncle_hash"`
+	UncleHash    string `db:"uncle_hash"`
-	UnclesReward       string `db:"uncles_reward"`
+	UnclesReward string `db:"uncles_reward"`
-	MappedUncleRewards map[string]map[string]*big.Int
+	Uncles       []Uncle
 }

pkg/core/uncle.go

@@ -0,0 +1,27 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package core
+
+type Uncle struct {
+	Id        int64
+	Miner     string
+	BlockHash string `db:"block_hash"`
+	Reward    string
+	Hash      string
+	Timestamp string `db:"block_timestamp"`
+	Raw       []byte
+}

pkg/datastore/postgres/repositories/block_repository.go

@@ -19,12 +19,10 @@ package repositories
 import (
 	"database/sql"
 	"errors"
-	"math/big"
-
 	"github.com/jmoiron/sqlx"
 	log "github.com/sirupsen/logrus"
-
 	"github.com/vulcanize/vulcanizedb/libraries/shared/utilities"
+
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
@@ -157,8 +155,8 @@ func (blockRepository BlockRepository) insertBlock(block core.Block) (int64, err
 		}
 		return 0, postgres.ErrDBInsertFailed(insertBlockErr)
 	}
-	if len(block.MappedUncleRewards) > 0 {
+	if len(block.Uncles) > 0 {
-		insertUncleErr := blockRepository.createUncleRewards(tx, blockId, block.Hash, block.MappedUncleRewards)
+		insertUncleErr := blockRepository.createUncles(tx, blockId, block.Hash, block.Uncles)
 		if insertUncleErr != nil {
 			tx.Rollback()
 			return 0, postgres.ErrDBInsertFailed(insertUncleErr)
@@ -185,25 +183,23 @@ func (blockRepository BlockRepository) insertBlock(block core.Block) (int64, err
 	return blockId, nil
 }
 
-func (blockRepository BlockRepository) createUncleRewards(tx *sqlx.Tx, blockId int64, blockHash string, mappedUncleRewards map[string]map[string]*big.Int) error {
+func (blockRepository BlockRepository) createUncles(tx *sqlx.Tx, blockId int64, blockHash string, uncles []core.Uncle) error {
-	for minerAddr, uncleRewards := range mappedUncleRewards {
+	for _, uncle := range uncles {
-		for uncleHash, reward := range uncleRewards {
+		err := blockRepository.createUncle(tx, blockId, uncle)
-			err := blockRepository.createUncleReward(tx, blockId, blockHash, minerAddr, uncleHash, reward.String())
+		if err != nil {
-			if err != nil {
+			return err
-				return err
-			}
 		}
 	}
 	return nil
 }
 
-func (blockRepository BlockRepository) createUncleReward(tx *sqlx.Tx, blockId int64, blockHash, minerAddr, uncleHash, amount string) error {
+func (blockRepository BlockRepository) createUncle(tx *sqlx.Tx, blockId int64, uncle core.Uncle) error {
 	_, err := tx.Exec(
-		`INSERT INTO uncle_rewards
+		`INSERT INTO uncles
-       (block_id, block_hash, miner_address, uncle_hash, uncle_reward)
+       (hash, block_id, block_hash, reward, miner, raw, block_timestamp, eth_node_id, eth_node_fingerprint)
-       VALUES ($1, $2, $3, $4, $5)
+       VALUES ($1, $2, $3, $4, $5, $6, $7::NUMERIC, $8, $9)
        RETURNING id`,
-		blockId, blockHash, minerAddr, uncleHash, utilities.NullToZero(amount))
+		uncle.Hash, blockId, uncle.BlockHash, utilities.NullToZero(uncle.Reward), uncle.Miner, uncle.Raw, uncle.Timestamp, blockRepository.database.NodeID, blockRepository.database.Node.ID)
 	return err
 }
 

pkg/datastore/postgres/repositories/block_repository_test.go

@@ -158,6 +158,72 @@ var _ = Describe("Saving blocks", func() {
 		Expect(len(savedBlock.Transactions)).To(Equal(2))
 	})
 
+	It("saves one uncle associated to the block", func() {
+		block := core.Block{
+			Hash:         fakes.FakeHash.String(),
+			Number:       123,
+			Transactions: []core.TransactionModel{fakes.FakeTransaction},
+			Uncles:       []core.Uncle{fakes.GetFakeUncle(common.BytesToHash([]byte{1, 2, 3}).String(), "100000")},
+			UnclesReward: "156250000000000000",
+		}
+
+		id, insertErr := blockRepository.CreateOrUpdateBlock(block)
+
+		Expect(insertErr).NotTo(HaveOccurred())
+		savedBlock, getErr := blockRepository.GetBlock(123)
+		Expect(getErr).NotTo(HaveOccurred())
+		Expect(len(savedBlock.Transactions)).To(Equal(1))
+		Expect(savedBlock.UnclesReward).To(Equal(big.NewInt(0).Div(big.NewInt(5000000000000000000), big.NewInt(32)).String()))
+
+		var uncleModel core.Uncle
+		err := db.Get(&uncleModel, `SELECT hash, block_hash, reward, miner, raw, block_timestamp FROM uncles 
+								WHERE block_id = $1 AND hash = $2`, id, common.BytesToHash([]byte{1, 2, 3}).Hex())
+		Expect(err).ToNot(HaveOccurred())
+		Expect(uncleModel.Hash).To(Equal(common.BytesToHash([]byte{1, 2, 3}).Hex()))
+		Expect(uncleModel.Reward).To(Equal("100000"))
+		Expect(uncleModel.Miner).To(Equal(fakes.FakeAddress.Hex()))
+		Expect(uncleModel.Timestamp).To(Equal("111111111"))
+	})
+
+	It("saves one uncle associated to the block", func() {
+		block := core.Block{
+			Hash:         fakes.FakeHash.String(),
+			Number:       123,
+			Transactions: []core.TransactionModel{fakes.FakeTransaction},
+			Uncles: []core.Uncle{
+				fakes.GetFakeUncle(common.BytesToHash([]byte{1, 2, 3}).String(), "100000"),
+				fakes.GetFakeUncle(common.BytesToHash([]byte{3, 2, 1}).String(), "90000")},
+			UnclesReward: "312500000000000000",
+		}
+
+		id, insertErr := blockRepository.CreateOrUpdateBlock(block)
+
+		Expect(insertErr).NotTo(HaveOccurred())
+		savedBlock, getErr := blockRepository.GetBlock(123)
+		Expect(getErr).NotTo(HaveOccurred())
+		Expect(len(savedBlock.Transactions)).To(Equal(1))
+		b := new(big.Int)
+		b.SetString("10000000000000000000", 10)
+		Expect(savedBlock.UnclesReward).To(Equal(big.NewInt(0).Div(b, big.NewInt(32)).String()))
+
+		var uncleModel core.Uncle
+		err := db.Get(&uncleModel, `SELECT hash, block_hash, reward, miner, raw, block_timestamp FROM uncles 
+								WHERE block_id = $1 AND hash = $2`, id, common.BytesToHash([]byte{1, 2, 3}).Hex())
+		Expect(err).ToNot(HaveOccurred())
+		Expect(uncleModel.Hash).To(Equal(common.BytesToHash([]byte{1, 2, 3}).Hex()))
+		Expect(uncleModel.Reward).To(Equal("100000"))
+		Expect(uncleModel.Miner).To(Equal(fakes.FakeAddress.Hex()))
+		Expect(uncleModel.Timestamp).To(Equal("111111111"))
+
+		err = db.Get(&uncleModel, `SELECT hash, block_hash, reward, miner, raw, block_timestamp FROM uncles 
+								WHERE block_id = $1 AND hash = $2`, id, common.BytesToHash([]byte{3, 2, 1}).Hex())
+		Expect(err).ToNot(HaveOccurred())
+		Expect(uncleModel.Hash).To(Equal(common.BytesToHash([]byte{3, 2, 1}).Hex()))
+		Expect(uncleModel.Reward).To(Equal("90000"))
+		Expect(uncleModel.Miner).To(Equal(fakes.FakeAddress.Hex()))
+		Expect(uncleModel.Timestamp).To(Equal("111111111"))
+	})
+
 	It(`replaces blocks and transactions associated to the block
 			when a more new block is in conflict (same block number + nodeid)`, func() {
 		blockOne := core.Block{

pkg/fakes/data.go

@@ -73,7 +73,7 @@ func GetFakeTransaction(hash string, receipt core.Receipt) core.TransactionModel
 	var raw bytes.Buffer
 	err := gethTransaction.EncodeRLP(&raw)
 	if err != nil {
-		panic("failed to marshal transaction creating test fake")
+		panic("failed to marshal transaction while creating test fake")
 	}
 	return core.TransactionModel{
 		Data:     []byte{},
@@ -89,3 +89,14 @@ func GetFakeTransaction(hash string, receipt core.Receipt) core.TransactionModel
 		Value:    "0",
 	}
 }
+
+func GetFakeUncle(hash, reward string) core.Uncle {
+	return core.Uncle{
+		Miner:     FakeAddress.String(),
+		Hash:      hash,
+		BlockHash: FakeHash.String(),
+		Reward:    reward,
+		Raw:       rawFakeHeader,
+		Timestamp: strconv.FormatInt(fakeTimestamp, 10),
+	}
+}

pkg/geth/converters/common/block_converter.go

@@ -17,12 +17,13 @@
 package common
 
 import (
+	"encoding/json"
+	"math/big"
 	"strings"
 
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/core/types"
 
-	"github.com/vulcanize/vulcanizedb/libraries/shared/utilities"
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 )
 
@@ -56,8 +57,41 @@ func (bc BlockConverter) ToCoreBlock(gethBlock *types.Block) (core.Block, error)
 		UncleHash:    gethBlock.UncleHash().Hex(),
 	}
 	coreBlock.Reward = CalcBlockReward(coreBlock, gethBlock.Uncles()).String()
-	uncleRewards, mappedUncleRewards := CalcUnclesReward(coreBlock, gethBlock.Uncles())
+	totalUncleReward, uncles := bc.ToCoreUncle(coreBlock, gethBlock.Uncles())
-	coreBlock.UnclesReward = utilities.NullToZero(uncleRewards.String())
+
-	coreBlock.MappedUncleRewards = mappedUncleRewards
+	coreBlock.UnclesReward = totalUncleReward.String()
+	coreBlock.Uncles = uncles
 	return coreBlock, nil
 }
+
+// Rewards for the miners of uncles is calculated as (U_n + 8 - B_n) * R / 8
+// Where U_n is the uncle block number, B_n is the parent block number and R is the static block reward at B_n
+// https://github.com/ethereum/go-ethereum/issues/1591
+// https://ethereum.stackexchange.com/questions/27172/different-uncles-reward
+// https://github.com/ethereum/homestead-guide/issues/399
+// Returns the total uncle reward and the individual processed uncles
+func (bc BlockConverter) ToCoreUncle(block core.Block, uncles []*types.Header) (*big.Int, []core.Uncle) {
+	totalUncleRewards := new(big.Int)
+	coreUncles := make([]core.Uncle, 0, len(uncles))
+	for _, uncle := range uncles {
+		staticBlockReward := staticRewardByBlockNumber(block.Number)
+		rewardDiv8 := staticBlockReward.Div(staticBlockReward, big.NewInt(8))
+		mainBlock := big.NewInt(block.Number)
+		uncleBlock := big.NewInt(uncle.Number.Int64())
+		uncleBlockPlus8 := uncleBlock.Add(uncleBlock, big.NewInt(8))
+		uncleBlockPlus8MinusMainBlock := uncleBlockPlus8.Sub(uncleBlockPlus8, mainBlock)
+		thisUncleReward := rewardDiv8.Mul(rewardDiv8, uncleBlockPlus8MinusMainBlock)
+		raw, _ := json.Marshal(uncle)
+		coreUncle := core.Uncle{
+			Miner:     uncle.Coinbase.Hex(),
+			BlockHash: block.Hash,
+			Hash:      uncle.Hash().Hex(),
+			Raw:       raw,
+			Reward:    thisUncleReward.String(),
+			Timestamp: uncle.Time.String(),
+		}
+		coreUncles = append(coreUncles, coreUncle)
+		totalUncleRewards.Add(totalUncleRewards, thisUncleReward)
+	}
+	return totalUncleRewards, coreUncles
+}

pkg/geth/converters/common/block_converter_test.go

@@ -159,15 +159,16 @@ var _ = Describe("Conversion of GethBlock to core.Block", func() {
 
 			expectedTotalReward := new(big.Int)
 			expectedTotalReward.SetString("6875000000000000000", 10)
-			totalReward, mappedRewards := vulcCommon.CalcUnclesReward(coreBlock, block.Uncles())
+			totalReward, coreUncles := blockConverter.ToCoreUncle(coreBlock, block.Uncles())
 			Expect(totalReward.String()).To(Equal(expectedTotalReward.String()))
 
-			Expect(len(mappedRewards)).To(Equal(1))
+			Expect(len(coreUncles)).To(Equal(2))
-			Expect(len(mappedRewards["0x0000000000000000000000000000000000000000"])).To(Equal(2))
+			Expect(coreUncles[0].Reward).To(Equal("3125000000000000000"))
-			Expect(mappedRewards["0x0000000000000000000000000000000000000000"]["0xb629de4014b6e30cf9555ee833f1806fa0d8b8516fde194405f9c98c2deb8772"].String()).
+			Expect(coreUncles[0].Miner).To(Equal("0x0000000000000000000000000000000000000000"))
-				To(Equal(big.NewInt(3125000000000000000).String()))
+			Expect(coreUncles[0].Hash).To(Equal("0xb629de4014b6e30cf9555ee833f1806fa0d8b8516fde194405f9c98c2deb8772"))
-			Expect(mappedRewards["0x0000000000000000000000000000000000000000"]["0x673f5231e4888a951e0bc8a25b5774b982e6e9e258362c21affaff6e02dd5a2b"].String()).
+			Expect(coreUncles[1].Reward).To(Equal("3750000000000000000"))
-				To(Equal(big.NewInt(3750000000000000000).String()))
+			Expect(coreUncles[1].Miner).To(Equal("0x0000000000000000000000000000000000000000"))
+			Expect(coreUncles[1].Hash).To(Equal("0x673f5231e4888a951e0bc8a25b5774b982e6e9e258362c21affaff6e02dd5a2b"))
 		})
 
 		It("decreases the static block reward from 5 to 3 for blocks after block 4,269,999", func() {

pkg/geth/converters/common/block_rewards.go

@@ -23,34 +23,6 @@ import (
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 )
 
-// (U_n + 8 - B_n) * R / 8
-// https://github.com/ethereum/go-ethereum/issues/1591
-// https://ethereum.stackexchange.com/questions/27172/different-uncles-reward
-// https://github.com/ethereum/homestead-guide/issues/399
-// Returns a map of miner addresses to a map of the uncles they mined (hashes) to the rewards received for that uncle
-func CalcUnclesReward(block core.Block, uncles []*types.Header) (*big.Int, map[string]map[string]*big.Int) {
-	uncleRewards := new(big.Int)
-	mappedUncleRewards := make(map[string]map[string]*big.Int)
-	for _, uncle := range uncles {
-		staticBlockReward := staticRewardByBlockNumber(block.Number)
-		rewardDiv8 := staticBlockReward.Div(staticBlockReward, big.NewInt(8))
-		uncleBlock := big.NewInt(uncle.Number.Int64())
-		uncleBlockPlus8 := uncleBlock.Add(uncleBlock, big.NewInt(8))
-		mainBlock := big.NewInt(block.Number)
-		uncleBlockPlus8MinusMainBlock := uncleBlockPlus8.Sub(uncleBlockPlus8, mainBlock)
-		thisUncleReward := rewardDiv8.Mul(rewardDiv8, uncleBlockPlus8MinusMainBlock)
-		uncleRewards = uncleRewards.Add(uncleRewards, thisUncleReward)
-		if mappedUncleRewards[uncle.Coinbase.Hex()] == nil {
-			mappedUncleRewards[uncle.Coinbase.Hex()] = make(map[string]*big.Int)
-		}
-		if mappedUncleRewards[uncle.Coinbase.Hex()][uncle.Hash().Hex()] == nil {
-			mappedUncleRewards[uncle.Coinbase.Hex()][uncle.Hash().Hex()] = new(big.Int)
-		}
-		mappedUncleRewards[uncle.Coinbase.Hex()][uncle.Hash().Hex()].Add(mappedUncleRewards[uncle.Coinbase.Hex()][uncle.Hash().Hex()], thisUncleReward)
-	}
-	return uncleRewards, mappedUncleRewards
-}
-
 func CalcBlockReward(block core.Block, uncles []*types.Header) *big.Int {
 	staticBlockReward := staticRewardByBlockNumber(block.Number)
 	transactionFees := calcTransactionFees(block)