From b934f6c9636375bd84a468822f7f1ad7251702dc Mon Sep 17 00:00:00 2001 From: Ian Norden Date: Tue, 17 Mar 2020 07:50:38 -0500 Subject: [PATCH] btc and eth unit tests for db cleaning --- pkg/super_node/btc/cleaner.go | 28 +- pkg/super_node/btc/cleaner_test.go | 288 ++++++++++++++ pkg/super_node/eth/cleaner.go | 74 ++-- pkg/super_node/eth/cleaner_test.go | 614 +++++++++++++++++++++++++++++ pkg/super_node/shared/data_type.go | 80 +++- 5 files changed, 1014 insertions(+), 70 deletions(-) create mode 100644 pkg/super_node/btc/cleaner_test.go create mode 100644 pkg/super_node/eth/cleaner_test.go diff --git a/pkg/super_node/btc/cleaner.go b/pkg/super_node/btc/cleaner.go index d3c90746..bf10e680 100644 --- a/pkg/super_node/btc/cleaner.go +++ b/pkg/super_node/btc/cleaner.go @@ -57,16 +57,8 @@ func (c *Cleaner) Clean(rngs [][2]uint64, t shared.DataType) error { func (c *Cleaner) clean(tx *sqlx.Tx, rng [2]uint64, t shared.DataType) error { switch t { - case shared.Full: + case shared.Full, shared.Headers: return c.cleanFull(tx, rng) - case shared.Headers: - if err := c.cleanTransactionIPLDs(tx, rng); err != nil { - return err - } - if err := c.cleanHeaderIPLDs(tx, rng); err != nil { - return err - } - return c.cleanHeaderMetaData(tx, rng) case shared.Transactions: if err := c.cleanTransactionIPLDs(tx, rng); err != nil { return err @@ -78,21 +70,13 @@ func (c *Cleaner) clean(tx *sqlx.Tx, rng [2]uint64, t shared.DataType) error { } func (c *Cleaner) cleanFull(tx *sqlx.Tx, rng [2]uint64) error { - // Clear all of the indexed iplds - pgStr := `DELETE FROM public.blocks A - USING btc.transaction_cids B, btc.header_cids C - WHERE (A.key = B.cid OR A.key = C.cid) - AND B.header_id = C.id - AND C.block_number BETWEEN $1 AND $2` - _, err := tx.Exec(pgStr, rng[0], rng[1]) - if err != nil { + if err := c.cleanTransactionIPLDs(tx, rng); err != nil { return err } - // Clear all the header_cids, this will cascade delete the rest of the index metadata - pgStr = `DELETE FROM eth.header_cids - WHERE block_number BETWEEN $1 AND $2` - _, err = tx.Exec(pgStr, rng[0], rng[1]) - return err + if err := c.cleanHeaderIPLDs(tx, rng); err != nil { + return err + } + return c.cleanHeaderMetaData(tx, rng) } func (c *Cleaner) cleanTransactionIPLDs(tx *sqlx.Tx, rng [2]uint64) error { diff --git a/pkg/super_node/btc/cleaner_test.go b/pkg/super_node/btc/cleaner_test.go new file mode 100644 index 00000000..a245a294 --- /dev/null +++ b/pkg/super_node/btc/cleaner_test.go @@ -0,0 +1,288 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package btc_test + +import ( + "math/big" + + "github.com/ethereum/go-ethereum/crypto" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vulcanize/vulcanizedb/pkg/postgres" + "github.com/vulcanize/vulcanizedb/pkg/super_node/btc" + "github.com/vulcanize/vulcanizedb/pkg/super_node/shared" +) + +var ( + // Block 0 + // header variables + blockHash1 = crypto.Keccak256Hash([]byte{00, 02}) + blocKNumber1 = big.NewInt(0) + headerCid1 = "mockHeader1CID" + parentHash = crypto.Keccak256Hash([]byte{00, 01}) + totalDifficulty = "50000000000000000000" + reward = "5000000000000000000" + headerModel1 = btc.HeaderModel{ + BlockHash: blockHash1.String(), + BlockNumber: blocKNumber1.String(), + ParentHash: parentHash.String(), + CID: headerCid1, + } + + // tx variables + tx1CID = "mockTx1CID" + tx2CID = "mockTx2CID" + tx1Hash = crypto.Keccak256Hash([]byte{01, 01}) + tx2Hash = crypto.Keccak256Hash([]byte{01, 02}) + opHash = crypto.Keccak256Hash([]byte{02, 01}) + txModels1 = []btc.TxModelWithInsAndOuts{ + { + Index: 0, + CID: tx1CID, + TxHash: tx1Hash.String(), + SegWit: true, + TxInputs: []btc.TxInput{ + { + Index: 0, + TxWitness: []string{"mockWitness"}, + SignatureScript: []byte{01}, + PreviousOutPointIndex: 0, + PreviousOutPointHash: opHash.String(), + }, + }, + TxOutputs: []btc.TxOutput{ + { + Index: 0, + Value: 50000000, + PkScript: []byte{02}, + ScriptClass: 0, + RequiredSigs: 1, + }, + }, + }, + { + Index: 1, + CID: tx2CID, + TxHash: tx2Hash.String(), + SegWit: true, + }, + } + mockCIDPayload1 = &btc.CIDPayload{ + HeaderCID: headerModel1, + TransactionCIDs: txModels1, + } + + // Block 1 + // header variables + blockHash2 = crypto.Keccak256Hash([]byte{00, 03}) + blocKNumber2 = big.NewInt(1) + headerCid2 = "mockHeaderCID2" + headerModel2 = btc.HeaderModel{ + BlockNumber: blocKNumber2.String(), + BlockHash: blockHash2.String(), + ParentHash: blockHash1.String(), + CID: headerCid2, + } + + // tx variables + tx3CID = "mockTx3CID" + tx3Hash = crypto.Keccak256Hash([]byte{01, 03}) + txModels2 = []btc.TxModelWithInsAndOuts{ + { + Index: 0, + CID: tx3CID, + TxHash: tx3Hash.String(), + SegWit: true, + }, + } + mockCIDPayload2 = &btc.CIDPayload{ + HeaderCID: headerModel2, + TransactionCIDs: txModels2, + } + rngs = [][2]uint64{{0, 1}} + cids = []string{ + headerCid1, + headerCid2, + tx1CID, + tx2CID, + tx3CID, + } + mockData = []byte{'\x01'} +) + +var _ = Describe("Cleaner", func() { + var ( + db *postgres.DB + repo *btc.CIDIndexer + cleaner *btc.Cleaner + ) + BeforeEach(func() { + var err error + db, err = shared.SetupDB() + Expect(err).ToNot(HaveOccurred()) + repo = btc.NewCIDIndexer(db) + cleaner = btc.NewCleaner(db) + }) + + Describe("Clean", func() { + BeforeEach(func() { + err := repo.Index(mockCIDPayload1) + Expect(err).ToNot(HaveOccurred()) + err = repo.Index(mockCIDPayload2) + Expect(err).ToNot(HaveOccurred()) + + for _, cid := range cids { + _, err = db.Exec(`INSERT INTO public.blocks (key, data) VALUES ($1, $2)`, cid, mockData) + Expect(err).ToNot(HaveOccurred()) + } + + tx, err := db.Beginx() + Expect(err).ToNot(HaveOccurred()) + var startingIPFSBlocksCount int + pgStr := `SELECT COUNT(*) FROM public.blocks` + err = tx.Get(&startingIPFSBlocksCount, pgStr) + Expect(err).ToNot(HaveOccurred()) + var startingTxCount int + pgStr = `SELECT COUNT(*) FROM btc.transaction_cids` + err = tx.Get(&startingTxCount, pgStr) + Expect(err).ToNot(HaveOccurred()) + var startingHeaderCount int + pgStr = `SELECT COUNT(*) FROM btc.header_cids` + err = tx.Get(&startingHeaderCount, pgStr) + Expect(err).ToNot(HaveOccurred()) + + err = tx.Commit() + Expect(err).ToNot(HaveOccurred()) + + Expect(startingIPFSBlocksCount).To(Equal(5)) + Expect(startingTxCount).To(Equal(3)) + Expect(startingHeaderCount).To(Equal(2)) + }) + AfterEach(func() { + btc.TearDownDB(db) + }) + It("Cleans everything", func() { + err := cleaner.Clean(rngs, shared.Full) + Expect(err).ToNot(HaveOccurred()) + + tx, err := db.Beginx() + Expect(err).ToNot(HaveOccurred()) + var txCount int + pgStr := `SELECT COUNT(*) FROM btc.transaction_cids` + err = tx.Get(&txCount, pgStr) + Expect(err).ToNot(HaveOccurred()) + var txInCount int + pgStr = `SELECT COUNT(*) FROM btc.tx_inputs` + err = tx.Get(&txInCount, pgStr) + Expect(err).ToNot(HaveOccurred()) + var txOutCount int + pgStr = `SELECT COUNT(*) FROM btc.tx_outputs` + err = tx.Get(&txOutCount, pgStr) + Expect(err).ToNot(HaveOccurred()) + var headerCount int + pgStr = `SELECT COUNT(*) FROM btc.header_cids` + err = tx.Get(&headerCount, pgStr) + Expect(err).ToNot(HaveOccurred()) + var blocksCount int + pgStr = `SELECT COUNT(*) FROM public.blocks` + err = tx.Get(&blocksCount, pgStr) + Expect(err).ToNot(HaveOccurred()) + + err = tx.Commit() + Expect(err).ToNot(HaveOccurred()) + + Expect(blocksCount).To(Equal(0)) + Expect(txCount).To(Equal(0)) + Expect(txInCount).To(Equal(0)) + Expect(txOutCount).To(Equal(0)) + Expect(headerCount).To(Equal(0)) + }) + It("Cleans headers and all linked data", func() { + err := cleaner.Clean(rngs, shared.Headers) + Expect(err).ToNot(HaveOccurred()) + + tx, err := db.Beginx() + Expect(err).ToNot(HaveOccurred()) + var txCount int + pgStr := `SELECT COUNT(*) FROM btc.transaction_cids` + err = tx.Get(&txCount, pgStr) + Expect(err).ToNot(HaveOccurred()) + var txInCount int + pgStr = `SELECT COUNT(*) FROM btc.tx_inputs` + err = tx.Get(&txInCount, pgStr) + Expect(err).ToNot(HaveOccurred()) + var txOutCount int + pgStr = `SELECT COUNT(*) FROM btc.tx_outputs` + err = tx.Get(&txOutCount, pgStr) + Expect(err).ToNot(HaveOccurred()) + var headerCount int + pgStr = `SELECT COUNT(*) FROM btc.header_cids` + err = tx.Get(&headerCount, pgStr) + Expect(err).ToNot(HaveOccurred()) + var blocksCount int + pgStr = `SELECT COUNT(*) FROM public.blocks` + err = tx.Get(&blocksCount, pgStr) + Expect(err).ToNot(HaveOccurred()) + + err = tx.Commit() + Expect(err).ToNot(HaveOccurred()) + + Expect(blocksCount).To(Equal(0)) + Expect(txCount).To(Equal(0)) + Expect(txInCount).To(Equal(0)) + Expect(txOutCount).To(Equal(0)) + Expect(headerCount).To(Equal(0)) + }) + It("Cleans transactions", func() { + err := cleaner.Clean(rngs, shared.Transactions) + Expect(err).ToNot(HaveOccurred()) + + tx, err := db.Beginx() + Expect(err).ToNot(HaveOccurred()) + var txCount int + pgStr := `SELECT COUNT(*) FROM btc.transaction_cids` + err = tx.Get(&txCount, pgStr) + Expect(err).ToNot(HaveOccurred()) + var txInCount int + pgStr = `SELECT COUNT(*) FROM btc.tx_inputs` + err = tx.Get(&txInCount, pgStr) + Expect(err).ToNot(HaveOccurred()) + var txOutCount int + pgStr = `SELECT COUNT(*) FROM btc.tx_outputs` + err = tx.Get(&txOutCount, pgStr) + Expect(err).ToNot(HaveOccurred()) + var headerCount int + pgStr = `SELECT COUNT(*) FROM btc.header_cids` + err = tx.Get(&headerCount, pgStr) + Expect(err).ToNot(HaveOccurred()) + var blocksCount int + pgStr = `SELECT COUNT(*) FROM public.blocks` + err = tx.Get(&blocksCount, pgStr) + Expect(err).ToNot(HaveOccurred()) + + err = tx.Commit() + Expect(err).ToNot(HaveOccurred()) + + Expect(blocksCount).To(Equal(2)) + Expect(txCount).To(Equal(0)) + Expect(txInCount).To(Equal(0)) + Expect(txOutCount).To(Equal(0)) + Expect(headerCount).To(Equal(2)) + }) + }) +}) diff --git a/pkg/super_node/eth/cleaner.go b/pkg/super_node/eth/cleaner.go index 6a56ed7c..18a4e466 100644 --- a/pkg/super_node/eth/cleaner.go +++ b/pkg/super_node/eth/cleaner.go @@ -57,25 +57,13 @@ func (c *Cleaner) Clean(rngs [][2]uint64, t shared.DataType) error { func (c *Cleaner) clean(tx *sqlx.Tx, rng [2]uint64, t shared.DataType) error { switch t { - case shared.Full: + case shared.Full, shared.Headers: return c.cleanFull(tx, rng) - case shared.Headers: - if err := c.cleanStorageIPLDs(tx, rng); err != nil { + case shared.Uncles: + if err := c.cleanUncleIPLDs(tx, rng); err != nil { return err } - if err := c.cleanStateIPLDs(tx, rng); err != nil { - return err - } - if err := c.cleanReceiptIPLDs(tx, rng); err != nil { - return err - } - if err := c.cleanTransactionIPLDs(tx, rng); err != nil { - return err - } - if err := c.cleanHeaderIPLDs(tx, rng); err != nil { - return err - } - return c.cleanHeaderMetaData(tx, rng) + return c.cleanUncleMetaData(tx, rng) case shared.Transactions: if err := c.cleanReceiptIPLDs(tx, rng); err != nil { return err @@ -108,24 +96,25 @@ func (c *Cleaner) clean(tx *sqlx.Tx, rng [2]uint64, t shared.DataType) error { } func (c *Cleaner) cleanFull(tx *sqlx.Tx, rng [2]uint64) error { - // Clear all of the indexed iplds - pgStr := `DELETE FROM public.blocks A - USING eth.storage_cids B, eth.state_cids C, eth.receipt_cids D, eth.transaction_cids E, eth.header_cids F - WHERE (A.key = B.cid OR A.key = C.cid OR A.key = D.cid OR A.key = E.cid OR A.key = F.cid) - AND B.state_id = C.id - AND C.header_id = F.id - AND D.tx_id = E.id - AND E.header_id = F.id - AND F.block_number BETWEEN $1 AND $2` - _, err := tx.Exec(pgStr, rng[0], rng[1]) - if err != nil { + if err := c.cleanStorageIPLDs(tx, rng); err != nil { return err } - // Clear all the header_cids, this will cascade delete the rest of the index metadata - pgStr = `DELETE FROM eth.header_cids - WHERE block_number BETWEEN $1 AND $2` - _, err = tx.Exec(pgStr, rng[0], rng[1]) - return err + if err := c.cleanStateIPLDs(tx, rng); err != nil { + return err + } + if err := c.cleanReceiptIPLDs(tx, rng); err != nil { + return err + } + if err := c.cleanTransactionIPLDs(tx, rng); err != nil { + return err + } + if err := c.cleanUncleIPLDs(tx, rng); err != nil { + return err + } + if err := c.cleanHeaderIPLDs(tx, rng); err != nil { + return err + } + return c.cleanHeaderMetaData(tx, rng) } func (c *Cleaner) cleanStorageIPLDs(tx *sqlx.Tx, rng [2]uint64) error { @@ -153,7 +142,7 @@ func (c *Cleaner) cleanStateIPLDs(tx *sqlx.Tx, rng [2]uint64) error { pgStr := `DELETE FROM public.blocks A USING eth.state_cids B, eth.header_cids C WHERE A.key = B.cid - AND B.header_cid = C.id + AND B.header_id = C.id AND C.block_number BETWEEN $1 AND $2` _, err := tx.Exec(pgStr, rng[0], rng[1]) return err @@ -208,6 +197,25 @@ func (c *Cleaner) cleanTransactionMetaData(tx *sqlx.Tx, rng [2]uint64) error { return err } +func (c *Cleaner) cleanUncleIPLDs(tx *sqlx.Tx, rng [2]uint64) error { + pgStr := `DELETE FROM public.blocks A + USING eth.uncle_cids B, eth.header_cids C + WHERE A.key = B.cid + AND B.header_id = C.id + AND C.block_number BETWEEN $1 AND $2` + _, err := tx.Exec(pgStr, rng[0], rng[1]) + return err +} + +func (c *Cleaner) cleanUncleMetaData(tx *sqlx.Tx, rng [2]uint64) error { + pgStr := `DELETE FROM eth.uncle_cids A + USING eth.header_cids B + WHERE A.header_id = B.id + AND B.block_number BETWEEN $1 AND $2` + _, err := tx.Exec(pgStr, rng[0], rng[1]) + return err +} + func (c *Cleaner) cleanHeaderIPLDs(tx *sqlx.Tx, rng [2]uint64) error { pgStr := `DELETE FROM public.blocks A USING eth.header_cids B diff --git a/pkg/super_node/eth/cleaner_test.go b/pkg/super_node/eth/cleaner_test.go new file mode 100644 index 00000000..351ce77d --- /dev/null +++ b/pkg/super_node/eth/cleaner_test.go @@ -0,0 +1,614 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package eth_test + +import ( + "math/big" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/crypto" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vulcanize/vulcanizedb/pkg/postgres" + "github.com/vulcanize/vulcanizedb/pkg/super_node/eth" + eth2 "github.com/vulcanize/vulcanizedb/pkg/super_node/eth" + "github.com/vulcanize/vulcanizedb/pkg/super_node/shared" +) + +var ( + // Block 0 + // header variables + blockHash1 = crypto.Keccak256Hash([]byte{00, 02}) + blocKNumber1 = big.NewInt(0) + headerCID1 = "mockHeader1CID" + parentHash = crypto.Keccak256Hash([]byte{00, 01}) + totalDifficulty = "50000000000000000000" + reward = "5000000000000000000" + headerModel = eth2.HeaderModel{ + BlockHash: blockHash1.String(), + BlockNumber: blocKNumber1.String(), + CID: headerCID1, + ParentHash: parentHash.String(), + TotalDifficulty: totalDifficulty, + Reward: reward, + } + + // tx variables + tx1CID = "mockTx1CID" + tx2CID = "mockTx2CID" + tx1Hash = crypto.Keccak256Hash([]byte{01, 01}) + tx2Hash = crypto.Keccak256Hash([]byte{01, 02}) + txSrc = common.HexToAddress("0x010a") + txDst = common.HexToAddress("0x020a") + txModels1 = []eth2.TxModel{ + { + CID: tx1CID, + TxHash: tx1Hash.String(), + Index: 0, + }, + { + CID: tx2CID, + TxHash: tx2Hash.String(), + Index: 1, + }, + } + + // uncle variables + uncleCID = "mockUncle1CID" + uncleHash = crypto.Keccak256Hash([]byte{02, 02}) + uncleParentHash = crypto.Keccak256Hash([]byte{02, 01}) + uncleReward = "1000000000000000000" + uncleModels1 = []eth2.UncleModel{ + { + CID: uncleCID, + Reward: uncleReward, + BlockHash: uncleHash.String(), + ParentHash: uncleParentHash.String(), + }, + } + + // receipt variables + rct1CID = "mockRct1CID" + rct2CID = "mockRct2CID" + rct1Contract = common.Address{} + rct2Contract = common.HexToAddress("0x010c") + receiptModels1 = map[common.Hash]eth2.ReceiptModel{ + tx1Hash: { + CID: rct1CID, + Contract: rct1Contract.String(), + }, + tx2Hash: { + CID: rct2CID, + Contract: rct2Contract.String(), + }, + } + + // state variables + state1CID1 = "mockState1CID1" + state1Path = []byte{'\x01'} + state1Key = crypto.Keccak256Hash(txSrc.Bytes()) + state2CID1 = "mockState2CID1" + state2Path = []byte{'\x02'} + state2Key = crypto.Keccak256Hash(txDst.Bytes()) + stateModels1 = []eth2.StateNodeModel{ + { + CID: state1CID1, + Path: state1Path, + NodeType: 2, + StateKey: state1Key.String(), + }, + { + CID: state2CID1, + Path: state2Path, + NodeType: 2, + StateKey: state2Key.String(), + }, + } + + // storage variables + storageCID = "mockStorageCID1" + storagePath = []byte{'\x01'} + storageKey = crypto.Keccak256Hash(common.Hex2Bytes("0x0000000000000000000000000000000000000000000000000000000000000000")) + storageModels1 = map[common.Hash][]eth2.StorageNodeModel{ + crypto.Keccak256Hash(state1Path): { + { + CID: storageCID, + StorageKey: storageKey.String(), + Path: storagePath, + NodeType: 2, + }, + }, + } + mockCIDPayload1 = ð.CIDPayload{ + HeaderCID: headerModel, + UncleCIDs: uncleModels1, + TransactionCIDs: txModels1, + ReceiptCIDs: receiptModels1, + StateNodeCIDs: stateModels1, + StorageNodeCIDs: storageModels1, + } + + // Block 1 + // header variables + blockHash2 = crypto.Keccak256Hash([]byte{00, 03}) + blocKNumber2 = big.NewInt(1) + headerCID2 = "mockHeaderCID2" + headerModel2 = eth2.HeaderModel{ + BlockHash: blockHash2.String(), + BlockNumber: blocKNumber2.String(), + CID: headerCID2, + ParentHash: blockHash1.String(), + TotalDifficulty: totalDifficulty, + Reward: reward, + } + // tx variables + tx3CID = "mockTx3CID" + tx3Hash = crypto.Keccak256Hash([]byte{01, 03}) + txModels2 = []eth2.TxModel{ + { + CID: tx3CID, + TxHash: tx3Hash.String(), + Index: 0, + }, + } + // receipt variables + rct3CID = "mockRct3CID" + receiptModels2 = map[common.Hash]eth2.ReceiptModel{ + tx3Hash: { + CID: rct3CID, + Contract: rct1Contract.String(), + }, + } + + // state variables + state1CID2 = "mockState1CID2" + stateModels2 = []eth2.StateNodeModel{ + { + CID: state1CID2, + Path: state1Path, + NodeType: 2, + StateKey: state1Key.String(), + }, + } + mockCIDPayload2 = ð.CIDPayload{ + HeaderCID: headerModel2, + TransactionCIDs: txModels2, + ReceiptCIDs: receiptModels2, + StateNodeCIDs: stateModels2, + } + rngs = [][2]uint64{{0, 1}} + cids = []string{ + headerCID1, + headerCID2, + uncleCID, + tx1CID, + tx2CID, + tx3CID, + rct1CID, + rct2CID, + rct3CID, + state1CID1, + state2CID1, + state1CID2, + storageCID, + } + mockData = []byte{'\x01'} +) + +var _ = Describe("Cleaner", func() { + var ( + db *postgres.DB + repo *eth2.CIDIndexer + cleaner *eth2.Cleaner + ) + BeforeEach(func() { + var err error + db, err = shared.SetupDB() + Expect(err).ToNot(HaveOccurred()) + repo = eth2.NewCIDIndexer(db) + cleaner = eth2.NewCleaner(db) + }) + Describe("Clean", func() { + BeforeEach(func() { + err := repo.Index(mockCIDPayload1) + Expect(err).ToNot(HaveOccurred()) + err = repo.Index(mockCIDPayload2) + Expect(err).ToNot(HaveOccurred()) + + for _, cid := range cids { + _, err = db.Exec(`INSERT INTO public.blocks (key, data) VALUES ($1, $2)`, cid, mockData) + Expect(err).ToNot(HaveOccurred()) + } + + tx, err := db.Beginx() + Expect(err).ToNot(HaveOccurred()) + + var startingIPFSBlocksCount int + pgStr := `SELECT COUNT(*) FROM public.blocks` + err = tx.Get(&startingIPFSBlocksCount, pgStr) + Expect(err).ToNot(HaveOccurred()) + var startingStorageCount int + pgStr = `SELECT COUNT(*) FROM eth.storage_cids` + err = tx.Get(&startingStorageCount, pgStr) + Expect(err).ToNot(HaveOccurred()) + var startingStateCount int + pgStr = `SELECT COUNT(*) FROM eth.state_cids` + err = tx.Get(&startingStateCount, pgStr) + Expect(err).ToNot(HaveOccurred()) + var startingReceiptCount int + pgStr = `SELECT COUNT(*) FROM eth.receipt_cids` + err = tx.Get(&startingReceiptCount, pgStr) + Expect(err).ToNot(HaveOccurred()) + var startingTxCount int + pgStr = `SELECT COUNT(*) FROM eth.transaction_cids` + err = tx.Get(&startingTxCount, pgStr) + Expect(err).ToNot(HaveOccurred()) + var startingUncleCount int + pgStr = `SELECT COUNT(*) FROM eth.uncle_cids` + err = tx.Get(&startingUncleCount, pgStr) + Expect(err).ToNot(HaveOccurred()) + var startingHeaderCount int + pgStr = `SELECT COUNT(*) FROM eth.header_cids` + err = tx.Get(&startingHeaderCount, pgStr) + Expect(err).ToNot(HaveOccurred()) + + err = tx.Commit() + Expect(err).ToNot(HaveOccurred()) + + Expect(startingIPFSBlocksCount).To(Equal(13)) + Expect(startingStorageCount).To(Equal(1)) + Expect(startingStateCount).To(Equal(3)) + Expect(startingReceiptCount).To(Equal(3)) + Expect(startingTxCount).To(Equal(3)) + Expect(startingUncleCount).To(Equal(1)) + Expect(startingHeaderCount).To(Equal(2)) + }) + AfterEach(func() { + eth.TearDownDB(db) + }) + It("Cleans everything", func() { + err := cleaner.Clean(rngs, shared.Full) + Expect(err).ToNot(HaveOccurred()) + + tx, err := db.Beginx() + Expect(err).ToNot(HaveOccurred()) + + pgStr := `SELECT COUNT(*) FROM eth.header_cids` + var headerCount int + err = tx.Get(&headerCount, pgStr) + Expect(err).ToNot(HaveOccurred()) + var uncleCount int + pgStr = `SELECT COUNT(*) FROM eth.uncle_cids` + err = tx.Get(&uncleCount, pgStr) + Expect(err).ToNot(HaveOccurred()) + var txCount int + pgStr = `SELECT COUNT(*) FROM eth.transaction_cids` + err = tx.Get(&txCount, pgStr) + Expect(err).ToNot(HaveOccurred()) + var rctCount int + pgStr = `SELECT COUNT(*) FROM eth.receipt_cids` + err = tx.Get(&rctCount, pgStr) + Expect(err).ToNot(HaveOccurred()) + var stateCount int + pgStr = `SELECT COUNT(*) FROM eth.state_cids` + err = tx.Get(&stateCount, pgStr) + Expect(err).ToNot(HaveOccurred()) + var storageCount int + pgStr = `SELECT COUNT(*) FROM eth.storage_cids` + err = tx.Get(&storageCount, pgStr) + Expect(err).ToNot(HaveOccurred()) + var blocksCount int + pgStr = `SELECT COUNT(*) FROM public.blocks` + err = tx.Get(&blocksCount, pgStr) + Expect(err).ToNot(HaveOccurred()) + + err = tx.Commit() + Expect(err).ToNot(HaveOccurred()) + + Expect(headerCount).To(Equal(0)) + Expect(uncleCount).To(Equal(0)) + Expect(txCount).To(Equal(0)) + Expect(rctCount).To(Equal(0)) + Expect(stateCount).To(Equal(0)) + Expect(storageCount).To(Equal(0)) + Expect(blocksCount).To(Equal(0)) + }) + It("Cleans headers and all linked data (same as full)", func() { + err := cleaner.Clean(rngs, shared.Headers) + Expect(err).ToNot(HaveOccurred()) + + tx, err := db.Beginx() + Expect(err).ToNot(HaveOccurred()) + + var headerCount int + pgStr := `SELECT COUNT(*) FROM eth.header_cids` + err = tx.Get(&headerCount, pgStr) + Expect(err).ToNot(HaveOccurred()) + var uncleCount int + pgStr = `SELECT COUNT(*) FROM eth.uncle_cids` + err = tx.Get(&uncleCount, pgStr) + Expect(err).ToNot(HaveOccurred()) + var txCount int + pgStr = `SELECT COUNT(*) FROM eth.transaction_cids` + err = tx.Get(&txCount, pgStr) + Expect(err).ToNot(HaveOccurred()) + var rctCount int + pgStr = `SELECT COUNT(*) FROM eth.receipt_cids` + err = tx.Get(&rctCount, pgStr) + Expect(err).ToNot(HaveOccurred()) + var stateCount int + pgStr = `SELECT COUNT(*) FROM eth.state_cids` + err = tx.Get(&stateCount, pgStr) + Expect(err).ToNot(HaveOccurred()) + var storageCount int + pgStr = `SELECT COUNT(*) FROM eth.storage_cids` + err = tx.Get(&storageCount, pgStr) + Expect(err).ToNot(HaveOccurred()) + var blocksCount int + pgStr = `SELECT COUNT(*) FROM public.blocks` + err = tx.Get(&blocksCount, pgStr) + Expect(err).ToNot(HaveOccurred()) + + err = tx.Commit() + Expect(err).ToNot(HaveOccurred()) + + Expect(headerCount).To(Equal(0)) + Expect(uncleCount).To(Equal(0)) + Expect(txCount).To(Equal(0)) + Expect(rctCount).To(Equal(0)) + Expect(stateCount).To(Equal(0)) + Expect(storageCount).To(Equal(0)) + Expect(blocksCount).To(Equal(0)) + }) + It("Cleans uncles", func() { + err := cleaner.Clean(rngs, shared.Uncles) + Expect(err).ToNot(HaveOccurred()) + + tx, err := db.Beginx() + Expect(err).ToNot(HaveOccurred()) + + var headerCount int + pgStr := `SELECT COUNT(*) FROM eth.header_cids` + err = tx.Get(&headerCount, pgStr) + Expect(err).ToNot(HaveOccurred()) + var uncleCount int + pgStr = `SELECT COUNT(*) FROM eth.uncle_cids` + err = tx.Get(&uncleCount, pgStr) + Expect(err).ToNot(HaveOccurred()) + var txCount int + pgStr = `SELECT COUNT(*) FROM eth.transaction_cids` + err = tx.Get(&txCount, pgStr) + Expect(err).ToNot(HaveOccurred()) + var rctCount int + pgStr = `SELECT COUNT(*) FROM eth.receipt_cids` + err = tx.Get(&rctCount, pgStr) + Expect(err).ToNot(HaveOccurred()) + var stateCount int + pgStr = `SELECT COUNT(*) FROM eth.state_cids` + err = tx.Get(&stateCount, pgStr) + Expect(err).ToNot(HaveOccurred()) + var storageCount int + pgStr = `SELECT COUNT(*) FROM eth.storage_cids` + err = tx.Get(&storageCount, pgStr) + Expect(err).ToNot(HaveOccurred()) + var blocksCount int + pgStr = `SELECT COUNT(*) FROM public.blocks` + err = tx.Get(&blocksCount, pgStr) + Expect(err).ToNot(HaveOccurred()) + + err = tx.Commit() + Expect(err).ToNot(HaveOccurred()) + + Expect(headerCount).To(Equal(2)) + Expect(uncleCount).To(Equal(0)) + Expect(txCount).To(Equal(3)) + Expect(rctCount).To(Equal(3)) + Expect(stateCount).To(Equal(3)) + Expect(storageCount).To(Equal(1)) + Expect(blocksCount).To(Equal(12)) + }) + It("Cleans transactions and linked receipts", func() { + err := cleaner.Clean(rngs, shared.Transactions) + Expect(err).ToNot(HaveOccurred()) + + tx, err := db.Beginx() + Expect(err).ToNot(HaveOccurred()) + + var headerCount int + pgStr := `SELECT COUNT(*) FROM eth.header_cids` + err = tx.Get(&headerCount, pgStr) + Expect(err).ToNot(HaveOccurred()) + var uncleCount int + pgStr = `SELECT COUNT(*) FROM eth.uncle_cids` + err = tx.Get(&uncleCount, pgStr) + Expect(err).ToNot(HaveOccurred()) + var txCount int + pgStr = `SELECT COUNT(*) FROM eth.transaction_cids` + err = tx.Get(&txCount, pgStr) + Expect(err).ToNot(HaveOccurred()) + var rctCount int + pgStr = `SELECT COUNT(*) FROM eth.receipt_cids` + err = tx.Get(&rctCount, pgStr) + Expect(err).ToNot(HaveOccurred()) + var stateCount int + pgStr = `SELECT COUNT(*) FROM eth.state_cids` + err = tx.Get(&stateCount, pgStr) + Expect(err).ToNot(HaveOccurred()) + var storageCount int + pgStr = `SELECT COUNT(*) FROM eth.storage_cids` + err = tx.Get(&storageCount, pgStr) + Expect(err).ToNot(HaveOccurred()) + var blocksCount int + pgStr = `SELECT COUNT(*) FROM public.blocks` + err = tx.Get(&blocksCount, pgStr) + Expect(err).ToNot(HaveOccurred()) + + err = tx.Commit() + Expect(err).ToNot(HaveOccurred()) + + Expect(headerCount).To(Equal(2)) + Expect(uncleCount).To(Equal(1)) + Expect(txCount).To(Equal(0)) + Expect(rctCount).To(Equal(0)) + Expect(stateCount).To(Equal(3)) + Expect(storageCount).To(Equal(1)) + Expect(blocksCount).To(Equal(7)) + }) + It("Cleans receipts", func() { + err := cleaner.Clean(rngs, shared.Receipts) + Expect(err).ToNot(HaveOccurred()) + + tx, err := db.Beginx() + Expect(err).ToNot(HaveOccurred()) + + var headerCount int + pgStr := `SELECT COUNT(*) FROM eth.header_cids` + err = tx.Get(&headerCount, pgStr) + Expect(err).ToNot(HaveOccurred()) + var uncleCount int + pgStr = `SELECT COUNT(*) FROM eth.uncle_cids` + err = tx.Get(&uncleCount, pgStr) + Expect(err).ToNot(HaveOccurred()) + var txCount int + pgStr = `SELECT COUNT(*) FROM eth.transaction_cids` + err = tx.Get(&txCount, pgStr) + Expect(err).ToNot(HaveOccurred()) + var rctCount int + pgStr = `SELECT COUNT(*) FROM eth.receipt_cids` + err = tx.Get(&rctCount, pgStr) + Expect(err).ToNot(HaveOccurred()) + var stateCount int + pgStr = `SELECT COUNT(*) FROM eth.state_cids` + err = tx.Get(&stateCount, pgStr) + Expect(err).ToNot(HaveOccurred()) + var storageCount int + pgStr = `SELECT COUNT(*) FROM eth.storage_cids` + err = tx.Get(&storageCount, pgStr) + Expect(err).ToNot(HaveOccurred()) + var blocksCount int + pgStr = `SELECT COUNT(*) FROM public.blocks` + err = tx.Get(&blocksCount, pgStr) + Expect(err).ToNot(HaveOccurred()) + + err = tx.Commit() + Expect(err).ToNot(HaveOccurred()) + + Expect(headerCount).To(Equal(2)) + Expect(uncleCount).To(Equal(1)) + Expect(txCount).To(Equal(3)) + Expect(rctCount).To(Equal(0)) + Expect(stateCount).To(Equal(3)) + Expect(storageCount).To(Equal(1)) + Expect(blocksCount).To(Equal(10)) + }) + It("Cleans state and linked storage", func() { + err := cleaner.Clean(rngs, shared.State) + Expect(err).ToNot(HaveOccurred()) + + tx, err := db.Beginx() + Expect(err).ToNot(HaveOccurred()) + + var headerCount int + pgStr := `SELECT COUNT(*) FROM eth.header_cids` + err = tx.Get(&headerCount, pgStr) + Expect(err).ToNot(HaveOccurred()) + var uncleCount int + pgStr = `SELECT COUNT(*) FROM eth.uncle_cids` + err = tx.Get(&uncleCount, pgStr) + Expect(err).ToNot(HaveOccurred()) + var txCount int + pgStr = `SELECT COUNT(*) FROM eth.transaction_cids` + err = tx.Get(&txCount, pgStr) + Expect(err).ToNot(HaveOccurred()) + var rctCount int + pgStr = `SELECT COUNT(*) FROM eth.receipt_cids` + err = tx.Get(&rctCount, pgStr) + Expect(err).ToNot(HaveOccurred()) + var stateCount int + pgStr = `SELECT COUNT(*) FROM eth.state_cids` + err = tx.Get(&stateCount, pgStr) + Expect(err).ToNot(HaveOccurred()) + var storageCount int + pgStr = `SELECT COUNT(*) FROM eth.storage_cids` + err = tx.Get(&storageCount, pgStr) + Expect(err).ToNot(HaveOccurred()) + var blocksCount int + pgStr = `SELECT COUNT(*) FROM public.blocks` + err = tx.Get(&blocksCount, pgStr) + Expect(err).ToNot(HaveOccurred()) + + err = tx.Commit() + Expect(err).ToNot(HaveOccurred()) + + Expect(headerCount).To(Equal(2)) + Expect(uncleCount).To(Equal(1)) + Expect(txCount).To(Equal(3)) + Expect(rctCount).To(Equal(3)) + Expect(stateCount).To(Equal(0)) + Expect(storageCount).To(Equal(0)) + Expect(blocksCount).To(Equal(9)) + }) + It("Cleans storage", func() { + err := cleaner.Clean(rngs, shared.Storage) + Expect(err).ToNot(HaveOccurred()) + + tx, err := db.Beginx() + Expect(err).ToNot(HaveOccurred()) + + var headerCount int + pgStr := `SELECT COUNT(*) FROM eth.header_cids` + err = tx.Get(&headerCount, pgStr) + Expect(err).ToNot(HaveOccurred()) + var uncleCount int + pgStr = `SELECT COUNT(*) FROM eth.uncle_cids` + err = tx.Get(&uncleCount, pgStr) + Expect(err).ToNot(HaveOccurred()) + var txCount int + pgStr = `SELECT COUNT(*) FROM eth.transaction_cids` + err = tx.Get(&txCount, pgStr) + Expect(err).ToNot(HaveOccurred()) + var rctCount int + pgStr = `SELECT COUNT(*) FROM eth.receipt_cids` + err = tx.Get(&rctCount, pgStr) + Expect(err).ToNot(HaveOccurred()) + var stateCount int + pgStr = `SELECT COUNT(*) FROM eth.state_cids` + err = tx.Get(&stateCount, pgStr) + Expect(err).ToNot(HaveOccurred()) + var storageCount int + pgStr = `SELECT COUNT(*) FROM eth.storage_cids` + err = tx.Get(&storageCount, pgStr) + Expect(err).ToNot(HaveOccurred()) + var blocksCount int + pgStr = `SELECT COUNT(*) FROM public.blocks` + err = tx.Get(&blocksCount, pgStr) + Expect(err).ToNot(HaveOccurred()) + + err = tx.Commit() + Expect(err).ToNot(HaveOccurred()) + + Expect(headerCount).To(Equal(2)) + Expect(uncleCount).To(Equal(1)) + Expect(txCount).To(Equal(3)) + Expect(rctCount).To(Equal(3)) + Expect(stateCount).To(Equal(3)) + Expect(storageCount).To(Equal(0)) + Expect(blocksCount).To(Equal(12)) + }) + }) +}) diff --git a/pkg/super_node/shared/data_type.go b/pkg/super_node/shared/data_type.go index 0c219c6a..8112988a 100644 --- a/pkg/super_node/shared/data_type.go +++ b/pkg/super_node/shared/data_type.go @@ -28,6 +28,7 @@ const ( UnknownDataType DataType = iota - 1 Full Headers + Uncles Transactions Receipts State @@ -41,6 +42,8 @@ func (r DataType) String() string { return "full" case Headers: return "headers" + case Uncles: + return "uncles" case Transactions: return "transactions" case Receipts: @@ -61,6 +64,8 @@ func GenerateResyncTypeFromString(str string) (DataType, error) { return Full, nil case "headers", "header", "h": return Headers, nil + case "uncles", "u": + return Uncles, nil case "transactions", "transaction", "trxs", "txs", "trx", "tx", "t": return Transactions, nil case "receipts", "receipt", "rcts", "rct", "r": @@ -74,21 +79,66 @@ func GenerateResyncTypeFromString(str string) (DataType, error) { } } -func SupportedResyncType(d DataType) bool { - switch d { - case Full: - return true - case Headers: - return false - case Transactions: - return false - case Receipts: - return false - case State: - return false - case Storage: - return false +func SupportedResyncType(d DataType, c ChainType) (bool, error) { + switch c { + case Ethereum: + switch d { + case Full: + return true, nil + case Headers: + return true, nil + case Uncles: + return true, nil + case Transactions: + return true, nil + case Receipts: + return true, nil + case State: + return true, nil + case Storage: + return true, nil + default: + return true, nil + } + case Bitcoin: + switch d { + case Full: + return true, nil + case Headers: + return true, nil + case Uncles: + return false, nil + case Transactions: + return true, nil + case Receipts: + return false, nil + case State: + return false, nil + case Storage: + return false, nil + default: + return false, nil + } + case Omni: + switch d { + case Full: + return false, nil + case Headers: + return false, nil + case Uncles: + return false, nil + case Transactions: + return false, nil + case Receipts: + return false, nil + case State: + return false, nil + case Storage: + return false, nil + default: + return false, nil + } default: - return false + return false, fmt.Errorf("unrecognized chain type %s", c.String()) } }