cerc-io/ipld-eth-server
12
0
Fork 1
Code Issues 10 Pull Requests 2 Actions Packages Projects Releases 37 Wiki Activity

update dependencies to work with update eth-block-extractor

Browse Source
This commit is contained in:
Ian Norden
2019-12-02 13:24:49 -06:00
parent 31a9017c4f
commit d702cb720c
3000 changed files with 433987 additions and 182448 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/ipfs/payload_converter.go → pkg/ipfs/converter.go
+12 -13
View File
@@ -40,12 +40,12 @@ type PayloadConverter struct {
// IPLDPayload is a custom type which packages ETH data for the IPFS publisher
type IPLDPayload struct {
HeaderRLP []byte
BlockNumber *big.Int
BlockHash common.Hash
BlockBody *types.Body
Receipts types.Receipts
StateLeafs map[common.Hash][]byte
HeaderRLP []byte
BlockNumber *big.Int
BlockHash common.Hash
BlockBody *types.Body
Receipts types.Receipts
StateLeafs map[common.Hash][]byte
StorageLeafs map[common.Hash]map[common.Hash][]byte
}
@@ -67,12 +67,12 @@ func (pc *PayloadConverter) Convert(payload statediff.Payload) (*IPLDPayload, er
return nil, err
}
convertedPayload := &IPLDPayload{
BlockHash: block.Hash(),
BlockNumber: block.Number(),
HeaderRLP: headerRlp,
BlockBody: block.Body(),
Receipts: make(types.Receipts, 0),
StateLeafs: make(map[common.Hash][]byte),
BlockHash: block.Hash(),
BlockNumber: block.Number(),
HeaderRLP: headerRlp,
BlockBody: block.Body(),
Receipts: make(types.Receipts, 0),
StateLeafs: make(map[common.Hash][]byte),
StorageLeafs: make(map[common.Hash]map[common.Hash][]byte),
}
for _, trx := range block.Transactions() {
@@ -112,4 +112,3 @@ func (pc *PayloadConverter) Convert(payload statediff.Payload) (*IPLDPayload, er
}
return convertedPayload, nil
}
pkg/ipfs/converter_test.go
+17
View File
@@ -0,0 +1,17 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package ipfs_test
pkg/ipfs/indexer.go
+16 -12
View File
@@ -19,8 +19,8 @@ package ipfs
import (
"sync"
log "github.com/sirupsen/logrus"
"github.com/ethereum/go-ethereum/statediff"
log "github.com/sirupsen/logrus"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
@@ -31,16 +31,16 @@ const payloadChanBufferSize = 800 // 1/10th max eth sub buffer size
// Indexer is an interface for streaming, converting to IPLDs, publishing, and indexing all Ethereum data
// This is the top-level interface used by the syncAndPublish command
type Indexer interface {
Index(wg sync.WaitGroup) error
Index(wg *sync.WaitGroup) error
}
// ipfsIndexer is the underlying struct for the Indexer interface
// we are not exporting this to enforce proper initialization through the NewIPFSIndexer function
type ipfsIndexer struct {
Streamer Streamer
Converter Converter
Publisher Publisher
Repository Repository
Streamer Streamer
Converter Converter
Publisher Publisher
Repository Repository
PayloadChan chan statediff.Payload
QuitChan chan bool
}
@@ -52,17 +52,17 @@ func NewIPFSIndexer(ipfsPath string, db *postgres.DB, ethClient core.EthClient,
return nil, err
}
return &ipfsIndexer{
Streamer: NewStateDiffSyncer(rpcClient),
Repository: NewCIDRepository(db),
Converter: NewPayloadConverter(ethClient),
Publisher: publisher,
Streamer: NewStateDiffSyncer(rpcClient),
Repository: NewCIDRepository(db),
Converter: NewPayloadConverter(ethClient),
Publisher: publisher,
PayloadChan: make(chan statediff.Payload, payloadChanBufferSize),
QuitChan: qc,
QuitChan: qc,
}, nil
}
// Index is the main processing loop
func (i *ipfsIndexer) Index(wg sync.WaitGroup) error {
func (i *ipfsIndexer) Index(wg *sync.WaitGroup) error {
sub, err := i.Streamer.Stream(i.PayloadChan)
if err != nil {
return err
@@ -73,6 +73,10 @@ func (i *ipfsIndexer) Index(wg sync.WaitGroup) error {
for {
select {
case payload := <-i.PayloadChan:
if payload.Err != nil {
log.Error(err)
continue
}
ipldPayload, err := i.Converter.Convert(payload)
if err != nil {
log.Error(err)
pkg/ipfs/indexer_test.go
+17
View File
@@ -0,0 +1,17 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package ipfs_test
pkg/ipfs/publisher.go
+22 -22
View File
@@ -19,8 +19,8 @@ package ipfs
import (
"errors"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/vulcanize/eth-block-extractor/pkg/ipfs"
"github.com/vulcanize/eth-block-extractor/pkg/ipfs/eth_block_header"
@@ -38,22 +38,22 @@ type Publisher interface {
// IPLDPublisher is the underlying struct for the Publisher interface
type IPLDPublisher struct {
Node *ipfs.IPFS
HeaderPutter *eth_block_header.BlockHeaderDagPutter
Node *ipfs.IPFS
HeaderPutter *eth_block_header.BlockHeaderDagPutter
TransactionPutter *eth_block_transactions.BlockTransactionsDagPutter
ReceiptPutter *eth_block_receipts.EthBlockReceiptDagPutter
StatePutter *eth_state_trie.StateTrieDagPutter
StoragePutter *eth_storage_trie.StorageTrieDagPutter
ReceiptPutter *eth_block_receipts.EthBlockReceiptDagPutter
StatePutter *eth_state_trie.StateTrieDagPutter
StoragePutter *eth_storage_trie.StorageTrieDagPutter
}
// CID payload is a struct to hold all the CIDs and their meta data
type CIDPayload struct {
BlockNumber string
BlockHash string
HeaderCID string
BlockNumber string
BlockHash string
HeaderCID string
TransactionCIDs map[common.Hash]string
ReceiptCIDs map[common.Hash]string
StateLeafCIDs map[common.Hash]string
ReceiptCIDs map[common.Hash]string
StateLeafCIDs map[common.Hash]string
StorageLeafCIDs map[common.Hash]map[common.Hash]string
}
@@ -65,12 +65,12 @@ func NewIPLDPublisher(ipfsPath string) (*IPLDPublisher, error) {
}
decoder := rlp.RlpDecoder{}
return &IPLDPublisher{
Node: node,
HeaderPutter: eth_block_header.NewBlockHeaderDagPutter(node, decoder),
Node: node,
HeaderPutter: eth_block_header.NewBlockHeaderDagPutter(node, decoder),
TransactionPutter: eth_block_transactions.NewBlockTransactionsDagPutter(node),
ReceiptPutter: eth_block_receipts.NewEthBlockReceiptDagPutter(node),
StatePutter: eth_state_trie.NewStateTrieDagPutter(node),
StoragePutter: eth_storage_trie.NewStorageTrieDagPutter(node),
ReceiptPutter: eth_block_receipts.NewEthBlockReceiptDagPutter(node),
StatePutter: eth_state_trie.NewStateTrieDagPutter(node),
StoragePutter: eth_storage_trie.NewStorageTrieDagPutter(node),
}, nil
}
@@ -108,12 +108,12 @@ func (pub *IPLDPublisher) Publish(payload *IPLDPayload) (*CIDPayload, error) {
// Package CIDs into a single struct
return &CIDPayload{
BlockHash: payload.BlockHash.Hex(),
BlockNumber: payload.BlockNumber.String(),
HeaderCID: headerCid,
BlockHash: payload.BlockHash.Hex(),
BlockNumber: payload.BlockNumber.String(),
HeaderCID: headerCid,
TransactionCIDs: transactionCids,
ReceiptCIDs: receiptsCids,
StateLeafCIDs: stateLeafCids,
ReceiptCIDs: receiptsCids,
StateLeafCIDs: stateLeafCids,
StorageLeafCIDs: storageLeafCids,
}, nil
}
@@ -190,4 +190,4 @@ func (pub *IPLDPublisher) publishStorageLeafs(storageLeafs map[common.Hash]map[c
}
}
return storageLeafCids, nil
}
}
pkg/ipfs/publisher_test.go
+17
View File
@@ -0,0 +1,17 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package ipfs_test
pkg/ipfs/repository.go
+4 -5
View File
@@ -60,13 +60,12 @@ func (repo *CIDRepository) Index(cidPayload *CIDPayload) error {
return tx.Commit()
}
func (repo *CIDRepository) indexHeaderCID(tx *sqlx.Tx, cid, blockNumber, hash string) (int64, error) {
var headerID int64
err := tx.QueryRowx(`INSERT INTO public.header_cids (block_number, block_hash, cid) VALUES ($1, $2, $3)
ON CONFLICT DO UPDATE SET cid = $3
RETURNING id`,
blockNumber, hash, cid).Scan(&headerID)
blockNumber, hash, cid).Scan(&headerID)
return headerID, err
}
@@ -76,7 +75,7 @@ func (repo *CIDRepository) indexTransactionAndReceiptCIDs(tx *sqlx.Tx, payload *
err := tx.QueryRowx(`INSERT INTO public.transaction_cids (header_id, tx_hash, cid) VALUES ($1, $2, $3)
ON CONFLICT DO UPDATE SET cid = $3
RETURNING id`,
headerID, hash.Hex(), trxCid).Scan(&txID)
headerID, hash.Hex(), trxCid).Scan(&txID)
if err != nil {
return err
}
@@ -103,7 +102,7 @@ func (repo *CIDRepository) indexStateAndStorageCIDs(tx *sqlx.Tx, payload *CIDPay
err := tx.QueryRowx(`INSERT INTO public.state_cids (header_id, account_key, cid) VALUES ($1, $2, $3)
ON CONFLICT DO UPDATE SET cid = $3
RETURNING id`,
headerID, accountKey.Hex(), stateCID).Scan(&stateID)
headerID, accountKey.Hex(), stateCID).Scan(&stateID)
if err != nil {
return err
}
@@ -121,4 +120,4 @@ func (repo *CIDRepository) indexStorageCID(tx *sqlx.Tx, key, cid string, stateId
_, err := repo.db.Exec(`INSERT INTO public.storage_cids (state_id, storage_key, cid) VALUES ($1, $2, $3)
ON CONFLICT DO UPDATE SET cid = $3`, stateId, key, cid)
return err
}
}
pkg/ipfs/repository_test.go
+17
View File
@@ -0,0 +1,17 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package ipfs_test
pkg/ipfs/streamer.go
+1 -1
View File
@@ -44,4 +44,4 @@ func NewStateDiffSyncer(client core.RpcClient) *StateDiffStreamer {
// Stream is the main loop for subscribing to data from the Geth state diff process
func (i *StateDiffStreamer) Stream(payloadChan chan statediff.Payload) (*rpc.ClientSubscription, error) {
return i.Client.Subscribe("statediff", i.PayloadChan)
}
}
pkg/ipfs/streamer_test.go
+17
View File
@@ -0,0 +1,17 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package ipfs_test
pkg/ipfs/test_helpers/fakes.go
+17
View File
@@ -0,0 +1,17 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package test_helpers
pkg/ipfs/test_helpers/mocks/converter.go
+17
View File
@@ -0,0 +1,17 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package mocks
pkg/ipfs/test_helpers/mocks/publisher.go
+17
View File
@@ -0,0 +1,17 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package mocks
pkg/ipfs/test_helpers/mocks/repository.go
+17
View File
@@ -0,0 +1,17 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package mocks
pkg/ipfs/test_helpers/mocks/streamer.go
+17
View File
@@ -0,0 +1,17 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package mocks
pkg/ipfs/test_helpers/test_helpers.go
+17
View File
@@ -0,0 +1,17 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package test_helpers