deep-stack/ipld-eth-server
4
0
Fork 0
forked from cerc-io/ipld-eth-server
Code Pull Requests Activity

fixes for issues uncovered in integration

Browse Source
This commit is contained in:
Ian Norden
2019-12-02 13:24:51 -06:00
parent b83c0371d9
commit 34393ffb3f
290 changed files with 108690 additions and 893 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/core/rpc_client.go
+1 -1
View File
@@ -29,5 +29,5 @@ type RPCClient interface {
BatchCall(batch []client.BatchElem) error
IpcPath() string
SupportedModules() (map[string]string, error)
Subscribe(namespace string, payloadChan interface{}, args ...interface{}) (*rpc.ClientSubscription, error)
Subscribe(namespace string, payloadChan interface{}, subName string, args ...interface{}) (*rpc.ClientSubscription, error)
}
pkg/fakes/mock_rpc_client.go
+1 -1
View File
@@ -188,6 +188,6 @@ func (client *MockRPCClient) AssertBatchCalledWith(method string, lengthOfBatch
Expect(client.passedMethod).To(Equal(method))
}
func (client *MockRpcClient) Subscribe(namespace string, payloadChan interface{}, args ...interface{}) (*rpc.ClientSubscription, error) {
func (client *MockRpcClient) Subscribe(namespace string, payloadChan interface{}, subName string, args ...interface{}) (*rpc.ClientSubscription, error) {
panic("implement me")
}
pkg/ipfs/api.go
+1 -1
View File
@@ -42,7 +42,7 @@ func NewPublicSeedNodeAPI(snp SyncPublishAndServe) *PublicSeedNodeAPI {
}
// Subscribe is the public method to setup a subscription that fires off state-diff payloads as they are created
func (api *PublicSeedNodeAPI) Subscribe(ctx context.Context, params *Params) (*rpc.Subscription, error) {
func (api *PublicSeedNodeAPI) Subscribe(ctx context.Context, payloadChan chan ResponsePayload, params *Params) (*rpc.Subscription, error) {
// ensure that the RPC connection supports subscriptions
notifier, supported := rpc.NotifierFromContext(ctx)
if !supported {
pkg/ipfs/converter.go
+8 -1
View File
@@ -73,7 +73,7 @@ func (pc *Converter) Convert(payload statediff.Payload) (*IPLDPayload, error) {
return nil, err
}
txMeta := &TrxMetaData{
To: trx.To().Hex(),
To: handleNullAddr(trx.To()),
From: from.Hex(),
}
// txMeta will have same index as its corresponding trx in the convertedPayload.BlockBody
@@ -152,3 +152,10 @@ func (pc *Converter) Convert(payload statediff.Payload) (*IPLDPayload, error) {
}
return convertedPayload, nil
}
func handleNullAddr(to *common.Address) string {
if to == nil {
return "0x0000000000000000000000000000000000000000000000000000000000000000"
}
return to.Hex()
}
pkg/ipfs/publisher.go
+17 -4
View File
@@ -22,6 +22,7 @@ import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
rlp2 "github.com/ethereum/go-ethereum/rlp"
"github.com/ipfs/go-ipfs/plugin/loader"
"github.com/vulcanize/eth-block-extractor/pkg/ipfs"
"github.com/vulcanize/eth-block-extractor/pkg/ipfs/eth_block_header"
@@ -49,6 +50,18 @@ type Publisher struct {
// NewIPLDPublisher creates a pointer to a new Publisher which satisfies the IPLDPublisher interface
func NewIPLDPublisher(ipfsPath string) (*Publisher, error) {
l, err := loader.NewPluginLoader("~/.ipfs/plugins")
if err != nil {
return nil, err
}
err = l.Initialize()
if err != nil {
return nil, err
}
err = l.Inject()
if err != nil {
return nil, err
}
node, err := ipfs.InitIPFSNode(ipfsPath)
if err != nil {
return nil, err
@@ -98,13 +111,13 @@ func (pub *Publisher) Publish(payload *IPLDPayload) (*CIDPayload, error) {
}
// Process and publish state leafs
stateLeafCids, err := pub.publishStateNodes(payload.StateNodes)
stateNodeCids, err := pub.publishStateNodes(payload.StateNodes)
if err != nil {
return nil, err
}
// Process and publish storage leafs
storageLeafCids, err := pub.publishStorageNodes(payload.StorageNodes)
storageNodeCids, err := pub.publishStorageNodes(payload.StorageNodes)
if err != nil {
return nil, err
}
@@ -117,8 +130,8 @@ func (pub *Publisher) Publish(payload *IPLDPayload) (*CIDPayload, error) {
UncleCIDS: uncleCids,
TransactionCIDs: transactionCids,
ReceiptCIDs: receiptsCids,
StateNodeCIDs: stateLeafCids,
StorageNodeCIDs: storageLeafCids,
StateNodeCIDs: stateNodeCids,
StorageNodeCIDs: storageNodeCids,
}, nil
}
pkg/ipfs/repository.go
+27 -17
View File
@@ -14,6 +14,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
// Still seeing some errors from tx and storage indexing processes... due to fk constraints being broken
package ipfs
import (
@@ -50,24 +51,27 @@ func (repo *Repository) Index(cidPayload *CIDPayload) error {
}
for uncleHash, cid := range cidPayload.UncleCIDS {
err = repo.indexUncleCID(tx, cid, cidPayload.BlockNumber, uncleHash.Hex())
if err != nil {
tx.Rollback()
return err
}
}
err = repo.indexTransactionAndReceiptCIDs(tx, cidPayload, headerID)
tx.Commit()
err = repo.indexTransactionAndReceiptCIDs(cidPayload, headerID)
if err != nil {
tx.Rollback()
return err
}
err = repo.indexStateAndStorageCIDs(tx, cidPayload, headerID)
err = repo.indexStateAndStorageCIDs(cidPayload, headerID)
if err != nil {
tx.Rollback()
return err
}
return tx.Commit()
return nil
}
func (repo *Repository) indexHeaderCID(tx *sqlx.Tx, cid, blockNumber, hash string) (int64, error) {
var headerID int64
err := tx.QueryRowx(`INSERT INTO public.header_cids (block_number, block_hash, cid, uncle) VALUES ($1, $2, $3, $4)
ON CONFLICT DO UPDATE SET (cid, uncle) = ($3, $4)
ON CONFLICT (block_number, block_hash) DO UPDATE SET (cid, uncle) = ($3, $4)
RETURNING id`,
blockNumber, hash, cid, false).Scan(&headerID)
return headerID, err
@@ -75,61 +79,67 @@ func (repo *Repository) indexHeaderCID(tx *sqlx.Tx, cid, blockNumber, hash strin
func (repo *Repository) indexUncleCID(tx *sqlx.Tx, cid, blockNumber, hash string) error {
_, err := tx.Queryx(`INSERT INTO public.header_cids (block_number, block_hash, cid, uncle) VALUES ($1, $2, $3, $4)
ON CONFLICT DO UPDATE SET (cid, uncle) = ($3, $4)`,
ON CONFLICT (block_number, block_hash) DO UPDATE SET (cid, uncle) = ($3, $4)`,
blockNumber, hash, cid, true)
return err
}
func (repo *Repository) indexTransactionAndReceiptCIDs(tx *sqlx.Tx, payload *CIDPayload, headerID int64) error {
func (repo *Repository) indexTransactionAndReceiptCIDs(payload *CIDPayload, headerID int64) error {
tx, _ := repo.db.Beginx()
for hash, trxCidMeta := range payload.TransactionCIDs {
var txID int64
err := tx.QueryRowx(`INSERT INTO public.transaction_cids (header_id, tx_hash, cid, dst, src) VALUES ($1, $2, $3, $4, $5)
ON CONFLICT DO UPDATE SET (cid, dst, src) = ($3, $4, $5)
ON CONFLICT (header_id, tx_hash) DO UPDATE SET (cid, dst, src) = ($3, $4, $5)
RETURNING id`,
headerID, hash.Hex(), trxCidMeta.CID, trxCidMeta.To, trxCidMeta.From).Scan(&txID)
if err != nil {
tx.Rollback()
return err
}
receiptCidMeta, ok := payload.ReceiptCIDs[hash]
if ok {
err = repo.indexReceiptCID(tx, receiptCidMeta, txID)
if err != nil {
tx.Rollback()
return err
}
}
}
return nil
return tx.Commit()
}
func (repo *Repository) indexReceiptCID(tx *sqlx.Tx, cidMeta *ReceiptMetaData, txID int64) error {
_, err := tx.Exec(`INSERT INTO public.receipt_cids (tx_id, cid, topic0s) VALUES ($1, $2, $3)
ON CONFLICT DO UPDATE SET (cid, topic0s) = ($2, $3)`, txID, cidMeta.CID, pq.Array(cidMeta.Topic0s))
_, err := tx.Exec(`INSERT INTO public.receipt_cids (tx_id, cid, topic0s) VALUES ($1, $2, $3)`,
txID, cidMeta.CID, pq.Array(cidMeta.Topic0s))
return err
}
func (repo *Repository) indexStateAndStorageCIDs(tx *sqlx.Tx, payload *CIDPayload, headerID int64) error {
func (repo *Repository) indexStateAndStorageCIDs(payload *CIDPayload, headerID int64) error {
tx, _ := repo.db.Beginx()
for accountKey, stateCID := range payload.StateNodeCIDs {
var stateID int64
err := tx.QueryRowx(`INSERT INTO public.state_cids (header_id, state_key, cid, leaf) VALUES ($1, $2, $3, $4)
ON CONFLICT DO UPDATE SET (cid, leaf) = ($3, $4)
ON CONFLICT (header_id, state_key) DO UPDATE SET (cid, leaf) = ($3, $4)
RETURNING id`,
headerID, accountKey.Hex(), stateCID.CID, stateCID.Leaf).Scan(&stateID)
if err != nil {
tx.Rollback()
return err
}
for _, storageCID := range payload.StorageNodeCIDs[accountKey] {
err = repo.indexStorageCID(tx, storageCID, stateID)
if err != nil {
tx.Rollback()
return err
}
}
}
return nil
return tx.Commit()
}
func (repo *Repository) indexStorageCID(tx *sqlx.Tx, storageCID StorageNodeCID, stateID int64) error {
_, err := repo.db.Exec(`INSERT INTO public.storage_cids (state_id, storage_key, cid, leaf) VALUES ($1, $2, $3, $4)
ON CONFLICT DO UPDATE SET (cid, leaf) = ($3, $4)`,
stateID, storageCID.Key, storageCID.CID, storageCID.Leaf)
ON CONFLICT (state_id, storage_key) DO UPDATE SET (cid, leaf) = ($3, $4)`,
stateID, storageCID.Key.Hex(), storageCID.CID, storageCID.Leaf)
return err
}
pkg/ipfs/streamer.go
+2 -3
View File
@@ -30,8 +30,7 @@ type StateDiffStreamer interface {
// Streamer is the underlying struct for the StateDiffStreamer interface
type Streamer struct {
Client core.RpcClient
PayloadChan chan statediff.Payload
Client core.RpcClient
}
// NewStateDiffStreamer creates a pointer to a new Streamer which satisfies the StateDiffStreamer interface
@@ -43,5 +42,5 @@ func NewStateDiffStreamer(client core.RpcClient) *Streamer {
// Stream is the main loop for subscribing to data from the Geth state diff process
func (sds *Streamer) Stream(payloadChan chan statediff.Payload) (*rpc.ClientSubscription, error) {
return sds.Client.Subscribe("statediff", sds.PayloadChan)
return sds.Client.Subscribe("statediff", payloadChan, "subscribe")
}