Merge pull request #171 from vulcanize/pm-error-logging

Add misssing error logs

.github/workflows/on-pr-publish.yaml

@@ -27,6 +27,10 @@ jobs:
       BUILD_HOSTNAME: ${{ secrets.BUILD_HOSTNAME }}
       BUILD_USERNAME: ${{ secrets.BUILD_USERNAME }}
       BUILD_KEY: ${{ secrets.BUILD_KEY }}
+    with:
+      STACK_ORCHESTRATOR_REF: "382aca8e42bc5e33f301f77cdd2e09cc80602fc3"
+      GO_ETHEREUM_REF: "c265fdc30915e01bb633203acbbd9d5009a7ddf2"
+      IPLD_ETH_DB_REF: "4e948c58ce20c20ab633289f986d2ed2a1fe02ec"
   build:
     name: Run docker build
     runs-on: ubuntu-latest

.github/workflows/run_unit_test.sh

@@ -8,30 +8,35 @@ temp_dir=$(mktemp -d)
 cd $temp_dir
 git clone -b $(cat /tmp/git_head_ref) "https://github.com/$(cat /tmp/git_repository).git"
 cd ipld-eth-server
-mkdir -p out
 
 ## Remove the branch and github related info. This way future runs wont be confused.
 rm -f /tmp/git_head_ref /tmp/git_repository
 
-# Remove existing docker-tsdb directory
-rm -rf out/docker-tsdb/
+# Setup the DB
+cd $temp_dir
+git clone "https://github.com/vulcanize/ipld-eth-db.git"; cd ipld-eth-db; git checkout $(cat /tmp/ipld_eth_db_ref)
 
-# Copy over files to setup TimescaleDB
-ID=$(docker create vulcanize/ipld-eth-db:v4.1.1-alpha)
-docker cp $ID:/app/docker-tsdb out/docker-tsdb/
-docker rm -v $ID
+# Spin Up DB using Stack Orchestrator
+cd $temp_dir
+git clone "https://github.com/vulcanize/stack-orchestrator.git"; cd stack-orchestrator; git checkout $(cat /tmp/stack_orchestrator_ref)
 
-# Spin up TimescaleDB
-docker-compose -f out/docker-tsdb/docker-compose.test.yml -f docker-compose.yml up ipld-eth-db
-trap "docker-compose -f out/docker-tsdb/docker-compose.test.yml -f docker-compose.yml down --remove-orphans --volumes; cd $start_dir ; rm -r $temp_dir" SIGINT SIGTERM ERR
-sleep 45
+cd ${temp_dir}/stack-orchestrator
+echo vulcanize_ipld_eth_db=${temp_dir}/ipld-eth-db > ./config.sh
+
+## Remove existing containers if they are present
+docker-compose -f docker/local/docker-compose-db-sharding.yml --env-file ./config.sh down -v --remove-orphans;
+
+trap 'cd ${temp_dir}/stack-orchestrator; docker-compose -f docker/local/docker-compose-db-sharding.yml --env-file ./config.sh down -v --remove-orphans; ' SIGINT SIGTERM
+docker-compose -f docker/local/docker-compose-db-sharding.yml --env-file ./config.sh up -d
 
 # Remove old logs so there's no confusion, then run test
 rm -f /tmp/test.log /tmp/return_test.txt
-PGPASSWORD=password DATABASE_USER=vdbm DATABASE_PORT=8066 DATABASE_PASSWORD=password DATABASE_HOSTNAME=127.0.0.1 DATABASE_NAME=vulcanize_testing_v4 make test >  /tmp/test.log
+cd ${temp_dir}/ipld-eth-server
+PGPASSWORD=password DATABASE_USER=vdbm DATABASE_PORT=8077 DATABASE_PASSWORD=password DATABASE_HOSTNAME=localhost DATABASE_NAME=vulcanize_testing make test >  /tmp/test.log
 echo $? > /tmp/return_test.txt
 
 # Clean up
-docker-compose -f out/docker-tsdb/docker-compose.test.yml -f docker-compose.yml down --remove-orphans --volumes
+
+cd ${temp_dir}/stack-orchestrator; docker-compose -f docker/local/docker-compose-db-sharding.yml --env-file ./config.sh down -v --remove-orphans
 cd $start_dir
 rm -fr $temp_dir

.github/workflows/tests.yaml

@@ -8,6 +8,16 @@ on:
         required: true
       BUILD_KEY:
         required: true
+    inputs:
+      STACK_ORCHESTRATOR_REF:
+        required: true
+        type: string
+      GO_ETHEREUM_REF:
+        required: true
+        type: string
+      IPLD_ETH_DB_REF:
+        required: true
+        type: string
 
 jobs:
   build:
@@ -36,18 +46,24 @@ jobs:
       - name: Output variables to files
         run: |
           echo $GITHUB_REPOSITORY > /tmp/git_repository
+          echo ${{ inputs.IPLD_ETH_DB_REF }} > /tmp/ipld_eth_db_ref
+          echo ${{ inputs.STACK_ORCHESTRATOR_REF }} > /tmp/stack_orchestrator_ref
           [ -z "$GITHUB_HEAD_REF" ] && echo $GITHUB_REF_NAME > /tmp/git_head_ref || echo $GITHUB_HEAD_REF > /tmp/git_head_ref
           echo "-----BEGIN OPENSSH PRIVATE KEY-----" >> /tmp/key
           echo ${{ env.BUILD_KEY }} >> /tmp/key
           echo "-----END OPENSSH PRIVATE KEY-----" >> /tmp/key
           chmod 400 /tmp/key
-          cat  /tmp/git_repository
-          cat  /tmp/git_head_ref
+          cat /tmp/git_repository
+          cat /tmp/git_head_ref
+          cat /tmp/stack_orchestrator_ref
+          cat /tmp/ipld_eth_db_ref
 
       - name: Raw SCP
         run: |
           scp -o 'StrictHostKeyChecking no' -o UserKnownHostsFile=/dev/null -q -i /tmp/key /tmp/git_repository ${{ env.BUILD_USERNAME }}@${{ env.BUILD_HOSTNAME }}:/tmp/git_repository
           scp -o 'StrictHostKeyChecking no' -o UserKnownHostsFile=/dev/null -q -i /tmp/key /tmp/git_head_ref ${{ env.BUILD_USERNAME }}@${{ env.BUILD_HOSTNAME }}:/tmp/git_head_ref
+          scp -o 'StrictHostKeyChecking no' -o UserKnownHostsFile=/dev/null -q -i /tmp/key /tmp/stack_orchestrator_ref ${{ env.BUILD_USERNAME }}@${{ env.BUILD_HOSTNAME }}:/tmp/stack_orchestrator_ref
+          scp -o 'StrictHostKeyChecking no' -o UserKnownHostsFile=/dev/null -q -i /tmp/key /tmp/ipld_eth_db_ref ${{ env.BUILD_USERNAME }}@${{ env.BUILD_HOSTNAME }}:/tmp/ipld_eth_db_ref
           scp -o 'StrictHostKeyChecking no' -o UserKnownHostsFile=/dev/null -q -i /tmp/key .github/workflows/run_unit_test.sh ${{ env.BUILD_USERNAME }}@${{ env.BUILD_HOSTNAME }}:/tmp/run_unit_test.sh
 
       - name: Trigger Unit Test
@@ -68,9 +84,6 @@ jobs:
   integrationtest:
     name: Run integration tests
     env:
-      STACK_ORCHESTRATOR_REF: 418957a1f745c921b21286c13bb033f922a91ae9
-      GO_ETHEREUM_REF: "v1.10.18-statediff-4.0.2-alpha"
-      IPLD_ETH_DB_REF: 91d30b9ea1acecd0a7f4307390a98bf3e289b8d7
       GOPATH: /tmp/go
       DB_WRITE: true
       ETH_FORWARD_ETH_CALLS: false
@@ -89,17 +102,17 @@ jobs:
           path: "./ipld-eth-server"
       - uses: actions/checkout@v2
         with:
-          ref: ${{ env.STACK_ORCHESTRATOR_REF }}
+          ref: ${{ inputs.STACK_ORCHESTRATOR_REF }}
           path: "./stack-orchestrator/"
           repository: vulcanize/stack-orchestrator
       - uses: actions/checkout@v2
         with:
-          ref: ${{ env.GO_ETHEREUM_REF }}
+          ref: ${{ inputs.GO_ETHEREUM_REF }}
           repository: vulcanize/go-ethereum
           path: "./go-ethereum/"
       - uses: actions/checkout@v2
         with:
-          ref: ${{ env.IPLD_ETH_DB_REF }}
+          ref: ${{ inputs.IPLD_ETH_DB_REF }}
           repository: vulcanize/ipld-eth-db
           path: "./ipld-eth-db/"
       - name: Create config file
@@ -112,9 +125,6 @@ jobs:
           echo eth_forward_eth_calls=$ETH_FORWARD_ETH_CALLS >> ./config.sh
           echo eth_proxy_on_error=$ETH_PROXY_ON_ERROR >> ./config.sh
           echo eth_http_path=$ETH_HTTP_PATH >> ./config.sh
-          echo ipld_eth_server_db_dependency=access-node >> ./config.sh
-          echo go_ethereum_db_dependency=access-node >> ./config.sh
-          echo connecting_db_name=vulcanize_testing_v4 >> ./config.sh
           echo watched_address_gap_filler_enabled=false >> ./config.sh
           echo watched_address_gap_filler_interval=5 >> ./config.sh
           cat ./config.sh
@@ -127,11 +137,10 @@ jobs:
       - name: Run docker compose
         run: |
           docker-compose  \
-          -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-db-migration.yml" \
+          -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-db-sharding.yml" \
           -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-go-ethereum.yml" \
           -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-ipld-eth-server.yml" \
           -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-contract.yml" \
-          -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/latest/docker-compose-timescale-db.yml" \
           --env-file "$GITHUB_WORKSPACE/config.sh" \
           up -d --build
       - name: Test
@@ -144,9 +153,6 @@ jobs:
   integrationtest_forwardethcalls:
     name: Run integration tests for direct proxy fall-through of eth_calls
     env:
-      STACK_ORCHESTRATOR_REF: 418957a1f745c921b21286c13bb033f922a91ae9
-      GO_ETHEREUM_REF: "v1.10.18-statediff-4.0.2-alpha"
-      IPLD_ETH_DB_REF: 91d30b9ea1acecd0a7f4307390a98bf3e289b8d7
       GOPATH: /tmp/go
       DB_WRITE: false
       ETH_FORWARD_ETH_CALLS: true
@@ -165,17 +171,17 @@ jobs:
           path: "./ipld-eth-server"
       - uses: actions/checkout@v2
         with:
-          ref: ${{ env.STACK_ORCHESTRATOR_REF }}
+          ref: ${{ inputs.STACK_ORCHESTRATOR_REF }}
           path: "./stack-orchestrator/"
           repository: vulcanize/stack-orchestrator
       - uses: actions/checkout@v2
         with:
-          ref: ${{ env.GO_ETHEREUM_REF }}
+          ref: ${{ inputs.GO_ETHEREUM_REF }}
           repository: vulcanize/go-ethereum
           path: "./go-ethereum/"
       - uses: actions/checkout@v2
         with:
-          ref: ${{ env.IPLD_ETH_DB_REF }}
+          ref: ${{ inputs.IPLD_ETH_DB_REF }}
           repository: vulcanize/ipld-eth-db
           path: "./ipld-eth-db/"
       - name: Create config file
@@ -188,9 +194,6 @@ jobs:
           echo eth_forward_eth_calls=$ETH_FORWARD_ETH_CALLS >> ./config.sh
           echo eth_proxy_on_error=$ETH_PROXY_ON_ERROR >> ./config.sh
           echo eth_http_path=$ETH_HTTP_PATH >> ./config.sh
-          echo ipld_eth_server_db_dependency=access-node >> ./config.sh
-          echo go_ethereum_db_dependency=access-node >> ./config.sh
-          echo connecting_db_name=vulcanize_testing_v4 >> ./config.sh
           echo watched_address_gap_filler_enabled=false >> ./config.sh
           echo watched_address_gap_filler_interval=5 >> ./config.sh
           cat ./config.sh
@@ -203,11 +206,10 @@ jobs:
       - name: Run docker compose
         run: |
           docker-compose  \
-          -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-db-migration.yml" \
+          -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-db-sharding.yml" \
           -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-go-ethereum.yml" \
           -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-ipld-eth-server.yml" \
           -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-contract.yml" \
-          -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/latest/docker-compose-timescale-db.yml" \
           --env-file "$GITHUB_WORKSPACE/config.sh" \
           up -d --build
       - name: Test

pkg/eth/api.go

@@ -37,6 +37,7 @@ import (
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/eth/filters"
 	"github.com/ethereum/go-ethereum/ethclient"
+	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/rpc"
 	"github.com/ethereum/go-ethereum/statediff"
@@ -1096,11 +1097,13 @@ func (pea *PublicEthAPI) writeStateDiffFor(blockHash common.Hash) {
 func (pea *PublicEthAPI) rpcMarshalBlock(b *types.Block, inclTx bool, fullTx bool) (map[string]interface{}, error) {
 	fields, err := RPCMarshalBlock(b, inclTx, fullTx)
 	if err != nil {
+		log.Error("error RPC marshalling block with hash", b.Hash().String(), err)
 		return nil, err
 	}
 	if inclTx {
 		td, err := pea.B.GetTd(b.Hash())
 		if err != nil {
+			log.Error("error getting td for block with hash", b.Hash().String(), err)
 			return nil, err
 		}
 		fields["totalDifficulty"] = (*hexutil.Big)(td)

pkg/eth/backend.go

@@ -342,6 +342,7 @@ func (b *Backend) BlockByNumber(ctx context.Context, blockNumber rpc.BlockNumber
 	var headerIPLD models.IPLDModel
 	headerIPLD, err = b.Fetcher.FetchHeader(tx, headerCID)
 	if err != nil {
+		log.Error("error fetching header ipld", err)
 		if err == sql.ErrNoRows {
 			return nil, nil
 		}
@@ -352,10 +353,12 @@ func (b *Backend) BlockByNumber(ctx context.Context, blockNumber rpc.BlockNumber
 	if err != nil {
 		return nil, err
 	}
+
 	// Fetch and decode the uncle IPLDs
 	var uncleIPLDs []models.IPLDModel
 	uncleIPLDs, err = b.Fetcher.FetchUncles(tx, uncleCIDs)
 	if err != nil {
+		log.Error("error fetching uncle iplds", err)
 		return nil, err
 	}
 	var uncles []*types.Header
@@ -367,10 +370,12 @@ func (b *Backend) BlockByNumber(ctx context.Context, blockNumber rpc.BlockNumber
 		}
 		uncles = append(uncles, &uncle)
 	}
+
 	// Fetch and decode the transaction IPLDs
 	var txIPLDs []models.IPLDModel
 	txIPLDs, err = b.Fetcher.FetchTrxs(tx, txCIDs)
 	if err != nil {
+		log.Error("error fetching tx iplds", err)
 		return nil, err
 	}
 	var transactions []*types.Transaction
@@ -386,6 +391,7 @@ func (b *Backend) BlockByNumber(ctx context.Context, blockNumber rpc.BlockNumber
 	var rctIPLDs []models.IPLDModel
 	rctIPLDs, err = b.Fetcher.FetchRcts(tx, rctCIDs)
 	if err != nil {
+		log.Error("error fetching rct iplds", err)
 		return nil, err
 	}
 	var receipts []*types.Receipt
@@ -438,6 +444,7 @@ func (b *Backend) BlockByHash(ctx context.Context, hash common.Hash) (*types.Blo
 	var headerIPLD models.IPLDModel
 	headerIPLD, err = b.Fetcher.FetchHeader(tx, headerCID)
 	if err != nil {
+		log.Error("error fetching header ipld", err)
 		if err == sql.ErrNoRows {
 			return nil, nil
 		}
@@ -452,6 +459,7 @@ func (b *Backend) BlockByHash(ctx context.Context, hash common.Hash) (*types.Blo
 	var uncleIPLDs []models.IPLDModel
 	uncleIPLDs, err = b.Fetcher.FetchUncles(tx, uncleCIDs)
 	if err != nil {
+		log.Error("error fetching uncle iplds", err)
 		if err == sql.ErrNoRows {
 			return nil, nil
 		}
@@ -470,6 +478,7 @@ func (b *Backend) BlockByHash(ctx context.Context, hash common.Hash) (*types.Blo
 	var txIPLDs []models.IPLDModel
 	txIPLDs, err = b.Fetcher.FetchTrxs(tx, txCIDs)
 	if err != nil {
+		log.Error("error fetching tx iplds", err)
 		if err == sql.ErrNoRows {
 			return nil, nil
 		}
@@ -488,6 +497,7 @@ func (b *Backend) BlockByHash(ctx context.Context, hash common.Hash) (*types.Blo
 	var rctIPLDs []models.IPLDModel
 	rctIPLDs, err = b.Fetcher.FetchRcts(tx, rctCIDs)
 	if err != nil {
+		log.Error("error fetching rct iplds", err)
 		if err == sql.ErrNoRows {
 			return nil, nil
 		}