Run integration tests for service to fill indexing gap in a separate script

Makefile

@@ -77,6 +77,7 @@ integrationtest_local: | $(GINKGO) $(GOOSE)
 	go fmt ./...
 	./scripts/run_integration_test.sh
 	./scripts/run_integration_test_forward_eth_calls.sh
+	./scripts/run_integration_test_watched_address_gap_filler.sh
 
 build:
 	go fmt ./...

scripts/run_integration_test.sh

@@ -5,6 +5,7 @@ export ETH_FORWARD_ETH_CALLS=false
 export DB_WRITE=true
 export ETH_HTTP_PATH=""
 export ETH_PROXY_ON_ERROR=false
+export WATCHED_ADDRESS_GAP_FILLER_ENABLED=false
 
 # Clear up existing docker images and volume.
 docker-compose down --remove-orphans --volumes

scripts/run_integration_test_forward_eth_calls.sh

@@ -5,6 +5,7 @@ export ETH_FORWARD_ETH_CALLS=true
 export DB_WRITE=false
 export ETH_HTTP_PATH="dapptools:8545"
 export ETH_PROXY_ON_ERROR=false
+export WATCHED_ADDRESS_GAP_FILLER_ENABLED=false
 
 # Clear up existing docker images and volume.
 docker-compose down --remove-orphans --volumes

scripts/run_integration_test_watched_address_gap_filler.sh

@@ -0,0 +1,28 @@
+set -e
+set -o xtrace
+
+export ETH_FORWARD_ETH_CALLS=false
+export DB_WRITE=true
+export ETH_HTTP_PATH=""
+export ETH_PROXY_ON_ERROR=false
+export WATCHED_ADDRESS_GAP_FILLER_ENABLED=true
+export WATCHED_ADDRESS_GAP_FILLER_INTERVAL=5
+
+# Clear up existing docker images and volume.
+docker-compose down --remove-orphans --volumes
+
+# Build and start the containers.
+# Note: Build only if `ipld-eth-server` or other container code is modified. Otherwise comment this line.
+docker-compose -f docker-compose.test.yml -f docker-compose.yml build eth-server
+docker-compose -f docker-compose.test.yml -f docker-compose.yml up -d ipld-eth-db dapptools contract eth-server
+
+export PGPASSWORD=password
+export DATABASE_USER=vdbm
+export DATABASE_PORT=8077
+export DATABASE_PASSWORD=password
+export DATABASE_HOSTNAME=127.0.0.1
+
+# Wait for containers to be up and execute the integration test.
+while [ "$(curl -s -o /dev/null -w ''%{http_code}'' localhost:8081)" != "200" ]; do echo "waiting for ipld-eth-server..." && sleep 5; done && \
+          while [ "$(curl -s -o /dev/null -w ''%{http_code}'' localhost:8545)" != "200" ]; do echo "waiting for geth-statediff..." && sleep 5; done && \
+          make integrationtest

test/direct_proxy_integration_test.go

@@ -20,6 +20,8 @@ import (
 var _ = Describe("Integration test", func() {
 	directProxyEthCalls, err := strconv.ParseBool(os.Getenv("ETH_FORWARD_ETH_CALLS"))
 	Expect(err).To(BeNil())
+	watchedAddressServiceEnabled, err := strconv.ParseBool(os.Getenv("WATCHED_ADDRESS_GAP_FILLER_ENABLED"))
+	Expect(err).To(BeNil())
 	gethHttpPath := "http://127.0.0.1:8545"
 	gethClient, err := ethclient.Dial(gethHttpPath)
 	Expect(err).ToNot(HaveOccurred())
@@ -38,11 +40,14 @@ var _ = Describe("Integration test", func() {
 	var txErr error
 	sleepInterval := 2 * time.Second
 
+	BeforeEach(func() {
+		if !directProxyEthCalls || watchedAddressServiceEnabled {
+			Skip("skipping direct-proxy-forwarding integration tests")
+		}
+	})
+
 	Describe("get Block", func() {
 		BeforeEach(func() {
-			if !directProxyEthCalls {
-				Skip("skipping direct-proxy-forwarding integration tests")
-			}
 			contract, contractErr = integration.DeployContract()
 			time.Sleep(sleepInterval)
 		})
@@ -94,9 +99,6 @@ var _ = Describe("Integration test", func() {
 
 	Describe("Transaction", func() {
 		BeforeEach(func() {
-			if !directProxyEthCalls {
-				Skip("skipping direct-proxy-forwarding integration tests")
-			}
 			contract, contractErr = integration.DeployContract()
 			time.Sleep(sleepInterval)
 		})
@@ -122,9 +124,6 @@ var _ = Describe("Integration test", func() {
 
 	Describe("Receipt", func() {
 		BeforeEach(func() {
-			if !directProxyEthCalls {
-				Skip("skipping direct-proxy-forwarding integration tests")
-			}
 			contract, contractErr = integration.DeployContract()
 			time.Sleep(sleepInterval)
 		})
@@ -142,9 +141,6 @@ var _ = Describe("Integration test", func() {
 
 	Describe("FilterLogs", func() {
 		BeforeEach(func() {
-			if !directProxyEthCalls {
-				Skip("skipping direct-proxy-forwarding integration tests")
-			}
 			contract, contractErr = integration.DeployContract()
 			time.Sleep(sleepInterval)
 		})
@@ -174,9 +170,6 @@ var _ = Describe("Integration test", func() {
 
 	Describe("CodeAt", func() {
 		BeforeEach(func() {
-			if !directProxyEthCalls {
-				Skip("skipping direct-proxy-forwarding integration tests")
-			}
 			contract, contractErr = integration.DeployContract()
 			time.Sleep(sleepInterval)
 		})
@@ -224,9 +217,6 @@ var _ = Describe("Integration test", func() {
 	Describe("Get balance", func() {
 		address := "0x1111111111111111111111111111111111111112"
 		BeforeEach(func() {
-			if !directProxyEthCalls {
-				Skip("skipping direct-proxy-forwarding integration tests")
-			}
 			tx, txErr = integration.SendEth(address, "0.01")
 			time.Sleep(sleepInterval)
 		})
@@ -286,9 +276,6 @@ var _ = Describe("Integration test", func() {
 
 	Describe("Get Storage", func() {
 		BeforeEach(func() {
-			if !directProxyEthCalls {
-				Skip("skipping direct-proxy-forwarding integration tests")
-			}
 			contract, contractErr = integration.DeployContract()
 			erc20TotalSupply, bigIntResult = new(big.Int).SetString("1000000000000000000000", 10)
 
@@ -392,9 +379,6 @@ var _ = Describe("Integration test", func() {
 
 	Describe("eth call", func() {
 		BeforeEach(func() {
-			if !directProxyEthCalls {
-				Skip("skipping direct-proxy-forwarding integration tests")
-			}
 			contract, contractErr = integration.DeployContract()
 			erc20TotalSupply, bigIntResult = new(big.Int).SetString("1000000000000000000000", 10)
 
@@ -445,9 +429,6 @@ var _ = Describe("Integration test", func() {
 
 	Describe("Chain ID", func() {
 		It("Check chain id", func() {
-			if !directProxyEthCalls {
-				Skip("skipping direct-proxy-forwarding integration tests")
-			}
 			_, err := gethClient.ChainID(ctx)
 			Expect(err).ToNot(HaveOccurred())
 

test/integration_test.go

@@ -31,6 +31,8 @@ var (
 var _ = Describe("Integration test", func() {
 	directProxyEthCalls, err := strconv.ParseBool(os.Getenv("ETH_FORWARD_ETH_CALLS"))
 	Expect(err).To(BeNil())
+	watchedAddressServiceEnabled, err := strconv.ParseBool(os.Getenv("WATCHED_ADDRESS_GAP_FILLER_ENABLED"))
+	Expect(err).To(BeNil())
 	gethHttpPath := "http://127.0.0.1:8545"
 	gethClient, err := ethclient.Dial(gethHttpPath)
 	Expect(err).ToNot(HaveOccurred())
@@ -49,11 +51,14 @@ var _ = Describe("Integration test", func() {
 	var txErr error
 	sleepInterval := 2 * time.Second
 
+	BeforeEach(func() {
+		if directProxyEthCalls || watchedAddressServiceEnabled {
+			Skip("skipping no-direct-proxy-forwarding integration tests")
+		}
+	})
+
 	Describe("get Block", func() {
 		BeforeEach(func() {
-			if directProxyEthCalls {
-				Skip("skipping no-direct-proxy-forwarding integration tests")
-			}
 			contract, contractErr = integration.DeployContract()
 			time.Sleep(sleepInterval)
 		})
@@ -123,9 +128,6 @@ var _ = Describe("Integration test", func() {
 
 	Describe("Transaction", func() {
 		BeforeEach(func() {
-			if directProxyEthCalls {
-				Skip("skipping no-direct-proxy-forwarding integration tests")
-			}
 			contract, contractErr = integration.DeployContract()
 			time.Sleep(sleepInterval)
 		})
@@ -157,9 +159,6 @@ var _ = Describe("Integration test", func() {
 
 	Describe("Receipt", func() {
 		BeforeEach(func() {
-			if directProxyEthCalls {
-				Skip("skipping no-direct-proxy-forwarding integration tests")
-			}
 			contract, contractErr = integration.DeployContract()
 			time.Sleep(sleepInterval)
 		})
@@ -187,9 +186,6 @@ var _ = Describe("Integration test", func() {
 
 	Describe("FilterLogs", func() {
 		BeforeEach(func() {
-			if directProxyEthCalls {
-				Skip("skipping no-direct-proxy-forwarding integration tests")
-			}
 			contract, contractErr = integration.DeployContract()
 			time.Sleep(sleepInterval)
 		})
@@ -220,9 +216,6 @@ var _ = Describe("Integration test", func() {
 
 	Describe("CodeAt", func() {
 		BeforeEach(func() {
-			if directProxyEthCalls {
-				Skip("skipping no-direct-proxy-forwarding integration tests")
-			}
 			contract, contractErr = integration.DeployContract()
 			time.Sleep(sleepInterval)
 		})
@@ -270,9 +263,6 @@ var _ = Describe("Integration test", func() {
 	Describe("Get balance", func() {
 		address := "0x1111111111111111111111111111111111111112"
 		BeforeEach(func() {
-			if directProxyEthCalls {
-				Skip("skipping no-direct-proxy-forwarding integration tests")
-			}
 			tx, txErr = integration.SendEth(address, "0.01")
 			time.Sleep(sleepInterval)
 		})
@@ -336,9 +326,6 @@ var _ = Describe("Integration test", func() {
 
 	Describe("Get Storage", func() {
 		BeforeEach(func() {
-			if directProxyEthCalls {
-				Skip("skipping no-direct-proxy-forwarding integration tests")
-			}
 			contract, contractErr = integration.DeployContract()
 			erc20TotalSupply, bigIntResult = new(big.Int).SetString("1000000000000000000000", 10)
 
@@ -452,9 +439,6 @@ var _ = Describe("Integration test", func() {
 
 	Describe("eth call", func() {
 		BeforeEach(func() {
-			if directProxyEthCalls {
-				Skip("skipping no-direct-proxy-forwarding integration tests")
-			}
 			contract, contractErr = integration.DeployContract()
 			erc20TotalSupply, bigIntResult = new(big.Int).SetString("1000000000000000000000", 10)
 
@@ -505,9 +489,6 @@ var _ = Describe("Integration test", func() {
 
 	Describe("Chain ID", func() {
 		It("Check chain id", func() {
-			if directProxyEthCalls {
-				Skip("skipping no-direct-proxy-forwarding integration tests")
-			}
 			gethChainId, err := gethClient.ChainID(ctx)
 			Expect(err).ToNot(HaveOccurred())
 

test/watch_address_integration_test.go

@@ -29,6 +29,9 @@ var _ = Describe("WatchAddress integration test", func() {
 	dbWrite, err := strconv.ParseBool(os.Getenv("DB_WRITE"))
 	Expect(err).To(BeNil())
 
+	watchedAddressServiceEnabled, err := strconv.ParseBool(os.Getenv("WATCHED_ADDRESS_GAP_FILLER_ENABLED"))
+	Expect(err).To(BeNil())
+
 	gethHttpPath := "http://127.0.0.1:8545"
 	gethRPCClient, err := rpc.Dial(gethHttpPath)
 	Expect(err).ToNot(HaveOccurred())
@@ -55,8 +58,8 @@ var _ = Describe("WatchAddress integration test", func() {
 	)
 
 	BeforeEach(func() {
-		if !dbWrite {
+		if !dbWrite || watchedAddressServiceEnabled {
-			Skip("skipping WatchAddress integration tests")
+			Skip("skipping WatchAddress API integration tests")
 		}
 	})
 

test/watched_address_gap_filling_service_integration_test.go

@@ -28,7 +28,7 @@ var _ = Describe("Watched address gap filling service integration test", func()
 	dbWrite, err := strconv.ParseBool(os.Getenv("DB_WRITE"))
 	Expect(err).To(BeNil())
 
-	serviceEnabled, err := strconv.ParseBool(os.Getenv("WATCHED_ADDRESS_GAP_FILLER_ENABLED"))
+	watchedAddressServiceEnabled, err := strconv.ParseBool(os.Getenv("WATCHED_ADDRESS_GAP_FILLER_ENABLED"))
 	Expect(err).To(BeNil())
 
 	serviceInterval, err := strconv.ParseInt(os.Getenv("WATCHED_ADDRESS_GAP_FILLER_INTERVAL"), 10, 0)
@@ -78,8 +78,8 @@ var _ = Describe("Watched address gap filling service integration test", func()
 	)
 
 	BeforeEach(func() {
-		if !dbWrite || !serviceEnabled {
+		if !dbWrite || !watchedAddressServiceEnabled {
-			Skip("skipping WatchAddress integration tests")
+			Skip("skipping watched address gap filling service integration tests")
 		}
 	})
 
@@ -152,6 +152,8 @@ var _ = Describe("Watched address gap filling service integration test", func()
 
 	Context("previously unwatched contract watched", func() {
 		It("indexes state only for watched contract", func() {
+			// WatchedAddresses = [GLD1]
+			// WatchedStorageSlots = []
 			// SLV1, countA
 			countA1Storage, err := ipldClient.StorageAt(ctx, common.HexToAddress(SLV1.Address), common.HexToHash(countAIndex), nil)
 			Expect(err).ToNot(HaveOccurred())
@@ -203,6 +205,8 @@ var _ = Describe("Watched address gap filling service integration test", func()
 			// Sleep for service interval + few extra seconds
 			time.Sleep(time.Duration(serviceInterval+2) * time.Second)
 
+			// WatchedAddresses = [GLD1, SLV1]
+			// WatchedStorageSlots = []
 			// SLV1, countA
 			countA1Storage, err := ipldClient.StorageAt(ctx, common.HexToAddress(SLV1.Address), common.HexToHash(countAIndex), nil)
 			Expect(err).ToNot(HaveOccurred())
@@ -266,6 +270,8 @@ var _ = Describe("Watched address gap filling service integration test", func()
 			// Sleep for service interval + few extra seconds
 			time.Sleep(time.Duration(serviceInterval+2) * time.Second)
 
+			// WatchedAddresses = [GLD1, SLV1, SLV2]
+			// WatchedStorageSlots = [countA]
 			// SLV2, countA
 			countA2Storage, err := ipldClient.StorageAt(ctx, common.HexToAddress(SLV2.Address), common.HexToHash(countAIndex), nil)
 			Expect(err).ToNot(HaveOccurred())
@@ -294,7 +300,7 @@ var _ = Describe("Watched address gap filling service integration test", func()
 
 	Context("previously unwatched storage slot watched", func() {
 		It("indexes past state only for watched storage slots updated after created at", func() {
-			// Watch countB
+			// Watch countB with created_at = SLV3.BlockNumber
 			args := []sdtypes.WatchAddressArg{
 				{
 					Address:   countBStorageHash.Hex(),
@@ -307,6 +313,8 @@ var _ = Describe("Watched address gap filling service integration test", func()
 			// Sleep for service interval + few extra seconds
 			time.Sleep(time.Duration(serviceInterval+2) * time.Second)
 
+			// WatchedAddresses = [GLD1, SLV1, SLV2]
+			// WatchedStorageSlots = [countA, countB] (countB -> created_at = SLV3.BlockNumber)
 			// SLV2, countA
 			countA2Storage, err := ipldClient.StorageAt(ctx, common.HexToAddress(SLV2.Address), common.HexToHash(countAIndex), nil)
 			Expect(err).ToNot(HaveOccurred())
@@ -333,19 +341,31 @@ var _ = Describe("Watched address gap filling service integration test", func()
 		})
 
 		It("indexes past state for watched storage slots of watched contracts", func() {
-			// Watch countB
+			// Unwatch countB
 			args := []sdtypes.WatchAddressArg{
+				{
+					Address:   countBStorageHash.Hex(),
+					CreatedAt: uint64(SLV3.BlockNumber),
+				},
+			}
+			ipldErr := ipldRPCClient.Call(nil, ipldMethod, statediff.RemoveStorageSlots, args)
+			Expect(ipldErr).ToNot(HaveOccurred())
+
+			// Watch countB with created_at = SLV1.BlockNumber
+			args = []sdtypes.WatchAddressArg{
 				{
 					Address:   countBStorageHash.Hex(),
 					CreatedAt: uint64(SLV1.BlockNumber),
 				},
 			}
-			ipldErr := ipldRPCClient.Call(nil, ipldMethod, statediff.AddStorageSlots, args)
+			ipldErr = ipldRPCClient.Call(nil, ipldMethod, statediff.AddStorageSlots, args)
 			Expect(ipldErr).ToNot(HaveOccurred())
 
 			// Sleep for service interval + few extra seconds
 			time.Sleep(time.Duration(serviceInterval+2) * time.Second)
 
+			// WatchedAddresses = [GLD1, SLV1, SLV2]
+			// WatchedStorageSlots = [countA, countB] (countB -> created_at = SLV1.BlockNumber)
 			// SLV2, countA
 			countA2Storage, err := ipldClient.StorageAt(ctx, common.HexToAddress(SLV2.Address), common.HexToHash(countAIndex), nil)
 			Expect(err).ToNot(HaveOccurred())
@@ -401,6 +421,8 @@ var _ = Describe("Watched address gap filling service integration test", func()
 			// Sleep for service interval + few extra seconds
 			time.Sleep(time.Duration(serviceInterval+2) * time.Second)
 
+			// WatchedAddresses = [GLD1, SLV1, SLV2, GLD2, SLV3]
+			// WatchedStorageSlots = [countA, countB, totalSupply]
 			// SLV3, countA
 			countA3Storage, err := ipldClient.StorageAt(ctx, common.HexToAddress(SLV3.Address), common.HexToHash(countAIndex), nil)
 			Expect(err).ToNot(HaveOccurred())
@@ -414,4 +436,7 @@ var _ = Describe("Watched address gap filling service integration test", func()
 			Expect(countB3.String()).To(Equal(updatedCountB3.String()))
 		})
 	})
+
+	// TODO:
+	// Add test: watched storage slot not having any previously watched related contracts.
 })