CI test

.gitea/workflows/test.yml

@@ -13,8 +13,8 @@ env:
   SYSTEM_TESTS_REF: roysc/test-blob-tx
 
 jobs:
-  test:
-    name: Run integration tests
+  test-beacon-collector:
+    name: Run Beacon collector tests
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
@@ -62,3 +62,36 @@ jobs:
           pip3 install --no-deps 'eth-account>=0.12.3'
           pip3 install 'pydantic>=2.0.0'
           python3 -m pytest -vv -k test_blob_tx
+
+  test-blobscan-scraper:
+    name: Run Blobscan scraper tests
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-go@v4
+        with:
+          go-version-file: 'go.mod'
+          check-latest: true
+      - name: "Install redis-cli"
+        env:
+          DEBIAN_FRONTEND: noninteractive
+        run: apt-get update && apt-get install -y redis-tools
+
+      - name: "Run Blob DB"
+        working-directory: ./test
+        run: docker compose up eth-blob-db --wait --quiet-pull
+        # Check a mainnet blob
+      - name: "Run tests"
+        env:
+          TEST_BLOCK: 20000000
+          TEST_VHASH: 0x017ba4bd9c166498865a3d08618e333ee84812941b5c3a356971b4a6ffffa574
+          # First 10 bytes of blob data
+          EXPECTED_VALUE: 2100000675789c8cd37b0a
+        run: |
+          REDIS_PORT=$(docker port test-eth-blob-db-1 6379 | cut -d: -f2)
+          go run ./cmd/blobscan-scraper \
+            --redis-addr localhost:$REDIS_PORT --log-level debug \
+            --from-block $TEST_BLOCK --to-block $((TEST_BLOCK + 1))
+
+          VALUE=$(redis-cli -p $REDIS_PORT GETRANGE blob:$TEST_VHASH 0 9 | od -An -tx1 | tr -d ' \n')
+          [ "$EXPECTED_VALUE" = "$VALUE" ]

cmd/blobscan-scraper/main.go

@@ -23,15 +23,19 @@ func main() {
 	cmd.Flags.StringVar(&BlobscanAddress,
 		"blobscan-addr", defaultBlobscanAddr, "Address of the Blobscan API")
 	cmd.Flags.Uint64Var(&FromBlock,
-		"block-start", 0, "(EL) Block number to start scanning from")
+		"from-block", 0, "EL block number to start scanning from")
 	cmd.Flags.Uint64Var(&ToBlock,
-		"block-end", 0, "(EL) Block number to scan up to (0 for no limit))")
+		"to-block", 0, "EL block number to scan up to (0 for no limit))")
 	cmd.Flags.Uint64Var(&BlockStep,
 		"block-step", blobscan.BlobscanMaxResults, "Number of blocks to fetch per API call")
 	cmd.ParseFlags()
 
-	indexer := blobscan.NewBlobscanScraper(BlobscanAddress)
+	if BlockStep > blobscan.BlobscanMaxResults {
+		slog.Warn("Block step exceeds max results supported by API",
+			"block-step", BlockStep, "max", blobscan.BlobscanMaxResults)
+	}
 
+	indexer := &blobscan.BlobscanScraper{URL: BlobscanAddress, BlockStep: BlockStep}
 	rdb := redis.NewClient(&redis.Options{
 		Addr: cmd.RedisAddress,
 	})

pkg/blobscan/service.go

@@ -17,17 +17,13 @@ const (
 )
 
 var (
-	// BlobscanBlocksPath = "/blocks/%s"
 	BlobscanBlocksRangePath = "/blocks?sort=asc&startBlock=%d&endBlock=%d"
 	BlobscanBlobsPath       = "/blobs/%s"
 )
 
 type BlobscanScraper struct {
-	URL string
-}
-
-func NewBlobscanScraper(rawurl string) *BlobscanScraper {
-	return &BlobscanScraper{URL: rawurl}
+	URL       string
+	BlockStep uint64
 }
 
 func (bs *BlobscanScraper) Endpoint(path string, args ...any) string {
@@ -40,11 +36,12 @@ func (bs *BlobscanScraper) ApiBlocksRange(from, to uint64) string {
 }
 
 func (bs *BlobscanScraper) ScrapeBlobs(ctx context.Context, from, to uint64, db storage.KvStorage) error {
-	for num := from; to == 0 || num <= to; num += BlobscanMaxResults {
+	for num := from; to == 0 || num <= to; num += bs.BlockStep {
 		if ctx.Err() != nil {
 			return ctx.Err()
 		}
-		endpoint := bs.ApiBlocksRange(num, num+BlobscanMaxResults)
+		end := min(to, num+bs.BlockStep)
+		endpoint := bs.ApiBlocksRange(num, end)
 		log.Debug("Fetching block", "endpoint", endpoint)
 		blocks, err := fetchBlocks(ctx, endpoint)
 		if err != nil {