Support making requests in parallel

packages/test/README.md

@@ -80,5 +80,5 @@
 * Run:
 
   ```bash
-  yarn eth-get-logs  -i <input-requests-json-file> -o <output-results-json-file> -c <output-curl-requests-file> -e http://127.0.0.1:1234/rpc/v1
+  yarn eth-get-logs  -i <input-requests-json-file> -o <output-results-json-file> -c <output-curl-requests-file> -e http://127.0.0.1:1234/rpc/v1 --parallel <true | false>
   ```

packages/test/data/requests-blockrange-near-head-1-address-1-topic.json

@@ -0,0 +1,122 @@
+[
+  {
+    "address": [
+      "0x497f5f88e0bad1a184e110514142dd9d94728ed5"
+    ],
+    "topics": [
+      [
+        "0x8c5be1e5ebec7d5bd14f71427d1e84f3dd0314c0f7b2291e5b200ac8c7c3b925"
+      ]
+    ],
+    "fromBlock": "0x405C02",
+    "toBlock": "0x405C66"
+  },
+  {
+    "address": [
+      "0x60e1773636cf5e4a227d9ac24f20feca034ee25a"
+    ],
+    "topics": [
+      [
+        "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef"
+      ]
+    ],
+    "fromBlock": "0x405C02",
+    "toBlock": "0x405C66"
+  },
+  {
+    "address": [
+      "0x60e1773636cf5e4a227d9ac24f20feca034ee25a"
+    ],
+    "topics": [
+      [
+        "0xe1fffcc4923d04b559f4d29a8bfc6cda04eb5b0d3c460751c2402c5c5cc9109c"
+      ]
+    ],
+    "fromBlock": "0x405C02",
+    "toBlock": "0x405C66"
+  },
+  {
+    "address": [
+      "0x60e1773636cf5e4a227d9ac24f20feca034ee25a"
+    ],
+    "topics": [
+      [
+        "0x8c5be1e5ebec7d5bd14f71427d1e84f3dd0314c0f7b2291e5b200ac8c7c3b925"
+      ]
+    ],
+    "fromBlock": "0x405C02",
+    "toBlock": "0x405C66"
+  },
+  {
+    "address": [
+      "0x57e3bb9f790185cfe70cc2c15ed5d6b84dcf4adb"
+    ],
+    "topics": [
+      [
+        "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef"
+      ]
+    ],
+    "fromBlock": "0x405C02",
+    "toBlock": "0x405C66"
+  },
+  {
+    "address": [
+      "0x443a6243a36ef0ae1c46523d563c15abd787f4e9"
+    ],
+    "topics": [
+      [
+        "0xc42079f94a6350d7e6235f29174924f928cc2ac818eb64fed8004e115fbcca67"
+      ]
+    ],
+    "fromBlock": "0x405C02",
+    "toBlock": "0x405C66"
+  },
+  {
+    "address": [
+      "0xaaa93ac72becfbbc9149f293466bbdaa4b5ef68c"
+    ],
+    "topics": [
+      [
+        "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef"
+      ]
+    ],
+    "fromBlock": "0x405C02",
+    "toBlock": "0x405C66"
+  },
+  {
+    "address": [
+      "0xaaa93ac72becfbbc9149f293466bbdaa4b5ef68c"
+    ],
+    "topics": [
+      [
+        "0x9d9c909296d9c674451c0c24f02cb64981eb3b727f99865939192f880a755dcb"
+      ]
+    ],
+    "fromBlock": "0x405C02",
+    "toBlock": "0x405C66"
+  },
+  {
+    "address": [
+      "0x763b29b97e75fb54923325d46bab2807ac8c43c5"
+    ],
+    "topics": [
+      [
+        "0xf154a899b3b867021c992026539485521c86f83735a958729ab118b9ce7a6407"
+      ]
+    ],
+    "fromBlock": "0x405C02",
+    "toBlock": "0x405C66"
+  },
+  {
+    "address": [
+      "0xd51cb0fa9a91f156a80188a18f039140704b8df7"
+    ],
+    "topics": [
+      [
+        "0xa4b3513a5f822f3e098a6a12338b3f07613cb130b75c90a250ab181402f4bb87"
+      ]
+    ],
+    "fromBlock": "0x405C02",
+    "toBlock": "0x405C66"
+  }
+]

packages/test/src/eth-get-logs.ts

@@ -1,3 +1,7 @@
+//
+// Copyright 2024 Vulcanize, Inc.
+//
+
 import { providers } from 'ethers';
 import * as fs from 'fs';
 import * as path from 'path';
@@ -132,6 +136,99 @@ async function getLogs(provider: providers.JsonRpcProvider, logParams: LogParams
   }
 }
 
+async function getLogsParallel(provider: providers.JsonRpcProvider, logParams: LogParams[], outputFilePath: string, curlRequestsOutputFilePath: string) {
+  const filters: any[] = [];
+  const results: any[] = [];
+
+  const latestBlockNumber = await provider.getBlockNumber();
+
+  for (const params of logParams) {
+    // Build the filter object
+    const filter: any = {
+      address: params.address.map(address => address.toLowerCase()),
+      topics: params.topics,
+    };
+
+    const result: any = {
+      ...filter,
+      address: params.address
+    };
+
+    let blockNumber: number;
+    if (params.blockHash) {
+      filter.blockHash = params.blockHash;
+      result.blockHash = params.blockHash;
+
+      const block = await provider.getBlock(params.blockHash);
+      blockNumber = block.number;
+      result.blockNumber = blockNumber;
+    } else {
+      assert(params.toBlock && params.fromBlock, 'fromBlock or toBlock not found');
+
+      filter.fromBlock = params.fromBlock;
+      filter.toBlock = params.toBlock;
+
+      result.fromBlock = params.fromBlock;
+      result.toBlock = params.toBlock;
+
+      blockNumber = parseInt(params.toBlock, 16);
+      result.blocksRange = parseInt(params.toBlock, 16) - parseInt(params.fromBlock, 16);
+    }
+
+    result.blocksBehindHead = latestBlockNumber - blockNumber
+
+    filters.push(filter);
+    results.push(result);
+
+    // Generate the curl command and write it to a file
+    const curlCommand = await generateCurlCommand('http://localhost:1234/rpc/v1', params);
+    fs.appendFileSync(curlRequestsOutputFilePath, curlCommand + '\n\n');
+  }
+
+  try {
+    // Record the start time
+    const startTime = Date.now();
+
+    await Promise.all(filters.map(async (filter, index) => {
+      // Fetch logs using the filter
+      const ethLogs = await provider.send(
+        'eth_getLogs',
+        [filter]
+      );
+
+      // Format raw eth_getLogs response
+      const logs: providers.Log[] = providers.Formatter.arrayOf(
+        provider.formatter.filterLog.bind(provider.formatter)
+      )(ethLogs);
+
+      // Store the result
+      results[index].numEvents = logs.length;
+    }));
+
+    // Record the end time and calculate the time taken
+    const endTime = Date.now();
+    const timeTakenMs = endTime - startTime;
+    const formattedTime = formatTime(timeTakenMs);
+    results.forEach(result => result.timeTaken = formattedTime);
+  } catch (error) {
+    console.error(`Error fetching logs:`, error);
+  } finally {
+    let existingData = [];
+
+    // Read existing outputfile
+    if (fs.existsSync(outputFilePath)) {
+      const data = fs.readFileSync(outputFilePath, 'utf-8');
+      existingData = JSON.parse(data || '[]');
+    }
+
+    // Append new result to existing data
+    existingData.push(...results);
+
+    // Write the updated data back to the JSON file
+    fs.writeFileSync(outputFilePath, JSON.stringify(existingData, null, 2));
+  }
+}
+
 async function main() {
   const argv = await yargs.parserConfiguration({
     'parse-numbers': false
@@ -160,6 +257,12 @@ async function main() {
       describe: 'Output file path for curl requests',
       type: 'string'
     },
+    parallel: {
+      alias: 'p',
+      default: false,
+      describe: 'Make requests in parallel',
+      type: 'boolean'
+    },
   }).argv;
 
   const outputFilePath = path.resolve(argv.output);
@@ -173,7 +276,13 @@ async function main() {
   const provider = new providers.JsonRpcProvider({ url: argv.endpoint, timeout });
 
   // Get logs and measure performance
-  await getLogs(provider, logParams, outputFilePath, curlRequestsOutputFilePath);
+  if (argv.parallel) {
+    log('Making parallel requests');
+    await getLogsParallel(provider, logParams, outputFilePath, curlRequestsOutputFilePath);
+  } else {
+    log('Making serial requests');
+    await getLogs(provider, logParams, outputFilePath, curlRequestsOutputFilePath);
+  }
 
   log(`Results written to ${outputFilePath}`);
   log(`CURL requests written to ${curlRequestsOutputFilePath}`);