Update fill prefetch CLI (#171)

* Update fill prefetch CLI to accept env variables

* Clean up assets before building util package

* Accomodate change in other watchers and codegen

* Add logs for debugging failing test in CI

packages/codegen/src/templates/fill-template.handlebars

@@ -26,7 +26,9 @@ const log = debug('vulcanize:server');
 export const main = async (): Promise<any> => {
   const argv = await yargs(hideBin(process.argv)).parserConfiguration({
     'parse-numbers': false
-  }).options({
+  }).env(
+    'FILL'
+  ).options({
     configFile: {
       alias: 'f',
       type: 'string',
@@ -62,10 +64,10 @@ export const main = async (): Promise<any> => {
   const db = new Database(config.database);
   await db.init();
   {{#if (subgraphPath)}}
-  
+
   const graphDb = new GraphDatabase(config.database, path.resolve(__dirname, 'entity/*'));
   await graphDb.init();
-  
+
   const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);
   {{/if}}
 

packages/eden-watcher/src/fill.ts

@@ -22,7 +22,9 @@ const log = debug('vulcanize:server');
 export const main = async (): Promise<any> => {
   const argv = await yargs(hideBin(process.argv)).parserConfiguration({
     'parse-numbers': false
-  }).options({
+  }).env(
+    'FILL'
+  ).options({
     configFile: {
       alias: 'f',
       type: 'string',

packages/erc20-watcher/src/fill.ts

@@ -20,7 +20,9 @@ const log = debug('vulcanize:server');
 export const main = async (): Promise<any> => {
   const argv = await yargs(hideBin(process.argv)).parserConfiguration({
     'parse-numbers': false
-  }).options({
+  }).env(
+    'FILL'
+  ).options({
     configFile: {
       alias: 'f',
       type: 'string',

packages/erc721-watcher/src/fill.ts

@@ -20,7 +20,9 @@ const log = debug('vulcanize:server');
 export const main = async (): Promise<any> => {
   const argv = await yargs(hideBin(process.argv)).parserConfiguration({
     'parse-numbers': false
-  }).options({
+  }).env(
+    'FILL'
+  ).options({
     configFile: {
       alias: 'f',
       type: 'string',

packages/graph-test-watcher/src/fill.ts

@@ -22,7 +22,9 @@ const log = debug('vulcanize:server');
 export const main = async (): Promise<any> => {
   const argv = await yargs(hideBin(process.argv)).parserConfiguration({
     'parse-numbers': false
-  }).options({
+  }).env(
+    'FILL'
+  ).options({
     configFile: {
       alias: 'f',
       type: 'string',

packages/mobymask-watcher/src/fill.ts

@@ -20,7 +20,9 @@ const log = debug('vulcanize:server');
 export const main = async (): Promise<any> => {
   const argv = await yargs(hideBin(process.argv)).parserConfiguration({
     'parse-numbers': false
-  }).options({
+  }).env(
+    'FILL'
+  ).options({
     configFile: {
       alias: 'f',
       type: 'string',

packages/solidity-mapper/src/storage.test.ts

@@ -514,6 +514,7 @@ describe('Get value from storage', () => {
       const bytesLength = Math.floor(Math.random() * 64);
       return ethers.utils.hexlify(ethers.utils.randomBytes(bytesLength));
     });
+    console.log('bytesArray', bytesArray);
 
     const addressArray = generateDummyAddresses(4);
 
@@ -647,6 +648,7 @@ describe('Get value from storage', () => {
     });
 
     it('get value for fixed size arrays of dynamic byte array type', async () => {
+      console.log('testFixedArrays.address', testFixedArrays.address);
       const { value, proof } = await getStorageValue(storageLayout, getStorageAt, blockHash, testFixedArrays.address, 'bytesArray');
       expect(value).to.eql(bytesArray);
       const proofData = JSON.parse(proof.data);
@@ -750,6 +752,7 @@ describe('Get value from storage', () => {
     });
 
     it('get value of dynamic bytes type array by index', async () => {
+      console.log('testFixedArrays.address', testFixedArrays.address);
       const arrayIndex = 2;
       const { value, proof } = await getStorageValue(storageLayout, getStorageAt, blockHash, testFixedArrays.address, 'bytesArray', arrayIndex);
       expect(value).to.eql(bytesArray[arrayIndex]);

packages/util/package.json

@@ -37,6 +37,8 @@
   },
   "scripts": {
     "lint": "eslint .",
-    "build": "tsc"
+    "build": "yarn clean && tsc && yarn copy-assets",
+    "clean": "rm -rf ./dist",
+    "copy-assets": "copyfiles -u 1 src/**/*.gql dist/"
   }
 }

packages/util/src/fill.ts

@@ -28,8 +28,18 @@ export const fillBlocks = async (
 ): Promise<any> => {
   let { startBlock, endBlock, prefetch = false, batchBlocks = DEFAULT_PREFETCH_BATCH_SIZE } = argv;
   assert(startBlock <= endBlock, 'endBlock should be greater than or equal to startBlock');
+
   const syncStatus = await indexer.getSyncStatus();
 
+  if (prefetch) {
+    if (syncStatus && startBlock <= syncStatus.chainHeadBlockNumber) {
+      throw new Error(`startBlock should be greater than chain head ${syncStatus.chainHeadBlockNumber}`);
+    }
+
+    await prefetchBlocks(indexer, blockDelayInMilliSecs, { startBlock, endBlock, batchBlocks });
+    return;
+  }
+
   if (syncStatus) {
     if (startBlock > syncStatus.chainHeadBlockNumber + 1) {
       throw new Error(`Missing blocks between startBlock ${startBlock} and chainHeadBlockNumber ${syncStatus.chainHeadBlockNumber}`);
@@ -38,11 +48,6 @@ export const fillBlocks = async (
     startBlock = syncStatus.chainHeadBlockNumber + 1;
   }
 
-  if (prefetch) {
-    await prefetchBlocks(indexer, blockDelayInMilliSecs, { startBlock, endBlock, batchBlocks });
-    return;
-  }
-
   await eventWatcher.initBlockProcessingOnCompleteHandler();
   await eventWatcher.initEventProcessingOnCompleteHandler();
 
@@ -131,6 +136,4 @@ const prefetchBlocks = async (
       process.exit(0);
     }
   }
-
-  console.timeEnd('time:fill#fillBlocks-process_blocks');
 };