Update codegen with changes implemented in mobymask watcher (#148)

* Update codegen with index-block CLI and remove graph-node

* Add filter logs by contract flag

* Skip generating GQL API for immutable variables

* Add config for maxEventsBlockRange

* Add new flags in existing watchers
This commit is contained in:
2022-07-22 13:17:56 +05:30
committed by GitHub
parent b577db287f
commit 1bcabd64f2
69 changed files with 825 additions and 575 deletions
+1 -10
View File
@@ -2,14 +2,12 @@
// Copyright 2021 Vulcanize, Inc.
//
import path from 'path';
import yargs from 'yargs';
import 'reflect-metadata';
import debug from 'debug';
import assert from 'assert';
import { Config, DEFAULT_CONFIG_PATH, getConfig, initClients, JobQueue } from '@vulcanize/util';
import { GraphWatcher, Database as GraphDatabase } from '@vulcanize/graph-node';
import { Database } from '../database';
import { Indexer } from '../indexer';
@@ -46,11 +44,6 @@ const main = async (): Promise<void> => {
const db = new Database(config.database);
await db.init();
const graphDb = new GraphDatabase(config.database, path.resolve(__dirname, 'entity/*'));
await graphDb.init();
const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);
const jobQueueConfig = config.jobQueue;
assert(jobQueueConfig, 'Missing job queue config');
@@ -60,11 +53,9 @@ const main = async (): Promise<void> => {
const jobQueue = new JobQueue({ dbConnectionString, maxCompletionLag: maxCompletionLagInSecs });
await jobQueue.start();
const indexer = new Indexer(config.server, db, ethClient, ethProvider, jobQueue, graphWatcher);
const indexer = new Indexer(config.server, db, ethClient, ethProvider, jobQueue);
await indexer.init();
graphWatcher.setIndexer(indexer);
const blockHash = await indexer.processCLICheckpoint(argv.address, argv.blockHash);
log(`Created a checkpoint for contract ${argv.address} at block-hash ${blockHash}`);
@@ -10,7 +10,6 @@ import fs from 'fs';
import path from 'path';
import { Config, DEFAULT_CONFIG_PATH, getConfig, initClients, JobQueue, StateKind } from '@vulcanize/util';
import { GraphWatcher, Database as GraphDatabase } from '@vulcanize/graph-node';
import * as codec from '@ipld/dag-cbor';
import { Database } from '../database';
@@ -43,11 +42,6 @@ const main = async (): Promise<void> => {
const db = new Database(config.database);
await db.init();
const graphDb = new GraphDatabase(config.database, path.resolve(__dirname, 'entity/*'));
await graphDb.init();
const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);
const jobQueueConfig = config.jobQueue;
assert(jobQueueConfig, 'Missing job queue config');
@@ -57,11 +51,9 @@ const main = async (): Promise<void> => {
const jobQueue = new JobQueue({ dbConnectionString, maxCompletionLag: maxCompletionLagInSecs });
await jobQueue.start();
const indexer = new Indexer(config.server, db, ethClient, ethProvider, jobQueue, graphWatcher);
const indexer = new Indexer(config.server, db, ethClient, ethProvider, jobQueue);
await indexer.init();
graphWatcher.setIndexer(indexer);
const exportData: any = {
snapshotBlock: {},
contracts: [],
@@ -12,7 +12,6 @@ import fs from 'fs';
import path from 'path';
import { getConfig, fillBlocks, JobQueue, DEFAULT_CONFIG_PATH, Config, initClients, StateKind } from '@vulcanize/util';
import { GraphWatcher, Database as GraphDatabase } from '@vulcanize/graph-node';
import * as codec from '@ipld/dag-cbor';
import { Database } from '../database';
@@ -47,11 +46,6 @@ export const main = async (): Promise<any> => {
const db = new Database(config.database);
await db.init();
const graphDb = new GraphDatabase(config.database, path.resolve(__dirname, 'entity/*'));
await graphDb.init();
const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);
// Note: In-memory pubsub works fine for now, as each watcher is a single process anyway.
// Later: https://www.apollographql.com/docs/apollo-server/data/subscriptions/#production-pubsub-libraries
const pubsub = new PubSub();
@@ -65,11 +59,9 @@ export const main = async (): Promise<any> => {
const jobQueue = new JobQueue({ dbConnectionString, maxCompletionLag: maxCompletionLagInSecs });
await jobQueue.start();
const indexer = new Indexer(config.server, db, ethClient, ethProvider, jobQueue, graphWatcher);
const indexer = new Indexer(config.server, db, ethClient, ethProvider, jobQueue);
await indexer.init();
graphWatcher.setIndexer(indexer);
const eventWatcher = new EventWatcher(config.upstream, ethClient, indexer, pubsub, jobQueue);
// Import data.
@@ -0,0 +1,63 @@
//
// Copyright 2022 Vulcanize, Inc.
//
import yargs from 'yargs';
import 'reflect-metadata';
import debug from 'debug';
import assert from 'assert';
import { Config, DEFAULT_CONFIG_PATH, getConfig, initClients, JobQueue, indexBlock } from '@vulcanize/util';
import { Database } from '../database';
import { Indexer } from '../indexer';
const log = debug('vulcanize:index-block');
const main = async (): Promise<void> => {
const argv = await yargs.parserConfiguration({
'parse-numbers': false
}).options({
configFile: {
alias: 'f',
type: 'string',
require: true,
demandOption: true,
describe: 'Configuration file path (toml)',
default: DEFAULT_CONFIG_PATH
},
block: {
type: 'number',
require: true,
demandOption: true,
describe: 'Block number to index'
}
}).argv;
const config: Config = await getConfig(argv.configFile);
const { ethClient, ethProvider } = await initClients(config);
const db = new Database(config.database);
await db.init();
const jobQueueConfig = config.jobQueue;
assert(jobQueueConfig, 'Missing job queue config');
const { dbConnectionString, maxCompletionLagInSecs } = jobQueueConfig;
assert(dbConnectionString, 'Missing job queue db connection string');
const jobQueue = new JobQueue({ dbConnectionString, maxCompletionLag: maxCompletionLagInSecs });
const indexer = new Indexer(config.server, db, ethClient, ethProvider, jobQueue);
await indexer.init();
await indexBlock(indexer, jobQueueConfig.eventsInBatch, argv);
await db.close();
};
main().catch(err => {
log(err);
}).finally(() => {
process.exit(0);
});
+1 -10
View File
@@ -2,7 +2,6 @@
// Copyright 2021 Vulcanize, Inc.
//
import path from 'path';
import assert from 'assert';
import yargs from 'yargs';
import 'reflect-metadata';
@@ -10,7 +9,6 @@ import debug from 'debug';
import util from 'util';
import { Config, DEFAULT_CONFIG_PATH, getConfig, initClients, JobQueue } from '@vulcanize/util';
import { GraphWatcher, Database as GraphDatabase } from '@vulcanize/graph-node';
import { Database } from '../database';
import { Indexer } from '../indexer';
@@ -43,11 +41,6 @@ const main = async (): Promise<void> => {
const db = new Database(config.database);
await db.init();
const graphDb = new GraphDatabase(config.database, path.resolve(__dirname, 'entity/*'));
await graphDb.init();
const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);
const jobQueueConfig = config.jobQueue;
assert(jobQueueConfig, 'Missing job queue config');
@@ -57,11 +50,9 @@ const main = async (): Promise<void> => {
const jobQueue = new JobQueue({ dbConnectionString, maxCompletionLag: maxCompletionLagInSecs });
await jobQueue.start();
const indexer = new Indexer(config.server, db, ethClient, ethProvider, jobQueue, graphWatcher);
const indexer = new Indexer(config.server, db, ethClient, ethProvider, jobQueue);
await indexer.init();
graphWatcher.setIndexer(indexer);
const ipldBlock = await indexer.getIPLDBlockByCid(argv.cid);
assert(ipldBlock, 'IPLDBlock for the provided CID doesn\'t exist.');
@@ -2,13 +2,11 @@
// Copyright 2021 Vulcanize, Inc.
//
import path from 'path';
import debug from 'debug';
import { MoreThan } from 'typeorm';
import assert from 'assert';
import { getConfig, initClients, resetJobs, JobQueue } from '@vulcanize/util';
import { GraphWatcher, Database as GraphDatabase } from '@vulcanize/graph-node';
import { Database } from '../../database';
import { Indexer } from '../../indexer';
@@ -50,11 +48,6 @@ export const handler = async (argv: any): Promise<void> => {
const db = new Database(config.database);
await db.init();
const graphDb = new GraphDatabase(config.database, path.resolve(__dirname, 'entity/*'));
await graphDb.init();
const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);
const jobQueueConfig = config.jobQueue;
assert(jobQueueConfig, 'Missing job queue config');
@@ -64,11 +57,9 @@ export const handler = async (argv: any): Promise<void> => {
const jobQueue = new JobQueue({ dbConnectionString, maxCompletionLag: maxCompletionLagInSecs });
await jobQueue.start();
const indexer = new Indexer(config.server, db, ethClient, ethProvider, jobQueue, graphWatcher);
const indexer = new Indexer(config.server, db, ethClient, ethProvider, jobQueue);
await indexer.init();
graphWatcher.setIndexer(indexer);
const blockProgresses = await indexer.getBlocksAtHeight(argv.blockNumber, false);
assert(blockProgresses.length, `No blocks at specified block number ${argv.blockNumber}`);
assert(!blockProgresses.some(block => !block.isComplete), `Incomplete block at block number ${argv.blockNumber} with unprocessed events`);
@@ -2,14 +2,12 @@
// Copyright 2021 Vulcanize, Inc.
//
import path from 'path';
import yargs from 'yargs';
import 'reflect-metadata';
import debug from 'debug';
import assert from 'assert';
import { Config, DEFAULT_CONFIG_PATH, getConfig, initClients, JobQueue } from '@vulcanize/util';
import { GraphWatcher, Database as GraphDatabase } from '@vulcanize/graph-node';
import { Database } from '../database';
import { Indexer } from '../indexer';
@@ -59,11 +57,6 @@ const main = async (): Promise<void> => {
const db = new Database(config.database);
await db.init();
const graphDb = new GraphDatabase(config.database, path.resolve(__dirname, 'entity/*'));
await graphDb.init();
const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);
const jobQueueConfig = config.jobQueue;
assert(jobQueueConfig, 'Missing job queue config');
@@ -73,11 +66,9 @@ const main = async (): Promise<void> => {
const jobQueue = new JobQueue({ dbConnectionString, maxCompletionLag: maxCompletionLagInSecs });
await jobQueue.start();
const indexer = new Indexer(config.server, db, ethClient, ethProvider, jobQueue, graphWatcher);
const indexer = new Indexer(config.server, db, ethClient, ethProvider, jobQueue);
await indexer.init();
graphWatcher.setIndexer(indexer);
await indexer.watchContract(argv.address, argv.kind, argv.checkpoint, argv.startingBlock);
await db.close();
@@ -2,7 +2,7 @@
// Copyright 2022 Vulcanize, Inc.
//
import { Entity, PrimaryColumn, Column, Index } from 'typeorm';
import { Entity, PrimaryColumn, Column } from 'typeorm';
@Entity()
export class TransferCount {
+1 -10
View File
@@ -2,7 +2,6 @@
// Copyright 2021 Vulcanize, Inc.
//
import path from 'path';
import assert from 'assert';
import 'reflect-metadata';
import yargs from 'yargs';
@@ -11,7 +10,6 @@ import debug from 'debug';
import { PubSub } from 'apollo-server-express';
import { Config, getConfig, fillBlocks, JobQueue, DEFAULT_CONFIG_PATH, initClients } from '@vulcanize/util';
import { GraphWatcher, Database as GraphDatabase } from '@vulcanize/graph-node';
import { Database } from './database';
import { Indexer } from './indexer';
@@ -58,11 +56,6 @@ export const main = async (): Promise<any> => {
const db = new Database(config.database);
await db.init();
const graphDb = new GraphDatabase(config.database, path.resolve(__dirname, 'entity/*'));
await graphDb.init();
const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);
const jobQueueConfig = config.jobQueue;
assert(jobQueueConfig, 'Missing job queue config');
@@ -72,11 +65,9 @@ export const main = async (): Promise<any> => {
const jobQueue = new JobQueue({ dbConnectionString, maxCompletionLag: maxCompletionLagInSecs });
await jobQueue.start();
const indexer = new Indexer(config.server, db, ethClient, ethProvider, jobQueue, graphWatcher);
const indexer = new Indexer(config.server, db, ethClient, ethProvider, jobQueue);
await indexer.init();
graphWatcher.setIndexer(indexer);
// Note: In-memory pubsub works fine for now, as each watcher is a single process anyway.
// Later: https://www.apollographql.com/docs/apollo-server/data/subscriptions/#production-pubsub-libraries
const pubsub = new PubSub();
+1 -1
View File
@@ -4,7 +4,7 @@
import assert from 'assert';
import { updateStateForMappingType, updateStateForElementaryType } from '@vulcanize/util';
import { updateStateForElementaryType } from '@vulcanize/util';
import { Indexer, ResultEvent } from './indexer';
import { TransferCount } from './entity/TransferCount';
+32 -41
View File
@@ -29,7 +29,6 @@ import {
StateKind,
IpldStatus as IpldStatusInterface
} from '@vulcanize/util';
import { GraphWatcher } from '@vulcanize/graph-node';
import ERC721Artifacts from './artifacts/ERC721.json';
import { Database } from './database';
@@ -94,7 +93,6 @@ export class Indexer implements IPLDIndexerInterface {
_ethProvider: BaseProvider
_baseIndexer: BaseIndexer
_serverConfig: ServerConfig
_graphWatcher: GraphWatcher;
_abiMap: Map<string, JsonFragment[]>
_storageLayoutMap: Map<string, StorageLayout>
@@ -102,10 +100,7 @@ export class Indexer implements IPLDIndexerInterface {
_ipfsClient: IPFSClient
_entityTypesMap: Map<string, { [key: string]: string }>
_relationsMap: Map<any, { [key: string]: any }>
constructor (serverConfig: ServerConfig, db: Database, ethClient: EthClient, ethProvider: BaseProvider, jobQueue: JobQueue, graphWatcher: GraphWatcher) {
constructor (serverConfig: ServerConfig, db: Database, ethClient: EthClient, ethProvider: BaseProvider, jobQueue: JobQueue) {
assert(db);
assert(ethClient);
@@ -115,7 +110,6 @@ export class Indexer implements IPLDIndexerInterface {
this._serverConfig = serverConfig;
this._ipfsClient = new IPFSClient(this._serverConfig.ipfsApiAddr);
this._baseIndexer = new BaseIndexer(this._serverConfig, this._db, this._ethClient, this._ethProvider, jobQueue, this._ipfsClient);
this._graphWatcher = graphWatcher;
this._abiMap = new Map();
this._storageLayoutMap = new Map();
@@ -131,13 +125,9 @@ export class Indexer implements IPLDIndexerInterface {
assert(ERC721StorageLayout);
this._storageLayoutMap.set(KIND_ERC721, ERC721StorageLayout);
this._contractMap.set(KIND_ERC721, new ethers.utils.Interface(ERC721ABI));
this._entityTypesMap = new Map();
this._relationsMap = new Map();
}
get serverConfig () {
get serverConfig (): ServerConfig {
return this._serverConfig;
}
@@ -452,9 +442,8 @@ export class Indexer implements IPLDIndexerInterface {
return res;
}
async saveOrUpdateTransferCount (transferCount: TransferCount) {
async saveOrUpdateTransferCount (transferCount: TransferCount): Promise<void> {
const dbTx = await this._db.createTransactionRunner();
let res;
try {
await this._db.saveTransferCount(dbTx, transferCount);
@@ -464,8 +453,6 @@ export class Indexer implements IPLDIndexerInterface {
} finally {
await dbTx.release();
}
return res;
}
async _name (blockHash: string, contractAddress: string, diff = false): Promise<ValueResult> {
@@ -784,20 +771,9 @@ export class Indexer implements IPLDIndexerInterface {
await this._baseIndexer.removeIPLDBlocks(blockNumber, kind);
}
async getSubgraphEntity<Entity> (entity: new () => Entity, id: string, block?: BlockHeight): Promise<any> {
const relations = this._relationsMap.get(entity) || {};
const data = await this._graphWatcher.getEntity(entity, id, relations, block);
return data;
}
async triggerIndexingOnEvent (event: Event): Promise<void> {
const resultEvent = this.getResultEvent(event);
// Call subgraph handler for event.
await this._graphWatcher.handleEvent(resultEvent);
// Call custom hook function for indexing on event.
await handleEvent(this, resultEvent);
}
@@ -810,9 +786,6 @@ export class Indexer implements IPLDIndexerInterface {
async processBlock (blockHash: string, blockNumber: number): Promise<void> {
// Call a function to create initial state for contracts.
await this._baseIndexer.createInit(this, blockHash, blockNumber);
// Call subgraph handler for block.
await this._graphWatcher.handleBlock(blockHash);
}
parseEventNameAndArgs (kind: string, logObj: any): any {
@@ -848,7 +821,7 @@ export class Indexer implements IPLDIndexerInterface {
switch (logDescription.name) {
case APPROVAL_EVENT: {
eventName = logDescription.name;
const { owner, approved, tokenId } = logDescription.args;
const [owner, approved, tokenId] = logDescription.args;
eventInfo = {
owner,
approved,
@@ -859,7 +832,7 @@ export class Indexer implements IPLDIndexerInterface {
}
case APPROVALFORALL_EVENT: {
eventName = logDescription.name;
const { owner, operator, approved } = logDescription.args;
const [owner, operator, approved] = logDescription.args;
eventInfo = {
owner,
operator,
@@ -870,7 +843,7 @@ export class Indexer implements IPLDIndexerInterface {
}
case TRANSFER_EVENT: {
eventName = logDescription.name;
const { from, to, tokenId } = logDescription.args;
const [from, to, tokenId] = logDescription.args;
eventInfo = {
from,
to,
@@ -1054,18 +1027,36 @@ export class Indexer implements IPLDIndexerInterface {
return this._baseIndexer.getAncestorAtDepth(blockHash, depth);
}
getEntityTypesMap (): Map<string, { [key: string]: string }> {
return this._entityTypesMap;
}
async _fetchAndSaveEvents ({ cid: blockCid, blockHash }: DeepPartial<BlockProgress>): Promise<BlockProgress> {
assert(blockHash);
const logsPromise = this._ethClient.getLogs({ blockHash });
const transactionsPromise = this._ethClient.getBlockWithTransactions({ blockHash });
const blockPromise = this._ethClient.getBlockByHash(blockHash);
let logs: any[];
if (this._serverConfig.filterLogs) {
const watchedContracts = this._baseIndexer.getWatchedContracts();
// TODO: Query logs by multiple contracts.
const contractlogsPromises = watchedContracts.map((watchedContract): Promise<any> => this._ethClient.getLogs({
blockHash,
contract: watchedContract.address
}));
const contractlogs = await Promise.all(contractlogsPromises);
// Flatten logs by contract and sort by index.
logs = contractlogs.map(data => {
return data.logs;
}).flat()
.sort((a, b) => {
return a.index - b.index;
});
} else {
({ logs } = await this._ethClient.getLogs({ blockHash }));
}
let [
{ block, logs },
{ block },
{
allEthHeaderCids: {
nodes: [
@@ -1077,7 +1068,7 @@ export class Indexer implements IPLDIndexerInterface {
]
}
}
] = await Promise.all([logsPromise, transactionsPromise]);
] = await Promise.all([blockPromise, transactionsPromise]);
const transactionMap = transactions.reduce((acc: {[key: string]: any}, transaction: {[key: string]: any}) => {
acc[transaction.txHash] = transaction;
+1 -10
View File
@@ -2,7 +2,6 @@
// Copyright 2021 Vulcanize, Inc.
//
import path from 'path';
import assert from 'assert';
import 'reflect-metadata';
import yargs from 'yargs';
@@ -24,7 +23,6 @@ import {
DEFAULT_CONFIG_PATH,
initClients
} from '@vulcanize/util';
import { GraphWatcher, Database as GraphDatabase } from '@vulcanize/graph-node';
import { Indexer } from './indexer';
import { Database } from './database';
@@ -253,11 +251,6 @@ export const main = async (): Promise<any> => {
const db = new Database(config.database);
await db.init();
const graphDb = new GraphDatabase(config.database, path.resolve(__dirname, 'entity/*'));
await graphDb.init();
const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);
const jobQueueConfig = config.jobQueue;
assert(jobQueueConfig, 'Missing job queue config');
@@ -267,11 +260,9 @@ export const main = async (): Promise<any> => {
const jobQueue = new JobQueue({ dbConnectionString, maxCompletionLag: maxCompletionLagInSecs });
await jobQueue.start();
const indexer = new Indexer(config.server, db, ethClient, ethProvider, jobQueue, graphWatcher);
const indexer = new Indexer(config.server, db, ethClient, ethProvider, jobQueue);
await indexer.init();
graphWatcher.setIndexer(indexer);
const jobRunner = new JobRunner(jobQueueConfig, indexer, jobQueue);
await jobRunner.start();
};
-1
View File
@@ -12,7 +12,6 @@ import { ValueResult, BlockHeight, StateKind } from '@vulcanize/util';
import { Indexer } from './indexer';
import { EventWatcher } from './events';
import { TransferCount } from './entity/TransferCount';
const log = debug('vulcanize:resolver');
+1 -9
View File
@@ -15,7 +15,6 @@ import 'graphql-import-node';
import { createServer } from 'http';
import { DEFAULT_CONFIG_PATH, getConfig, Config, JobQueue, KIND_ACTIVE, initClients } from '@vulcanize/util';
import { GraphWatcher, Database as GraphDatabase } from '@vulcanize/graph-node';
import { createResolvers } from './resolvers';
import { Indexer } from './indexer';
@@ -43,11 +42,6 @@ export const main = async (): Promise<any> => {
const db = new Database(config.database);
await db.init();
const graphDb = new GraphDatabase(config.database, path.resolve(__dirname, 'entity/*'));
await graphDb.init();
const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);
// Note: In-memory pubsub works fine for now, as each watcher is a single process anyway.
// Later: https://www.apollographql.com/docs/apollo-server/data/subscriptions/#production-pubsub-libraries
const pubsub = new PubSub();
@@ -60,11 +54,9 @@ export const main = async (): Promise<any> => {
const jobQueue = new JobQueue({ dbConnectionString, maxCompletionLag: maxCompletionLagInSecs });
const indexer = new Indexer(config.server, db, ethClient, ethProvider, jobQueue, graphWatcher);
const indexer = new Indexer(config.server, db, ethClient, ethProvider, jobQueue);
await indexer.init();
graphWatcher.setIndexer(indexer);
const eventWatcher = new EventWatcher(config.upstream, ethClient, indexer, pubsub, jobQueue);
if (watcherKind === KIND_ACTIVE) {