mirror of
https://github.com/cerc-io/watcher-ts
synced 2025-01-21 10:39:06 +00:00
Refactor state creation code (#204)
* Remove support for pushing state to IPFS * Move job handlers for state creation to util * Rename state creation related methods and objects * Update mock indexer used in graph-node testing * Fetch and merge diffs in batches while creating a state checkpoint * Fix timing logs while for state checkpoint creation * Refactor method to get state query result to util * Accept contracts for state verification in compare CLI config * Make method to update state status map synchronous
This commit is contained in:
prathamesh0
GitHub
parent
ce182bce85
commit
5af90bd388
@ -82,18 +82,10 @@
|
||||
yarn
|
||||
```
|
||||
|
||||
* Run the IPFS (go-ipfs version 0.12.2) daemon:
|
||||
|
||||
```bash
|
||||
ipfs daemon
|
||||
```
|
||||
|
||||
* In the config file (`environments/local.toml`):
|
||||
|
||||
* Update the state checkpoint settings.
|
||||
|
||||
* Update the IPFS API address in `environments/local.toml`.
|
||||
|
||||
* Create the databases configured in `environments/local.toml`.
|
||||
|
||||
### Customize
|
||||
@ -106,11 +98,11 @@
|
||||
|
||||
* Generating state:
|
||||
|
||||
* Edit the custom hook function `createInitialState` (triggered if the watcher passes the start block, checkpoint: `true`) in `src/hooks.ts` to save an initial state `IPLDBlock` using the `Indexer` object.
|
||||
* Edit the custom hook function `createInitialState` (triggered if the watcher passes the start block, checkpoint: `true`) in `src/hooks.ts` to save an initial `State` using the `Indexer` object.
|
||||
|
||||
* Edit the custom hook function `createStateDiff` (triggered on a block) in `src/hooks.ts` to save the state in a `diff` `IPLDBlock` using the `Indexer` object. The default state (if exists) is updated.
|
||||
* Edit the custom hook function `createStateDiff` (triggered on a block) in `src/hooks.ts` to save the state in a `diff` `State` using the `Indexer` object. The default state (if exists) is updated.
|
||||
|
||||
* Edit the custom hook function `createStateCheckpoint` (triggered just before default and CLI checkpoint) in `src/hooks.ts` to save the state in a `checkpoint` `IPLDBlock` using the `Indexer` object.
|
||||
* Edit the custom hook function `createStateCheckpoint` (triggered just before default and CLI checkpoint) in `src/hooks.ts` to save the state in a `checkpoint` `State` using the `Indexer` object.
|
||||
|
||||
### Run
|
||||
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
className: IPLDBlock
|
||||
className: State
|
||||
indexOn:
|
||||
- columns:
|
||||
- cid
|
||||
@ -1,10 +1,10 @@
|
||||
className: IpldStatus
|
||||
className: StateSyncStatus
|
||||
indexOn: []
|
||||
columns:
|
||||
- name: id
|
||||
tsType: number
|
||||
columnType: PrimaryGeneratedColumn
|
||||
- name: latestHooksBlockNumber
|
||||
- name: latestIndexedBlockNumber
|
||||
pgType: integer
|
||||
tsType: number
|
||||
columnType: Column
|
||||
@ -12,10 +12,6 @@ columns:
|
||||
pgType: integer
|
||||
tsType: number
|
||||
columnType: Column
|
||||
- name: latestIPFSBlockNumber
|
||||
pgType: integer
|
||||
tsType: number
|
||||
columnType: Column
|
||||
imports:
|
||||
- toImport:
|
||||
- Entity
|
||||
@ -178,8 +178,8 @@ export class Entity {
|
||||
this._addSyncStatusEntity();
|
||||
this._addContractEntity();
|
||||
this._addBlockProgressEntity();
|
||||
this._addIPLDBlockEntity();
|
||||
this._addIpldStatusEntity();
|
||||
this._addStateEntity();
|
||||
this._addStateSyncStatusEntity();
|
||||
|
||||
const template = Handlebars.compile(this._templateString);
|
||||
this._entities.forEach(entityObj => {
|
||||
@ -278,13 +278,13 @@ export class Entity {
|
||||
this._entities.push(entity);
|
||||
}
|
||||
|
||||
_addIPLDBlockEntity (): void {
|
||||
const entity = yaml.load(fs.readFileSync(path.resolve(__dirname, TABLES_DIR, 'IPLDBlock.yaml'), 'utf8'));
|
||||
_addStateEntity (): void {
|
||||
const entity = yaml.load(fs.readFileSync(path.resolve(__dirname, TABLES_DIR, 'State.yaml'), 'utf8'));
|
||||
this._entities.push(entity);
|
||||
}
|
||||
|
||||
_addIpldStatusEntity (): void {
|
||||
const entity = yaml.load(fs.readFileSync(path.resolve(__dirname, TABLES_DIR, 'IpldStatus.yaml'), 'utf8'));
|
||||
_addStateSyncStatusEntity (): void {
|
||||
const entity = yaml.load(fs.readFileSync(path.resolve(__dirname, TABLES_DIR, 'StateSyncStatus.yaml'), 'utf8'));
|
||||
this._entities.push(entity);
|
||||
}
|
||||
|
||||
|
||||
@ -297,25 +297,12 @@ function generateWatcher (visitor: Visitor, contracts: any[], config: any) {
|
||||
: process.stdout;
|
||||
visitor.exportClient(outStream, schemaContent, path.join(outputDir, 'src/gql'));
|
||||
|
||||
let resetOutStream, resetJQOutStream, resetStateOutStream, resetIPLDStateOutStream;
|
||||
const resetOutStream = fs.createWriteStream(path.join(outputDir, 'src/cli/reset.ts'));
|
||||
const resetJQOutStream = fs.createWriteStream(path.join(outputDir, 'src/cli/reset-cmds/job-queue.ts'));
|
||||
const resetWatcherOutStream = fs.createWriteStream(path.join(outputDir, 'src/cli/reset-cmds/watcher.ts'));
|
||||
const resetStateOutStream = fs.createWriteStream(path.join(outputDir, 'src/cli/reset-cmds/state.ts'));
|
||||
|
||||
if (outputDir) {
|
||||
resetOutStream = fs.createWriteStream(path.join(outputDir, 'src/cli/reset.ts'));
|
||||
resetJQOutStream = fs.createWriteStream(path.join(outputDir, 'src/cli/reset-cmds/job-queue.ts'));
|
||||
resetStateOutStream = fs.createWriteStream(path.join(outputDir, 'src/cli/reset-cmds/state.ts'));
|
||||
if (config.subgraphPath) {
|
||||
resetIPLDStateOutStream = fs.createWriteStream(path.join(outputDir, 'src/cli/reset-cmds/ipld-state.ts'));
|
||||
}
|
||||
} else {
|
||||
resetOutStream = process.stdout;
|
||||
resetJQOutStream = process.stdout;
|
||||
resetStateOutStream = process.stdout;
|
||||
if (config.subgraphPath) {
|
||||
resetIPLDStateOutStream = process.stdout;
|
||||
}
|
||||
}
|
||||
|
||||
visitor.exportReset(resetOutStream, resetJQOutStream, resetStateOutStream, resetIPLDStateOutStream, config.subgraphPath);
|
||||
visitor.exportReset(resetOutStream, resetJQOutStream, resetWatcherOutStream, resetStateOutStream, config.subgraphPath);
|
||||
|
||||
outStream = outputDir
|
||||
? fs.createWriteStream(path.join(outputDir, 'src/cli/export-state.ts'))
|
||||
|
||||
@ -9,22 +9,22 @@ import { Writable } from 'stream';
|
||||
|
||||
const RESET_TEMPLATE_FILE = './templates/reset-template.handlebars';
|
||||
const RESET_JQ_TEMPLATE_FILE = './templates/reset-job-queue-template.handlebars';
|
||||
const RESET_WATCHER_TEMPLATE_FILE = './templates/reset-watcher-template.handlebars';
|
||||
const RESET_STATE_TEMPLATE_FILE = './templates/reset-state-template.handlebars';
|
||||
const RESET_IPLD_STATE_TEMPLATE_FILE = './templates/reset-ipld-state-template.handlebars';
|
||||
|
||||
export class Reset {
|
||||
_queries: Array<any>;
|
||||
_resetTemplateString: string;
|
||||
_resetJQTemplateString: string;
|
||||
_resetWatcherTemplateString: string;
|
||||
_resetStateTemplateString: string;
|
||||
_resetIPLDStateTemplateString: string;
|
||||
|
||||
constructor () {
|
||||
this._queries = [];
|
||||
this._resetTemplateString = fs.readFileSync(path.resolve(__dirname, RESET_TEMPLATE_FILE)).toString();
|
||||
this._resetJQTemplateString = fs.readFileSync(path.resolve(__dirname, RESET_JQ_TEMPLATE_FILE)).toString();
|
||||
this._resetWatcherTemplateString = fs.readFileSync(path.resolve(__dirname, RESET_WATCHER_TEMPLATE_FILE)).toString();
|
||||
this._resetStateTemplateString = fs.readFileSync(path.resolve(__dirname, RESET_STATE_TEMPLATE_FILE)).toString();
|
||||
this._resetIPLDStateTemplateString = fs.readFileSync(path.resolve(__dirname, RESET_IPLD_STATE_TEMPLATE_FILE)).toString();
|
||||
}
|
||||
|
||||
/**
|
||||
@ -71,13 +71,13 @@ export class Reset {
|
||||
*/
|
||||
|
||||
/**
|
||||
* Writes the reset.ts, job-queue.ts, state.ts files generated from templates to respective streams.
|
||||
* Writes the reset.ts, job-queue.ts, watcher.ts, state.ts files generated from templates to respective streams.
|
||||
* @param resetOutStream A writable output stream to write the reset file to.
|
||||
* @param resetJQOutStream A writable output stream to write the reset job-queue file to.
|
||||
* @param resetWatcherOutStream A writable output stream to write the reset watcher file to.
|
||||
* @param resetStateOutStream A writable output stream to write the reset state file to.
|
||||
* @param resetIPLDStateOutStream A writable output stream to write the reset IPLD state file to.
|
||||
*/
|
||||
exportReset (resetOutStream: Writable, resetJQOutStream: Writable, resetStateOutStream: Writable, resetIPLDStateOutStream: Writable | undefined, subgraphPath: string): void {
|
||||
exportReset (resetOutStream: Writable, resetJQOutStream: Writable, resetWatcherOutStream: Writable, resetStateOutStream: Writable, subgraphPath: string): void {
|
||||
const resetTemplate = Handlebars.compile(this._resetTemplateString);
|
||||
const resetString = resetTemplate({});
|
||||
resetOutStream.write(resetString);
|
||||
@ -86,18 +86,16 @@ export class Reset {
|
||||
const resetJQString = resetJQTemplate({});
|
||||
resetJQOutStream.write(resetJQString);
|
||||
|
||||
const resetStateTemplate = Handlebars.compile(this._resetStateTemplateString);
|
||||
const resetWatcherTemplate = Handlebars.compile(this._resetWatcherTemplateString);
|
||||
const obj = {
|
||||
queries: this._queries,
|
||||
subgraphPath
|
||||
};
|
||||
const resetState = resetStateTemplate(obj);
|
||||
resetStateOutStream.write(resetState);
|
||||
const resetWatcher = resetWatcherTemplate(obj);
|
||||
resetWatcherOutStream.write(resetWatcher);
|
||||
|
||||
if (resetIPLDStateOutStream) {
|
||||
const resetIPLDStateTemplate = Handlebars.compile(this._resetIPLDStateTemplateString);
|
||||
const resetIPLDStateString = resetIPLDStateTemplate({});
|
||||
resetIPLDStateOutStream.write(resetIPLDStateString);
|
||||
}
|
||||
const resetStateTemplate = Handlebars.compile(this._resetStateTemplateString);
|
||||
const resetState = resetStateTemplate({});
|
||||
resetStateOutStream.write(resetState);
|
||||
}
|
||||
}
|
||||
|
||||
@ -98,9 +98,9 @@ export class Schema {
|
||||
// Add type and query for SyncStatus.
|
||||
this._addSyncStatus();
|
||||
|
||||
// Add IPLDBlock type and queries.
|
||||
this._addIPLDType();
|
||||
this._addIPLDQuery();
|
||||
// Add State type and queries.
|
||||
this._addStateType();
|
||||
this._addStateQuery();
|
||||
|
||||
// Build the schema.
|
||||
return this._composer.buildSchema();
|
||||
@ -354,9 +354,9 @@ export class Schema {
|
||||
});
|
||||
}
|
||||
|
||||
_addIPLDType (): void {
|
||||
_addStateType (): void {
|
||||
const typeComposer = this._composer.createObjectTC({
|
||||
name: 'ResultIPLDBlock',
|
||||
name: 'ResultState',
|
||||
fields: {
|
||||
block: () => this._composer.getOTC('_Block_').NonNull,
|
||||
contractAddress: 'String!',
|
||||
@ -368,10 +368,10 @@ export class Schema {
|
||||
this._composer.addSchemaMustHaveType(typeComposer);
|
||||
}
|
||||
|
||||
_addIPLDQuery (): void {
|
||||
_addStateQuery (): void {
|
||||
this._composer.Query.addFields({
|
||||
getStateByCID: {
|
||||
type: this._composer.getOTC('ResultIPLDBlock'),
|
||||
type: this._composer.getOTC('ResultState'),
|
||||
args: {
|
||||
cid: 'String!'
|
||||
}
|
||||
@ -380,7 +380,7 @@ export class Schema {
|
||||
|
||||
this._composer.Query.addFields({
|
||||
getState: {
|
||||
type: this._composer.getOTC('ResultIPLDBlock'),
|
||||
type: this._composer.getOTC('ResultState'),
|
||||
args: {
|
||||
blockHash: 'String!',
|
||||
contractAddress: 'String!',
|
||||
|
||||
@ -54,12 +54,12 @@ export const handler = async (argv: any): Promise<void> => {
|
||||
graphWatcher.setIndexer(indexer);
|
||||
await graphWatcher.init();
|
||||
|
||||
const ipldBlock = await indexer.getIPLDBlockByCid(argv.cid);
|
||||
assert(ipldBlock, 'IPLDBlock for the provided CID doesn\'t exist.');
|
||||
const data = indexer.getIPLDData(ipldBlock);
|
||||
const state = await indexer.getStateByCID(argv.cid);
|
||||
assert(state, 'State for the provided CID doesn\'t exist.');
|
||||
const data = indexer.getStateData(state);
|
||||
|
||||
log(`Verifying checkpoint data for contract ${ipldBlock.contractAddress}`);
|
||||
await verifyCheckpointData(graphDb, ipldBlock.block, data);
|
||||
log(`Verifying checkpoint data for contract ${state.contractAddress}`);
|
||||
await verifyCheckpointData(graphDb, state.block, data);
|
||||
log('Checkpoint data verified');
|
||||
|
||||
await db.close();
|
||||
|
||||
@ -9,9 +9,6 @@
|
||||
# Checkpoint interval in number of blocks.
|
||||
checkpointInterval = 2000
|
||||
|
||||
# IPFS API address (can be taken from the output on running the IPFS daemon).
|
||||
ipfsApiAddr = "/ip4/127.0.0.1/tcp/5001"
|
||||
|
||||
{{#if (subgraphPath)}}
|
||||
subgraphPath = "{{subgraphPath}}"
|
||||
|
||||
|
||||
@ -11,9 +11,9 @@ import { Database as BaseDatabase, DatabaseInterface, QueryOptions, StateKind, W
|
||||
import { Contract } from './entity/Contract';
|
||||
import { Event } from './entity/Event';
|
||||
import { SyncStatus } from './entity/SyncStatus';
|
||||
import { IpldStatus } from './entity/IpldStatus';
|
||||
import { StateSyncStatus } from './entity/StateSyncStatus';
|
||||
import { BlockProgress } from './entity/BlockProgress';
|
||||
import { IPLDBlock } from './entity/IPLDBlock';
|
||||
import { State } from './entity/State';
|
||||
{{#each queries as | query |}}
|
||||
import { {{query.entityName}} } from './entity/{{query.entityName}}';
|
||||
{{/each}}
|
||||
@ -79,75 +79,69 @@ export class Database implements DatabaseInterface {
|
||||
}
|
||||
|
||||
{{/each}}
|
||||
getNewIPLDBlock (): IPLDBlock {
|
||||
return new IPLDBlock();
|
||||
getNewState (): State {
|
||||
return new State();
|
||||
}
|
||||
|
||||
async getIPLDBlocks (where: FindConditions<IPLDBlock>): Promise<IPLDBlock[]> {
|
||||
const repo = this._conn.getRepository(IPLDBlock);
|
||||
async getStates (where: FindConditions<State>): Promise<State[]> {
|
||||
const repo = this._conn.getRepository(State);
|
||||
|
||||
return this._baseDatabase.getIPLDBlocks(repo, where);
|
||||
return this._baseDatabase.getStates(repo, where);
|
||||
}
|
||||
|
||||
async getLatestIPLDBlock (contractAddress: string, kind: StateKind | null, blockNumber?: number): Promise<IPLDBlock | undefined> {
|
||||
const repo = this._conn.getRepository(IPLDBlock);
|
||||
async getLatestState (contractAddress: string, kind: StateKind | null, blockNumber?: number): Promise<State | undefined> {
|
||||
const repo = this._conn.getRepository(State);
|
||||
|
||||
return this._baseDatabase.getLatestIPLDBlock(repo, contractAddress, kind, blockNumber);
|
||||
return this._baseDatabase.getLatestState(repo, contractAddress, kind, blockNumber);
|
||||
}
|
||||
|
||||
async getPrevIPLDBlock (blockHash: string, contractAddress: string, kind?: string): Promise<IPLDBlock | undefined> {
|
||||
const repo = this._conn.getRepository(IPLDBlock);
|
||||
async getPrevState (blockHash: string, contractAddress: string, kind?: string): Promise<State | undefined> {
|
||||
const repo = this._conn.getRepository(State);
|
||||
|
||||
return this._baseDatabase.getPrevIPLDBlock(repo, blockHash, contractAddress, kind);
|
||||
return this._baseDatabase.getPrevState(repo, blockHash, contractAddress, kind);
|
||||
}
|
||||
|
||||
// Fetch all diff IPLDBlocks after the specified block number.
|
||||
async getDiffIPLDBlocksInRange (contractAddress: string, startBlock: number, endBlock: number): Promise<IPLDBlock[]> {
|
||||
const repo = this._conn.getRepository(IPLDBlock);
|
||||
// Fetch all diff States after the specified block number.
|
||||
async getDiffStatesInRange (contractAddress: string, startblock: number, endBlock: number): Promise<State[]> {
|
||||
const repo = this._conn.getRepository(State);
|
||||
|
||||
return this._baseDatabase.getDiffIPLDBlocksInRange(repo, contractAddress, startBlock, endBlock);
|
||||
return this._baseDatabase.getDiffStatesInRange(repo, contractAddress, startblock, endBlock);
|
||||
}
|
||||
|
||||
async saveOrUpdateIPLDBlock (dbTx: QueryRunner, ipldBlock: IPLDBlock): Promise<IPLDBlock> {
|
||||
const repo = dbTx.manager.getRepository(IPLDBlock);
|
||||
async saveOrUpdateState (dbTx: QueryRunner, state: State): Promise<State> {
|
||||
const repo = dbTx.manager.getRepository(State);
|
||||
|
||||
return this._baseDatabase.saveOrUpdateIPLDBlock(repo, ipldBlock);
|
||||
return this._baseDatabase.saveOrUpdateState(repo, state);
|
||||
}
|
||||
|
||||
async removeIPLDBlocks (dbTx: QueryRunner, blockNumber: number, kind: string): Promise<void> {
|
||||
const repo = dbTx.manager.getRepository(IPLDBlock);
|
||||
async removeStates (dbTx: QueryRunner, blockNumber: number, kind: string): Promise<void> {
|
||||
const repo = dbTx.manager.getRepository(State);
|
||||
|
||||
await this._baseDatabase.removeIPLDBlocks(repo, blockNumber, kind);
|
||||
await this._baseDatabase.removeStates(repo, blockNumber, kind);
|
||||
}
|
||||
|
||||
async removeIPLDBlocksAfterBlock (dbTx: QueryRunner, blockNumber: number): Promise<void> {
|
||||
const repo = dbTx.manager.getRepository(IPLDBlock);
|
||||
async removeStatesAfterBlock (dbTx: QueryRunner, blockNumber: number): Promise<void> {
|
||||
const repo = dbTx.manager.getRepository(State);
|
||||
|
||||
await this._baseDatabase.removeIPLDBlocksAfterBlock(repo, blockNumber);
|
||||
await this._baseDatabase.removeStatesAfterBlock(repo, blockNumber);
|
||||
}
|
||||
|
||||
async getIPLDStatus (): Promise<IpldStatus | undefined> {
|
||||
const repo = this._conn.getRepository(IpldStatus);
|
||||
async getStateSyncStatus (): Promise<StateSyncStatus | undefined> {
|
||||
const repo = this._conn.getRepository(StateSyncStatus);
|
||||
|
||||
return this._baseDatabase.getIPLDStatus(repo);
|
||||
return this._baseDatabase.getStateSyncStatus(repo);
|
||||
}
|
||||
|
||||
async updateIPLDStatusHooksBlock (queryRunner: QueryRunner, blockNumber: number, force?: boolean): Promise<IpldStatus> {
|
||||
const repo = queryRunner.manager.getRepository(IpldStatus);
|
||||
async updateStateSyncStatusIndexedBlock (queryRunner: QueryRunner, blockNumber: number, force?: boolean): Promise<StateSyncStatus> {
|
||||
const repo = queryRunner.manager.getRepository(StateSyncStatus);
|
||||
|
||||
return this._baseDatabase.updateIPLDStatusHooksBlock(repo, blockNumber, force);
|
||||
return this._baseDatabase.updateStateSyncStatusIndexedBlock(repo, blockNumber, force);
|
||||
}
|
||||
|
||||
async updateIPLDStatusCheckpointBlock (queryRunner: QueryRunner, blockNumber: number, force?: boolean): Promise<IpldStatus> {
|
||||
const repo = queryRunner.manager.getRepository(IpldStatus);
|
||||
async updateStateSyncStatusCheckpointBlock (queryRunner: QueryRunner, blockNumber: number, force?: boolean): Promise<StateSyncStatus> {
|
||||
const repo = queryRunner.manager.getRepository(StateSyncStatus);
|
||||
|
||||
return this._baseDatabase.updateIPLDStatusCheckpointBlock(repo, blockNumber, force);
|
||||
}
|
||||
|
||||
async updateIPLDStatusIPFSBlock (queryRunner: QueryRunner, blockNumber: number, force?: boolean): Promise<IpldStatus> {
|
||||
const repo = queryRunner.manager.getRepository(IpldStatus);
|
||||
|
||||
return this._baseDatabase.updateIPLDStatusIPFSBlock(repo, blockNumber, force);
|
||||
return this._baseDatabase.updateStateSyncStatusCheckpointBlock(repo, blockNumber, force);
|
||||
}
|
||||
|
||||
async getContracts (): Promise<Contract[]> {
|
||||
|
||||
@ -85,18 +85,18 @@ const main = async (): Promise<void> => {
|
||||
const exportData: any = {
|
||||
snapshotBlock: {},
|
||||
contracts: [],
|
||||
ipldCheckpoints: []
|
||||
stateCheckpoints: []
|
||||
};
|
||||
|
||||
const contracts = await db.getContracts();
|
||||
|
||||
// Get latest block with hooks processed.
|
||||
let block = await indexer.getLatestHooksProcessedBlock();
|
||||
let block = await indexer.getLatestStateIndexedBlock();
|
||||
assert(block);
|
||||
|
||||
if (argv.blockNumber) {
|
||||
if (argv.blockNumber > block.blockNumber) {
|
||||
throw new Error(`Export snapshot block height ${argv.blockNumber} should be less than latest hooks processed block height ${block.blockNumber}`);
|
||||
throw new Error(`Export snapshot block height ${argv.blockNumber} should be less than latest state indexed block height ${block.blockNumber}`);
|
||||
}
|
||||
|
||||
const blocksAtSnapshotHeight = await indexer.getBlocksAtHeight(argv.blockNumber, false);
|
||||
@ -133,19 +133,15 @@ const main = async (): Promise<void> => {
|
||||
if (contract.checkpoint) {
|
||||
await indexer.createCheckpoint(contract.address, block.blockHash);
|
||||
|
||||
const ipldBlock = await indexer.getLatestIPLDBlock(contract.address, StateKind.Checkpoint, block.blockNumber);
|
||||
assert(ipldBlock);
|
||||
const state = await indexer.getLatestState(contract.address, StateKind.Checkpoint, block.blockNumber);
|
||||
assert(state);
|
||||
|
||||
const data = indexer.getIPLDData(ipldBlock);
|
||||
const data = indexer.getStateData(state);
|
||||
|
||||
if (indexer.isIPFSConfigured()) {
|
||||
await indexer.pushToIPFS(data);
|
||||
}
|
||||
|
||||
exportData.ipldCheckpoints.push({
|
||||
contractAddress: ipldBlock.contractAddress,
|
||||
cid: ipldBlock.cid,
|
||||
kind: ipldBlock.kind,
|
||||
exportData.stateCheckpoints.push({
|
||||
contractAddress: state.contractAddress,
|
||||
cid: state.cid,
|
||||
kind: state.kind,
|
||||
data
|
||||
});
|
||||
}
|
||||
|
||||
@ -31,7 +31,7 @@ export const fillState = async (
|
||||
log(`Filling state for subgraph entities in range: [${startBlock}, ${endBlock}]`);
|
||||
|
||||
// Check that there are no existing diffs in this range
|
||||
const existingStates = await indexer.getIPLDBlocks({ block: { blockNumber: Between(startBlock, endBlock) } });
|
||||
const existingStates = await indexer.getStates({ block: { blockNumber: Between(startBlock, endBlock) } });
|
||||
if (existingStates.length > 0) {
|
||||
log('found existing state(s) in the given range');
|
||||
process.exit(1);
|
||||
@ -93,12 +93,12 @@ export const fillState = async (
|
||||
|
||||
// Persist subgraph state to the DB
|
||||
await indexer.dumpSubgraphState(blockHash, true);
|
||||
await indexer.updateStateSyncStatusIndexedBlock(blockNumber);
|
||||
|
||||
// Create checkpoints
|
||||
await indexer.processCheckpoint(blockHash);
|
||||
await indexer.updateStateSyncStatusCheckpointBlock(blockNumber);
|
||||
}
|
||||
|
||||
// TODO: Push state to IPFS
|
||||
|
||||
log(`Filled state for subgraph entities in range: [${startBlock}, ${endBlock}]`);
|
||||
};
|
||||
|
||||
@ -20,19 +20,19 @@ export async function createInitialState (indexer: Indexer, contractAddress: str
|
||||
assert(blockHash);
|
||||
assert(contractAddress);
|
||||
|
||||
// Store the desired initial state in an IPLDBlock.
|
||||
const ipldBlockData: any = {
|
||||
// Store an empty State.
|
||||
const stateData: any = {
|
||||
state: {}
|
||||
};
|
||||
|
||||
// Use updateStateForElementaryType to update initial state with an elementary property.
|
||||
// Eg. const ipldBlockData = updateStateForElementaryType(ipldBlockData, '_totalBalance', result.value.toString());
|
||||
// Eg. const stateData = updateStateForElementaryType(stateData, '_totalBalance', result.value.toString());
|
||||
|
||||
// Use updateStateForMappingType to update initial state with a nested property.
|
||||
// Eg. const ipldBlockData = updateStateForMappingType(ipldBlockData, '_allowances', [owner, spender], allowance.value.toString());
|
||||
// Eg. const stateData = updateStateForMappingType(stateData, '_allowances', [owner, spender], allowance.value.toString());
|
||||
|
||||
// Return initial state data to be saved.
|
||||
return ipldBlockData;
|
||||
return stateData;
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@ -20,7 +20,7 @@ import * as codec from '@ipld/dag-cbor';
|
||||
import { Database } from '../database';
|
||||
import { Indexer } from '../indexer';
|
||||
import { EventWatcher } from '../events';
|
||||
import { IPLDBlock } from '../entity/IPLDBlock';
|
||||
import { State } from '../entity/State';
|
||||
|
||||
const log = debug('vulcanize:import-state');
|
||||
|
||||
@ -106,18 +106,18 @@ export const main = async (): Promise<any> => {
|
||||
const block = await indexer.getBlockProgress(importData.snapshotBlock.blockHash);
|
||||
assert(block);
|
||||
|
||||
// Fill the IPLDBlocks.
|
||||
for (const checkpoint of importData.ipldCheckpoints) {
|
||||
let ipldBlock = new IPLDBlock();
|
||||
// Fill the States.
|
||||
for (const checkpoint of importData.stateCheckpoints) {
|
||||
let state = new State();
|
||||
|
||||
ipldBlock = Object.assign(ipldBlock, checkpoint);
|
||||
ipldBlock.block = block;
|
||||
state = Object.assign(state, checkpoint);
|
||||
state.block = block;
|
||||
|
||||
ipldBlock.data = Buffer.from(codec.encode(ipldBlock.data));
|
||||
state.data = Buffer.from(codec.encode(state.data));
|
||||
|
||||
ipldBlock = await indexer.saveOrUpdateIPLDBlock(ipldBlock);
|
||||
state = await indexer.saveOrUpdateState(state);
|
||||
{{#if (subgraphPath)}}
|
||||
await graphWatcher.updateEntitiesFromIPLDState(ipldBlock);
|
||||
await graphWatcher.updateEntitiesFromState(state);
|
||||
{{/if}}
|
||||
}
|
||||
|
||||
@ -126,12 +126,12 @@ export const main = async (): Promise<any> => {
|
||||
await indexer.updateBlockProgress(block, block.lastProcessedEventIndex);
|
||||
await indexer.updateSyncStatusChainHead(block.blockHash, block.blockNumber);
|
||||
await indexer.updateSyncStatusIndexedBlock(block.blockHash, block.blockNumber);
|
||||
await indexer.updateIPLDStatusHooksBlock(block.blockNumber);
|
||||
await indexer.updateIPLDStatusCheckpointBlock(block.blockNumber);
|
||||
await indexer.updateStateSyncStatusIndexedBlock(block.blockNumber);
|
||||
await indexer.updateStateSyncStatusCheckpointBlock(block.blockNumber);
|
||||
|
||||
// The 'diff_staged' and 'init' IPLD blocks are unnecessary as checkpoints have been already created for the snapshot block.
|
||||
await indexer.removeIPLDBlocks(block.blockNumber, StateKind.Init);
|
||||
await indexer.removeIPLDBlocks(block.blockNumber, StateKind.DiffStaged);
|
||||
// The 'diff_staged' and 'init' State entries are unnecessary as checkpoints have been already created for the snapshot block.
|
||||
await indexer.removeStates(block.blockNumber, StateKind.Init);
|
||||
await indexer.removeStates(block.blockNumber, StateKind.DiffStaged);
|
||||
|
||||
log(`Import completed for snapshot block at height ${block.blockNumber}`);
|
||||
};
|
||||
|
||||
@ -8,11 +8,12 @@ import { DeepPartial, FindConditions, FindManyOptions } from 'typeorm';
|
||||
import JSONbig from 'json-bigint';
|
||||
import { ethers } from 'ethers';
|
||||
import _ from 'lodash';
|
||||
{{#if (subgraphPath)}}
|
||||
import { SelectionNode } from 'graphql';
|
||||
{{/if}}
|
||||
|
||||
import { JsonFragment } from '@ethersproject/abi';
|
||||
import { BaseProvider } from '@ethersproject/providers';
|
||||
import * as codec from '@ipld/dag-cbor';
|
||||
import { EthClient } from '@cerc-io/ipld-eth-client';
|
||||
import { MappingKey, StorageLayout } from '@cerc-io/solidity-mapper';
|
||||
import {
|
||||
@ -29,10 +30,8 @@ import {
|
||||
{{#if (subgraphPath)}}
|
||||
BlockHeight,
|
||||
{{/if}}
|
||||
IPFSClient,
|
||||
StateKind,
|
||||
IpldStatus as IpldStatusInterface,
|
||||
ResultIPLDBlock
|
||||
StateStatus
|
||||
} from '@cerc-io/util';
|
||||
{{#if (subgraphPath)}}
|
||||
import { GraphWatcher } from '@cerc-io/graph-node';
|
||||
@ -46,9 +45,9 @@ import { createInitialState, handleEvent, createStateDiff, createStateCheckpoint
|
||||
import { Contract } from './entity/Contract';
|
||||
import { Event } from './entity/Event';
|
||||
import { SyncStatus } from './entity/SyncStatus';
|
||||
import { IpldStatus } from './entity/IpldStatus';
|
||||
import { StateSyncStatus } from './entity/StateSyncStatus';
|
||||
import { BlockProgress } from './entity/BlockProgress';
|
||||
import { IPLDBlock } from './entity/IPLDBlock';
|
||||
import { State } from './entity/State';
|
||||
|
||||
{{#each subgraphEntities as | subgraphEntity |}}
|
||||
import { {{subgraphEntity.className}} } from './entity/{{subgraphEntity.className}}';
|
||||
@ -103,8 +102,6 @@ export class Indexer implements IndexerInterface {
|
||||
_storageLayoutMap: Map<string, StorageLayout>
|
||||
_contractMap: Map<string, ethers.utils.Interface>
|
||||
|
||||
_ipfsClient: IPFSClient
|
||||
|
||||
{{#if (subgraphPath)}}
|
||||
_entityTypesMap: Map<string, { [key: string]: string }>
|
||||
_relationsMap: Map<any, { [key: string]: any }>
|
||||
@ -120,8 +117,7 @@ export class Indexer implements IndexerInterface {
|
||||
this._ethClient = ethClient;
|
||||
this._ethProvider = ethProvider;
|
||||
this._serverConfig = serverConfig;
|
||||
this._ipfsClient = new IPFSClient(this._serverConfig.ipfsApiAddr);
|
||||
this._baseIndexer = new BaseIndexer(this._serverConfig, this._db, this._ethClient, this._ethProvider, jobQueue, this._ipfsClient);
|
||||
this._baseIndexer = new BaseIndexer(this._serverConfig, this._db, this._ethClient, this._ethProvider, jobQueue);
|
||||
{{#if (subgraphPath)}}
|
||||
this._graphWatcher = graphWatcher;
|
||||
{{/if}}
|
||||
@ -170,7 +166,7 @@ export class Indexer implements IndexerInterface {
|
||||
|
||||
async init (): Promise<void> {
|
||||
await this._baseIndexer.fetchContracts();
|
||||
await this._baseIndexer.fetchIPLDStatus();
|
||||
await this._baseIndexer.fetchStateStatus();
|
||||
}
|
||||
|
||||
getResultEvent (event: Event): ResultEvent {
|
||||
@ -208,26 +204,6 @@ export class Indexer implements IndexerInterface {
|
||||
};
|
||||
}
|
||||
|
||||
getResultIPLDBlock (ipldBlock: IPLDBlock): ResultIPLDBlock {
|
||||
const block = ipldBlock.block;
|
||||
|
||||
const data = codec.decode(Buffer.from(ipldBlock.data)) as any;
|
||||
|
||||
return {
|
||||
block: {
|
||||
cid: block.cid,
|
||||
hash: block.blockHash,
|
||||
number: block.blockNumber,
|
||||
timestamp: block.blockTimestamp,
|
||||
parentHash: block.parentHash
|
||||
},
|
||||
contractAddress: ipldBlock.contractAddress,
|
||||
cid: ipldBlock.cid,
|
||||
kind: ipldBlock.kind,
|
||||
data: JSON.stringify(data)
|
||||
};
|
||||
}
|
||||
|
||||
{{#each queries as | query |}}
|
||||
async {{query.name}} (blockHash: string, contractAddress: string
|
||||
{{~#each query.params}}, {{this.name~}}: {{this.type~}} {{/each}}
|
||||
@ -324,10 +300,6 @@ export class Indexer implements IndexerInterface {
|
||||
);
|
||||
}
|
||||
|
||||
async pushToIPFS (data: any): Promise<void> {
|
||||
await this._baseIndexer.pushToIPFS(data);
|
||||
}
|
||||
|
||||
async processInitialState (contractAddress: string, blockHash: string): Promise<any> {
|
||||
// Call initial state hook.
|
||||
return createInitialState(this, contractAddress, blockHash);
|
||||
@ -362,32 +334,28 @@ export class Indexer implements IndexerInterface {
|
||||
return this._baseIndexer.processCLICheckpoint(this, contractAddress, blockHash);
|
||||
}
|
||||
|
||||
async getPrevIPLDBlock (blockHash: string, contractAddress: string, kind?: string): Promise<IPLDBlock | undefined> {
|
||||
return this._db.getPrevIPLDBlock(blockHash, contractAddress, kind);
|
||||
async getPrevState (blockHash: string, contractAddress: string, kind?: string): Promise<State | undefined> {
|
||||
return this._db.getPrevState(blockHash, contractAddress, kind);
|
||||
}
|
||||
|
||||
async getLatestIPLDBlock (contractAddress: string, kind: StateKind | null, blockNumber?: number): Promise<IPLDBlock | undefined> {
|
||||
return this._db.getLatestIPLDBlock(contractAddress, kind, blockNumber);
|
||||
async getLatestState (contractAddress: string, kind: StateKind | null, blockNumber?: number): Promise<State | undefined> {
|
||||
return this._db.getLatestState(contractAddress, kind, blockNumber);
|
||||
}
|
||||
|
||||
async getIPLDBlocksByHash (blockHash: string): Promise<IPLDBlock[]> {
|
||||
return this._baseIndexer.getIPLDBlocksByHash(blockHash);
|
||||
async getStatesByHash (blockHash: string): Promise<State[]> {
|
||||
return this._baseIndexer.getStatesByHash(blockHash);
|
||||
}
|
||||
|
||||
async getIPLDBlockByCid (cid: string): Promise<IPLDBlock | undefined> {
|
||||
return this._baseIndexer.getIPLDBlockByCid(cid);
|
||||
async getStateByCID (cid: string): Promise<State | undefined> {
|
||||
return this._baseIndexer.getStateByCID(cid);
|
||||
}
|
||||
|
||||
async getIPLDBlocks (where: FindConditions<IPLDBlock>): Promise<IPLDBlock[]> {
|
||||
return this._db.getIPLDBlocks(where);
|
||||
async getStates (where: FindConditions<State>): Promise<State[]> {
|
||||
return this._db.getStates(where);
|
||||
}
|
||||
|
||||
getIPLDData (ipldBlock: IPLDBlock): any {
|
||||
return this._baseIndexer.getIPLDData(ipldBlock);
|
||||
}
|
||||
|
||||
isIPFSConfigured (): boolean {
|
||||
return this._baseIndexer.isIPFSConfigured();
|
||||
getStateData (state: State): any {
|
||||
return this._baseIndexer.getStateData(state);
|
||||
}
|
||||
|
||||
// Method used to create auto diffs (diff_staged).
|
||||
@ -425,12 +393,12 @@ export class Indexer implements IndexerInterface {
|
||||
await this._baseIndexer.createInit(this, blockHash, blockNumber);
|
||||
}
|
||||
|
||||
async saveOrUpdateIPLDBlock (ipldBlock: IPLDBlock): Promise<IPLDBlock> {
|
||||
return this._baseIndexer.saveOrUpdateIPLDBlock(ipldBlock);
|
||||
async saveOrUpdateState (state: State): Promise<State> {
|
||||
return this._baseIndexer.saveOrUpdateState(state);
|
||||
}
|
||||
|
||||
async removeIPLDBlocks (blockNumber: number, kind: StateKind): Promise<void> {
|
||||
await this._baseIndexer.removeIPLDBlocks(blockNumber, kind);
|
||||
async removeStates (blockNumber: number, kind: StateKind): Promise<void> {
|
||||
await this._baseIndexer.removeStates(blockNumber, kind);
|
||||
}
|
||||
|
||||
{{#if (subgraphPath)}}
|
||||
@ -466,8 +434,8 @@ export class Indexer implements IndexerInterface {
|
||||
async processBlock (blockProgress: BlockProgress): Promise<void> {
|
||||
// Call a function to create initial state for contracts.
|
||||
await this._baseIndexer.createInit(this, blockProgress.blockHash, blockProgress.blockNumber);
|
||||
|
||||
{{#if (subgraphPath)}}
|
||||
|
||||
this._graphWatcher.updateEntityCacheFrothyBlocks(blockProgress);
|
||||
{{/if}}
|
||||
}
|
||||
@ -499,16 +467,16 @@ export class Indexer implements IndexerInterface {
|
||||
};
|
||||
}
|
||||
|
||||
async getIPLDStatus (): Promise<IpldStatus | undefined> {
|
||||
return this._db.getIPLDStatus();
|
||||
async getStateSyncStatus (): Promise<StateSyncStatus | undefined> {
|
||||
return this._db.getStateSyncStatus();
|
||||
}
|
||||
|
||||
async updateIPLDStatusHooksBlock (blockNumber: number, force?: boolean): Promise<IpldStatus> {
|
||||
async updateStateSyncStatusIndexedBlock (blockNumber: number, force?: boolean): Promise<StateSyncStatus> {
|
||||
const dbTx = await this._db.createTransactionRunner();
|
||||
let res;
|
||||
|
||||
try {
|
||||
res = await this._db.updateIPLDStatusHooksBlock(dbTx, blockNumber, force);
|
||||
res = await this._db.updateStateSyncStatusIndexedBlock(dbTx, blockNumber, force);
|
||||
await dbTx.commitTransaction();
|
||||
} catch (error) {
|
||||
await dbTx.rollbackTransaction();
|
||||
@ -520,29 +488,12 @@ export class Indexer implements IndexerInterface {
|
||||
return res;
|
||||
}
|
||||
|
||||
async updateIPLDStatusCheckpointBlock (blockNumber: number, force?: boolean): Promise<IpldStatus> {
|
||||
async updateStateSyncStatusCheckpointBlock (blockNumber: number, force?: boolean): Promise<StateSyncStatus> {
|
||||
const dbTx = await this._db.createTransactionRunner();
|
||||
let res;
|
||||
|
||||
try {
|
||||
res = await this._db.updateIPLDStatusCheckpointBlock(dbTx, blockNumber, force);
|
||||
await dbTx.commitTransaction();
|
||||
} catch (error) {
|
||||
await dbTx.rollbackTransaction();
|
||||
throw error;
|
||||
} finally {
|
||||
await dbTx.release();
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
async updateIPLDStatusIPFSBlock (blockNumber: number, force?: boolean): Promise<IpldStatus> {
|
||||
const dbTx = await this._db.createTransactionRunner();
|
||||
let res;
|
||||
|
||||
try {
|
||||
res = await this._db.updateIPLDStatusIPFSBlock(dbTx, blockNumber, force);
|
||||
res = await this._db.updateStateSyncStatusCheckpointBlock(dbTx, blockNumber, force);
|
||||
await dbTx.commitTransaction();
|
||||
} catch (error) {
|
||||
await dbTx.rollbackTransaction();
|
||||
@ -564,16 +515,16 @@ export class Indexer implements IndexerInterface {
|
||||
return latestCanonicalBlock;
|
||||
}
|
||||
|
||||
async getLatestHooksProcessedBlock (): Promise<BlockProgress> {
|
||||
return this._baseIndexer.getLatestHooksProcessedBlock();
|
||||
async getLatestStateIndexedBlock (): Promise<BlockProgress> {
|
||||
return this._baseIndexer.getLatestStateIndexedBlock();
|
||||
}
|
||||
|
||||
async watchContract (address: string, kind: string, checkpoint: boolean, startingBlock: number): Promise<void> {
|
||||
return this._baseIndexer.watchContract(address, kind, checkpoint, startingBlock);
|
||||
}
|
||||
|
||||
async updateIPLDStatusMap (address: string, ipldStatus: IpldStatusInterface): Promise<void> {
|
||||
await this._baseIndexer.updateIPLDStatusMap(address, ipldStatus);
|
||||
updateStateStatusMap (address: string, stateStatus: StateStatus): void {
|
||||
this._baseIndexer.updateStateStatusMap(address, stateStatus);
|
||||
}
|
||||
|
||||
cacheContract (contract: Contract): void {
|
||||
|
||||
@ -71,12 +71,12 @@ const main = async (): Promise<void> => {
|
||||
await graphWatcher.init();
|
||||
{{/if}}
|
||||
|
||||
const ipldBlock = await indexer.getIPLDBlockByCid(argv.cid);
|
||||
assert(ipldBlock, 'IPLDBlock for the provided CID doesn\'t exist.');
|
||||
const state = await indexer.getStateByCID(argv.cid);
|
||||
assert(state, 'State for the provided CID doesn\'t exist.');
|
||||
|
||||
const ipldData = await indexer.getIPLDData(ipldBlock);
|
||||
const stateData = await indexer.getStateData(state);
|
||||
|
||||
log(util.inspect(ipldData, false, null));
|
||||
log(util.inspect(stateData, false, null));
|
||||
};
|
||||
|
||||
main().catch(err => {
|
||||
|
||||
@ -20,7 +20,6 @@ import {
|
||||
QUEUE_EVENT_PROCESSING,
|
||||
QUEUE_BLOCK_CHECKPOINT,
|
||||
QUEUE_HOOKS,
|
||||
QUEUE_IPFS,
|
||||
JOB_KIND_PRUNE,
|
||||
JobQueueConfig,
|
||||
DEFAULT_CONFIG_PATH,
|
||||
@ -54,21 +53,11 @@ export class JobRunner {
|
||||
await this.subscribeEventProcessingQueue();
|
||||
await this.subscribeBlockCheckpointQueue();
|
||||
await this.subscribeHooksQueue();
|
||||
await this.subscribeIPFSQueue();
|
||||
}
|
||||
|
||||
async subscribeBlockProcessingQueue (): Promise<void> {
|
||||
await this._jobQueue.subscribe(QUEUE_BLOCK_PROCESSING, async (job) => {
|
||||
await this._baseJobRunner.processBlock(job);
|
||||
|
||||
const { data: { kind } } = job;
|
||||
|
||||
// If it's a pruning job: Create a hooks job.
|
||||
if (kind === JOB_KIND_PRUNE) {
|
||||
await this.createHooksJob();
|
||||
}
|
||||
|
||||
await this._jobQueue.markComplete(job);
|
||||
});
|
||||
}
|
||||
|
||||
@ -80,165 +69,15 @@ export class JobRunner {
|
||||
|
||||
async subscribeHooksQueue (): Promise<void> {
|
||||
await this._jobQueue.subscribe(QUEUE_HOOKS, async (job) => {
|
||||
const { data: { blockHash, blockNumber } } = job;
|
||||
|
||||
// Get the current IPLD Status.
|
||||
const ipldStatus = await this._indexer.getIPLDStatus();
|
||||
|
||||
if (ipldStatus) {
|
||||
if (ipldStatus.latestHooksBlockNumber < (blockNumber - 1)) {
|
||||
// Create hooks job for parent block.
|
||||
const [parentBlock] = await this._indexer.getBlocksAtHeight(blockNumber - 1, false);
|
||||
await this.createHooksJob(parentBlock.blockHash, parentBlock.blockNumber);
|
||||
|
||||
const message = `Hooks for blockNumber ${blockNumber - 1} not processed yet, aborting`;
|
||||
log(message);
|
||||
|
||||
throw new Error(message);
|
||||
}
|
||||
|
||||
if (ipldStatus.latestHooksBlockNumber > (blockNumber - 1)) {
|
||||
log(`Hooks for blockNumber ${blockNumber} already processed`);
|
||||
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// Process the hooks for the given block number.
|
||||
await this._indexer.processCanonicalBlock(blockHash, blockNumber);
|
||||
|
||||
// Update the IPLD status.
|
||||
await this._indexer.updateIPLDStatusHooksBlock(blockNumber);
|
||||
|
||||
// Create a checkpoint job after completion of a hook job.
|
||||
await this.createCheckpointJob(blockHash, blockNumber);
|
||||
|
||||
await this._jobQueue.markComplete(job);
|
||||
await this._baseJobRunner.processHooks(job);
|
||||
});
|
||||
}
|
||||
|
||||
async subscribeBlockCheckpointQueue (): Promise<void> {
|
||||
await this._jobQueue.subscribe(QUEUE_BLOCK_CHECKPOINT, async (job) => {
|
||||
const { data: { blockHash, blockNumber } } = job;
|
||||
|
||||
// Get the current IPLD Status.
|
||||
const ipldStatus = await this._indexer.getIPLDStatus();
|
||||
assert(ipldStatus);
|
||||
|
||||
if (ipldStatus.latestCheckpointBlockNumber >= 0) {
|
||||
if (ipldStatus.latestCheckpointBlockNumber < (blockNumber - 1)) {
|
||||
// Create a checkpoint job for parent block.
|
||||
const [parentBlock] = await this._indexer.getBlocksAtHeight(blockNumber - 1, false);
|
||||
await this.createCheckpointJob(parentBlock.blockHash, parentBlock.blockNumber);
|
||||
|
||||
const message = `Checkpoints for blockNumber ${blockNumber - 1} not processed yet, aborting`;
|
||||
log(message);
|
||||
|
||||
throw new Error(message);
|
||||
}
|
||||
|
||||
if (ipldStatus.latestCheckpointBlockNumber > (blockNumber - 1)) {
|
||||
log(`Checkpoints for blockNumber ${blockNumber} already processed`);
|
||||
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// Process checkpoints for the given block.
|
||||
await this._indexer.processCheckpoint(blockHash);
|
||||
|
||||
// Update the IPLD status.
|
||||
await this._indexer.updateIPLDStatusCheckpointBlock(blockNumber);
|
||||
|
||||
// Create an IPFS job after completion of a checkpoint job.
|
||||
if (this._indexer.isIPFSConfigured()) {
|
||||
await this.createIPFSPutJob(blockHash, blockNumber);
|
||||
}
|
||||
|
||||
await this._jobQueue.markComplete(job);
|
||||
await this._baseJobRunner.processCheckpoint(job);
|
||||
});
|
||||
}
|
||||
|
||||
async subscribeIPFSQueue (): Promise<void> {
|
||||
await this._jobQueue.subscribe(QUEUE_IPFS, async (job) => {
|
||||
const { data: { blockHash, blockNumber } } = job;
|
||||
|
||||
const ipldStatus = await this._indexer.getIPLDStatus();
|
||||
assert(ipldStatus);
|
||||
|
||||
if (ipldStatus.latestIPFSBlockNumber >= 0) {
|
||||
if (ipldStatus.latestIPFSBlockNumber < (blockNumber - 1)) {
|
||||
// Create a IPFS job for parent block.
|
||||
const [parentBlock] = await this._indexer.getBlocksAtHeight(blockNumber - 1, false);
|
||||
await this.createIPFSPutJob(parentBlock.blockHash, parentBlock.blockNumber);
|
||||
|
||||
const message = `IPFS for blockNumber ${blockNumber - 1} not processed yet, aborting`;
|
||||
log(message);
|
||||
|
||||
throw new Error(message);
|
||||
}
|
||||
|
||||
if (ipldStatus.latestIPFSBlockNumber > (blockNumber - 1)) {
|
||||
log(`IPFS for blockNumber ${blockNumber} already processed`);
|
||||
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// Get IPLDBlocks for the given blocHash.
|
||||
const ipldBlocks = await this._indexer.getIPLDBlocksByHash(blockHash);
|
||||
|
||||
// Push all the IPLDBlocks to IPFS.
|
||||
for (const ipldBlock of ipldBlocks) {
|
||||
const data = this._indexer.getIPLDData(ipldBlock);
|
||||
await this._indexer.pushToIPFS(data);
|
||||
}
|
||||
|
||||
// Update the IPLD status.
|
||||
await this._indexer.updateIPLDStatusIPFSBlock(blockNumber);
|
||||
|
||||
await this._jobQueue.markComplete(job);
|
||||
});
|
||||
}
|
||||
|
||||
async createHooksJob (blockHash?: string, blockNumber?: number): Promise<void> {
|
||||
if (!blockNumber || !blockHash) {
|
||||
// Get the latest canonical block
|
||||
const latestCanonicalBlock = await this._indexer.getLatestCanonicalBlock();
|
||||
|
||||
// Create a hooks job for parent block of latestCanonicalBlock because pruning for first block is skipped as it is assumed to be a canonical block.
|
||||
blockHash = latestCanonicalBlock.parentHash;
|
||||
blockNumber = latestCanonicalBlock.blockNumber - 1;
|
||||
}
|
||||
|
||||
await this._jobQueue.pushJob(
|
||||
QUEUE_HOOKS,
|
||||
{
|
||||
blockHash,
|
||||
blockNumber
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
async createCheckpointJob (blockHash: string, blockNumber: number): Promise<void> {
|
||||
await this._jobQueue.pushJob(
|
||||
QUEUE_BLOCK_CHECKPOINT,
|
||||
{
|
||||
blockHash,
|
||||
blockNumber
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
async createIPFSPutJob (blockHash: string, blockNumber: number): Promise<void> {
|
||||
await this._jobQueue.pushJob(
|
||||
QUEUE_IPFS,
|
||||
{
|
||||
blockHash,
|
||||
blockNumber
|
||||
}
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export const main = async (): Promise<any> => {
|
||||
|
||||
@ -8,12 +8,6 @@
|
||||
yarn
|
||||
```
|
||||
|
||||
* Run the IPFS (go-ipfs version 0.12.2) daemon:
|
||||
|
||||
```bash
|
||||
ipfs daemon
|
||||
```
|
||||
|
||||
* Create a postgres12 database for the watcher:
|
||||
|
||||
```bash
|
||||
@ -47,7 +41,7 @@
|
||||
|
||||
* Update the `upstream` config and provide the `ipld-eth-server` GQL API endpoint.
|
||||
|
||||
* Update the `server` config with state checkpoint settings and provide the IPFS API address.
|
||||
* Update the `server` config with state checkpoint settings.
|
||||
|
||||
## Customize
|
||||
|
||||
@ -59,11 +53,11 @@
|
||||
|
||||
* Generating state:
|
||||
|
||||
* Edit the custom hook function `createInitialState` (triggered if the watcher passes the start block, checkpoint: `true`) in [hooks.ts](./src/hooks.ts) to save an initial state `IPLDBlock` using the `Indexer` object.
|
||||
* Edit the custom hook function `createInitialState` (triggered if the watcher passes the start block, checkpoint: `true`) in [hooks.ts](./src/hooks.ts) to save an initial `State` using the `Indexer` object.
|
||||
|
||||
* Edit the custom hook function `createStateDiff` (triggered on a block) in [hooks.ts](./src/hooks.ts) to save the state in a `diff` `IPLDBlock` using the `Indexer` object. The default state (if exists) is updated.
|
||||
* Edit the custom hook function `createStateDiff` (triggered on a block) in [hooks.ts](./src/hooks.ts) to save the state in a `diff` `State` using the `Indexer` object. The default state (if exists) is updated.
|
||||
|
||||
* Edit the custom hook function `createStateCheckpoint` (triggered just before default and CLI checkpoint) in [hooks.ts](./src/hooks.ts) to save the state in a `checkpoint` `IPLDBlock` using the `Indexer` object.
|
||||
* Edit the custom hook function `createStateCheckpoint` (triggered just before default and CLI checkpoint) in [hooks.ts](./src/hooks.ts) to save the state in a `checkpoint` `State` using the `Indexer` object.
|
||||
|
||||
## Run
|
||||
|
||||
@ -134,14 +128,14 @@ GQL console: http://localhost:{{port}}/graphql
|
||||
```
|
||||
|
||||
`cid`: CID of the checkpoint for which to verify.
|
||||
|
||||
|
||||
{{/if}}
|
||||
* To reset the watcher to a previous block number:
|
||||
|
||||
* Reset state:
|
||||
* Reset watcher:
|
||||
|
||||
```bash
|
||||
yarn reset state --block-number <previous-block-number>
|
||||
yarn reset watcher --block-number <previous-block-number>
|
||||
```
|
||||
|
||||
* Reset job-queue:
|
||||
@ -150,6 +144,12 @@ GQL console: http://localhost:{{port}}/graphql
|
||||
yarn reset job-queue --block-number <previous-block-number>
|
||||
```
|
||||
|
||||
* Reset state:
|
||||
|
||||
```bash
|
||||
yarn reset state --block-number <previous-block-number>
|
||||
```
|
||||
|
||||
* `block-number`: Block number to which to reset the watcher.
|
||||
|
||||
* To export and import the watcher state:
|
||||
|
||||
@ -1,66 +0,0 @@
|
||||
//
|
||||
// Copyright 2022 Vulcanize, Inc.
|
||||
//
|
||||
|
||||
import debug from 'debug';
|
||||
|
||||
import { getConfig } from '@cerc-io/util';
|
||||
|
||||
import { Database } from '../../database';
|
||||
|
||||
const log = debug('vulcanize:reset-ipld-state');
|
||||
|
||||
export const command = 'ipld-state';
|
||||
|
||||
export const desc = 'Reset IPLD state in the given range';
|
||||
|
||||
export const builder = {
|
||||
blockNumber: {
|
||||
type: 'number'
|
||||
}
|
||||
};
|
||||
|
||||
export const handler = async (argv: any): Promise<void> => {
|
||||
const { blockNumber } = argv;
|
||||
const config = await getConfig(argv.configFile);
|
||||
|
||||
// Initialize database
|
||||
const db = new Database(config.database);
|
||||
await db.init();
|
||||
|
||||
// Create a DB transaction
|
||||
const dbTx = await db.createTransactionRunner();
|
||||
|
||||
console.time('time:reset-ipld-state');
|
||||
try {
|
||||
// Delete all IPLDBlock entries in the given range
|
||||
await db.removeIPLDBlocksAfterBlock(dbTx, blockNumber);
|
||||
|
||||
// Reset the IPLD status.
|
||||
const ipldStatus = await db.getIPLDStatus();
|
||||
|
||||
if (ipldStatus) {
|
||||
if (ipldStatus.latestHooksBlockNumber > blockNumber) {
|
||||
await db.updateIPLDStatusHooksBlock(dbTx, blockNumber, true);
|
||||
}
|
||||
|
||||
if (ipldStatus.latestCheckpointBlockNumber > blockNumber) {
|
||||
await db.updateIPLDStatusCheckpointBlock(dbTx, blockNumber, true);
|
||||
}
|
||||
|
||||
if (ipldStatus.latestIPFSBlockNumber > blockNumber) {
|
||||
await db.updateIPLDStatusIPFSBlock(dbTx, blockNumber, true);
|
||||
}
|
||||
}
|
||||
|
||||
dbTx.commitTransaction();
|
||||
} catch (error) {
|
||||
await dbTx.rollbackTransaction();
|
||||
throw error;
|
||||
} finally {
|
||||
await dbTx.release();
|
||||
}
|
||||
console.timeEnd('time:reset-ipld-state');
|
||||
|
||||
log(`Reset ipld-state successfully to block ${blockNumber}`);
|
||||
};
|
||||
@ -1,32 +1,18 @@
|
||||
//
|
||||
// Copyright 2021 Vulcanize, Inc.
|
||||
// Copyright 2022 Vulcanize, Inc.
|
||||
//
|
||||
|
||||
{{#if (subgraphPath)}}
|
||||
import path from 'path';
|
||||
{{/if}}
|
||||
import debug from 'debug';
|
||||
import { MoreThan } from 'typeorm';
|
||||
import assert from 'assert';
|
||||
|
||||
import { getConfig, initClients, resetJobs, JobQueue } from '@cerc-io/util';
|
||||
{{#if (subgraphPath)}}
|
||||
import { GraphWatcher, Database as GraphDatabase } from '@cerc-io/graph-node';
|
||||
{{/if}}
|
||||
import { getConfig } from '@cerc-io/util';
|
||||
|
||||
import { Database } from '../../database';
|
||||
import { Indexer } from '../../indexer';
|
||||
import { BlockProgress } from '../../entity/BlockProgress';
|
||||
|
||||
{{#each queries as | query |}}
|
||||
import { {{query.entityName}} } from '../../entity/{{query.entityName}}';
|
||||
{{/each}}
|
||||
|
||||
const log = debug('vulcanize:reset-state');
|
||||
|
||||
export const command = 'state';
|
||||
|
||||
export const desc = 'Reset state to block number';
|
||||
export const desc = 'Reset State to a given block number';
|
||||
|
||||
export const builder = {
|
||||
blockNumber: {
|
||||
@ -35,87 +21,34 @@ export const builder = {
|
||||
};
|
||||
|
||||
export const handler = async (argv: any): Promise<void> => {
|
||||
const { blockNumber } = argv;
|
||||
const config = await getConfig(argv.configFile);
|
||||
await resetJobs(config);
|
||||
const { ethClient, ethProvider } = await initClients(config);
|
||||
|
||||
// Initialize database.
|
||||
// Initialize database
|
||||
const db = new Database(config.database);
|
||||
await db.init();
|
||||
{{#if (subgraphPath)}}
|
||||
|
||||
const graphDb = new GraphDatabase(config.database, path.resolve(__dirname, '../../entity/*'));
|
||||
await graphDb.init();
|
||||
|
||||
const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);
|
||||
{{/if}}
|
||||
|
||||
const jobQueueConfig = config.jobQueue;
|
||||
assert(jobQueueConfig, 'Missing job queue config');
|
||||
|
||||
const { dbConnectionString, maxCompletionLagInSecs } = jobQueueConfig;
|
||||
assert(dbConnectionString, 'Missing job queue db connection string');
|
||||
|
||||
const jobQueue = new JobQueue({ dbConnectionString, maxCompletionLag: maxCompletionLagInSecs });
|
||||
await jobQueue.start();
|
||||
|
||||
const indexer = new Indexer(config.server, db, ethClient, ethProvider, jobQueue{{#if (subgraphPath)}}, graphWatcher{{/if}});
|
||||
await indexer.init();
|
||||
{{#if (subgraphPath)}}
|
||||
|
||||
graphWatcher.setIndexer(indexer);
|
||||
await graphWatcher.init();
|
||||
{{/if}}
|
||||
|
||||
const blockProgresses = await indexer.getBlocksAtHeight(argv.blockNumber, false);
|
||||
assert(blockProgresses.length, `No blocks at specified block number ${argv.blockNumber}`);
|
||||
assert(!blockProgresses.some(block => !block.isComplete), `Incomplete block at block number ${argv.blockNumber} with unprocessed events`);
|
||||
const [blockProgress] = blockProgresses;
|
||||
|
||||
// Create a DB transaction
|
||||
const dbTx = await db.createTransactionRunner();
|
||||
|
||||
console.time('time:reset-state');
|
||||
try {
|
||||
const entities = [BlockProgress
|
||||
{{~#each queries as | query |~}}
|
||||
, {{query.entityName}}
|
||||
{{~/each~}}
|
||||
];
|
||||
// Delete all State entries in the given range
|
||||
await db.removeStatesAfterBlock(dbTx, blockNumber);
|
||||
|
||||
const removeEntitiesPromise = entities.map(async entityClass => {
|
||||
return db.removeEntities<any>(dbTx, entityClass, { blockNumber: MoreThan(argv.blockNumber) });
|
||||
});
|
||||
// Reset the stateSyncStatus.
|
||||
const stateSyncStatus = await db.getStateSyncStatus();
|
||||
|
||||
await Promise.all(removeEntitiesPromise);
|
||||
|
||||
const syncStatus = await indexer.getSyncStatus();
|
||||
assert(syncStatus, 'Missing syncStatus');
|
||||
|
||||
if (syncStatus.latestIndexedBlockNumber > blockProgress.blockNumber) {
|
||||
await indexer.updateSyncStatusIndexedBlock(blockProgress.blockHash, blockProgress.blockNumber, true);
|
||||
}
|
||||
|
||||
if (syncStatus.latestCanonicalBlockNumber > blockProgress.blockNumber) {
|
||||
await indexer.updateSyncStatusCanonicalBlock(blockProgress.blockHash, blockProgress.blockNumber, true);
|
||||
}
|
||||
|
||||
const ipldStatus = await indexer.getIPLDStatus();
|
||||
|
||||
if (ipldStatus) {
|
||||
if (ipldStatus.latestHooksBlockNumber > blockProgress.blockNumber) {
|
||||
await indexer.updateIPLDStatusHooksBlock(blockProgress.blockNumber, true);
|
||||
if (stateSyncStatus) {
|
||||
if (stateSyncStatus.latestIndexedBlockNumber > blockNumber) {
|
||||
await db.updateStateSyncStatusIndexedBlock(dbTx, blockNumber, true);
|
||||
}
|
||||
|
||||
if (ipldStatus.latestCheckpointBlockNumber > blockProgress.blockNumber) {
|
||||
await indexer.updateIPLDStatusCheckpointBlock(blockProgress.blockNumber, true);
|
||||
}
|
||||
|
||||
if (ipldStatus.latestIPFSBlockNumber > blockProgress.blockNumber) {
|
||||
await indexer.updateIPLDStatusIPFSBlock(blockProgress.blockNumber, true);
|
||||
if (stateSyncStatus.latestCheckpointBlockNumber > blockNumber) {
|
||||
await db.updateStateSyncStatusCheckpointBlock(dbTx, blockNumber, true);
|
||||
}
|
||||
}
|
||||
|
||||
await indexer.updateSyncStatusChainHead(blockProgress.blockHash, blockProgress.blockNumber, true);
|
||||
|
||||
dbTx.commitTransaction();
|
||||
} catch (error) {
|
||||
await dbTx.rollbackTransaction();
|
||||
@ -123,6 +56,7 @@ export const handler = async (argv: any): Promise<void> => {
|
||||
} finally {
|
||||
await dbTx.release();
|
||||
}
|
||||
console.timeEnd('time:reset-state');
|
||||
|
||||
log('Reset state successfully');
|
||||
log(`Reset State successfully to block ${blockNumber}`);
|
||||
};
|
||||
|
||||
124
packages/codegen/src/templates/reset-watcher-template.handlebars
Normal file
124
packages/codegen/src/templates/reset-watcher-template.handlebars
Normal file
@ -0,0 +1,124 @@
|
||||
//
|
||||
// Copyright 2021 Vulcanize, Inc.
|
||||
//
|
||||
|
||||
{{#if (subgraphPath)}}
|
||||
import path from 'path';
|
||||
{{/if}}
|
||||
import debug from 'debug';
|
||||
import { MoreThan } from 'typeorm';
|
||||
import assert from 'assert';
|
||||
|
||||
import { getConfig, initClients, resetJobs, JobQueue } from '@cerc-io/util';
|
||||
{{#if (subgraphPath)}}
|
||||
import { GraphWatcher, Database as GraphDatabase } from '@cerc-io/graph-node';
|
||||
{{/if}}
|
||||
|
||||
import { Database } from '../../database';
|
||||
import { Indexer } from '../../indexer';
|
||||
import { BlockProgress } from '../../entity/BlockProgress';
|
||||
|
||||
{{#each queries as | query |}}
|
||||
import { {{query.entityName}} } from '../../entity/{{query.entityName}}';
|
||||
{{/each}}
|
||||
|
||||
const log = debug('vulcanize:reset-watcher');
|
||||
|
||||
export const command = 'watcher';
|
||||
|
||||
export const desc = 'Reset watcher to a block number';
|
||||
|
||||
export const builder = {
|
||||
blockNumber: {
|
||||
type: 'number'
|
||||
}
|
||||
};
|
||||
|
||||
export const handler = async (argv: any): Promise<void> => {
|
||||
const config = await getConfig(argv.configFile);
|
||||
await resetJobs(config);
|
||||
const { ethClient, ethProvider } = await initClients(config);
|
||||
|
||||
// Initialize database.
|
||||
const db = new Database(config.database);
|
||||
await db.init();
|
||||
{{#if (subgraphPath)}}
|
||||
|
||||
const graphDb = new GraphDatabase(config.database, path.resolve(__dirname, '../../entity/*'));
|
||||
await graphDb.init();
|
||||
|
||||
const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);
|
||||
{{/if}}
|
||||
|
||||
const jobQueueConfig = config.jobQueue;
|
||||
assert(jobQueueConfig, 'Missing job queue config');
|
||||
|
||||
const { dbConnectionString, maxCompletionLagInSecs } = jobQueueConfig;
|
||||
assert(dbConnectionString, 'Missing job queue db connection string');
|
||||
|
||||
const jobQueue = new JobQueue({ dbConnectionString, maxCompletionLag: maxCompletionLagInSecs });
|
||||
await jobQueue.start();
|
||||
|
||||
const indexer = new Indexer(config.server, db, ethClient, ethProvider, jobQueue{{#if (subgraphPath)}}, graphWatcher{{/if}});
|
||||
await indexer.init();
|
||||
{{#if (subgraphPath)}}
|
||||
|
||||
graphWatcher.setIndexer(indexer);
|
||||
await graphWatcher.init();
|
||||
{{/if}}
|
||||
|
||||
const blockProgresses = await indexer.getBlocksAtHeight(argv.blockNumber, false);
|
||||
assert(blockProgresses.length, `No blocks at specified block number ${argv.blockNumber}`);
|
||||
assert(!blockProgresses.some(block => !block.isComplete), `Incomplete block at block number ${argv.blockNumber} with unprocessed events`);
|
||||
const [blockProgress] = blockProgresses;
|
||||
|
||||
const dbTx = await db.createTransactionRunner();
|
||||
|
||||
try {
|
||||
const entities = [BlockProgress
|
||||
{{~#each queries as | query |~}}
|
||||
, {{query.entityName}}
|
||||
{{~/each~}}
|
||||
];
|
||||
|
||||
const removeEntitiesPromise = entities.map(async entityClass => {
|
||||
return db.removeEntities<any>(dbTx, entityClass, { blockNumber: MoreThan(argv.blockNumber) });
|
||||
});
|
||||
|
||||
await Promise.all(removeEntitiesPromise);
|
||||
|
||||
const syncStatus = await indexer.getSyncStatus();
|
||||
assert(syncStatus, 'Missing syncStatus');
|
||||
|
||||
if (syncStatus.latestIndexedBlockNumber > blockProgress.blockNumber) {
|
||||
await indexer.updateSyncStatusIndexedBlock(blockProgress.blockHash, blockProgress.blockNumber, true);
|
||||
}
|
||||
|
||||
if (syncStatus.latestCanonicalBlockNumber > blockProgress.blockNumber) {
|
||||
await indexer.updateSyncStatusCanonicalBlock(blockProgress.blockHash, blockProgress.blockNumber, true);
|
||||
}
|
||||
|
||||
const stateSyncStatus = await indexer.getStateSyncStatus();
|
||||
|
||||
if (stateSyncStatus) {
|
||||
if (stateSyncStatus.latestIndexedBlockNumber > blockProgress.blockNumber) {
|
||||
await indexer.updateStateSyncStatusIndexedBlock(blockProgress.blockNumber, true);
|
||||
}
|
||||
|
||||
if (stateSyncStatus.latestCheckpointBlockNumber > blockProgress.blockNumber) {
|
||||
await indexer.updateStateSyncStatusCheckpointBlock(blockProgress.blockNumber, true);
|
||||
}
|
||||
}
|
||||
|
||||
await indexer.updateSyncStatusChainHead(blockProgress.blockHash, blockProgress.blockNumber, true);
|
||||
|
||||
dbTx.commitTransaction();
|
||||
} catch (error) {
|
||||
await dbTx.rollbackTransaction();
|
||||
throw error;
|
||||
} finally {
|
||||
await dbTx.release();
|
||||
}
|
||||
|
||||
log('Reset watcher successfully');
|
||||
};
|
||||
@ -8,7 +8,7 @@ import debug from 'debug';
|
||||
import Decimal from 'decimal.js';
|
||||
import { GraphQLResolveInfo, GraphQLScalarType } from 'graphql';
|
||||
|
||||
import { ValueResult, BlockHeight, gqlTotalQueryCount, gqlQueryCount, jsonBigIntStringReplacer } from '@cerc-io/util';
|
||||
import { ValueResult, BlockHeight, gqlTotalQueryCount, gqlQueryCount, jsonBigIntStringReplacer, getResultState } from '@cerc-io/util';
|
||||
|
||||
import { Indexer } from './indexer';
|
||||
import { EventWatcher } from './events';
|
||||
@ -126,9 +126,9 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch
|
||||
gqlTotalQueryCount.inc(1);
|
||||
gqlQueryCount.labels('getStateByCID').inc(1);
|
||||
|
||||
const ipldBlock = await indexer.getIPLDBlockByCid(cid);
|
||||
const state = await indexer.getStateByCID(cid);
|
||||
|
||||
return ipldBlock && ipldBlock.block.isComplete ? indexer.getResultIPLDBlock(ipldBlock) : undefined;
|
||||
return state && state.block.isComplete ? getResultState(state) : undefined;
|
||||
},
|
||||
|
||||
getState: async (_: any, { blockHash, contractAddress, kind }: { blockHash: string, contractAddress: string, kind: string }) => {
|
||||
@ -136,9 +136,9 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch
|
||||
gqlTotalQueryCount.inc(1);
|
||||
gqlQueryCount.labels('getState').inc(1);
|
||||
|
||||
const ipldBlock = await indexer.getPrevIPLDBlock(blockHash, contractAddress, kind);
|
||||
const state = await indexer.getPrevState(blockHash, contractAddress, kind);
|
||||
|
||||
return ipldBlock && ipldBlock.block.isComplete ? indexer.getResultIPLDBlock(ipldBlock) : undefined;
|
||||
return state && state.block.isComplete ? getResultState(state) : undefined;
|
||||
},
|
||||
|
||||
getSyncStatus: async () => {
|
||||
|
||||
@ -210,13 +210,14 @@ export class Visitor {
|
||||
}
|
||||
|
||||
/**
|
||||
* Writes the reset.ts, job-queue.ts, state.ts files generated from templates to respective streams.
|
||||
* Writes the reset.ts, job-queue.ts, watcher.ts, state.ts files generated from templates to respective streams.
|
||||
* @param resetOutStream A writable output stream to write the reset file to.
|
||||
* @param resetJQOutStream A writable output stream to write the reset job-queue file to.
|
||||
* @param resetWatcherOutStream A writable output stream to write the reset watcher file to.
|
||||
* @param resetStateOutStream A writable output stream to write the reset state file to.
|
||||
*/
|
||||
exportReset (resetOutStream: Writable, resetJQOutStream: Writable, resetStateOutStream: Writable, resetIPLDStateOutStream: Writable | undefined, subgraphPath: string): void {
|
||||
this._reset.exportReset(resetOutStream, resetJQOutStream, resetStateOutStream, resetIPLDStateOutStream, subgraphPath);
|
||||
exportReset (resetOutStream: Writable, resetJQOutStream: Writable, resetWatcherOutStream: Writable, resetStateOutStream: Writable, subgraphPath: string): void {
|
||||
this._reset.exportReset(resetOutStream, resetJQOutStream, resetWatcherOutStream, resetStateOutStream, subgraphPath);
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@ -8,12 +8,6 @@
|
||||
yarn
|
||||
```
|
||||
|
||||
* Run the IPFS (go-ipfs version 0.12.2) daemon:
|
||||
|
||||
```bash
|
||||
ipfs daemon
|
||||
```
|
||||
|
||||
* Create a postgres12 database for the watcher:
|
||||
|
||||
```bash
|
||||
@ -47,7 +41,7 @@
|
||||
|
||||
* Update the `upstream` config and provide the `ipld-eth-server` GQL API endpoint.
|
||||
|
||||
* Update the `server` config with state checkpoint settings and provide the IPFS API address.
|
||||
* Update the `server` config with state checkpoint settings.
|
||||
|
||||
## Customize
|
||||
|
||||
@ -59,11 +53,11 @@
|
||||
|
||||
* Generating state:
|
||||
|
||||
* Edit the custom hook function `createInitialState` (triggered if the watcher passes the start block, checkpoint: `true`) in [hooks.ts](./src/hooks.ts) to save an initial state `IPLDBlock` using the `Indexer` object.
|
||||
* Edit the custom hook function `createInitialState` (triggered if the watcher passes the start block, checkpoint: `true`) in [hooks.ts](./src/hooks.ts) to save an initial `State` using the `Indexer` object.
|
||||
|
||||
* Edit the custom hook function `createStateDiff` (triggered on a block) in [hooks.ts](./src/hooks.ts) to save the state in a `diff` `IPLDBlock` using the `Indexer` object. The default state (if exists) is updated.
|
||||
* Edit the custom hook function `createStateDiff` (triggered on a block) in [hooks.ts](./src/hooks.ts) to save the state in a `diff` `State` using the `Indexer` object. The default state (if exists) is updated.
|
||||
|
||||
* Edit the custom hook function `createStateCheckpoint` (triggered just before default and CLI checkpoint) in [hooks.ts](./src/hooks.ts) to save the state in a `checkpoint` `IPLDBlock` using the `Indexer` object.
|
||||
* Edit the custom hook function `createStateCheckpoint` (triggered just before default and CLI checkpoint) in [hooks.ts](./src/hooks.ts) to save the state in a `checkpoint` `State` using the `Indexer` object.
|
||||
|
||||
## Run
|
||||
|
||||
@ -136,10 +130,10 @@ GQL console: http://localhost:3012/graphql
|
||||
|
||||
* To reset the watcher to a previous block number:
|
||||
|
||||
* Reset state:
|
||||
* Reset watcher:
|
||||
|
||||
```bash
|
||||
yarn reset state --block-number <previous-block-number>
|
||||
yarn reset watcher --block-number <previous-block-number>
|
||||
```
|
||||
|
||||
* Reset job-queue:
|
||||
@ -148,6 +142,12 @@ GQL console: http://localhost:3012/graphql
|
||||
yarn reset job-queue --block-number <previous-block-number>
|
||||
```
|
||||
|
||||
* Reset state:
|
||||
|
||||
```bash
|
||||
yarn reset state --block-number <previous-block-number>
|
||||
```
|
||||
|
||||
* `block-number`: Block number to which to reset the watcher.
|
||||
|
||||
* To export and import the watcher state:
|
||||
|
||||
@ -9,9 +9,6 @@
|
||||
# Checkpoint interval in number of blocks.
|
||||
checkpointInterval = 2000
|
||||
|
||||
# IPFS API address (can be taken from the output on running the IPFS daemon).
|
||||
# ipfsApiAddr = "/ip4/127.0.0.1/tcp/5001"
|
||||
|
||||
subgraphPath = "../graph-node/test/subgraph/eden"
|
||||
|
||||
# Disable creation of state from subgraph entity updates
|
||||
|
||||
@ -54,12 +54,12 @@ export const handler = async (argv: any): Promise<void> => {
|
||||
graphWatcher.setIndexer(indexer);
|
||||
await graphWatcher.init();
|
||||
|
||||
const ipldBlock = await indexer.getIPLDBlockByCid(argv.cid);
|
||||
assert(ipldBlock, 'IPLDBlock for the provided CID doesn\'t exist.');
|
||||
const data = indexer.getIPLDData(ipldBlock);
|
||||
const state = await indexer.getStateByCID(argv.cid);
|
||||
assert(state, 'State for the provided CID doesn\'t exist.');
|
||||
const data = indexer.getStateData(state);
|
||||
|
||||
log(`Verifying checkpoint data for contract ${ipldBlock.contractAddress}`);
|
||||
await verifyCheckpointData(graphDb, ipldBlock.block, data);
|
||||
log(`Verifying checkpoint data for contract ${state.contractAddress}`);
|
||||
await verifyCheckpointData(graphDb, state.block, data);
|
||||
log('Checkpoint data verified');
|
||||
|
||||
await db.close();
|
||||
|
||||
@ -70,16 +70,16 @@ const main = async (): Promise<void> => {
|
||||
const exportData: any = {
|
||||
snapshotBlock: {},
|
||||
contracts: [],
|
||||
ipldCheckpoints: []
|
||||
stateCheckpoints: []
|
||||
};
|
||||
|
||||
const contracts = await db.getContracts();
|
||||
let block = await indexer.getLatestHooksProcessedBlock();
|
||||
let block = await indexer.getLatestStateIndexedBlock();
|
||||
assert(block);
|
||||
|
||||
if (argv.blockNumber) {
|
||||
if (argv.blockNumber > block.blockNumber) {
|
||||
throw new Error(`Export snapshot block height ${argv.blockNumber} should be less than latest hooks processed block height ${block.blockNumber}`);
|
||||
throw new Error(`Export snapshot block height ${argv.blockNumber} should be less than latest state indexed block height ${block.blockNumber}`);
|
||||
}
|
||||
|
||||
const blocksAtSnapshotHeight = await indexer.getBlocksAtHeight(argv.blockNumber, false);
|
||||
@ -116,19 +116,15 @@ const main = async (): Promise<void> => {
|
||||
if (contract.checkpoint) {
|
||||
await indexer.createCheckpoint(contract.address, block.blockHash);
|
||||
|
||||
const ipldBlock = await indexer.getLatestIPLDBlock(contract.address, StateKind.Checkpoint, block.blockNumber);
|
||||
assert(ipldBlock);
|
||||
const state = await indexer.getLatestState(contract.address, StateKind.Checkpoint, block.blockNumber);
|
||||
assert(state);
|
||||
|
||||
const data = indexer.getIPLDData(ipldBlock);
|
||||
const data = indexer.getStateData(state);
|
||||
|
||||
if (indexer.isIPFSConfigured()) {
|
||||
await indexer.pushToIPFS(data);
|
||||
}
|
||||
|
||||
exportData.ipldCheckpoints.push({
|
||||
contractAddress: ipldBlock.contractAddress,
|
||||
cid: ipldBlock.cid,
|
||||
kind: ipldBlock.kind,
|
||||
exportData.stateCheckpoints.push({
|
||||
contractAddress: state.contractAddress,
|
||||
cid: state.cid,
|
||||
kind: state.kind,
|
||||
data
|
||||
});
|
||||
}
|
||||
|
||||
@ -18,7 +18,7 @@ import * as codec from '@ipld/dag-cbor';
|
||||
import { Database } from '../database';
|
||||
import { Indexer } from '../indexer';
|
||||
import { EventWatcher } from '../events';
|
||||
import { IPLDBlock } from '../entity/IPLDBlock';
|
||||
import { State } from '../entity/State';
|
||||
|
||||
const log = debug('vulcanize:import-state');
|
||||
|
||||
@ -100,17 +100,17 @@ export const main = async (): Promise<any> => {
|
||||
const block = await indexer.getBlockProgress(importData.snapshotBlock.blockHash);
|
||||
assert(block);
|
||||
|
||||
// Fill the IPLDBlocks.
|
||||
for (const checkpoint of importData.ipldCheckpoints) {
|
||||
let ipldBlock = new IPLDBlock();
|
||||
// Fill the States.
|
||||
for (const checkpoint of importData.stateCheckpoints) {
|
||||
let state = new State();
|
||||
|
||||
ipldBlock = Object.assign(ipldBlock, checkpoint);
|
||||
ipldBlock.block = block;
|
||||
state = Object.assign(state, checkpoint);
|
||||
state.block = block;
|
||||
|
||||
ipldBlock.data = Buffer.from(codec.encode(ipldBlock.data));
|
||||
state.data = Buffer.from(codec.encode(state.data));
|
||||
|
||||
ipldBlock = await indexer.saveOrUpdateIPLDBlock(ipldBlock);
|
||||
await graphWatcher.updateEntitiesFromIPLDState(ipldBlock);
|
||||
state = await indexer.saveOrUpdateState(state);
|
||||
await graphWatcher.updateEntitiesFromState(state);
|
||||
}
|
||||
|
||||
// Mark snapshot block as completely processed.
|
||||
@ -118,12 +118,12 @@ export const main = async (): Promise<any> => {
|
||||
await indexer.updateBlockProgress(block, block.lastProcessedEventIndex);
|
||||
await indexer.updateSyncStatusChainHead(block.blockHash, block.blockNumber);
|
||||
await indexer.updateSyncStatusIndexedBlock(block.blockHash, block.blockNumber);
|
||||
await indexer.updateIPLDStatusHooksBlock(block.blockNumber);
|
||||
await indexer.updateIPLDStatusCheckpointBlock(block.blockNumber);
|
||||
await indexer.updateStateSyncStatusIndexedBlock(block.blockNumber);
|
||||
await indexer.updateStateSyncStatusCheckpointBlock(block.blockNumber);
|
||||
|
||||
// The 'diff_staged' and 'init' IPLD blocks are unnecessary as checkpoints have been already created for the snapshot block.
|
||||
await indexer.removeIPLDBlocks(block.blockNumber, StateKind.Init);
|
||||
await indexer.removeIPLDBlocks(block.blockNumber, StateKind.DiffStaged);
|
||||
// The 'diff_staged' and 'init' State entries are unnecessary as checkpoints have been already created for the snapshot block.
|
||||
await indexer.removeStates(block.blockNumber, StateKind.Init);
|
||||
await indexer.removeStates(block.blockNumber, StateKind.DiffStaged);
|
||||
|
||||
log(`Import completed for snapshot block at height ${block.blockNumber}`);
|
||||
};
|
||||
|
||||
@ -63,12 +63,12 @@ const main = async (): Promise<void> => {
|
||||
graphWatcher.setIndexer(indexer);
|
||||
await graphWatcher.init();
|
||||
|
||||
const ipldBlock = await indexer.getIPLDBlockByCid(argv.cid);
|
||||
assert(ipldBlock, 'IPLDBlock for the provided CID doesn\'t exist.');
|
||||
const state = await indexer.getStateByCID(argv.cid);
|
||||
assert(state, 'State for the provided CID doesn\'t exist.');
|
||||
|
||||
const ipldData = await indexer.getIPLDData(ipldBlock);
|
||||
const stateData = await indexer.getStateData(state);
|
||||
|
||||
log(util.inspect(ipldData, false, null));
|
||||
log(util.inspect(stateData, false, null));
|
||||
};
|
||||
|
||||
main().catch(err => {
|
||||
|
||||
@ -1,66 +0,0 @@
|
||||
//
|
||||
// Copyright 2022 Vulcanize, Inc.
|
||||
//
|
||||
|
||||
import debug from 'debug';
|
||||
|
||||
import { getConfig } from '@cerc-io/util';
|
||||
|
||||
import { Database } from '../../database';
|
||||
|
||||
const log = debug('vulcanize:reset-ipld-state');
|
||||
|
||||
export const command = 'ipld-state';
|
||||
|
||||
export const desc = 'Reset IPLD state in the given range';
|
||||
|
||||
export const builder = {
|
||||
blockNumber: {
|
||||
type: 'number'
|
||||
}
|
||||
};
|
||||
|
||||
export const handler = async (argv: any): Promise<void> => {
|
||||
const { blockNumber } = argv;
|
||||
const config = await getConfig(argv.configFile);
|
||||
|
||||
// Initialize database
|
||||
const db = new Database(config.database);
|
||||
await db.init();
|
||||
|
||||
// Create a DB transaction
|
||||
const dbTx = await db.createTransactionRunner();
|
||||
|
||||
console.time('time:reset-ipld-state');
|
||||
try {
|
||||
// Delete all IPLDBlock entries in the given range
|
||||
await db.removeIPLDBlocksAfterBlock(dbTx, blockNumber);
|
||||
|
||||
// Reset the IPLD status.
|
||||
const ipldStatus = await db.getIPLDStatus();
|
||||
|
||||
if (ipldStatus) {
|
||||
if (ipldStatus.latestHooksBlockNumber > blockNumber) {
|
||||
await db.updateIPLDStatusHooksBlock(dbTx, blockNumber, true);
|
||||
}
|
||||
|
||||
if (ipldStatus.latestCheckpointBlockNumber > blockNumber) {
|
||||
await db.updateIPLDStatusCheckpointBlock(dbTx, blockNumber, true);
|
||||
}
|
||||
|
||||
if (ipldStatus.latestIPFSBlockNumber > blockNumber) {
|
||||
await db.updateIPLDStatusIPFSBlock(dbTx, blockNumber, true);
|
||||
}
|
||||
}
|
||||
|
||||
dbTx.commitTransaction();
|
||||
} catch (error) {
|
||||
await dbTx.rollbackTransaction();
|
||||
throw error;
|
||||
} finally {
|
||||
await dbTx.release();
|
||||
}
|
||||
console.timeEnd('time:reset-ipld-state');
|
||||
|
||||
log(`Reset ipld-state successfully to block ${blockNumber}`);
|
||||
};
|
||||
@ -1,42 +1,18 @@
|
||||
//
|
||||
// Copyright 2021 Vulcanize, Inc.
|
||||
// Copyright 2022 Vulcanize, Inc.
|
||||
//
|
||||
|
||||
import path from 'path';
|
||||
import debug from 'debug';
|
||||
import { MoreThan } from 'typeorm';
|
||||
import assert from 'assert';
|
||||
|
||||
import { getConfig, initClients, resetJobs, JobQueue } from '@cerc-io/util';
|
||||
import { GraphWatcher, Database as GraphDatabase } from '@cerc-io/graph-node';
|
||||
import { getConfig } from '@cerc-io/util';
|
||||
|
||||
import { Database } from '../../database';
|
||||
import { Indexer } from '../../indexer';
|
||||
import { BlockProgress } from '../../entity/BlockProgress';
|
||||
import { Producer } from '../../entity/Producer';
|
||||
import { ProducerSet } from '../../entity/ProducerSet';
|
||||
import { ProducerSetChange } from '../../entity/ProducerSetChange';
|
||||
import { ProducerRewardCollectorChange } from '../../entity/ProducerRewardCollectorChange';
|
||||
import { RewardScheduleEntry } from '../../entity/RewardScheduleEntry';
|
||||
import { RewardSchedule } from '../../entity/RewardSchedule';
|
||||
import { ProducerEpoch } from '../../entity/ProducerEpoch';
|
||||
import { Block } from '../../entity/Block';
|
||||
import { Epoch } from '../../entity/Epoch';
|
||||
import { SlotClaim } from '../../entity/SlotClaim';
|
||||
import { Slot } from '../../entity/Slot';
|
||||
import { Staker } from '../../entity/Staker';
|
||||
import { Network } from '../../entity/Network';
|
||||
import { Distributor } from '../../entity/Distributor';
|
||||
import { Distribution } from '../../entity/Distribution';
|
||||
import { Claim } from '../../entity/Claim';
|
||||
import { Slash } from '../../entity/Slash';
|
||||
import { Account } from '../../entity/Account';
|
||||
|
||||
const log = debug('vulcanize:reset-state');
|
||||
|
||||
export const command = 'state';
|
||||
|
||||
export const desc = 'Reset state to block number';
|
||||
export const desc = 'Reset State to a given block number';
|
||||
|
||||
export const builder = {
|
||||
blockNumber: {
|
||||
@ -45,77 +21,34 @@ export const builder = {
|
||||
};
|
||||
|
||||
export const handler = async (argv: any): Promise<void> => {
|
||||
const { blockNumber } = argv;
|
||||
const config = await getConfig(argv.configFile);
|
||||
await resetJobs(config);
|
||||
const { ethClient, ethProvider } = await initClients(config);
|
||||
|
||||
// Initialize database.
|
||||
// Initialize database
|
||||
const db = new Database(config.database);
|
||||
await db.init();
|
||||
|
||||
const graphDb = new GraphDatabase(config.database, path.resolve(__dirname, '../../entity/*'));
|
||||
await graphDb.init();
|
||||
|
||||
const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);
|
||||
|
||||
const jobQueueConfig = config.jobQueue;
|
||||
assert(jobQueueConfig, 'Missing job queue config');
|
||||
|
||||
const { dbConnectionString, maxCompletionLagInSecs } = jobQueueConfig;
|
||||
assert(dbConnectionString, 'Missing job queue db connection string');
|
||||
|
||||
const jobQueue = new JobQueue({ dbConnectionString, maxCompletionLag: maxCompletionLagInSecs });
|
||||
await jobQueue.start();
|
||||
|
||||
const indexer = new Indexer(config.server, db, ethClient, ethProvider, jobQueue, graphWatcher);
|
||||
await indexer.init();
|
||||
|
||||
graphWatcher.setIndexer(indexer);
|
||||
await graphWatcher.init();
|
||||
|
||||
const blockProgresses = await indexer.getBlocksAtHeight(argv.blockNumber, false);
|
||||
assert(blockProgresses.length, `No blocks at specified block number ${argv.blockNumber}`);
|
||||
assert(!blockProgresses.some(block => !block.isComplete), `Incomplete block at block number ${argv.blockNumber} with unprocessed events`);
|
||||
const [blockProgress] = blockProgresses;
|
||||
|
||||
// Create a DB transaction
|
||||
const dbTx = await db.createTransactionRunner();
|
||||
|
||||
console.time('time:reset-state');
|
||||
try {
|
||||
const entities = [BlockProgress, Producer, ProducerSet, ProducerSetChange, ProducerRewardCollectorChange, RewardScheduleEntry, RewardSchedule, ProducerEpoch, Block, Epoch, SlotClaim, Slot, Staker, Network, Distributor, Distribution, Claim, Slash, Account];
|
||||
// Delete all State entries after the given block
|
||||
await db.removeStatesAfterBlock(dbTx, blockNumber);
|
||||
|
||||
for (const entity of entities) {
|
||||
await db.deleteEntitiesByConditions<any>(dbTx, entity, { blockNumber: MoreThan(argv.blockNumber) });
|
||||
}
|
||||
// Reset the stateSyncStatus.
|
||||
const stateSyncStatus = await db.getStateSyncStatus();
|
||||
|
||||
const syncStatus = await indexer.getSyncStatus();
|
||||
assert(syncStatus, 'Missing syncStatus');
|
||||
|
||||
if (syncStatus.latestIndexedBlockNumber > blockProgress.blockNumber) {
|
||||
await indexer.updateSyncStatusIndexedBlock(blockProgress.blockHash, blockProgress.blockNumber, true);
|
||||
}
|
||||
|
||||
if (syncStatus.latestCanonicalBlockNumber > blockProgress.blockNumber) {
|
||||
await indexer.updateSyncStatusCanonicalBlock(blockProgress.blockHash, blockProgress.blockNumber, true);
|
||||
}
|
||||
|
||||
const ipldStatus = await indexer.getIPLDStatus();
|
||||
|
||||
if (ipldStatus) {
|
||||
if (ipldStatus.latestHooksBlockNumber > blockProgress.blockNumber) {
|
||||
await indexer.updateIPLDStatusHooksBlock(blockProgress.blockNumber, true);
|
||||
if (stateSyncStatus) {
|
||||
if (stateSyncStatus.latestIndexedBlockNumber > blockNumber) {
|
||||
await db.updateStateSyncStatusIndexedBlock(dbTx, blockNumber, true);
|
||||
}
|
||||
|
||||
if (ipldStatus.latestCheckpointBlockNumber > blockProgress.blockNumber) {
|
||||
await indexer.updateIPLDStatusCheckpointBlock(blockProgress.blockNumber, true);
|
||||
}
|
||||
|
||||
if (ipldStatus.latestIPFSBlockNumber > blockProgress.blockNumber) {
|
||||
await indexer.updateIPLDStatusIPFSBlock(blockProgress.blockNumber, true);
|
||||
if (stateSyncStatus.latestCheckpointBlockNumber > blockNumber) {
|
||||
await db.updateStateSyncStatusCheckpointBlock(dbTx, blockNumber, true);
|
||||
}
|
||||
}
|
||||
|
||||
await indexer.updateSyncStatusChainHead(blockProgress.blockHash, blockProgress.blockNumber, true);
|
||||
|
||||
dbTx.commitTransaction();
|
||||
} catch (error) {
|
||||
await dbTx.rollbackTransaction();
|
||||
@ -123,6 +56,7 @@ export const handler = async (argv: any): Promise<void> => {
|
||||
} finally {
|
||||
await dbTx.release();
|
||||
}
|
||||
console.timeEnd('time:reset-state');
|
||||
|
||||
log('Reset state successfully');
|
||||
log(`Reset state successfully to block ${blockNumber}`);
|
||||
};
|
||||
|
||||
124
packages/eden-watcher/src/cli/reset-cmds/watcher.ts
Normal file
124
packages/eden-watcher/src/cli/reset-cmds/watcher.ts
Normal file
@ -0,0 +1,124 @@
|
||||
//
|
||||
// Copyright 2021 Vulcanize, Inc.
|
||||
//
|
||||
|
||||
import path from 'path';
|
||||
import debug from 'debug';
|
||||
import { MoreThan } from 'typeorm';
|
||||
import assert from 'assert';
|
||||
|
||||
import { getConfig, initClients, resetJobs, JobQueue } from '@cerc-io/util';
|
||||
import { GraphWatcher, Database as GraphDatabase } from '@cerc-io/graph-node';
|
||||
|
||||
import { Database } from '../../database';
|
||||
import { Indexer } from '../../indexer';
|
||||
import { BlockProgress } from '../../entity/BlockProgress';
|
||||
import { Producer } from '../../entity/Producer';
|
||||
import { ProducerSet } from '../../entity/ProducerSet';
|
||||
import { ProducerSetChange } from '../../entity/ProducerSetChange';
|
||||
import { ProducerRewardCollectorChange } from '../../entity/ProducerRewardCollectorChange';
|
||||
import { RewardScheduleEntry } from '../../entity/RewardScheduleEntry';
|
||||
import { RewardSchedule } from '../../entity/RewardSchedule';
|
||||
import { ProducerEpoch } from '../../entity/ProducerEpoch';
|
||||
import { Block } from '../../entity/Block';
|
||||
import { Epoch } from '../../entity/Epoch';
|
||||
import { SlotClaim } from '../../entity/SlotClaim';
|
||||
import { Slot } from '../../entity/Slot';
|
||||
import { Staker } from '../../entity/Staker';
|
||||
import { Network } from '../../entity/Network';
|
||||
import { Distributor } from '../../entity/Distributor';
|
||||
import { Distribution } from '../../entity/Distribution';
|
||||
import { Claim } from '../../entity/Claim';
|
||||
import { Slash } from '../../entity/Slash';
|
||||
import { Account } from '../../entity/Account';
|
||||
|
||||
const log = debug('vulcanize:reset-watcher');
|
||||
|
||||
export const command = 'watcher';
|
||||
|
||||
export const desc = 'Reset watcher to a block number';
|
||||
|
||||
export const builder = {
|
||||
blockNumber: {
|
||||
type: 'number'
|
||||
}
|
||||
};
|
||||
|
||||
export const handler = async (argv: any): Promise<void> => {
|
||||
const config = await getConfig(argv.configFile);
|
||||
await resetJobs(config);
|
||||
const { ethClient, ethProvider } = await initClients(config);
|
||||
|
||||
// Initialize database.
|
||||
const db = new Database(config.database);
|
||||
await db.init();
|
||||
|
||||
const graphDb = new GraphDatabase(config.database, path.resolve(__dirname, '../../entity/*'));
|
||||
await graphDb.init();
|
||||
|
||||
const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);
|
||||
|
||||
const jobQueueConfig = config.jobQueue;
|
||||
assert(jobQueueConfig, 'Missing job queue config');
|
||||
|
||||
const { dbConnectionString, maxCompletionLagInSecs } = jobQueueConfig;
|
||||
assert(dbConnectionString, 'Missing job queue db connection string');
|
||||
|
||||
const jobQueue = new JobQueue({ dbConnectionString, maxCompletionLag: maxCompletionLagInSecs });
|
||||
await jobQueue.start();
|
||||
|
||||
const indexer = new Indexer(config.server, db, ethClient, ethProvider, jobQueue, graphWatcher);
|
||||
await indexer.init();
|
||||
|
||||
graphWatcher.setIndexer(indexer);
|
||||
await graphWatcher.init();
|
||||
|
||||
const blockProgresses = await indexer.getBlocksAtHeight(argv.blockNumber, false);
|
||||
assert(blockProgresses.length, `No blocks at specified block number ${argv.blockNumber}`);
|
||||
assert(!blockProgresses.some(block => !block.isComplete), `Incomplete block at block number ${argv.blockNumber} with unprocessed events`);
|
||||
const [blockProgress] = blockProgresses;
|
||||
|
||||
const dbTx = await db.createTransactionRunner();
|
||||
|
||||
try {
|
||||
const entities = [BlockProgress, Producer, ProducerSet, ProducerSetChange, ProducerRewardCollectorChange, RewardScheduleEntry, RewardSchedule, ProducerEpoch, Block, Epoch, SlotClaim, Slot, Staker, Network, Distributor, Distribution, Claim, Slash, Account];
|
||||
|
||||
for (const entity of entities) {
|
||||
await db.deleteEntitiesByConditions<any>(dbTx, entity, { blockNumber: MoreThan(argv.blockNumber) });
|
||||
}
|
||||
|
||||
const syncStatus = await indexer.getSyncStatus();
|
||||
assert(syncStatus, 'Missing syncStatus');
|
||||
|
||||
if (syncStatus.latestIndexedBlockNumber > blockProgress.blockNumber) {
|
||||
await indexer.updateSyncStatusIndexedBlock(blockProgress.blockHash, blockProgress.blockNumber, true);
|
||||
}
|
||||
|
||||
if (syncStatus.latestCanonicalBlockNumber > blockProgress.blockNumber) {
|
||||
await indexer.updateSyncStatusCanonicalBlock(blockProgress.blockHash, blockProgress.blockNumber, true);
|
||||
}
|
||||
|
||||
const stateSyncStatus = await indexer.getStateSyncStatus();
|
||||
|
||||
if (stateSyncStatus) {
|
||||
if (stateSyncStatus.latestIndexedBlockNumber > blockProgress.blockNumber) {
|
||||
await indexer.updateStateSyncStatusIndexedBlock(blockProgress.blockNumber, true);
|
||||
}
|
||||
|
||||
if (stateSyncStatus.latestCheckpointBlockNumber > blockProgress.blockNumber) {
|
||||
await indexer.updateStateSyncStatusCheckpointBlock(blockProgress.blockNumber, true);
|
||||
}
|
||||
}
|
||||
|
||||
await indexer.updateSyncStatusChainHead(blockProgress.blockHash, blockProgress.blockNumber, true);
|
||||
|
||||
dbTx.commitTransaction();
|
||||
} catch (error) {
|
||||
await dbTx.rollbackTransaction();
|
||||
throw error;
|
||||
} finally {
|
||||
await dbTx.release();
|
||||
}
|
||||
|
||||
log('Reset watcher successfully');
|
||||
};
|
||||
@ -11,9 +11,9 @@ import { Database as BaseDatabase, DatabaseInterface, QueryOptions, StateKind, W
|
||||
import { Contract } from './entity/Contract';
|
||||
import { Event } from './entity/Event';
|
||||
import { SyncStatus } from './entity/SyncStatus';
|
||||
import { IpldStatus } from './entity/IpldStatus';
|
||||
import { StateSyncStatus } from './entity/StateSyncStatus';
|
||||
import { BlockProgress } from './entity/BlockProgress';
|
||||
import { IPLDBlock } from './entity/IPLDBlock';
|
||||
import { State } from './entity/State';
|
||||
|
||||
export class Database implements DatabaseInterface {
|
||||
_config: ConnectionOptions;
|
||||
@ -39,75 +39,69 @@ export class Database implements DatabaseInterface {
|
||||
return this._baseDatabase.close();
|
||||
}
|
||||
|
||||
getNewIPLDBlock (): IPLDBlock {
|
||||
return new IPLDBlock();
|
||||
getNewState (): State {
|
||||
return new State();
|
||||
}
|
||||
|
||||
async getIPLDBlocks (where: FindConditions<IPLDBlock>): Promise<IPLDBlock[]> {
|
||||
const repo = this._conn.getRepository(IPLDBlock);
|
||||
async getStates (where: FindConditions<State>): Promise<State[]> {
|
||||
const repo = this._conn.getRepository(State);
|
||||
|
||||
return this._baseDatabase.getIPLDBlocks(repo, where);
|
||||
return this._baseDatabase.getStates(repo, where);
|
||||
}
|
||||
|
||||
async getLatestIPLDBlock (contractAddress: string, kind: StateKind | null, blockNumber?: number): Promise<IPLDBlock | undefined> {
|
||||
const repo = this._conn.getRepository(IPLDBlock);
|
||||
async getLatestState (contractAddress: string, kind: StateKind | null, blockNumber?: number): Promise<State | undefined> {
|
||||
const repo = this._conn.getRepository(State);
|
||||
|
||||
return this._baseDatabase.getLatestIPLDBlock(repo, contractAddress, kind, blockNumber);
|
||||
return this._baseDatabase.getLatestState(repo, contractAddress, kind, blockNumber);
|
||||
}
|
||||
|
||||
async getPrevIPLDBlock (blockHash: string, contractAddress: string, kind?: string): Promise<IPLDBlock | undefined> {
|
||||
const repo = this._conn.getRepository(IPLDBlock);
|
||||
async getPrevState (blockHash: string, contractAddress: string, kind?: string): Promise<State | undefined> {
|
||||
const repo = this._conn.getRepository(State);
|
||||
|
||||
return this._baseDatabase.getPrevIPLDBlock(repo, blockHash, contractAddress, kind);
|
||||
return this._baseDatabase.getPrevState(repo, blockHash, contractAddress, kind);
|
||||
}
|
||||
|
||||
// Fetch all diff IPLDBlocks after the specified block number.
|
||||
async getDiffIPLDBlocksInRange (contractAddress: string, startblock: number, endBlock: number): Promise<IPLDBlock[]> {
|
||||
const repo = this._conn.getRepository(IPLDBlock);
|
||||
// Fetch all diff States after the specified block number.
|
||||
async getDiffStatesInRange (contractAddress: string, startblock: number, endBlock: number): Promise<State[]> {
|
||||
const repo = this._conn.getRepository(State);
|
||||
|
||||
return this._baseDatabase.getDiffIPLDBlocksInRange(repo, contractAddress, startblock, endBlock);
|
||||
return this._baseDatabase.getDiffStatesInRange(repo, contractAddress, startblock, endBlock);
|
||||
}
|
||||
|
||||
async saveOrUpdateIPLDBlock (dbTx: QueryRunner, ipldBlock: IPLDBlock): Promise<IPLDBlock> {
|
||||
const repo = dbTx.manager.getRepository(IPLDBlock);
|
||||
async saveOrUpdateState (dbTx: QueryRunner, state: State): Promise<State> {
|
||||
const repo = dbTx.manager.getRepository(State);
|
||||
|
||||
return this._baseDatabase.saveOrUpdateIPLDBlock(repo, ipldBlock);
|
||||
return this._baseDatabase.saveOrUpdateState(repo, state);
|
||||
}
|
||||
|
||||
async removeIPLDBlocks (dbTx: QueryRunner, blockNumber: number, kind: string): Promise<void> {
|
||||
const repo = dbTx.manager.getRepository(IPLDBlock);
|
||||
async removeStates (dbTx: QueryRunner, blockNumber: number, kind: string): Promise<void> {
|
||||
const repo = dbTx.manager.getRepository(State);
|
||||
|
||||
await this._baseDatabase.removeIPLDBlocks(repo, blockNumber, kind);
|
||||
await this._baseDatabase.removeStates(repo, blockNumber, kind);
|
||||
}
|
||||
|
||||
async removeIPLDBlocksAfterBlock (dbTx: QueryRunner, blockNumber: number): Promise<void> {
|
||||
const repo = dbTx.manager.getRepository(IPLDBlock);
|
||||
async removeStatesAfterBlock (dbTx: QueryRunner, blockNumber: number): Promise<void> {
|
||||
const repo = dbTx.manager.getRepository(State);
|
||||
|
||||
await this._baseDatabase.removeIPLDBlocksAfterBlock(repo, blockNumber);
|
||||
await this._baseDatabase.removeStatesAfterBlock(repo, blockNumber);
|
||||
}
|
||||
|
||||
async getIPLDStatus (): Promise<IpldStatus | undefined> {
|
||||
const repo = this._conn.getRepository(IpldStatus);
|
||||
async getStateSyncStatus (): Promise<StateSyncStatus | undefined> {
|
||||
const repo = this._conn.getRepository(StateSyncStatus);
|
||||
|
||||
return this._baseDatabase.getIPLDStatus(repo);
|
||||
return this._baseDatabase.getStateSyncStatus(repo);
|
||||
}
|
||||
|
||||
async updateIPLDStatusHooksBlock (queryRunner: QueryRunner, blockNumber: number, force?: boolean): Promise<IpldStatus> {
|
||||
const repo = queryRunner.manager.getRepository(IpldStatus);
|
||||
async updateStateSyncStatusIndexedBlock (queryRunner: QueryRunner, blockNumber: number, force?: boolean): Promise<StateSyncStatus> {
|
||||
const repo = queryRunner.manager.getRepository(StateSyncStatus);
|
||||
|
||||
return this._baseDatabase.updateIPLDStatusHooksBlock(repo, blockNumber, force);
|
||||
return this._baseDatabase.updateStateSyncStatusIndexedBlock(repo, blockNumber, force);
|
||||
}
|
||||
|
||||
async updateIPLDStatusCheckpointBlock (queryRunner: QueryRunner, blockNumber: number, force?: boolean): Promise<IpldStatus> {
|
||||
const repo = queryRunner.manager.getRepository(IpldStatus);
|
||||
async updateStateSyncStatusCheckpointBlock (queryRunner: QueryRunner, blockNumber: number, force?: boolean): Promise<StateSyncStatus> {
|
||||
const repo = queryRunner.manager.getRepository(StateSyncStatus);
|
||||
|
||||
return this._baseDatabase.updateIPLDStatusCheckpointBlock(repo, blockNumber, force);
|
||||
}
|
||||
|
||||
async updateIPLDStatusIPFSBlock (queryRunner: QueryRunner, blockNumber: number, force?: boolean): Promise<IpldStatus> {
|
||||
const repo = queryRunner.manager.getRepository(IpldStatus);
|
||||
|
||||
return this._baseDatabase.updateIPLDStatusIPFSBlock(repo, blockNumber, force);
|
||||
return this._baseDatabase.updateStateSyncStatusCheckpointBlock(repo, blockNumber, force);
|
||||
}
|
||||
|
||||
async getContracts (): Promise<Contract[]> {
|
||||
|
||||
@ -12,7 +12,7 @@ import { BlockProgress } from './BlockProgress';
|
||||
@Index(['cid'], { unique: true })
|
||||
@Index(['block', 'contractAddress'])
|
||||
@Index(['block', 'contractAddress', 'kind'], { unique: true })
|
||||
export class IPLDBlock {
|
||||
export class State {
|
||||
@PrimaryGeneratedColumn()
|
||||
id!: number;
|
||||
|
||||
@ -5,16 +5,13 @@
|
||||
import { Entity, PrimaryGeneratedColumn, Column } from 'typeorm';
|
||||
|
||||
@Entity()
|
||||
export class IpldStatus {
|
||||
export class StateSyncStatus {
|
||||
@PrimaryGeneratedColumn()
|
||||
id!: number;
|
||||
|
||||
@Column('integer')
|
||||
latestHooksBlockNumber!: number;
|
||||
latestIndexedBlockNumber!: number;
|
||||
|
||||
@Column('integer', { nullable: true })
|
||||
latestCheckpointBlockNumber!: number;
|
||||
|
||||
@Column('integer', { nullable: true })
|
||||
latestIPFSBlockNumber!: number;
|
||||
}
|
||||
@ -31,7 +31,7 @@ export const fillState = async (
|
||||
log(`Filling state for subgraph entities in range: [${startBlock}, ${endBlock}]`);
|
||||
|
||||
// Check that there are no existing diffs in this range
|
||||
const existingStates = await indexer.getIPLDBlocks({ block: { blockNumber: Between(startBlock, endBlock) } });
|
||||
const existingStates = await indexer.getStates({ block: { blockNumber: Between(startBlock, endBlock) } });
|
||||
if (existingStates.length > 0) {
|
||||
log('found existing state(s) in the given range');
|
||||
process.exit(1);
|
||||
@ -97,26 +97,11 @@ export const fillState = async (
|
||||
|
||||
// Persist subgraph state to the DB
|
||||
await indexer.dumpSubgraphState(blockHash, true);
|
||||
await indexer.updateIPLDStatusHooksBlock(blockNumber);
|
||||
await indexer.updateStateSyncStatusIndexedBlock(blockNumber);
|
||||
|
||||
// Create checkpoints
|
||||
await indexer.processCheckpoint(blockHash);
|
||||
await indexer.updateIPLDStatusCheckpointBlock(blockNumber);
|
||||
|
||||
// TODO: Push state to IPFS in separate process.
|
||||
if (indexer.isIPFSConfigured()) {
|
||||
// Get IPLDBlocks for the given blocHash.
|
||||
const ipldBlocks = await indexer.getIPLDBlocksByHash(blockHash);
|
||||
|
||||
// Push all the IPLDBlocks to IPFS.
|
||||
for (const ipldBlock of ipldBlocks) {
|
||||
const data = indexer.getIPLDData(ipldBlock);
|
||||
await indexer.pushToIPFS(data);
|
||||
}
|
||||
|
||||
// Update the IPLD status.
|
||||
await indexer.updateIPLDStatusIPFSBlock(blockNumber);
|
||||
}
|
||||
await indexer.updateStateSyncStatusCheckpointBlock(blockNumber);
|
||||
|
||||
console.timeEnd(`time:fill-state-${blockNumber}`);
|
||||
}
|
||||
|
||||
@ -2,15 +2,10 @@
|
||||
// Copyright 2021 Vulcanize, Inc.
|
||||
//
|
||||
|
||||
import { IPLDBlockInterface, StateKind } from '@cerc-io/util';
|
||||
import assert from 'assert';
|
||||
import * as codec from '@ipld/dag-cbor';
|
||||
import _ from 'lodash';
|
||||
|
||||
import { Indexer, ResultEvent } from './indexer';
|
||||
|
||||
const IPLD_BATCH_BLOCKS = 10000;
|
||||
|
||||
/**
|
||||
* Hook function to store an initial state.
|
||||
* @param indexer Indexer instance.
|
||||
@ -23,13 +18,13 @@ export async function createInitialState (indexer: Indexer, contractAddress: str
|
||||
assert(blockHash);
|
||||
assert(contractAddress);
|
||||
|
||||
// Store an empty state in an IPLDBlock.
|
||||
const ipldBlockData: any = {
|
||||
// Store an empty State.
|
||||
const stateData: any = {
|
||||
state: {}
|
||||
};
|
||||
|
||||
// Return initial state data to be saved.
|
||||
return ipldBlockData;
|
||||
return stateData;
|
||||
}
|
||||
|
||||
/**
|
||||
@ -56,64 +51,11 @@ export async function createStateCheckpoint (indexer: Indexer, contractAddress:
|
||||
assert(blockHash);
|
||||
assert(contractAddress);
|
||||
|
||||
// TODO: Pass blockProgress instead of blockHash to hook method.
|
||||
const block = await indexer.getBlockProgress(blockHash);
|
||||
assert(block);
|
||||
// Use indexer.createStateCheckpoint() method to create a custom checkpoint.
|
||||
|
||||
// Fetch the latest 'checkpoint' | 'init' for the contract to fetch diffs after it.
|
||||
let prevNonDiffBlock: IPLDBlockInterface;
|
||||
let diffStartBlockNumber: number;
|
||||
const checkpointBlock = await indexer.getLatestIPLDBlock(contractAddress, StateKind.Checkpoint, block.blockNumber - 1);
|
||||
|
||||
if (checkpointBlock) {
|
||||
const checkpointBlockNumber = checkpointBlock.block.blockNumber;
|
||||
|
||||
prevNonDiffBlock = checkpointBlock;
|
||||
diffStartBlockNumber = checkpointBlockNumber;
|
||||
|
||||
// Update IPLD status map with the latest checkpoint info.
|
||||
// Essential while importing state as checkpoint at the snapshot block is added by import-state CLI.
|
||||
// (job-runner won't have the updated ipld status)
|
||||
indexer.updateIPLDStatusMap(contractAddress, { checkpoint: checkpointBlockNumber });
|
||||
} else {
|
||||
// There should be an initial state at least.
|
||||
const initBlock = await indexer.getLatestIPLDBlock(contractAddress, StateKind.Init);
|
||||
assert(initBlock, 'No initial state found');
|
||||
|
||||
prevNonDiffBlock = initBlock;
|
||||
// Take block number previous to initial state block to include any diff state at that block.
|
||||
diffStartBlockNumber = initBlock.block.blockNumber - 1;
|
||||
}
|
||||
|
||||
const prevNonDiffBlockData = codec.decode(Buffer.from(prevNonDiffBlock.data)) as any;
|
||||
const data = {
|
||||
state: prevNonDiffBlockData.state
|
||||
};
|
||||
|
||||
console.time('time:hooks#createStateCheckpoint');
|
||||
|
||||
// Fetching and merging all diff blocks after the latest 'checkpoint' | 'init' in batch.
|
||||
for (let i = diffStartBlockNumber; i < block.blockNumber;) {
|
||||
const endBlockHeight = Math.min(i + IPLD_BATCH_BLOCKS, block.blockNumber);
|
||||
console.time(`time:hooks#createStateCheckpoint-batch-merge-diff-${i}-${endBlockHeight}`);
|
||||
const diffBlocks = await indexer.getDiffIPLDBlocksInRange(contractAddress, i, endBlockHeight);
|
||||
|
||||
// Merge all diff blocks after previous checkpoint.
|
||||
for (const diffBlock of diffBlocks) {
|
||||
const diff = codec.decode(Buffer.from(diffBlock.data)) as any;
|
||||
data.state = _.merge(data.state, diff.state);
|
||||
}
|
||||
|
||||
console.timeEnd(`time:hooks#createStateCheckpoint-batch-merge-diff-${i}-${endBlockHeight}`);
|
||||
i = endBlockHeight;
|
||||
}
|
||||
|
||||
console.time('time:hooks#createStateCheckpoint-db-save-checkpoint');
|
||||
await indexer.createStateCheckpoint(contractAddress, blockHash, data);
|
||||
console.timeEnd('time:hooks#createStateCheckpoint-db-save-checkpoint');
|
||||
|
||||
console.timeEnd('time:hooks#createStateCheckpoint');
|
||||
return true;
|
||||
// Return false to update the state created by this hook by auto-generated checkpoint state.
|
||||
// Return true to disable update of the state created by this hook by auto-generated checkpoint state.
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@ -12,7 +12,6 @@ import { SelectionNode } from 'graphql';
|
||||
|
||||
import { JsonFragment } from '@ethersproject/abi';
|
||||
import { BaseProvider } from '@ethersproject/providers';
|
||||
import * as codec from '@ipld/dag-cbor';
|
||||
import { EthClient } from '@cerc-io/ipld-eth-client';
|
||||
import { MappingKey, StorageLayout } from '@cerc-io/solidity-mapper';
|
||||
import {
|
||||
@ -23,12 +22,10 @@ import {
|
||||
Where,
|
||||
QueryOptions,
|
||||
BlockHeight,
|
||||
IPFSClient,
|
||||
StateKind,
|
||||
IndexerInterface,
|
||||
IpldStatus as IpldStatusInterface,
|
||||
ValueResult,
|
||||
ResultIPLDBlock
|
||||
StateStatus,
|
||||
ValueResult
|
||||
} from '@cerc-io/util';
|
||||
import { GraphWatcher } from '@cerc-io/graph-node';
|
||||
|
||||
@ -36,9 +33,9 @@ import { Database } from './database';
|
||||
import { Contract } from './entity/Contract';
|
||||
import { Event } from './entity/Event';
|
||||
import { SyncStatus } from './entity/SyncStatus';
|
||||
import { IpldStatus } from './entity/IpldStatus';
|
||||
import { StateSyncStatus } from './entity/StateSyncStatus';
|
||||
import { BlockProgress } from './entity/BlockProgress';
|
||||
import { IPLDBlock } from './entity/IPLDBlock';
|
||||
import { State } from './entity/State';
|
||||
import EdenNetworkArtifacts from './artifacts/EdenNetwork.json';
|
||||
import MerkleDistributorArtifacts from './artifacts/MerkleDistributor.json';
|
||||
import DistributorGovernanceArtifacts from './artifacts/DistributorGovernance.json';
|
||||
@ -103,8 +100,6 @@ export class Indexer implements IndexerInterface {
|
||||
_storageLayoutMap: Map<string, StorageLayout>
|
||||
_contractMap: Map<string, ethers.utils.Interface>
|
||||
|
||||
_ipfsClient: IPFSClient
|
||||
|
||||
_entityTypesMap: Map<string, { [key: string]: string }>
|
||||
_relationsMap: Map<any, { [key: string]: any }>
|
||||
|
||||
@ -118,8 +113,7 @@ export class Indexer implements IndexerInterface {
|
||||
this._ethClient = ethClient;
|
||||
this._ethProvider = ethProvider;
|
||||
this._serverConfig = serverConfig;
|
||||
this._ipfsClient = new IPFSClient(this._serverConfig.ipfsApiAddr);
|
||||
this._baseIndexer = new BaseIndexer(this._serverConfig, this._db, this._ethClient, this._ethProvider, jobQueue, this._ipfsClient);
|
||||
this._baseIndexer = new BaseIndexer(this._serverConfig, this._db, this._ethClient, this._ethProvider, jobQueue);
|
||||
this._graphWatcher = graphWatcher;
|
||||
|
||||
this._abiMap = new Map();
|
||||
@ -170,7 +164,7 @@ export class Indexer implements IndexerInterface {
|
||||
|
||||
async init (): Promise<void> {
|
||||
await this._baseIndexer.fetchContracts();
|
||||
await this._baseIndexer.fetchIPLDStatus();
|
||||
await this._baseIndexer.fetchStateStatus();
|
||||
}
|
||||
|
||||
getResultEvent (event: Event): ResultEvent {
|
||||
@ -208,26 +202,6 @@ export class Indexer implements IndexerInterface {
|
||||
};
|
||||
}
|
||||
|
||||
getResultIPLDBlock (ipldBlock: IPLDBlock): ResultIPLDBlock {
|
||||
const block = ipldBlock.block;
|
||||
|
||||
const data = codec.decode(Buffer.from(ipldBlock.data)) as any;
|
||||
|
||||
return {
|
||||
block: {
|
||||
cid: block.cid,
|
||||
hash: block.blockHash,
|
||||
number: block.blockNumber,
|
||||
timestamp: block.blockTimestamp,
|
||||
parentHash: block.parentHash
|
||||
},
|
||||
contractAddress: ipldBlock.contractAddress,
|
||||
cid: ipldBlock.cid,
|
||||
kind: ipldBlock.kind,
|
||||
data: JSON.stringify(data)
|
||||
};
|
||||
}
|
||||
|
||||
async getStorageValue (storageLayout: StorageLayout, blockHash: string, contractAddress: string, variable: string, ...mappingKeys: MappingKey[]): Promise<ValueResult> {
|
||||
return this._baseIndexer.getStorageValue(
|
||||
storageLayout,
|
||||
@ -238,10 +212,6 @@ export class Indexer implements IndexerInterface {
|
||||
);
|
||||
}
|
||||
|
||||
async pushToIPFS (data: any): Promise<void> {
|
||||
await this._baseIndexer.pushToIPFS(data);
|
||||
}
|
||||
|
||||
async processInitialState (contractAddress: string, blockHash: string): Promise<any> {
|
||||
// Call initial state hook.
|
||||
return createInitialState(this, contractAddress, blockHash);
|
||||
@ -280,36 +250,32 @@ export class Indexer implements IndexerInterface {
|
||||
return this._baseIndexer.processCLICheckpoint(this, contractAddress, blockHash);
|
||||
}
|
||||
|
||||
async getPrevIPLDBlock (blockHash: string, contractAddress: string, kind?: string): Promise<IPLDBlock | undefined> {
|
||||
return this._db.getPrevIPLDBlock(blockHash, contractAddress, kind);
|
||||
async getPrevState (blockHash: string, contractAddress: string, kind?: string): Promise<State | undefined> {
|
||||
return this._db.getPrevState(blockHash, contractAddress, kind);
|
||||
}
|
||||
|
||||
async getLatestIPLDBlock (contractAddress: string, kind: StateKind | null, blockNumber?: number): Promise<IPLDBlock | undefined> {
|
||||
return this._db.getLatestIPLDBlock(contractAddress, kind, blockNumber);
|
||||
async getLatestState (contractAddress: string, kind: StateKind | null, blockNumber?: number): Promise<State | undefined> {
|
||||
return this._db.getLatestState(contractAddress, kind, blockNumber);
|
||||
}
|
||||
|
||||
async getIPLDBlocksByHash (blockHash: string): Promise<IPLDBlock[]> {
|
||||
return this._baseIndexer.getIPLDBlocksByHash(blockHash);
|
||||
async getStatesByHash (blockHash: string): Promise<State[]> {
|
||||
return this._baseIndexer.getStatesByHash(blockHash);
|
||||
}
|
||||
|
||||
async getIPLDBlockByCid (cid: string): Promise<IPLDBlock | undefined> {
|
||||
return this._baseIndexer.getIPLDBlockByCid(cid);
|
||||
async getStateByCID (cid: string): Promise<State | undefined> {
|
||||
return this._baseIndexer.getStateByCID(cid);
|
||||
}
|
||||
|
||||
async getIPLDBlocks (where: FindConditions<IPLDBlock>): Promise<IPLDBlock[]> {
|
||||
return this._db.getIPLDBlocks(where);
|
||||
async getStates (where: FindConditions<State>): Promise<State[]> {
|
||||
return this._db.getStates(where);
|
||||
}
|
||||
|
||||
async getDiffIPLDBlocksInRange (contractAddress: string, startBlock: number, endBlock: number): Promise<IPLDBlock[]> {
|
||||
return this._db.getDiffIPLDBlocksInRange(contractAddress, startBlock, endBlock);
|
||||
async getDiffStatesInRange (contractAddress: string, startBlock: number, endBlock: number): Promise<State[]> {
|
||||
return this._db.getDiffStatesInRange(contractAddress, startBlock, endBlock);
|
||||
}
|
||||
|
||||
getIPLDData (ipldBlock: IPLDBlock): any {
|
||||
return this._baseIndexer.getIPLDData(ipldBlock);
|
||||
}
|
||||
|
||||
isIPFSConfigured (): boolean {
|
||||
return this._baseIndexer.isIPFSConfigured();
|
||||
getStateData (state: State): any {
|
||||
return this._baseIndexer.getStateData(state);
|
||||
}
|
||||
|
||||
// Method used to create auto diffs (diff_staged).
|
||||
@ -351,12 +317,12 @@ export class Indexer implements IndexerInterface {
|
||||
await this._baseIndexer.createInit(this, blockHash, blockNumber);
|
||||
}
|
||||
|
||||
async saveOrUpdateIPLDBlock (ipldBlock: IPLDBlock): Promise<IPLDBlock> {
|
||||
return this._baseIndexer.saveOrUpdateIPLDBlock(ipldBlock);
|
||||
async saveOrUpdateState (state: State): Promise<State> {
|
||||
return this._baseIndexer.saveOrUpdateState(state);
|
||||
}
|
||||
|
||||
async removeIPLDBlocks (blockNumber: number, kind: StateKind): Promise<void> {
|
||||
await this._baseIndexer.removeIPLDBlocks(blockNumber, kind);
|
||||
async removeStates (blockNumber: number, kind: StateKind): Promise<void> {
|
||||
await this._baseIndexer.removeStates(blockNumber, kind);
|
||||
}
|
||||
|
||||
async getSubgraphEntity<Entity> (entity: new () => Entity, id: string, block: BlockHeight, selections: ReadonlyArray<SelectionNode> = []): Promise<any> {
|
||||
@ -436,16 +402,16 @@ export class Indexer implements IndexerInterface {
|
||||
};
|
||||
}
|
||||
|
||||
async getIPLDStatus (): Promise<IpldStatus | undefined> {
|
||||
return this._db.getIPLDStatus();
|
||||
async getStateSyncStatus (): Promise<StateSyncStatus | undefined> {
|
||||
return this._db.getStateSyncStatus();
|
||||
}
|
||||
|
||||
async updateIPLDStatusHooksBlock (blockNumber: number, force?: boolean): Promise<IpldStatus> {
|
||||
async updateStateSyncStatusIndexedBlock (blockNumber: number, force?: boolean): Promise<StateSyncStatus> {
|
||||
const dbTx = await this._db.createTransactionRunner();
|
||||
let res;
|
||||
|
||||
try {
|
||||
res = await this._db.updateIPLDStatusHooksBlock(dbTx, blockNumber, force);
|
||||
res = await this._db.updateStateSyncStatusIndexedBlock(dbTx, blockNumber, force);
|
||||
await dbTx.commitTransaction();
|
||||
} catch (error) {
|
||||
await dbTx.rollbackTransaction();
|
||||
@ -457,29 +423,12 @@ export class Indexer implements IndexerInterface {
|
||||
return res;
|
||||
}
|
||||
|
||||
async updateIPLDStatusCheckpointBlock (blockNumber: number, force?: boolean): Promise<IpldStatus> {
|
||||
async updateStateSyncStatusCheckpointBlock (blockNumber: number, force?: boolean): Promise<StateSyncStatus> {
|
||||
const dbTx = await this._db.createTransactionRunner();
|
||||
let res;
|
||||
|
||||
try {
|
||||
res = await this._db.updateIPLDStatusCheckpointBlock(dbTx, blockNumber, force);
|
||||
await dbTx.commitTransaction();
|
||||
} catch (error) {
|
||||
await dbTx.rollbackTransaction();
|
||||
throw error;
|
||||
} finally {
|
||||
await dbTx.release();
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
async updateIPLDStatusIPFSBlock (blockNumber: number, force?: boolean): Promise<IpldStatus> {
|
||||
const dbTx = await this._db.createTransactionRunner();
|
||||
let res;
|
||||
|
||||
try {
|
||||
res = await this._db.updateIPLDStatusIPFSBlock(dbTx, blockNumber, force);
|
||||
res = await this._db.updateStateSyncStatusCheckpointBlock(dbTx, blockNumber, force);
|
||||
await dbTx.commitTransaction();
|
||||
} catch (error) {
|
||||
await dbTx.rollbackTransaction();
|
||||
@ -501,16 +450,16 @@ export class Indexer implements IndexerInterface {
|
||||
return latestCanonicalBlock;
|
||||
}
|
||||
|
||||
async getLatestHooksProcessedBlock (): Promise<BlockProgress> {
|
||||
return this._baseIndexer.getLatestHooksProcessedBlock();
|
||||
async getLatestStateIndexedBlock (): Promise<BlockProgress> {
|
||||
return this._baseIndexer.getLatestStateIndexedBlock();
|
||||
}
|
||||
|
||||
async watchContract (address: string, kind: string, checkpoint: boolean, startingBlock: number): Promise<void> {
|
||||
return this._baseIndexer.watchContract(address, kind, checkpoint, startingBlock);
|
||||
}
|
||||
|
||||
async updateIPLDStatusMap (address: string, ipldStatus: IpldStatusInterface): Promise<void> {
|
||||
await this._baseIndexer.updateIPLDStatusMap(address, ipldStatus);
|
||||
updateStateStatusMap (address: string, stateStatus: StateStatus): void {
|
||||
this._baseIndexer.updateStateStatusMap(address, stateStatus);
|
||||
}
|
||||
|
||||
cacheContract (contract: Contract): void {
|
||||
|
||||
@ -18,8 +18,6 @@ import {
|
||||
QUEUE_EVENT_PROCESSING,
|
||||
QUEUE_BLOCK_CHECKPOINT,
|
||||
QUEUE_HOOKS,
|
||||
QUEUE_IPFS,
|
||||
JOB_KIND_PRUNE,
|
||||
JobQueueConfig,
|
||||
DEFAULT_CONFIG_PATH,
|
||||
initClients,
|
||||
@ -50,22 +48,12 @@ export class JobRunner {
|
||||
await this.subscribeEventProcessingQueue();
|
||||
await this.subscribeBlockCheckpointQueue();
|
||||
await this.subscribeHooksQueue();
|
||||
await this.subscribeIPFSQueue();
|
||||
this._baseJobRunner.handleShutdown();
|
||||
}
|
||||
|
||||
async subscribeBlockProcessingQueue (): Promise<void> {
|
||||
await this._jobQueue.subscribe(QUEUE_BLOCK_PROCESSING, async (job) => {
|
||||
await this._baseJobRunner.processBlock(job);
|
||||
|
||||
const { data: { kind } } = job;
|
||||
|
||||
// If it's a pruning job: Create a hooks job.
|
||||
if (kind === JOB_KIND_PRUNE) {
|
||||
await this.createHooksJob();
|
||||
}
|
||||
|
||||
await this._jobQueue.markComplete(job);
|
||||
});
|
||||
}
|
||||
|
||||
@ -77,165 +65,15 @@ export class JobRunner {
|
||||
|
||||
async subscribeHooksQueue (): Promise<void> {
|
||||
await this._jobQueue.subscribe(QUEUE_HOOKS, async (job) => {
|
||||
const { data: { blockHash, blockNumber } } = job;
|
||||
|
||||
// Get the current IPLD Status.
|
||||
const ipldStatus = await this._indexer.getIPLDStatus();
|
||||
|
||||
if (ipldStatus) {
|
||||
if (ipldStatus.latestHooksBlockNumber < (blockNumber - 1)) {
|
||||
// Create hooks job for parent block.
|
||||
const [parentBlock] = await this._indexer.getBlocksAtHeight(blockNumber - 1, false);
|
||||
await this.createHooksJob(parentBlock.blockHash, parentBlock.blockNumber);
|
||||
|
||||
const message = `Hooks for blockNumber ${blockNumber - 1} not processed yet, aborting`;
|
||||
log(message);
|
||||
|
||||
throw new Error(message);
|
||||
}
|
||||
|
||||
if (ipldStatus.latestHooksBlockNumber > (blockNumber - 1)) {
|
||||
log(`Hooks for blockNumber ${blockNumber} already processed`);
|
||||
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// Process the hooks for the given block number.
|
||||
await this._indexer.processCanonicalBlock(blockHash, blockNumber);
|
||||
|
||||
// Update the IPLD status.
|
||||
await this._indexer.updateIPLDStatusHooksBlock(blockNumber);
|
||||
|
||||
// Create a checkpoint job after completion of a hook job.
|
||||
await this.createCheckpointJob(blockHash, blockNumber);
|
||||
|
||||
await this._jobQueue.markComplete(job);
|
||||
await this._baseJobRunner.processHooks(job);
|
||||
});
|
||||
}
|
||||
|
||||
async subscribeBlockCheckpointQueue (): Promise<void> {
|
||||
await this._jobQueue.subscribe(QUEUE_BLOCK_CHECKPOINT, async (job) => {
|
||||
const { data: { blockHash, blockNumber } } = job;
|
||||
|
||||
// Get the current IPLD Status.
|
||||
const ipldStatus = await this._indexer.getIPLDStatus();
|
||||
assert(ipldStatus);
|
||||
|
||||
if (ipldStatus.latestCheckpointBlockNumber >= 0) {
|
||||
if (ipldStatus.latestCheckpointBlockNumber < (blockNumber - 1)) {
|
||||
// Create a checkpoint job for parent block.
|
||||
const [parentBlock] = await this._indexer.getBlocksAtHeight(blockNumber - 1, false);
|
||||
await this.createCheckpointJob(parentBlock.blockHash, parentBlock.blockNumber);
|
||||
|
||||
const message = `Checkpoints for blockNumber ${blockNumber - 1} not processed yet, aborting`;
|
||||
log(message);
|
||||
|
||||
throw new Error(message);
|
||||
}
|
||||
|
||||
if (ipldStatus.latestCheckpointBlockNumber > (blockNumber - 1)) {
|
||||
log(`Checkpoints for blockNumber ${blockNumber} already processed`);
|
||||
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// Process checkpoints for the given block.
|
||||
await this._indexer.processCheckpoint(blockHash);
|
||||
|
||||
// Update the IPLD status.
|
||||
await this._indexer.updateIPLDStatusCheckpointBlock(blockNumber);
|
||||
|
||||
// Create an IPFS job after completion of a checkpoint job.
|
||||
if (this._indexer.isIPFSConfigured()) {
|
||||
await this.createIPFSPutJob(blockHash, blockNumber);
|
||||
}
|
||||
|
||||
await this._jobQueue.markComplete(job);
|
||||
await this._baseJobRunner.processCheckpoint(job);
|
||||
});
|
||||
}
|
||||
|
||||
async subscribeIPFSQueue (): Promise<void> {
|
||||
await this._jobQueue.subscribe(QUEUE_IPFS, async (job) => {
|
||||
const { data: { blockHash, blockNumber } } = job;
|
||||
|
||||
const ipldStatus = await this._indexer.getIPLDStatus();
|
||||
assert(ipldStatus);
|
||||
|
||||
if (ipldStatus.latestIPFSBlockNumber >= 0) {
|
||||
if (ipldStatus.latestIPFSBlockNumber < (blockNumber - 1)) {
|
||||
// Create a IPFS job for parent block.
|
||||
const [parentBlock] = await this._indexer.getBlocksAtHeight(blockNumber - 1, false);
|
||||
await this.createIPFSPutJob(parentBlock.blockHash, parentBlock.blockNumber);
|
||||
|
||||
const message = `IPFS for blockNumber ${blockNumber - 1} not processed yet, aborting`;
|
||||
log(message);
|
||||
|
||||
throw new Error(message);
|
||||
}
|
||||
|
||||
if (ipldStatus.latestIPFSBlockNumber > (blockNumber - 1)) {
|
||||
log(`IPFS for blockNumber ${blockNumber} already processed`);
|
||||
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// Get IPLDBlocks for the given blocHash.
|
||||
const ipldBlocks = await this._indexer.getIPLDBlocksByHash(blockHash);
|
||||
|
||||
// Push all the IPLDBlocks to IPFS.
|
||||
for (const ipldBlock of ipldBlocks) {
|
||||
const data = this._indexer.getIPLDData(ipldBlock);
|
||||
await this._indexer.pushToIPFS(data);
|
||||
}
|
||||
|
||||
// Update the IPLD status.
|
||||
await this._indexer.updateIPLDStatusIPFSBlock(blockNumber);
|
||||
|
||||
await this._jobQueue.markComplete(job);
|
||||
});
|
||||
}
|
||||
|
||||
async createHooksJob (blockHash?: string, blockNumber?: number): Promise<void> {
|
||||
if (!blockNumber || !blockHash) {
|
||||
// Get the latest canonical block
|
||||
const latestCanonicalBlock = await this._indexer.getLatestCanonicalBlock();
|
||||
|
||||
// Create a hooks job for parent block of latestCanonicalBlock because pruning for first block is skipped as it is assumed to be a canonical block.
|
||||
blockHash = latestCanonicalBlock.parentHash;
|
||||
blockNumber = latestCanonicalBlock.blockNumber - 1;
|
||||
}
|
||||
|
||||
await this._jobQueue.pushJob(
|
||||
QUEUE_HOOKS,
|
||||
{
|
||||
blockHash,
|
||||
blockNumber
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
async createCheckpointJob (blockHash: string, blockNumber: number): Promise<void> {
|
||||
await this._jobQueue.pushJob(
|
||||
QUEUE_BLOCK_CHECKPOINT,
|
||||
{
|
||||
blockHash,
|
||||
blockNumber
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
async createIPFSPutJob (blockHash: string, blockNumber: number): Promise<void> {
|
||||
await this._jobQueue.pushJob(
|
||||
QUEUE_IPFS,
|
||||
{
|
||||
blockHash,
|
||||
blockNumber
|
||||
}
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export const main = async (): Promise<any> => {
|
||||
|
||||
@ -8,7 +8,7 @@ import debug from 'debug';
|
||||
import Decimal from 'decimal.js';
|
||||
import { GraphQLResolveInfo, GraphQLScalarType } from 'graphql';
|
||||
|
||||
import { BlockHeight, OrderDirection, gqlTotalQueryCount, gqlQueryCount, jsonBigIntStringReplacer } from '@cerc-io/util';
|
||||
import { BlockHeight, OrderDirection, gqlTotalQueryCount, gqlQueryCount, jsonBigIntStringReplacer, getResultState } from '@cerc-io/util';
|
||||
|
||||
import { Indexer } from './indexer';
|
||||
import { EventWatcher } from './events';
|
||||
@ -442,9 +442,9 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch
|
||||
gqlTotalQueryCount.inc(1);
|
||||
gqlQueryCount.labels('getStateByCID').inc(1);
|
||||
|
||||
const ipldBlock = await indexer.getIPLDBlockByCid(cid);
|
||||
const state = await indexer.getStateByCID(cid);
|
||||
|
||||
return ipldBlock && ipldBlock.block.isComplete ? indexer.getResultIPLDBlock(ipldBlock) : undefined;
|
||||
return state && state.block.isComplete ? getResultState(state) : undefined;
|
||||
},
|
||||
|
||||
getState: async (_: any, { blockHash, contractAddress, kind }: { blockHash: string, contractAddress: string, kind: string }) => {
|
||||
@ -452,9 +452,9 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch
|
||||
gqlTotalQueryCount.inc(1);
|
||||
gqlQueryCount.labels('getState').inc(1);
|
||||
|
||||
const ipldBlock = await indexer.getPrevIPLDBlock(blockHash, contractAddress, kind);
|
||||
const state = await indexer.getPrevState(blockHash, contractAddress, kind);
|
||||
|
||||
return ipldBlock && ipldBlock.block.isComplete ? indexer.getResultIPLDBlock(ipldBlock) : undefined;
|
||||
return state && state.block.isComplete ? getResultState(state) : undefined;
|
||||
},
|
||||
|
||||
getSyncStatus: async () => {
|
||||
|
||||
@ -203,7 +203,7 @@ type RoleRevokedEvent {
|
||||
sender: String!
|
||||
}
|
||||
|
||||
type ResultIPLDBlock {
|
||||
type ResultState {
|
||||
block: _Block_!
|
||||
contractAddress: String!
|
||||
cid: String!
|
||||
@ -248,8 +248,8 @@ type Query {
|
||||
claim(id: String!, block: Block_height): Claim!
|
||||
slash(id: String!, block: Block_height): Slash!
|
||||
account(id: String!, block: Block_height): Account!
|
||||
getStateByCID(cid: String!): ResultIPLDBlock
|
||||
getState(blockHash: String!, contractAddress: String!, kind: String): ResultIPLDBlock
|
||||
getStateByCID(cid: String!): ResultState
|
||||
getState(blockHash: String!, contractAddress: String!, kind: String): ResultState
|
||||
getSyncStatus: SyncStatus
|
||||
}
|
||||
|
||||
|
||||
@ -14,11 +14,11 @@ import { BlockProgress } from '../../entity/BlockProgress';
|
||||
import { Allowance } from '../../entity/Allowance';
|
||||
import { Balance } from '../../entity/Balance';
|
||||
|
||||
const log = debug('vulcanize:reset-state');
|
||||
const log = debug('vulcanize:reset-watcher');
|
||||
|
||||
export const command = 'state';
|
||||
export const command = 'watcher';
|
||||
|
||||
export const desc = 'Reset state to block number';
|
||||
export const desc = 'Reset watcher to a block number';
|
||||
|
||||
export const builder = {
|
||||
blockNumber: {
|
||||
@ -78,5 +78,5 @@ export const handler = async (argv: any): Promise<void> => {
|
||||
await dbTx.release();
|
||||
}
|
||||
|
||||
log('Reset state successfully');
|
||||
log('Reset watcher successfully');
|
||||
};
|
||||
@ -14,8 +14,8 @@ import { Contract } from './entity/Contract';
|
||||
import { Event } from './entity/Event';
|
||||
import { SyncStatus } from './entity/SyncStatus';
|
||||
import { BlockProgress } from './entity/BlockProgress';
|
||||
import { IPLDBlock } from './entity/IPLDBlock';
|
||||
import { IpldStatus } from './entity/IpldStatus';
|
||||
import { State } from './entity/State';
|
||||
import { StateSyncStatus } from './entity/StateSyncStatus';
|
||||
|
||||
export class Database implements DatabaseInterface {
|
||||
_config: ConnectionOptions
|
||||
@ -41,45 +41,57 @@ export class Database implements DatabaseInterface {
|
||||
return this._baseDatabase.close();
|
||||
}
|
||||
|
||||
getNewIPLDBlock (): IPLDBlock {
|
||||
return new IPLDBlock();
|
||||
getNewState (): State {
|
||||
return new State();
|
||||
}
|
||||
|
||||
async getIPLDBlocks (where: FindConditions<IPLDBlock>): Promise<IPLDBlock[]> {
|
||||
const repo = this._conn.getRepository(IPLDBlock);
|
||||
async getStates (where: FindConditions<State>): Promise<State[]> {
|
||||
const repo = this._conn.getRepository(State);
|
||||
|
||||
return this._baseDatabase.getIPLDBlocks(repo, where);
|
||||
return this._baseDatabase.getStates(repo, where);
|
||||
}
|
||||
|
||||
async getLatestIPLDBlock (contractAddress: string, kind: StateKind | null, blockNumber?: number): Promise<IPLDBlock | undefined> {
|
||||
const repo = this._conn.getRepository(IPLDBlock);
|
||||
async getLatestState (contractAddress: string, kind: StateKind | null, blockNumber?: number): Promise<State | undefined> {
|
||||
const repo = this._conn.getRepository(State);
|
||||
|
||||
return this._baseDatabase.getLatestIPLDBlock(repo, contractAddress, kind, blockNumber);
|
||||
return this._baseDatabase.getLatestState(repo, contractAddress, kind, blockNumber);
|
||||
}
|
||||
|
||||
// Fetch all diff IPLDBlocks after the specified block number.
|
||||
async getDiffIPLDBlocksInRange (contractAddress: string, startblock: number, endBlock: number): Promise<IPLDBlock[]> {
|
||||
const repo = this._conn.getRepository(IPLDBlock);
|
||||
async getPrevState (blockHash: string, contractAddress: string, kind?: string): Promise<State | undefined> {
|
||||
const repo = this._conn.getRepository(State);
|
||||
|
||||
return this._baseDatabase.getDiffIPLDBlocksInRange(repo, contractAddress, startblock, endBlock);
|
||||
return this._baseDatabase.getPrevState(repo, blockHash, contractAddress, kind);
|
||||
}
|
||||
|
||||
async saveOrUpdateIPLDBlock (dbTx: QueryRunner, ipldBlock: IPLDBlock): Promise<IPLDBlock> {
|
||||
const repo = dbTx.manager.getRepository(IPLDBlock);
|
||||
// Fetch all diff States after the specified block number.
|
||||
async getDiffStatesInRange (contractAddress: string, startblock: number, endBlock: number): Promise<State[]> {
|
||||
const repo = this._conn.getRepository(State);
|
||||
|
||||
return this._baseDatabase.saveOrUpdateIPLDBlock(repo, ipldBlock);
|
||||
return this._baseDatabase.getDiffStatesInRange(repo, contractAddress, startblock, endBlock);
|
||||
}
|
||||
|
||||
async removeIPLDBlocks (dbTx: QueryRunner, blockNumber: number, kind: string): Promise<void> {
|
||||
const repo = dbTx.manager.getRepository(IPLDBlock);
|
||||
async saveOrUpdateState (dbTx: QueryRunner, state: State): Promise<State> {
|
||||
const repo = dbTx.manager.getRepository(State);
|
||||
|
||||
await this._baseDatabase.removeIPLDBlocks(repo, blockNumber, kind);
|
||||
return this._baseDatabase.saveOrUpdateState(repo, state);
|
||||
}
|
||||
|
||||
async getIPLDStatus (): Promise<IpldStatus | undefined> {
|
||||
const repo = this._conn.getRepository(IpldStatus);
|
||||
async removeStates (dbTx: QueryRunner, blockNumber: number, kind: string): Promise<void> {
|
||||
const repo = dbTx.manager.getRepository(State);
|
||||
|
||||
return this._baseDatabase.getIPLDStatus(repo);
|
||||
await this._baseDatabase.removeStates(repo, blockNumber, kind);
|
||||
}
|
||||
|
||||
async removeStatesAfterBlock (dbTx: QueryRunner, blockNumber: number): Promise<void> {
|
||||
const repo = dbTx.manager.getRepository(State);
|
||||
|
||||
await this._baseDatabase.removeStatesAfterBlock(repo, blockNumber);
|
||||
}
|
||||
|
||||
async getStateSyncStatus (): Promise<StateSyncStatus | undefined> {
|
||||
const repo = this._conn.getRepository(StateSyncStatus);
|
||||
|
||||
return this._baseDatabase.getStateSyncStatus(repo);
|
||||
}
|
||||
|
||||
async getBalance ({ blockHash, token, owner }: { blockHash: string, token: string, owner: string }): Promise<Balance | undefined> {
|
||||
|
||||
@ -1,20 +0,0 @@
|
||||
//
|
||||
// Copyright 2022 Vulcanize, Inc.
|
||||
//
|
||||
|
||||
import { Entity, PrimaryGeneratedColumn, Column } from 'typeorm';
|
||||
|
||||
@Entity()
|
||||
export class IpldStatus {
|
||||
@PrimaryGeneratedColumn()
|
||||
id!: number;
|
||||
|
||||
@Column('integer')
|
||||
latestHooksBlockNumber!: number;
|
||||
|
||||
@Column('integer', { nullable: true })
|
||||
latestCheckpointBlockNumber!: number;
|
||||
|
||||
@Column('integer', { nullable: true })
|
||||
latestIPFSBlockNumber!: number;
|
||||
}
|
||||
@ -12,7 +12,7 @@ import { BlockProgress } from './BlockProgress';
|
||||
@Index(['cid'], { unique: true })
|
||||
@Index(['block', 'contractAddress'])
|
||||
@Index(['block', 'contractAddress', 'kind'], { unique: true })
|
||||
export class IPLDBlock {
|
||||
export class State {
|
||||
@PrimaryGeneratedColumn()
|
||||
id!: number;
|
||||
|
||||
@ -5,16 +5,13 @@
|
||||
import { Entity, PrimaryGeneratedColumn, Column } from 'typeorm';
|
||||
|
||||
@Entity()
|
||||
export class IpldStatus {
|
||||
export class StateSyncStatus {
|
||||
@PrimaryGeneratedColumn()
|
||||
id!: number;
|
||||
|
||||
@Column('integer')
|
||||
latestHooksBlockNumber!: number;
|
||||
latestIndexedBlockNumber!: number;
|
||||
|
||||
@Column('integer', { nullable: true })
|
||||
latestCheckpointBlockNumber!: number;
|
||||
|
||||
@Column('integer', { nullable: true })
|
||||
latestIPFSBlockNumber!: number;
|
||||
}
|
||||
@ -12,16 +12,17 @@ import { BaseProvider } from '@ethersproject/providers';
|
||||
|
||||
import { EthClient } from '@cerc-io/ipld-eth-client';
|
||||
import { MappingKey, StorageLayout } from '@cerc-io/solidity-mapper';
|
||||
import { IndexerInterface, Indexer as BaseIndexer, ValueResult, UNKNOWN_EVENT_NAME, JobQueue, Where, QueryOptions, ServerConfig, IPFSClient, IpldStatus as IpldStatusInterface } from '@cerc-io/util';
|
||||
import { IndexerInterface, Indexer as BaseIndexer, ValueResult, UNKNOWN_EVENT_NAME, JobQueue, Where, QueryOptions, ServerConfig, StateStatus } from '@cerc-io/util';
|
||||
|
||||
import { Database } from './database';
|
||||
import { Event } from './entity/Event';
|
||||
import { fetchTokenDecimals, fetchTokenName, fetchTokenSymbol, fetchTokenTotalSupply } from './utils';
|
||||
import { SyncStatus } from './entity/SyncStatus';
|
||||
import { StateSyncStatus } from './entity/StateSyncStatus';
|
||||
import artifacts from './artifacts/ERC20.json';
|
||||
import { BlockProgress } from './entity/BlockProgress';
|
||||
import { Contract } from './entity/Contract';
|
||||
import { IPLDBlock } from './entity/IPLDBlock';
|
||||
import { State } from './entity/State';
|
||||
|
||||
const log = debug('vulcanize:indexer');
|
||||
const JSONbigNative = JSONbig({ useNativeBigInt: true });
|
||||
@ -64,8 +65,7 @@ export class Indexer implements IndexerInterface {
|
||||
this._ethProvider = ethProvider;
|
||||
this._serverConfig = serverConfig;
|
||||
this._serverMode = serverConfig.mode;
|
||||
const ipfsClient = new IPFSClient(this._serverConfig.ipfsApiAddr);
|
||||
this._baseIndexer = new BaseIndexer(serverConfig, this._db, this._ethClient, this._ethProvider, jobQueue, ipfsClient);
|
||||
this._baseIndexer = new BaseIndexer(serverConfig, this._db, this._ethClient, this._ethProvider, jobQueue);
|
||||
|
||||
const { abi, storageLayout } = artifacts;
|
||||
|
||||
@ -250,8 +250,16 @@ export class Indexer implements IndexerInterface {
|
||||
);
|
||||
}
|
||||
|
||||
getIPLDData (ipldBlock: IPLDBlock): any {
|
||||
return this._baseIndexer.getIPLDData(ipldBlock);
|
||||
async processCanonicalBlock (blockHash: string, blockNumber: number): Promise<void> {
|
||||
// TODO Implement
|
||||
}
|
||||
|
||||
async processCheckpoint (blockHash: string): Promise<void> {
|
||||
// TODO Implement
|
||||
}
|
||||
|
||||
getStateData (state: State): any {
|
||||
return this._baseIndexer.getStateData(state);
|
||||
}
|
||||
|
||||
async triggerIndexingOnEvent (event: Event): Promise<void> {
|
||||
@ -298,6 +306,30 @@ export class Indexer implements IndexerInterface {
|
||||
return { eventName, eventInfo };
|
||||
}
|
||||
|
||||
async getStateSyncStatus (): Promise<StateSyncStatus | undefined> {
|
||||
return this._db.getStateSyncStatus();
|
||||
}
|
||||
|
||||
async updateStateSyncStatusIndexedBlock (blockNumber: number, force?: boolean): Promise<StateSyncStatus> {
|
||||
// TODO Implement
|
||||
return {} as StateSyncStatus;
|
||||
}
|
||||
|
||||
async updateStateSyncStatusCheckpointBlock (blockNumber: number, force?: boolean): Promise<StateSyncStatus> {
|
||||
// TODO Implement
|
||||
return {} as StateSyncStatus;
|
||||
}
|
||||
|
||||
async getLatestCanonicalBlock (): Promise<BlockProgress> {
|
||||
const syncStatus = await this.getSyncStatus();
|
||||
assert(syncStatus);
|
||||
|
||||
const latestCanonicalBlock = await this.getBlockProgress(syncStatus.latestCanonicalBlockHash);
|
||||
assert(latestCanonicalBlock);
|
||||
|
||||
return latestCanonicalBlock;
|
||||
}
|
||||
|
||||
async getEventsByFilter (blockHash: string, contract: string, name?: string): Promise<Array<Event>> {
|
||||
return this._baseIndexer.getEventsByFilter(blockHash, contract, name);
|
||||
}
|
||||
@ -310,8 +342,8 @@ export class Indexer implements IndexerInterface {
|
||||
return this._baseIndexer.watchContract(address, kind, checkpoint, startingBlock);
|
||||
}
|
||||
|
||||
async updateIPLDStatusMap (address: string, ipldStatus: IpldStatusInterface): Promise<void> {
|
||||
await this._baseIndexer.updateIPLDStatusMap(address, ipldStatus);
|
||||
updateStateStatusMap (address: string, stateStatus: StateStatus): void {
|
||||
this._baseIndexer.updateStateStatusMap(address, stateStatus);
|
||||
}
|
||||
|
||||
cacheContract (contract: Contract): void {
|
||||
|
||||
@ -8,12 +8,6 @@
|
||||
yarn
|
||||
```
|
||||
|
||||
* Run the IPFS (go-ipfs version 0.12.2) daemon:
|
||||
|
||||
```bash
|
||||
ipfs daemon
|
||||
```
|
||||
|
||||
* Create a postgres12 database for the watcher:
|
||||
|
||||
```bash
|
||||
@ -42,9 +36,9 @@
|
||||
```
|
||||
|
||||
* The following core services should be setup and running on localhost:
|
||||
|
||||
|
||||
* `vulcanize/go-ethereum` [v1.10.18-statediff-4.0.2-alpha](https://github.com/vulcanize/go-ethereum/releases/tag/v1.10.18-statediff-4.0.2-alpha) on port 8545
|
||||
|
||||
|
||||
* `vulcanize/ipld-eth-server` [v4.0.3-alpha](https://github.com/vulcanize/ipld-eth-server/releases/tag/v4.0.3-alpha) with native GQL API enabled, on port 8082
|
||||
|
||||
* In the [config file](./environments/local.toml):
|
||||
@ -53,7 +47,7 @@
|
||||
|
||||
* Update the `upstream` config and provide the `ipld-eth-server` GQL API endpoint.
|
||||
|
||||
* Update the `server` config with state checkpoint settings and provide the IPFS API address.
|
||||
* Update the `server` config with state checkpoint settings.
|
||||
|
||||
## Customize
|
||||
|
||||
@ -65,11 +59,11 @@
|
||||
|
||||
* Generating state:
|
||||
|
||||
* Edit the custom hook function `createInitialState` (triggered if the watcher passes the start block, checkpoint: `true`) in [hooks.ts](./src/hooks.ts) to save an initial state `IPLDBlock` using the `Indexer` object.
|
||||
* Edit the custom hook function `createInitialState` (triggered if the watcher passes the start block, checkpoint: `true`) in [hooks.ts](./src/hooks.ts) to save an initial `State` using the `Indexer` object.
|
||||
|
||||
* Edit the custom hook function `createStateDiff` (triggered on a block) in [hooks.ts](./src/hooks.ts) to save the state in a `diff` `IPLDBlock` using the `Indexer` object. The default state (if exists) is updated.
|
||||
* Edit the custom hook function `createStateDiff` (triggered on a block) in [hooks.ts](./src/hooks.ts) to save the state in a `diff` `State` using the `Indexer` object. The default state (if exists) is updated.
|
||||
|
||||
* Edit the custom hook function `createStateCheckpoint` (triggered just before default and CLI checkpoint) in [hooks.ts](./src/hooks.ts) to save the state in a `checkpoint` `IPLDBlock` using the `Indexer` object.
|
||||
* Edit the custom hook function `createStateCheckpoint` (triggered just before default and CLI checkpoint) in [hooks.ts](./src/hooks.ts) to save the state in a `checkpoint` `State` using the `Indexer` object.
|
||||
|
||||
## Run
|
||||
|
||||
@ -136,10 +130,10 @@ GQL console: http://localhost:3006/graphql
|
||||
|
||||
* To reset the watcher to a previous block number:
|
||||
|
||||
* Reset state:
|
||||
* Reset watcher:
|
||||
|
||||
```bash
|
||||
yarn reset state --block-number <previous-block-number>
|
||||
yarn reset watcher --block-number <previous-block-number>
|
||||
```
|
||||
|
||||
* Reset job-queue:
|
||||
|
||||
@ -26,14 +26,14 @@
|
||||
|
||||
```bash
|
||||
docker-compose version
|
||||
|
||||
|
||||
# docker-compose version 1.29.2, build 5becea4c
|
||||
```
|
||||
|
||||
* Run the stack-orchestrator
|
||||
|
||||
```bash
|
||||
cd stack-orchestrator/helper-scripts
|
||||
cd stack-orchestrator/helper-scripts
|
||||
```
|
||||
|
||||
```bash
|
||||
@ -45,22 +45,11 @@
|
||||
-p ../config.sh
|
||||
```
|
||||
|
||||
* Run the IPFS (go-ipfs version 0.12.2) daemon:
|
||||
|
||||
```bash
|
||||
ipfs daemon
|
||||
|
||||
# API server listening on /ip4/127.0.0.1/tcp/5001
|
||||
```
|
||||
The IPFS API address can be seen in the output.
|
||||
|
||||
* In the [config file](./environments/local.toml) update the `server.ipfsApiAddr` config with the IPFS API address.
|
||||
|
||||
* Create a postgres12 database for the watcher:
|
||||
|
||||
```bash
|
||||
sudo su - postgres
|
||||
|
||||
|
||||
# If database already exists
|
||||
# dropdb erc721-watcher
|
||||
|
||||
@ -135,7 +124,7 @@
|
||||
```
|
||||
|
||||
* Get the signer account address and export to a shell variable:
|
||||
|
||||
|
||||
```bash
|
||||
yarn account
|
||||
```
|
||||
@ -262,11 +251,11 @@
|
||||
|
||||
* A Transfer event to SIGNER_ADDRESS shall be visible in the subscription at endpoint.
|
||||
|
||||
* An auto-generated `diff_staged` IPLDBlock should be added with parent CID pointing to the initial checkpoint IPLDBlock.
|
||||
* An auto-generated `diff_staged` `State` should be added with parent CID pointing to the initial `checkpoint` `State`.
|
||||
|
||||
* Custom property `transferCount` should be 1 initially.
|
||||
|
||||
* Run the `getState` query at the endpoint to get the latest IPLDBlock for NFT_ADDRESS:
|
||||
* Run the `getState` query at the endpoint to get the latest `State` for NFT_ADDRESS:
|
||||
|
||||
```graphql
|
||||
query {
|
||||
@ -291,7 +280,7 @@
|
||||
}
|
||||
```
|
||||
|
||||
* `diff` IPLDBlocks get created corresponding to the `diff_staged` blocks when their respective eth_blocks reach the pruned region.
|
||||
* `diff` States get created corresponding to the `diff_staged` blocks when their respective eth_blocks reach the pruned region.
|
||||
|
||||
* `data` contains the default state and also the custom state property `transferCount` that is indexed in [hooks.ts](./src/hooks.ts) file.
|
||||
|
||||
@ -360,9 +349,9 @@
|
||||
|
||||
* A Transfer event to $RECIPIENT_ADDRESS shall be visible in the subscription at endpoint.
|
||||
|
||||
* An auto-generated `diff_staged` IPLDBlock should be added with parent CID pointing to the previous IPLDBlock.
|
||||
* An auto-generated `diff_staged` State should be added with parent CID pointing to the previous State.
|
||||
|
||||
* Custom property `transferCount` should be incremented after transfer. This can be checked in the `getState` query and in IPFS webUI mentioned in the later steps.
|
||||
* Custom property `transferCount` should be incremented after transfer. This can be checked in the `getState` query.
|
||||
|
||||
* Get the latest blockHash and replace the blockHash in the above `eth_call` query. The result should be different and the token should be transferred to the recipient.
|
||||
|
||||
@ -380,11 +369,7 @@
|
||||
|
||||
* The latest checkpoint should have the aggregate of state diffs since the last checkpoint.
|
||||
|
||||
* The IPLDBlock entries can be seen in pg-admin in table ipld_block.
|
||||
|
||||
* All the diff and checkpoint IPLDBlocks should pushed to IPFS.
|
||||
|
||||
* Open IPFS WebUI http://127.0.0.1:5001/webui and search for IPLDBlocks using their CIDs.
|
||||
* The `State` entries can be seen in pg-admin in table `state`.
|
||||
|
||||
* The state should have auto indexed data and also custom property `transferCount` according to code in [hooks](./src/hooks.ts) file `handleEvent` method.
|
||||
|
||||
|
||||
@ -9,9 +9,6 @@
|
||||
# Checkpoint interval in number of blocks.
|
||||
checkpointInterval = 2000
|
||||
|
||||
# IPFS API address (can be taken from the output on running the IPFS daemon).
|
||||
ipfsApiAddr = "/ip4/127.0.0.1/tcp/5001"
|
||||
|
||||
# Boolean to filter logs by contract.
|
||||
filterLogs = false
|
||||
|
||||
|
||||
@ -61,16 +61,16 @@ const main = async (): Promise<void> => {
|
||||
const exportData: any = {
|
||||
snapshotBlock: {},
|
||||
contracts: [],
|
||||
ipldCheckpoints: []
|
||||
stateCheckpoints: []
|
||||
};
|
||||
|
||||
const contracts = await db.getContracts();
|
||||
let block = await indexer.getLatestHooksProcessedBlock();
|
||||
let block = await indexer.getLatestStateIndexedBlock();
|
||||
assert(block);
|
||||
|
||||
if (argv.blockNumber) {
|
||||
if (argv.blockNumber > block.blockNumber) {
|
||||
throw new Error(`Export snapshot block height ${argv.blockNumber} should be less than latest hooks processed block height ${block.blockNumber}`);
|
||||
throw new Error(`Export snapshot block height ${argv.blockNumber} should be less than latest state indexed block height ${block.blockNumber}`);
|
||||
}
|
||||
|
||||
const blocksAtSnapshotHeight = await indexer.getBlocksAtHeight(argv.blockNumber, false);
|
||||
@ -107,19 +107,15 @@ const main = async (): Promise<void> => {
|
||||
if (contract.checkpoint) {
|
||||
await indexer.createCheckpoint(contract.address, block.blockHash);
|
||||
|
||||
const ipldBlock = await indexer.getLatestIPLDBlock(contract.address, StateKind.Checkpoint, block.blockNumber);
|
||||
assert(ipldBlock);
|
||||
const state = await indexer.getLatestState(contract.address, StateKind.Checkpoint, block.blockNumber);
|
||||
assert(state);
|
||||
|
||||
const data = indexer.getIPLDData(ipldBlock);
|
||||
const data = indexer.getStateData(state);
|
||||
|
||||
if (indexer.isIPFSConfigured()) {
|
||||
await indexer.pushToIPFS(data);
|
||||
}
|
||||
|
||||
exportData.ipldCheckpoints.push({
|
||||
contractAddress: ipldBlock.contractAddress,
|
||||
cid: ipldBlock.cid,
|
||||
kind: ipldBlock.kind,
|
||||
exportData.stateCheckpoints.push({
|
||||
contractAddress: state.contractAddress,
|
||||
cid: state.cid,
|
||||
kind: state.kind,
|
||||
data
|
||||
});
|
||||
}
|
||||
|
||||
@ -17,7 +17,7 @@ import * as codec from '@ipld/dag-cbor';
|
||||
import { Database } from '../database';
|
||||
import { Indexer } from '../indexer';
|
||||
import { EventWatcher } from '../events';
|
||||
import { IPLDBlock } from '../entity/IPLDBlock';
|
||||
import { State } from '../entity/State';
|
||||
|
||||
const log = debug('vulcanize:import-state');
|
||||
|
||||
@ -91,16 +91,16 @@ export const main = async (): Promise<any> => {
|
||||
const block = await indexer.getBlockProgress(importData.snapshotBlock.blockHash);
|
||||
assert(block);
|
||||
|
||||
// Fill the IPLDBlocks.
|
||||
for (const checkpoint of importData.ipldCheckpoints) {
|
||||
let ipldBlock = new IPLDBlock();
|
||||
// Fill the States.
|
||||
for (const checkpoint of importData.stateCheckpoints) {
|
||||
let state = new State();
|
||||
|
||||
ipldBlock = Object.assign(ipldBlock, checkpoint);
|
||||
ipldBlock.block = block;
|
||||
state = Object.assign(state, checkpoint);
|
||||
state.block = block;
|
||||
|
||||
ipldBlock.data = Buffer.from(codec.encode(ipldBlock.data));
|
||||
state.data = Buffer.from(codec.encode(state.data));
|
||||
|
||||
ipldBlock = await indexer.saveOrUpdateIPLDBlock(ipldBlock);
|
||||
state = await indexer.saveOrUpdateState(state);
|
||||
}
|
||||
|
||||
// Mark snapshot block as completely processed.
|
||||
@ -108,12 +108,12 @@ export const main = async (): Promise<any> => {
|
||||
await indexer.updateBlockProgress(block, block.lastProcessedEventIndex);
|
||||
await indexer.updateSyncStatusChainHead(block.blockHash, block.blockNumber);
|
||||
await indexer.updateSyncStatusIndexedBlock(block.blockHash, block.blockNumber);
|
||||
await indexer.updateIPLDStatusHooksBlock(block.blockNumber);
|
||||
await indexer.updateIPLDStatusCheckpointBlock(block.blockNumber);
|
||||
await indexer.updateStateSyncStatusIndexedBlock(block.blockNumber);
|
||||
await indexer.updateStateSyncStatusCheckpointBlock(block.blockNumber);
|
||||
|
||||
// The 'diff_staged' and 'init' IPLD blocks are unnecessary as checkpoints have been already created for the snapshot block.
|
||||
await indexer.removeIPLDBlocks(block.blockNumber, StateKind.Init);
|
||||
await indexer.removeIPLDBlocks(block.blockNumber, StateKind.DiffStaged);
|
||||
// The 'diff_staged' and 'init' State entries are unnecessary as checkpoints have been already created for the snapshot block.
|
||||
await indexer.removeStates(block.blockNumber, StateKind.Init);
|
||||
await indexer.removeStates(block.blockNumber, StateKind.DiffStaged);
|
||||
|
||||
log(`Import completed for snapshot block at height ${block.blockNumber}`);
|
||||
};
|
||||
|
||||
@ -53,12 +53,12 @@ const main = async (): Promise<void> => {
|
||||
const indexer = new Indexer(config.server, db, ethClient, ethProvider, jobQueue);
|
||||
await indexer.init();
|
||||
|
||||
const ipldBlock = await indexer.getIPLDBlockByCid(argv.cid);
|
||||
assert(ipldBlock, 'IPLDBlock for the provided CID doesn\'t exist.');
|
||||
const state = await indexer.getStateByCID(argv.cid);
|
||||
assert(state, 'State for the provided CID doesn\'t exist.');
|
||||
|
||||
const ipldData = await indexer.getIPLDData(ipldBlock);
|
||||
const stateData = await indexer.getStateData(state);
|
||||
|
||||
log(util.inspect(ipldData, false, null));
|
||||
log(util.inspect(stateData, false, null));
|
||||
};
|
||||
|
||||
main().catch(err => {
|
||||
|
||||
@ -27,11 +27,11 @@ import { _Balances } from '../../entity/_Balances';
|
||||
import { _TokenApprovals } from '../../entity/_TokenApprovals';
|
||||
import { _OperatorApprovals } from '../../entity/_OperatorApprovals';
|
||||
|
||||
const log = debug('vulcanize:reset-state');
|
||||
const log = debug('vulcanize:reset-watcher');
|
||||
|
||||
export const command = 'state';
|
||||
export const command = 'watcher';
|
||||
|
||||
export const desc = 'Reset state to block number';
|
||||
export const desc = 'Reset watcher to a block number';
|
||||
|
||||
export const builder = {
|
||||
blockNumber: {
|
||||
@ -85,19 +85,15 @@ export const handler = async (argv: any): Promise<void> => {
|
||||
await indexer.updateSyncStatusCanonicalBlock(blockProgress.blockHash, blockProgress.blockNumber, true);
|
||||
}
|
||||
|
||||
const ipldStatus = await indexer.getIPLDStatus();
|
||||
const stateSyncStatus = await indexer.getStateSyncStatus();
|
||||
|
||||
if (ipldStatus) {
|
||||
if (ipldStatus.latestHooksBlockNumber > blockProgress.blockNumber) {
|
||||
await indexer.updateIPLDStatusHooksBlock(blockProgress.blockNumber, true);
|
||||
if (stateSyncStatus) {
|
||||
if (stateSyncStatus.latestIndexedBlockNumber > blockProgress.blockNumber) {
|
||||
await indexer.updateStateSyncStatusIndexedBlock(blockProgress.blockNumber, true);
|
||||
}
|
||||
|
||||
if (ipldStatus.latestCheckpointBlockNumber > blockProgress.blockNumber) {
|
||||
await indexer.updateIPLDStatusCheckpointBlock(blockProgress.blockNumber, true);
|
||||
}
|
||||
|
||||
if (ipldStatus.latestIPFSBlockNumber > blockProgress.blockNumber) {
|
||||
await indexer.updateIPLDStatusIPFSBlock(blockProgress.blockNumber, true);
|
||||
if (stateSyncStatus.latestCheckpointBlockNumber > blockProgress.blockNumber) {
|
||||
await indexer.updateStateSyncStatusCheckpointBlock(blockProgress.blockNumber, true);
|
||||
}
|
||||
}
|
||||
|
||||
@ -111,5 +107,5 @@ export const handler = async (argv: any): Promise<void> => {
|
||||
await dbTx.release();
|
||||
}
|
||||
|
||||
log('Reset state successfully');
|
||||
log('Reset watcher successfully');
|
||||
};
|
||||
@ -11,9 +11,9 @@ import { Database as BaseDatabase, DatabaseInterface, QueryOptions, StateKind, W
|
||||
import { Contract } from './entity/Contract';
|
||||
import { Event } from './entity/Event';
|
||||
import { SyncStatus } from './entity/SyncStatus';
|
||||
import { IpldStatus } from './entity/IpldStatus';
|
||||
import { StateSyncStatus } from './entity/StateSyncStatus';
|
||||
import { BlockProgress } from './entity/BlockProgress';
|
||||
import { IPLDBlock } from './entity/IPLDBlock';
|
||||
import { State } from './entity/State';
|
||||
import { SupportsInterface } from './entity/SupportsInterface';
|
||||
import { BalanceOf } from './entity/BalanceOf';
|
||||
import { OwnerOf } from './entity/OwnerOf';
|
||||
@ -300,69 +300,69 @@ export class Database implements DatabaseInterface {
|
||||
return repo.save(entity);
|
||||
}
|
||||
|
||||
getNewIPLDBlock (): IPLDBlock {
|
||||
return new IPLDBlock();
|
||||
getNewState (): State {
|
||||
return new State();
|
||||
}
|
||||
|
||||
async getIPLDBlocks (where: FindConditions<IPLDBlock>): Promise<IPLDBlock[]> {
|
||||
const repo = this._conn.getRepository(IPLDBlock);
|
||||
async getStates (where: FindConditions<State>): Promise<State[]> {
|
||||
const repo = this._conn.getRepository(State);
|
||||
|
||||
return this._baseDatabase.getIPLDBlocks(repo, where);
|
||||
return this._baseDatabase.getStates(repo, where);
|
||||
}
|
||||
|
||||
async getLatestIPLDBlock (contractAddress: string, kind: StateKind | null, blockNumber?: number): Promise<IPLDBlock | undefined> {
|
||||
const repo = this._conn.getRepository(IPLDBlock);
|
||||
async getLatestState (contractAddress: string, kind: StateKind | null, blockNumber?: number): Promise<State | undefined> {
|
||||
const repo = this._conn.getRepository(State);
|
||||
|
||||
return this._baseDatabase.getLatestIPLDBlock(repo, contractAddress, kind, blockNumber);
|
||||
return this._baseDatabase.getLatestState(repo, contractAddress, kind, blockNumber);
|
||||
}
|
||||
|
||||
async getPrevIPLDBlock (blockHash: string, contractAddress: string, kind?: string): Promise<IPLDBlock | undefined> {
|
||||
const repo = this._conn.getRepository(IPLDBlock);
|
||||
async getPrevState (blockHash: string, contractAddress: string, kind?: string): Promise<State | undefined> {
|
||||
const repo = this._conn.getRepository(State);
|
||||
|
||||
return this._baseDatabase.getPrevIPLDBlock(repo, blockHash, contractAddress, kind);
|
||||
return this._baseDatabase.getPrevState(repo, blockHash, contractAddress, kind);
|
||||
}
|
||||
|
||||
// Fetch all diff IPLDBlocks after the specified block number.
|
||||
async getDiffIPLDBlocksInRange (contractAddress: string, startBlock: number, endBlock: number): Promise<IPLDBlock[]> {
|
||||
const repo = this._conn.getRepository(IPLDBlock);
|
||||
// Fetch all diff States after the specified block number.
|
||||
async getDiffStatesInRange (contractAddress: string, startblock: number, endBlock: number): Promise<State[]> {
|
||||
const repo = this._conn.getRepository(State);
|
||||
|
||||
return this._baseDatabase.getDiffIPLDBlocksInRange(repo, contractAddress, startBlock, endBlock);
|
||||
return this._baseDatabase.getDiffStatesInRange(repo, contractAddress, startblock, endBlock);
|
||||
}
|
||||
|
||||
async saveOrUpdateIPLDBlock (dbTx: QueryRunner, ipldBlock: IPLDBlock): Promise<IPLDBlock> {
|
||||
const repo = dbTx.manager.getRepository(IPLDBlock);
|
||||
async saveOrUpdateState (dbTx: QueryRunner, state: State): Promise<State> {
|
||||
const repo = dbTx.manager.getRepository(State);
|
||||
|
||||
return this._baseDatabase.saveOrUpdateIPLDBlock(repo, ipldBlock);
|
||||
return this._baseDatabase.saveOrUpdateState(repo, state);
|
||||
}
|
||||
|
||||
async removeIPLDBlocks (dbTx: QueryRunner, blockNumber: number, kind: string): Promise<void> {
|
||||
const repo = dbTx.manager.getRepository(IPLDBlock);
|
||||
async removeStates (dbTx: QueryRunner, blockNumber: number, kind: string): Promise<void> {
|
||||
const repo = dbTx.manager.getRepository(State);
|
||||
|
||||
await this._baseDatabase.removeIPLDBlocks(repo, blockNumber, kind);
|
||||
await this._baseDatabase.removeStates(repo, blockNumber, kind);
|
||||
}
|
||||
|
||||
async getIPLDStatus (): Promise<IpldStatus | undefined> {
|
||||
const repo = this._conn.getRepository(IpldStatus);
|
||||
async removeStatesAfterBlock (dbTx: QueryRunner, blockNumber: number): Promise<void> {
|
||||
const repo = dbTx.manager.getRepository(State);
|
||||
|
||||
return this._baseDatabase.getIPLDStatus(repo);
|
||||
await this._baseDatabase.removeStatesAfterBlock(repo, blockNumber);
|
||||
}
|
||||
|
||||
async updateIPLDStatusHooksBlock (queryRunner: QueryRunner, blockNumber: number, force?: boolean): Promise<IpldStatus> {
|
||||
const repo = queryRunner.manager.getRepository(IpldStatus);
|
||||
async getStateSyncStatus (): Promise<StateSyncStatus | undefined> {
|
||||
const repo = this._conn.getRepository(StateSyncStatus);
|
||||
|
||||
return this._baseDatabase.updateIPLDStatusHooksBlock(repo, blockNumber, force);
|
||||
return this._baseDatabase.getStateSyncStatus(repo);
|
||||
}
|
||||
|
||||
async updateIPLDStatusCheckpointBlock (queryRunner: QueryRunner, blockNumber: number, force?: boolean): Promise<IpldStatus> {
|
||||
const repo = queryRunner.manager.getRepository(IpldStatus);
|
||||
async updateStateSyncStatusIndexedBlock (queryRunner: QueryRunner, blockNumber: number, force?: boolean): Promise<StateSyncStatus> {
|
||||
const repo = queryRunner.manager.getRepository(StateSyncStatus);
|
||||
|
||||
return this._baseDatabase.updateIPLDStatusCheckpointBlock(repo, blockNumber, force);
|
||||
return this._baseDatabase.updateStateSyncStatusIndexedBlock(repo, blockNumber, force);
|
||||
}
|
||||
|
||||
async updateIPLDStatusIPFSBlock (queryRunner: QueryRunner, blockNumber: number, force?: boolean): Promise<IpldStatus> {
|
||||
const repo = queryRunner.manager.getRepository(IpldStatus);
|
||||
async updateStateSyncStatusCheckpointBlock (queryRunner: QueryRunner, blockNumber: number, force?: boolean): Promise<StateSyncStatus> {
|
||||
const repo = queryRunner.manager.getRepository(StateSyncStatus);
|
||||
|
||||
return this._baseDatabase.updateIPLDStatusIPFSBlock(repo, blockNumber, force);
|
||||
return this._baseDatabase.updateStateSyncStatusCheckpointBlock(repo, blockNumber, force);
|
||||
}
|
||||
|
||||
async getContracts (): Promise<Contract[]> {
|
||||
|
||||
@ -12,7 +12,7 @@ import { BlockProgress } from './BlockProgress';
|
||||
@Index(['cid'], { unique: true })
|
||||
@Index(['block', 'contractAddress'])
|
||||
@Index(['block', 'contractAddress', 'kind'], { unique: true })
|
||||
export class IPLDBlock {
|
||||
export class State {
|
||||
@PrimaryGeneratedColumn()
|
||||
id!: number;
|
||||
|
||||
@ -5,16 +5,13 @@
|
||||
import { Entity, PrimaryGeneratedColumn, Column } from 'typeorm';
|
||||
|
||||
@Entity()
|
||||
export class IpldStatus {
|
||||
export class StateSyncStatus {
|
||||
@PrimaryGeneratedColumn()
|
||||
id!: number;
|
||||
|
||||
@Column('integer')
|
||||
latestHooksBlockNumber!: number;
|
||||
latestIndexedBlockNumber!: number;
|
||||
|
||||
@Column('integer')
|
||||
@Column('integer', { nullable: true })
|
||||
latestCheckpointBlockNumber!: number;
|
||||
|
||||
@Column('integer')
|
||||
latestIPFSBlockNumber!: number;
|
||||
}
|
||||
@ -21,19 +21,19 @@ export async function createInitialState (indexer: Indexer, contractAddress: str
|
||||
assert(blockHash);
|
||||
assert(contractAddress);
|
||||
|
||||
// Store the desired initial state in an IPLDBlock.
|
||||
const ipldBlockData: any = {
|
||||
// Store an empty State.
|
||||
const stateData: any = {
|
||||
state: {}
|
||||
};
|
||||
|
||||
// Use updateStateForElementaryType to update initial state with an elementary property.
|
||||
// Eg. const ipldBlockData = updateStateForElementaryType(ipldBlockData, '_totalBalance', result.value.toString());
|
||||
// Eg. const stateData = updateStateForElementaryType(stateData, '_totalBalance', result.value.toString());
|
||||
|
||||
// Use updateStateForMappingType to update initial state with a nested property.
|
||||
// Eg. const ipldBlockData = updateStateForMappingType(ipldBlockData, '_allowances', [owner, spender], allowance.value.toString());
|
||||
// Eg. const stateData = updateStateForMappingType(stateData, '_allowances', [owner, spender], allowance.value.toString());
|
||||
|
||||
// Return initial state data to be saved.
|
||||
return ipldBlockData;
|
||||
return stateData;
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@ -10,7 +10,6 @@ import { ethers } from 'ethers';
|
||||
|
||||
import { JsonFragment } from '@ethersproject/abi';
|
||||
import { BaseProvider } from '@ethersproject/providers';
|
||||
import * as codec from '@ipld/dag-cbor';
|
||||
import { EthClient } from '@cerc-io/ipld-eth-client';
|
||||
import { MappingKey, StorageLayout } from '@cerc-io/solidity-mapper';
|
||||
import {
|
||||
@ -25,10 +24,8 @@ import {
|
||||
updateStateForElementaryType,
|
||||
updateStateForMappingType,
|
||||
BlockHeight,
|
||||
IPFSClient,
|
||||
StateKind,
|
||||
IpldStatus as IpldStatusInterface,
|
||||
ResultIPLDBlock
|
||||
StateStatus
|
||||
} from '@cerc-io/util';
|
||||
|
||||
import ERC721Artifacts from './artifacts/ERC721.json';
|
||||
@ -37,9 +34,9 @@ import { createInitialState, handleEvent, createStateDiff, createStateCheckpoint
|
||||
import { Contract } from './entity/Contract';
|
||||
import { Event } from './entity/Event';
|
||||
import { SyncStatus } from './entity/SyncStatus';
|
||||
import { IpldStatus } from './entity/IpldStatus';
|
||||
import { StateSyncStatus } from './entity/StateSyncStatus';
|
||||
import { BlockProgress } from './entity/BlockProgress';
|
||||
import { IPLDBlock } from './entity/IPLDBlock';
|
||||
import { State } from './entity/State';
|
||||
import { TransferCount } from './entity/TransferCount';
|
||||
|
||||
const log = debug('vulcanize:indexer');
|
||||
@ -82,8 +79,6 @@ export class Indexer implements IndexerInterface {
|
||||
_storageLayoutMap: Map<string, StorageLayout>
|
||||
_contractMap: Map<string, ethers.utils.Interface>
|
||||
|
||||
_ipfsClient: IPFSClient
|
||||
|
||||
constructor (serverConfig: ServerConfig, db: Database, ethClient: EthClient, ethProvider: BaseProvider, jobQueue: JobQueue) {
|
||||
assert(db);
|
||||
assert(ethClient);
|
||||
@ -92,8 +87,7 @@ export class Indexer implements IndexerInterface {
|
||||
this._ethClient = ethClient;
|
||||
this._ethProvider = ethProvider;
|
||||
this._serverConfig = serverConfig;
|
||||
this._ipfsClient = new IPFSClient(this._serverConfig.ipfsApiAddr);
|
||||
this._baseIndexer = new BaseIndexer(this._serverConfig, this._db, this._ethClient, this._ethProvider, jobQueue, this._ipfsClient);
|
||||
this._baseIndexer = new BaseIndexer(this._serverConfig, this._db, this._ethClient, this._ethProvider, jobQueue);
|
||||
|
||||
this._abiMap = new Map();
|
||||
this._storageLayoutMap = new Map();
|
||||
@ -121,7 +115,7 @@ export class Indexer implements IndexerInterface {
|
||||
|
||||
async init (): Promise<void> {
|
||||
await this._baseIndexer.fetchContracts();
|
||||
await this._baseIndexer.fetchIPLDStatus();
|
||||
await this._baseIndexer.fetchStateStatus();
|
||||
}
|
||||
|
||||
getResultEvent (event: Event): ResultEvent {
|
||||
@ -159,26 +153,6 @@ export class Indexer implements IndexerInterface {
|
||||
};
|
||||
}
|
||||
|
||||
getResultIPLDBlock (ipldBlock: IPLDBlock): ResultIPLDBlock {
|
||||
const block = ipldBlock.block;
|
||||
|
||||
const data = codec.decode(Buffer.from(ipldBlock.data)) as any;
|
||||
|
||||
return {
|
||||
block: {
|
||||
cid: block.cid,
|
||||
hash: block.blockHash,
|
||||
number: block.blockNumber,
|
||||
timestamp: block.blockTimestamp,
|
||||
parentHash: block.parentHash
|
||||
},
|
||||
contractAddress: ipldBlock.contractAddress,
|
||||
cid: ipldBlock.cid,
|
||||
kind: ipldBlock.kind,
|
||||
data: JSON.stringify(data)
|
||||
};
|
||||
}
|
||||
|
||||
async supportsInterface (blockHash: string, contractAddress: string, interfaceId: string): Promise<ValueResult> {
|
||||
const entity = await this._db.getSupportsInterface({ blockHash, contractAddress, interfaceId });
|
||||
if (entity) {
|
||||
@ -674,10 +648,6 @@ export class Indexer implements IndexerInterface {
|
||||
);
|
||||
}
|
||||
|
||||
async pushToIPFS (data: any): Promise<void> {
|
||||
await this._baseIndexer.pushToIPFS(data);
|
||||
}
|
||||
|
||||
async processInitialState (contractAddress: string, blockHash: string): Promise<any> {
|
||||
// Call initial state hook.
|
||||
return createInitialState(this, contractAddress, blockHash);
|
||||
@ -708,28 +678,24 @@ export class Indexer implements IndexerInterface {
|
||||
return this._baseIndexer.processCLICheckpoint(this, contractAddress, blockHash);
|
||||
}
|
||||
|
||||
async getPrevIPLDBlock (blockHash: string, contractAddress: string, kind?: string): Promise<IPLDBlock | undefined> {
|
||||
return this._db.getPrevIPLDBlock(blockHash, contractAddress, kind);
|
||||
async getPrevState (blockHash: string, contractAddress: string, kind?: string): Promise<State | undefined> {
|
||||
return this._db.getPrevState(blockHash, contractAddress, kind);
|
||||
}
|
||||
|
||||
async getLatestIPLDBlock (contractAddress: string, kind: StateKind | null, blockNumber?: number): Promise<IPLDBlock | undefined> {
|
||||
return this._db.getLatestIPLDBlock(contractAddress, kind, blockNumber);
|
||||
async getLatestState (contractAddress: string, kind: StateKind | null, blockNumber?: number): Promise<State | undefined> {
|
||||
return this._db.getLatestState(contractAddress, kind, blockNumber);
|
||||
}
|
||||
|
||||
async getIPLDBlocksByHash (blockHash: string): Promise<IPLDBlock[]> {
|
||||
return this._baseIndexer.getIPLDBlocksByHash(blockHash);
|
||||
async getStatesByHash (blockHash: string): Promise<State[]> {
|
||||
return this._baseIndexer.getStatesByHash(blockHash);
|
||||
}
|
||||
|
||||
async getIPLDBlockByCid (cid: string): Promise<IPLDBlock | undefined> {
|
||||
return this._baseIndexer.getIPLDBlockByCid(cid);
|
||||
async getStateByCID (cid: string): Promise<State | undefined> {
|
||||
return this._baseIndexer.getStateByCID(cid);
|
||||
}
|
||||
|
||||
getIPLDData (ipldBlock: IPLDBlock): any {
|
||||
return this._baseIndexer.getIPLDData(ipldBlock);
|
||||
}
|
||||
|
||||
isIPFSConfigured (): boolean {
|
||||
return this._baseIndexer.isIPFSConfigured();
|
||||
getStateData (state: State): any {
|
||||
return this._baseIndexer.getStateData(state);
|
||||
}
|
||||
|
||||
// Method used to create auto diffs (diff_staged).
|
||||
@ -761,12 +727,12 @@ export class Indexer implements IndexerInterface {
|
||||
return this._baseIndexer.createCheckpoint(this, contractAddress, block);
|
||||
}
|
||||
|
||||
async saveOrUpdateIPLDBlock (ipldBlock: IPLDBlock): Promise<IPLDBlock> {
|
||||
return this._baseIndexer.saveOrUpdateIPLDBlock(ipldBlock);
|
||||
async saveOrUpdateState (state: State): Promise<State> {
|
||||
return this._baseIndexer.saveOrUpdateState(state);
|
||||
}
|
||||
|
||||
async removeIPLDBlocks (blockNumber: number, kind: StateKind): Promise<void> {
|
||||
await this._baseIndexer.removeIPLDBlocks(blockNumber, kind);
|
||||
async removeStates (blockNumber: number, kind: StateKind): Promise<void> {
|
||||
await this._baseIndexer.removeStates(blockNumber, kind);
|
||||
}
|
||||
|
||||
async triggerIndexingOnEvent (event: Event): Promise<void> {
|
||||
@ -803,16 +769,16 @@ export class Indexer implements IndexerInterface {
|
||||
};
|
||||
}
|
||||
|
||||
async getIPLDStatus (): Promise<IpldStatus | undefined> {
|
||||
return this._db.getIPLDStatus();
|
||||
async getStateSyncStatus (): Promise<StateSyncStatus | undefined> {
|
||||
return this._db.getStateSyncStatus();
|
||||
}
|
||||
|
||||
async updateIPLDStatusHooksBlock (blockNumber: number, force?: boolean): Promise<IpldStatus> {
|
||||
async updateStateSyncStatusIndexedBlock (blockNumber: number, force?: boolean): Promise<StateSyncStatus> {
|
||||
const dbTx = await this._db.createTransactionRunner();
|
||||
let res;
|
||||
|
||||
try {
|
||||
res = await this._db.updateIPLDStatusHooksBlock(dbTx, blockNumber, force);
|
||||
res = await this._db.updateStateSyncStatusIndexedBlock(dbTx, blockNumber, force);
|
||||
await dbTx.commitTransaction();
|
||||
} catch (error) {
|
||||
await dbTx.rollbackTransaction();
|
||||
@ -824,29 +790,12 @@ export class Indexer implements IndexerInterface {
|
||||
return res;
|
||||
}
|
||||
|
||||
async updateIPLDStatusCheckpointBlock (blockNumber: number, force?: boolean): Promise<IpldStatus> {
|
||||
async updateStateSyncStatusCheckpointBlock (blockNumber: number, force?: boolean): Promise<StateSyncStatus> {
|
||||
const dbTx = await this._db.createTransactionRunner();
|
||||
let res;
|
||||
|
||||
try {
|
||||
res = await this._db.updateIPLDStatusCheckpointBlock(dbTx, blockNumber, force);
|
||||
await dbTx.commitTransaction();
|
||||
} catch (error) {
|
||||
await dbTx.rollbackTransaction();
|
||||
throw error;
|
||||
} finally {
|
||||
await dbTx.release();
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
async updateIPLDStatusIPFSBlock (blockNumber: number, force?: boolean): Promise<IpldStatus> {
|
||||
const dbTx = await this._db.createTransactionRunner();
|
||||
let res;
|
||||
|
||||
try {
|
||||
res = await this._db.updateIPLDStatusIPFSBlock(dbTx, blockNumber, force);
|
||||
res = await this._db.updateStateSyncStatusCheckpointBlock(dbTx, blockNumber, force);
|
||||
await dbTx.commitTransaction();
|
||||
} catch (error) {
|
||||
await dbTx.rollbackTransaction();
|
||||
@ -868,16 +817,16 @@ export class Indexer implements IndexerInterface {
|
||||
return latestCanonicalBlock;
|
||||
}
|
||||
|
||||
async getLatestHooksProcessedBlock (): Promise<BlockProgress> {
|
||||
return this._baseIndexer.getLatestHooksProcessedBlock();
|
||||
async getLatestStateIndexedBlock (): Promise<BlockProgress> {
|
||||
return this._baseIndexer.getLatestStateIndexedBlock();
|
||||
}
|
||||
|
||||
async watchContract (address: string, kind: string, checkpoint: boolean, startingBlock: number): Promise<void> {
|
||||
return this._baseIndexer.watchContract(address, kind, checkpoint, startingBlock);
|
||||
}
|
||||
|
||||
async updateIPLDStatusMap (address: string, ipldStatus: IpldStatusInterface): Promise<void> {
|
||||
await this._baseIndexer.updateIPLDStatusMap(address, ipldStatus);
|
||||
updateStateStatusMap (address: string, stateStatus: StateStatus): void {
|
||||
this._baseIndexer.updateStateStatusMap(address, stateStatus);
|
||||
}
|
||||
|
||||
cacheContract (contract: Contract): void {
|
||||
|
||||
@ -17,8 +17,6 @@ import {
|
||||
QUEUE_EVENT_PROCESSING,
|
||||
QUEUE_BLOCK_CHECKPOINT,
|
||||
QUEUE_HOOKS,
|
||||
QUEUE_IPFS,
|
||||
JOB_KIND_PRUNE,
|
||||
JobQueueConfig,
|
||||
DEFAULT_CONFIG_PATH,
|
||||
initClients,
|
||||
@ -48,22 +46,12 @@ export class JobRunner {
|
||||
await this.subscribeEventProcessingQueue();
|
||||
await this.subscribeBlockCheckpointQueue();
|
||||
await this.subscribeHooksQueue();
|
||||
await this.subscribeIPFSQueue();
|
||||
this._baseJobRunner.handleShutdown();
|
||||
}
|
||||
|
||||
async subscribeBlockProcessingQueue (): Promise<void> {
|
||||
await this._jobQueue.subscribe(QUEUE_BLOCK_PROCESSING, async (job) => {
|
||||
await this._baseJobRunner.processBlock(job);
|
||||
|
||||
const { data: { kind } } = job;
|
||||
|
||||
// If it's a pruning job: Create a hooks job.
|
||||
if (kind === JOB_KIND_PRUNE) {
|
||||
await this.createHooksJob();
|
||||
}
|
||||
|
||||
await this._jobQueue.markComplete(job);
|
||||
});
|
||||
}
|
||||
|
||||
@ -75,165 +63,15 @@ export class JobRunner {
|
||||
|
||||
async subscribeHooksQueue (): Promise<void> {
|
||||
await this._jobQueue.subscribe(QUEUE_HOOKS, async (job) => {
|
||||
const { data: { blockHash, blockNumber } } = job;
|
||||
|
||||
// Get the current IPLD Status.
|
||||
const ipldStatus = await this._indexer.getIPLDStatus();
|
||||
|
||||
if (ipldStatus) {
|
||||
if (ipldStatus.latestHooksBlockNumber < (blockNumber - 1)) {
|
||||
// Create hooks job for parent block.
|
||||
const [parentBlock] = await this._indexer.getBlocksAtHeight(blockNumber - 1, false);
|
||||
await this.createHooksJob(parentBlock.blockHash, parentBlock.blockNumber);
|
||||
|
||||
const message = `Hooks for blockNumber ${blockNumber - 1} not processed yet, aborting`;
|
||||
log(message);
|
||||
|
||||
throw new Error(message);
|
||||
}
|
||||
|
||||
if (ipldStatus.latestHooksBlockNumber > (blockNumber - 1)) {
|
||||
log(`Hooks for blockNumber ${blockNumber} already processed`);
|
||||
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// Process the hooks for the given block number.
|
||||
await this._indexer.processCanonicalBlock(blockHash);
|
||||
|
||||
// Update the IPLD status.
|
||||
await this._indexer.updateIPLDStatusHooksBlock(blockNumber);
|
||||
|
||||
// Create a checkpoint job after completion of a hook job.
|
||||
await this.createCheckpointJob(blockHash, blockNumber);
|
||||
|
||||
await this._jobQueue.markComplete(job);
|
||||
await this._baseJobRunner.processHooks(job);
|
||||
});
|
||||
}
|
||||
|
||||
async subscribeBlockCheckpointQueue (): Promise<void> {
|
||||
await this._jobQueue.subscribe(QUEUE_BLOCK_CHECKPOINT, async (job) => {
|
||||
const { data: { blockHash, blockNumber } } = job;
|
||||
|
||||
// Get the current IPLD Status.
|
||||
const ipldStatus = await this._indexer.getIPLDStatus();
|
||||
assert(ipldStatus);
|
||||
|
||||
if (ipldStatus.latestCheckpointBlockNumber >= 0) {
|
||||
if (ipldStatus.latestCheckpointBlockNumber < (blockNumber - 1)) {
|
||||
// Create a checkpoint job for parent block.
|
||||
const [parentBlock] = await this._indexer.getBlocksAtHeight(blockNumber - 1, false);
|
||||
await this.createCheckpointJob(parentBlock.blockHash, parentBlock.blockNumber);
|
||||
|
||||
const message = `Checkpoints for blockNumber ${blockNumber - 1} not processed yet, aborting`;
|
||||
log(message);
|
||||
|
||||
throw new Error(message);
|
||||
}
|
||||
|
||||
if (ipldStatus.latestCheckpointBlockNumber > (blockNumber - 1)) {
|
||||
log(`Checkpoints for blockNumber ${blockNumber} already processed`);
|
||||
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// Process checkpoints for the given block.
|
||||
await this._indexer.processCheckpoint(blockHash);
|
||||
|
||||
// Update the IPLD status.
|
||||
await this._indexer.updateIPLDStatusCheckpointBlock(blockNumber);
|
||||
|
||||
// Create an IPFS job after completion of a checkpoint job.
|
||||
if (this._indexer.isIPFSConfigured()) {
|
||||
await this.createIPFSPutJob(blockHash, blockNumber);
|
||||
}
|
||||
|
||||
await this._jobQueue.markComplete(job);
|
||||
await this._baseJobRunner.processCheckpoint(job);
|
||||
});
|
||||
}
|
||||
|
||||
async subscribeIPFSQueue (): Promise<void> {
|
||||
await this._jobQueue.subscribe(QUEUE_IPFS, async (job) => {
|
||||
const { data: { blockHash, blockNumber } } = job;
|
||||
|
||||
const ipldStatus = await this._indexer.getIPLDStatus();
|
||||
assert(ipldStatus);
|
||||
|
||||
if (ipldStatus.latestIPFSBlockNumber >= 0) {
|
||||
if (ipldStatus.latestIPFSBlockNumber < (blockNumber - 1)) {
|
||||
// Create a IPFS job for parent block.
|
||||
const [parentBlock] = await this._indexer.getBlocksAtHeight(blockNumber - 1, false);
|
||||
await this.createIPFSPutJob(parentBlock.blockHash, parentBlock.blockNumber);
|
||||
|
||||
const message = `IPFS for blockNumber ${blockNumber - 1} not processed yet, aborting`;
|
||||
log(message);
|
||||
|
||||
throw new Error(message);
|
||||
}
|
||||
|
||||
if (ipldStatus.latestIPFSBlockNumber > (blockNumber - 1)) {
|
||||
log(`IPFS for blockNumber ${blockNumber} already processed`);
|
||||
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// Get IPLDBlocks for the given blocHash.
|
||||
const ipldBlocks = await this._indexer.getIPLDBlocksByHash(blockHash);
|
||||
|
||||
// Push all the IPLDBlocks to IPFS.
|
||||
for (const ipldBlock of ipldBlocks) {
|
||||
const data = this._indexer.getIPLDData(ipldBlock);
|
||||
await this._indexer.pushToIPFS(data);
|
||||
}
|
||||
|
||||
// Update the IPLD status.
|
||||
await this._indexer.updateIPLDStatusIPFSBlock(blockNumber);
|
||||
|
||||
await this._jobQueue.markComplete(job);
|
||||
});
|
||||
}
|
||||
|
||||
async createHooksJob (blockHash?: string, blockNumber?: number): Promise<void> {
|
||||
if (!blockNumber || !blockHash) {
|
||||
// Get the latest canonical block
|
||||
const latestCanonicalBlock = await this._indexer.getLatestCanonicalBlock();
|
||||
|
||||
// Create a hooks job for parent block of latestCanonicalBlock because pruning for first block is skipped as it is assumed to be a canonical block.
|
||||
blockHash = latestCanonicalBlock.parentHash;
|
||||
blockNumber = latestCanonicalBlock.blockNumber - 1;
|
||||
}
|
||||
|
||||
await this._jobQueue.pushJob(
|
||||
QUEUE_HOOKS,
|
||||
{
|
||||
blockHash,
|
||||
blockNumber
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
async createCheckpointJob (blockHash: string, blockNumber: number): Promise<void> {
|
||||
await this._jobQueue.pushJob(
|
||||
QUEUE_BLOCK_CHECKPOINT,
|
||||
{
|
||||
blockHash,
|
||||
blockNumber
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
async createIPFSPutJob (blockHash: string, blockNumber: number): Promise<void> {
|
||||
await this._jobQueue.pushJob(
|
||||
QUEUE_IPFS,
|
||||
{
|
||||
blockHash,
|
||||
blockNumber
|
||||
}
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export const main = async (): Promise<any> => {
|
||||
|
||||
@ -8,7 +8,7 @@ import debug from 'debug';
|
||||
import Decimal from 'decimal.js';
|
||||
import { GraphQLScalarType } from 'graphql';
|
||||
|
||||
import { ValueResult, BlockHeight } from '@cerc-io/util';
|
||||
import { ValueResult, BlockHeight, getResultState } from '@cerc-io/util';
|
||||
|
||||
import { Indexer } from './indexer';
|
||||
import { EventWatcher } from './events';
|
||||
@ -161,17 +161,17 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch
|
||||
getStateByCID: async (_: any, { cid }: { cid: string }) => {
|
||||
log('getStateByCID', cid);
|
||||
|
||||
const ipldBlock = await indexer.getIPLDBlockByCid(cid);
|
||||
const state = await indexer.getStateByCID(cid);
|
||||
|
||||
return ipldBlock && ipldBlock.block.isComplete ? indexer.getResultIPLDBlock(ipldBlock) : undefined;
|
||||
return state && state.block.isComplete ? getResultState(state) : undefined;
|
||||
},
|
||||
|
||||
getState: async (_: any, { blockHash, contractAddress, kind }: { blockHash: string, contractAddress: string, kind: string }) => {
|
||||
log('getState', blockHash, contractAddress, kind);
|
||||
|
||||
const ipldBlock = await indexer.getPrevIPLDBlock(blockHash, contractAddress, kind);
|
||||
const state = await indexer.getPrevState(blockHash, contractAddress, kind);
|
||||
|
||||
return ipldBlock && ipldBlock.block.isComplete ? indexer.getResultIPLDBlock(ipldBlock) : undefined;
|
||||
return state && state.block.isComplete ? getResultState(state) : undefined;
|
||||
},
|
||||
|
||||
getSyncStatus: async () => {
|
||||
|
||||
@ -77,7 +77,7 @@ type TransferEvent {
|
||||
tokenId: BigInt!
|
||||
}
|
||||
|
||||
type ResultIPLDBlock {
|
||||
type ResultState {
|
||||
block: _Block_!
|
||||
contractAddress: String!
|
||||
cid: String!
|
||||
@ -109,8 +109,8 @@ type Query {
|
||||
_balances(blockHash: String!, contractAddress: String!, key0: String!): ResultBigInt!
|
||||
_tokenApprovals(blockHash: String!, contractAddress: String!, key0: BigInt!): ResultString!
|
||||
_operatorApprovals(blockHash: String!, contractAddress: String!, key0: String!, key1: String!): ResultBoolean!
|
||||
getStateByCID(cid: String!): ResultIPLDBlock
|
||||
getState(blockHash: String!, contractAddress: String!, kind: String): ResultIPLDBlock
|
||||
getStateByCID(cid: String!): ResultState
|
||||
getState(blockHash: String!, contractAddress: String!, kind: String): ResultState
|
||||
getSyncStatus: SyncStatus
|
||||
transferCount(id: String!, block: Block_height): TransferCount!
|
||||
}
|
||||
|
||||
@ -69,13 +69,13 @@
|
||||
```
|
||||
|
||||
The queries will be fired if the corresponding entities are updated.
|
||||
|
||||
|
||||
* Run the CLI:
|
||||
|
||||
```bash
|
||||
./bin/compare-blocks --config-file environments/compare-cli-config.toml --start-block 1 --end-block 10
|
||||
```
|
||||
|
||||
|
||||
* For comparing entities after fetching updated entity ids from watcher database:
|
||||
|
||||
* Set the watcher config file path and entities directory.
|
||||
@ -90,13 +90,13 @@
|
||||
[queries.names]
|
||||
author = "Author"
|
||||
blog = "Blog"
|
||||
|
||||
|
||||
[watcher]
|
||||
configPath = "../../graph-test-watcher/environments/local.toml"
|
||||
entitiesDir = "../../graph-test-watcher/dist/entity/*"
|
||||
```
|
||||
|
||||
* To verify diff IPLD state generated at each block, set the watcher endpoint and `verifyState` flag to true
|
||||
|
||||
* To verify `diff` State state generated at each block, set the watcher endpoint and `verifyState` flag to true
|
||||
|
||||
```toml
|
||||
[watcher]
|
||||
@ -105,7 +105,7 @@
|
||||
endpoint = "gqlEndpoint2"
|
||||
verifyState = true
|
||||
```
|
||||
|
||||
|
||||
* Run the CLI with `fetch-ids` flag set to true:\
|
||||
|
||||
```bash
|
||||
|
||||
@ -12,7 +12,17 @@ import _ from 'lodash';
|
||||
import { getConfig as getWatcherConfig, wait } from '@cerc-io/util';
|
||||
import { GraphQLClient } from '@cerc-io/ipld-eth-client';
|
||||
|
||||
import { checkGQLEntityInIPLDState, compareQuery, Config, getIPLDsByBlock, checkIPLDMetaData, combineIPLDState, getClients, getConfig, checkGQLEntitiesInIPLDState } from './utils';
|
||||
import {
|
||||
checkGQLEntityInState,
|
||||
compareQuery,
|
||||
Config,
|
||||
getStatesByBlock,
|
||||
checkStateMetaData,
|
||||
combineState,
|
||||
getClients,
|
||||
getConfig,
|
||||
checkGQLEntitiesInState
|
||||
} from './utils';
|
||||
import { Database } from '../../database';
|
||||
import { getSubgraphConfig } from '../../utils';
|
||||
|
||||
@ -117,8 +127,14 @@ export const main = async (): Promise<void> => {
|
||||
await db.init();
|
||||
|
||||
if (config.watcher.verifyState) {
|
||||
const { dataSources } = await getSubgraphConfig(watcherConfig.server.subgraphPath);
|
||||
subgraphContracts = dataSources.map((dataSource: any) => dataSource.source.address);
|
||||
// Use provided contracts if available; else read from subraph config.
|
||||
if (config.watcher.contracts) {
|
||||
subgraphContracts = config.watcher.contracts;
|
||||
} else {
|
||||
const { dataSources } = await getSubgraphConfig(watcherConfig.server.subgraphPath);
|
||||
subgraphContracts = dataSources.map((dataSource: any) => dataSource.source.address);
|
||||
}
|
||||
|
||||
const watcherEndpoint = config.endpoints[config.watcher.endpoint] as string;
|
||||
subgraphGQLClient = new GraphQLClient({ gqlEndpoint: watcherEndpoint });
|
||||
}
|
||||
@ -134,7 +150,7 @@ export const main = async (): Promise<void> => {
|
||||
const block = { number: blockNumber };
|
||||
const updatedEntityIds: { [entityName: string]: string[] } = {};
|
||||
const updatedEntities: Set<string> = new Set();
|
||||
let ipldStateByBlock = {};
|
||||
let stateByBlock = {};
|
||||
assert(db);
|
||||
console.time(`time:compare-block-${blockNumber}`);
|
||||
|
||||
@ -166,18 +182,18 @@ export const main = async (): Promise<void> => {
|
||||
assert(db);
|
||||
const [block] = await db.getBlocksAtHeight(blockNumber, false);
|
||||
assert(subgraphGQLClient);
|
||||
const contractIPLDsByBlock = await getIPLDsByBlock(subgraphGQLClient, subgraphContracts, block.blockHash);
|
||||
const contractStatesByBlock = await getStatesByBlock(subgraphGQLClient, subgraphContracts, block.blockHash);
|
||||
|
||||
// Check meta data for each IPLD block found
|
||||
contractIPLDsByBlock.flat().forEach(contractIPLD => {
|
||||
const ipldMetaDataDiff = checkIPLDMetaData(contractIPLD, contractLatestStateCIDMap, rawJson);
|
||||
if (ipldMetaDataDiff) {
|
||||
log('Results mismatch for IPLD meta data:', ipldMetaDataDiff);
|
||||
// Check meta data for each State entry found
|
||||
contractStatesByBlock.flat().forEach(contractStateEntry => {
|
||||
const stateMetaDataDiff = checkStateMetaData(contractStateEntry, contractLatestStateCIDMap, rawJson);
|
||||
if (stateMetaDataDiff) {
|
||||
log('Results mismatch for State meta data:', stateMetaDataDiff);
|
||||
diffFound = true;
|
||||
}
|
||||
});
|
||||
|
||||
ipldStateByBlock = combineIPLDState(contractIPLDsByBlock.flat());
|
||||
stateByBlock = combineState(contractStatesByBlock.flat());
|
||||
}
|
||||
|
||||
await blockDelay;
|
||||
@ -205,10 +221,10 @@ export const main = async (): Promise<void> => {
|
||||
);
|
||||
|
||||
if (config.watcher.verifyState) {
|
||||
const ipldDiff = await checkGQLEntityInIPLDState(ipldStateByBlock, entityName, result[queryName], id, rawJson, config.watcher.skipFields);
|
||||
const stateDiff = await checkGQLEntityInState(stateByBlock, entityName, result[queryName], id, rawJson, config.watcher.skipFields);
|
||||
|
||||
if (ipldDiff) {
|
||||
log('Results mismatch for IPLD state:', ipldDiff);
|
||||
if (stateDiff) {
|
||||
log('Results mismatch for State:', stateDiff);
|
||||
diffFound = true;
|
||||
}
|
||||
}
|
||||
@ -236,10 +252,10 @@ export const main = async (): Promise<void> => {
|
||||
));
|
||||
|
||||
if (config.watcher.verifyState) {
|
||||
const ipldDiff = await checkGQLEntitiesInIPLDState(ipldStateByBlock, entityName, result[queryName], rawJson, config.watcher.skipFields);
|
||||
const stateDiff = await checkGQLEntitiesInState(stateByBlock, entityName, result[queryName], rawJson, config.watcher.skipFields);
|
||||
|
||||
if (ipldDiff) {
|
||||
log('Results mismatch for IPLD state:', ipldDiff);
|
||||
if (stateDiff) {
|
||||
log('Results mismatch for State:', stateDiff);
|
||||
diffFound = true;
|
||||
}
|
||||
}
|
||||
|
||||
@ -21,7 +21,7 @@ import { DEFAULT_LIMIT } from '../../database';
|
||||
|
||||
const log = debug('vulcanize:compare-utils');
|
||||
|
||||
const IPLD_STATE_QUERY = `
|
||||
const STATE_QUERY = `
|
||||
query getState($blockHash: String!, $contractAddress: String!, $kind: String){
|
||||
getState(blockHash: $blockHash, contractAddress: $contractAddress, kind: $kind){
|
||||
block {
|
||||
@ -63,7 +63,8 @@ export interface Config {
|
||||
entitiesDir: string;
|
||||
verifyState: boolean;
|
||||
endpoint: keyof EndpointConfig;
|
||||
skipFields: EntitySkipFields[]
|
||||
skipFields: EntitySkipFields[];
|
||||
contracts: string[];
|
||||
}
|
||||
cache: {
|
||||
endpoint: keyof EndpointConfig;
|
||||
@ -169,24 +170,24 @@ export const getClients = async (config: Config, timeDiff: boolean, queryDir?: s
|
||||
};
|
||||
};
|
||||
|
||||
export const getIPLDsByBlock = async (client: GraphQLClient, contracts: string[], blockHash: string): Promise<{[key: string]: any}[][]> => {
|
||||
// Fetch IPLD states for all contracts
|
||||
export const getStatesByBlock = async (client: GraphQLClient, contracts: string[], blockHash: string): Promise<{[key: string]: any}[][]> => {
|
||||
// Fetch States for all contracts
|
||||
return Promise.all(contracts.map(async contract => {
|
||||
const { getState } = await client.query(
|
||||
gql(IPLD_STATE_QUERY),
|
||||
gql(STATE_QUERY),
|
||||
{
|
||||
blockHash,
|
||||
contractAddress: contract
|
||||
}
|
||||
);
|
||||
|
||||
const stateIPLDs = [];
|
||||
const states = [];
|
||||
|
||||
// If 'checkpoint' is found at the same block, fetch 'diff' as well
|
||||
if (getState && getState.kind === 'checkpoint' && getState.block.hash === blockHash) {
|
||||
// Check if 'init' present at the same block
|
||||
const { getState: getInitState } = await client.query(
|
||||
gql(IPLD_STATE_QUERY),
|
||||
gql(STATE_QUERY),
|
||||
{
|
||||
blockHash,
|
||||
contractAddress: contract,
|
||||
@ -195,13 +196,13 @@ export const getIPLDsByBlock = async (client: GraphQLClient, contracts: string[]
|
||||
);
|
||||
|
||||
if (getInitState && getInitState.block.hash === blockHash) {
|
||||
// Append the 'init' IPLD block to the result
|
||||
stateIPLDs.push(getInitState);
|
||||
// Append the 'init' state to the result
|
||||
states.push(getInitState);
|
||||
}
|
||||
|
||||
// Check if 'diff' present at the same block
|
||||
// Check if 'diff' state present at the same block
|
||||
const { getState: getDiffState } = await client.query(
|
||||
gql(IPLD_STATE_QUERY),
|
||||
gql(STATE_QUERY),
|
||||
{
|
||||
blockHash,
|
||||
contractAddress: contract,
|
||||
@ -210,25 +211,25 @@ export const getIPLDsByBlock = async (client: GraphQLClient, contracts: string[]
|
||||
);
|
||||
|
||||
if (getDiffState && getDiffState.block.hash === blockHash) {
|
||||
// Append the 'diff' IPLD block to the result
|
||||
stateIPLDs.push(getDiffState);
|
||||
// Append the 'diff' state to the result
|
||||
states.push(getDiffState);
|
||||
}
|
||||
}
|
||||
|
||||
// Append the IPLD block to the result
|
||||
stateIPLDs.push(getState);
|
||||
// Append the state to the result
|
||||
states.push(getState);
|
||||
|
||||
return stateIPLDs;
|
||||
return states;
|
||||
}));
|
||||
};
|
||||
|
||||
export const checkIPLDMetaData = (contractIPLD: {[key: string]: any}, contractLatestStateCIDMap: Map<string, { diff: string, checkpoint: string }>, rawJson: boolean) => {
|
||||
// Return if IPLD for a contract not found
|
||||
if (!contractIPLD) {
|
||||
export const checkStateMetaData = (contractState: {[key: string]: any}, contractLatestStateCIDMap: Map<string, { diff: string, checkpoint: string }>, rawJson: boolean) => {
|
||||
// Return if State for a contract not found
|
||||
if (!contractState) {
|
||||
return;
|
||||
}
|
||||
|
||||
const { contractAddress, cid, kind, block } = contractIPLD;
|
||||
const { contractAddress, cid, kind, block } = contractState;
|
||||
|
||||
const parentCIDs = contractLatestStateCIDMap.get(contractAddress);
|
||||
assert(parentCIDs);
|
||||
@ -246,7 +247,7 @@ export const checkIPLDMetaData = (contractIPLD: {[key: string]: any}, contractLa
|
||||
contractLatestStateCIDMap.set(contractAddress, nextParentCIDs);
|
||||
|
||||
// Actual meta data from the GQL result
|
||||
const data = JSON.parse(contractIPLD.data);
|
||||
const data = JSON.parse(contractState.data);
|
||||
|
||||
// If parentCID not initialized (is empty at start)
|
||||
// Take the expected parentCID from the actual data itself
|
||||
@ -279,13 +280,13 @@ export const checkIPLDMetaData = (contractIPLD: {[key: string]: any}, contractLa
|
||||
return compareObjects(expectedMetaData, data.meta, rawJson);
|
||||
};
|
||||
|
||||
export const combineIPLDState = (contractIPLDs: {[key: string]: any}[]): {[key: string]: any} => {
|
||||
const contractIPLDStates: {[key: string]: any}[] = contractIPLDs.map(contractIPLD => {
|
||||
if (!contractIPLD) {
|
||||
export const combineState = (contractStateEntries: {[key: string]: any}[]): {[key: string]: any} => {
|
||||
const contractStates: {[key: string]: any}[] = contractStateEntries.map(contractStateEntry => {
|
||||
if (!contractStateEntry) {
|
||||
return {};
|
||||
}
|
||||
|
||||
const data = JSON.parse(contractIPLD.data);
|
||||
const data = JSON.parse(contractStateEntry.data);
|
||||
|
||||
// Apply default limit and sort by id on array type relation fields.
|
||||
Object.values(data.state)
|
||||
@ -309,18 +310,18 @@ export const combineIPLDState = (contractIPLDs: {[key: string]: any}[]): {[key:
|
||||
return data.state;
|
||||
});
|
||||
|
||||
return contractIPLDStates.reduce((acc, state) => _.merge(acc, state));
|
||||
return contractStates.reduce((acc, state) => _.merge(acc, state));
|
||||
};
|
||||
|
||||
export const checkGQLEntityInIPLDState = async (
|
||||
ipldState: {[key: string]: any},
|
||||
export const checkGQLEntityInState = async (
|
||||
state: {[key: string]: any},
|
||||
entityName: string,
|
||||
entityResult: {[key: string]: any},
|
||||
id: string,
|
||||
rawJson: boolean,
|
||||
skipFields: EntitySkipFields[] = []
|
||||
): Promise<string> => {
|
||||
const ipldEntity = ipldState[entityName][id];
|
||||
const stateEntity = state[entityName][id];
|
||||
|
||||
// Filter __typename key in GQL result.
|
||||
entityResult = omitDeep(entityResult, '__typename');
|
||||
@ -329,24 +330,24 @@ export const checkGQLEntityInIPLDState = async (
|
||||
skipFields.forEach(({ entity, fields }) => {
|
||||
if (entityName === entity) {
|
||||
omitDeep(entityResult, fields);
|
||||
omitDeep(ipldEntity, fields);
|
||||
omitDeep(stateEntity, fields);
|
||||
}
|
||||
});
|
||||
|
||||
const diff = compareObjects(entityResult, ipldEntity, rawJson);
|
||||
const diff = compareObjects(entityResult, stateEntity, rawJson);
|
||||
|
||||
return diff;
|
||||
};
|
||||
|
||||
export const checkGQLEntitiesInIPLDState = async (
|
||||
ipldState: {[key: string]: any},
|
||||
export const checkGQLEntitiesInState = async (
|
||||
state: {[key: string]: any},
|
||||
entityName: string,
|
||||
entitiesResult: any[],
|
||||
rawJson: boolean,
|
||||
skipFields: EntitySkipFields[] = []
|
||||
): Promise<string> => {
|
||||
// Form entities from state to compare with GQL result
|
||||
const stateEntities = ipldState[entityName];
|
||||
const stateEntities = state[entityName];
|
||||
|
||||
for (const entityResult of entitiesResult) {
|
||||
const stateEntity = stateEntities[entityResult.id];
|
||||
|
||||
@ -31,8 +31,6 @@ import { Block, fromEntityValue, fromStateEntityValues, toEntityValue } from './
|
||||
|
||||
export const DEFAULT_LIMIT = 100;
|
||||
|
||||
const log = debug('vulcanize:graph-node-database');
|
||||
|
||||
interface CachedEntities {
|
||||
frothyBlocks: Map<
|
||||
string,
|
||||
@ -641,7 +639,7 @@ export class Database {
|
||||
}, {});
|
||||
}
|
||||
|
||||
fromIPLDState (block: BlockProgressInterface, entity: string, stateEntity: any, relations: { [key: string]: any } = {}): any {
|
||||
fromState (block: BlockProgressInterface, entity: string, stateEntity: any, relations: { [key: string]: any } = {}): any {
|
||||
const repo = this._conn.getRepository(entity);
|
||||
const entityFields = repo.metadata.columns;
|
||||
|
||||
@ -678,7 +676,7 @@ export class Database {
|
||||
}, {});
|
||||
}
|
||||
|
||||
cacheUpdatedEntity<Entity> (entityName: string, entity: any, pruned = false): void {
|
||||
cacheUpdatedEntity (entityName: string, entity: any, pruned = false): void {
|
||||
const repo = this._conn.getRepository(entityName);
|
||||
const tableName = repo.metadata.tableName;
|
||||
|
||||
|
||||
@ -12,7 +12,7 @@ import { SelectionNode } from 'graphql';
|
||||
|
||||
import { ResultObject } from '@vulcanize/assemblyscript/lib/loader';
|
||||
import { EthClient } from '@cerc-io/ipld-eth-client';
|
||||
import { getFullBlock, BlockHeight, ServerConfig, getFullTransaction, QueryOptions, IPLDBlockInterface, IndexerInterface, BlockProgressInterface, cachePrunedEntitiesCount } from '@cerc-io/util';
|
||||
import { getFullBlock, BlockHeight, ServerConfig, getFullTransaction, QueryOptions, StateInterface, IndexerInterface, BlockProgressInterface, cachePrunedEntitiesCount } from '@cerc-io/util';
|
||||
|
||||
import { createBlock, createEvent, getSubgraphConfig, resolveEntityFieldConflicts, Transaction } from './utils';
|
||||
import { Context, GraphData, instantiate } from './loader';
|
||||
@ -349,9 +349,9 @@ export class GraphWatcher {
|
||||
}
|
||||
}
|
||||
|
||||
async updateEntitiesFromIPLDState (ipldBlock: IPLDBlockInterface) {
|
||||
async updateEntitiesFromState (state: StateInterface) {
|
||||
assert(this._indexer);
|
||||
const data = this._indexer.getIPLDData(ipldBlock);
|
||||
const data = this._indexer.getStateData(state);
|
||||
|
||||
for (const [entityName, entities] of Object.entries(data.state)) {
|
||||
// Get relations for subgraph entity
|
||||
@ -363,13 +363,13 @@ export class GraphWatcher {
|
||||
|
||||
const relations = result ? result[1] : {};
|
||||
|
||||
log(`Updating entities from IPLD state for entity ${entityName}`);
|
||||
console.time(`time:watcher#GraphWatcher-updateEntitiesFromIPLDState-IPLD-update-entity-${entityName}`);
|
||||
log(`Updating entities from State for entity ${entityName}`);
|
||||
console.time(`time:watcher#GraphWatcher-updateEntitiesFromState-update-entity-${entityName}`);
|
||||
for (const [id, entityData] of Object.entries(entities as any)) {
|
||||
const dbData = this._database.fromIPLDState(ipldBlock.block, entityName, entityData, relations);
|
||||
const dbData = this._database.fromState(state.block, entityName, entityData, relations);
|
||||
await this._database.saveEntity(entityName, dbData);
|
||||
}
|
||||
console.timeEnd(`time:watcher#GraphWatcher-updateEntitiesFromIPLDState-IPLD-update-entity-${entityName}`);
|
||||
console.timeEnd(`time:watcher#GraphWatcher-updateEntitiesFromState-update-entity-${entityName}`);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -9,8 +9,9 @@ import {
|
||||
ServerConfig as ServerConfigInterface,
|
||||
ValueResult,
|
||||
ContractInterface,
|
||||
IpldStatus as IpldStatusInterface,
|
||||
IPLDBlockInterface
|
||||
StateStatus,
|
||||
StateSyncStatusInterface,
|
||||
StateInterface
|
||||
} from '@cerc-io/util';
|
||||
import { EthClient } from '@cerc-io/ipld-eth-client';
|
||||
import { GetStorageAt, getStorageValue, MappingKey, StorageLayout } from '@cerc-io/solidity-mapper';
|
||||
@ -107,7 +108,7 @@ export class Indexer implements IndexerInterface {
|
||||
assert(blockHash);
|
||||
assert(blockNumber);
|
||||
|
||||
return new SyncStatus();
|
||||
return {} as SyncStatusInterface;
|
||||
}
|
||||
|
||||
async updateSyncStatusIndexedBlock (blockHash: string, blockNumber: number, force?: boolean): Promise<SyncStatusInterface> {
|
||||
@ -115,7 +116,7 @@ export class Indexer implements IndexerInterface {
|
||||
assert(blockHash);
|
||||
assert(force);
|
||||
|
||||
return new SyncStatus();
|
||||
return {} as SyncStatusInterface;
|
||||
}
|
||||
|
||||
async updateSyncStatusCanonicalBlock (blockHash: string, blockNumber: number, force?: boolean): Promise<SyncStatusInterface> {
|
||||
@ -123,7 +124,7 @@ export class Indexer implements IndexerInterface {
|
||||
assert(blockHash);
|
||||
assert(force);
|
||||
|
||||
return new SyncStatus();
|
||||
return {} as SyncStatusInterface;
|
||||
}
|
||||
|
||||
async markBlocksAsPruned (blocks: BlockProgressInterface[]): Promise<void> {
|
||||
@ -157,6 +158,22 @@ export class Indexer implements IndexerInterface {
|
||||
assert(event);
|
||||
}
|
||||
|
||||
async getStateSyncStatus (): Promise<StateSyncStatusInterface | undefined> {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
async updateStateSyncStatusIndexedBlock (blockNumber: number, force?: boolean): Promise<StateSyncStatusInterface> {
|
||||
return {} as StateSyncStatusInterface;
|
||||
}
|
||||
|
||||
async updateStateSyncStatusCheckpointBlock (blockNumber: number, force?: boolean): Promise<StateSyncStatusInterface> {
|
||||
return {} as StateSyncStatusInterface;
|
||||
}
|
||||
|
||||
async getLatestCanonicalBlock (): Promise<BlockProgressInterface> {
|
||||
return {} as BlockProgressInterface;
|
||||
}
|
||||
|
||||
isWatchedContract (address : string): ContractInterface | undefined {
|
||||
return undefined;
|
||||
}
|
||||
@ -165,36 +182,20 @@ export class Indexer implements IndexerInterface {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
getIPLDData (ipldBlock: IPLDBlockInterface): any {
|
||||
async processCanonicalBlock (blockHash: string, blockNumber: number): Promise<void> {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
async updateIPLDStatusMap (address: string, ipldStatus: IpldStatusInterface): Promise<void> {
|
||||
async processCheckpoint (blockHash: string): Promise<void> {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
class SyncStatus implements SyncStatusInterface {
|
||||
id: number;
|
||||
chainHeadBlockHash: string;
|
||||
chainHeadBlockNumber: number;
|
||||
latestIndexedBlockHash: string;
|
||||
latestIndexedBlockNumber: number;
|
||||
latestCanonicalBlockHash: string;
|
||||
latestCanonicalBlockNumber: number;
|
||||
initialIndexedBlockHash: string;
|
||||
initialIndexedBlockNumber: number;
|
||||
getStateData (state: StateInterface): any {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
constructor () {
|
||||
this.id = 0;
|
||||
this.chainHeadBlockHash = '0';
|
||||
this.chainHeadBlockNumber = 0;
|
||||
this.latestIndexedBlockHash = '0';
|
||||
this.latestIndexedBlockNumber = 0;
|
||||
this.latestCanonicalBlockHash = '0';
|
||||
this.latestCanonicalBlockNumber = 0;
|
||||
this.initialIndexedBlockHash = '0';
|
||||
this.initialIndexedBlockNumber = 0;
|
||||
updateStateStatusMap (address: string, stateStatus: StateStatus): void {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
@ -205,7 +206,6 @@ class ServerConfig implements ServerConfigInterface {
|
||||
kind: string;
|
||||
checkpointing: boolean;
|
||||
checkpointInterval: number;
|
||||
ipfsApiAddr: string;
|
||||
subgraphPath: string;
|
||||
disableSubgraphState: boolean;
|
||||
wasmRestartBlocksInterval: number;
|
||||
@ -220,7 +220,6 @@ class ServerConfig implements ServerConfigInterface {
|
||||
this.kind = '';
|
||||
this.checkpointing = false;
|
||||
this.checkpointInterval = 0;
|
||||
this.ipfsApiAddr = '';
|
||||
this.subgraphPath = '';
|
||||
this.disableSubgraphState = false;
|
||||
this.wasmRestartBlocksInterval = 0;
|
||||
|
||||
@ -8,12 +8,6 @@
|
||||
yarn
|
||||
```
|
||||
|
||||
* Run the IPFS (go-ipfs version 0.12.2) daemon:
|
||||
|
||||
```bash
|
||||
ipfs daemon
|
||||
```
|
||||
|
||||
* Create a postgres12 database for the watcher:
|
||||
|
||||
```bash
|
||||
@ -47,7 +41,7 @@
|
||||
|
||||
* Update the `upstream` config and provide the `ipld-eth-server` GQL API endpoint.
|
||||
|
||||
* Update the `server` config with state checkpoint settings and provide the IPFS API address.
|
||||
* Update the `server` config with state checkpoint settings.
|
||||
|
||||
## Customize
|
||||
|
||||
@ -59,11 +53,11 @@
|
||||
|
||||
* Generating state:
|
||||
|
||||
* Edit the custom hook function `createInitialCheckpoint` (triggered on watch-contract, checkpoint: `true`) in [hooks.ts](./src/hooks.ts) to save an initial checkpoint `IPLDBlock` using the `Indexer` object.
|
||||
* Edit the custom hook function `createInitialCheckpoint` (triggered on watch-contract, checkpoint: `true`) in [hooks.ts](./src/hooks.ts) to save an initial checkpoint `State` using the `Indexer` object.
|
||||
|
||||
* Edit the custom hook function `createStateDiff` (triggered on a block) in [hooks.ts](./src/hooks.ts) to save the state in a `diff` `IPLDBlock` using the `Indexer` object. The default state (if exists) is updated.
|
||||
* Edit the custom hook function `createStateDiff` (triggered on a block) in [hooks.ts](./src/hooks.ts) to save the state in a `diff` `State` using the `Indexer` object. The default state (if exists) is updated.
|
||||
|
||||
* Edit the custom hook function `createStateCheckpoint` (triggered just before default and CLI checkpoint) in [hooks.ts](./src/hooks.ts) to save the state in a `checkpoint` `IPLDBlock` using the `Indexer` object.
|
||||
* Edit the custom hook function `createStateCheckpoint` (triggered just before default and CLI checkpoint) in [hooks.ts](./src/hooks.ts) to save the state in a `checkpoint` `State` using the `Indexer` object.
|
||||
|
||||
* The existing example hooks in [hooks.ts](./src/hooks.ts) are for an `ERC20` contract.
|
||||
|
||||
@ -138,10 +132,10 @@ GQL console: http://localhost:3008/graphql
|
||||
|
||||
* To reset the watcher to a previous block number:
|
||||
|
||||
* Reset state:
|
||||
* Reset watcher:
|
||||
|
||||
```bash
|
||||
yarn reset state --block-number <previous-block-number>
|
||||
yarn reset watcher --block-number <previous-block-number>
|
||||
```
|
||||
|
||||
* Reset job-queue:
|
||||
|
||||
@ -9,9 +9,6 @@
|
||||
# Checkpoint interval in number of blocks.
|
||||
checkpointInterval = 2000
|
||||
|
||||
# IPFS API address (can be taken from the output on running the IPFS daemon).
|
||||
ipfsApiAddr = "/ip4/127.0.0.1/tcp/5001"
|
||||
|
||||
subgraphPath = "../graph-node/test/subgraph/example1/build"
|
||||
wasmRestartBlocksInterval = 20
|
||||
|
||||
|
||||
@ -54,12 +54,12 @@ export const handler = async (argv: any): Promise<void> => {
|
||||
graphWatcher.setIndexer(indexer);
|
||||
await graphWatcher.init();
|
||||
|
||||
const ipldBlock = await indexer.getIPLDBlockByCid(argv.cid);
|
||||
assert(ipldBlock, 'IPLDBlock for the provided CID doesn\'t exist.');
|
||||
const data = indexer.getIPLDData(ipldBlock);
|
||||
const state = await indexer.getStateByCID(argv.cid);
|
||||
assert(state, 'State for the provided CID doesn\'t exist.');
|
||||
const data = indexer.getStateData(state);
|
||||
|
||||
log(`Verifying checkpoint data for contract ${ipldBlock.contractAddress}`);
|
||||
await verifyCheckpointData(graphDb, ipldBlock.block, data);
|
||||
log(`Verifying checkpoint data for contract ${state.contractAddress}`);
|
||||
await verifyCheckpointData(graphDb, state.block, data);
|
||||
log('Checkpoint data verified');
|
||||
|
||||
await db.close();
|
||||
|
||||
@ -70,16 +70,16 @@ const main = async (): Promise<void> => {
|
||||
const exportData: any = {
|
||||
snapshotBlock: {},
|
||||
contracts: [],
|
||||
ipldCheckpoints: []
|
||||
stateCheckpoints: []
|
||||
};
|
||||
|
||||
const contracts = await db.getContracts();
|
||||
let block = await indexer.getLatestHooksProcessedBlock();
|
||||
let block = await indexer.getLatestStateIndexedBlock();
|
||||
assert(block);
|
||||
|
||||
if (argv.blockNumber) {
|
||||
if (argv.blockNumber > block.blockNumber) {
|
||||
throw new Error(`Export snapshot block height ${argv.blockNumber} should be less than latest hooks processed block height ${block.blockNumber}`);
|
||||
throw new Error(`Export snapshot block height ${argv.blockNumber} should be less than latest state indexed block height ${block.blockNumber}`);
|
||||
}
|
||||
|
||||
const blocksAtSnapshotHeight = await indexer.getBlocksAtHeight(argv.blockNumber, false);
|
||||
@ -116,19 +116,15 @@ const main = async (): Promise<void> => {
|
||||
if (contract.checkpoint) {
|
||||
await indexer.createCheckpoint(contract.address, block.blockHash);
|
||||
|
||||
const ipldBlock = await indexer.getLatestIPLDBlock(contract.address, StateKind.Checkpoint, block.blockNumber);
|
||||
assert(ipldBlock);
|
||||
const state = await indexer.getLatestState(contract.address, StateKind.Checkpoint, block.blockNumber);
|
||||
assert(state);
|
||||
|
||||
const data = indexer.getIPLDData(ipldBlock);
|
||||
const data = indexer.getStateData(state);
|
||||
|
||||
if (indexer.isIPFSConfigured()) {
|
||||
await indexer.pushToIPFS(data);
|
||||
}
|
||||
|
||||
exportData.ipldCheckpoints.push({
|
||||
contractAddress: ipldBlock.contractAddress,
|
||||
cid: ipldBlock.cid,
|
||||
kind: ipldBlock.kind,
|
||||
exportData.stateCheckpoints.push({
|
||||
contractAddress: state.contractAddress,
|
||||
cid: state.cid,
|
||||
kind: state.kind,
|
||||
data
|
||||
});
|
||||
}
|
||||
|
||||
@ -18,7 +18,7 @@ import * as codec from '@ipld/dag-cbor';
|
||||
import { Database } from '../database';
|
||||
import { Indexer } from '../indexer';
|
||||
import { EventWatcher } from '../events';
|
||||
import { IPLDBlock } from '../entity/IPLDBlock';
|
||||
import { State } from '../entity/State';
|
||||
|
||||
const log = debug('vulcanize:import-state');
|
||||
|
||||
@ -100,17 +100,17 @@ export const main = async (): Promise<any> => {
|
||||
const block = await indexer.getBlockProgress(importData.snapshotBlock.blockHash);
|
||||
assert(block);
|
||||
|
||||
// Fill the IPLDBlocks.
|
||||
for (const checkpoint of importData.ipldCheckpoints) {
|
||||
let ipldBlock = new IPLDBlock();
|
||||
// Fill the States.
|
||||
for (const checkpoint of importData.stateCheckpoints) {
|
||||
let state = new State();
|
||||
|
||||
ipldBlock = Object.assign(ipldBlock, checkpoint);
|
||||
ipldBlock.block = block;
|
||||
state = Object.assign(state, checkpoint);
|
||||
state.block = block;
|
||||
|
||||
ipldBlock.data = Buffer.from(codec.encode(ipldBlock.data));
|
||||
state.data = Buffer.from(codec.encode(state.data));
|
||||
|
||||
ipldBlock = await indexer.saveOrUpdateIPLDBlock(ipldBlock);
|
||||
await graphWatcher.updateEntitiesFromIPLDState(ipldBlock);
|
||||
state = await indexer.saveOrUpdateState(state);
|
||||
await graphWatcher.updateEntitiesFromState(state);
|
||||
}
|
||||
|
||||
// Mark snapshot block as completely processed.
|
||||
@ -118,12 +118,12 @@ export const main = async (): Promise<any> => {
|
||||
await indexer.updateBlockProgress(block, block.lastProcessedEventIndex);
|
||||
await indexer.updateSyncStatusChainHead(block.blockHash, block.blockNumber);
|
||||
await indexer.updateSyncStatusIndexedBlock(block.blockHash, block.blockNumber);
|
||||
await indexer.updateIPLDStatusHooksBlock(block.blockNumber);
|
||||
await indexer.updateIPLDStatusCheckpointBlock(block.blockNumber);
|
||||
await indexer.updateStateSyncStatusIndexedBlock(block.blockNumber);
|
||||
await indexer.updateStateSyncStatusCheckpointBlock(block.blockNumber);
|
||||
|
||||
// The 'diff_staged' and 'init' IPLD blocks are unnecessary as checkpoints have been already created for the snapshot block.
|
||||
await indexer.removeIPLDBlocks(block.blockNumber, StateKind.Init);
|
||||
await indexer.removeIPLDBlocks(block.blockNumber, StateKind.DiffStaged);
|
||||
// The 'diff_staged' and 'init' State entries are unnecessary as checkpoints have been already created for the snapshot block.
|
||||
await indexer.removeStates(block.blockNumber, StateKind.Init);
|
||||
await indexer.removeStates(block.blockNumber, StateKind.DiffStaged);
|
||||
|
||||
log(`Import completed for snapshot block at height ${block.blockNumber}`);
|
||||
};
|
||||
|
||||
@ -63,12 +63,12 @@ const main = async (): Promise<void> => {
|
||||
graphWatcher.setIndexer(indexer);
|
||||
await graphWatcher.init();
|
||||
|
||||
const ipldBlock = await indexer.getIPLDBlockByCid(argv.cid);
|
||||
assert(ipldBlock, 'IPLDBlock for the provided CID doesn\'t exist.');
|
||||
const state = await indexer.getStateByCID(argv.cid);
|
||||
assert(state, 'State for the provided CID doesn\'t exist.');
|
||||
|
||||
const ipldData = await indexer.getIPLDData(ipldBlock);
|
||||
const stateData = await indexer.getStateData(state);
|
||||
|
||||
log(util.inspect(ipldData, false, null));
|
||||
log(util.inspect(stateData, false, null));
|
||||
};
|
||||
|
||||
main().catch(err => {
|
||||
|
||||
@ -20,11 +20,11 @@ import { Author } from '../../entity/Author';
|
||||
import { Blog } from '../../entity/Blog';
|
||||
import { Category } from '../../entity/Category';
|
||||
|
||||
const log = debug('vulcanize:reset-state');
|
||||
const log = debug('vulcanize:reset-watcher');
|
||||
|
||||
export const command = 'state';
|
||||
export const command = 'watcher';
|
||||
|
||||
export const desc = 'Reset state to block number';
|
||||
export const desc = 'Reset watcher to a block number';
|
||||
|
||||
export const builder = {
|
||||
blockNumber: {
|
||||
@ -86,18 +86,15 @@ export const handler = async (argv: any): Promise<void> => {
|
||||
await indexer.updateSyncStatusCanonicalBlock(blockProgress.blockHash, blockProgress.blockNumber, true);
|
||||
}
|
||||
|
||||
const ipldStatus = await indexer.getIPLDStatus();
|
||||
if (ipldStatus) {
|
||||
if (ipldStatus.latestHooksBlockNumber > blockProgress.blockNumber) {
|
||||
await indexer.updateIPLDStatusHooksBlock(blockProgress.blockNumber, true);
|
||||
const stateSyncStatus = await indexer.getStateSyncStatus();
|
||||
|
||||
if (stateSyncStatus) {
|
||||
if (stateSyncStatus.latestIndexedBlockNumber > blockProgress.blockNumber) {
|
||||
await indexer.updateStateSyncStatusIndexedBlock(blockProgress.blockNumber, true);
|
||||
}
|
||||
|
||||
if (ipldStatus.latestCheckpointBlockNumber > blockProgress.blockNumber) {
|
||||
await indexer.updateIPLDStatusCheckpointBlock(blockProgress.blockNumber, true);
|
||||
}
|
||||
|
||||
if (ipldStatus.latestIPFSBlockNumber > blockProgress.blockNumber) {
|
||||
await indexer.updateIPLDStatusIPFSBlock(blockProgress.blockNumber, true);
|
||||
if (stateSyncStatus.latestCheckpointBlockNumber > blockProgress.blockNumber) {
|
||||
await indexer.updateStateSyncStatusCheckpointBlock(blockProgress.blockNumber, true);
|
||||
}
|
||||
}
|
||||
|
||||
@ -111,5 +108,5 @@ export const handler = async (argv: any): Promise<void> => {
|
||||
await dbTx.release();
|
||||
}
|
||||
|
||||
log('Reset state successfully');
|
||||
log('Reset watcher successfully');
|
||||
};
|
||||
@ -11,9 +11,9 @@ import { Database as BaseDatabase, DatabaseInterface, QueryOptions, StateKind, W
|
||||
import { Contract } from './entity/Contract';
|
||||
import { Event } from './entity/Event';
|
||||
import { SyncStatus } from './entity/SyncStatus';
|
||||
import { IpldStatus } from './entity/IpldStatus';
|
||||
import { StateSyncStatus } from './entity/StateSyncStatus';
|
||||
import { BlockProgress } from './entity/BlockProgress';
|
||||
import { IPLDBlock } from './entity/IPLDBlock';
|
||||
import { State } from './entity/State';
|
||||
|
||||
import { GetMethod } from './entity/GetMethod';
|
||||
import { _Test } from './entity/_Test';
|
||||
@ -73,69 +73,63 @@ export class Database implements DatabaseInterface {
|
||||
return repo.save(entity);
|
||||
}
|
||||
|
||||
getNewIPLDBlock (): IPLDBlock {
|
||||
return new IPLDBlock();
|
||||
getNewState (): State {
|
||||
return new State();
|
||||
}
|
||||
|
||||
async getIPLDBlocks (where: FindConditions<IPLDBlock>): Promise<IPLDBlock[]> {
|
||||
const repo = this._conn.getRepository(IPLDBlock);
|
||||
async getStates (where: FindConditions<State>): Promise<State[]> {
|
||||
const repo = this._conn.getRepository(State);
|
||||
|
||||
return this._baseDatabase.getIPLDBlocks(repo, where);
|
||||
return this._baseDatabase.getStates(repo, where);
|
||||
}
|
||||
|
||||
async getLatestIPLDBlock (contractAddress: string, kind: StateKind | null, blockNumber?: number): Promise<IPLDBlock | undefined> {
|
||||
const repo = this._conn.getRepository(IPLDBlock);
|
||||
async getLatestState (contractAddress: string, kind: StateKind | null, blockNumber?: number): Promise<State | undefined> {
|
||||
const repo = this._conn.getRepository(State);
|
||||
|
||||
return this._baseDatabase.getLatestIPLDBlock(repo, contractAddress, kind, blockNumber);
|
||||
return this._baseDatabase.getLatestState(repo, contractAddress, kind, blockNumber);
|
||||
}
|
||||
|
||||
async getPrevIPLDBlock (blockHash: string, contractAddress: string, kind?: string): Promise<IPLDBlock | undefined> {
|
||||
const repo = this._conn.getRepository(IPLDBlock);
|
||||
async getPrevState (blockHash: string, contractAddress: string, kind?: string): Promise<State | undefined> {
|
||||
const repo = this._conn.getRepository(State);
|
||||
|
||||
return this._baseDatabase.getPrevIPLDBlock(repo, blockHash, contractAddress, kind);
|
||||
return this._baseDatabase.getPrevState(repo, blockHash, contractAddress, kind);
|
||||
}
|
||||
|
||||
// Fetch all diff IPLDBlocks after the specified block number.
|
||||
async getDiffIPLDBlocksInRange (contractAddress: string, startBlock: number, endBlock: number): Promise<IPLDBlock[]> {
|
||||
const repo = this._conn.getRepository(IPLDBlock);
|
||||
// Fetch all diff States after the specified block number.
|
||||
async getDiffStatesInRange (contractAddress: string, startblock: number, endBlock: number): Promise<State[]> {
|
||||
const repo = this._conn.getRepository(State);
|
||||
|
||||
return this._baseDatabase.getDiffIPLDBlocksInRange(repo, contractAddress, startBlock, endBlock);
|
||||
return this._baseDatabase.getDiffStatesInRange(repo, contractAddress, startblock, endBlock);
|
||||
}
|
||||
|
||||
async saveOrUpdateIPLDBlock (dbTx: QueryRunner, ipldBlock: IPLDBlock): Promise<IPLDBlock> {
|
||||
const repo = dbTx.manager.getRepository(IPLDBlock);
|
||||
async saveOrUpdateState (dbTx: QueryRunner, state: State): Promise<State> {
|
||||
const repo = dbTx.manager.getRepository(State);
|
||||
|
||||
return this._baseDatabase.saveOrUpdateIPLDBlock(repo, ipldBlock);
|
||||
return this._baseDatabase.saveOrUpdateState(repo, state);
|
||||
}
|
||||
|
||||
async removeIPLDBlocks (dbTx: QueryRunner, blockNumber: number, kind: string): Promise<void> {
|
||||
const repo = dbTx.manager.getRepository(IPLDBlock);
|
||||
async removeStates (dbTx: QueryRunner, blockNumber: number, kind: string): Promise<void> {
|
||||
const repo = dbTx.manager.getRepository(State);
|
||||
|
||||
await this._baseDatabase.removeIPLDBlocks(repo, blockNumber, kind);
|
||||
await this._baseDatabase.removeStates(repo, blockNumber, kind);
|
||||
}
|
||||
|
||||
async getIPLDStatus (): Promise<IpldStatus | undefined> {
|
||||
const repo = this._conn.getRepository(IpldStatus);
|
||||
async getStateSyncStatus (): Promise<StateSyncStatus | undefined> {
|
||||
const repo = this._conn.getRepository(StateSyncStatus);
|
||||
|
||||
return this._baseDatabase.getIPLDStatus(repo);
|
||||
return this._baseDatabase.getStateSyncStatus(repo);
|
||||
}
|
||||
|
||||
async updateIPLDStatusHooksBlock (queryRunner: QueryRunner, blockNumber: number, force?: boolean): Promise<IpldStatus> {
|
||||
const repo = queryRunner.manager.getRepository(IpldStatus);
|
||||
async updateStateSyncStatusIndexedBlock (queryRunner: QueryRunner, blockNumber: number, force?: boolean): Promise<StateSyncStatus> {
|
||||
const repo = queryRunner.manager.getRepository(StateSyncStatus);
|
||||
|
||||
return this._baseDatabase.updateIPLDStatusHooksBlock(repo, blockNumber, force);
|
||||
return this._baseDatabase.updateStateSyncStatusIndexedBlock(repo, blockNumber, force);
|
||||
}
|
||||
|
||||
async updateIPLDStatusCheckpointBlock (queryRunner: QueryRunner, blockNumber: number, force?: boolean): Promise<IpldStatus> {
|
||||
const repo = queryRunner.manager.getRepository(IpldStatus);
|
||||
async updateStateSyncStatusCheckpointBlock (queryRunner: QueryRunner, blockNumber: number, force?: boolean): Promise<StateSyncStatus> {
|
||||
const repo = queryRunner.manager.getRepository(StateSyncStatus);
|
||||
|
||||
return this._baseDatabase.updateIPLDStatusCheckpointBlock(repo, blockNumber, force);
|
||||
}
|
||||
|
||||
async updateIPLDStatusIPFSBlock (queryRunner: QueryRunner, blockNumber: number, force?: boolean): Promise<IpldStatus> {
|
||||
const repo = queryRunner.manager.getRepository(IpldStatus);
|
||||
|
||||
return this._baseDatabase.updateIPLDStatusIPFSBlock(repo, blockNumber, force);
|
||||
return this._baseDatabase.updateStateSyncStatusCheckpointBlock(repo, blockNumber, force);
|
||||
}
|
||||
|
||||
async getContracts (): Promise<Contract[]> {
|
||||
|
||||
@ -3,14 +3,16 @@
|
||||
//
|
||||
|
||||
import { Entity, PrimaryGeneratedColumn, Column, Index, ManyToOne } from 'typeorm';
|
||||
|
||||
import { StateKind } from '@cerc-io/util';
|
||||
|
||||
import { BlockProgress } from './BlockProgress';
|
||||
|
||||
@Entity()
|
||||
@Index(['cid'], { unique: true })
|
||||
@Index(['block', 'contractAddress'])
|
||||
@Index(['block', 'contractAddress', 'kind'], { unique: true })
|
||||
export class IPLDBlock {
|
||||
export class State {
|
||||
@PrimaryGeneratedColumn()
|
||||
id!: number;
|
||||
|
||||
@ -23,7 +25,10 @@ export class IPLDBlock {
|
||||
@Column('varchar')
|
||||
cid!: string;
|
||||
|
||||
@Column({ type: 'enum', enum: StateKind })
|
||||
@Column({
|
||||
type: 'enum',
|
||||
enum: StateKind
|
||||
})
|
||||
kind!: StateKind;
|
||||
|
||||
@Column('bytea')
|
||||
@ -5,16 +5,13 @@
|
||||
import { Entity, PrimaryGeneratedColumn, Column } from 'typeorm';
|
||||
|
||||
@Entity()
|
||||
export class IpldStatus {
|
||||
export class StateSyncStatus {
|
||||
@PrimaryGeneratedColumn()
|
||||
id!: number;
|
||||
|
||||
@Column('integer')
|
||||
latestHooksBlockNumber!: number;
|
||||
latestIndexedBlockNumber!: number;
|
||||
|
||||
@Column('integer')
|
||||
@Column('integer', { nullable: true })
|
||||
latestCheckpointBlockNumber!: number;
|
||||
|
||||
@Column('integer')
|
||||
latestIPFSBlockNumber!: number;
|
||||
}
|
||||
@ -18,13 +18,13 @@ export async function createInitialState (indexer: Indexer, contractAddress: str
|
||||
assert(blockHash);
|
||||
assert(contractAddress);
|
||||
|
||||
// Store an empty state in an IPLDBlock.
|
||||
const ipldBlockData: any = {
|
||||
// Store an empty State.
|
||||
const stateData: any = {
|
||||
state: {}
|
||||
};
|
||||
|
||||
// Return initial state data to be saved.
|
||||
return ipldBlockData;
|
||||
return stateData;
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@ -12,7 +12,6 @@ import { SelectionNode } from 'graphql';
|
||||
|
||||
import { JsonFragment } from '@ethersproject/abi';
|
||||
import { BaseProvider } from '@ethersproject/providers';
|
||||
import * as codec from '@ipld/dag-cbor';
|
||||
import { EthClient } from '@cerc-io/ipld-eth-client';
|
||||
import { StorageLayout, MappingKey } from '@cerc-io/solidity-mapper';
|
||||
import {
|
||||
@ -25,11 +24,9 @@ import {
|
||||
Where,
|
||||
QueryOptions,
|
||||
BlockHeight,
|
||||
IPFSClient,
|
||||
StateKind,
|
||||
IndexerInterface,
|
||||
IpldStatus as IpldStatusInterface,
|
||||
ResultIPLDBlock
|
||||
StateStatus
|
||||
} from '@cerc-io/util';
|
||||
import { GraphWatcher } from '@cerc-io/graph-node';
|
||||
|
||||
@ -37,9 +34,9 @@ import { Database } from './database';
|
||||
import { Contract } from './entity/Contract';
|
||||
import { Event } from './entity/Event';
|
||||
import { SyncStatus } from './entity/SyncStatus';
|
||||
import { IpldStatus } from './entity/IpldStatus';
|
||||
import { StateSyncStatus } from './entity/StateSyncStatus';
|
||||
import { BlockProgress } from './entity/BlockProgress';
|
||||
import { IPLDBlock } from './entity/IPLDBlock';
|
||||
import { State } from './entity/State';
|
||||
import Example1Artifacts from './artifacts/Example.json';
|
||||
import { createInitialState, handleEvent, createStateDiff, createStateCheckpoint } from './hooks';
|
||||
import { Author } from './entity/Author';
|
||||
@ -87,8 +84,6 @@ export class Indexer implements IndexerInterface {
|
||||
_storageLayoutMap: Map<string, StorageLayout>
|
||||
_contractMap: Map<string, ethers.utils.Interface>
|
||||
|
||||
_ipfsClient: IPFSClient
|
||||
|
||||
_entityTypesMap: Map<string, { [key: string]: string }>
|
||||
_relationsMap: Map<any, { [key: string]: any }>
|
||||
|
||||
@ -102,8 +97,7 @@ export class Indexer implements IndexerInterface {
|
||||
this._ethClient = ethClient;
|
||||
this._ethProvider = ethProvider;
|
||||
this._serverConfig = serverConfig;
|
||||
this._ipfsClient = new IPFSClient(this._serverConfig.ipfsApiAddr);
|
||||
this._baseIndexer = new BaseIndexer(this._serverConfig, this._db, this._ethClient, this._ethProvider, jobQueue, this._ipfsClient);
|
||||
this._baseIndexer = new BaseIndexer(this._serverConfig, this._db, this._ethClient, this._ethProvider, jobQueue);
|
||||
this._graphWatcher = graphWatcher;
|
||||
|
||||
this._abiMap = new Map();
|
||||
@ -140,7 +134,7 @@ export class Indexer implements IndexerInterface {
|
||||
|
||||
async init (): Promise<void> {
|
||||
await this._baseIndexer.fetchContracts();
|
||||
await this._baseIndexer.fetchIPLDStatus();
|
||||
await this._baseIndexer.fetchStateStatus();
|
||||
}
|
||||
|
||||
getResultEvent (event: Event): ResultEvent {
|
||||
@ -178,26 +172,6 @@ export class Indexer implements IndexerInterface {
|
||||
};
|
||||
}
|
||||
|
||||
getResultIPLDBlock (ipldBlock: IPLDBlock): ResultIPLDBlock {
|
||||
const block = ipldBlock.block;
|
||||
|
||||
const data = codec.decode(Buffer.from(ipldBlock.data)) as any;
|
||||
|
||||
return {
|
||||
block: {
|
||||
cid: block.cid,
|
||||
hash: block.blockHash,
|
||||
number: block.blockNumber,
|
||||
timestamp: block.blockTimestamp,
|
||||
parentHash: block.parentHash
|
||||
},
|
||||
contractAddress: ipldBlock.contractAddress,
|
||||
cid: ipldBlock.cid,
|
||||
kind: ipldBlock.kind,
|
||||
data: JSON.stringify(data)
|
||||
};
|
||||
}
|
||||
|
||||
async getMethod (blockHash: string, contractAddress: string): Promise<ValueResult> {
|
||||
const entity = await this._db.getGetMethod({ blockHash, contractAddress });
|
||||
if (entity) {
|
||||
@ -273,10 +247,6 @@ export class Indexer implements IndexerInterface {
|
||||
);
|
||||
}
|
||||
|
||||
async pushToIPFS (data: any): Promise<void> {
|
||||
await this._baseIndexer.pushToIPFS(data);
|
||||
}
|
||||
|
||||
async processInitialState (contractAddress: string, blockHash: string): Promise<any> {
|
||||
// Call initial state hook.
|
||||
return createInitialState(this, contractAddress, blockHash);
|
||||
@ -309,32 +279,28 @@ export class Indexer implements IndexerInterface {
|
||||
return this._baseIndexer.processCLICheckpoint(this, contractAddress, blockHash);
|
||||
}
|
||||
|
||||
async getPrevIPLDBlock (blockHash: string, contractAddress: string, kind?: string): Promise<IPLDBlock | undefined> {
|
||||
return this._db.getPrevIPLDBlock(blockHash, contractAddress, kind);
|
||||
async getPrevState (blockHash: string, contractAddress: string, kind?: string): Promise<State | undefined> {
|
||||
return this._db.getPrevState(blockHash, contractAddress, kind);
|
||||
}
|
||||
|
||||
async getLatestIPLDBlock (contractAddress: string, kind: StateKind | null, blockNumber?: number): Promise<IPLDBlock | undefined> {
|
||||
return this._db.getLatestIPLDBlock(contractAddress, kind, blockNumber);
|
||||
async getLatestState (contractAddress: string, kind: StateKind | null, blockNumber?: number): Promise<State | undefined> {
|
||||
return this._db.getLatestState(contractAddress, kind, blockNumber);
|
||||
}
|
||||
|
||||
async getIPLDBlocksByHash (blockHash: string): Promise<IPLDBlock[]> {
|
||||
return this._baseIndexer.getIPLDBlocksByHash(blockHash);
|
||||
async getStatesByHash (blockHash: string): Promise<State[]> {
|
||||
return this._baseIndexer.getStatesByHash(blockHash);
|
||||
}
|
||||
|
||||
async getIPLDBlockByCid (cid: string): Promise<IPLDBlock | undefined> {
|
||||
return this._baseIndexer.getIPLDBlockByCid(cid);
|
||||
async getStateByCID (cid: string): Promise<State | undefined> {
|
||||
return this._baseIndexer.getStateByCID(cid);
|
||||
}
|
||||
|
||||
async getIPLDBlocks (where: FindConditions<IPLDBlock>): Promise<IPLDBlock[]> {
|
||||
return this._db.getIPLDBlocks(where);
|
||||
async getStates (where: FindConditions<State>): Promise<State[]> {
|
||||
return this._db.getStates(where);
|
||||
}
|
||||
|
||||
getIPLDData (ipldBlock: IPLDBlock): any {
|
||||
return this._baseIndexer.getIPLDData(ipldBlock);
|
||||
}
|
||||
|
||||
isIPFSConfigured (): boolean {
|
||||
return this._baseIndexer.isIPFSConfigured();
|
||||
getStateData (state: State): any {
|
||||
return this._baseIndexer.getStateData(state);
|
||||
}
|
||||
|
||||
// Method used to create auto diffs (diff_staged).
|
||||
@ -366,12 +332,12 @@ export class Indexer implements IndexerInterface {
|
||||
return this._baseIndexer.createCheckpoint(this, contractAddress, block);
|
||||
}
|
||||
|
||||
async saveOrUpdateIPLDBlock (ipldBlock: IPLDBlock): Promise<IPLDBlock> {
|
||||
return this._baseIndexer.saveOrUpdateIPLDBlock(ipldBlock);
|
||||
async saveOrUpdateState (state: State): Promise<State> {
|
||||
return this._baseIndexer.saveOrUpdateState(state);
|
||||
}
|
||||
|
||||
async removeIPLDBlocks (blockNumber: number, kind: StateKind): Promise<void> {
|
||||
await this._baseIndexer.removeIPLDBlocks(blockNumber, kind);
|
||||
async removeStates (blockNumber: number, kind: StateKind): Promise<void> {
|
||||
await this._baseIndexer.removeStates(blockNumber, kind);
|
||||
}
|
||||
|
||||
async getSubgraphEntity<Entity> (
|
||||
@ -432,16 +398,16 @@ export class Indexer implements IndexerInterface {
|
||||
};
|
||||
}
|
||||
|
||||
async getIPLDStatus (): Promise<IpldStatus | undefined> {
|
||||
return this._db.getIPLDStatus();
|
||||
async getStateSyncStatus (): Promise<StateSyncStatus | undefined> {
|
||||
return this._db.getStateSyncStatus();
|
||||
}
|
||||
|
||||
async updateIPLDStatusHooksBlock (blockNumber: number, force?: boolean): Promise<IpldStatus> {
|
||||
async updateStateSyncStatusIndexedBlock (blockNumber: number, force?: boolean): Promise<StateSyncStatus> {
|
||||
const dbTx = await this._db.createTransactionRunner();
|
||||
let res;
|
||||
|
||||
try {
|
||||
res = await this._db.updateIPLDStatusHooksBlock(dbTx, blockNumber, force);
|
||||
res = await this._db.updateStateSyncStatusIndexedBlock(dbTx, blockNumber, force);
|
||||
await dbTx.commitTransaction();
|
||||
} catch (error) {
|
||||
await dbTx.rollbackTransaction();
|
||||
@ -453,29 +419,12 @@ export class Indexer implements IndexerInterface {
|
||||
return res;
|
||||
}
|
||||
|
||||
async updateIPLDStatusCheckpointBlock (blockNumber: number, force?: boolean): Promise<IpldStatus> {
|
||||
async updateStateSyncStatusCheckpointBlock (blockNumber: number, force?: boolean): Promise<StateSyncStatus> {
|
||||
const dbTx = await this._db.createTransactionRunner();
|
||||
let res;
|
||||
|
||||
try {
|
||||
res = await this._db.updateIPLDStatusCheckpointBlock(dbTx, blockNumber, force);
|
||||
await dbTx.commitTransaction();
|
||||
} catch (error) {
|
||||
await dbTx.rollbackTransaction();
|
||||
throw error;
|
||||
} finally {
|
||||
await dbTx.release();
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
async updateIPLDStatusIPFSBlock (blockNumber: number, force?: boolean): Promise<IpldStatus> {
|
||||
const dbTx = await this._db.createTransactionRunner();
|
||||
let res;
|
||||
|
||||
try {
|
||||
res = await this._db.updateIPLDStatusIPFSBlock(dbTx, blockNumber, force);
|
||||
res = await this._db.updateStateSyncStatusCheckpointBlock(dbTx, blockNumber, force);
|
||||
await dbTx.commitTransaction();
|
||||
} catch (error) {
|
||||
await dbTx.rollbackTransaction();
|
||||
@ -497,16 +446,16 @@ export class Indexer implements IndexerInterface {
|
||||
return latestCanonicalBlock;
|
||||
}
|
||||
|
||||
async getLatestHooksProcessedBlock (): Promise<BlockProgress> {
|
||||
return this._baseIndexer.getLatestHooksProcessedBlock();
|
||||
async getLatestStateIndexedBlock (): Promise<BlockProgress> {
|
||||
return this._baseIndexer.getLatestStateIndexedBlock();
|
||||
}
|
||||
|
||||
async watchContract (address: string, kind: string, checkpoint: boolean, startingBlock: number): Promise<void> {
|
||||
return this._baseIndexer.watchContract(address, kind, checkpoint, startingBlock);
|
||||
}
|
||||
|
||||
async updateIPLDStatusMap (address: string, ipldStatus: IpldStatusInterface): Promise<void> {
|
||||
await this._baseIndexer.updateIPLDStatusMap(address, ipldStatus);
|
||||
updateStateStatusMap (address: string, stateStatus: StateStatus): void {
|
||||
this._baseIndexer.updateStateStatusMap(address, stateStatus);
|
||||
}
|
||||
|
||||
cacheContract (contract: Contract): void {
|
||||
|
||||
@ -18,8 +18,6 @@ import {
|
||||
QUEUE_EVENT_PROCESSING,
|
||||
QUEUE_BLOCK_CHECKPOINT,
|
||||
QUEUE_HOOKS,
|
||||
QUEUE_IPFS,
|
||||
JOB_KIND_PRUNE,
|
||||
JobQueueConfig,
|
||||
DEFAULT_CONFIG_PATH,
|
||||
initClients,
|
||||
@ -50,22 +48,12 @@ export class JobRunner {
|
||||
await this.subscribeEventProcessingQueue();
|
||||
await this.subscribeBlockCheckpointQueue();
|
||||
await this.subscribeHooksQueue();
|
||||
await this.subscribeIPFSQueue();
|
||||
this._baseJobRunner.handleShutdown();
|
||||
}
|
||||
|
||||
async subscribeBlockProcessingQueue (): Promise<void> {
|
||||
await this._jobQueue.subscribe(QUEUE_BLOCK_PROCESSING, async (job) => {
|
||||
await this._baseJobRunner.processBlock(job);
|
||||
|
||||
const { data: { kind } } = job;
|
||||
|
||||
// If it's a pruning job: Create a hooks job.
|
||||
if (kind === JOB_KIND_PRUNE) {
|
||||
await this.createHooksJob();
|
||||
}
|
||||
|
||||
await this._jobQueue.markComplete(job);
|
||||
});
|
||||
}
|
||||
|
||||
@ -77,165 +65,15 @@ export class JobRunner {
|
||||
|
||||
async subscribeHooksQueue (): Promise<void> {
|
||||
await this._jobQueue.subscribe(QUEUE_HOOKS, async (job) => {
|
||||
const { data: { blockHash, blockNumber } } = job;
|
||||
|
||||
// Get the current IPLD Status.
|
||||
const ipldStatus = await this._indexer.getIPLDStatus();
|
||||
|
||||
if (ipldStatus) {
|
||||
if (ipldStatus.latestHooksBlockNumber < (blockNumber - 1)) {
|
||||
// Create hooks job for parent block.
|
||||
const [parentBlock] = await this._indexer.getBlocksAtHeight(blockNumber - 1, false);
|
||||
await this.createHooksJob(parentBlock.blockHash, parentBlock.blockNumber);
|
||||
|
||||
const message = `Hooks for blockNumber ${blockNumber - 1} not processed yet, aborting`;
|
||||
log(message);
|
||||
|
||||
throw new Error(message);
|
||||
}
|
||||
|
||||
if (ipldStatus.latestHooksBlockNumber > (blockNumber - 1)) {
|
||||
log(`Hooks for blockNumber ${blockNumber} already processed`);
|
||||
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// Process the hooks for the given block number.
|
||||
await this._indexer.processCanonicalBlock(blockHash, blockNumber);
|
||||
|
||||
// Update the IPLD status.
|
||||
await this._indexer.updateIPLDStatusHooksBlock(blockNumber);
|
||||
|
||||
// Create a checkpoint job after completion of a hook job.
|
||||
await this.createCheckpointJob(blockHash, blockNumber);
|
||||
|
||||
await this._jobQueue.markComplete(job);
|
||||
await this._baseJobRunner.processHooks(job);
|
||||
});
|
||||
}
|
||||
|
||||
async subscribeBlockCheckpointQueue (): Promise<void> {
|
||||
await this._jobQueue.subscribe(QUEUE_BLOCK_CHECKPOINT, async (job) => {
|
||||
const { data: { blockHash, blockNumber } } = job;
|
||||
|
||||
// Get the current IPLD Status.
|
||||
const ipldStatus = await this._indexer.getIPLDStatus();
|
||||
assert(ipldStatus);
|
||||
|
||||
if (ipldStatus.latestCheckpointBlockNumber >= 0) {
|
||||
if (ipldStatus.latestCheckpointBlockNumber < (blockNumber - 1)) {
|
||||
// Create a checkpoint job for parent block.
|
||||
const [parentBlock] = await this._indexer.getBlocksAtHeight(blockNumber - 1, false);
|
||||
await this.createCheckpointJob(parentBlock.blockHash, parentBlock.blockNumber);
|
||||
|
||||
const message = `Checkpoints for blockNumber ${blockNumber - 1} not processed yet, aborting`;
|
||||
log(message);
|
||||
|
||||
throw new Error(message);
|
||||
}
|
||||
|
||||
if (ipldStatus.latestCheckpointBlockNumber > (blockNumber - 1)) {
|
||||
log(`Checkpoints for blockNumber ${blockNumber} already processed`);
|
||||
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// Process checkpoints for the given block.
|
||||
await this._indexer.processCheckpoint(blockHash);
|
||||
|
||||
// Update the IPLD status.
|
||||
await this._indexer.updateIPLDStatusCheckpointBlock(blockNumber);
|
||||
|
||||
// Create an IPFS job after completion of a checkpoint job.
|
||||
if (this._indexer.isIPFSConfigured()) {
|
||||
await this.createIPFSPutJob(blockHash, blockNumber);
|
||||
}
|
||||
|
||||
await this._jobQueue.markComplete(job);
|
||||
await this._baseJobRunner.processCheckpoint(job);
|
||||
});
|
||||
}
|
||||
|
||||
async subscribeIPFSQueue (): Promise<void> {
|
||||
await this._jobQueue.subscribe(QUEUE_IPFS, async (job) => {
|
||||
const { data: { blockHash, blockNumber } } = job;
|
||||
|
||||
const ipldStatus = await this._indexer.getIPLDStatus();
|
||||
assert(ipldStatus);
|
||||
|
||||
if (ipldStatus.latestIPFSBlockNumber >= 0) {
|
||||
if (ipldStatus.latestIPFSBlockNumber < (blockNumber - 1)) {
|
||||
// Create a IPFS job for parent block.
|
||||
const [parentBlock] = await this._indexer.getBlocksAtHeight(blockNumber - 1, false);
|
||||
await this.createIPFSPutJob(parentBlock.blockHash, parentBlock.blockNumber);
|
||||
|
||||
const message = `IPFS for blockNumber ${blockNumber - 1} not processed yet, aborting`;
|
||||
log(message);
|
||||
|
||||
throw new Error(message);
|
||||
}
|
||||
|
||||
if (ipldStatus.latestIPFSBlockNumber > (blockNumber - 1)) {
|
||||
log(`IPFS for blockNumber ${blockNumber} already processed`);
|
||||
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// Get IPLDBlocks for the given blocHash.
|
||||
const ipldBlocks = await this._indexer.getIPLDBlocksByHash(blockHash);
|
||||
|
||||
// Push all the IPLDBlocks to IPFS.
|
||||
for (const ipldBlock of ipldBlocks) {
|
||||
const data = this._indexer.getIPLDData(ipldBlock);
|
||||
await this._indexer.pushToIPFS(data);
|
||||
}
|
||||
|
||||
// Update the IPLD status.
|
||||
await this._indexer.updateIPLDStatusIPFSBlock(blockNumber);
|
||||
|
||||
await this._jobQueue.markComplete(job);
|
||||
});
|
||||
}
|
||||
|
||||
async createHooksJob (blockHash?: string, blockNumber?: number): Promise<void> {
|
||||
if (!blockNumber || !blockHash) {
|
||||
// Get the latest canonical block
|
||||
const latestCanonicalBlock = await this._indexer.getLatestCanonicalBlock();
|
||||
|
||||
// Create a hooks job for parent block of latestCanonicalBlock because pruning for first block is skipped as it is assumed to be a canonical block.
|
||||
blockHash = latestCanonicalBlock.parentHash;
|
||||
blockNumber = latestCanonicalBlock.blockNumber - 1;
|
||||
}
|
||||
|
||||
await this._jobQueue.pushJob(
|
||||
QUEUE_HOOKS,
|
||||
{
|
||||
blockHash,
|
||||
blockNumber
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
async createCheckpointJob (blockHash: string, blockNumber: number): Promise<void> {
|
||||
await this._jobQueue.pushJob(
|
||||
QUEUE_BLOCK_CHECKPOINT,
|
||||
{
|
||||
blockHash,
|
||||
blockNumber
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
async createIPFSPutJob (blockHash: string, blockNumber: number): Promise<void> {
|
||||
await this._jobQueue.pushJob(
|
||||
QUEUE_IPFS,
|
||||
{
|
||||
blockHash,
|
||||
blockNumber
|
||||
}
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export const main = async (): Promise<any> => {
|
||||
|
||||
@ -8,7 +8,7 @@ import debug from 'debug';
|
||||
import Decimal from 'decimal.js';
|
||||
import { GraphQLResolveInfo, GraphQLScalarType } from 'graphql';
|
||||
|
||||
import { ValueResult, BlockHeight, gqlTotalQueryCount, gqlQueryCount, jsonBigIntStringReplacer } from '@cerc-io/util';
|
||||
import { ValueResult, BlockHeight, gqlTotalQueryCount, gqlQueryCount, jsonBigIntStringReplacer, getResultState } from '@cerc-io/util';
|
||||
|
||||
import { Indexer } from './indexer';
|
||||
import { EventWatcher } from './events';
|
||||
@ -153,9 +153,9 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch
|
||||
gqlTotalQueryCount.inc(1);
|
||||
gqlQueryCount.labels('getStateByCID').inc(1);
|
||||
|
||||
const ipldBlock = await indexer.getIPLDBlockByCid(cid);
|
||||
const state = await indexer.getStateByCID(cid);
|
||||
|
||||
return ipldBlock && ipldBlock.block.isComplete ? indexer.getResultIPLDBlock(ipldBlock) : undefined;
|
||||
return state && state.block.isComplete ? getResultState(state) : undefined;
|
||||
},
|
||||
|
||||
getState: async (_: any, { blockHash, contractAddress, kind }: { blockHash: string, contractAddress: string, kind: string }) => {
|
||||
@ -163,9 +163,9 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch
|
||||
gqlTotalQueryCount.inc(1);
|
||||
gqlQueryCount.labels('getState').inc(1);
|
||||
|
||||
const ipldBlock = await indexer.getPrevIPLDBlock(blockHash, contractAddress, kind);
|
||||
const state = await indexer.getPrevState(blockHash, contractAddress, kind);
|
||||
|
||||
return ipldBlock && ipldBlock.block.isComplete ? indexer.getResultIPLDBlock(ipldBlock) : undefined;
|
||||
return state && state.block.isComplete ? getResultState(state) : undefined;
|
||||
},
|
||||
|
||||
getSyncStatus: async () => {
|
||||
|
||||
@ -65,7 +65,7 @@ type TestEvent {
|
||||
param3: BigInt!
|
||||
}
|
||||
|
||||
type ResultIPLDBlock {
|
||||
type ResultState {
|
||||
block: Block!
|
||||
contractAddress: String!
|
||||
cid: String!
|
||||
@ -88,8 +88,8 @@ type Query {
|
||||
blog(id: String!, block: Block_height): Blog!
|
||||
author(id: String!, block: Block_height): Author!
|
||||
category(id: String!, block: Block_height): Category!
|
||||
getStateByCID(cid: String!): ResultIPLDBlock
|
||||
getState(blockHash: String!, contractAddress: String!, kind: String): ResultIPLDBlock
|
||||
getStateByCID(cid: String!): ResultState
|
||||
getState(blockHash: String!, contractAddress: String!, kind: String): ResultState
|
||||
getSyncStatus: SyncStatus
|
||||
}
|
||||
|
||||
|
||||
@ -8,12 +8,6 @@
|
||||
yarn
|
||||
```
|
||||
|
||||
* Run the IPFS (go-ipfs version 0.12.2) daemon:
|
||||
|
||||
```bash
|
||||
ipfs daemon
|
||||
```
|
||||
|
||||
* Create a postgres12 database for the watcher:
|
||||
|
||||
```bash
|
||||
@ -47,7 +41,7 @@
|
||||
|
||||
* Update the `upstream` config and provide the `ipld-eth-server` GQL API endpoint.
|
||||
|
||||
* Update the `server` config with state checkpoint settings and provide the IPFS API address.
|
||||
* Update the `server` config with state checkpoint settings.
|
||||
|
||||
## Customize
|
||||
|
||||
@ -59,11 +53,11 @@
|
||||
|
||||
* Generating state:
|
||||
|
||||
* Edit the custom hook function `createInitialState` (triggered if the watcher passes the start block, checkpoint: `true`) in [hooks.ts](./src/hooks.ts) to save an initial state `IPLDBlock` using the `Indexer` object.
|
||||
* Edit the custom hook function `createInitialState` (triggered if the watcher passes the start block, checkpoint: `true`) in [hooks.ts](./src/hooks.ts) to save an initial `State` using the `Indexer` object.
|
||||
|
||||
* Edit the custom hook function `createStateDiff` (triggered on a block) in [hooks.ts](./src/hooks.ts) to save the state in a `diff` `IPLDBlock` using the `Indexer` object. The default state (if exists) is updated.
|
||||
* Edit the custom hook function `createStateDiff` (triggered on a block) in [hooks.ts](./src/hooks.ts) to save the state in a `diff` `State` using the `Indexer` object. The default state (if exists) is updated.
|
||||
|
||||
* Edit the custom hook function `createStateCheckpoint` (triggered just before default and CLI checkpoint) in [hooks.ts](./src/hooks.ts) to save the state in a `checkpoint` `IPLDBlock` using the `Indexer` object.
|
||||
* Edit the custom hook function `createStateCheckpoint` (triggered just before default and CLI checkpoint) in [hooks.ts](./src/hooks.ts) to save the state in a `checkpoint` `State` using the `Indexer` object.
|
||||
|
||||
## Run
|
||||
|
||||
@ -130,10 +124,10 @@ GQL console: http://localhost:3010/graphql
|
||||
|
||||
* To reset the watcher to a previous block number:
|
||||
|
||||
* Reset state:
|
||||
* Reset watcher:
|
||||
|
||||
```bash
|
||||
yarn reset state --block-number <previous-block-number>
|
||||
yarn reset watcher --block-number <previous-block-number>
|
||||
```
|
||||
|
||||
* Reset job-queue:
|
||||
|
||||
@ -9,9 +9,6 @@
|
||||
# Checkpoint interval in number of blocks.
|
||||
checkpointInterval = 2000
|
||||
|
||||
# IPFS API address (can be taken from the output on running the IPFS daemon).
|
||||
# ipfsApiAddr = "/ip4/127.0.0.1/tcp/5001"
|
||||
|
||||
# Boolean to filter logs by contract.
|
||||
filterLogs = true
|
||||
|
||||
|
||||
@ -61,16 +61,16 @@ const main = async (): Promise<void> => {
|
||||
const exportData: any = {
|
||||
snapshotBlock: {},
|
||||
contracts: [],
|
||||
ipldCheckpoints: []
|
||||
stateCheckpoints: []
|
||||
};
|
||||
|
||||
const contracts = await db.getContracts();
|
||||
let block = await indexer.getLatestHooksProcessedBlock();
|
||||
let block = await indexer.getLatestStateIndexedBlock();
|
||||
assert(block);
|
||||
|
||||
if (argv.blockNumber) {
|
||||
if (argv.blockNumber > block.blockNumber) {
|
||||
throw new Error(`Export snapshot block height ${argv.blockNumber} should be less than latest hooks processed block height ${block.blockNumber}`);
|
||||
throw new Error(`Export snapshot block height ${argv.blockNumber} should be less than latest state indexed block height ${block.blockNumber}`);
|
||||
}
|
||||
|
||||
const blocksAtSnapshotHeight = await indexer.getBlocksAtHeight(argv.blockNumber, false);
|
||||
@ -107,19 +107,15 @@ const main = async (): Promise<void> => {
|
||||
if (contract.checkpoint) {
|
||||
await indexer.createCheckpoint(contract.address, block.blockHash);
|
||||
|
||||
const ipldBlock = await indexer.getLatestIPLDBlock(contract.address, StateKind.Checkpoint, block.blockNumber);
|
||||
assert(ipldBlock);
|
||||
const state = await indexer.getLatestState(contract.address, StateKind.Checkpoint, block.blockNumber);
|
||||
assert(state);
|
||||
|
||||
const data = indexer.getIPLDData(ipldBlock);
|
||||
const data = indexer.getStateData(state);
|
||||
|
||||
if (indexer.isIPFSConfigured()) {
|
||||
await indexer.pushToIPFS(data);
|
||||
}
|
||||
|
||||
exportData.ipldCheckpoints.push({
|
||||
contractAddress: ipldBlock.contractAddress,
|
||||
cid: ipldBlock.cid,
|
||||
kind: ipldBlock.kind,
|
||||
exportData.stateCheckpoints.push({
|
||||
contractAddress: state.contractAddress,
|
||||
cid: state.cid,
|
||||
kind: state.kind,
|
||||
data
|
||||
});
|
||||
}
|
||||
|
||||
@ -17,7 +17,7 @@ import * as codec from '@ipld/dag-cbor';
|
||||
import { Database } from '../database';
|
||||
import { Indexer } from '../indexer';
|
||||
import { EventWatcher } from '../events';
|
||||
import { IPLDBlock } from '../entity/IPLDBlock';
|
||||
import { State } from '../entity/State';
|
||||
|
||||
const log = debug('vulcanize:import-state');
|
||||
|
||||
@ -91,16 +91,16 @@ export const main = async (): Promise<any> => {
|
||||
const block = await indexer.getBlockProgress(importData.snapshotBlock.blockHash);
|
||||
assert(block);
|
||||
|
||||
// Fill the IPLDBlocks.
|
||||
for (const checkpoint of importData.ipldCheckpoints) {
|
||||
let ipldBlock = new IPLDBlock();
|
||||
// Fill the States.
|
||||
for (const checkpoint of importData.stateCheckpoints) {
|
||||
let state = new State();
|
||||
|
||||
ipldBlock = Object.assign(ipldBlock, checkpoint);
|
||||
ipldBlock.block = block;
|
||||
state = Object.assign(state, checkpoint);
|
||||
state.block = block;
|
||||
|
||||
ipldBlock.data = Buffer.from(codec.encode(ipldBlock.data));
|
||||
state.data = Buffer.from(codec.encode(state.data));
|
||||
|
||||
ipldBlock = await indexer.saveOrUpdateIPLDBlock(ipldBlock);
|
||||
state = await indexer.saveOrUpdateState(state);
|
||||
}
|
||||
|
||||
// Mark snapshot block as completely processed.
|
||||
@ -108,12 +108,12 @@ export const main = async (): Promise<any> => {
|
||||
await indexer.updateBlockProgress(block, block.lastProcessedEventIndex);
|
||||
await indexer.updateSyncStatusChainHead(block.blockHash, block.blockNumber);
|
||||
await indexer.updateSyncStatusIndexedBlock(block.blockHash, block.blockNumber);
|
||||
await indexer.updateIPLDStatusHooksBlock(block.blockNumber);
|
||||
await indexer.updateIPLDStatusCheckpointBlock(block.blockNumber);
|
||||
await indexer.updateStateSyncStatusIndexedBlock(block.blockNumber);
|
||||
await indexer.updateStateSyncStatusCheckpointBlock(block.blockNumber);
|
||||
|
||||
// The 'diff_staged' and 'init' IPLD blocks are unnecessary as checkpoints have been already created for the snapshot block.
|
||||
await indexer.removeIPLDBlocks(block.blockNumber, StateKind.Init);
|
||||
await indexer.removeIPLDBlocks(block.blockNumber, StateKind.DiffStaged);
|
||||
// The 'diff_staged' and 'init' State entries are unnecessary as checkpoints have been already created for the snapshot block.
|
||||
await indexer.removeStates(block.blockNumber, StateKind.Init);
|
||||
await indexer.removeStates(block.blockNumber, StateKind.DiffStaged);
|
||||
|
||||
log(`Import completed for snapshot block at height ${block.blockNumber}`);
|
||||
};
|
||||
|
||||
@ -53,12 +53,12 @@ const main = async (): Promise<void> => {
|
||||
const indexer = new Indexer(config.server, db, ethClient, ethProvider, jobQueue);
|
||||
await indexer.init();
|
||||
|
||||
const ipldBlock = await indexer.getIPLDBlockByCid(argv.cid);
|
||||
assert(ipldBlock, 'IPLDBlock for the provided CID doesn\'t exist.');
|
||||
const state = await indexer.getStateByCID(argv.cid);
|
||||
assert(state, 'State for the provided CID doesn\'t exist.');
|
||||
|
||||
const ipldData = await indexer.getIPLDData(ipldBlock);
|
||||
const stateData = await indexer.getStateData(state);
|
||||
|
||||
log(util.inspect(ipldData, false, null));
|
||||
log(util.inspect(stateData, false, null));
|
||||
};
|
||||
|
||||
main().catch(err => {
|
||||
|
||||
@ -18,11 +18,11 @@ import { IsRevoked } from '../../entity/IsRevoked';
|
||||
import { IsPhisher } from '../../entity/IsPhisher';
|
||||
import { IsMember } from '../../entity/IsMember';
|
||||
|
||||
const log = debug('vulcanize:reset-state');
|
||||
const log = debug('vulcanize:reset-watcher');
|
||||
|
||||
export const command = 'state';
|
||||
export const command = 'watcher';
|
||||
|
||||
export const desc = 'Reset state to block number';
|
||||
export const desc = 'Reset watcher to a block number';
|
||||
|
||||
export const builder = {
|
||||
blockNumber: {
|
||||
@ -76,19 +76,15 @@ export const handler = async (argv: any): Promise<void> => {
|
||||
await indexer.updateSyncStatusCanonicalBlock(blockProgress.blockHash, blockProgress.blockNumber, true);
|
||||
}
|
||||
|
||||
const ipldStatus = await indexer.getIPLDStatus();
|
||||
const stateSyncStatus = await indexer.getStateSyncStatus();
|
||||
|
||||
if (ipldStatus) {
|
||||
if (ipldStatus.latestHooksBlockNumber > blockProgress.blockNumber) {
|
||||
await indexer.updateIPLDStatusHooksBlock(blockProgress.blockNumber, true);
|
||||
if (stateSyncStatus) {
|
||||
if (stateSyncStatus.latestIndexedBlockNumber > blockProgress.blockNumber) {
|
||||
await indexer.updateStateSyncStatusIndexedBlock(blockProgress.blockNumber, true);
|
||||
}
|
||||
|
||||
if (ipldStatus.latestCheckpointBlockNumber > blockProgress.blockNumber) {
|
||||
await indexer.updateIPLDStatusCheckpointBlock(blockProgress.blockNumber, true);
|
||||
}
|
||||
|
||||
if (ipldStatus.latestIPFSBlockNumber > blockProgress.blockNumber) {
|
||||
await indexer.updateIPLDStatusIPFSBlock(blockProgress.blockNumber, true);
|
||||
if (stateSyncStatus.latestCheckpointBlockNumber > blockProgress.blockNumber) {
|
||||
await indexer.updateStateSyncStatusCheckpointBlock(blockProgress.blockNumber, true);
|
||||
}
|
||||
}
|
||||
|
||||
@ -102,5 +98,5 @@ export const handler = async (argv: any): Promise<void> => {
|
||||
await dbTx.release();
|
||||
}
|
||||
|
||||
log('Reset state successfully');
|
||||
log('Reset watcher successfully');
|
||||
};
|
||||
@ -11,9 +11,9 @@ import { Database as BaseDatabase, DatabaseInterface, QueryOptions, StateKind, W
|
||||
import { Contract } from './entity/Contract';
|
||||
import { Event } from './entity/Event';
|
||||
import { SyncStatus } from './entity/SyncStatus';
|
||||
import { IpldStatus } from './entity/IpldStatus';
|
||||
import { StateSyncStatus } from './entity/StateSyncStatus';
|
||||
import { BlockProgress } from './entity/BlockProgress';
|
||||
import { IPLDBlock } from './entity/IPLDBlock';
|
||||
import { State } from './entity/State';
|
||||
import { MultiNonce } from './entity/MultiNonce';
|
||||
import { _Owner } from './entity/_Owner';
|
||||
import { IsRevoked } from './entity/IsRevoked';
|
||||
@ -132,69 +132,63 @@ export class Database implements DatabaseInterface {
|
||||
return repo.save(entity);
|
||||
}
|
||||
|
||||
getNewIPLDBlock (): IPLDBlock {
|
||||
return new IPLDBlock();
|
||||
getNewState (): State {
|
||||
return new State();
|
||||
}
|
||||
|
||||
async getIPLDBlocks (where: FindConditions<IPLDBlock>): Promise<IPLDBlock[]> {
|
||||
const repo = this._conn.getRepository(IPLDBlock);
|
||||
async getStates (where: FindConditions<State>): Promise<State[]> {
|
||||
const repo = this._conn.getRepository(State);
|
||||
|
||||
return this._baseDatabase.getIPLDBlocks(repo, where);
|
||||
return this._baseDatabase.getStates(repo, where);
|
||||
}
|
||||
|
||||
async getLatestIPLDBlock (contractAddress: string, kind: StateKind | null, blockNumber?: number): Promise<IPLDBlock | undefined> {
|
||||
const repo = this._conn.getRepository(IPLDBlock);
|
||||
async getLatestState (contractAddress: string, kind: StateKind | null, blockNumber?: number): Promise<State | undefined> {
|
||||
const repo = this._conn.getRepository(State);
|
||||
|
||||
return this._baseDatabase.getLatestIPLDBlock(repo, contractAddress, kind, blockNumber);
|
||||
return this._baseDatabase.getLatestState(repo, contractAddress, kind, blockNumber);
|
||||
}
|
||||
|
||||
async getPrevIPLDBlock (blockHash: string, contractAddress: string, kind?: string): Promise<IPLDBlock | undefined> {
|
||||
const repo = this._conn.getRepository(IPLDBlock);
|
||||
async getPrevState (blockHash: string, contractAddress: string, kind?: string): Promise<State | undefined> {
|
||||
const repo = this._conn.getRepository(State);
|
||||
|
||||
return this._baseDatabase.getPrevIPLDBlock(repo, blockHash, contractAddress, kind);
|
||||
return this._baseDatabase.getPrevState(repo, blockHash, contractAddress, kind);
|
||||
}
|
||||
|
||||
// Fetch all diff IPLDBlocks after the specified block number.
|
||||
async getDiffIPLDBlocksInRange (contractAddress: string, startBlock: number, endBlock: number): Promise<IPLDBlock[]> {
|
||||
const repo = this._conn.getRepository(IPLDBlock);
|
||||
// Fetch all diff States after the specified block number.
|
||||
async getDiffStatesInRange (contractAddress: string, startblock: number, endBlock: number): Promise<State[]> {
|
||||
const repo = this._conn.getRepository(State);
|
||||
|
||||
return this._baseDatabase.getDiffIPLDBlocksInRange(repo, contractAddress, startBlock, endBlock);
|
||||
return this._baseDatabase.getDiffStatesInRange(repo, contractAddress, startblock, endBlock);
|
||||
}
|
||||
|
||||
async saveOrUpdateIPLDBlock (dbTx: QueryRunner, ipldBlock: IPLDBlock): Promise<IPLDBlock> {
|
||||
const repo = dbTx.manager.getRepository(IPLDBlock);
|
||||
async saveOrUpdateState (dbTx: QueryRunner, state: State): Promise<State> {
|
||||
const repo = dbTx.manager.getRepository(State);
|
||||
|
||||
return this._baseDatabase.saveOrUpdateIPLDBlock(repo, ipldBlock);
|
||||
return this._baseDatabase.saveOrUpdateState(repo, state);
|
||||
}
|
||||
|
||||
async removeIPLDBlocks (dbTx: QueryRunner, blockNumber: number, kind: string): Promise<void> {
|
||||
const repo = dbTx.manager.getRepository(IPLDBlock);
|
||||
async removeStates (dbTx: QueryRunner, blockNumber: number, kind: string): Promise<void> {
|
||||
const repo = dbTx.manager.getRepository(State);
|
||||
|
||||
await this._baseDatabase.removeIPLDBlocks(repo, blockNumber, kind);
|
||||
await this._baseDatabase.removeStates(repo, blockNumber, kind);
|
||||
}
|
||||
|
||||
async getIPLDStatus (): Promise<IpldStatus | undefined> {
|
||||
const repo = this._conn.getRepository(IpldStatus);
|
||||
async getStateSyncStatus (): Promise<StateSyncStatus | undefined> {
|
||||
const repo = this._conn.getRepository(StateSyncStatus);
|
||||
|
||||
return this._baseDatabase.getIPLDStatus(repo);
|
||||
return this._baseDatabase.getStateSyncStatus(repo);
|
||||
}
|
||||
|
||||
async updateIPLDStatusHooksBlock (queryRunner: QueryRunner, blockNumber: number, force?: boolean): Promise<IpldStatus> {
|
||||
const repo = queryRunner.manager.getRepository(IpldStatus);
|
||||
async updateStateSyncStatusIndexedBlock (queryRunner: QueryRunner, blockNumber: number, force?: boolean): Promise<StateSyncStatus> {
|
||||
const repo = queryRunner.manager.getRepository(StateSyncStatus);
|
||||
|
||||
return this._baseDatabase.updateIPLDStatusHooksBlock(repo, blockNumber, force);
|
||||
return this._baseDatabase.updateStateSyncStatusIndexedBlock(repo, blockNumber, force);
|
||||
}
|
||||
|
||||
async updateIPLDStatusCheckpointBlock (queryRunner: QueryRunner, blockNumber: number, force?: boolean): Promise<IpldStatus> {
|
||||
const repo = queryRunner.manager.getRepository(IpldStatus);
|
||||
async updateStateSyncStatusCheckpointBlock (queryRunner: QueryRunner, blockNumber: number, force?: boolean): Promise<StateSyncStatus> {
|
||||
const repo = queryRunner.manager.getRepository(StateSyncStatus);
|
||||
|
||||
return this._baseDatabase.updateIPLDStatusCheckpointBlock(repo, blockNumber, force);
|
||||
}
|
||||
|
||||
async updateIPLDStatusIPFSBlock (queryRunner: QueryRunner, blockNumber: number, force?: boolean): Promise<IpldStatus> {
|
||||
const repo = queryRunner.manager.getRepository(IpldStatus);
|
||||
|
||||
return this._baseDatabase.updateIPLDStatusIPFSBlock(repo, blockNumber, force);
|
||||
return this._baseDatabase.updateStateSyncStatusCheckpointBlock(repo, blockNumber, force);
|
||||
}
|
||||
|
||||
async getContracts (): Promise<Contract[]> {
|
||||
|
||||
@ -1,31 +0,0 @@
|
||||
//
|
||||
// Copyright 2021 Vulcanize, Inc.
|
||||
//
|
||||
|
||||
import { Entity, PrimaryGeneratedColumn, Column, Index, ManyToOne } from 'typeorm';
|
||||
import { StateKind } from '@cerc-io/util';
|
||||
import { BlockProgress } from './BlockProgress';
|
||||
|
||||
@Entity()
|
||||
@Index(['cid'], { unique: true })
|
||||
@Index(['block', 'contractAddress'])
|
||||
@Index(['block', 'contractAddress', 'kind'], { unique: true })
|
||||
export class IPLDBlock {
|
||||
@PrimaryGeneratedColumn()
|
||||
id!: number;
|
||||
|
||||
@ManyToOne(() => BlockProgress, { onDelete: 'CASCADE' })
|
||||
block!: BlockProgress;
|
||||
|
||||
@Column('varchar', { length: 42 })
|
||||
contractAddress!: string;
|
||||
|
||||
@Column('varchar')
|
||||
cid!: string;
|
||||
|
||||
@Column({ type: 'enum', enum: StateKind })
|
||||
kind!: StateKind;
|
||||
|
||||
@Column('bytea')
|
||||
data!: Buffer;
|
||||
}
|
||||
36
packages/mobymask-watcher/src/entity/State.ts
Normal file
36
packages/mobymask-watcher/src/entity/State.ts
Normal file
@ -0,0 +1,36 @@
|
||||
//
|
||||
// Copyright 2021 Vulcanize, Inc.
|
||||
//
|
||||
|
||||
import { Entity, PrimaryGeneratedColumn, Column, Index, ManyToOne } from 'typeorm';
|
||||
|
||||
import { StateKind } from '@cerc-io/util';
|
||||
|
||||
import { BlockProgress } from './BlockProgress';
|
||||
|
||||
@Entity()
|
||||
@Index(['cid'], { unique: true })
|
||||
@Index(['block', 'contractAddress'])
|
||||
@Index(['block', 'contractAddress', 'kind'], { unique: true })
|
||||
export class State {
|
||||
@PrimaryGeneratedColumn()
|
||||
id!: number;
|
||||
|
||||
@ManyToOne(() => BlockProgress, { onDelete: 'CASCADE' })
|
||||
block!: BlockProgress;
|
||||
|
||||
@Column('varchar', { length: 42 })
|
||||
contractAddress!: string;
|
||||
|
||||
@Column('varchar')
|
||||
cid!: string;
|
||||
|
||||
@Column({
|
||||
type: 'enum',
|
||||
enum: StateKind
|
||||
})
|
||||
kind!: StateKind;
|
||||
|
||||
@Column('bytea')
|
||||
data!: Buffer;
|
||||
}
|
||||
17
packages/mobymask-watcher/src/entity/StateSyncStatus.ts
Normal file
17
packages/mobymask-watcher/src/entity/StateSyncStatus.ts
Normal file
@ -0,0 +1,17 @@
|
||||
//
|
||||
// Copyright 2021 Vulcanize, Inc.
|
||||
//
|
||||
|
||||
import { Entity, PrimaryGeneratedColumn, Column } from 'typeorm';
|
||||
|
||||
@Entity()
|
||||
export class StateSyncStatus {
|
||||
@PrimaryGeneratedColumn()
|
||||
id!: number;
|
||||
|
||||
@Column('integer')
|
||||
latestIndexedBlockNumber!: number;
|
||||
|
||||
@Column('integer', { nullable: true })
|
||||
latestCheckpointBlockNumber!: number;
|
||||
}
|
||||
@ -25,19 +25,19 @@ export async function createInitialState (indexer: Indexer, contractAddress: str
|
||||
assert(blockHash);
|
||||
assert(contractAddress);
|
||||
|
||||
// Store the desired initial state in an IPLDBlock.
|
||||
const ipldBlockData: any = {
|
||||
// Store an empty State.
|
||||
const stateData: any = {
|
||||
state: {}
|
||||
};
|
||||
|
||||
// Use updateStateForElementaryType to update initial state with an elementary property.
|
||||
// Eg. const ipldBlockData = updateStateForElementaryType(ipldBlockData, '_totalBalance', result.value.toString());
|
||||
// Eg. const stateData = updateStateForElementaryType(stateData, '_totalBalance', result.value.toString());
|
||||
|
||||
// Use updateStateForMappingType to update initial state with a nested property.
|
||||
// Eg. const ipldBlockData = updateStateForMappingType(ipldBlockData, '_allowances', [owner, spender], allowance.value.toString());
|
||||
// Eg. const stateData = updateStateForMappingType(stateData, '_allowances', [owner, spender], allowance.value.toString());
|
||||
|
||||
// Return initial state data to be saved.
|
||||
return ipldBlockData;
|
||||
return stateData;
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@ -10,7 +10,6 @@ import { ethers } from 'ethers';
|
||||
|
||||
import { JsonFragment } from '@ethersproject/abi';
|
||||
import { JsonRpcProvider } from '@ethersproject/providers';
|
||||
import * as codec from '@ipld/dag-cbor';
|
||||
import { EthClient } from '@cerc-io/ipld-eth-client';
|
||||
import { MappingKey, StorageLayout } from '@cerc-io/solidity-mapper';
|
||||
import {
|
||||
@ -25,11 +24,9 @@ import {
|
||||
updateStateForElementaryType,
|
||||
updateStateForMappingType,
|
||||
BlockHeight,
|
||||
IPFSClient,
|
||||
StateKind,
|
||||
IpldStatus as IpldStatusInterface,
|
||||
getFullTransaction,
|
||||
ResultIPLDBlock
|
||||
StateStatus,
|
||||
getFullTransaction
|
||||
} from '@cerc-io/util';
|
||||
|
||||
import PhisherRegistryArtifacts from './artifacts/PhisherRegistry.json';
|
||||
@ -38,9 +35,9 @@ import { createInitialState, handleEvent, createStateDiff, createStateCheckpoint
|
||||
import { Contract } from './entity/Contract';
|
||||
import { Event } from './entity/Event';
|
||||
import { SyncStatus } from './entity/SyncStatus';
|
||||
import { IpldStatus } from './entity/IpldStatus';
|
||||
import { StateSyncStatus } from './entity/StateSyncStatus';
|
||||
import { BlockProgress } from './entity/BlockProgress';
|
||||
import { IPLDBlock } from './entity/IPLDBlock';
|
||||
import { State } from './entity/State';
|
||||
import { IsMember } from './entity/IsMember';
|
||||
import { IsPhisher } from './entity/IsPhisher';
|
||||
import { IsRevoked } from './entity/IsRevoked';
|
||||
@ -87,8 +84,6 @@ export class Indexer implements IndexerInterface {
|
||||
_storageLayoutMap: Map<string, StorageLayout>
|
||||
_contractMap: Map<string, ethers.utils.Interface>
|
||||
|
||||
_ipfsClient: IPFSClient
|
||||
|
||||
constructor (serverConfig: ServerConfig, db: Database, ethClient: EthClient, ethProvider: JsonRpcProvider, jobQueue: JobQueue) {
|
||||
assert(db);
|
||||
assert(ethClient);
|
||||
@ -97,8 +92,7 @@ export class Indexer implements IndexerInterface {
|
||||
this._ethClient = ethClient;
|
||||
this._ethProvider = ethProvider;
|
||||
this._serverConfig = serverConfig;
|
||||
this._ipfsClient = new IPFSClient(this._serverConfig.ipfsApiAddr);
|
||||
this._baseIndexer = new BaseIndexer(this._serverConfig, this._db, this._ethClient, this._ethProvider, jobQueue, this._ipfsClient);
|
||||
this._baseIndexer = new BaseIndexer(this._serverConfig, this._db, this._ethClient, this._ethProvider, jobQueue);
|
||||
|
||||
this._abiMap = new Map();
|
||||
this._storageLayoutMap = new Map();
|
||||
@ -126,7 +120,7 @@ export class Indexer implements IndexerInterface {
|
||||
|
||||
async init (): Promise<void> {
|
||||
await this._baseIndexer.fetchContracts();
|
||||
await this._baseIndexer.fetchIPLDStatus();
|
||||
await this._baseIndexer.fetchStateStatus();
|
||||
}
|
||||
|
||||
getResultEvent (event: Event): ResultEvent {
|
||||
@ -164,26 +158,6 @@ export class Indexer implements IndexerInterface {
|
||||
};
|
||||
}
|
||||
|
||||
getResultIPLDBlock (ipldBlock: IPLDBlock): ResultIPLDBlock {
|
||||
const block = ipldBlock.block;
|
||||
|
||||
const data = codec.decode(Buffer.from(ipldBlock.data)) as any;
|
||||
|
||||
return {
|
||||
block: {
|
||||
cid: block.cid,
|
||||
hash: block.blockHash,
|
||||
number: block.blockNumber,
|
||||
timestamp: block.blockTimestamp,
|
||||
parentHash: block.parentHash
|
||||
},
|
||||
contractAddress: ipldBlock.contractAddress,
|
||||
cid: ipldBlock.cid,
|
||||
kind: ipldBlock.kind,
|
||||
data: JSON.stringify(data)
|
||||
};
|
||||
}
|
||||
|
||||
async multiNonce (blockHash: string, contractAddress: string, key0: string, key1: bigint, diff = false): Promise<ValueResult> {
|
||||
let entity = await this._db.getMultiNonce({ blockHash, contractAddress, key0, key1 });
|
||||
|
||||
@ -401,10 +375,6 @@ export class Indexer implements IndexerInterface {
|
||||
);
|
||||
}
|
||||
|
||||
async pushToIPFS (data: any): Promise<void> {
|
||||
await this._baseIndexer.pushToIPFS(data);
|
||||
}
|
||||
|
||||
async processInitialState (contractAddress: string, blockHash: string): Promise<any> {
|
||||
// Call initial state hook.
|
||||
return createInitialState(this, contractAddress, blockHash);
|
||||
@ -435,28 +405,24 @@ export class Indexer implements IndexerInterface {
|
||||
return this._baseIndexer.processCLICheckpoint(this, contractAddress, blockHash);
|
||||
}
|
||||
|
||||
async getPrevIPLDBlock (blockHash: string, contractAddress: string, kind?: string): Promise<IPLDBlock | undefined> {
|
||||
return this._db.getPrevIPLDBlock(blockHash, contractAddress, kind);
|
||||
async getPrevState (blockHash: string, contractAddress: string, kind?: string): Promise<State | undefined> {
|
||||
return this._db.getPrevState(blockHash, contractAddress, kind);
|
||||
}
|
||||
|
||||
async getLatestIPLDBlock (contractAddress: string, kind: StateKind | null, blockNumber?: number): Promise<IPLDBlock | undefined> {
|
||||
return this._db.getLatestIPLDBlock(contractAddress, kind, blockNumber);
|
||||
async getLatestState (contractAddress: string, kind: StateKind | null, blockNumber?: number): Promise<State | undefined> {
|
||||
return this._db.getLatestState(contractAddress, kind, blockNumber);
|
||||
}
|
||||
|
||||
async getIPLDBlocksByHash (blockHash: string): Promise<IPLDBlock[]> {
|
||||
return this._baseIndexer.getIPLDBlocksByHash(blockHash);
|
||||
async getStatesByHash (blockHash: string): Promise<State[]> {
|
||||
return this._baseIndexer.getStatesByHash(blockHash);
|
||||
}
|
||||
|
||||
async getIPLDBlockByCid (cid: string): Promise<IPLDBlock | undefined> {
|
||||
return this._baseIndexer.getIPLDBlockByCid(cid);
|
||||
async getStateByCID (cid: string): Promise<State | undefined> {
|
||||
return this._baseIndexer.getStateByCID(cid);
|
||||
}
|
||||
|
||||
getIPLDData (ipldBlock: IPLDBlock): any {
|
||||
return this._baseIndexer.getIPLDData(ipldBlock);
|
||||
}
|
||||
|
||||
isIPFSConfigured (): boolean {
|
||||
return this._baseIndexer.isIPFSConfigured();
|
||||
getStateData (state: State): any {
|
||||
return this._baseIndexer.getStateData(state);
|
||||
}
|
||||
|
||||
// Method used to create auto diffs (diff_staged).
|
||||
@ -488,12 +454,12 @@ export class Indexer implements IndexerInterface {
|
||||
return this._baseIndexer.createCheckpoint(this, contractAddress, block);
|
||||
}
|
||||
|
||||
async saveOrUpdateIPLDBlock (ipldBlock: IPLDBlock): Promise<IPLDBlock> {
|
||||
return this._baseIndexer.saveOrUpdateIPLDBlock(ipldBlock);
|
||||
async saveOrUpdateState (state: State): Promise<State> {
|
||||
return this._baseIndexer.saveOrUpdateState(state);
|
||||
}
|
||||
|
||||
async removeIPLDBlocks (blockNumber: number, kind: StateKind): Promise<void> {
|
||||
await this._baseIndexer.removeIPLDBlocks(blockNumber, kind);
|
||||
async removeStates (blockNumber: number, kind: StateKind): Promise<void> {
|
||||
await this._baseIndexer.removeStates(blockNumber, kind);
|
||||
}
|
||||
|
||||
async triggerIndexingOnEvent (event: Event): Promise<void> {
|
||||
@ -530,16 +496,16 @@ export class Indexer implements IndexerInterface {
|
||||
};
|
||||
}
|
||||
|
||||
async getIPLDStatus (): Promise<IpldStatus | undefined> {
|
||||
return this._db.getIPLDStatus();
|
||||
async getStateSyncStatus (): Promise<StateSyncStatus | undefined> {
|
||||
return this._db.getStateSyncStatus();
|
||||
}
|
||||
|
||||
async updateIPLDStatusHooksBlock (blockNumber: number, force?: boolean): Promise<IpldStatus> {
|
||||
async updateStateSyncStatusIndexedBlock (blockNumber: number, force?: boolean): Promise<StateSyncStatus> {
|
||||
const dbTx = await this._db.createTransactionRunner();
|
||||
let res;
|
||||
|
||||
try {
|
||||
res = await this._db.updateIPLDStatusHooksBlock(dbTx, blockNumber, force);
|
||||
res = await this._db.updateStateSyncStatusIndexedBlock(dbTx, blockNumber, force);
|
||||
await dbTx.commitTransaction();
|
||||
} catch (error) {
|
||||
await dbTx.rollbackTransaction();
|
||||
@ -551,29 +517,12 @@ export class Indexer implements IndexerInterface {
|
||||
return res;
|
||||
}
|
||||
|
||||
async updateIPLDStatusCheckpointBlock (blockNumber: number, force?: boolean): Promise<IpldStatus> {
|
||||
async updateStateSyncStatusCheckpointBlock (blockNumber: number, force?: boolean): Promise<StateSyncStatus> {
|
||||
const dbTx = await this._db.createTransactionRunner();
|
||||
let res;
|
||||
|
||||
try {
|
||||
res = await this._db.updateIPLDStatusCheckpointBlock(dbTx, blockNumber, force);
|
||||
await dbTx.commitTransaction();
|
||||
} catch (error) {
|
||||
await dbTx.rollbackTransaction();
|
||||
throw error;
|
||||
} finally {
|
||||
await dbTx.release();
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
async updateIPLDStatusIPFSBlock (blockNumber: number, force?: boolean): Promise<IpldStatus> {
|
||||
const dbTx = await this._db.createTransactionRunner();
|
||||
let res;
|
||||
|
||||
try {
|
||||
res = await this._db.updateIPLDStatusIPFSBlock(dbTx, blockNumber, force);
|
||||
res = await this._db.updateStateSyncStatusCheckpointBlock(dbTx, blockNumber, force);
|
||||
await dbTx.commitTransaction();
|
||||
} catch (error) {
|
||||
await dbTx.rollbackTransaction();
|
||||
@ -595,16 +544,16 @@ export class Indexer implements IndexerInterface {
|
||||
return latestCanonicalBlock;
|
||||
}
|
||||
|
||||
async getLatestHooksProcessedBlock (): Promise<BlockProgress> {
|
||||
return this._baseIndexer.getLatestHooksProcessedBlock();
|
||||
async getLatestStateIndexedBlock (): Promise<BlockProgress> {
|
||||
return this._baseIndexer.getLatestStateIndexedBlock();
|
||||
}
|
||||
|
||||
async watchContract (address: string, kind: string, checkpoint: boolean, startingBlock: number): Promise<void> {
|
||||
return this._baseIndexer.watchContract(address, kind, checkpoint, startingBlock);
|
||||
}
|
||||
|
||||
async updateIPLDStatusMap (address: string, ipldStatus: IpldStatusInterface): Promise<void> {
|
||||
await this._baseIndexer.updateIPLDStatusMap(address, ipldStatus);
|
||||
updateStateStatusMap (address: string, stateStatus: StateStatus): void {
|
||||
this._baseIndexer.updateStateStatusMap(address, stateStatus);
|
||||
}
|
||||
|
||||
cacheContract (contract: Contract): void {
|
||||
|
||||
@ -17,8 +17,6 @@ import {
|
||||
QUEUE_EVENT_PROCESSING,
|
||||
QUEUE_BLOCK_CHECKPOINT,
|
||||
QUEUE_HOOKS,
|
||||
QUEUE_IPFS,
|
||||
JOB_KIND_PRUNE,
|
||||
JobQueueConfig,
|
||||
DEFAULT_CONFIG_PATH,
|
||||
initClients,
|
||||
@ -48,22 +46,12 @@ export class JobRunner {
|
||||
await this.subscribeEventProcessingQueue();
|
||||
await this.subscribeBlockCheckpointQueue();
|
||||
await this.subscribeHooksQueue();
|
||||
await this.subscribeIPFSQueue();
|
||||
this._baseJobRunner.handleShutdown();
|
||||
}
|
||||
|
||||
async subscribeBlockProcessingQueue (): Promise<void> {
|
||||
await this._jobQueue.subscribe(QUEUE_BLOCK_PROCESSING, async (job) => {
|
||||
await this._baseJobRunner.processBlock(job);
|
||||
|
||||
const { data: { kind } } = job;
|
||||
|
||||
// If it's a pruning job: Create a hooks job.
|
||||
if (kind === JOB_KIND_PRUNE) {
|
||||
await this.createHooksJob();
|
||||
}
|
||||
|
||||
await this._jobQueue.markComplete(job);
|
||||
});
|
||||
}
|
||||
|
||||
@ -75,165 +63,15 @@ export class JobRunner {
|
||||
|
||||
async subscribeHooksQueue (): Promise<void> {
|
||||
await this._jobQueue.subscribe(QUEUE_HOOKS, async (job) => {
|
||||
const { data: { blockHash, blockNumber } } = job;
|
||||
|
||||
// Get the current IPLD Status.
|
||||
const ipldStatus = await this._indexer.getIPLDStatus();
|
||||
|
||||
if (ipldStatus) {
|
||||
if (ipldStatus.latestHooksBlockNumber < (blockNumber - 1)) {
|
||||
// Create hooks job for parent block.
|
||||
const [parentBlock] = await this._indexer.getBlocksAtHeight(blockNumber - 1, false);
|
||||
await this.createHooksJob(parentBlock.blockHash, parentBlock.blockNumber);
|
||||
|
||||
const message = `Hooks for blockNumber ${blockNumber - 1} not processed yet, aborting`;
|
||||
log(message);
|
||||
|
||||
throw new Error(message);
|
||||
}
|
||||
|
||||
if (ipldStatus.latestHooksBlockNumber > (blockNumber - 1)) {
|
||||
log(`Hooks for blockNumber ${blockNumber} already processed`);
|
||||
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// Process the hooks for the given block number.
|
||||
await this._indexer.processCanonicalBlock(blockHash);
|
||||
|
||||
// Update the IPLD status.
|
||||
await this._indexer.updateIPLDStatusHooksBlock(blockNumber);
|
||||
|
||||
// Create a checkpoint job after completion of a hook job.
|
||||
await this.createCheckpointJob(blockHash, blockNumber);
|
||||
|
||||
await this._jobQueue.markComplete(job);
|
||||
await this._baseJobRunner.processHooks(job);
|
||||
});
|
||||
}
|
||||
|
||||
async subscribeBlockCheckpointQueue (): Promise<void> {
|
||||
await this._jobQueue.subscribe(QUEUE_BLOCK_CHECKPOINT, async (job) => {
|
||||
const { data: { blockHash, blockNumber } } = job;
|
||||
|
||||
// Get the current IPLD Status.
|
||||
const ipldStatus = await this._indexer.getIPLDStatus();
|
||||
assert(ipldStatus);
|
||||
|
||||
if (ipldStatus.latestCheckpointBlockNumber >= 0) {
|
||||
if (ipldStatus.latestCheckpointBlockNumber < (blockNumber - 1)) {
|
||||
// Create a checkpoint job for parent block.
|
||||
const [parentBlock] = await this._indexer.getBlocksAtHeight(blockNumber - 1, false);
|
||||
await this.createCheckpointJob(parentBlock.blockHash, parentBlock.blockNumber);
|
||||
|
||||
const message = `Checkpoints for blockNumber ${blockNumber - 1} not processed yet, aborting`;
|
||||
log(message);
|
||||
|
||||
throw new Error(message);
|
||||
}
|
||||
|
||||
if (ipldStatus.latestCheckpointBlockNumber > (blockNumber - 1)) {
|
||||
log(`Checkpoints for blockNumber ${blockNumber} already processed`);
|
||||
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// Process checkpoints for the given block.
|
||||
await this._indexer.processCheckpoint(blockHash);
|
||||
|
||||
// Update the IPLD status.
|
||||
await this._indexer.updateIPLDStatusCheckpointBlock(blockNumber);
|
||||
|
||||
// Create an IPFS job after completion of a checkpoint job.
|
||||
if (this._indexer.isIPFSConfigured()) {
|
||||
await this.createIPFSPutJob(blockHash, blockNumber);
|
||||
}
|
||||
|
||||
await this._jobQueue.markComplete(job);
|
||||
await this._baseJobRunner.processCheckpoint(job);
|
||||
});
|
||||
}
|
||||
|
||||
async subscribeIPFSQueue (): Promise<void> {
|
||||
await this._jobQueue.subscribe(QUEUE_IPFS, async (job) => {
|
||||
const { data: { blockHash, blockNumber } } = job;
|
||||
|
||||
const ipldStatus = await this._indexer.getIPLDStatus();
|
||||
assert(ipldStatus);
|
||||
|
||||
if (ipldStatus.latestIPFSBlockNumber >= 0) {
|
||||
if (ipldStatus.latestIPFSBlockNumber < (blockNumber - 1)) {
|
||||
// Create a IPFS job for parent block.
|
||||
const [parentBlock] = await this._indexer.getBlocksAtHeight(blockNumber - 1, false);
|
||||
await this.createIPFSPutJob(parentBlock.blockHash, parentBlock.blockNumber);
|
||||
|
||||
const message = `IPFS for blockNumber ${blockNumber - 1} not processed yet, aborting`;
|
||||
log(message);
|
||||
|
||||
throw new Error(message);
|
||||
}
|
||||
|
||||
if (ipldStatus.latestIPFSBlockNumber > (blockNumber - 1)) {
|
||||
log(`IPFS for blockNumber ${blockNumber} already processed`);
|
||||
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// Get IPLDBlocks for the given blocHash.
|
||||
const ipldBlocks = await this._indexer.getIPLDBlocksByHash(blockHash);
|
||||
|
||||
// Push all the IPLDBlocks to IPFS.
|
||||
for (const ipldBlock of ipldBlocks) {
|
||||
const data = this._indexer.getIPLDData(ipldBlock);
|
||||
await this._indexer.pushToIPFS(data);
|
||||
}
|
||||
|
||||
// Update the IPLD status.
|
||||
await this._indexer.updateIPLDStatusIPFSBlock(blockNumber);
|
||||
|
||||
await this._jobQueue.markComplete(job);
|
||||
});
|
||||
}
|
||||
|
||||
async createHooksJob (blockHash?: string, blockNumber?: number): Promise<void> {
|
||||
if (!blockNumber || !blockHash) {
|
||||
// Get the latest canonical block
|
||||
const latestCanonicalBlock = await this._indexer.getLatestCanonicalBlock();
|
||||
|
||||
// Create a hooks job for parent block of latestCanonicalBlock because pruning for first block is skipped as it is assumed to be a canonical block.
|
||||
blockHash = latestCanonicalBlock.parentHash;
|
||||
blockNumber = latestCanonicalBlock.blockNumber - 1;
|
||||
}
|
||||
|
||||
await this._jobQueue.pushJob(
|
||||
QUEUE_HOOKS,
|
||||
{
|
||||
blockHash,
|
||||
blockNumber
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
async createCheckpointJob (blockHash: string, blockNumber: number): Promise<void> {
|
||||
await this._jobQueue.pushJob(
|
||||
QUEUE_BLOCK_CHECKPOINT,
|
||||
{
|
||||
blockHash,
|
||||
blockNumber
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
async createIPFSPutJob (blockHash: string, blockNumber: number): Promise<void> {
|
||||
await this._jobQueue.pushJob(
|
||||
QUEUE_IPFS,
|
||||
{
|
||||
blockHash,
|
||||
blockNumber
|
||||
}
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export const main = async (): Promise<any> => {
|
||||
|
||||
@ -8,7 +8,7 @@ import debug from 'debug';
|
||||
import Decimal from 'decimal.js';
|
||||
import { GraphQLScalarType } from 'graphql';
|
||||
|
||||
import { ValueResult, gqlTotalQueryCount, gqlQueryCount } from '@cerc-io/util';
|
||||
import { ValueResult, gqlTotalQueryCount, gqlQueryCount, getResultState } from '@cerc-io/util';
|
||||
|
||||
import { Indexer } from './indexer';
|
||||
import { EventWatcher } from './events';
|
||||
@ -126,9 +126,9 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch
|
||||
gqlTotalQueryCount.inc(1);
|
||||
gqlQueryCount.labels('getStateByCID').inc(1);
|
||||
|
||||
const ipldBlock = await indexer.getIPLDBlockByCid(cid);
|
||||
const state = await indexer.getStateByCID(cid);
|
||||
|
||||
return ipldBlock && ipldBlock.block.isComplete ? indexer.getResultIPLDBlock(ipldBlock) : undefined;
|
||||
return state && state.block.isComplete ? getResultState(state) : undefined;
|
||||
},
|
||||
|
||||
getState: async (_: any, { blockHash, contractAddress, kind }: { blockHash: string, contractAddress: string, kind: string }) => {
|
||||
@ -136,9 +136,9 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch
|
||||
gqlTotalQueryCount.inc(1);
|
||||
gqlQueryCount.labels('getState').inc(1);
|
||||
|
||||
const ipldBlock = await indexer.getPrevIPLDBlock(blockHash, contractAddress, kind);
|
||||
const state = await indexer.getPrevState(blockHash, contractAddress, kind);
|
||||
|
||||
return ipldBlock && ipldBlock.block.isComplete ? indexer.getResultIPLDBlock(ipldBlock) : undefined;
|
||||
return state && state.block.isComplete ? getResultState(state) : undefined;
|
||||
},
|
||||
|
||||
getSyncStatus: async () => {
|
||||
|
||||
@ -79,7 +79,7 @@ type PhisherStatusUpdatedEvent {
|
||||
isPhisher: Boolean!
|
||||
}
|
||||
|
||||
type ResultIPLDBlock {
|
||||
type ResultState {
|
||||
block: _Block_!
|
||||
contractAddress: String!
|
||||
cid: String!
|
||||
@ -102,8 +102,8 @@ type Query {
|
||||
isRevoked(blockHash: String!, contractAddress: String!, key0: String!): ResultBoolean!
|
||||
isPhisher(blockHash: String!, contractAddress: String!, key0: String!): ResultBoolean!
|
||||
isMember(blockHash: String!, contractAddress: String!, key0: String!): ResultBoolean!
|
||||
getStateByCID(cid: String!): ResultIPLDBlock
|
||||
getState(blockHash: String!, contractAddress: String!, kind: String): ResultIPLDBlock
|
||||
getStateByCID(cid: String!): ResultState
|
||||
getState(blockHash: String!, contractAddress: String!, kind: String): ResultState
|
||||
getSyncStatus: SyncStatus
|
||||
latestBlock: Block_height
|
||||
}
|
||||
|
||||
@ -12,9 +12,8 @@ export * from './src/events';
|
||||
export * from './src/types';
|
||||
export * from './src/indexer';
|
||||
export * from './src/job-runner';
|
||||
export * from './src/ipld-helper';
|
||||
export * from './src/state-helper';
|
||||
export * from './src/graph-decimal';
|
||||
export * from './src/ipfs';
|
||||
export * from './src/index-block';
|
||||
export * from './src/metrics';
|
||||
export * from './src/gql-metrics';
|
||||
|
||||
@ -2,7 +2,14 @@ import debug from 'debug';
|
||||
import assert from 'assert';
|
||||
import { DeepPartial } from 'typeorm';
|
||||
|
||||
import { QUEUE_BLOCK_PROCESSING, JOB_KIND_PRUNE, JOB_KIND_INDEX, UNKNOWN_EVENT_NAME } from './constants';
|
||||
import {
|
||||
QUEUE_BLOCK_PROCESSING,
|
||||
QUEUE_HOOKS,
|
||||
QUEUE_BLOCK_CHECKPOINT,
|
||||
JOB_KIND_PRUNE,
|
||||
JOB_KIND_INDEX,
|
||||
UNKNOWN_EVENT_NAME
|
||||
} from './constants';
|
||||
import { JobQueue } from './job-queue';
|
||||
import { BlockProgressInterface, IndexerInterface, EventInterface } from './types';
|
||||
import { wait } from './misc';
|
||||
@ -18,30 +25,6 @@ export interface PrefetchedBlock {
|
||||
events: DeepPartial<EventInterface>[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Create pruning job in QUEUE_BLOCK_PROCESSING.
|
||||
* @param jobQueue
|
||||
* @param latestCanonicalBlockNumber
|
||||
* @param priority
|
||||
*/
|
||||
export const createPruningJob = async (jobQueue: JobQueue, latestCanonicalBlockNumber: number, priority = 0): Promise<void> => {
|
||||
const pruneBlockHeight = latestCanonicalBlockNumber + 1;
|
||||
const newPriority = priority + 1;
|
||||
|
||||
// Create a job to prune at block height (latestCanonicalBlockNumber + 1).
|
||||
return jobQueue.pushJob(
|
||||
QUEUE_BLOCK_PROCESSING,
|
||||
{
|
||||
kind: JOB_KIND_PRUNE,
|
||||
pruneBlockHeight,
|
||||
priority: newPriority
|
||||
},
|
||||
{
|
||||
priority: newPriority
|
||||
}
|
||||
);
|
||||
};
|
||||
|
||||
/**
|
||||
* Method to fetch block by number and push to job queue.
|
||||
* @param jobQueue
|
||||
@ -382,6 +365,62 @@ export const processBatchEvents = async (indexer: IndexerInterface, block: Block
|
||||
console.timeEnd('time:common#processBatchEvents-updateBlockProgress');
|
||||
};
|
||||
|
||||
/**
|
||||
* Create pruning job in QUEUE_BLOCK_PROCESSING.
|
||||
* @param jobQueue
|
||||
* @param latestCanonicalBlockNumber
|
||||
* @param priority
|
||||
*/
|
||||
export const createPruningJob = async (jobQueue: JobQueue, latestCanonicalBlockNumber: number, priority = 0): Promise<void> => {
|
||||
const pruneBlockHeight = latestCanonicalBlockNumber + 1;
|
||||
const newPriority = priority + 1;
|
||||
|
||||
// Create a job to prune at block height (latestCanonicalBlockNumber + 1).
|
||||
return jobQueue.pushJob(
|
||||
QUEUE_BLOCK_PROCESSING,
|
||||
{
|
||||
kind: JOB_KIND_PRUNE,
|
||||
pruneBlockHeight,
|
||||
priority: newPriority
|
||||
},
|
||||
{
|
||||
priority: newPriority
|
||||
}
|
||||
);
|
||||
};
|
||||
|
||||
/**
|
||||
* Create a job in QUEUE_HOOKS.
|
||||
* @param jobQueue
|
||||
* @param blockHash
|
||||
* @param blockNumber
|
||||
*/
|
||||
export const createHooksJob = async (jobQueue: JobQueue, blockHash: string, blockNumber: number): Promise<void> => {
|
||||
await jobQueue.pushJob(
|
||||
QUEUE_HOOKS,
|
||||
{
|
||||
blockHash,
|
||||
blockNumber
|
||||
}
|
||||
);
|
||||
};
|
||||
|
||||
/**
|
||||
* Create a job in QUEUE_BLOCK_CHECKPOINT.
|
||||
* @param jobQueue
|
||||
* @param blockHash
|
||||
* @param blockNumber
|
||||
*/
|
||||
export const createCheckpointJob = async (jobQueue: JobQueue, blockHash: string, blockNumber: number): Promise<void> => {
|
||||
await jobQueue.pushJob(
|
||||
QUEUE_BLOCK_CHECKPOINT,
|
||||
{
|
||||
blockHash,
|
||||
blockNumber
|
||||
}
|
||||
);
|
||||
};
|
||||
|
||||
const getPrefetchedBlocksAtHeight = (prefetchedBlocksMap: Map<string, PrefetchedBlock>, blockNumber: number):any[] => {
|
||||
return Array.from(prefetchedBlocksMap.values())
|
||||
.filter(({ block }) => Number(block.blockNumber) === blockNumber)
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user