mirror of
https://github.com/cerc-io/watcher-ts
synced 2026-09-09 09:14:06 +00:00
Clear cache of latest entities on event processing error (#201)
* Clear cache of latest entities on event processing error * Remove lighthouse-watcher and update ethersjs version * Handle GraphDecimal type in state entity * Add option for comparing all entities using paginate * Clear pruned cached entities at intervals * Move ipld-demo to graph-node package and remove reset-dbs script * Implement changes in all watchers and codegen
This commit is contained in:
@@ -22,6 +22,9 @@
|
||||
# Interval to restart wasm instance periodically
|
||||
wasmRestartBlocksInterval = 20
|
||||
|
||||
# Interval in number of blocks at which to clear entities cache.
|
||||
clearEntitiesCacheInterval = 1000
|
||||
|
||||
{{/if}}
|
||||
# Boolean to filter logs by contract.
|
||||
filterLogs = false
|
||||
|
||||
@@ -118,6 +118,10 @@ const main = async (): Promise<void> => {
|
||||
|
||||
// Export contracts and checkpoints.
|
||||
for (const contract of contracts) {
|
||||
if (contract.startingBlock > block.blockNumber) {
|
||||
continue;
|
||||
}
|
||||
|
||||
exportData.contracts.push({
|
||||
address: contract.address,
|
||||
kind: contract.kind,
|
||||
|
||||
@@ -569,8 +569,6 @@ export class Indexer implements IndexerInterface {
|
||||
}
|
||||
|
||||
async watchContract (address: string, kind: string, checkpoint: boolean, startingBlock: number): Promise<void> {
|
||||
await this.updateIPLDStatusMap(address, {});
|
||||
|
||||
return this._baseIndexer.watchContract(address, kind, checkpoint, startingBlock);
|
||||
}
|
||||
|
||||
|
||||
@@ -28,6 +28,9 @@
|
||||
# Use -1 for skipping check on block range.
|
||||
maxEventsBlockRange = 1000
|
||||
|
||||
# Interval in number of blocks at which to clear entities cache.
|
||||
clearEntitiesCacheInterval = 1000
|
||||
|
||||
[metrics]
|
||||
host = "127.0.0.1"
|
||||
port = 9000
|
||||
|
||||
@@ -101,6 +101,10 @@ const main = async (): Promise<void> => {
|
||||
|
||||
// Export contracts and checkpoints.
|
||||
for (const contract of contracts) {
|
||||
if (contract.startingBlock > block.blockNumber) {
|
||||
continue;
|
||||
}
|
||||
|
||||
exportData.contracts.push({
|
||||
address: contract.address,
|
||||
kind: contract.kind,
|
||||
|
||||
@@ -506,8 +506,6 @@ export class Indexer implements IndexerInterface {
|
||||
}
|
||||
|
||||
async watchContract (address: string, kind: string, checkpoint: boolean, startingBlock: number): Promise<void> {
|
||||
await this.updateIPLDStatusMap(address, {});
|
||||
|
||||
return this._baseIndexer.watchContract(address, kind, checkpoint, startingBlock);
|
||||
}
|
||||
|
||||
|
||||
@@ -92,6 +92,10 @@ const main = async (): Promise<void> => {
|
||||
|
||||
// Export contracts and checkpoints.
|
||||
for (const contract of contracts) {
|
||||
if (contract.startingBlock > block.blockNumber) {
|
||||
continue;
|
||||
}
|
||||
|
||||
exportData.contracts.push({
|
||||
address: contract.address,
|
||||
kind: contract.kind,
|
||||
|
||||
@@ -873,8 +873,6 @@ export class Indexer implements IndexerInterface {
|
||||
}
|
||||
|
||||
async watchContract (address: string, kind: string, checkpoint: boolean, startingBlock: number): Promise<void> {
|
||||
await this.updateIPLDStatusMap(address, {});
|
||||
|
||||
return this._baseIndexer.watchContract(address, kind, checkpoint, startingBlock);
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,208 @@
|
||||
# Demo for IPLD statediff and checkpointing
|
||||
|
||||
* In the root of `watcher-ts`, run:
|
||||
|
||||
```bash
|
||||
yarn && yarn build
|
||||
```
|
||||
|
||||
* In console, run the IPFS daemon:
|
||||
|
||||
```bash
|
||||
# Verify ipfs version
|
||||
ipfs version
|
||||
# ipfs version 0.12.2
|
||||
|
||||
ipfs daemon
|
||||
```
|
||||
|
||||
* The following services should be running to work with watcher:
|
||||
|
||||
* [vulcanize/go-ethereum](https://github.com/vulcanize/go-ethereum) ([v1.10.17-statediff-3.2.0](https://github.com/vulcanize/go-ethereum/releases/tag/v1.10.17-statediff-3.2.0)) on port 8545.
|
||||
* [vulcanize/ipld-eth-server](https://github.com/vulcanize/ipld-eth-server) ([v3.0.0](https://github.com/vulcanize/ipld-eth-server/releases/tag/v3.0.0)) with native GQL API enabled on port 8082 and RPC API enabled on port 8081.
|
||||
|
||||
* Deploy `Example` contract:
|
||||
|
||||
```bash
|
||||
cd packages/graph-node
|
||||
|
||||
yarn example:deploy
|
||||
```
|
||||
|
||||
* Set the returned address to the variable `$EXAMPLE_ADDRESS`:
|
||||
|
||||
```bash
|
||||
EXAMPLE_ADDRESS=
|
||||
```
|
||||
|
||||
* In `packages/graph-node`, run:
|
||||
|
||||
```bash
|
||||
cp .env.example .env
|
||||
```
|
||||
|
||||
* In `.env` file, set `EXAMPLE_CONTRACT_ADDRESS` to the `EXAMPLE_ADDRESS`.
|
||||
|
||||
* In [packages/graph-node/test/subgraph/example1/subgraph.yaml](./packages/graph-node/test/subgraph/example1/subgraph.yaml), set the source address for `Example1` datasource to the `EXAMPLE_ADDRESS`.
|
||||
|
||||
```bash
|
||||
yarn build:example
|
||||
```
|
||||
|
||||
* In `packages/codegen`, create a `config.yaml` file with the following contents:
|
||||
|
||||
```yaml
|
||||
contracts:
|
||||
- name: Example
|
||||
path: ../graph-node/test/contracts/Example.sol
|
||||
kind: Example1
|
||||
|
||||
outputFolder: ../demo-example-watcher
|
||||
mode: all
|
||||
kind: active
|
||||
port: 3008
|
||||
flatten: true
|
||||
subgraphPath: ../graph-node/test/subgraph/example1/build
|
||||
```
|
||||
|
||||
Reference: [packages/codegen/README.md](./packages/codegen/README.md#run)
|
||||
|
||||
* Generate watcher:
|
||||
|
||||
```bash
|
||||
cd packages/codegen
|
||||
|
||||
yarn codegen --config-file ./config.yaml
|
||||
```
|
||||
|
||||
* In `packages/demo-example-watcher`, run:
|
||||
|
||||
```bash
|
||||
yarn
|
||||
```
|
||||
|
||||
* Create dbs:
|
||||
|
||||
```bash
|
||||
sudo su - postgres
|
||||
# Delete databases if they already exist.
|
||||
dropdb demo-example-watcher
|
||||
dropdb demo-example-watcher-job-queue
|
||||
|
||||
# Create databases
|
||||
createdb demo-example-watcher
|
||||
createdb demo-example-watcher-job-queue
|
||||
```
|
||||
|
||||
Enable the `pgcrypto` extension.
|
||||
```
|
||||
psql -U postgres -h localhost demo-example-watcher-job-queue
|
||||
|
||||
demo-example-watcher-job-queue=# CREATE EXTENSION pgcrypto;
|
||||
demo-example-watcher-job-queue=# exit
|
||||
```
|
||||
|
||||
* In a new terminal, in `packages/demo-example-watcher`, run:
|
||||
|
||||
```bash
|
||||
yarn server
|
||||
```
|
||||
|
||||
```bash
|
||||
yarn job-runner
|
||||
```
|
||||
|
||||
* Run the following GQL subscription at the [graphql endpoint](http://127.0.0.1:3008/graphql):
|
||||
|
||||
```graphql
|
||||
subscription {
|
||||
onEvent {
|
||||
event {
|
||||
__typename
|
||||
... on TestEvent {
|
||||
param1
|
||||
param2
|
||||
param3
|
||||
},
|
||||
},
|
||||
block {
|
||||
number
|
||||
hash
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
* Trigger the `Test` event by calling example contract method:
|
||||
|
||||
```bash
|
||||
cd packages/graph-node
|
||||
|
||||
yarn example:test --address $EXAMPLE_ADDRESS
|
||||
```
|
||||
|
||||
A `Test` event shall be visible in the subscription at endpoint.
|
||||
|
||||
* Run the `getState` query at the endpoint to get the latest `IPLDBlock` for `EXAMPLE_ADDRESS`:
|
||||
|
||||
```graphql
|
||||
query {
|
||||
getState (
|
||||
blockHash: "EVENT_BLOCK_HASH"
|
||||
contractAddress: "EXAMPLE_ADDRESS"
|
||||
kind: "diff_staged"
|
||||
) {
|
||||
cid
|
||||
block {
|
||||
cid
|
||||
hash
|
||||
number
|
||||
timestamp
|
||||
parentHash
|
||||
}
|
||||
contractAddress
|
||||
data
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
* Run the query for entity at the endpoint:
|
||||
|
||||
```graphql
|
||||
query {
|
||||
author (
|
||||
block: {
|
||||
hash: "EVENT_BLOCK_HASH"
|
||||
}
|
||||
id: "0xdc7d7a8920c8eecc098da5b7522a5f31509b5bfc"
|
||||
) {
|
||||
__typename
|
||||
name
|
||||
paramInt
|
||||
paramBigInt
|
||||
paramBytes
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
* `diff` IPLDBlocks get created corresponding to the `diff_staged` blocks when their respective `eth_block`s reach the pruned region.
|
||||
|
||||
* In `packages/demo-example-watcher`:
|
||||
|
||||
* After the `diff` block has been created, create a `checkpoint`:
|
||||
|
||||
```bash
|
||||
cd packages/demo-example-watcher
|
||||
|
||||
yarn checkpoint --address $EXAMPLE_ADDRESS
|
||||
```
|
||||
|
||||
* A `checkpoint` IPLDBlock should be created at the latest canonical block hash.
|
||||
|
||||
* Run the `getState` query again at the endpoint with the output `blockHash` and kind `checkpoint`.
|
||||
|
||||
* All the `IPLDBlock` entries can be seen in `pg-admin` in table `ipld_block`.
|
||||
|
||||
* All the `diff` and `checkpoint` IPLDBlocks should be pushed to `IPFS`.
|
||||
|
||||
* Open IPFS WebUI http://127.0.0.1:5001/webui and search for `IPLDBlock`s using their `CID`s.
|
||||
@@ -16,6 +16,8 @@ import { checkGQLEntityInIPLDState, compareQuery, Config, getIPLDsByBlock, check
|
||||
import { Database } from '../../database';
|
||||
import { getSubgraphConfig } from '../../utils';
|
||||
|
||||
const DEFAULT_ENTITIES_LIMIT = 100;
|
||||
|
||||
const log = debug('vulcanize:compare-blocks');
|
||||
|
||||
export const main = async (): Promise<void> => {
|
||||
@@ -70,10 +72,33 @@ export const main = async (): Promise<void> => {
|
||||
type: 'boolean',
|
||||
describe: 'Compare time taken between GQL queries',
|
||||
default: false
|
||||
},
|
||||
queryEntitiesLimit: {
|
||||
type: 'number',
|
||||
default: DEFAULT_ENTITIES_LIMIT,
|
||||
describe: 'Limit for entities returned in query'
|
||||
},
|
||||
paginate: {
|
||||
type: 'boolean',
|
||||
describe: 'Paginate in multiple entities query and compare',
|
||||
default: false
|
||||
}
|
||||
}).argv;
|
||||
|
||||
const { startBlock, endBlock, batchSize, interval, rawJson, queryDir, fetchIds, configFile, timeDiff } = argv;
|
||||
const {
|
||||
startBlock,
|
||||
endBlock,
|
||||
batchSize,
|
||||
interval,
|
||||
rawJson,
|
||||
queryDir,
|
||||
fetchIds,
|
||||
configFile,
|
||||
timeDiff,
|
||||
queryEntitiesLimit,
|
||||
paginate
|
||||
} = argv;
|
||||
|
||||
const config: Config = await getConfig(configFile);
|
||||
const snakeNamingStrategy = new SnakeNamingStrategy();
|
||||
const clients = await getClients(config, timeDiff, queryDir);
|
||||
@@ -195,23 +220,41 @@ export const main = async (): Promise<void> => {
|
||||
} else {
|
||||
if (updatedEntities.has(entityName)) {
|
||||
let result;
|
||||
let skip = 0;
|
||||
|
||||
({ diff: resultDiff, result1: result } = await compareQuery(
|
||||
clients,
|
||||
queryName,
|
||||
{ block },
|
||||
rawJson,
|
||||
timeDiff
|
||||
));
|
||||
do {
|
||||
({ diff: resultDiff, result1: result } = await compareQuery(
|
||||
clients,
|
||||
queryName,
|
||||
{
|
||||
block,
|
||||
skip,
|
||||
first: queryEntitiesLimit
|
||||
},
|
||||
rawJson,
|
||||
timeDiff
|
||||
));
|
||||
|
||||
if (config.watcher.verifyState) {
|
||||
const ipldDiff = await checkGQLEntitiesInIPLDState(ipldStateByBlock, entityName, result[queryName], rawJson, config.watcher.skipFields);
|
||||
if (config.watcher.verifyState) {
|
||||
const ipldDiff = await checkGQLEntitiesInIPLDState(ipldStateByBlock, entityName, result[queryName], rawJson, config.watcher.skipFields);
|
||||
|
||||
if (ipldDiff) {
|
||||
log('Results mismatch for IPLD state:', ipldDiff);
|
||||
diffFound = true;
|
||||
if (ipldDiff) {
|
||||
log('Results mismatch for IPLD state:', ipldDiff);
|
||||
diffFound = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
skip += queryEntitiesLimit;
|
||||
} while (
|
||||
// Check if needed to query more entities.
|
||||
result[queryName].length === queryEntitiesLimit &&
|
||||
// Check if diff found.
|
||||
!diffFound &&
|
||||
!resultDiff &&
|
||||
// Check paginate flag
|
||||
// eslint-disable-next-line no-unmodified-loop-condition
|
||||
paginate
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -12,6 +12,7 @@ import {
|
||||
QueryRunner,
|
||||
Repository
|
||||
} from 'typeorm';
|
||||
import { ColumnMetadata } from 'typeorm/metadata/ColumnMetadata';
|
||||
import { SelectionNode } from 'graphql';
|
||||
import _ from 'lodash';
|
||||
import debug from 'debug';
|
||||
@@ -647,9 +648,9 @@ export class Database {
|
||||
return this.getStateEntityValues(block, stateEntity, entityFields, relations);
|
||||
}
|
||||
|
||||
getStateEntityValues (block: BlockProgressInterface, stateEntity: any, entityFields: any, relations: { [key: string]: any } = {}): { [key: string]: any } {
|
||||
const entityValues = entityFields.map((field: any) => {
|
||||
const { propertyName } = field;
|
||||
getStateEntityValues (block: BlockProgressInterface, stateEntity: any, entityFields: ColumnMetadata[], relations: { [key: string]: any } = {}): { [key: string]: any } {
|
||||
const entityValues = entityFields.map((field) => {
|
||||
const { propertyName, transformer } = field;
|
||||
|
||||
// Get blockHash property for db entry from block instance.
|
||||
if (propertyName === 'blockHash') {
|
||||
@@ -663,10 +664,10 @@ export class Database {
|
||||
|
||||
// Get blockNumber as _blockNumber and blockHash as _blockHash from the entityInstance (wasm).
|
||||
if (['_blockNumber', '_blockHash'].includes(propertyName)) {
|
||||
return fromStateEntityValues(stateEntity, propertyName.slice(1), relations);
|
||||
return fromStateEntityValues(stateEntity, propertyName.slice(1), relations, transformer);
|
||||
}
|
||||
|
||||
return fromStateEntityValues(stateEntity, propertyName, relations);
|
||||
return fromStateEntityValues(stateEntity, propertyName, relations, transformer);
|
||||
}, {});
|
||||
|
||||
return entityFields.reduce((acc: { [key: string]: any }, field: any, index: number) => {
|
||||
|
||||
@@ -3,6 +3,7 @@ import path from 'path';
|
||||
import fs from 'fs-extra';
|
||||
import debug from 'debug';
|
||||
import yaml from 'js-yaml';
|
||||
import { ValueTransformer } from 'typeorm';
|
||||
import { ColumnMetadata } from 'typeorm/metadata/ColumnMetadata';
|
||||
import assert from 'assert';
|
||||
|
||||
@@ -840,7 +841,12 @@ export const prepareEntityState = (updatedEntity: any, entityName: string, relat
|
||||
return diffData;
|
||||
};
|
||||
|
||||
export const fromStateEntityValues = (stateEntity: any, propertyName: string, relations: { [key: string]: any } = {}): any => {
|
||||
export const fromStateEntityValues = (
|
||||
stateEntity: any,
|
||||
propertyName: string,
|
||||
relations: { [key: string]: any } = {},
|
||||
transformer?: ValueTransformer | ValueTransformer[]
|
||||
): any => {
|
||||
// Parse DB data value from state entity data.
|
||||
if (relations) {
|
||||
const relation = relations[propertyName];
|
||||
@@ -854,5 +860,16 @@ export const fromStateEntityValues = (stateEntity: any, propertyName: string, re
|
||||
}
|
||||
}
|
||||
|
||||
if (transformer) {
|
||||
if (Array.isArray(transformer)) {
|
||||
// Apply transformer in reverse order similar to when reading from DB.
|
||||
return transformer.reduceRight((acc, elTransformer) => {
|
||||
return elTransformer.from(acc);
|
||||
}, stateEntity[propertyName]);
|
||||
}
|
||||
|
||||
return transformer.from(stateEntity[propertyName]);
|
||||
}
|
||||
|
||||
return stateEntity[propertyName];
|
||||
};
|
||||
|
||||
@@ -12,7 +12,7 @@ import { SelectionNode } from 'graphql';
|
||||
|
||||
import { ResultObject } from '@vulcanize/assemblyscript/lib/loader';
|
||||
import { EthClient } from '@cerc-io/ipld-eth-client';
|
||||
import { getFullBlock, BlockHeight, ServerConfig, getFullTransaction, QueryOptions, IPLDBlockInterface, IndexerInterface, BlockProgressInterface } from '@cerc-io/util';
|
||||
import { getFullBlock, BlockHeight, ServerConfig, getFullTransaction, QueryOptions, IPLDBlockInterface, IndexerInterface, BlockProgressInterface, cachePrunedEntitiesCount } from '@cerc-io/util';
|
||||
|
||||
import { createBlock, createEvent, getSubgraphConfig, resolveEntityFieldConflicts, Transaction } from './utils';
|
||||
import { Context, GraphData, instantiate } from './loader';
|
||||
@@ -186,8 +186,12 @@ export class GraphWatcher {
|
||||
|
||||
// Create ethereum event to be passed to the wasm event handler.
|
||||
const ethereumEvent = await createEvent(instanceExports, contract, data);
|
||||
|
||||
await this._handleMemoryError(instanceExports[eventHandler.handler](ethereumEvent), dataSource.name);
|
||||
try {
|
||||
await this._handleMemoryError(instanceExports[eventHandler.handler](ethereumEvent), dataSource.name);
|
||||
} catch (error) {
|
||||
this._clearCachedEntities();
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async handleBlock (blockHash: string) {
|
||||
@@ -249,7 +253,12 @@ export class GraphWatcher {
|
||||
await instanceExports[blockHandler.handler](ethereumBlock);
|
||||
});
|
||||
|
||||
await this._handleMemoryError(Promise.all(blockHandlerPromises), dataSource.name);
|
||||
try {
|
||||
await this._handleMemoryError(Promise.all(blockHandlerPromises), dataSource.name);
|
||||
} catch (error) {
|
||||
this._clearCachedEntities();
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -375,8 +384,18 @@ export class GraphWatcher {
|
||||
entities: new Map()
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
log(`Size of cachedEntities.frothyBlocks map: ${this._database.cachedEntities.frothyBlocks.size}`);
|
||||
log(`Size of cachedEntities.frothyBlocks map: ${this._database.cachedEntities.frothyBlocks.size}`);
|
||||
this._measureCachedPrunedEntities();
|
||||
|
||||
assert(this._indexer);
|
||||
// Check if it is time to clear entities cache.
|
||||
if (blockProgress.blockNumber % this._indexer.serverConfig.clearEntitiesCacheInterval === 0) {
|
||||
log(`Clearing cachedEntities.latestPrunedEntities at block ${blockProgress.blockNumber}`);
|
||||
// Clearing only pruned region as frothy region cache gets updated in pruning queue.
|
||||
this._database.cachedEntities.latestPrunedEntities.clear();
|
||||
log(`Cleared cachedEntities.latestPrunedEntities. Map size: ${this._database.cachedEntities.latestPrunedEntities.size}`);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -407,6 +426,19 @@ export class GraphWatcher {
|
||||
prunedBlockHashes.forEach(blockHash => this._database.cachedEntities.frothyBlocks.delete(blockHash));
|
||||
}
|
||||
|
||||
_clearCachedEntities () {
|
||||
this._database.cachedEntities.frothyBlocks.clear();
|
||||
this._database.cachedEntities.latestPrunedEntities.clear();
|
||||
}
|
||||
|
||||
_measureCachedPrunedEntities () {
|
||||
const totalEntities = Array.from(this._database.cachedEntities.latestPrunedEntities.values())
|
||||
.reduce((acc, idEntitiesMap) => acc + idEntitiesMap.size, 0);
|
||||
|
||||
log(`Total entities in cachedEntities.latestPrunedEntities map: ${totalEntities}`);
|
||||
cachePrunedEntitiesCount.set(totalEntities);
|
||||
}
|
||||
|
||||
/**
|
||||
* Method to reinstantiate WASM instance for specified dataSource.
|
||||
* @param dataSourceName
|
||||
|
||||
@@ -211,6 +211,7 @@ class ServerConfig implements ServerConfigInterface {
|
||||
wasmRestartBlocksInterval: number;
|
||||
filterLogs: boolean;
|
||||
maxEventsBlockRange: number;
|
||||
clearEntitiesCacheInterval: number;
|
||||
|
||||
constructor () {
|
||||
this.host = '';
|
||||
@@ -225,5 +226,6 @@ class ServerConfig implements ServerConfigInterface {
|
||||
this.wasmRestartBlocksInterval = 0;
|
||||
this.filterLogs = false;
|
||||
this.maxEventsBlockRange = 0;
|
||||
this.clearEntitiesCacheInterval = 0;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -22,6 +22,9 @@
|
||||
# Use -1 for skipping check on block range.
|
||||
maxEventsBlockRange = 1000
|
||||
|
||||
# Interval in number of blocks at which to clear entities cache.
|
||||
clearEntitiesCacheInterval = 1000
|
||||
|
||||
[metrics]
|
||||
host = "127.0.0.1"
|
||||
port = 9000
|
||||
|
||||
@@ -101,6 +101,10 @@ const main = async (): Promise<void> => {
|
||||
|
||||
// Export contracts and checkpoints.
|
||||
for (const contract of contracts) {
|
||||
if (contract.startingBlock > block.blockNumber) {
|
||||
continue;
|
||||
}
|
||||
|
||||
exportData.contracts.push({
|
||||
address: contract.address,
|
||||
kind: contract.kind,
|
||||
|
||||
@@ -502,8 +502,6 @@ export class Indexer implements IndexerInterface {
|
||||
}
|
||||
|
||||
async watchContract (address: string, kind: string, checkpoint: boolean, startingBlock: number): Promise<void> {
|
||||
await this.updateIPLDStatusMap(address, {});
|
||||
|
||||
return this._baseIndexer.watchContract(address, kind, checkpoint, startingBlock);
|
||||
}
|
||||
|
||||
|
||||
@@ -1,5 +0,0 @@
|
||||
# Don't lint node_modules.
|
||||
node_modules
|
||||
|
||||
# Don't lint build output.
|
||||
dist
|
||||
@@ -1,27 +0,0 @@
|
||||
{
|
||||
"env": {
|
||||
"browser": true,
|
||||
"es2021": true
|
||||
},
|
||||
"extends": [
|
||||
"semistandard",
|
||||
"plugin:@typescript-eslint/recommended"
|
||||
],
|
||||
"parser": "@typescript-eslint/parser",
|
||||
"parserOptions": {
|
||||
"ecmaVersion": 12,
|
||||
"sourceType": "module"
|
||||
},
|
||||
"plugins": [
|
||||
"@typescript-eslint"
|
||||
],
|
||||
"rules": {
|
||||
"@typescript-eslint/no-explicit-any": "off",
|
||||
"@typescript-eslint/explicit-module-boundary-types": [
|
||||
"warn",
|
||||
{
|
||||
"allowArgumentsExplicitlyTypedAsAny": true
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -1,10 +0,0 @@
|
||||
.idea/
|
||||
.vscode/
|
||||
node_modules/
|
||||
build/
|
||||
tmp/
|
||||
temp/
|
||||
|
||||
#Hardhat files
|
||||
cache
|
||||
artifacts
|
||||
@@ -1,4 +0,0 @@
|
||||
timeout: '10000'
|
||||
bail: true
|
||||
exit: true # TODO: Find out why the program doesn't exit on its own.
|
||||
require: 'ts-node/register'
|
||||
@@ -1,89 +0,0 @@
|
||||
# Lighthouse Watcher
|
||||
|
||||
## Setup
|
||||
|
||||
Deploy a Lighthouse contract:
|
||||
|
||||
```bash
|
||||
yarn lighthouse:deploy
|
||||
```
|
||||
|
||||
Use the Lighthouse contract address and set `environments/local.toml` to watch the contract.
|
||||
|
||||
```toml
|
||||
[watch]
|
||||
lighthouse = "0xLighthouseContractAddress"
|
||||
```
|
||||
|
||||
## Run
|
||||
|
||||
Build files:
|
||||
|
||||
```bash
|
||||
$ yarn build
|
||||
```
|
||||
|
||||
Run the server:
|
||||
|
||||
```bash
|
||||
$ yarn server
|
||||
|
||||
# For development.
|
||||
$ yarn server:dev
|
||||
|
||||
# For specifying config file.
|
||||
$ yarn server -f environments/local.toml
|
||||
```
|
||||
|
||||
## Test
|
||||
|
||||
To test the watcher locally:
|
||||
|
||||
Open graphql playground at http://127.0.0.1:3005/graphql and set a subscription query
|
||||
|
||||
```graphql
|
||||
subscription {
|
||||
onEvent {
|
||||
block {
|
||||
hash
|
||||
number
|
||||
timestamp
|
||||
}
|
||||
tx {
|
||||
hash
|
||||
}
|
||||
contract
|
||||
eventIndex
|
||||
event {
|
||||
__typename
|
||||
... on StorageRequestEvent {
|
||||
uploader
|
||||
cid
|
||||
config
|
||||
fileCost
|
||||
}
|
||||
}
|
||||
proof {
|
||||
data
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
To trigger StorageRequest event locally, run:
|
||||
|
||||
```bash
|
||||
yarn lighthouse:store --lighthouse 0xLighthouseContractAddress --cid testCid --store-config testConfig --file-cost 10
|
||||
```
|
||||
|
||||
### Smoke test
|
||||
|
||||
To run a smoke test:
|
||||
|
||||
* Start the server.
|
||||
|
||||
* Run:
|
||||
|
||||
```bash
|
||||
$ yarn smoke-test
|
||||
```
|
||||
@@ -1,15 +0,0 @@
|
||||
[server]
|
||||
host = "127.0.0.1"
|
||||
port = 3005
|
||||
|
||||
[watch]
|
||||
lighthouse = "0xbDA876401576281a1912a20de135F60de6D7d711"
|
||||
|
||||
[upstream]
|
||||
[upstream.ethServer]
|
||||
gqlApiEndpoint = "http://127.0.0.1:8082/graphql"
|
||||
|
||||
[upstream.cache]
|
||||
name = "requests"
|
||||
enabled = false
|
||||
deleteOnStart = false
|
||||
@@ -1,14 +0,0 @@
|
||||
import { HardhatUserConfig } from 'hardhat/config';
|
||||
|
||||
import './tasks/lighthouse-deploy';
|
||||
import './tasks/lighthouse-store';
|
||||
|
||||
const config: HardhatUserConfig = {
|
||||
defaultNetwork: 'localhost',
|
||||
solidity: '0.7.3',
|
||||
paths: {
|
||||
sources: './test/contracts'
|
||||
}
|
||||
};
|
||||
|
||||
export default config;
|
||||
@@ -1,54 +0,0 @@
|
||||
{
|
||||
"name": "@cerc-io/lighthouse-watcher",
|
||||
"version": "0.2.13",
|
||||
"main": "index.js",
|
||||
"license": "AGPL-3.0",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"lint": "eslint .",
|
||||
"test": "mocha -r ts-node/register src/**/*.test.ts",
|
||||
"build": "tsc",
|
||||
"server": "DEBUG=vulcanize:* node dist/server.js",
|
||||
"server:dev": "DEBUG=vulcanize:* nodemon --watch src src/server.ts",
|
||||
"smoke-test": "mocha src/smoke.test.ts",
|
||||
"lighthouse:deploy": "hardhat lighthouse-deploy",
|
||||
"lighthouse:store": "hardhat lighthouse-store"
|
||||
},
|
||||
"dependencies": {
|
||||
"@apollo/client": "^3.3.19",
|
||||
"@cerc-io/cache": "^0.2.13",
|
||||
"@cerc-io/ipld-eth-client": "^0.2.13",
|
||||
"@cerc-io/util": "^0.2.13",
|
||||
"apollo-server-express": "^2.25.0",
|
||||
"apollo-type-bigint": "^0.1.3",
|
||||
"debug": "^4.3.1",
|
||||
"ethers": "^5.4.4",
|
||||
"express": "^4.17.1",
|
||||
"graphql-request": "^3.4.0",
|
||||
"json-bigint": "^1.0.0",
|
||||
"lodash": "^4.17.21",
|
||||
"reflect-metadata": "^0.1.13",
|
||||
"yargs": "^17.0.1"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@nomiclabs/hardhat-ethers": "^2.0.2",
|
||||
"@types/chai": "^4.2.19",
|
||||
"@types/express": "^4.17.11",
|
||||
"@types/json-bigint": "^1.0.0",
|
||||
"@types/mocha": "^8.2.2",
|
||||
"@types/yargs": "^17.0.0",
|
||||
"@typescript-eslint/eslint-plugin": "^4.25.0",
|
||||
"@typescript-eslint/parser": "^4.25.0",
|
||||
"chai": "^4.3.4",
|
||||
"eslint": "^7.27.0",
|
||||
"eslint-config-semistandard": "^15.0.1",
|
||||
"eslint-config-standard": "^16.0.3",
|
||||
"eslint-plugin-import": "^2.23.3",
|
||||
"eslint-plugin-node": "^11.1.0",
|
||||
"eslint-plugin-promise": "^5.1.0",
|
||||
"eslint-plugin-standard": "^5.0.0",
|
||||
"hardhat": "^2.3.0",
|
||||
"mocha": "^8.4.0",
|
||||
"nodemon": "^2.0.7"
|
||||
}
|
||||
}
|
||||
@@ -1,107 +0,0 @@
|
||||
[
|
||||
{
|
||||
"anonymous": false,
|
||||
"inputs": [
|
||||
{
|
||||
"indexed": false,
|
||||
"internalType": "address",
|
||||
"name": "uploader",
|
||||
"type": "address"
|
||||
},
|
||||
{
|
||||
"indexed": false,
|
||||
"internalType": "string",
|
||||
"name": "cid",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"indexed": false,
|
||||
"internalType": "string",
|
||||
"name": "config",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"indexed": false,
|
||||
"internalType": "uint256",
|
||||
"name": "fileCost",
|
||||
"type": "uint256"
|
||||
}
|
||||
],
|
||||
"name": "StorageRequest",
|
||||
"type": "event"
|
||||
},
|
||||
{
|
||||
"stateMutability": "payable",
|
||||
"type": "fallback"
|
||||
},
|
||||
{
|
||||
"inputs": [
|
||||
{
|
||||
"internalType": "uint256",
|
||||
"name": "amount",
|
||||
"type": "uint256"
|
||||
},
|
||||
{
|
||||
"internalType": "address payable",
|
||||
"name": "recipient",
|
||||
"type": "address"
|
||||
}
|
||||
],
|
||||
"name": "getPaid",
|
||||
"outputs": [],
|
||||
"stateMutability": "nonpayable",
|
||||
"type": "function"
|
||||
},
|
||||
{
|
||||
"inputs": [
|
||||
{
|
||||
"internalType": "address",
|
||||
"name": "",
|
||||
"type": "address"
|
||||
},
|
||||
{
|
||||
"internalType": "string",
|
||||
"name": "",
|
||||
"type": "string"
|
||||
}
|
||||
],
|
||||
"name": "requests",
|
||||
"outputs": [
|
||||
{
|
||||
"internalType": "string",
|
||||
"name": "cid",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"internalType": "string",
|
||||
"name": "config",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"internalType": "uint256",
|
||||
"name": "fileCost",
|
||||
"type": "uint256"
|
||||
}
|
||||
],
|
||||
"stateMutability": "view",
|
||||
"type": "function"
|
||||
},
|
||||
{
|
||||
"inputs": [
|
||||
{
|
||||
"internalType": "string",
|
||||
"name": "cid",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"internalType": "string",
|
||||
"name": "config",
|
||||
"type": "string"
|
||||
}
|
||||
],
|
||||
"name": "store",
|
||||
"outputs": [],
|
||||
"stateMutability": "payable",
|
||||
"type": "function"
|
||||
}
|
||||
]
|
||||
@@ -1,28 +0,0 @@
|
||||
//
|
||||
// Copyright 2021 Vulcanize, Inc.
|
||||
//
|
||||
|
||||
import { gql } from '@apollo/client/core';
|
||||
import { GraphQLClient, GraphQLConfig } from '@cerc-io/ipld-eth-client';
|
||||
|
||||
import { subscribeEvents } from './queries';
|
||||
|
||||
export class Client {
|
||||
_config: GraphQLConfig;
|
||||
_client: GraphQLClient;
|
||||
|
||||
constructor (config: GraphQLConfig) {
|
||||
this._config = config;
|
||||
|
||||
this._client = new GraphQLClient(config);
|
||||
}
|
||||
|
||||
async watchEvents (onNext: (value: any) => void): Promise<ZenObservable.Subscription> {
|
||||
return this._client.subscribe(
|
||||
gql(subscribeEvents),
|
||||
({ data }) => {
|
||||
onNext(data.onEvent);
|
||||
}
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,72 +0,0 @@
|
||||
//
|
||||
// Copyright 2021 Vulcanize, Inc.
|
||||
//
|
||||
|
||||
import assert from 'assert';
|
||||
import debug from 'debug';
|
||||
import _ from 'lodash';
|
||||
import { PubSub } from 'apollo-server-express';
|
||||
|
||||
import { EthClient } from '@cerc-io/ipld-eth-client';
|
||||
|
||||
import { Indexer, ResultEvent, UNKNOWN_EVENT_NAME } from './indexer';
|
||||
const log = debug('vulcanize:events');
|
||||
|
||||
export const LighthouseEvent = 'lighthouse-event';
|
||||
|
||||
export class EventWatcher {
|
||||
_ethClient: EthClient
|
||||
_indexer: Indexer
|
||||
_subscription: ZenObservable.Subscription | undefined
|
||||
_pubsub: PubSub
|
||||
|
||||
constructor (ethClient: EthClient, indexer: Indexer, pubsub: PubSub) {
|
||||
this._ethClient = ethClient;
|
||||
this._indexer = indexer;
|
||||
this._pubsub = pubsub;
|
||||
}
|
||||
|
||||
getEventIterator (): AsyncIterator<any> {
|
||||
return this._pubsub.asyncIterator([LighthouseEvent]);
|
||||
}
|
||||
|
||||
async start (): Promise<void> {
|
||||
assert(!this._subscription, 'subscription already started');
|
||||
|
||||
await this.watchBlocksAtChainHead();
|
||||
}
|
||||
|
||||
async watchBlocksAtChainHead (): Promise<void> {
|
||||
log('Started watching upstream blocks...');
|
||||
|
||||
// TODO: Update to pull based watcher.
|
||||
// this._subscription = await this._ethClient.watchBlocks(async (value) => {
|
||||
// const { blockHash, blockNumber } = _.get(value, 'data.listen.relatedNode');
|
||||
// log('watchBlock', blockHash, blockNumber);
|
||||
|
||||
// const events = await this._indexer.getOrFetchBlockEvents(blockHash);
|
||||
|
||||
// for (let ei = 0; ei < events.length; ei++) {
|
||||
// await this.publishLighthouseEventToSubscribers(events[ei]);
|
||||
// }
|
||||
// });
|
||||
}
|
||||
|
||||
async publishLighthouseEventToSubscribers (resultEvent: ResultEvent): Promise<void> {
|
||||
if (resultEvent.event.__typename !== UNKNOWN_EVENT_NAME) {
|
||||
log(`pushing event to GQL subscribers: ${resultEvent.event.__typename}`);
|
||||
|
||||
// Publishing the event here will result in pushing the payload to GQL subscribers for `onEvent`.
|
||||
await this._pubsub.publish(LighthouseEvent, {
|
||||
onEvent: resultEvent
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async stop (): Promise<void> {
|
||||
if (this._subscription) {
|
||||
log('Stopped watching upstream blocks');
|
||||
this._subscription.unsubscribe();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,181 +0,0 @@
|
||||
//
|
||||
// Copyright 2021 Vulcanize, Inc.
|
||||
//
|
||||
|
||||
import debug from 'debug';
|
||||
import JSONbig from 'json-bigint';
|
||||
import { ethers } from 'ethers';
|
||||
import assert from 'assert';
|
||||
|
||||
import { EthClient } from '@cerc-io/ipld-eth-client';
|
||||
import { Config as BaseConfig } from '@cerc-io/util';
|
||||
|
||||
import lighthouseABI from './abi/Lighthouse.json';
|
||||
|
||||
export const UNKNOWN_EVENT_NAME = '__unknown__';
|
||||
|
||||
const log = debug('vulcanize:indexer');
|
||||
const JSONbigNative = JSONbig({ useNativeBigInt: true });
|
||||
|
||||
export type ResultEvent = {
|
||||
block: any;
|
||||
tx: any;
|
||||
|
||||
contract: string;
|
||||
|
||||
eventIndex: number;
|
||||
event: any;
|
||||
|
||||
proof: any;
|
||||
};
|
||||
|
||||
export interface Config extends BaseConfig {
|
||||
watch?: {
|
||||
lighthouse: string
|
||||
}
|
||||
}
|
||||
|
||||
export class Indexer {
|
||||
_config: Config
|
||||
_ethClient: EthClient
|
||||
|
||||
_lighthouseContract: ethers.utils.Interface
|
||||
|
||||
constructor (config: Config, ethClient: EthClient) {
|
||||
assert(config.watch);
|
||||
this._config = config;
|
||||
this._ethClient = ethClient;
|
||||
|
||||
this._lighthouseContract = new ethers.utils.Interface(lighthouseABI);
|
||||
}
|
||||
|
||||
// Note: Some event names might be unknown at this point, as earlier events might not yet be processed.
|
||||
async getOrFetchBlockEvents (blockHash: string): Promise<Array<ResultEvent>> {
|
||||
// Fetch and save events first and make a note in the event sync progress table.
|
||||
log(`getBlockEvents: fetching from upstream server ${blockHash}`);
|
||||
const events = await this.fetchEvents(blockHash);
|
||||
|
||||
log(`getBlockEvents: ${blockHash} num events: ${events.length}`);
|
||||
|
||||
return events;
|
||||
}
|
||||
|
||||
parseEventNameAndArgs (logObj: any): any {
|
||||
let eventName = UNKNOWN_EVENT_NAME;
|
||||
let eventInfo = {};
|
||||
|
||||
const { topics, data } = logObj;
|
||||
|
||||
const logDescription = this._lighthouseContract.parseLog({ data, topics });
|
||||
switch (logDescription.name) {
|
||||
case 'StorageRequest': {
|
||||
eventName = logDescription.name;
|
||||
const { uploader, cid, config, fileCost } = logDescription.args;
|
||||
eventInfo = { uploader, cid, config, fileCost };
|
||||
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return { eventName, eventInfo };
|
||||
}
|
||||
|
||||
async fetchEvents (blockHash: string): Promise<Array<ResultEvent>> {
|
||||
assert(this._config.watch);
|
||||
const contract = this._config.watch.lighthouse;
|
||||
|
||||
const [{ logs }, { block }] = await Promise.all([
|
||||
this._ethClient.getLogs({ blockHash, addresses: [contract] }),
|
||||
this._ethClient.getBlockByHash(blockHash)
|
||||
]);
|
||||
|
||||
const {
|
||||
allEthHeaderCids: {
|
||||
nodes: [
|
||||
{
|
||||
ethTransactionCidsByHeaderId: {
|
||||
nodes: transactions
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
} = await this._ethClient.getBlockWithTransactions({ blockHash });
|
||||
|
||||
const transactionMap = transactions.reduce((acc: {[key: string]: any}, transaction: {[key: string]: any}) => {
|
||||
acc[transaction.txHash] = transaction;
|
||||
return acc;
|
||||
}, {});
|
||||
|
||||
const events: Array<ResultEvent> = [];
|
||||
|
||||
for (let li = 0; li < logs.length; li++) {
|
||||
const logObj = logs[li];
|
||||
const {
|
||||
index: logIndex,
|
||||
cid,
|
||||
ipldBlock,
|
||||
account: {
|
||||
address
|
||||
},
|
||||
transaction: {
|
||||
hash: txHash
|
||||
},
|
||||
receiptCID,
|
||||
status
|
||||
} = logObj;
|
||||
|
||||
if (status) {
|
||||
const tx = transactionMap[txHash];
|
||||
assert(ethers.utils.getAddress(address) === contract);
|
||||
|
||||
const eventDetails = this.parseEventNameAndArgs(logObj);
|
||||
const eventName = eventDetails.eventName;
|
||||
const eventInfo = eventDetails.eventInfo;
|
||||
|
||||
const {
|
||||
hash,
|
||||
number,
|
||||
timestamp,
|
||||
parent: {
|
||||
hash: parentHash
|
||||
}
|
||||
} = block;
|
||||
|
||||
events.push({
|
||||
block: {
|
||||
hash,
|
||||
number,
|
||||
timestamp,
|
||||
parentHash
|
||||
},
|
||||
eventIndex: logIndex,
|
||||
tx: {
|
||||
hash: txHash,
|
||||
index: tx.index,
|
||||
from: tx.src,
|
||||
to: tx.dst
|
||||
},
|
||||
contract,
|
||||
event: {
|
||||
__typename: `${eventName}Event`,
|
||||
...eventInfo
|
||||
},
|
||||
proof: {
|
||||
data: JSONbigNative.stringify({
|
||||
blockHash,
|
||||
receiptCID,
|
||||
log: {
|
||||
cid,
|
||||
ipldBlock
|
||||
}
|
||||
})
|
||||
}
|
||||
});
|
||||
} else {
|
||||
log(`Skipping event for receipt ${receiptCID} due to failed transaction.`);
|
||||
}
|
||||
}
|
||||
|
||||
return events;
|
||||
}
|
||||
}
|
||||
@@ -1,46 +0,0 @@
|
||||
//
|
||||
// Copyright 2021 Vulcanize, Inc.
|
||||
//
|
||||
|
||||
import { gql } from 'graphql-request';
|
||||
|
||||
const resultEvent = `
|
||||
{
|
||||
block {
|
||||
number
|
||||
hash
|
||||
timestamp
|
||||
parentHash
|
||||
}
|
||||
tx {
|
||||
hash
|
||||
from
|
||||
to
|
||||
index
|
||||
}
|
||||
contract
|
||||
eventIndex
|
||||
|
||||
event {
|
||||
__typename
|
||||
|
||||
... on StorageRequestEvent {
|
||||
uploader
|
||||
cid
|
||||
config
|
||||
fileCost
|
||||
}
|
||||
}
|
||||
|
||||
proof {
|
||||
data
|
||||
}
|
||||
}
|
||||
`;
|
||||
|
||||
export const subscribeEvents = gql`
|
||||
subscription SubscriptionEvents {
|
||||
onEvent
|
||||
${resultEvent}
|
||||
}
|
||||
`;
|
||||
@@ -1,28 +0,0 @@
|
||||
//
|
||||
// Copyright 2021 Vulcanize, Inc.
|
||||
//
|
||||
|
||||
import BigInt from 'apollo-type-bigint';
|
||||
import assert from 'assert';
|
||||
|
||||
import { EventWatcher } from './events';
|
||||
|
||||
export const createResolvers = async (eventWatcher: EventWatcher): Promise<any> => {
|
||||
return {
|
||||
BigInt: new BigInt('bigInt'),
|
||||
|
||||
Event: {
|
||||
__resolveType: (obj: any) => {
|
||||
assert(obj.__typename);
|
||||
|
||||
return obj.__typename;
|
||||
}
|
||||
},
|
||||
|
||||
Subscription: {
|
||||
onEvent: {
|
||||
subscribe: () => eventWatcher.getEventIterator()
|
||||
}
|
||||
}
|
||||
};
|
||||
};
|
||||
@@ -1,79 +0,0 @@
|
||||
//
|
||||
// Copyright 2021 Vulcanize, Inc.
|
||||
//
|
||||
|
||||
import { gql } from '@apollo/client/core';
|
||||
|
||||
export default gql`
|
||||
# Types
|
||||
|
||||
# Support uint256 values.
|
||||
scalar BigInt
|
||||
|
||||
# Ethereum types
|
||||
|
||||
type Block {
|
||||
hash: String!
|
||||
number: Int!
|
||||
timestamp: Int!
|
||||
parentHash: String!
|
||||
}
|
||||
|
||||
type Transaction {
|
||||
hash: String!
|
||||
index: Int!
|
||||
from: String!
|
||||
to: String!
|
||||
}
|
||||
|
||||
# event StorageRequest(address uploader, string cid, string config, uint fileCost);
|
||||
type StorageRequestEvent {
|
||||
uploader: String!
|
||||
cid: String!
|
||||
config: String!
|
||||
fileCost: BigInt!
|
||||
}
|
||||
|
||||
# All events emitted by the watcher.
|
||||
union Event = StorageRequestEvent
|
||||
|
||||
# Proof for returned data. Serialized blob for now.
|
||||
# Will be converted into a well defined structure later.
|
||||
type Proof {
|
||||
data: String!
|
||||
}
|
||||
|
||||
# Result event, include additional context over and above the event data.
|
||||
type ResultEvent {
|
||||
# Block and tx data for the event.
|
||||
block: Block!
|
||||
tx: Transaction!
|
||||
|
||||
# Contract that generated the event.
|
||||
contract: String!
|
||||
|
||||
# Index of the event in the block.
|
||||
eventIndex: Int!
|
||||
|
||||
event: Event!
|
||||
|
||||
# Proof from receipts trie.
|
||||
proof: Proof
|
||||
}
|
||||
|
||||
#
|
||||
# Queries
|
||||
#
|
||||
type Query {
|
||||
# https://github.com/ardatan/graphql-tools/issues/764#issuecomment-419556241
|
||||
dummy: String
|
||||
}
|
||||
|
||||
#
|
||||
# Subscriptions
|
||||
#
|
||||
type Subscription {
|
||||
# Watch for Lighthouse events (at head of chain).
|
||||
onEvent: ResultEvent!
|
||||
}
|
||||
`;
|
||||
@@ -1,88 +0,0 @@
|
||||
//
|
||||
// Copyright 2021 Vulcanize, Inc.
|
||||
//
|
||||
|
||||
import assert from 'assert';
|
||||
import 'reflect-metadata';
|
||||
import express, { Application } from 'express';
|
||||
import { ApolloServer, PubSub } from 'apollo-server-express';
|
||||
import yargs from 'yargs';
|
||||
import { hideBin } from 'yargs/helpers';
|
||||
import debug from 'debug';
|
||||
import { createServer } from 'http';
|
||||
|
||||
import { getCache } from '@cerc-io/cache';
|
||||
import { EthClient } from '@cerc-io/ipld-eth-client';
|
||||
import { DEFAULT_CONFIG_PATH, getConfig } from '@cerc-io/util';
|
||||
|
||||
import typeDefs from './schema';
|
||||
|
||||
import { createResolvers } from './resolvers';
|
||||
import { Indexer } from './indexer';
|
||||
import { EventWatcher } from './events';
|
||||
|
||||
const log = debug('vulcanize:server');
|
||||
|
||||
export const main = async (): Promise<any> => {
|
||||
const argv = await yargs(hideBin(process.argv))
|
||||
.option('f', {
|
||||
alias: 'config-file',
|
||||
demandOption: true,
|
||||
describe: 'configuration file path (toml)',
|
||||
type: 'string',
|
||||
default: DEFAULT_CONFIG_PATH
|
||||
})
|
||||
.argv;
|
||||
|
||||
const config = await getConfig(argv.f);
|
||||
|
||||
assert(config.server, 'Missing server config');
|
||||
|
||||
const { host, port } = config.server;
|
||||
|
||||
const { upstream } = config;
|
||||
|
||||
assert(upstream, 'Missing upstream config');
|
||||
const { ethServer: { gqlApiEndpoint }, cache: cacheConfig } = upstream;
|
||||
assert(gqlApiEndpoint, 'Missing upstream ethServer.gqlApiEndpoint');
|
||||
|
||||
const cache = await getCache(cacheConfig);
|
||||
const ethClient = new EthClient({
|
||||
gqlEndpoint: gqlApiEndpoint,
|
||||
cache
|
||||
});
|
||||
|
||||
const indexer = new Indexer(config, ethClient);
|
||||
|
||||
// Note: In-memory pubsub works fine for now, as each watcher is a single process anyway.
|
||||
// Later: https://www.apollographql.com/docs/apollo-server/data/subscriptions/#production-pubsub-libraries
|
||||
const pubsub = new PubSub();
|
||||
const eventWatcher = new EventWatcher(ethClient, indexer, pubsub);
|
||||
await eventWatcher.start();
|
||||
|
||||
const resolvers = await createResolvers(eventWatcher);
|
||||
|
||||
const app: Application = express();
|
||||
const server = new ApolloServer({
|
||||
typeDefs,
|
||||
resolvers
|
||||
});
|
||||
|
||||
await server.start();
|
||||
server.applyMiddleware({ app });
|
||||
|
||||
const httpServer = createServer(app);
|
||||
server.installSubscriptionHandlers(httpServer);
|
||||
|
||||
httpServer.listen(port, host, () => {
|
||||
log(`Server is listening on host ${host} port ${port}`);
|
||||
});
|
||||
|
||||
return { app, server };
|
||||
};
|
||||
|
||||
main().then(() => {
|
||||
log('Starting server...');
|
||||
}).catch(err => {
|
||||
log(err);
|
||||
});
|
||||
@@ -1,78 +0,0 @@
|
||||
//
|
||||
// Copyright 2021 Vulcanize, Inc.
|
||||
//
|
||||
|
||||
import { expect } from 'chai';
|
||||
import assert from 'assert';
|
||||
import { ethers, Contract, ContractTransaction, Signer, utils } from 'ethers';
|
||||
import 'mocha';
|
||||
|
||||
import {
|
||||
getConfig
|
||||
} from '@cerc-io/util';
|
||||
|
||||
import lighthouseABI from './abi/Lighthouse.json';
|
||||
import { Config } from './indexer';
|
||||
import { Client } from './client';
|
||||
|
||||
const NETWORK_RPC_URL = 'http://localhost:8545';
|
||||
|
||||
describe('lighthouse-watcher', () => {
|
||||
let lighthouse: Contract;
|
||||
|
||||
let config: Config;
|
||||
let signer: Signer;
|
||||
let client: Client;
|
||||
|
||||
before(async () => {
|
||||
const configFile = './environments/local.toml';
|
||||
config = await getConfig(configFile);
|
||||
|
||||
const { server: { host, port }, watch } = config;
|
||||
assert(watch);
|
||||
|
||||
const endpoint = `http://${host}:${port}/graphql`;
|
||||
const gqlEndpoint = endpoint;
|
||||
const gqlSubscriptionEndpoint = endpoint;
|
||||
client = new Client({
|
||||
gqlEndpoint,
|
||||
gqlSubscriptionEndpoint
|
||||
});
|
||||
|
||||
const provider = new ethers.providers.JsonRpcProvider(NETWORK_RPC_URL);
|
||||
signer = provider.getSigner();
|
||||
lighthouse = new Contract(watch.lighthouse, lighthouseABI, signer);
|
||||
});
|
||||
|
||||
it('should trigger StorageRequest event', done => {
|
||||
(async () => {
|
||||
const cid = 'testCid';
|
||||
const config = 'testConfig';
|
||||
const fileCost = '10';
|
||||
const signerAddress = await signer.getAddress();
|
||||
|
||||
// Subscribe using UniClient.
|
||||
const subscription = await client.watchEvents((value: any) => {
|
||||
if (value.event.__typename === 'StorageRequestEvent') {
|
||||
expect(value.event.uploader).to.equal(signerAddress);
|
||||
expect(value.event.cid).to.equal(cid);
|
||||
expect(value.event.config).to.equal(config);
|
||||
expect(value.event.fileCost).to.equal(fileCost);
|
||||
|
||||
if (subscription) {
|
||||
subscription.unsubscribe();
|
||||
}
|
||||
|
||||
done();
|
||||
}
|
||||
});
|
||||
|
||||
// Pool mint.
|
||||
const value = utils.parseUnits(fileCost, 'wei');
|
||||
const transaction: ContractTransaction = await lighthouse.store(cid, config, { value });
|
||||
await transaction.wait();
|
||||
})().catch((error) => {
|
||||
done(error);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,6 +0,0 @@
|
||||
//
|
||||
// Copyright 2021 Vulcanize, Inc.
|
||||
//
|
||||
|
||||
// https://medium.com/@steveruiz/using-a-javascript-library-without-type-declarations-in-a-typescript-project-3643490015f3
|
||||
declare module 'canonical-json'
|
||||
@@ -1,6 +0,0 @@
|
||||
{
|
||||
"name": "common",
|
||||
"version": "0.1.0",
|
||||
"license": "AGPL-3.0",
|
||||
"typings": "main.d.ts"
|
||||
}
|
||||
@@ -1,16 +0,0 @@
|
||||
//
|
||||
// Copyright 2021 Vulcanize, Inc.
|
||||
//
|
||||
|
||||
import { task } from 'hardhat/config';
|
||||
import '@nomiclabs/hardhat-ethers';
|
||||
|
||||
task('lighthouse-deploy', 'Deploys Lighthouse contract')
|
||||
.setAction(async (_, hre) => {
|
||||
await hre.run('compile');
|
||||
|
||||
const lighthouseFactory = await hre.ethers.getContractFactory('Lighthouse');
|
||||
const lighthouse = await lighthouseFactory.deploy();
|
||||
|
||||
console.log('Lighthouse deployed to:', lighthouse.address);
|
||||
});
|
||||
@@ -1,45 +0,0 @@
|
||||
//
|
||||
// Copyright 2021 Vulcanize, Inc.
|
||||
//
|
||||
|
||||
import { task, types } from 'hardhat/config';
|
||||
import '@nomiclabs/hardhat-ethers';
|
||||
import { ContractTransaction, utils } from 'ethers';
|
||||
|
||||
task('lighthouse-store', 'Call Lighthouse store method')
|
||||
.addParam('lighthouse', 'Address of Lighthouse contract', undefined, types.string)
|
||||
.addParam('cid', 'store cid', undefined, types.string)
|
||||
.addParam('storeConfig', 'store config', undefined, types.string)
|
||||
.addParam('fileCost', 'store fileCost (wei)', undefined, types.float)
|
||||
.setAction(async (args, hre) => {
|
||||
const {
|
||||
lighthouse: lighthouseAddress,
|
||||
cid,
|
||||
storeConfig: config,
|
||||
fileCost
|
||||
} = args;
|
||||
|
||||
await hre.run('compile');
|
||||
|
||||
const Ligthouse = await hre.ethers.getContractFactory('Lighthouse');
|
||||
const lighthouse = Ligthouse.attach(lighthouseAddress);
|
||||
const value = utils.parseUnits(String(fileCost), 'wei');
|
||||
|
||||
const transaction: ContractTransaction = await lighthouse.store(cid, config, { value });
|
||||
|
||||
const receipt = await transaction.wait();
|
||||
|
||||
if (receipt.events) {
|
||||
console.log('receipt blockHash', receipt.blockHash);
|
||||
|
||||
const storageRequestEvent = receipt.events.find(el => el.event === 'StorageRequest');
|
||||
|
||||
if (storageRequestEvent && storageRequestEvent.args) {
|
||||
console.log('StorageRequest Event');
|
||||
console.log('uploader:', storageRequestEvent.args.uploader);
|
||||
console.log('cid:', storageRequestEvent.args.cid);
|
||||
console.log('config:', storageRequestEvent.args.config);
|
||||
console.log('fileCost:', storageRequestEvent.args.fileCost.toString());
|
||||
}
|
||||
}
|
||||
});
|
||||
@@ -1,38 +0,0 @@
|
||||
// Original: https://github.com/nandit123/lighthouse/blob/master/contracts/Lighthouse.sol
|
||||
// License:
|
||||
// https://github.com/nandit123/lighthouse/blob/master/LICENSE-APACHE
|
||||
// https://github.com/nandit123/lighthouse/blob/master/LICENSE-MIT
|
||||
|
||||
pragma solidity >=0.4.22 <0.8.0;
|
||||
|
||||
contract Lighthouse {
|
||||
address owner = msg.sender;
|
||||
|
||||
struct Content {
|
||||
string cid;
|
||||
string config;
|
||||
uint fileCost;
|
||||
}
|
||||
|
||||
event StorageRequest(address uploader, string cid, string config, uint fileCost);
|
||||
|
||||
mapping(address => mapping(string => Content)) public requests;
|
||||
|
||||
function store(string calldata cid, string calldata config)
|
||||
external
|
||||
payable
|
||||
{
|
||||
uint fileCost = msg.value;
|
||||
requests[msg.sender][cid] = Content(cid, config, fileCost);
|
||||
emit StorageRequest(msg.sender, cid, config, msg.value);
|
||||
}
|
||||
|
||||
function getPaid(uint amount, address payable recipient)
|
||||
external
|
||||
{
|
||||
require(msg.sender == owner);
|
||||
recipient.transfer(amount);
|
||||
}
|
||||
|
||||
fallback () external payable {}
|
||||
}
|
||||
@@ -1,77 +0,0 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
/* Visit https://aka.ms/tsconfig.json to read more about this file */
|
||||
|
||||
/* Basic Options */
|
||||
// "incremental": true, /* Enable incremental compilation */
|
||||
"target": "es5", /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019', 'ES2020', 'ES2021', or 'ESNEXT'. */
|
||||
"module": "commonjs", /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', 'es2020', or 'ESNext'. */
|
||||
"lib": [ "ES5", "ES6", "ES2020" ], /* Specify library files to be included in the compilation. */
|
||||
// "allowJs": true, /* Allow javascript files to be compiled. */
|
||||
// "checkJs": true, /* Report errors in .js files. */
|
||||
// "jsx": "preserve", /* Specify JSX code generation: 'preserve', 'react-native', 'react', 'react-jsx' or 'react-jsxdev'. */
|
||||
"declaration": true, /* Generates corresponding '.d.ts' file. */
|
||||
// "declarationMap": true, /* Generates a sourcemap for each corresponding '.d.ts' file. */
|
||||
"sourceMap": true, /* Generates corresponding '.map' file. */
|
||||
// "outFile": "./", /* Concatenate and emit output to single file. */
|
||||
"outDir": "dist", /* Redirect output structure to the directory. */
|
||||
// "rootDir": "./", /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */
|
||||
// "composite": true, /* Enable project compilation */
|
||||
// "tsBuildInfoFile": "./", /* Specify file to store incremental compilation information */
|
||||
// "removeComments": true, /* Do not emit comments to output. */
|
||||
// "noEmit": true, /* Do not emit outputs. */
|
||||
// "importHelpers": true, /* Import emit helpers from 'tslib'. */
|
||||
"downlevelIteration": true, /* Provide full support for iterables in 'for-of', spread, and destructuring when targeting 'ES5' or 'ES3'. */
|
||||
// "isolatedModules": true, /* Transpile each file as a separate module (similar to 'ts.transpileModule'). */
|
||||
|
||||
/* Strict Type-Checking Options */
|
||||
"strict": true, /* Enable all strict type-checking options. */
|
||||
// "noImplicitAny": true, /* Raise error on expressions and declarations with an implied 'any' type. */
|
||||
// "strictNullChecks": true, /* Enable strict null checks. */
|
||||
// "strictFunctionTypes": true, /* Enable strict checking of function types. */
|
||||
// "strictBindCallApply": true, /* Enable strict 'bind', 'call', and 'apply' methods on functions. */
|
||||
"strictPropertyInitialization": false, /* Enable strict checking of property initialization in classes. */
|
||||
// "noImplicitThis": true, /* Raise error on 'this' expressions with an implied 'any' type. */
|
||||
// "alwaysStrict": true, /* Parse in strict mode and emit "use strict" for each source file. */
|
||||
|
||||
/* Additional Checks */
|
||||
// "noUnusedLocals": true, /* Report errors on unused locals. */
|
||||
// "noUnusedParameters": true, /* Report errors on unused parameters. */
|
||||
// "noImplicitReturns": true, /* Report error when not all code paths in function return a value. */
|
||||
// "noFallthroughCasesInSwitch": true, /* Report errors for fallthrough cases in switch statement. */
|
||||
// "noUncheckedIndexedAccess": true, /* Include 'undefined' in index signature results */
|
||||
// "noImplicitOverride": true, /* Ensure overriding members in derived classes are marked with an 'override' modifier. */
|
||||
// "noPropertyAccessFromIndexSignature": true, /* Require undeclared properties from index signatures to use element accesses. */
|
||||
|
||||
/* Module Resolution Options */
|
||||
"moduleResolution": "node", /* Specify module resolution strategy: 'node' (Node.js) or 'classic' (TypeScript pre-1.6). */
|
||||
// "baseUrl": "./", /* Base directory to resolve non-absolute module names. */
|
||||
// "paths": {}, /* A series of entries which re-map imports to lookup locations relative to the 'baseUrl'. */
|
||||
// "rootDirs": [], /* List of root folders whose combined content represents the structure of the project at runtime. */
|
||||
"typeRoots": [
|
||||
"./src/types"
|
||||
], /* List of folders to include type definitions from. */
|
||||
// "types": [], /* Type declaration files to be included in compilation. */
|
||||
// "allowSyntheticDefaultImports": true, /* Allow default imports from modules with no default export. This does not affect code emit, just typechecking. */
|
||||
"esModuleInterop": true, /* Enables emit interoperability between CommonJS and ES Modules via creation of namespace objects for all imports. Implies 'allowSyntheticDefaultImports'. */
|
||||
// "preserveSymlinks": true, /* Do not resolve the real path of symlinks. */
|
||||
// "allowUmdGlobalAccess": true, /* Allow accessing UMD globals from modules. */
|
||||
|
||||
/* Source Map Options */
|
||||
// "sourceRoot": "", /* Specify the location where debugger should locate TypeScript files instead of source locations. */
|
||||
// "mapRoot": "", /* Specify the location where debugger should locate map files instead of generated locations. */
|
||||
// "inlineSourceMap": true, /* Emit a single file with source maps instead of having a separate file. */
|
||||
// "inlineSources": true, /* Emit the source alongside the sourcemaps within a single file; requires '--inlineSourceMap' or '--sourceMap' to be set. */
|
||||
|
||||
/* Experimental Options */
|
||||
"experimentalDecorators": true, /* Enables experimental support for ES7 decorators. */
|
||||
// "emitDecoratorMetadata": true, /* Enables experimental support for emitting type metadata for decorators. */
|
||||
|
||||
/* Advanced Options */
|
||||
"skipLibCheck": true, /* Skip type checking of declaration files. */
|
||||
"forceConsistentCasingInFileNames": true, /* Disallow inconsistently-cased references to the same file. */
|
||||
"resolveJsonModule": true /* Enabling the option allows importing JSON, and validating the types in that JSON file. */
|
||||
},
|
||||
"include": ["src"],
|
||||
"exclude": ["dist"]
|
||||
}
|
||||
@@ -92,6 +92,10 @@ const main = async (): Promise<void> => {
|
||||
|
||||
// Export contracts and checkpoints.
|
||||
for (const contract of contracts) {
|
||||
if (contract.startingBlock > block.blockNumber) {
|
||||
continue;
|
||||
}
|
||||
|
||||
exportData.contracts.push({
|
||||
address: contract.address,
|
||||
kind: contract.kind,
|
||||
|
||||
@@ -600,8 +600,6 @@ export class Indexer implements IndexerInterface {
|
||||
}
|
||||
|
||||
async watchContract (address: string, kind: string, checkpoint: boolean, startingBlock: number): Promise<void> {
|
||||
await this.updateIPLDStatusMap(address, {});
|
||||
|
||||
return this._baseIndexer.watchContract(address, kind, checkpoint, startingBlock);
|
||||
}
|
||||
|
||||
|
||||
@@ -806,6 +806,7 @@ describe('Get value from storage', () => {
|
||||
const bytesLength = Math.floor(Math.random() * 64);
|
||||
return ethers.utils.hexlify(ethers.utils.randomBytes(bytesLength));
|
||||
});
|
||||
console.log('bytesArray', bytesArray);
|
||||
|
||||
before(async () => {
|
||||
({ contract: testDynamicArrays, storageLayout } = contracts.TestDynamicArrays);
|
||||
@@ -952,6 +953,7 @@ describe('Get value from storage', () => {
|
||||
});
|
||||
|
||||
it('get value for dynamic sized array of bytes', async () => {
|
||||
console.log('testFixedArrays.address', testDynamicArrays.address);
|
||||
let { value, proof } = await getStorageValue(storageLayout, getStorageAt, blockHash, testDynamicArrays.address, 'bytesArray');
|
||||
expect(value).to.eql(bytesArray);
|
||||
const proofData = JSON.parse(proof.data);
|
||||
|
||||
@@ -42,6 +42,7 @@ export interface ServerConfig {
|
||||
wasmRestartBlocksInterval: number;
|
||||
filterLogs: boolean;
|
||||
maxEventsBlockRange: number;
|
||||
clearEntitiesCacheInterval: number;
|
||||
}
|
||||
|
||||
export interface UpstreamConfig {
|
||||
|
||||
@@ -404,6 +404,7 @@ export class Indexer {
|
||||
|
||||
async watchContract (address: string, kind: string, checkpoint: boolean, startingBlock: number): Promise<void> {
|
||||
assert(this._db.saveContract);
|
||||
this.updateIPLDStatusMap(address, {});
|
||||
const dbTx = await this._db.createTransactionRunner();
|
||||
|
||||
// Use the checksum address (https://docs.ethers.io/v5/api/utils/address/#utils-getAddress) if input to address is a contract address.
|
||||
@@ -838,14 +839,14 @@ export class Indexer {
|
||||
try {
|
||||
res = await this._db.saveOrUpdateIPLDBlock(dbTx, ipldBlock);
|
||||
|
||||
await dbTx.commitTransaction();
|
||||
|
||||
// Get IPLD status for the contract.
|
||||
const ipldStatus = this._ipldStatusMap[res.contractAddress];
|
||||
assert(ipldStatus, `IPLD status for contract ${res.contractAddress} not found`);
|
||||
|
||||
// Update the IPLD status for the kind.
|
||||
ipldStatus[res.kind] = res.block.blockNumber;
|
||||
|
||||
await dbTx.commitTransaction();
|
||||
} catch (error) {
|
||||
await dbTx.rollbackTransaction();
|
||||
throw error;
|
||||
@@ -888,7 +889,7 @@ export class Indexer {
|
||||
}
|
||||
}
|
||||
|
||||
async updateIPLDStatusMap (address: string, ipldStatus: IpldStatus): Promise<void> {
|
||||
updateIPLDStatusMap (address: string, ipldStatus: IpldStatus): void {
|
||||
// Get and update IPLD status for the contract.
|
||||
const ipldStatusOld = this._ipldStatusMap[address];
|
||||
this._ipldStatusMap[address] = _.merge(ipldStatusOld, ipldStatus);
|
||||
|
||||
@@ -68,6 +68,11 @@ export const eventProcessingLoadEntityDBQueryDuration = new client.Histogram({
|
||||
help: 'Duration of DB query made in event processing'
|
||||
});
|
||||
|
||||
export const cachePrunedEntitiesCount = new client.Gauge({
|
||||
name: 'cached_pruned_entities_total',
|
||||
help: 'Total entities in pruned region of cache'
|
||||
});
|
||||
|
||||
export const eventProcessingEthCallDuration = new client.Histogram({
|
||||
name: 'event_processing_eth_call_duration_seconds',
|
||||
help: 'Duration of eth_calls made in event processing'
|
||||
|
||||
Reference in New Issue
Block a user