Address watcher package scaffolding (#75)

* Rename erc20 watcher folder.

* Update repo URLs.

* Address watcher package scaffolding.
This commit is contained in:
Ashwin Phatak 2021-06-17 17:56:38 +05:30 committed by GitHub
parent 56fb7c69c0
commit 8507bc8b9c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
49 changed files with 590 additions and 34 deletions

View File

@ -1,20 +1,4 @@
# ERC20 Watcher
## Overview
* Create developer facing GQL schema (`erc20.graphql`) for ERC20 contracts
* GQL `queries` that return useful information
* Individual token data corresponding to the ERC20 ABI
* Aggregate data like running 1-day, 7-day & 30-day `transfer` counts and volumes
* GQL `mutation` to add a new ERC20 contract to watch
* Create a server (`erc20-info-server`) to expose the above GQL API
* Initally, the GQL resolvers will return mock data
* Create a basic `React` app (`erc20-dashboard`) that consumes the GQL API from `erc20-info-server`.
* Create a new watcher (`erc20-watcher-ts`) that is capable of watching multiple ERC20 tokens, capturing their events and state
* Update the `erc20-info-server` GQL resolver to return data by querying the lower-layer `erc20-watcher-ts` GQL API
* For GQL result data, at a minimum, return the request and list of CIDs/mhKeys required to generate that result.
* Note: This implies, for example, performing aggregation in code instead of at the SQL layer.
* Create an ERC20 watcher factory (`erc20-watcher-factory-ts`) that auto-detects ERC20 tokens created on-chain and calls `erc20-info-server` to request watching them.
# watcher-ts
## Setup
@ -26,6 +10,8 @@ Install packages (Node.JS v15.11.0):
yarn
```
## ERC20 Watcher
Create a postgres12 database and provide connection settings in `environments/local.toml`.
For example:
@ -38,7 +24,7 @@ createdb erc20-watcher
Run the watcher:
```bash
cd packages/watcher
cd packages/erc20-watcher
yarn run server
```
@ -47,16 +33,16 @@ GQL console: http://localhost:3001/graphql
To run tests (GQL queries) against the mock server:
```
cd packages/watcher
cd packages/erc20-watcher
yarn run server:mock
```
```bash
cd packages/watcher
cd packages/erc20-watcher
yarn test
```
## Example GQL Queries
### Example GQL Queries
```text
{

View File

@ -1,5 +1,5 @@
{
"name": "erc20-watcher",
"name": "watcher-ts",
"private": true,
"workspaces": [
"packages/*"

View File

@ -0,0 +1,31 @@
[server]
host = "127.0.0.1"
port = 3002
[database]
type = "postgres"
host = "localhost"
port = 5432
database = "address-watcher"
username = "postgres"
password = "postgres"
synchronize = true
logging = false
entities = [ "src/entity/**/*.ts" ]
migrations = [ "src/migration/**/*.ts" ]
subscribers = [ "src/subscriber/**/*.ts" ]
[database.cli]
entitiesDir = "src/entity"
migrationsDir = "src/migration"
subscribersDir = "src/subscriber"
[upstream]
gqlEndpoint = "http://127.0.0.1:8083/graphql"
gqlSubscriptionEndpoint = "http://127.0.0.1:5000/graphql"
[upstream.cache]
name = "requests"
enabled = false
deleteOnStart = false

View File

@ -0,0 +1,67 @@
{
"name": "@vulcanize/address-watcher",
"version": "0.1.0",
"description": "Address Watcher",
"private": true,
"scripts": {
"server": "DEBUG=vulcanize:* nodemon src/server.ts -f environments/local.toml",
"server:mock": "MOCK=1 nodemon src/server.ts -f environments/local.toml",
"test": "mocha -r ts-node/register src/**/*.spec.ts",
"lint": "eslint .",
"build": "tsc"
},
"repository": {
"type": "git",
"url": "git+https://github.com/vulcanize/watcher-ts.git"
},
"author": "",
"license": "UNLICENSED",
"bugs": {
"url": "https://github.com/vulcanize/watcher-ts/issues"
},
"homepage": "https://github.com/vulcanize/watcher-ts#readme",
"dependencies": {
"@graphql-tools/schema": "^7.1.5",
"@types/lodash": "^4.14.168",
"@vulcanize/cache": "^0.1.0",
"@vulcanize/ipld-eth-client": "^0.1.0",
"@vulcanize/solidity-mapper": "^0.1.0",
"apollo-server-express": "^2.25.0",
"apollo-type-bigint": "^0.1.3",
"debug": "^4.3.1",
"ethers": "^5.2.0",
"express": "^4.17.1",
"express-graphql": "^0.12.0",
"fs-extra": "^10.0.0",
"graphql": "^15.5.0",
"graphql-import-node": "^0.0.4",
"graphql-request": "^3.4.0",
"json-bigint": "^1.0.0",
"lodash": "^4.17.21",
"pg": "^8.6.0",
"reflect-metadata": "^0.1.13",
"toml": "^3.0.0",
"typeorm": "^0.2.32",
"typeorm-naming-strategies": "^2.0.0",
"yargs": "^17.0.1"
},
"devDependencies": {
"@ethersproject/abi": "^5.3.0",
"@types/express": "^4.17.11",
"@types/fs-extra": "^9.0.11",
"@types/json-bigint": "^1.0.0",
"@types/yargs": "^17.0.0",
"@typescript-eslint/eslint-plugin": "^4.25.0",
"@typescript-eslint/parser": "^4.25.0",
"chai": "^4.3.4",
"eslint": "^7.27.0",
"eslint-config-semistandard": "^15.0.1",
"eslint-config-standard": "^16.0.3",
"eslint-plugin-import": "^2.23.3",
"eslint-plugin-node": "^11.1.0",
"eslint-plugin-promise": "^5.1.0",
"eslint-plugin-standard": "^5.0.0",
"mocha": "^8.4.0",
"nodemon": "^2.0.7"
}
}

View File

@ -0,0 +1,45 @@
import assert from 'assert';
import yargs from 'yargs';
import 'reflect-metadata';
import { ethers } from 'ethers';
import { Config, getConfig } from '../config';
import { Database } from '../database';
(async () => {
const argv = await yargs.parserConfiguration({
'parse-numbers': false
}).options({
configFile: {
type: 'string',
require: true,
demandOption: true,
describe: 'configuration file path (toml)'
},
address: {
type: 'string',
require: true,
demandOption: true,
describe: 'Address to watch'
},
startingBlock: {
type: 'number',
default: 1,
describe: 'Starting block'
}
}).argv;
const config: Config = await getConfig(argv.configFile);
const { database: dbConfig } = config;
assert(dbConfig);
const db = new Database(dbConfig);
await db.init();
// Always use the checksum address (https://docs.ethers.io/v5/api/utils/address/#utils-getAddress).
const address = ethers.utils.getAddress(argv.address);
await db.saveAddress(address, argv.startingBlock);
await db.close();
})();

View File

@ -0,0 +1,53 @@
import assert from 'assert';
import { Connection, ConnectionOptions, createConnection } from 'typeorm';
import { SnakeNamingStrategy } from 'typeorm-naming-strategies';
import { Address } from './entity/Address';
export class Database {
_config: ConnectionOptions
_conn!: Connection
constructor (config: ConnectionOptions) {
assert(config);
this._config = config;
}
async init (): Promise<void> {
assert(!this._conn);
this._conn = await createConnection({
...this._config,
namingStrategy: new SnakeNamingStrategy()
});
}
async close (): Promise<void> {
return this._conn.close();
}
async isWatchedAddress (address: string): Promise<boolean> {
const numRows = await this._conn.getRepository(Address)
.createQueryBuilder()
.where('address = :address', { address })
.getCount();
return numRows > 0;
}
async saveAddress (address: string, startingBlock: number): Promise<void> {
await this._conn.transaction(async (tx) => {
const repo = tx.getRepository(Address);
const numRows = await repo
.createQueryBuilder()
.where('address = :address', { address })
.getCount();
if (numRows === 0) {
const entity = repo.create({ address, startingBlock: BigInt(startingBlock) });
await repo.save(entity);
}
});
}
}

View File

@ -0,0 +1,14 @@
import { Entity, PrimaryGeneratedColumn, Column, Index } from 'typeorm';
@Entity()
@Index(['address'], { unique: true })
export class Address {
@PrimaryGeneratedColumn()
id!: number;
@Column('varchar', { length: 42 })
address!: string;
@Column('numeric')
startingBlock!: bigint;
}

View File

@ -0,0 +1,46 @@
import assert from 'assert';
// import debug from 'debug';
import { ethers } from 'ethers';
import { PubSub } from 'apollo-server-express';
import { EthClient } from '@vulcanize/ipld-eth-client';
import { GetStorageAt } from '@vulcanize/solidity-mapper';
import { Database } from './database';
// const log = debug('vulcanize:indexer');
export class Indexer {
_db: Database
_ethClient: EthClient
_pubsub: PubSub
_getStorageAt: GetStorageAt
constructor (db: Database, ethClient: EthClient, pubsub: PubSub) {
assert(db);
assert(ethClient);
assert(pubsub);
this._db = db;
this._ethClient = ethClient;
this._pubsub = pubsub;
this._getStorageAt = this._ethClient.getStorageAt.bind(this._ethClient);
}
getEventIterator (): AsyncIterator<any> {
return this._pubsub.asyncIterator(['event']);
}
async isWatchedAddress (address : string): Promise<boolean> {
assert(address);
return this._db.isWatchedAddress(ethers.utils.getAddress(address));
}
async watchAddress (address: string, startingBlock: number): Promise<boolean> {
// Always use the checksum address (https://docs.ethers.io/v5/api/utils/address/#utils-getAddress).
await this._db.saveAddress(ethers.utils.getAddress(address), startingBlock);
return true;
}
}

View File

View File

@ -0,0 +1,27 @@
import assert from 'assert';
import debug from 'debug';
import { Indexer } from './indexer';
const log = debug('vulcanize:resolver');
export const createResolvers = async (indexer: Indexer): Promise<any> => {
assert(indexer);
return {
Subscription: {
onAddressEvent: {
subscribe: () => indexer.getEventIterator()
}
},
Mutation: {
watchAddress: (_: any, { address, startingBlock = 1 }: { address: string, startingBlock: number }): Promise<boolean> => {
log('watchAddress', address, startingBlock);
return indexer.watchAddress(address, startingBlock);
}
},
Query: {}
};
};

View File

@ -0,0 +1,45 @@
import { gql } from '@apollo/client/core';
export default gql`
# Types
# Watched event, include additional context over and above the event data.
type WatchedEvent {
blockHash: String!
txHash: String!
address: String!
}
#
# Queries
#
type Query {
queryAppearances(
address: String!
txHash: String!
): [String!]
}
#
# Subscriptions
#
type Subscription {
# Watch for token events (at head of chain).
onAddressEvent: WatchedEvent!
}
#
# Mutations
#
type Mutation {
# Actively watch and index data for the address.
watchAddress(
address: String!
startingBlock: Int
): Boolean!
}
`;

View File

@ -0,0 +1,85 @@
import assert from 'assert';
import 'reflect-metadata';
import express, { Application } from 'express';
import { ApolloServer, PubSub } from 'apollo-server-express';
import yargs from 'yargs';
import { hideBin } from 'yargs/helpers';
import debug from 'debug';
import 'graphql-import-node';
import { createServer } from 'http';
import { getCache } from '@vulcanize/cache';
import { EthClient } from '@vulcanize/ipld-eth-client';
import typeDefs from './schema';
import { createResolvers } from './resolvers';
import { Indexer } from './indexer';
import { Database } from './database';
import { getConfig } from './config';
const log = debug('vulcanize:server');
export const main = async (): Promise<any> => {
const argv = await yargs(hideBin(process.argv))
.option('f', {
alias: 'config-file',
demandOption: true,
describe: 'configuration file path (toml)',
type: 'string'
})
.argv;
const config = await getConfig(argv.f);
assert(config.server, 'Missing server config');
const { host, port } = config.server;
const { upstream, database: dbConfig } = config;
assert(dbConfig, 'Missing database config');
const db = new Database(dbConfig);
await db.init();
assert(upstream, 'Missing upstream config');
const { gqlEndpoint, gqlSubscriptionEndpoint, cache: cacheConfig } = upstream;
assert(gqlEndpoint, 'Missing upstream gqlEndpoint');
assert(gqlSubscriptionEndpoint, 'Missing upstream gqlSubscriptionEndpoint');
const cache = await getCache(cacheConfig);
const ethClient = new EthClient({ gqlEndpoint, gqlSubscriptionEndpoint, cache });
// Note: In-memory pubsub works fine for now, as each watcher is a single process anyway.
// Later: https://www.apollographql.com/docs/apollo-server/data/subscriptions/#production-pubsub-libraries
const pubsub = new PubSub();
const indexer = new Indexer(db, ethClient, pubsub);
const resolvers = await createResolvers(indexer);
const app: Application = express();
const server = new ApolloServer({
typeDefs,
resolvers
});
await server.start();
server.applyMiddleware({ app });
const httpServer = createServer(app);
server.installSubscriptionHandlers(httpServer);
httpServer.listen(port, host, () => {
log(`Server is listening on host ${host} port ${port}`);
});
return { app, server };
};
main().then(() => {
log('Starting server...');
}).catch(err => {
log(err);
});

View File

@ -11,14 +11,14 @@
},
"repository": {
"type": "git",
"url": "git+https://github.com/vulcanize/erc20-watcher.git"
"url": "git+https://github.com/vulcanize/watcher-ts.git"
},
"author": "",
"license": "UNLICENSED",
"bugs": {
"url": "https://github.com/vulcanize/erc20-watcher/issues"
"url": "https://github.com/vulcanize/watcher-ts/issues"
},
"homepage": "https://github.com/vulcanize/erc20-watcher#readme",
"homepage": "https://github.com/vulcanize/watcher-ts#readme",
"dependencies": {
"canonical-json": "^0.0.4",
"debug": "^4.3.1",

View File

@ -0,0 +1,5 @@
# Don't lint node_modules.
node_modules
# Don't lint build output.
dist

View File

@ -0,0 +1,27 @@
{
"env": {
"browser": true,
"es2021": true
},
"extends": [
"semistandard",
"plugin:@typescript-eslint/recommended"
],
"parser": "@typescript-eslint/parser",
"parserOptions": {
"ecmaVersion": 12,
"sourceType": "module"
},
"plugins": [
"@typescript-eslint"
],
"rules": {
"@typescript-eslint/no-explicit-any": "off",
"@typescript-eslint/explicit-module-boundary-types": [
"warn",
{
"allowArgumentsExplicitlyTypedAsAny": true
}
]
}
}

6
packages/erc20-watcher/.gitignore vendored Normal file
View File

@ -0,0 +1,6 @@
.idea/
.vscode/
node_modules/
build/
tmp/
temp/

View File

@ -12,14 +12,14 @@
},
"repository": {
"type": "git",
"url": "git+https://github.com/vulcanize/erc20-watcher.git"
"url": "git+https://github.com/vulcanize/watcher-ts.git"
},
"author": "",
"license": "UNLICENSED",
"bugs": {
"url": "https://github.com/vulcanize/erc20-watcher/issues"
"url": "https://github.com/vulcanize/watcher-ts/issues"
},
"homepage": "https://github.com/vulcanize/erc20-watcher#readme",
"homepage": "https://github.com/vulcanize/watcher-ts#readme",
"dependencies": {
"@graphql-tools/schema": "^7.1.5",
"@types/lodash": "^4.14.168",

View File

@ -0,0 +1,35 @@
import fs from 'fs-extra';
import path from 'path';
import toml from 'toml';
import debug from 'debug';
import { ConnectionOptions } from 'typeorm';
import { Config as CacheConfig } from '@vulcanize/cache';
const log = debug('vulcanize:config');
export interface Config {
server: {
host: string;
port: number;
};
database: ConnectionOptions;
upstream: {
gqlEndpoint: string;
gqlSubscriptionEndpoint: string;
cache: CacheConfig
}
}
export const getConfig = async (configFile: string): Promise<Config> => {
const configFilePath = path.resolve(configFile);
const fileExists = await fs.pathExists(configFilePath);
if (!fileExists) {
throw new Error(`Config file not found: ${configFilePath}`);
}
const config = toml.parse(await fs.readFile(configFilePath, 'utf8'));
log('config', JSON.stringify(config, null, 2));
return config;
};

View File

@ -0,0 +1,2 @@
// https://medium.com/@steveruiz/using-a-javascript-library-without-type-declarations-in-a-typescript-project-3643490015f3
declare module 'canonical-json'

View File

@ -0,0 +1,5 @@
{
"name": "common",
"version": "0.1.0",
"typings": "main.d.ts"
}

View File

@ -0,0 +1,77 @@
{
"compilerOptions": {
/* Visit https://aka.ms/tsconfig.json to read more about this file */
/* Basic Options */
// "incremental": true, /* Enable incremental compilation */
"target": "es5", /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019', 'ES2020', 'ES2021', or 'ESNEXT'. */
"module": "commonjs", /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', 'es2020', or 'ESNext'. */
"lib": [ "ES5", "ES6", "ES2020" ], /* Specify library files to be included in the compilation. */
// "allowJs": true, /* Allow javascript files to be compiled. */
// "checkJs": true, /* Report errors in .js files. */
// "jsx": "preserve", /* Specify JSX code generation: 'preserve', 'react-native', 'react', 'react-jsx' or 'react-jsxdev'. */
"declaration": true, /* Generates corresponding '.d.ts' file. */
// "declarationMap": true, /* Generates a sourcemap for each corresponding '.d.ts' file. */
// "sourceMap": true, /* Generates corresponding '.map' file. */
// "outFile": "./", /* Concatenate and emit output to single file. */
"outDir": "dist", /* Redirect output structure to the directory. */
// "rootDir": "./", /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */
// "composite": true, /* Enable project compilation */
// "tsBuildInfoFile": "./", /* Specify file to store incremental compilation information */
// "removeComments": true, /* Do not emit comments to output. */
// "noEmit": true, /* Do not emit outputs. */
// "importHelpers": true, /* Import emit helpers from 'tslib'. */
"downlevelIteration": true, /* Provide full support for iterables in 'for-of', spread, and destructuring when targeting 'ES5' or 'ES3'. */
// "isolatedModules": true, /* Transpile each file as a separate module (similar to 'ts.transpileModule'). */
/* Strict Type-Checking Options */
"strict": true, /* Enable all strict type-checking options. */
// "noImplicitAny": true, /* Raise error on expressions and declarations with an implied 'any' type. */
// "strictNullChecks": true, /* Enable strict null checks. */
// "strictFunctionTypes": true, /* Enable strict checking of function types. */
// "strictBindCallApply": true, /* Enable strict 'bind', 'call', and 'apply' methods on functions. */
// "strictPropertyInitialization": true, /* Enable strict checking of property initialization in classes. */
// "noImplicitThis": true, /* Raise error on 'this' expressions with an implied 'any' type. */
// "alwaysStrict": true, /* Parse in strict mode and emit "use strict" for each source file. */
/* Additional Checks */
// "noUnusedLocals": true, /* Report errors on unused locals. */
// "noUnusedParameters": true, /* Report errors on unused parameters. */
// "noImplicitReturns": true, /* Report error when not all code paths in function return a value. */
// "noFallthroughCasesInSwitch": true, /* Report errors for fallthrough cases in switch statement. */
// "noUncheckedIndexedAccess": true, /* Include 'undefined' in index signature results */
// "noImplicitOverride": true, /* Ensure overriding members in derived classes are marked with an 'override' modifier. */
// "noPropertyAccessFromIndexSignature": true, /* Require undeclared properties from index signatures to use element accesses. */
/* Module Resolution Options */
"moduleResolution": "node", /* Specify module resolution strategy: 'node' (Node.js) or 'classic' (TypeScript pre-1.6). */
// "baseUrl": "./", /* Base directory to resolve non-absolute module names. */
// "paths": {}, /* A series of entries which re-map imports to lookup locations relative to the 'baseUrl'. */
// "rootDirs": [], /* List of root folders whose combined content represents the structure of the project at runtime. */
"typeRoots": [
"./src/types"
], /* List of folders to include type definitions from. */
// "types": [], /* Type declaration files to be included in compilation. */
// "allowSyntheticDefaultImports": true, /* Allow default imports from modules with no default export. This does not affect code emit, just typechecking. */
"esModuleInterop": true, /* Enables emit interoperability between CommonJS and ES Modules via creation of namespace objects for all imports. Implies 'allowSyntheticDefaultImports'. */
// "preserveSymlinks": true, /* Do not resolve the real path of symlinks. */
// "allowUmdGlobalAccess": true, /* Allow accessing UMD globals from modules. */
/* Source Map Options */
// "sourceRoot": "", /* Specify the location where debugger should locate TypeScript files instead of source locations. */
// "mapRoot": "", /* Specify the location where debugger should locate map files instead of generated locations. */
// "inlineSourceMap": true, /* Emit a single file with source maps instead of having a separate file. */
// "inlineSources": true, /* Emit the source alongside the sourcemaps within a single file; requires '--inlineSourceMap' or '--sourceMap' to be set. */
/* Experimental Options */
"experimentalDecorators": true, /* Enables experimental support for ES7 decorators. */
// "emitDecoratorMetadata": true, /* Enables experimental support for emitting type metadata for decorators. */
/* Advanced Options */
"skipLibCheck": true, /* Skip type checking of declaration files. */
"forceConsistentCasingInFileNames": true, /* Disallow inconsistently-cased references to the same file. */
"resolveJsonModule": true /* Enabling the option allows importing JSON, and validating the types in that JSON file. */
},
"include": ["src"],
"exclude": ["dist"]
}

View File

@ -11,14 +11,14 @@
},
"repository": {
"type": "git",
"url": "git+https://github.com/vulcanize/erc20-watcher.git"
"url": "git+https://github.com/vulcanize/watcher-ts.git"
},
"author": "",
"license": "UNLICENSED",
"bugs": {
"url": "https://github.com/vulcanize/erc20-watcher/issues"
"url": "https://github.com/vulcanize/watcher-ts/issues"
},
"homepage": "https://github.com/vulcanize/erc20-watcher#readme",
"homepage": "https://github.com/vulcanize/watcher-ts#readme",
"dependencies": {
"@apollo/client": "^3.3.19",
"@vulcanize/cache": "^0.1.0",

View File

@ -11,14 +11,14 @@
},
"repository": {
"type": "git",
"url": "git+https://github.com/vulcanize/erc20-watcher.git"
"url": "git+https://github.com/vulcanize/watcher-ts.git"
},
"author": "",
"license": "UNLICENSED",
"bugs": {
"url": "https://github.com/vulcanize/erc20-watcher/issues"
"url": "https://github.com/vulcanize/watcher-ts/issues"
},
"homepage": "https://github.com/vulcanize/erc20-watcher#readme",
"homepage": "https://github.com/vulcanize/watcher-ts#readme",
"dependencies": {
"canonical-json": "^0.0.4",
"debug": "^4.3.1",