From efbc86bdeaa2144f012a821363159972d3db6338 Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Tue, 20 Dec 2022 14:06:48 -0600 Subject: [PATCH 01/11] Moved repos. (#72) --- app/data/repository-list.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/data/repository-list.txt b/app/data/repository-list.txt index bf9c4427..72d778b7 100644 --- a/app/data/repository-list.txt +++ b/app/data/repository-list.txt @@ -5,8 +5,8 @@ cerc-io/ipld-eth-server cerc-io/eth-statediff-service vulcanize/eth-statediff-fill-service vulcanize/ipld-eth-db-validator -vulcanize/ipld-eth-beacon-indexer -vulcanize/ipld-eth-beacon-db +cerc-io/ipld-eth-beacon-indexer +cerc-io/ipld-eth-beacon-db cerc-io/laconicd cerc-io/laconic-sdk cerc-io/laconic-registry-cli From 8f63328d608cbf3c45da90dce7e60c5840795451 Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Tue, 20 Dec 2022 14:08:10 -0600 Subject: [PATCH 02/11] fixturenet-eth README (#73) * fixturenet-eth README * Fix bullets * Minor edit * Update README.md --- stacks/fixturenet-eth/README.md | 88 +++++++++++++++++++++++++++++++++ stacks/fixturenet-eth/stack.yml | 12 +++++ 2 files changed, 100 insertions(+) create mode 100644 stacks/fixturenet-eth/README.md create mode 100644 stacks/fixturenet-eth/stack.yml diff --git a/stacks/fixturenet-eth/README.md b/stacks/fixturenet-eth/README.md new file mode 100644 index 00000000..d4c37fe9 --- /dev/null +++ b/stacks/fixturenet-eth/README.md @@ -0,0 +1,88 @@ +# fixturenet-eth + +Instructions for deploying a local a geth + lighthouse blockchain "fixturenet" for development and testing purposes using laconic-stack-orchestrator (the installation of which is covered [here](https://github.com/cerc-io/stack-orchestrator#user-mode)): + +## Clone required repositories +``` +$ laconic-so setup-repositories --include cerc-io/go-ethereum +``` + +## Build the fixturenet-eth containers +``` +$ laconic-so build-containers --include cerc/go-ethereum,cerc/lighthouse,cerc/fixturenet-eth-geth,cerc/fixturenet-eth-lighthouse +``` +This should create several container in the local image registry: + +* cerc/go-ethereum +* cerc/lighthouse +* cerc/fixturenet-eth-geth +* cerc/fixturenet-eth-lighthouse + +## Deploy the stack +``` +$ laconic-so deploy-system --include fixturenet-eth up +``` + +## Check status + +``` +$ container-build/cerc-fixturenet-eth-lighthouse/scripts/status.sh +Waiting for geth to generate DAG ..................................................... DONE! +Waiting for beacon phase0 .... DONE! +Waiting for beacon altair .... DONE! +Waiting for beacon bellatrix pre-merge .... DONE! +Waiting for beacon bellatrix merge .... DONE! +``` + +## Additional pieces + +Several other containers can used with the basic `fixturenet-eth`: + +* `ipld-eth-db` (enables statediffing) +* `ipld-eth-server` (GQL and Ethereum API server, requires `ipld-eth-db`) +* `ipld-eth-beacon-db` and `ipld-eth-beacon-indexer` (for indexing Beacon chain blocks) +* `eth-probe` (captures eth1 tx gossip) +* `keycloak` (nginx proxy with keycloak auth for API authentication) + +It is not necessary to use them all at once, but a complete example follows: + +``` +# Setup +$ laconic-so setup-repositories --include cerc-io/go-ethereum,cerc-io/ipld-eth-db,cerc-io/ipld-eth-server,cerc-io/ipld-eth-beacon-db,cerc-io/ipld-eth-beacon-indexer,cerc-io/eth-probe + +# Build +$ laconic-so build-containers --include cerc/go-ethereum,cerc/lighthouse,cerc/fixturenet-eth-geth,cerc/fixturenet-eth-lighthouse,cerc/ipld-eth-db,cerc/ipld-eth-server,cerc/ipld-eth-beacon-db,cerc/ipld-eth-beacon-indexer,cerc/eth-probe,cerc/keycloak + +# Deploy +$ laconic-so deploy-system --include db,fixturenet-eth,ipld-eth-server,ipld-eth-beacon-db,ipld-eth-beacon-indexer,eth-probe,keycloak up + +# Status + +$ container-build/cerc-fixturenet-eth-lighthouse/scripts/status.sh +Waiting for geth to generate DAG.... DONE! +Waiting for beacon phase0.... DONE! +Waiting for beacon altair.... DONE! +Waiting for beacon bellatrix pre-merge.... DONE! +Waiting for beacon bellatrix merge.... DONE! + +$ docker ps -f 'name=laconic' +CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES +fe60af64140c cerc/ipld-eth-beacon-indexer:local "./entrypoint.sh" 4 minutes ago Up 3 minutes laconic-7ca8f4a970d7999a235e0ee27588a5ab-ipld-eth-beacon-indexer-1 +015583b318c2 cerc/eth-probe:local "/app/run.sh" 4 minutes ago Up 3 minutes laconic-7ca8f4a970d7999a235e0ee27588a5ab-eth-probe-probe-1 +180b03993b1e nginx:1.23-alpine "/docker-entrypoint.…" 4 minutes ago Up 4 minutes 0.0.0.0:54162->80/tcp laconic-7ca8f4a970d7999a235e0ee27588a5ab-keycloak-nginx-1 +2e963e3c4b44 cerc/ipld-eth-server:local "/app/entrypoint.sh" 4 minutes ago Up 4 minutes 127.0.0.1:8081-8082->8081-8082/tcp laconic-7ca8f4a970d7999a235e0ee27588a5ab-ipld-eth-server-1 +913cefc5cecf cerc/fixturenet-eth-lighthouse:local "/opt/testnet/run.sh" 4 minutes ago Up 4 minutes (healthy) 0.0.0.0:54161->8001/tcp laconic-7ca8f4a970d7999a235e0ee27588a5ab-fixturenet-eth-lighthouse-1-1 +c0659d0204ff cerc/fixturenet-eth-lighthouse:local "/opt/testnet/run.sh" 4 minutes ago Up 3 minutes (healthy) laconic-7ca8f4a970d7999a235e0ee27588a5ab-fixturenet-eth-lighthouse-2-1 +1636ed1013a6 cerc/keycloak:local "/opt/keycloak/bin/k…" 4 minutes ago Up 4 minutes 8443/tcp, 0.0.0.0:54160->8080/tcp laconic-7ca8f4a970d7999a235e0ee27588a5ab-keycloak-1 +439b017d75c1 cerc/ipld-eth-db:local "/app/startup_script…" 4 minutes ago Up 4 minutes laconic-7ca8f4a970d7999a235e0ee27588a5ab-migrations-1 +0a2c740a8e12 cerc/eth-probe:local "/app/run.sh" 4 minutes ago Up 4 minutes (healthy) laconic-7ca8f4a970d7999a235e0ee27588a5ab-eth-probe-mq-1 +35a816e0bac1 cerc/fixturenet-eth-geth:local "/opt/testnet/run.sh" 4 minutes ago Up 4 minutes (healthy) 8546/tcp, 30303/tcp, 30303/udp, 0.0.0.0:54154->8545/tcp laconic-7ca8f4a970d7999a235e0ee27588a5ab-fixturenet-eth-geth-1-1 +6691e5988519 cerc/fixturenet-eth-geth:local "/opt/testnet/run.sh" 4 minutes ago Up 4 minutes (healthy) 8545-8546/tcp, 30303/tcp, 30303/udp laconic-7ca8f4a970d7999a235e0ee27588a5ab-fixturenet-eth-geth-2-1 +06602dc7e3d0 timescale/timescaledb:latest-pg14 "docker-entrypoint.s…" 4 minutes ago Up 4 minutes (healthy) 0.0.0.0:54153->5432/tcp laconic-7ca8f4a970d7999a235e0ee27588a5ab-eth-probe-db-1 +91d73ec45b97 cerc/ipld-eth-beacon-db:local "docker-entrypoint.s…" 4 minutes ago Up 4 minutes (healthy) 127.0.0.1:8076->5432/tcp laconic-7ca8f4a970d7999a235e0ee27588a5ab-ipld-eth-beacon-db-1 +48459978329c postgres:14-alpine "docker-entrypoint.s…" 4 minutes ago Up 4 minutes (healthy) 0.0.0.0:54152->5432/tcp laconic-7ca8f4a970d7999a235e0ee27588a5ab-keycloak-db-1 +f48169806b54 timescale/timescaledb:2.8.1-pg14 "docker-entrypoint.s…" 4 minutes ago Up 4 minutes (healthy) 127.0.0.1:8077->5432/tcp laconic-7ca8f4a970d7999a235e0ee27588a5ab-ipld-eth-db-1 +56686c4e004f cerc/fixturenet-eth-geth:local "/opt/testnet/run.sh" 4 minutes ago Up 4 minutes 8545-8546/tcp, 30303/udp, 0.0.0.0:54151->9898/tcp, 0.0.0.0:54150->30303/tcp laconic-7ca8f4a970d7999a235e0ee27588a5ab-fixturenet-eth-bootnode-geth-1 +95f073c5e956 cerc/fixturenet-eth-lighthouse:local "/opt/testnet/run.sh" 4 minutes ago Up 4 minutes laconic-7ca8f4a970d7999a235e0ee27588a5ab-fixturenet-eth-bootnode-lighthouse-1 +``` + diff --git a/stacks/fixturenet-eth/stack.yml b/stacks/fixturenet-eth/stack.yml new file mode 100644 index 00000000..0030fdd4 --- /dev/null +++ b/stacks/fixturenet-eth/stack.yml @@ -0,0 +1,12 @@ +version: "1.0" +name: fixturenet-eth +repos: + - cerc-io/go-ethereum + - cerc-io/lighthouse +containers: + - cerc/go-ethereum + - cerc/lighthouse + - cerc/fixturenet-eth-geth + - cerc/fixturenet-eth-lighthouse +pods: + - fixturenet-eth \ No newline at end of file From 543afe18088d21a8284551dd98b308b65567e53f Mon Sep 17 00:00:00 2001 From: David Boreham Date: Tue, 20 Dec 2022 19:10:23 -0700 Subject: [PATCH 03/11] Fix typo --- stacks/fixturenet-eth/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stacks/fixturenet-eth/README.md b/stacks/fixturenet-eth/README.md index d4c37fe9..e2087f82 100644 --- a/stacks/fixturenet-eth/README.md +++ b/stacks/fixturenet-eth/README.md @@ -11,7 +11,7 @@ $ laconic-so setup-repositories --include cerc-io/go-ethereum ``` $ laconic-so build-containers --include cerc/go-ethereum,cerc/lighthouse,cerc/fixturenet-eth-geth,cerc/fixturenet-eth-lighthouse ``` -This should create several container in the local image registry: +This should create several container images in the local image registry: * cerc/go-ethereum * cerc/lighthouse From d138bb1a72091c7b41b9cf77b02b7b53f7d85da1 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Tue, 20 Dec 2022 19:15:36 -0700 Subject: [PATCH 04/11] Add warning on user mode install --- README.md | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 2deac75d..49a7f2f5 100644 --- a/README.md +++ b/README.md @@ -3,9 +3,7 @@ Stack Orchestrator allows building and deployment of a Laconic stack on a single machine with minimial prerequisites. ## Setup -### User Mode -User mode runs the orchestrator from a "binary" single-file release and does not require special Python environment setup. Use this mode unless you plan to make changes to the orchestrator source code. -#### Prerequisites +### Prerequisites Stack Orchestrator is a Python3 CLI tool that runs on any OS with Python3 and Docker. Tested on: Ubuntu 20/22. Ensure that the following are already installed: @@ -30,7 +28,13 @@ Ensure that the following are already installed: # see https://docs.docker.com/compose/install/linux/#install-the-plugin-manually for further details # or to install for all users. ``` -#### Install + +### User Mode Install + +User mode runs the orchestrator from a "binary" single-file release and does not require special Python environment setup. Use this mode unless you plan to make changes to the orchestrator source code. + +*NOTE: User Mode is currently broken, use "Developer mode" described below for now.* + 1. Download the latest release from [this page](https://github.com/cerc-io/stack-orchestrator/tags), into a suitable directory (e.g. `~/bin`): ``` $ cd ~/bin @@ -56,7 +60,7 @@ Ensure that the following are already installed: deploy-system deploy a stack setup-repositories git clone the set of repositories required to build... ``` -### Developer mode +### Developer mode Install Suitable for developers either modifying or debugging the orchestrator Python code: #### Prerequisites In addition to the binary install prerequisites listed above, the following are required: From 2c2df6ebbda84eebd44d8f1727a261e0ddc92da1 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Tue, 20 Dec 2022 20:37:33 -0700 Subject: [PATCH 05/11] Remove extraneous repo This repo was included erroneously in the original move from old-stack-orchestrator. --- app/data/repository-list.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/app/data/repository-list.txt b/app/data/repository-list.txt index 72d778b7..3da6b4f6 100644 --- a/app/data/repository-list.txt +++ b/app/data/repository-list.txt @@ -1,4 +1,3 @@ -vulcanize/ops cerc-io/ipld-eth-db cerc-io/go-ethereum cerc-io/ipld-eth-server From e7296c2ce324b037ec15924a7fee7a3f1e4faf54 Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Wed, 21 Dec 2022 09:59:40 -0600 Subject: [PATCH 06/11] Add tx-spammer (#74) * Add tx-spammer * Update README --- app/data/container-image-list.txt | 1 + app/data/pod-list.txt | 1 + app/data/repository-list.txt | 1 + compose/docker-compose-tx-spammer.yml | 18 ++++++++ config/tx-spammer/tx-spammer.env | 2 + container-build/cerc-tx-spammer/build.sh | 3 ++ stacks/fixturenet-eth/README.md | 58 ++++++++++++++---------- 7 files changed, 60 insertions(+), 24 deletions(-) create mode 100644 compose/docker-compose-tx-spammer.yml create mode 100644 config/tx-spammer/tx-spammer.env create mode 100755 container-build/cerc-tx-spammer/build.sh diff --git a/app/data/container-image-list.txt b/app/data/container-image-list.txt index ba70e6f6..12db6164 100644 --- a/app/data/container-image-list.txt +++ b/app/data/container-image-list.txt @@ -17,3 +17,4 @@ cerc/test-container cerc/eth-probe cerc/builder-js cerc/keycloak +cerc/tx-spammer diff --git a/app/data/pod-list.txt b/app/data/pod-list.txt index a15507de..659eeb10 100644 --- a/app/data/pod-list.txt +++ b/app/data/pod-list.txt @@ -14,3 +14,4 @@ watcher-mobymask test eth-probe keycloak +tx-spammer diff --git a/app/data/repository-list.txt b/app/data/repository-list.txt index 3da6b4f6..c73ce29b 100644 --- a/app/data/repository-list.txt +++ b/app/data/repository-list.txt @@ -13,3 +13,4 @@ cerc-io/mobymask-watcher cerc-io/watcher-ts vulcanize/assemblyscript cerc-io/eth-probe +cerc-io/tx-spammer diff --git a/compose/docker-compose-tx-spammer.yml b/compose/docker-compose-tx-spammer.yml new file mode 100644 index 00000000..2d203b10 --- /dev/null +++ b/compose/docker-compose-tx-spammer.yml @@ -0,0 +1,18 @@ +version: '3.2' + +services: + tx-spammer: + restart: always + image: cerc/tx-spammer:local + env_file: + - ../config/tx-spammer/tx-spammer.env + environment: + ACCOUNTS_CSV_URL: http://fixturenet-eth-bootnode-geth:9898/accounts.csv + ETH_HTTP_PATH: http://fixturenet-eth-geth-1:8545 + LOG_LEVEL: debug + SPAMMER_COMMAND: autoSend + depends_on: + fixturenet-eth-bootnode-geth: + condition: service_started + fixturenet-eth-geth-1: + condition: service_healthy diff --git a/config/tx-spammer/tx-spammer.env b/config/tx-spammer/tx-spammer.env new file mode 100644 index 00000000..35534265 --- /dev/null +++ b/config/tx-spammer/tx-spammer.env @@ -0,0 +1,2 @@ +ETH_CALL_FREQ=1000 +ETH_SEND_FREQ=1000 diff --git a/container-build/cerc-tx-spammer/build.sh b/container-build/cerc-tx-spammer/build.sh new file mode 100755 index 00000000..b6752dd7 --- /dev/null +++ b/container-build/cerc-tx-spammer/build.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +# Build cerc/tx-spammer +docker build -t cerc/tx-spammer:local ${CERC_REPO_BASE_DIR}/tx-spammer diff --git a/stacks/fixturenet-eth/README.md b/stacks/fixturenet-eth/README.md index e2087f82..bf77c25c 100644 --- a/stacks/fixturenet-eth/README.md +++ b/stacks/fixturenet-eth/README.md @@ -32,6 +32,15 @@ Waiting for beacon phase0 .... DONE! Waiting for beacon altair .... DONE! Waiting for beacon bellatrix pre-merge .... DONE! Waiting for beacon bellatrix merge .... DONE! + +$ docker ps -f 'name=laconic' --format 'table {{.Names}}\t{{.Ports}}' | cut -d'-' -f3- | sort +NAMES PORTS +fixturenet-eth-bootnode-geth-1 8545-8546/tcp, 30303/udp, 0.0.0.0:55847->9898/tcp, 0.0.0.0:55848->30303/tcp +fixturenet-eth-bootnode-lighthouse-1 +fixturenet-eth-geth-1-1 8546/tcp, 30303/tcp, 30303/udp, 0.0.0.0:55851->8545/tcp +fixturenet-eth-geth-2-1 8545-8546/tcp, 30303/tcp, 30303/udp +fixturenet-eth-lighthouse-1-1 0.0.0.0:55858->8001/tcp +fixturenet-eth-lighthouse-2-1 ``` ## Additional pieces @@ -43,18 +52,19 @@ Several other containers can used with the basic `fixturenet-eth`: * `ipld-eth-beacon-db` and `ipld-eth-beacon-indexer` (for indexing Beacon chain blocks) * `eth-probe` (captures eth1 tx gossip) * `keycloak` (nginx proxy with keycloak auth for API authentication) +* `tx-spammer` (generates and sends automated transactions to the fixturenet) It is not necessary to use them all at once, but a complete example follows: ``` # Setup -$ laconic-so setup-repositories --include cerc-io/go-ethereum,cerc-io/ipld-eth-db,cerc-io/ipld-eth-server,cerc-io/ipld-eth-beacon-db,cerc-io/ipld-eth-beacon-indexer,cerc-io/eth-probe +$ laconic-so setup-repositories --include cerc-io/go-ethereum,cerc-io/ipld-eth-db,cerc-io/ipld-eth-server,cerc-io/ipld-eth-beacon-db,cerc-io/ipld-eth-beacon-indexer,cerc-io/eth-probe,cerc-io/tx-spammer # Build -$ laconic-so build-containers --include cerc/go-ethereum,cerc/lighthouse,cerc/fixturenet-eth-geth,cerc/fixturenet-eth-lighthouse,cerc/ipld-eth-db,cerc/ipld-eth-server,cerc/ipld-eth-beacon-db,cerc/ipld-eth-beacon-indexer,cerc/eth-probe,cerc/keycloak +$ laconic-so build-containers --include cerc/go-ethereum,cerc/lighthouse,cerc/fixturenet-eth-geth,cerc/fixturenet-eth-lighthouse,cerc/ipld-eth-db,cerc/ipld-eth-server,cerc/ipld-eth-beacon-db,cerc/ipld-eth-beacon-indexer,cerc/eth-probe,cerc/keycloak,cerc/tx-spammer # Deploy -$ laconic-so deploy-system --include db,fixturenet-eth,ipld-eth-server,ipld-eth-beacon-db,ipld-eth-beacon-indexer,eth-probe,keycloak up +$ laconic-so deploy-system --include db,fixturenet-eth,ipld-eth-server,ipld-eth-beacon-db,ipld-eth-beacon-indexer,eth-probe,keycloak,tx-spammer up # Status @@ -65,24 +75,24 @@ Waiting for beacon altair.... DONE! Waiting for beacon bellatrix pre-merge.... DONE! Waiting for beacon bellatrix merge.... DONE! -$ docker ps -f 'name=laconic' -CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES -fe60af64140c cerc/ipld-eth-beacon-indexer:local "./entrypoint.sh" 4 minutes ago Up 3 minutes laconic-7ca8f4a970d7999a235e0ee27588a5ab-ipld-eth-beacon-indexer-1 -015583b318c2 cerc/eth-probe:local "/app/run.sh" 4 minutes ago Up 3 minutes laconic-7ca8f4a970d7999a235e0ee27588a5ab-eth-probe-probe-1 -180b03993b1e nginx:1.23-alpine "/docker-entrypoint.…" 4 minutes ago Up 4 minutes 0.0.0.0:54162->80/tcp laconic-7ca8f4a970d7999a235e0ee27588a5ab-keycloak-nginx-1 -2e963e3c4b44 cerc/ipld-eth-server:local "/app/entrypoint.sh" 4 minutes ago Up 4 minutes 127.0.0.1:8081-8082->8081-8082/tcp laconic-7ca8f4a970d7999a235e0ee27588a5ab-ipld-eth-server-1 -913cefc5cecf cerc/fixturenet-eth-lighthouse:local "/opt/testnet/run.sh" 4 minutes ago Up 4 minutes (healthy) 0.0.0.0:54161->8001/tcp laconic-7ca8f4a970d7999a235e0ee27588a5ab-fixturenet-eth-lighthouse-1-1 -c0659d0204ff cerc/fixturenet-eth-lighthouse:local "/opt/testnet/run.sh" 4 minutes ago Up 3 minutes (healthy) laconic-7ca8f4a970d7999a235e0ee27588a5ab-fixturenet-eth-lighthouse-2-1 -1636ed1013a6 cerc/keycloak:local "/opt/keycloak/bin/k…" 4 minutes ago Up 4 minutes 8443/tcp, 0.0.0.0:54160->8080/tcp laconic-7ca8f4a970d7999a235e0ee27588a5ab-keycloak-1 -439b017d75c1 cerc/ipld-eth-db:local "/app/startup_script…" 4 minutes ago Up 4 minutes laconic-7ca8f4a970d7999a235e0ee27588a5ab-migrations-1 -0a2c740a8e12 cerc/eth-probe:local "/app/run.sh" 4 minutes ago Up 4 minutes (healthy) laconic-7ca8f4a970d7999a235e0ee27588a5ab-eth-probe-mq-1 -35a816e0bac1 cerc/fixturenet-eth-geth:local "/opt/testnet/run.sh" 4 minutes ago Up 4 minutes (healthy) 8546/tcp, 30303/tcp, 30303/udp, 0.0.0.0:54154->8545/tcp laconic-7ca8f4a970d7999a235e0ee27588a5ab-fixturenet-eth-geth-1-1 -6691e5988519 cerc/fixturenet-eth-geth:local "/opt/testnet/run.sh" 4 minutes ago Up 4 minutes (healthy) 8545-8546/tcp, 30303/tcp, 30303/udp laconic-7ca8f4a970d7999a235e0ee27588a5ab-fixturenet-eth-geth-2-1 -06602dc7e3d0 timescale/timescaledb:latest-pg14 "docker-entrypoint.s…" 4 minutes ago Up 4 minutes (healthy) 0.0.0.0:54153->5432/tcp laconic-7ca8f4a970d7999a235e0ee27588a5ab-eth-probe-db-1 -91d73ec45b97 cerc/ipld-eth-beacon-db:local "docker-entrypoint.s…" 4 minutes ago Up 4 minutes (healthy) 127.0.0.1:8076->5432/tcp laconic-7ca8f4a970d7999a235e0ee27588a5ab-ipld-eth-beacon-db-1 -48459978329c postgres:14-alpine "docker-entrypoint.s…" 4 minutes ago Up 4 minutes (healthy) 0.0.0.0:54152->5432/tcp laconic-7ca8f4a970d7999a235e0ee27588a5ab-keycloak-db-1 -f48169806b54 timescale/timescaledb:2.8.1-pg14 "docker-entrypoint.s…" 4 minutes ago Up 4 minutes (healthy) 127.0.0.1:8077->5432/tcp laconic-7ca8f4a970d7999a235e0ee27588a5ab-ipld-eth-db-1 -56686c4e004f cerc/fixturenet-eth-geth:local "/opt/testnet/run.sh" 4 minutes ago Up 4 minutes 8545-8546/tcp, 30303/udp, 0.0.0.0:54151->9898/tcp, 0.0.0.0:54150->30303/tcp laconic-7ca8f4a970d7999a235e0ee27588a5ab-fixturenet-eth-bootnode-geth-1 -95f073c5e956 cerc/fixturenet-eth-lighthouse:local "/opt/testnet/run.sh" 4 minutes ago Up 4 minutes laconic-7ca8f4a970d7999a235e0ee27588a5ab-fixturenet-eth-bootnode-lighthouse-1 -``` - +$ docker ps -f 'name=laconic' --format 'table {{.Names}}\t{{.Ports}}' | cut -d'-' -f3- | sort +NAMES PORTS +eth-probe-db-1 0.0.0.0:55849->5432/tcp +eth-probe-mq-1 +eth-probe-probe-1 +fixturenet-eth-bootnode-geth-1 8545-8546/tcp, 30303/udp, 0.0.0.0:55847->9898/tcp, 0.0.0.0:55848->30303/tcp +fixturenet-eth-bootnode-lighthouse-1 +fixturenet-eth-geth-1-1 8546/tcp, 30303/tcp, 30303/udp, 0.0.0.0:55851->8545/tcp +fixturenet-eth-geth-2-1 8545-8546/tcp, 30303/tcp, 30303/udp +fixturenet-eth-lighthouse-1-1 0.0.0.0:55858->8001/tcp +fixturenet-eth-lighthouse-2-1 +ipld-eth-beacon-db-1 127.0.0.1:8076->5432/tcp +ipld-eth-beacon-indexer-1 +ipld-eth-db-1 127.0.0.1:8077->5432/tcp +ipld-eth-server-1 127.0.0.1:8081-8082->8081-8082/tcp +keycloak-1 8443/tcp, 0.0.0.0:55857->8080/tcp +keycloak-db-1 0.0.0.0:55850->5432/tcp +keycloak-nginx-1 0.0.0.0:55859->80/tcp +migrations-1 +tx-spammer-1 +``` \ No newline at end of file From a7221ddc24130b1ae554a3b7c87b287a2f64909f Mon Sep 17 00:00:00 2001 From: prathamesh0 <42446521+prathamesh0@users.noreply.github.com> Date: Thu, 22 Dec 2022 00:03:39 -0600 Subject: [PATCH 07/11] Add a local stack for ERC20 watcher (#80) * Add a healthcheck for ipld-eth-server * Update ipld-eth-server ports in mobymask watcher config * Update clique period to 5s in geth-foundry config * Add a stack for erc20 watcher * Add a demo for erc20 watcher * Upgrade watcher-ts version * Update instructions * Pass option to pull while setting up repos --- app/data/container-image-list.txt | 1 + app/data/pod-list.txt | 1 + app/deploy_system.py | 2 +- compose/docker-compose-ipld-eth-server.yml | 7 +- compose/docker-compose-watcher-erc20.yml | 49 ++++++ config/watcher-erc20/erc20-watcher.toml | 41 +++++ config/watcher-mobymask/mobymask-watcher.toml | 4 +- .../genesis-automine.json | 2 +- container-build/cerc-watcher-erc20/Dockerfile | 13 ++ container-build/cerc-watcher-erc20/build.sh | 7 + stacks/erc20/README.md | 157 ++++++++++++++++++ stacks/erc20/stack.yml | 18 ++ 12 files changed, 297 insertions(+), 5 deletions(-) create mode 100644 compose/docker-compose-watcher-erc20.yml create mode 100644 config/watcher-erc20/erc20-watcher.toml create mode 100644 container-build/cerc-watcher-erc20/Dockerfile create mode 100755 container-build/cerc-watcher-erc20/build.sh create mode 100644 stacks/erc20/README.md create mode 100644 stacks/erc20/stack.yml diff --git a/app/data/container-image-list.txt b/app/data/container-image-list.txt index 12db6164..19ee8da6 100644 --- a/app/data/container-image-list.txt +++ b/app/data/container-image-list.txt @@ -13,6 +13,7 @@ cerc/laconic-cns-cli cerc/fixturenet-eth-geth cerc/fixturenet-eth-lighthouse cerc/watcher-mobymask +cerc/watcher-erc20 cerc/test-container cerc/eth-probe cerc/builder-js diff --git a/app/data/pod-list.txt b/app/data/pod-list.txt index 659eeb10..0e02bc4c 100644 --- a/app/data/pod-list.txt +++ b/app/data/pod-list.txt @@ -11,6 +11,7 @@ laconicd fixturenet-laconicd fixturenet-eth watcher-mobymask +watcher-erc20 test eth-probe keycloak diff --git a/app/deploy_system.py b/app/deploy_system.py index b3fb618a..91232094 100644 --- a/app/deploy_system.py +++ b/app/deploy_system.py @@ -73,7 +73,7 @@ def command(ctx, include, exclude, cluster, command, services): docker = DockerClient(compose_files=compose_files, compose_project_name=cluster) services_list = list(services) or None - + if not dry_run: if command == "up": if verbose: diff --git a/compose/docker-compose-ipld-eth-server.yml b/compose/docker-compose-ipld-eth-server.yml index 21dfd2b2..ee9c8885 100644 --- a/compose/docker-compose-ipld-eth-server.yml +++ b/compose/docker-compose-ipld-eth-server.yml @@ -30,4 +30,9 @@ services: ports: - "127.0.0.1:8081:8081" - "127.0.0.1:8082:8082" - + healthcheck: + test: ["CMD", "nc", "-v", "localhost", "8081"] + interval: 20s + timeout: 5s + retries: 15 + start_period: 5s diff --git a/compose/docker-compose-watcher-erc20.yml b/compose/docker-compose-watcher-erc20.yml new file mode 100644 index 00000000..18beacb0 --- /dev/null +++ b/compose/docker-compose-watcher-erc20.yml @@ -0,0 +1,49 @@ +version: '3.2' + +services: + + watcher-db: + restart: unless-stopped + image: postgres:14-alpine + environment: + - POSTGRES_USER=vdbm + - POSTGRES_MULTIPLE_DATABASES=erc20-watcher,erc20-watcher-job-queue + - POSTGRES_EXTENSION=erc20-watcher-job-queue:pgcrypto + - POSTGRES_PASSWORD=password + volumes: + - ../config/postgresql/multiple-postgressql-databases.sh:/docker-entrypoint-initdb.d/multiple-postgressql-databases.sh + - watcher_db_data:/var/lib/postgresql/data + ports: + - "0.0.0.0:15432:5432" + healthcheck: + test: ["CMD", "nc", "-v", "localhost", "5432"] + interval: 20s + timeout: 5s + retries: 15 + start_period: 10s + + erc20-watcher: + restart: unless-stopped + depends_on: + ipld-eth-server: + condition: service_healthy + watcher-db: + condition: service_healthy + image: cerc/watcher-erc20:local + environment: + - ETH_RPC_URL=http://go-ethereum:8545 + command: ["sh", "-c", "yarn server"] + volumes: + - ../config/watcher-erc20/erc20-watcher.toml:/app/packages/erc20-watcher/environments/local.toml + ports: + - "0.0.0.0:3001:3001" + - "0.0.0.0:9001:9001" + healthcheck: + test: ["CMD", "nc", "-v", "localhost", "3001"] + interval: 20s + timeout: 5s + retries: 15 + start_period: 5s + +volumes: + watcher_db_data: diff --git a/config/watcher-erc20/erc20-watcher.toml b/config/watcher-erc20/erc20-watcher.toml new file mode 100644 index 00000000..5b8b4ce5 --- /dev/null +++ b/config/watcher-erc20/erc20-watcher.toml @@ -0,0 +1,41 @@ +[server] + host = "0.0.0.0" + port = 3001 + mode = "storage" + kind = "lazy" + +[metrics] + host = "127.0.0.1" + port = 9000 + [metrics.gql] + port = 9001 + +[database] + type = "postgres" + host = "watcher-db" + port = 5432 + database = "erc20-watcher" + username = "vdbm" + password = "password" + synchronize = true + logging = false + maxQueryExecutionTime = 100 + +[upstream] + [upstream.ethServer] + gqlApiEndpoint = "http://ipld-eth-server:8082/graphql" + rpcProviderEndpoint = "http://ipld-eth-server:8081" + + [upstream.cache] + name = "requests" + enabled = false + deleteOnStart = false + +[jobQueue] + dbConnectionString = "postgres://vdbm:password@watcher-db:5432/erc20-watcher-job-queue" + maxCompletionLagInSecs = 300 + jobDelayInMilliSecs = 100 + eventsInBatch = 50 + blockDelayInMilliSecs = 2000 + prefetchBlocksInMem = true + prefetchBlockCount = 10 diff --git a/config/watcher-mobymask/mobymask-watcher.toml b/config/watcher-mobymask/mobymask-watcher.toml index 4dba25b9..3e9b2db5 100644 --- a/config/watcher-mobymask/mobymask-watcher.toml +++ b/config/watcher-mobymask/mobymask-watcher.toml @@ -37,8 +37,8 @@ [upstream] [upstream.ethServer] - gqlApiEndpoint = "http://ipld-eth-server:8083/graphql" - rpcProviderEndpoint = "http://ipld-eth-server:8082" + gqlApiEndpoint = "http://ipld-eth-server:8082/graphql" + rpcProviderEndpoint = "http://ipld-eth-server:8081" blockDelayInMilliSecs = 60000 [upstream.cache] diff --git a/container-build/cerc-go-ethereum-foundry/genesis-automine.json b/container-build/cerc-go-ethereum-foundry/genesis-automine.json index a00a614e..6cbf7f26 100644 --- a/container-build/cerc-go-ethereum-foundry/genesis-automine.json +++ b/container-build/cerc-go-ethereum-foundry/genesis-automine.json @@ -10,7 +10,7 @@ "petersburgBlock": 0, "istanbulBlock": 0, "clique": { - "period": 2, + "period": 5, "epoch": 3000 } }, diff --git a/container-build/cerc-watcher-erc20/Dockerfile b/container-build/cerc-watcher-erc20/Dockerfile new file mode 100644 index 00000000..5ec94987 --- /dev/null +++ b/container-build/cerc-watcher-erc20/Dockerfile @@ -0,0 +1,13 @@ +FROM node:16.17.1-alpine3.16 + +RUN apk --update --no-cache add git python3 alpine-sdk + +WORKDIR /app + +COPY . . + +RUN echo "Building watcher-ts" && \ + git checkout v0.2.19 && \ + yarn && yarn build + +WORKDIR /app/packages/erc20-watcher diff --git a/container-build/cerc-watcher-erc20/build.sh b/container-build/cerc-watcher-erc20/build.sh new file mode 100755 index 00000000..b800b73d --- /dev/null +++ b/container-build/cerc-watcher-erc20/build.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +# Build cerc/watcher-erc20 + +# See: https://stackoverflow.com/a/246128/1701505 +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +docker build -t cerc/watcher-erc20:local -f ${SCRIPT_DIR}/Dockerfile ${CERC_REPO_BASE_DIR}/watcher-ts diff --git a/stacks/erc20/README.md b/stacks/erc20/README.md new file mode 100644 index 00000000..eec3d9f4 --- /dev/null +++ b/stacks/erc20/README.md @@ -0,0 +1,157 @@ +# ERC20 Watcher + +Instructions to deploy a local ERC20 watcher stack (core + watcher) for demonstration and testing purposes using [laconic-stack-orchestrator](../../README.md#setup) + +## Setup + +* Clone / pull required repositories: + + ```bash + $ laconic-so setup-repositories --include cerc-io/go-ethereum,cerc-io/ipld-eth-db,cerc-io/ipld-eth-server,cerc-io/watcher-ts --pull + ``` + +* Build the core and watcher container images: + + ```bash + $ laconic-so build-containers --include cerc/go-ethereum,cerc/go-ethereum-foundry,cerc/ipld-eth-db,cerc/ipld-eth-server,cerc/watcher-erc20 + ``` + + This should create the required docker images in the local image registry. + +* Deploy the stack: + + ```bash + $ laconic-so deploy-system --include db,go-ethereum-foundry,ipld-eth-server,watcher-erc20 up + ``` + +## Demo + +* Find the watcher container's id using `docker ps` and export it for later use: + + ```bash + $ export CONTAINER_ID= + ``` + +* Deploy an ERC20 token: + + ```bash + $ docker exec $CONTAINER_ID yarn token:deploy:docker + ``` + + Export the address of the deployed token to a shell variable for later use: + + ```bash + $ export TOKEN_ADDRESS= + ``` + +* Open `http://localhost:3001/graphql` (GraphQL Playground) in a browser window + +* Connect MetaMask to `http://localhost:8545` (with chain ID `99`) + +* Add the deployed token as an asset in MetaMask and check that the initial balance is zero + +* Export your MetaMask account (second account) address to a shell variable for later use: + + ```bash + $ export RECIPIENT_ADDRESS= + ``` + +* To get the primary account's address, run: + + ```bash + $ docker exec $CONTAINER_ID yarn account:docker + ``` + +* To get the current block hash at any time, run: + + ```bash + $ docker exec $CONTAINER_ID yarn block:latest:docker + ``` + +* Fire a GQL query in the playground to get the name, symbol and total supply of the deployed token: + + ```graphql + query { + name( + blockHash: "LATEST_BLOCK_HASH" + token: "TOKEN_ADDRESS" + ) { + value + proof { + data + } + } + + symbol( + blockHash: "LATEST_BLOCK_HASH" + token: "TOKEN_ADDRESS" + ) { + value + proof { + data + } + } + + totalSupply( + blockHash: "LATEST_BLOCK_HASH" + token: "TOKEN_ADDRESS" + ) { + value + proof { + data + } + } + } + ``` + +* Fire the following query to get balances for the primary and the recipient account at the latest block hash: + + ```graphql + query { + fromBalanceOf: balanceOf( + blockHash: "LATEST_BLOCK_HASH" + token: "TOKEN_ADDRESS", + # primary account having all the balance initially + owner: "PRIMARY_ADDRESS" + ) { + value + proof { + data + } + } + toBalanceOf: balanceOf( + blockHash: "LATEST_BLOCK_HASH" + token: "TOKEN_ADDRESS", + owner: "RECIPIENT_ADDRESS" + ) { + value + proof { + data + } + } + } + ``` + + * The initial balance for the primary account should be `1000000000000000000000` + * The initial balance for the recipient should be `0` + +* Transfer tokens to the recipient account: + + ```bash + $ docker exec $CONTAINER_ID yarn token:transfer:docker --token $TOKEN_ADDRESS --to $RECIPIENT_ADDRESS --amount 100 + ``` + +* Fire the above GQL query again with the latest block hash to get updated balances for the primary (`from`) and the recipient (`to`) account: + + * The balance for the primary account should be reduced by the transfer amount (`100`) + * The balance for the recipient account should be equal to the transfer amount (`100`) + +* Transfer funds between different accounts using MetaMask and use the playground to query the balance before and after the transfer. + +## Clean up + +* To stop all the services running in background run: + + ```bash + $ laconic-so deploy-system --include db,go-ethereum-foundry,ipld-eth-server,watcher-erc20 down + ``` diff --git a/stacks/erc20/stack.yml b/stacks/erc20/stack.yml new file mode 100644 index 00000000..e9b0eb6d --- /dev/null +++ b/stacks/erc20/stack.yml @@ -0,0 +1,18 @@ +version: "1.0" +name: erc20-watcher +repos: + - cerc-io/go-ethereum + - cerc-io/ipld-eth-db + - cerc-io/ipld-eth-server + - cerc-io/watcher-ts +containers: + - cerc/go-ethereum + - cerc/go-ethereum-foundry + - cerc/ipld-eth-db + - cerc/ipld-eth-server + - cerc/watcher-erc20 +pods: + - go-ethereum-foundry + - db + - ipld-eth-server + - watcher-erc20 From 27c3efabe2741d6eb908bc5986b07de75be2c4a7 Mon Sep 17 00:00:00 2001 From: prathamesh0 <42446521+prathamesh0@users.noreply.github.com> Date: Thu, 22 Dec 2022 13:13:59 +0530 Subject: [PATCH 08/11] Add a local stack for ERC721 watcher (#81) * Add a stack for erc721 watcher * Add a demo * Avoid watcher db service and volume name conflicts across watchers * Avoid host port conflicts across watchers * Avoid watcher-db port conflicts --- app/data/container-image-list.txt | 1 + app/data/pod-list.txt | 1 + compose/docker-compose-watcher-erc20.yml | 16 +- compose/docker-compose-watcher-erc721.yml | 49 ++++ config/watcher-erc20/erc20-watcher.toml | 4 +- config/watcher-erc721/erc721-watcher.toml | 56 +++++ .../cerc-watcher-erc721/Dockerfile | 13 ++ container-build/cerc-watcher-erc721/build.sh | 7 + stacks/erc721/README.md | 214 ++++++++++++++++++ stacks/erc721/stack.yml | 18 ++ 10 files changed, 369 insertions(+), 10 deletions(-) create mode 100644 compose/docker-compose-watcher-erc721.yml create mode 100644 config/watcher-erc721/erc721-watcher.toml create mode 100644 container-build/cerc-watcher-erc721/Dockerfile create mode 100755 container-build/cerc-watcher-erc721/build.sh create mode 100644 stacks/erc721/README.md create mode 100644 stacks/erc721/stack.yml diff --git a/app/data/container-image-list.txt b/app/data/container-image-list.txt index 19ee8da6..4c7c052b 100644 --- a/app/data/container-image-list.txt +++ b/app/data/container-image-list.txt @@ -14,6 +14,7 @@ cerc/fixturenet-eth-geth cerc/fixturenet-eth-lighthouse cerc/watcher-mobymask cerc/watcher-erc20 +cerc/watcher-erc721 cerc/test-container cerc/eth-probe cerc/builder-js diff --git a/app/data/pod-list.txt b/app/data/pod-list.txt index 0e02bc4c..42c46acf 100644 --- a/app/data/pod-list.txt +++ b/app/data/pod-list.txt @@ -12,6 +12,7 @@ fixturenet-laconicd fixturenet-eth watcher-mobymask watcher-erc20 +watcher-erc721 test eth-probe keycloak diff --git a/compose/docker-compose-watcher-erc20.yml b/compose/docker-compose-watcher-erc20.yml index 18beacb0..82e76120 100644 --- a/compose/docker-compose-watcher-erc20.yml +++ b/compose/docker-compose-watcher-erc20.yml @@ -2,7 +2,7 @@ version: '3.2' services: - watcher-db: + erc20-watcher-db: restart: unless-stopped image: postgres:14-alpine environment: @@ -12,9 +12,9 @@ services: - POSTGRES_PASSWORD=password volumes: - ../config/postgresql/multiple-postgressql-databases.sh:/docker-entrypoint-initdb.d/multiple-postgressql-databases.sh - - watcher_db_data:/var/lib/postgresql/data + - erc20_watcher_db_data:/var/lib/postgresql/data ports: - - "0.0.0.0:15432:5432" + - "0.0.0.0:15433:5432" healthcheck: test: ["CMD", "nc", "-v", "localhost", "5432"] interval: 20s @@ -27,7 +27,7 @@ services: depends_on: ipld-eth-server: condition: service_healthy - watcher-db: + erc20-watcher-db: condition: service_healthy image: cerc/watcher-erc20:local environment: @@ -36,14 +36,14 @@ services: volumes: - ../config/watcher-erc20/erc20-watcher.toml:/app/packages/erc20-watcher/environments/local.toml ports: - - "0.0.0.0:3001:3001" - - "0.0.0.0:9001:9001" + - "0.0.0.0:3002:3001" + - "0.0.0.0:9002:9001" healthcheck: - test: ["CMD", "nc", "-v", "localhost", "3001"] + test: ["CMD", "nc", "-v", "localhost", "3002"] interval: 20s timeout: 5s retries: 15 start_period: 5s volumes: - watcher_db_data: + erc20_watcher_db_data: diff --git a/compose/docker-compose-watcher-erc721.yml b/compose/docker-compose-watcher-erc721.yml new file mode 100644 index 00000000..cdfcdbc7 --- /dev/null +++ b/compose/docker-compose-watcher-erc721.yml @@ -0,0 +1,49 @@ +version: '3.2' + +services: + + erc721-watcher-db: + restart: unless-stopped + image: postgres:14-alpine + environment: + - POSTGRES_USER=vdbm + - POSTGRES_MULTIPLE_DATABASES=erc721-watcher,erc721-watcher-job-queue + - POSTGRES_EXTENSION=erc721-watcher-job-queue:pgcrypto + - POSTGRES_PASSWORD=password + volumes: + - ../config/postgresql/multiple-postgressql-databases.sh:/docker-entrypoint-initdb.d/multiple-postgressql-databases.sh + - erc721_watcher_db_data:/var/lib/postgresql/data + ports: + - "0.0.0.0:15434:5432" + healthcheck: + test: ["CMD", "nc", "-v", "localhost", "5432"] + interval: 20s + timeout: 5s + retries: 15 + start_period: 10s + + erc721-watcher: + restart: unless-stopped + depends_on: + ipld-eth-server: + condition: service_healthy + erc721-watcher-db: + condition: service_healthy + image: cerc/watcher-erc721:local + environment: + - ETH_RPC_URL=http://go-ethereum:8545 + command: ["sh", "-c", "yarn server"] + volumes: + - ../config/watcher-erc721/erc721-watcher.toml:/app/packages/erc721-watcher/environments/local.toml + ports: + - "0.0.0.0:3009:3009" + - "0.0.0.0:9003:9001" + healthcheck: + test: ["CMD", "nc", "-v", "localhost", "3009"] + interval: 20s + timeout: 5s + retries: 15 + start_period: 5s + +volumes: + erc721_watcher_db_data: diff --git a/config/watcher-erc20/erc20-watcher.toml b/config/watcher-erc20/erc20-watcher.toml index 5b8b4ce5..20773321 100644 --- a/config/watcher-erc20/erc20-watcher.toml +++ b/config/watcher-erc20/erc20-watcher.toml @@ -12,7 +12,7 @@ [database] type = "postgres" - host = "watcher-db" + host = "erc20-watcher-db" port = 5432 database = "erc20-watcher" username = "vdbm" @@ -32,7 +32,7 @@ deleteOnStart = false [jobQueue] - dbConnectionString = "postgres://vdbm:password@watcher-db:5432/erc20-watcher-job-queue" + dbConnectionString = "postgres://vdbm:password@erc20-watcher-db:5432/erc20-watcher-job-queue" maxCompletionLagInSecs = 300 jobDelayInMilliSecs = 100 eventsInBatch = 50 diff --git a/config/watcher-erc721/erc721-watcher.toml b/config/watcher-erc721/erc721-watcher.toml new file mode 100644 index 00000000..8e877411 --- /dev/null +++ b/config/watcher-erc721/erc721-watcher.toml @@ -0,0 +1,56 @@ +[server] + host = "0.0.0.0" + port = 3009 + kind = "lazy" + + # Checkpointing state. + checkpointing = true + + # Checkpoint interval in number of blocks. + checkpointInterval = 2000 + + # Enable state creation + enableState = true + + # Boolean to filter logs by contract. + filterLogs = false + + # Max block range for which to return events in eventsInRange GQL query. + # Use -1 for skipping check on block range. + maxEventsBlockRange = 1000 + +[metrics] + host = "127.0.0.1" + port = 9000 + [metrics.gql] + port = 9001 + +[database] + type = "postgres" + host = "erc721-watcher-db" + port = 5432 + database = "erc721-watcher" + username = "vdbm" + password = "password" + synchronize = true + logging = false + maxQueryExecutionTime = 100 + +[upstream] + [upstream.ethServer] + gqlApiEndpoint = "http://ipld-eth-server:8082/graphql" + rpcProviderEndpoint = "http://ipld-eth-server:8081" + + [upstream.cache] + name = "requests" + enabled = false + deleteOnStart = false + +[jobQueue] + dbConnectionString = "postgres://vdbm:password@erc721-watcher-db:5432/erc721-watcher-job-queue" + maxCompletionLagInSecs = 300 + jobDelayInMilliSecs = 100 + eventsInBatch = 50 + blockDelayInMilliSecs = 2000 + prefetchBlocksInMem = true + prefetchBlockCount = 10 diff --git a/container-build/cerc-watcher-erc721/Dockerfile b/container-build/cerc-watcher-erc721/Dockerfile new file mode 100644 index 00000000..6358df9a --- /dev/null +++ b/container-build/cerc-watcher-erc721/Dockerfile @@ -0,0 +1,13 @@ +FROM node:16.17.1-alpine3.16 + +RUN apk --update --no-cache add git python3 alpine-sdk + +WORKDIR /app + +COPY . . + +RUN echo "Building watcher-ts" && \ + git checkout v0.2.19 && \ + yarn && yarn build + +WORKDIR /app/packages/erc721-watcher diff --git a/container-build/cerc-watcher-erc721/build.sh b/container-build/cerc-watcher-erc721/build.sh new file mode 100755 index 00000000..b9c9c0b8 --- /dev/null +++ b/container-build/cerc-watcher-erc721/build.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +# Build cerc/watcher-erc721 + +# See: https://stackoverflow.com/a/246128/1701505 +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +docker build -t cerc/watcher-erc721:local -f ${SCRIPT_DIR}/Dockerfile ${CERC_REPO_BASE_DIR}/watcher-ts diff --git a/stacks/erc721/README.md b/stacks/erc721/README.md new file mode 100644 index 00000000..ed5f9c80 --- /dev/null +++ b/stacks/erc721/README.md @@ -0,0 +1,214 @@ +# ERC721 Watcher + +Instructions to deploy a local ERC721 watcher stack (core + watcher) for demonstration and testing purposes using [laconic-stack-orchestrator](../../README.md#setup) + +## Setup + +* Clone / pull required repositories: + + ```bash + $ laconic-so setup-repositories --include cerc-io/go-ethereum,cerc-io/ipld-eth-db,cerc-io/ipld-eth-server,cerc-io/watcher-ts --pull + ``` + +* Build the core and watcher container images: + + ```bash + $ laconic-so build-containers --include cerc/go-ethereum,cerc/go-ethereum-foundry,cerc/ipld-eth-db,cerc/ipld-eth-server,cerc/watcher-erc721 + ``` + + This should create the required docker images in the local image registry. + +* Deploy the stack: + + ```bash + $ laconic-so deploy-system --include db,go-ethereum-foundry,ipld-eth-server,watcher-erc721 up + ``` + +## Demo + +* Find the watcher container's id using `docker ps` and export it for later use: + + ```bash + $ export CONTAINER_ID= + ``` + +* Deploy an ERC721 token: + + ```bash + $ docker exec $CONTAINER_ID yarn nft:deploy:docker + ``` + + Export the address of the deployed token to a shell variable for later use: + + ```bash + $ export NFT_ADDRESS= + ``` + +* Open `http://localhost:3009/graphql` (GraphQL Playground) in a browser window + +* Connect MetaMask to `http://localhost:8545` (with chain ID `99`) + +* Export your MetaMask account (second account) address to a shell variable for later use: + + ```bash + $ export RECIPIENT_ADDRESS= + ``` + +* To get the primary account's address, run: + + ```bash + $ docker exec $CONTAINER_ID yarn account:docker + ``` + + Export it to shell variable for later use: + + ```bash + $ export PRIMARY_ADDRESS= + ``` + +* To get the current block hash at any time, run: + + ```bash + $ docker exec $CONTAINER_ID yarn block:latest:docker + ``` + +* Fire the following GQL query (uses `eth_call`) in the playground: + + ```graphql + query { + name( + blockHash: "LATEST_BLOCK_HASH" + contractAddress: "NFT_ADDRESS" + ) { + value + proof { + data + } + } + + symbol( + blockHash: "LATEST_BLOCK_HASH" + contractAddress: "NFT_ADDRESS" + ) { + value + proof { + data + } + } + + balanceOf( + blockHash: "LATEST_BLOCK_HASH" + contractAddress: "NFT_ADDRESS" + owner: "PRIMARY_ADDRESS" + ) { + value + proof { + data + } + } + } + ``` + + Balance for the `PRIMARY_ADDRESS` should be `0` as the token is yet to be minted. + +* Fire the following GQL query (uses `storage` calls) in the playground: + + ```graphql + query { + _name( + blockHash: "LATEST_BLOCK_HASH" + contractAddress: "NFT_ADDRESS" + ) { + value + proof { + data + } + } + + _symbol( + blockHash: "LATEST_BLOCK_HASH" + contractAddress: "NFT_ADDRESS" + ) { + value + proof { + data + } + } + + _balances( + blockHash: "LATEST_BLOCK_HASH" + contractAddress: "NFT_ADDRESS" + key0: "PRIMARY_ADDRESS" + ) { + value + proof { + data + } + } + } + ``` + +* Mint the token: + + ```bash + $ docker exec $CONTAINER_ID yarn nft:mint:docker --nft $NFT_ADDRESS --to $PRIMARY_ADDRESS --token-id 1 + ``` + + Fire the GQL query above again with latest block hash. The balance should increase to `1`. + +* Get the latest block hash and run the following GQL query in the playground for `balanceOf` and `ownerOf` (`eth_call`): + + ```graphql + query { + fromBalanceOf: balanceOf( + blockHash: "LATEST_BLOCK_HASH" + contractAddress: "NFT_ADDRESS" + owner: "PRIMARY_ADDRESS" + ) { + value + proof { + data + } + } + + toBalanceOf: balanceOf( + blockHash: "LATEST_BLOCK_HASH" + contractAddress: "NFT_ADDRESS" + owner: "RECIPIENT_ADDRESS" + ) { + value + proof { + data + } + } + + ownerOf( + blockHash: "LATEST_BLOCK_HASH" + contractAddress: "NFT_ADDRESS" + tokenId: 1 + ) { + value + proof { + data + } + } + } + ``` + + Balance should be `1` for the `PRIMARY_ADDRESS`, `0` for the `RECIPIENT_ADDRESS` and owner value of the token should be equal to the `PRIMARY_ADDRESS`. + +* Transfer the token: + + ```bash + $ docker exec $CONTAINER_ID yarn nft:transfer:docker --nft $NFT_ADDRESS --from $PRIMARY_ADDRESS --to $RECIPIENT_ADDRESS --token-id 1 + ``` + + Fire the GQL query above again with the latest block hash. The token should be transferred to the recipient. + +## Clean up + +* To stop all the services running in background: + + ```bash + $ laconic-so deploy-system --include db,go-ethereum-foundry,ipld-eth-server,watcher-erc721 down + ``` diff --git a/stacks/erc721/stack.yml b/stacks/erc721/stack.yml new file mode 100644 index 00000000..85072d4f --- /dev/null +++ b/stacks/erc721/stack.yml @@ -0,0 +1,18 @@ +version: "1.0" +name: erc721-watcher +repos: + - cerc-io/go-ethereum + - cerc-io/ipld-eth-db + - cerc-io/ipld-eth-server + - cerc-io/watcher-ts +containers: + - cerc/go-ethereum + - cerc/go-ethereum-foundry + - cerc/ipld-eth-db + - cerc/ipld-eth-server + - cerc/watcher-erc721 +pods: + - go-ethereum-foundry + - db + - ipld-eth-server + - watcher-erc721 From d51508b01de197ec8577572ccd4d36731b95f9ce Mon Sep 17 00:00:00 2001 From: prathamesh0 <42446521+prathamesh0@users.noreply.github.com> Date: Thu, 22 Dec 2022 19:30:56 +0530 Subject: [PATCH 09/11] [WIP] Add a Uniswap v3 watcher-only stack (#82) * Add a stack for uniswap watcher with info app * Add deployment instructions * Use docker interactive option while loading the db dumps --- app/data/container-image-list.txt | 2 + app/data/pod-list.txt | 1 + app/data/repository-list.txt | 2 + compose/docker-compose-watcher-uniswap-v3.yml | 170 ++++++++++++++++++ .../postgresql/create-pg-stat-statements.sql | 1 + config/watcher-uniswap-v3/erc20-watcher.toml | 39 ++++ config/watcher-uniswap-v3/run.sh | 10 ++ .../watcher-uniswap-v3/uni-info-watcher.toml | 90 ++++++++++ config/watcher-uniswap-v3/uni-watcher.toml | 41 +++++ config/watcher-uniswap-v3/watch-contract.sh | 10 ++ .../cerc-uniswap-v3-info/Dockerfile | 13 ++ container-build/cerc-uniswap-v3-info/build.sh | 7 + .../cerc-watcher-uniswap-v3/Dockerfile | 11 ++ .../cerc-watcher-uniswap-v3/build.sh | 7 + stacks/erc20/README.md | 2 +- stacks/uniswap-v3/README.md | 83 +++++++++ stacks/uniswap-v3/stack.yml | 10 ++ 17 files changed, 498 insertions(+), 1 deletion(-) create mode 100644 compose/docker-compose-watcher-uniswap-v3.yml create mode 100644 config/postgresql/create-pg-stat-statements.sql create mode 100644 config/watcher-uniswap-v3/erc20-watcher.toml create mode 100755 config/watcher-uniswap-v3/run.sh create mode 100644 config/watcher-uniswap-v3/uni-info-watcher.toml create mode 100644 config/watcher-uniswap-v3/uni-watcher.toml create mode 100755 config/watcher-uniswap-v3/watch-contract.sh create mode 100644 container-build/cerc-uniswap-v3-info/Dockerfile create mode 100755 container-build/cerc-uniswap-v3-info/build.sh create mode 100644 container-build/cerc-watcher-uniswap-v3/Dockerfile create mode 100755 container-build/cerc-watcher-uniswap-v3/build.sh create mode 100644 stacks/uniswap-v3/README.md create mode 100644 stacks/uniswap-v3/stack.yml diff --git a/app/data/container-image-list.txt b/app/data/container-image-list.txt index 4c7c052b..c697fe21 100644 --- a/app/data/container-image-list.txt +++ b/app/data/container-image-list.txt @@ -15,6 +15,8 @@ cerc/fixturenet-eth-lighthouse cerc/watcher-mobymask cerc/watcher-erc20 cerc/watcher-erc721 +cerc/watcher-uniswap-v3 +cerc/uniswap-v3-info cerc/test-container cerc/eth-probe cerc/builder-js diff --git a/app/data/pod-list.txt b/app/data/pod-list.txt index 42c46acf..e5cadd5a 100644 --- a/app/data/pod-list.txt +++ b/app/data/pod-list.txt @@ -13,6 +13,7 @@ fixturenet-eth watcher-mobymask watcher-erc20 watcher-erc721 +watcher-uniswap-v3 test eth-probe keycloak diff --git a/app/data/repository-list.txt b/app/data/repository-list.txt index c73ce29b..cdf40b25 100644 --- a/app/data/repository-list.txt +++ b/app/data/repository-list.txt @@ -11,6 +11,8 @@ cerc-io/laconic-sdk cerc-io/laconic-registry-cli cerc-io/mobymask-watcher cerc-io/watcher-ts +vulcanize/uniswap-watcher-ts +vulcanize/uniswap-v3-info vulcanize/assemblyscript cerc-io/eth-probe cerc-io/tx-spammer diff --git a/compose/docker-compose-watcher-uniswap-v3.yml b/compose/docker-compose-watcher-uniswap-v3.yml new file mode 100644 index 00000000..b98f6a70 --- /dev/null +++ b/compose/docker-compose-watcher-uniswap-v3.yml @@ -0,0 +1,170 @@ +version: '3.2' + +services: + + uniswap-watcher-db: + restart: unless-stopped + image: postgres:14-alpine + environment: + - POSTGRES_USER=vdbm + - POSTGRES_MULTIPLE_DATABASES=erc20-watcher,uni-watcher,uni-info-watcher,erc20-watcher-job-queue,uni-watcher-job-queue,uni-info-watcher-job-queue + - POSTGRES_EXTENSION=erc20-watcher-job-queue:pgcrypto,uni-watcher-job-queue:pgcrypto,uni-info-watcher-job-queue:pgcrypto + - POSTGRES_PASSWORD=password + command: ["postgres", "-c", "shared_preload_libraries=pg_stat_statements", "-c", "pg_stat_statements.track=all", "-c", "work_mem=2GB"] + volumes: + - ../config/postgresql/multiple-postgressql-databases.sh:/docker-entrypoint-initdb.d/multiple-postgressql-databases.sh + - ../config/postgresql/create-pg-stat-statements.sql:/docker-entrypoint-initdb.d/create-pg-stat-statements.sql + - uniswap_watcher_db_data:/var/lib/postgresql/data + ports: + - "0.0.0.0:15435:5432" + healthcheck: + test: ["CMD", "nc", "-v", "localhost", "5432"] + interval: 20s + timeout: 5s + retries: 15 + start_period: 10s + shm_size: '8GB' + + erc20-watcher-server: + restart: unless-stopped + depends_on: + uniswap-watcher-db: + condition: service_healthy + image: cerc/watcher-uniswap-v3:local + working_dir: /app/packages/erc20-watcher + environment: + - DEBUG=vulcanize:* + command: ["node", "--enable-source-maps", "dist/server.js"] + volumes: + - ../config/watcher-uniswap-v3/erc20-watcher.toml:/app/packages/erc20-watcher/environments/local.toml + ports: + - "0.0.0.0:3005:3001" + healthcheck: + test: ["CMD", "nc", "-v", "localhost", "3001"] + interval: 20s + timeout: 5s + retries: 15 + start_period: 5s + extra_hosts: + - "host.docker.internal:host-gateway" + + uni-watcher-job-runner: + restart: unless-stopped + depends_on: + uniswap-watcher-db: + condition: service_healthy + image: cerc/watcher-uniswap-v3:local + working_dir: /app/packages/uni-watcher + environment: + - DEBUG=vulcanize:* + command: ["sh", "-c", "./watch-contract.sh && node --enable-source-maps dist/job-runner.js"] + volumes: + - ../config/watcher-uniswap-v3/uni-watcher.toml:/app/packages/uni-watcher/environments/local.toml + - ../config/watcher-uniswap-v3/watch-contract.sh:/app/packages/uni-watcher/watch-contract.sh + ports: + - "0.0.0.0:9004:9000" + healthcheck: + test: ["CMD", "nc", "-v", "localhost", "9000"] + interval: 20s + timeout: 5s + retries: 15 + start_period: 5s + extra_hosts: + - "host.docker.internal:host-gateway" + + uni-watcher-server: + restart: unless-stopped + depends_on: + uniswap-watcher-db: + condition: service_healthy + uni-watcher-job-runner: + condition: service_healthy + image: cerc/watcher-uniswap-v3:local + environment: + - UNISWAP_START_BLOCK=12369621 + - DEBUG=vulcanize:* + working_dir: /app/packages/uni-watcher + command: ["./run.sh"] + volumes: + - ../config/watcher-uniswap-v3/uni-watcher.toml:/app/packages/uni-watcher/environments/local.toml + - ../config/watcher-uniswap-v3/run.sh:/app/packages/uni-watcher/run.sh + ports: + - "0.0.0.0:3003:3003" + - "0.0.0.0:9005:9001" + healthcheck: + test: ["CMD", "nc", "-v", "localhost", "3003"] + interval: 20s + timeout: 5s + retries: 15 + start_period: 5s + extra_hosts: + - "host.docker.internal:host-gateway" + + uni-info-watcher-job-runner: + restart: unless-stopped + depends_on: + uniswap-watcher-db: + condition: service_healthy + erc20-watcher-server: + condition: service_healthy + uni-watcher-server: + condition: service_healthy + image: cerc/watcher-uniswap-v3:local + working_dir: /app/packages/uni-info-watcher + environment: + - DEBUG=vulcanize:* + command: ["node", "--enable-source-maps", "dist/job-runner.js"] + volumes: + - ../config/watcher-uniswap-v3/uni-info-watcher.toml:/app/packages/uni-info-watcher/environments/local.toml + ports: + - "0.0.0.0:9006:9002" + healthcheck: + test: ["CMD", "nc", "-v", "localhost", "9002"] + interval: 20s + timeout: 5s + retries: 15 + start_period: 5s + extra_hosts: + - "host.docker.internal:host-gateway" + + uni-info-watcher-server: + restart: unless-stopped + depends_on: + uniswap-watcher-db: + condition: service_healthy + erc20-watcher-server: + condition: service_healthy + uni-watcher-server: + condition: service_healthy + uni-info-watcher-job-runner: + condition: service_healthy + image: cerc/watcher-uniswap-v3:local + environment: + - UNISWAP_START_BLOCK=12369621 + working_dir: /app/packages/uni-info-watcher + command: ["./run.sh"] + volumes: + - ../config/watcher-uniswap-v3/uni-info-watcher.toml:/app/packages/uni-info-watcher/environments/local.toml + - ../config/watcher-uniswap-v3/run.sh:/app/packages/uni-info-watcher/run.sh + ports: + - "0.0.0.0:3004:3004" + - "0.0.0.0:9007:9003" + healthcheck: + test: ["CMD", "nc", "-v", "localhost", "3004"] + interval: 20s + timeout: 5s + retries: 15 + start_period: 5s + extra_hosts: + - "host.docker.internal:host-gateway" + + uniswap-v3-info: + depends_on: + uni-info-watcher-server: + condition: service_healthy + image: cerc/uniswap-v3-info:local + ports: + - "0.0.0.0:3006:3000" + +volumes: + uniswap_watcher_db_data: diff --git a/config/postgresql/create-pg-stat-statements.sql b/config/postgresql/create-pg-stat-statements.sql new file mode 100644 index 00000000..d3c6679b --- /dev/null +++ b/config/postgresql/create-pg-stat-statements.sql @@ -0,0 +1 @@ +CREATE EXTENSION pg_stat_statements; diff --git a/config/watcher-uniswap-v3/erc20-watcher.toml b/config/watcher-uniswap-v3/erc20-watcher.toml new file mode 100644 index 00000000..3e258eeb --- /dev/null +++ b/config/watcher-uniswap-v3/erc20-watcher.toml @@ -0,0 +1,39 @@ +[server] + host = "0.0.0.0" + port = 3001 + mode = "eth_call" + kind = "lazy" + +[metrics] + host = "127.0.0.1" + port = 9000 + [metrics.gql] + port = 9001 + +[database] + type = "postgres" + host = "uniswap-watcher-db" + port = 5432 + database = "erc20-watcher" + username = "vdbm" + password = "password" + synchronize = true + logging = false + maxQueryExecutionTime = 100 + +[upstream] + [upstream.ethServer] + gqlApiEndpoint = "http://ipld-eth-server.example.com:8083/graphql" + rpcProviderEndpoint = "http://ipld-eth-server.example.com:8082" + + [upstream.cache] + name = "requests" + enabled = false + deleteOnStart = false + +[jobQueue] + dbConnectionString = "postgres://vdbm:password@uniswap-watcher-db:5432/erc20-watcher-job-queue" + maxCompletionLagInSecs = 300 + jobDelayInMilliSecs = 100 + eventsInBatch = 50 + blockDelayInMilliSecs = 2000 diff --git a/config/watcher-uniswap-v3/run.sh b/config/watcher-uniswap-v3/run.sh new file mode 100755 index 00000000..d8dcffb8 --- /dev/null +++ b/config/watcher-uniswap-v3/run.sh @@ -0,0 +1,10 @@ +#!/bin/sh + +set -e +set -u + +echo "Initializing watcher..." +yarn fill --start-block $UNISWAP_START_BLOCK --end-block $((UNISWAP_START_BLOCK + 1)) + +echo "Running active server" +DEBUG=vulcanize:* exec node --enable-source-maps dist/server.js diff --git a/config/watcher-uniswap-v3/uni-info-watcher.toml b/config/watcher-uniswap-v3/uni-info-watcher.toml new file mode 100644 index 00000000..2f092845 --- /dev/null +++ b/config/watcher-uniswap-v3/uni-info-watcher.toml @@ -0,0 +1,90 @@ +[server] + host = "0.0.0.0" + port = 3004 + mode = "prod" + kind = "active" + + # Checkpointing state. + checkpointing = true + + # Checkpoint interval in number of blocks. + checkpointInterval = 50000 + + # Enable state creation + enableState = false + + # Max block range for which to return events in eventsInRange GQL query. + # Use -1 for skipping check on block range. + maxEventsBlockRange = 1000 + + # Interval in number of blocks at which to clear entities cache. + clearEntitiesCacheInterval = 1000 + + # Boolean to skip updating entity fields required in state creation and not required in the frontend. + skipStateFieldsUpdate = false + + # Boolean to load GQL query nested entity relations sequentially. + loadRelationsSequential = false + + # Max GQL API requests to process simultaneously (defaults to 1). + maxSimultaneousRequests = 1 + + # GQL cache settings + [server.gqlCache] + enabled = true + + # Max in-memory cache size (in bytes) (default 8 MB) + # maxCacheSize + + # GQL cache-control max-age settings (in seconds) + maxAge = 15 + timeTravelMaxAge = 86400 # 1 day + +[metrics] + host = "0.0.0.0" + port = 9002 + [metrics.gql] + port = 9003 + +[database] + type = "postgres" + host = "uniswap-watcher-db" + port = 5432 + database = "uni-info-watcher" + username = "vdbm" + password = "password" + synchronize = true + logging = false + maxQueryExecutionTime = 100 + + [database.extra] + # maximum number of clients the pool should contain + max = 20 + +[upstream] + [upstream.ethServer] + gqlApiEndpoint = "http://ipld-eth-server.example.com:8083/graphql" + rpcProviderEndpoint = "http://ipld-eth-server.example.com:8082" + + [upstream.cache] + name = "requests" + enabled = false + deleteOnStart = false + + [upstream.uniWatcher] + gqlEndpoint = "http://uni-watcher-server:3003/graphql" + gqlSubscriptionEndpoint = "ws://uni-watcher-server:3003/graphql" + + [upstream.tokenWatcher] + gqlEndpoint = "http://erc20-watcher-server:3001/graphql" + gqlSubscriptionEndpoint = "ws://erc20-watcher-server:3001/graphql" + +[jobQueue] + dbConnectionString = "postgres://vdbm:password@uniswap-watcher-db:5432/uni-info-watcher-job-queue" + maxCompletionLagInSecs = 300 + jobDelayInMilliSecs = 1000 + eventsInBatch = 50 + subgraphEventsOrder = true + blockDelayInMilliSecs = 2000 + prefetchBlocksInMem = true + prefetchBlockCount = 10 diff --git a/config/watcher-uniswap-v3/uni-watcher.toml b/config/watcher-uniswap-v3/uni-watcher.toml new file mode 100644 index 00000000..bc583e9b --- /dev/null +++ b/config/watcher-uniswap-v3/uni-watcher.toml @@ -0,0 +1,41 @@ +[server] + host = "0.0.0.0" + port = 3003 + kind = "active" + +[metrics] + host = "0.0.0.0" + port = 9000 + [metrics.gql] + port = 9001 + +[database] + type = "postgres" + host = "uniswap-watcher-db" + port = 5432 + database = "uni-watcher" + username = "vdbm" + password = "password" + synchronize = true + logging = false + maxQueryExecutionTime = 100 + +[upstream] + [upstream.ethServer] + gqlApiEndpoint = "http://ipld-eth-server.example.com:8083/graphql" + rpcProviderEndpoint = "http://ipld-eth-server.example.com:8082" + + [upstream.cache] + name = "requests" + enabled = false + deleteOnStart = false + +[jobQueue] + dbConnectionString = "postgres://vdbm:password@uniswap-watcher-db:5432/uni-watcher-job-queue" + maxCompletionLagInSecs = 300 + jobDelayInMilliSecs = 0 + eventsInBatch = 50 + lazyUpdateBlockProgress = true + blockDelayInMilliSecs = 2000 + prefetchBlocksInMem = true + prefetchBlockCount = 10 diff --git a/config/watcher-uniswap-v3/watch-contract.sh b/config/watcher-uniswap-v3/watch-contract.sh new file mode 100755 index 00000000..aaed7e1d --- /dev/null +++ b/config/watcher-uniswap-v3/watch-contract.sh @@ -0,0 +1,10 @@ +#!/bin/sh + +set -e +set -u + +echo "Watching factory contract 0x1F98431c8aD98523631AE4a59f267346ea31F984" +yarn watch:contract --address 0x1F98431c8aD98523631AE4a59f267346ea31F984 --kind factory --startingBlock 12369621 --checkpoint + +echo "Watching nfpm contract 0xC36442b4a4522E871399CD717aBDD847Ab11FE88" +yarn watch:contract --address 0xC36442b4a4522E871399CD717aBDD847Ab11FE88 --kind nfpm --startingBlock 12369651 --checkpoint diff --git a/container-build/cerc-uniswap-v3-info/Dockerfile b/container-build/cerc-uniswap-v3-info/Dockerfile new file mode 100644 index 00000000..a01e2ed2 --- /dev/null +++ b/container-build/cerc-uniswap-v3-info/Dockerfile @@ -0,0 +1,13 @@ +FROM node:15.3.0-alpine3.10 + +RUN apk --update --no-cache add make git + +WORKDIR /app + +COPY . . + +RUN echo "Building uniswap-v3-info" && \ + git checkout v0.1.1 && \ + yarn + +CMD ["sh", "-c", "yarn start"] diff --git a/container-build/cerc-uniswap-v3-info/build.sh b/container-build/cerc-uniswap-v3-info/build.sh new file mode 100755 index 00000000..bda1bb5e --- /dev/null +++ b/container-build/cerc-uniswap-v3-info/build.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +# Build cerc/uniswap-v3-info + +# See: https://stackoverflow.com/a/246128/1701505 +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +docker build -t cerc/uniswap-v3-info:local -f ${SCRIPT_DIR}/Dockerfile ${CERC_REPO_BASE_DIR}/uniswap-v3-info diff --git a/container-build/cerc-watcher-uniswap-v3/Dockerfile b/container-build/cerc-watcher-uniswap-v3/Dockerfile new file mode 100644 index 00000000..f400755b --- /dev/null +++ b/container-build/cerc-watcher-uniswap-v3/Dockerfile @@ -0,0 +1,11 @@ +FROM node:16.17.1-alpine3.16 + +RUN apk --update --no-cache add git python3 alpine-sdk + +WORKDIR /app + +COPY . . + +RUN echo "Building uniswap-watcher-ts" && \ + git checkout v0.3.4 && \ + yarn && yarn build && yarn build:contracts diff --git a/container-build/cerc-watcher-uniswap-v3/build.sh b/container-build/cerc-watcher-uniswap-v3/build.sh new file mode 100755 index 00000000..432feb02 --- /dev/null +++ b/container-build/cerc-watcher-uniswap-v3/build.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +# Build cerc/watcher-uniswap-v3 + +# See: https://stackoverflow.com/a/246128/1701505 +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +docker build -t cerc/watcher-uniswap-v3:local -f ${SCRIPT_DIR}/Dockerfile ${CERC_REPO_BASE_DIR}/uniswap-watcher-ts diff --git a/stacks/erc20/README.md b/stacks/erc20/README.md index eec3d9f4..3405b703 100644 --- a/stacks/erc20/README.md +++ b/stacks/erc20/README.md @@ -44,7 +44,7 @@ Instructions to deploy a local ERC20 watcher stack (core + watcher) for demonstr $ export TOKEN_ADDRESS= ``` -* Open `http://localhost:3001/graphql` (GraphQL Playground) in a browser window +* Open `http://localhost:3002/graphql` (GraphQL Playground) in a browser window * Connect MetaMask to `http://localhost:8545` (with chain ID `99`) diff --git a/stacks/uniswap-v3/README.md b/stacks/uniswap-v3/README.md new file mode 100644 index 00000000..99f55050 --- /dev/null +++ b/stacks/uniswap-v3/README.md @@ -0,0 +1,83 @@ +# Uniswap v3 + +Instructions to deploy Uniswap v3 watcher stack (watcher + uniswap-v3-info frontend app) using [laconic-stack-orchestrator](../../README.md#setup) + +## Prerequisites + +* Access to [uniswap-watcher-ts](https://github.com/vulcanize/uniswap-watcher-ts). + +* This deployment expects core services to be running; specifically, it requires `ipld-eth-server` RPC and GQL endpoints. Update the `upstream.ethServer` endpoints in the [watcher config files](../../config/watcher-uniswap-v3) accordingly: + + ```toml + [upstream] + [upstream.ethServer] + gqlApiEndpoint = "http://ipld-eth-server.example.com:8083/graphql" + rpcProviderEndpoint = "http://ipld-eth-server.example.com:8082" + ``` + +* `uni-watcher` and `uni-info-watcher` database dumps (optional). + +## Setup + +* Clone / pull required repositories: + + ```bash + $ laconic-so setup-repositories --include vulcanize/uniswap-watcher-ts,vulcanize/uniswap-v3-info --git-ssh --pull + ``` + +* Build watcher and info app container images: + + ```bash + $ laconic-so build-containers --include cerc/watcher-uniswap-v3,cerc/uniswap-v3-info + ``` + + This should create the required docker images in the local image registry. + +## Deploy + +* (Optional) Initialize the watcher database with existing database dumps if available: + + * Start the watcher database to be initialized: + + ```bash + $ laconic-so deploy-system --include watcher-uniswap-v3 up uniswap-watcher-db + ``` + + * Find the watcher database container's id using `docker ps` and export it for further usage: + + ```bash + $ export CONTAINER_ID= + ``` + + * Load watcher database dumps: + + ```bash + # uni-watcher database + $ docker exec -i $CONTAINER_ID psql -U vdbm uni-watcher < UNI_WATCHER_DB_DUMP_FILE_PATH.sql + + # uni-info-watcher database + $ docker exec -i $CONTAINER_ID psql -U vdbm uni-info-watcher < UNI_INFO_WATCHER_DB_DUMP_FILE_PATH.sql + ``` + +* Start all the watcher and info app services: + + ```bash + $ laconic-so deploy-system --include watcher-uniswap-v3 up + ``` + +* Check that all the services are up and healthy: + + ```bash + $ docker ps + ``` + + * The `uni-info-watcher` GraphQL Playground can be accessed at `http://localhost:3004/graphql` + * The frontend app can be accessed at `http://localhost:3006` + +## Clean up + +* To stop all the services running in background: + + ```bash + $ laconic-so deploy-system --include watcher-uniswap-v3 down + ``` diff --git a/stacks/uniswap-v3/stack.yml b/stacks/uniswap-v3/stack.yml new file mode 100644 index 00000000..8eec8f87 --- /dev/null +++ b/stacks/uniswap-v3/stack.yml @@ -0,0 +1,10 @@ +version: "1.0" +name: uniswap-v3 +repos: + - vulcanize/uniswap-watcher-ts + - vulcanize/uniswap-v3-info +containers: + - cerc/watcher-uniswap-v3 + - cerc/uniswap-v3-info +pods: + - watcher-uniswap-v3 From 3b031329b4f0e80d439ac378674fcbd16117b843 Mon Sep 17 00:00:00 2001 From: prathamesh0 <42446521+prathamesh0@users.noreply.github.com> Date: Thu, 22 Dec 2022 19:43:13 +0530 Subject: [PATCH 10/11] Upgrade watcher-ts version in the mobymask stack (#83) * Upgrade watcher-ts version in mobymask setup * Change watcher db service name * Add a clean-up step --- compose/docker-compose-watcher-mobymask.yml | 16 ++++---- config/watcher-mobymask/mobymask-watcher.toml | 8 ++-- .../cerc-watcher-mobymask/Dockerfile | 23 +++-------- .../cerc-watcher-mobymask/build.sh | 5 ++- stacks/mobymask/README.md | 41 +++++++++++++------ stacks/mobymask/stack.yml | 3 +- 6 files changed, 48 insertions(+), 48 deletions(-) diff --git a/compose/docker-compose-watcher-mobymask.yml b/compose/docker-compose-watcher-mobymask.yml index 6ce410cc..f54e1454 100644 --- a/compose/docker-compose-watcher-mobymask.yml +++ b/compose/docker-compose-watcher-mobymask.yml @@ -4,7 +4,7 @@ version: '3.2' services: - watcher-db: + mobymask-watcher-db: restart: unless-stopped image: postgres:14-alpine environment: @@ -14,7 +14,7 @@ services: - POSTGRES_PASSWORD=password volumes: - ../config/postgresql/multiple-postgressql-databases.sh:/docker-entrypoint-initdb.d/multiple-postgressql-databases.sh - - watcher_db_data:/var/lib/postgresql/data + - mobymask_watcher_db_data:/var/lib/postgresql/data ports: - "0.0.0.0:15432:5432" healthcheck: @@ -27,12 +27,12 @@ services: mobymask-watcher-server: restart: unless-stopped depends_on: - watcher-db: + mobymask-watcher-db: condition: service_healthy image: cerc/watcher-mobymask:local command: ["sh", "-c", "yarn server"] volumes: - - ../config/watcher-mobymask/mobymask-watcher.toml:/app/watcher-ts/packages/mobymask-watcher/environments/local.toml + - ../config/watcher-mobymask/mobymask-watcher.toml:/app/packages/mobymask-watcher/environments/local.toml ports: - "0.0.0.0:3001:3001" - "0.0.0.0:9001:9001" @@ -50,18 +50,16 @@ services: depends_on: mobymask-watcher-server: condition: service_healthy - watcher-db: + mobymask-watcher-db: condition: service_healthy image: cerc/watcher-mobymask:local command: ["sh", "-c", "yarn job-runner"] volumes: - - ../config/watcher-mobymask/mobymask-watcher.toml:/app/watcher-ts/packages/mobymask-watcher/environments/local.toml + - ../config/watcher-mobymask/mobymask-watcher.toml:/app/packages/mobymask-watcher/environments/local.toml ports: - "0.0.0.0:9000:9000" extra_hosts: - "ipld-eth-server:host-gateway" volumes: - indexer_db_data: - watcher_db_data: - \ No newline at end of file + mobymask_watcher_db_data: diff --git a/config/watcher-mobymask/mobymask-watcher.toml b/config/watcher-mobymask/mobymask-watcher.toml index 3e9b2db5..dd91cf8f 100644 --- a/config/watcher-mobymask/mobymask-watcher.toml +++ b/config/watcher-mobymask/mobymask-watcher.toml @@ -27,7 +27,7 @@ [database] type = "postgres" - host = "watcher-db" + host = "mobymask-watcher-db" port = 5432 database = "mobymask-watcher" username = "vdbm" @@ -37,8 +37,8 @@ [upstream] [upstream.ethServer] - gqlApiEndpoint = "http://ipld-eth-server:8082/graphql" - rpcProviderEndpoint = "http://ipld-eth-server:8081" + gqlApiEndpoint = "http://ipld-eth-server:8083/graphql" + rpcProviderEndpoint = "http://ipld-eth-server:8082" blockDelayInMilliSecs = 60000 [upstream.cache] @@ -47,7 +47,7 @@ deleteOnStart = false [jobQueue] - dbConnectionString = "postgres://vdbm:password@watcher-db/mobymask-watcher-job-queue" + dbConnectionString = "postgres://vdbm:password@mobymask-watcher-db/mobymask-watcher-job-queue" maxCompletionLagInSecs = 300 jobDelayInMilliSecs = 100 eventsInBatch = 50 diff --git a/container-build/cerc-watcher-mobymask/Dockerfile b/container-build/cerc-watcher-mobymask/Dockerfile index b27c6f96..7994237a 100644 --- a/container-build/cerc-watcher-mobymask/Dockerfile +++ b/container-build/cerc-watcher-mobymask/Dockerfile @@ -5,23 +5,10 @@ RUN apk --update --no-cache add git python3 alpine-sdk WORKDIR /app -COPY assemblyscript assemblyscript -COPY watcher-ts watcher-ts +COPY . . -# TODO: needs branch ng-integrate-asyncify -# We use a mixture of npm and yarn below because the upstream -# project checked in an npm package-log.json file -RUN echo "Building assemblyscript" && \ - cd assemblyscript && \ - npm install && npm run build && yarn link +RUN echo "Building watcher-ts" && \ + git checkout v0.2.19 && \ + yarn && yarn build -# TODO: needs branch v0.2.13 -# The shenanigans below is due to yarn and lerna being a dumpster-fire -RUN echo "Linking watcher-ts to local assemblyscript" && \ - cd watcher-ts/packages/graph-node && yarn remove @vulcanize/assemblyscript && \ - yarn add https://github.com/vulcanize/assemblyscript.git#ng-integrate-asyncify - -RUN echo "Building watcher-tst" && \ - cd watcher-ts && yarn && yarn link "@vulcanize/assemblyscript" && yarn build - -WORKDIR /app/watcher-ts/packages/mobymask-watcher +WORKDIR /app/packages/mobymask-watcher diff --git a/container-build/cerc-watcher-mobymask/build.sh b/container-build/cerc-watcher-mobymask/build.sh index a4a97cdd..f965c770 100755 --- a/container-build/cerc-watcher-mobymask/build.sh +++ b/container-build/cerc-watcher-mobymask/build.sh @@ -4,5 +4,6 @@ # See: https://stackoverflow.com/a/246128/1701505 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -# TODO: add a mechanism to pass two repos into a container rather than the parent directory as below -docker build -t cerc/watcher-mobymask:local -f ${SCRIPT_DIR}/Dockerfile ${CERC_REPO_BASE_DIR} +docker build -t cerc/watcher-mobymask:local -f ${SCRIPT_DIR}/Dockerfile ${CERC_REPO_BASE_DIR}/watcher-ts + +# TODO: add a mechanism to pass two repos into a container rather than the parent directory diff --git a/stacks/mobymask/README.md b/stacks/mobymask/README.md index 8b197754..d048ba38 100644 --- a/stacks/mobymask/README.md +++ b/stacks/mobymask/README.md @@ -3,36 +3,51 @@ The MobyMask watcher is a Laconic Network component that provides efficient access to MobyMask contract data from Ethereum, along with evidence allowing users to verify the correctness of that data. The watcher source code is available in [this repository](https://github.com/cerc-io/watcher-ts/tree/main/packages/mobymask-watcher) and a developer-oriented Docker Compose setup for the watcher can be found [here](https://github.com/cerc-io/mobymask-watcher). The watcher can be deployed automatically using the Laconic Stack Orchestrator tool as detailed below: ## Deploy the MobyMask Watcher + The instructions below show how to deploy a MobyMask watcher using laconic-stack-orchestrator (the installation of which is covered [here](https://github.com/cerc-io/stack-orchestrator#user-mode)). This deployment expects that ipld-eth-server's endpoints are available on the local machine at http://ipld-eth-server.example.com:8083/graphql and http://ipld-eth-server.example.com:8082. More advanced configurations are supported by modifying the watcher's [config file](../../config/watcher-mobymask/mobymask-watcher.toml). + ## Clone required repositories + ``` -$ laconic-so setup-repositories --include vulcanize/assemblyscript,cerc-io/watcher-ts -``` -Checkout required branches for the current release: -``` -$ cd ~/cerc/assemblyscript -$ git checkout ng-integrate-asyncify -$ cd ~/cerc/watcher-ts -$ git checkout v0.2.13 +$ laconic-so setup-repositories --include cerc-io/watcher-ts ``` + ## Build the watcher container + ``` -$ laconic-sh build-containers --include cerc/watcher-mobymask +$ laconic-so build-containers --include cerc/watcher-mobymask ``` + This should create a container with tag `cerc/watcher-mobymask` in the local image registry. + ## Deploy the stack -First the watcher database has to be initialized. Start only the watcher-db service: + +First the watcher database has to be initialized. Start only the mobymask-watcher-db service: + ``` -$ laconic-so deploy-system --include watcher-mobymask up watcher-db +$ laconic-so deploy-system --include watcher-mobymask up mobymask-watcher-db ``` + Next find the container's id using `docker ps` then run the following command to initialize the database: + ``` -$ docker exec -i psql -U vdbm mobymask-watcher < config/watcher-mobymask/mobymask-watcher-db.sql +$ docker exec -i psql -U vdbm mobymask-watcher < config/watcher-mobymask/mobymask-watcher-db.sql ``` + Finally start the remaining containers: + ``` -$ laconic-so deploy-system --include watcher-mobymask +$ laconic-so deploy-system --include watcher-mobymask up ``` + Correct operation should be verified by following the instructions [here](https://github.com/cerc-io/mobymask-watcher/tree/main/mainnet-watcher-only#run), checking GraphQL queries return valid results in the watcher's [playground](http://127.0.0.1:3001/graphql). + +## Clean up + +Stop all the services running in background: + +```bash +$ laconic-so deploy-system --include watcher-mobymask down +``` diff --git a/stacks/mobymask/stack.yml b/stacks/mobymask/stack.yml index 8925cf28..794ec4ab 100644 --- a/stacks/mobymask/stack.yml +++ b/stacks/mobymask/stack.yml @@ -1,8 +1,7 @@ version: "1.0" name: mobymask-watcher repos: - - cerc-io/watcher-ts/v0.2.13 - - vulcanize/assemblyscript/ng-integrate-asyncify + - cerc-io/watcher-ts/v0.2.19 containers: - cerc/watcher-mobymask pods: From 63378a5c7e71b37ec10cb53b3261a12f46160693 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Thu, 22 Dec 2022 12:52:23 -0700 Subject: [PATCH 11/11] Add redirect follow to curl command for github --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 49a7f2f5..32618100 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ User mode runs the orchestrator from a "binary" single-file release and does not 1. Download the latest release from [this page](https://github.com/cerc-io/stack-orchestrator/tags), into a suitable directory (e.g. `~/bin`): ``` $ cd ~/bin - $ curl https://github.com/cerc-io/stack-orchestrator/releases/download/v1.0.3-alpha/laconic-so + $ curl -L https://github.com/cerc-io/stack-orchestrator/releases/download/v1.0.3-alpha/laconic-so ``` 1. Ensure `laconic-so` is on the `PATH` 1. Verify operation: