remove references to vulcanizedb repo and some other minor changes/additions
This commit is contained in:
Ian Norden
parent
b9fbd2da53
commit
1137436ac0
24
Dockerfile
24
Dockerfile
@ -1,24 +0,0 @@
|
||||
FROM golang:alpine as builder
|
||||
RUN apk --update --no-cache add make git g++
|
||||
|
||||
# Build statically linked vDB binary (wonky path because of Dep)
|
||||
RUN mkdir -p /go/src/github.com/vulcanize/ipfs-blockchain-watcher
|
||||
ADD . /go/src/github.com/vulcanize/ipfs-blockchain-watcher
|
||||
WORKDIR /go/src/github.com/vulcanize/ipfs-blockchain-watcher
|
||||
RUN GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' .
|
||||
|
||||
# Build migration tool
|
||||
RUN go get -u -d github.com/pressly/goose/cmd/goose
|
||||
WORKDIR /go/src/github.com/pressly/goose/cmd/goose
|
||||
RUN GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -tags='no_mysql no_sqlite' -o goose
|
||||
|
||||
# Second stage
|
||||
FROM alpine
|
||||
COPY --from=builder /go/src/github.com/vulcanize/ipfs-blockchain-watcher/vulcanizedb /app/vulcanizedb
|
||||
COPY --from=builder /go/src/github.com/vulcanize/ipfs-blockchain-watcher/environments/staging.toml /app/environments/
|
||||
COPY --from=builder /go/src/github.com/vulcanize/ipfs-blockchain-watcher/dockerfiles/startup_script.sh /app/
|
||||
COPY --from=builder /go/src/github.com/vulcanize/ipfs-blockchain-watcher/db/migrations/* /app/
|
||||
COPY --from=builder /go/src/github.com/pressly/goose/cmd/goose/goose /app/goose
|
||||
|
||||
WORKDIR /app
|
||||
CMD ["./startup_script.sh"]
|
||||
25
Makefile
25
Makefile
@ -105,7 +105,7 @@ rollback: $(GOOSE) checkdbvars
|
||||
pg_dump -O -s $(CONNECT_STRING) > db/schema.sql
|
||||
|
||||
|
||||
## Rollbackt to a select migration (id/timestamp)
|
||||
## Rollback to a select migration (id/timestamp)
|
||||
.PHONY: rollback_to
|
||||
rollback_to: $(GOOSE) checkmigration checkdbvars
|
||||
$(GOOSE) -dir db/migrations postgres "$(CONNECT_STRING)" down-to "$(MIGRATION)"
|
||||
@ -137,26 +137,3 @@ version_migrations:
|
||||
import:
|
||||
test -n "$(NAME)" # $$NAME
|
||||
psql $(NAME) < db/schema.sql
|
||||
|
||||
|
||||
# Docker actions
|
||||
## Rinkeby docker environment
|
||||
RINKEBY_COMPOSE_FILE=dockerfiles/rinkeby/docker-compose.yml
|
||||
|
||||
.PHONY: rinkeby_env_up
|
||||
rinkeby_env_up:
|
||||
docker-compose -f $(RINKEBY_COMPOSE_FILE) up -d geth
|
||||
docker-compose -f $(RINKEBY_COMPOSE_FILE) up --build migrations
|
||||
docker-compose -f $(RINKEBY_COMPOSE_FILE) up -d --build vulcanizedb
|
||||
|
||||
.PHONY: rinkeby_env_deploy
|
||||
rinkeby_env_deploy:
|
||||
docker-compose -f $(RINKEBY_COMPOSE_FILE) up -d --build vulcanizedb
|
||||
|
||||
.PHONY: dev_env_migrate
|
||||
rinkeby_env_migrate:
|
||||
docker-compose -f $(RINKEBY_COMPOSE_FILE) up --build migrations
|
||||
|
||||
.PHONY: rinkeby_env_down
|
||||
rinkeby_env_down:
|
||||
docker-compose -f $(RINKEBY_COMPOSE_FILE) down
|
||||
|
||||
10
README.md
10
README.md
@ -2,7 +2,7 @@
|
||||
|
||||
[](https://goreportcard.com/report/github.com/vulcanize/ipfs-blockchain-watcher)
|
||||
|
||||
> Tool for extracting and indexing blockchain data on PG-IPFS
|
||||
> ipfs-blockchain-watcher is used to extract, transform, and load all eth or btc data into an IPFS-backing Postgres datastore while generating useful secondary indexes around that data in other Postgres tables
|
||||
|
||||
## Table of Contents
|
||||
1. [Background](#background)
|
||||
@ -30,7 +30,7 @@ More details on the design of ipfs-blockchain-watcher can be found in [here](./d
|
||||
|
||||
### Goose
|
||||
[goose](https://github.com/pressly/goose) is used for migration management. While it is not necessary to use `goose` for manual setup, it
|
||||
is required for running the automated tests and by the `make migrate` command.
|
||||
is required for running the automated tests and is used by the `make migrate` command.
|
||||
|
||||
### Postgres
|
||||
1. [Install Postgres](https://wiki.postgresql.org/wiki/Detailed_installation_guides)
|
||||
@ -52,12 +52,12 @@ localhost. To allow access on Ubuntu, set localhost connections via hostname, ip
|
||||
|
||||
### IPFS
|
||||
Data is stored in an [IPFS-backing Postgres datastore](https://github.com/ipfs/go-ds-sql).
|
||||
By default data is writen directly to the ipfs blockstore in Postgres; the public.blocks table.
|
||||
By default data is written directly to the ipfs blockstore in Postgres; the public.blocks table.
|
||||
In this case no further IPFS configuration is needed at this time.
|
||||
|
||||
Optionally, ipfs-blockchain-watcher can be configured to function through an internal ipfs node interface using the flag: `-ipfs-mode=interface`.
|
||||
Operating through the ipfs interface provides the option to configure a block exchange that can search remotely for IPLD data found missing in the local datastore.
|
||||
This option is irrelevant in most cases and this mode some disadvantages, namely:
|
||||
This option is irrelevant in most cases and this mode has some disadvantages, namely:
|
||||
|
||||
1. Environment must have IPFS configured
|
||||
1. Process will contend with the lockfile at `$IPFS_PATH`
|
||||
@ -120,7 +120,7 @@ The default http url is "127.0.0.1:8332". We will use the http endpoint as both
|
||||
### Watcher
|
||||
Finally, setup the watcher process itself.
|
||||
|
||||
Start by downloading vulcanizedb and moving into the repo:
|
||||
Start by downloading ipfs-blockchain-watcher and moving into the repo:
|
||||
|
||||
`go get github.com/vulcanize/ipfs-blockchain-watcher`
|
||||
|
||||
|
||||
@ -1,32 +0,0 @@
|
||||
S
|
||||
`Dockerfile` will build an alpine image containing:
|
||||
- vDB as a binary with runtime deps statically linked: `/app/vulcanizedb`
|
||||
- The migration tool goose: `/app/goose`
|
||||
- Two services for running `headerSync` and `continuousLogSync`, started with the default configuration `environments/staging.toml`.
|
||||
|
||||
By default, vDB is configured towards the Kovan deploy. The configuration values can be overridden using environment variables, using the same hierarchical naming pattern but in CAPS and using underscores. For example, the contract address for the `Pit` can be set with the variable `CONTRACT_ADDRESS_PIT="0x123..."`.
|
||||
|
||||
## To use the container:
|
||||
1. Setup a postgres database with superuser `vulcanize`
|
||||
2. Set the env variables `DATABASE_NAME`, `DATABASE_HOSTNAME`,
|
||||
`DATABASE_PORT`, `DATABASE_USER` & `DATABASE_PASSWORD`
|
||||
3. Run the DB migrations:
|
||||
* `./goose postgres "postgresql://$(DATABASE_USER):$(DATABASE_PASSWORD)@$(DATABASE_HOSTNAME):$(DATABASE_PORT)/$(DATABASE_NAME)?sslmode=disable"
|
||||
e`
|
||||
4. Set `CLIENT_IPCPATH` to a node endpoint
|
||||
5. Set the contract variables:
|
||||
* `CONTRACT_ADDRESS_[CONTRACT NAME]=0x123...`
|
||||
* `CONTRACT_ABI_[CONTRACT NAME]="ABI STRING"`
|
||||
* `CONTRACT_DEPLOYMENT-BLOCK_[CONTRACT NAME]=0` (doesn't really matter on a short chain, just avoids long unnecessary searching)
|
||||
6. Start the `headerSync` and `continuousLogSync` services:
|
||||
* `./vulcanizedb headerSync --config environments/staging.toml`
|
||||
* `./vulcanizedb continuousLogSync --config environments/staging.toml`
|
||||
|
||||
### Automated
|
||||
The steps above have been rolled into a script: `/app/startup_script.sh`, which just assumes the DB env variables have been set, and defaults the rest to Kovan according to `environments/staging.toml`. This can be called with something like:
|
||||
|
||||
`docker run -d -e DATABASE_NAME=vulcanize_public -e DATABASE_HOSTNAME=localhost -e DATABASE_PORT=5432 -e DATABASE_USER=vulcanize -e DATABASE_PASSWORD=vulcanize m0ar/images:vDB`
|
||||
|
||||
### Logging
|
||||
When running, vDB services log to `/app/vulcanizedb.log`.
|
||||
|
||||
@ -7,7 +7,7 @@ RUN apk add busybox-extras
|
||||
# this is probably a noob move, but I want apk from alpine for the above but need to avoid Go 1.13 below as this error still occurs https://github.com/ipfs/go-ipfs/issues/6603
|
||||
FROM golang:1.12.4 as builder
|
||||
|
||||
# Get and build vulcanizedb
|
||||
# Get and build ipfs-blockchain-watcher
|
||||
ADD . /go/src/github.com/vulcanize/ipfs-blockchain-watcher
|
||||
|
||||
# Build migration tool
|
||||
|
||||
@ -1,29 +0,0 @@
|
||||
#!/bin/sh
|
||||
# Runs the migrations and starts the headerSync and continuousLogSync services
|
||||
|
||||
# Exit if the variable tests fail
|
||||
set -e
|
||||
|
||||
# Check the database variables are set
|
||||
test $DATABASE_NAME
|
||||
test $DATABASE_HOSTNAME
|
||||
test $DATABASE_PORT
|
||||
test $DATABASE_USER
|
||||
test $DATABASE_PASSWORD
|
||||
|
||||
# Construct the connection string for postgres
|
||||
CONNECT_STRING=postgresql://$DATABASE_USER:$DATABASE_PASSWORD@$DATABASE_HOSTNAME:$DATABASE_PORT/$DATABASE_NAME?sslmode=disable
|
||||
echo "Connecting with: $CONNECT_STRING"
|
||||
|
||||
set +e
|
||||
|
||||
# Run the DB migrations
|
||||
./goose postgres "$CONNECT_STRING" up
|
||||
if [ $? -eq 0 ]; then
|
||||
# Fire up the services
|
||||
./vulcanizedb headerSync --config environments/staging.toml &
|
||||
./vulcanizedb continuousLogSync --config environments/staging.toml &
|
||||
else
|
||||
echo "Could not run migrations. Are the database details correct?"
|
||||
fi
|
||||
wait
|
||||
@ -4,10 +4,10 @@ RUN apk --update --no-cache add make git g++ linux-headers
|
||||
# DEBUG
|
||||
RUN apk add busybox-extras
|
||||
|
||||
# Get and build vulcanizedb
|
||||
# Get and build ipfs-blockchain-watcher
|
||||
ADD . /go/src/github.com/vulcanize/ipfs-blockchain-watcher
|
||||
WORKDIR /go/src/github.com/vulcanize/ipfs-blockchain-watcher
|
||||
RUN GO111MODULE=on GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -o vulcanizedb .
|
||||
RUN GO111MODULE=on GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -o ipfs-blockchain-watcher .
|
||||
|
||||
# Build migration tool
|
||||
RUN go get -u -d github.com/pressly/goose/cmd/goose
|
||||
@ -37,7 +37,7 @@ COPY --chown=5000:5000 --from=builder /go/src/github.com/vulcanize/ipfs-blockcha
|
||||
|
||||
|
||||
# keep binaries immutable
|
||||
COPY --from=builder /go/src/github.com/vulcanize/ipfs-blockchain-watcher/vulcanizedb vulcanizedb
|
||||
COPY --from=builder /go/src/github.com/vulcanize/ipfs-blockchain-watcher/ipfs-blockchain-watcher ipfs-blockchain-watcher
|
||||
COPY --from=builder /go/src/github.com/pressly/goose/cmd/goose/goose goose
|
||||
COPY --from=builder /go/src/github.com/vulcanize/ipfs-blockchain-watcher/db/migrations migrations/vulcanizedb
|
||||
COPY --from=builder /go/src/github.com/vulcanize/ipfs-blockchain-watcher/environments environments
|
||||
|
||||
@ -36,7 +36,7 @@ echo "Beginning the vulcanizedb process"
|
||||
VDB_CONFIG_FILE=${VDB_CONFIG_FILE:-config.toml}
|
||||
DEFAULT_OPTIONS="--config=$VDB_CONFIG_FILE"
|
||||
VDB_FULL_CL=${VDB_FULL_CL:-$VDB_COMMAND $DEFAULT_OPTIONS}
|
||||
echo running: ./vulcanizedb $VDB_FULL_CL $@
|
||||
echo running: ./ipfs-blockchain-watcher $VDB_FULL_CL $@
|
||||
|
||||
case "$1" in
|
||||
"/bin/sh" )
|
||||
@ -50,8 +50,8 @@ if [[ -z "$vdb_args" ]]; then
|
||||
vdb_args="--config=config.toml"
|
||||
fi
|
||||
|
||||
echo running: ./vulcanizedb $vdb_args
|
||||
./vulcanizedb $vdb_args
|
||||
echo running: ./ipfs-blockchain-watcher $vdb_args
|
||||
./ipfs-blockchain-watcher $vdb_args
|
||||
rv=$?
|
||||
|
||||
if [ $rv != 0 ]; then
|
||||
|
||||
@ -50,7 +50,7 @@ fi
|
||||
# If IPFS initialization was successful
|
||||
if [[ $? -eq 0 ]]; then
|
||||
echo "Running the VulcanizeDB process"
|
||||
./vulcanizedb ${VDB_COMMAND} --config=config.toml
|
||||
./ipfs-blockchain-watcher ${VDB_COMMAND} --config=config.toml
|
||||
else
|
||||
echo "Could not initialize IPFS."
|
||||
exit 1
|
||||
|
||||
@ -11,9 +11,9 @@ fi
|
||||
|
||||
db=$1
|
||||
dir=$(basename "$(pwd)")
|
||||
if [ $dir != "vulcanizedb" ]
|
||||
if [ $dir != "ipfs-blockchain-watcher" ]
|
||||
then
|
||||
echo "Run me from the vulcanizedb root dir"
|
||||
echo "Run me from the ipfs-blockchain-watcher root dir"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user