cerc-io/ipld-eth-server
18
0
Fork 1
Code Issues 10 Pull Requests 1 Actions Packages Projects Releases 37 Wiki Activity

update dockerfile and documentation

Browse Source
This commit is contained in:
Ian Norden 2019-09-05 17:47:47 -05:00
parent 8b31d12716
commit 2c997921cb
6 changed files with 107 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
dockerfiles/seed_node/Dockerfile
View File

@ -1,16 +1,17 @@
FROM golang:alpine as builder FROM golang:alpine
ENV GO111MODULE=on
RUN apk --update --no-cache add make git g++ linux-headers RUN apk --update --no-cache add make git g++ linux-headers
# DEBUG # DEBUG
RUN apk add busybox-extras RUN apk add busybox-extras
# this is probably a noob move, but I want apk from alpine for the above but need to avoid Go 1.13 below as this error still occurs https://github.com/ipfs/go-ipfs/issues/6603
FROM golang:1.12.4 as builder
# Get and build vulcanizedb ipfs_concurreny fork # Get and build vulcanizedb ipfs_concurreny fork
RUN go get -u -d github.com/vulcanize/vulcanizedb RUN go get -u -d github.com/vulcanize/vulcanizedb
WORKDIR /go/src/github.com/vulcanize/vulcanizedb WORKDIR /go/src/github.com/vulcanize/vulcanizedb
RUN git checkout ipfs_concurrency RUN git checkout ipfs_concurrency
RUN GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -o vulcanizedb . RUN GO111MODULE=on GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -o vulcanizedb .
# Get and build vulcanize's go-ipfs fork # Get and build vulcanize's go-ipfs fork
RUN go get -u -d github.com/ipfs/go-ipfs RUN go get -u -d github.com/ipfs/go-ipfs
@ -18,14 +19,14 @@ WORKDIR /go/src/github.com/ipfs/go-ipfs
RUN git remote add vulcanize https://github.com/vulcanize/go-ipfs.git RUN git remote add vulcanize https://github.com/vulcanize/go-ipfs.git
RUN git fetch vulcanize RUN git fetch vulcanize
RUN git checkout -b pg_ipfs vulcanize/postgres_update RUN git checkout -b pg_ipfs vulcanize/postgres_update
RUN GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -o ipfs ./cmd/ipfs RUN GO111MODULE=on GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -o ipfs ./cmd/ipfs
# Get and build vulcanize's geth fork # Get and build vulcanize's geth fork
RUN go get -u -d github.com/ethereum/go-ethereum RUN go get -u -d github.com/ethereum/go-ethereum
WORKDIR /go/src/github.com/ethereum/go-ethereum WORKDIR /go/src/github.com/ethereum/go-ethereum
RUN git remote add vulcanize https://github.com/vulcanize/go-ethereum.git RUN git remote add vulcanize https://github.com/vulcanize/go-ethereum.git
RUN git fetch vulcanize RUN git fetch vulcanize
RUN git checkout -b statediff_geth vulcanize/rpc_statediffing RUN git checkout -b statediff_geth vulcanize/statediffing
RUN GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -o geth ./cmd/geth RUN GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -o geth ./cmd/geth
# Build migration tool # Build migration tool
@ -69,9 +70,6 @@ COPY --from=builder /go/src/github.com/vulcanize/vulcanizedb/db/migrations migra
COPY --from=builder /go/src/github.com/ipfs/go-ipfs/ipfs ipfs COPY --from=builder /go/src/github.com/ipfs/go-ipfs/ipfs ipfs
COPY --from=builder /go/src/github.com/ethereum/go-ethereum/geth geth COPY --from=builder /go/src/github.com/ethereum/go-ethereum/geth geth
# DEBUG EXPOSE 8080
COPY --from=builder /usr/bin/telnet /bin/telnet
EXPOSE 80
CMD ["./startup_script.sh"] CMD ["./startup_script.sh"]

2
dockerfiles/seed_node/startup_script.sh
View File

@ -56,7 +56,7 @@ if [ $? -eq 0 ]; then
echo "Beginning the syncPublishScreenAndServe vulcanizedb process" echo "Beginning the syncPublishScreenAndServe vulcanizedb process"
./vulcanizedb syncPublishScreenAndServe --config=config.toml 2>&1 | tee -a log.txt & ./vulcanizedb syncPublishScreenAndServe --config=config.toml 2>&1 | tee -a log.txt &
else else
echo "Could not initialize Postgres backed IPFS profile. Are the database details correct?" echo "Could not initialize state-diffing Geth."
exit exit
fi fi

106
documentation/seed-node.md
View File

@ -1,11 +1,11 @@
# Seed node commands # Seed Node
Vulcanizedb can act as an index for Ethereum data stored on IPFS through the use of the `syncAndPublish` and Vulcanizedb can act as an index for Ethereum data stored on IPFS through the use of the `syncAndPublish` and
`syncPublishScreenAndServe` commands. `syncPublishScreenAndServe` commands.
## Setup ## Manual Setup
These commands work in conjunction with a [state-diffing full Geth node](https://github.com/vulcanize/go-ethereum/tree/rpc_statediffing) These commands work in conjunction with a [state-diffing full Geth node](https://github.com/vulcanize/go-ethereum/tree/statediffing)
and IPFS. and IPFS.
### IPFS ### IPFS
@ -17,9 +17,7 @@ To start, download and install [IPFS](https://github.com/vulcanize/go-ipfs)
`make install` `make install`
If we want to use Postgres as our backing datastore, we need to use the vulcanize fork of go-ipfs. This fork supports If we want to use Postgres as our backing datastore, we need to use the vulcanize fork of go-ipfs.
the Postgres datastore plugin and has been adjusted to use `dep` instead of Go modules since Go modules cannot work with
un-versioned forks and we need to use an un-versioned fork of go-ipfs-config.
Start by adding the fork and switching over to it: Start by adding the fork and switching over to it:
@ -29,11 +27,9 @@ Start by adding the fork and switching over to it:
`git checkout -b postgres_update vulcanize/postgres_update` `git checkout -b postgres_update vulcanize/postgres_update`
Now install this fork of ipfs, first be sure to remove any previous installation. It is important to use the below command Now install this fork of ipfs, first be sure to remove any previous installation.
instead of using `make install`, as `make install` will default to using Go modules and this will wreck our dependencies
since Go modules cannot work with un-versioned forks.
`go install ./cmd/ipfs` `make install`
Check that is installed properly by running Check that is installed properly by running
@ -65,7 +61,7 @@ which has usage:
and will ask us to enter the password, avoiding storing it to an ENV variable. and will ask us to enter the password, avoiding storing it to an ENV variable.
Once we have initialized ipfs, that is all we need to do with it- we do not need to run a daemon during the subsequent processes. Once we have initialized ipfs, that is all we need to do with it- we do not need to run a daemon during the subsequent processes (in fact, we can't).
### Geth ### Geth
For Geth, we currently *require* a special fork, and we can set this up as follows: For Geth, we currently *require* a special fork, and we can set this up as follows:
@ -80,7 +76,7 @@ Begin by downloading geth and switching to the vulcanize/rpc_statediffing branch
`git fetch vulcanize` `git fetch vulcanize`
`git checkout -b rpc_statediffing vulcanize/rpc_statediffing` `git checkout -b statediffing vulcanize/statediffing`
Now, install this fork of geth (make sure any old versions have been uninstalled/binaries removed first) Now, install this fork of geth (make sure any old versions have been uninstalled/binaries removed first)
@ -159,7 +155,88 @@ The additional `server.ipcPath` and `server.wsEndpoint` fields are used to set w
the `syncPublishScreenAndServe` rpc server will expose itself to subscribing transformers over, respectively. the `syncPublishScreenAndServe` rpc server will expose itself to subscribing transformers over, respectively.
Any valid and available path and endpoint is acceptable, but keep in mind that this path and endpoint need to be known by transformers for them to subscribe to the seed node. Any valid and available path and endpoint is acceptable, but keep in mind that this path and endpoint need to be known by transformers for them to subscribe to the seed node.
#### Subscribing
## Dockerfile Setup
The below provides step-by-step directions for how to setup the seed node using the provided Dockerfile on an AWS Linux AMI instance.
Note that the instance will need sufficient memory and storage for this to work.
1. Install basic dependencies
```
sudo yum update
sudo yum install -y curl gpg gcc gcc-c++ make git
```
2. Install Go 1.12
```
wget https://dl.google.com/go/go1.12.6.linux-amd64.tar.gz
tar -xzf go1.12.6.linux-amd64.tar.gz
sudo mv go /usr/local
```
3. Edit .bash_profile to export GOPATH
```
export GOROOT=/usr/local/go
export GOPATH=$HOME/go
export PATH=$GOPATH/bin:$GOROOT/bin:$PATH
```
4. Install and setup Postgres
```
sudo yum install postgresql postgresql96-server
sudo service postgresql96 initdb
sudo service postgresql96 start
sudo -u postgres createuser -s ec2-user
sudo -u postgres createdb ec2-user
sudo su postgres
psql
ALTER USER "ec2-user" WITH SUPERUSER;
/q
exit
```
4b. Edit hba_file to trust connections
```
psql
SHOW hba_file;
/q
sudo vim {PATH_TO_FILE}
```
4c. Stop and restart Postgres server to affect changes
```
sudo service postgresql96 stop
sudo service postgresql96 start
```
5. Install and start Docker (exit and re-enter ec2 instance afterwards to affect changes)
```
sudo yum install -y docker
sudo service docker start
sudo usermod -aG docker ec2-user
```
6. Fetch the repository and switch to this working branch
```
go get github.com/vulcanize/vulcanizedb
cd $GOPATH/src/github.com/vulcanize/vulcanizedb
git checkout ipfs_concurrency
```
7. Create the db
```
createdb vulcanize_public
```
8. Build and run the Docker image
```
cd $GOPATH/src/github.com/vulcanize/vulcanizedb/dockerfiles/seed_node
docker build .
docker run --network host -e VDB_PG_CONNECT=postgres://localhost:5432/vulcanize_public?sslmode=disable {IMAGE_ID}
```
## Subscribing
A transformer can subscribe to the `syncPublishScreenAndServe` service over its ipc or ws endpoints, when subscribing the transformer A transformer can subscribe to the `syncPublishScreenAndServe` service over its ipc or ws endpoints, when subscribing the transformer
specifies which subsets of the synced data it is interested in and the server will forward only these data. specifies which subsets of the synced data it is interested in and the server will forward only these data.
@ -176,7 +253,7 @@ The config for `streamSubscribe` has the `subscribe` set of parameters, for exam
```toml ```toml
[subscription] [subscription]
path = "ws://127.0.0.1:2019" path = "ws://127.0.0.1:8080"
backfill = true backfill = true
backfillOnly = false backfillOnly = false
startingBlock = 0 startingBlock = 0
@ -253,4 +330,3 @@ if it has any addresses then the seed-node will only send storage nodes from the
array that can be filled with storage keys we want to filter storage data for. It is important to note that the storageKeys are the actual keccak256 hashes, whereas array that can be filled with storage keys we want to filter storage data for. It is important to note that the storageKeys are the actual keccak256 hashes, whereas
the addresses in the `addresses` fields are the ETH addresses and not their keccak256 hashes that serve as the actual state keys. By default the seed-node the addresses in the `addresses` fields are the ETH addresses and not their keccak256 hashes that serve as the actual state keys. By default the seed-node
only sends along storage leafs, if we want to receive branch and extension nodes as well `intermediateNodes` can be set to `true`. only sends along storage leafs, if we want to receive branch and extension nodes as well `intermediateNodes` can be set to `true`.

3
environments/syncPublishScreenAndServe.toml
View File

@ -2,6 +2,7 @@
name = "vulcanize_public" name = "vulcanize_public"
hostname = "localhost" hostname = "localhost"
port = 5432 port = 5432
user = "ec2-user"
[client] [client]
ipcPath = "ws://127.0.0.1:8546" ipcPath = "ws://127.0.0.1:8546"
@ -9,4 +10,4 @@
[server] [server]
ipcPath = "/root/.vulcanize/vulcanize.ipc" ipcPath = "/root/.vulcanize/vulcanize.ipc"
wsEndpoint = "127.0.0.1:80" wsEndpoint = "127.0.0.1:8080"

2
go.mod
View File

@ -94,7 +94,7 @@ replace github.com/dgraph-io/badger v2.0.0-rc.2+incompatible => github.com/dgrap
replace github.com/dgraph-io/badger/v2 v2.0.0-rc2 => github.com/dgraph-io/badger v1.6.0-rc1 replace github.com/dgraph-io/badger/v2 v2.0.0-rc2 => github.com/dgraph-io/badger v1.6.0-rc1
replace github.com/ipfs/go-ipfs v0.4.21 => github.com/vulcanize/go-ipfs v0.4.22-alpha replace github.com/ipfs/go-ipfs v0.4.22 => github.com/vulcanize/go-ipfs v0.4.22-alpha
replace github.com/ipfs/go-ipfs-config v0.0.3 => github.com/vulcanize/go-ipfs-config v0.0.8-alpha replace github.com/ipfs/go-ipfs-config v0.0.3 => github.com/vulcanize/go-ipfs-config v0.0.8-alpha

4
pkg/config/database.go
View File

@ -31,5 +31,9 @@ func DbConnectionString(dbConfig Database) string {
return fmt.Sprintf("postgresql://%s:%s@%s:%d/%s?sslmode=disable", return fmt.Sprintf("postgresql://%s:%s@%s:%d/%s?sslmode=disable",
dbConfig.User, dbConfig.Password, dbConfig.Hostname, dbConfig.Port, dbConfig.Name) dbConfig.User, dbConfig.Password, dbConfig.Hostname, dbConfig.Port, dbConfig.Name)
} }
if len(dbConfig.User) > 0 && len(dbConfig.Password) == 0 {
return fmt.Sprintf("postgresql://%s@%s:%d/%s?sslmode=disable",
dbConfig.User, dbConfig.Hostname, dbConfig.Port, dbConfig.Name)
}
return fmt.Sprintf("postgresql://%s:%d/%s?sslmode=disable", dbConfig.Hostname, dbConfig.Port, dbConfig.Name) return fmt.Sprintf("postgresql://%s:%d/%s?sslmode=disable", dbConfig.Hostname, dbConfig.Port, dbConfig.Name)
} }