cerc-io/ipld-eth-server
18
0
Fork 1
Code Issues 10 Pull Requests 1 Actions Packages Projects Releases 37 Wiki Activity

Merge pull request #6 from 8thlight/VDB-337-Dockerize-vDB

Browse Source 
VDB-337 Dockerize vDB
This commit is contained in:
Edvard Hübinette 2019-02-11 12:37:56 +01:00 committed by GitHub
commit f73623789e
14 changed files with 176 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
.dockerignore Normal file
View File

@ -0,0 +1,16 @@
.git
.travis.yml
.idea
bin
.gitignore
integration_test
LICENSE
postgraphile
.private_blockchain_password
README.md
scripts
Supfile
test_config
.travis.yml
vulcanizedb.log
Dockerfile

24
Dockerfile Normal file
View File

@ -0,0 +1,24 @@
FROM golang:alpine as builder
RUN apk --update --no-cache add make git g++
# Build statically linked vDB binary (wonky path because of Dep)
RUN mkdir -p /go/src/github.com/vulcanize/vulcanizedb
ADD . /go/src/github.com/vulcanize/vulcanizedb
WORKDIR /go/src/github.com/vulcanize/vulcanizedb
RUN GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' .
# Build migration tool
RUN go get -u -d github.com/pressly/goose/cmd/goose
WORKDIR /go/src/github.com/pressly/goose/cmd/goose
RUN GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -tags='no_mysql no_sqlite' -o goose
# Second stage
FROM alpine
COPY --from=builder /go/src/github.com/vulcanize/vulcanizedb/vulcanizedb /app/vulcanizedb
COPY --from=builder /go/src/github.com/vulcanize/vulcanizedb/environments/staging.toml /app/environments/
COPY --from=builder /go/src/github.com/vulcanize/vulcanizedb/dockerfiles/startup_script.sh /app/
COPY --from=builder /go/src/github.com/vulcanize/vulcanizedb/db/migrations/* /app/
COPY --from=builder /go/src/github.com/pressly/goose/cmd/goose/goose /app/goose
WORKDIR /app
CMD ["./startup_script.sh"]

4
cmd/root.go
View File

@ -19,6 +19,7 @@ package cmd
import (
"fmt"
"os"
"strings"
"time"
"github.com/ethereum/go-ethereum/ethclient"
@ -78,6 +79,9 @@ func database(cmd *cobra.Command, args []string) {
func init() {
cobra.OnInitialize(initConfig)
// When searching for env variables, replace dots in config keys with underscores
viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_"))
viper.AutomaticEnv()
rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file location")
rootCmd.PersistentFlags().String("database-name", "vulcanize_public", "database name")

32
dockerfiles/README.md Normal file
View File

@ -0,0 +1,32 @@
S
`Dockerfile` will build an alpine image containing:
- vDB as a binary with runtime deps statically linked: `/app/vulcanizedb`
- The migration tool goose: `/app/goose`
- Two services for running `lightSync` and `continuousLogSync`, started with the default configuration `environments/staging.toml`.
By default, vDB is configured towards the Kovan deploy. The configuration values can be overridden using environment variables, using the same hierarchical naming pattern but in CAPS and using underscores. For example, the contract address for the `Pit` can be set with the variable `CONTRACT_ADDRESS_PIT="0x123..."`.
## To use the container:
1. Setup a postgres database with superuser `vulcanize`
2. Set the env variables `DATABASE_NAME`, `DATABASE_HOSTNAME`,
`DATABASE_PORT`, `DATABASE_USER` & `DATABASE_PASSWORD`
3. Run the DB migrations:
* `./goose postgres "postgresql://$(DATABASE_USER):$(DATABASE_PASSWORD)@$(DATABASE_HOSTNAME):$(DATABASE_PORT)/$(DATABASE_NAME)?sslmode=disable"
e`
4. Set `CLIENT_IPCPATH` to a node endpoint
5. Set the contract variables:
* `CONTRACT_ADDRESS_[CONTRACT NAME]=0x123...`
* `CONTRACT_ABI_[CONTRACT NAME]="ABI STRING"`
* `CONTRACT_DEPLOYMENT-BLOCK_[CONTRACT NAME]=0` (doesn't really matter on a short chain, just avoids long unnecessary searching)
6. Start the `lightSync` and `continuousLogSync` services:
* `./vulcanizedb lightSync --config environments/staging.toml`
* `./vulcanizedb continuousLogSync --config environments/staging.toml`
### Automated
The steps above have been rolled into a script: `/app/startup_script.sh`, which just assumes the DB env variables have been set, and defaults the rest to Kovan according to `environments/staging.toml`. This can be called with something like:
`docker run -d -e DATABASE_NAME=vulcanize_public -e DATABASE_HOSTNAME=localhost -e DATABASE_PORT=5432 -e DATABASE_USER=vulcanize -e DATABASE_PASSWORD=vulcanize m0ar/images:vDB`
### Logging
When running, vDB services log to `/app/vulcanizedb.log`.

29
dockerfiles/startup_script.sh Executable file
View File

@ -0,0 +1,29 @@
#!/bin/sh
# Runs the migrations and starts the lightSync and continuousLogSync services
# Exit if the variable tests fail
set -e
# Check the database variables are set
test $DATABASE_NAME
test $DATABASE_HOSTNAME
test $DATABASE_PORT
test $DATABASE_USER
test $DATABASE_PASSWORD
# Construct the connection string for postgres
CONNECT_STRING=postgresql://$DATABASE_USER:$DATABASE_PASSWORD@$DATABASE_HOSTNAME:$DATABASE_PORT/$DATABASE_NAME?sslmode=disable
echo "Connecting with: $CONNECT_STRING"
set +e
# Run the DB migrations
./goose postgres "$CONNECT_STRING" up
if [ $? -eq 0 ]; then
# Fire up the services
./vulcanizedb lightSync --config environments/staging.toml &
./vulcanizedb continuousLogSync --config environments/staging.toml &
else
echo "Could not run migrations. Are the database details correct?"
fi
wait

4
environments/staging.toml
View File

@ -1,8 +1,4 @@
[database]
name = "vulcanize_public"
hostname = "localhost"
user = "vulcanize"
password = "vulcanize"
port = 5432
[client]

2
pkg/datastore/postgres/postgres.go
View File

@ -18,6 +18,7 @@ package postgres
import (
"errors"
"github.com/sirupsen/logrus"
"github.com/jmoiron/sqlx"
_ "github.com/lib/pq" //postgres driver
@ -40,6 +41,7 @@ var (
func NewDB(databaseConfig config.Database, node core.Node) (*DB, error) {
connectString := config.DbConnectionString(databaseConfig)
logrus.Info("Using connection string: ", connectString)
db, err := sqlx.Connect("postgres", connectString)
if err != nil {
return &DB{}, ErrDBConnectionFailed

20
pkg/transformers/shared/constants/external.go
View File

@ -6,23 +6,7 @@ import (
"github.com/spf13/viper"
)
var initialized = false
func initConfig() {
if initialized {
return
}
if err := viper.ReadInConfig(); err == nil {
fmt.Printf("Using config file: %s\n\n", viper.ConfigFileUsed())
} else {
panic(fmt.Sprintf("Could not find environment file: %v", err))
}
initialized = true
}
func getEnvironmentString(key string) string {
initConfig()
value := viper.GetString(key)
if value == "" {
panic(fmt.Sprintf("No environment configuration variable set for key: \"%v\"", key))
@ -30,11 +14,11 @@ func getEnvironmentString(key string) string {
return value
}
// Returns an int from the environment, defaulting to 0 if it does not exist
func getEnvironmentInt64(key string) int64 {
initConfig()
value := viper.GetInt64(key)
if value == -1 {
panic(fmt.Sprintf("No environment configuration variable set for key: \"%v\"", key))
return 0
}
return value
}

7
postgraphile/.dockerignore Normal file
View File

@ -0,0 +1,7 @@
package-lock.json
yarn.lock
node_modules
Dockerfile
README.md
spec
.dockerignore

9
postgraphile/Dockerfile Normal file
View File

@ -0,0 +1,9 @@
FROM mhart/alpine-node:10
RUN apk --update --no-cache add make g++ python findutils postgresql-dev
WORKDIR /app
COPY . /app
run yarn install
RUN ["./node_modules/typescript/bin/tsc"]
EXPOSE 3000
CMD ["node", "/app/build/dist/index.js"]

12
postgraphile/README.md
View File

@ -2,6 +2,18 @@
This application utilizes Postgraphile to expose GraphQL endpoints for exposure of the varied data that VulcanizeDB tracks.
## Docker use
_Note: currently this image is ~500MB large (unpacked)_
Build the docker image in this directory. Start the `GraphiQL` frontend by:
* Setting the env variables for the database connection: `DATABASE_HOST`,
`DATABASE_NAME`, `DATABASE_USER`, `DATABASE_PASSWORD` (and optionally
`DATABASE_PORT` if running on non-standard port).
* The specified user needs to be `superuser` on the vulcanizeDB database
* Run the container (ex. `docker run -e DATABASE_HOST=localhost -e DATABASE_NAME=vulcanize_public -e DATABASE_USER=vulcanize -e DATABASE_PASSWORD=vulcanize -d m0ar/images:postgraphile-alpine`)
* GraphiQL is available at `:3000/graphiql`
## Building
*This application assumes the use of the [Yarn package manager](https://yarnpkg.com/en/). The use of npm may produce unexpected results.*

10
postgraphile/package.json
View File

@ -22,22 +22,22 @@
"homepage": "https://github.com/vulcanize/vulcanizedb",
"dependencies": {
"express-session": "1.15.6",
"graphql-subscriptions": "0.5.8",
"lodash": "4.17.10",
"passport": "0.4.0",
"pg": "6.4.2",
"pg-native": "3.0.0",
"postgraphile": "4.0.0-rc.4",
"graphql-subscriptions": "0.5.8",
"subscriptions-transport-ws": "0.9.14",
"toml": "2.3.3",
"pg": "6.4.2"
"toml": "2.3.3"
},
"devDependencies": {
"@types/graphql": "^0.13.4",
"@types/express": "4.16.0",
"@types/express-session": "1.15.10",
"@types/graphql": "^0.13.4",
"@types/jasmine": "2.8.8",
"@types/lodash": "4.14.116",
"@types/node": "10.9.3",
"@types/node": "^10.12.21",
"@types/passport": "0.4.6",
"awesome-typescript-loader": "5.2.0",
"jasmine": "3.2.0",

43
postgraphile/src/config/parse.ts
View File

@ -6,34 +6,49 @@ export const MISSING_PATH_MESSAGE = `No path to config toml file provided, `
+ `please check the value of ${CONFIG_PATH_KEY} in your environment`;
export const MISSING_HOST_MESSAGE = 'No database host provided in config toml';
export const MISSING_DATABASE_MESSAGE = 'No database name provided in config '
+ 'toml';
export const MISSING_USER_MESSAGE = 'No database user & password provided in config toml';
export const MISSING_DATABASE_MESSAGE = 'No database name provided in config toml';
export function parseConfig(
readCallback: ReadFileSyncCallback,
tomlParseCallback: TomlParseCallback,
configPath?: string
): DatabaseConfig {
if (!configPath || configPath.length < 1) {
throw new Error(MISSING_PATH_MESSAGE);
let host = '';
let port = '';
let database = '';
let user = '';
let password = '';
if (configPath) {
const tomlContents = readCallback(`${configPath}`).toString();
const parsedToml = tomlParseCallback(tomlContents);
host = parsedToml['database']['hostname'];
port = parsedToml['database']['port'];
database = parsedToml['database']['name'];
user = parsedToml['database']['user'];
password = parsedToml['database']['password'];
}
const tomlContents = readCallback(`${configPath}`).toString();
const parsedToml = tomlParseCallback(tomlContents);
// Overwrite config values with env. vars if such are set
host = process.env.DATABASE_HOST || host;
port = process.env.DATABASE_PORT || port;
database = process.env.DATABASE_NAME || database;
user = process.env.DATABASE_USER || user;
password = process.env.DATABASE_PASSWORD || password;
const host = parsedToml['database']['hostname'];
const port = parsedToml['database']['port'];
const database = parsedToml['database']['name'];
const user = parsedToml['database']['user'] || '';
const password = parsedToml['database']['password'] || '';
if (!host || host.length < 1) {
if (!host) {
throw new Error(MISSING_HOST_MESSAGE);
}
if (!database || database.length < 1) {
if (!database) {
throw new Error(MISSING_DATABASE_MESSAGE);
}
if (!user || !password) {
throw new Error(MISSING_USER_MESSAGE);
}
return { host: `postgres://${user}:${password}@${host}:${port}`, database };
}

7
postgraphile/yarn.lock
View File

@ -108,9 +108,10 @@
version "10.9.4"
resolved "https://registry.yarnpkg.com/@types/node/-/node-10.9.4.tgz#0f4cb2dc7c1de6096055357f70179043c33e9897"
"@types/node@10.9.3":
version "10.9.3"
resolved "https://registry.yarnpkg.com/@types/node/-/node-10.9.3.tgz#85f288502503ade0b3bfc049fe1777b05d0327d5"
"@types/node@^10.12.21":
version "10.12.21"
resolved "https://registry.yarnpkg.com/@types/node/-/node-10.12.21.tgz#7e8a0c34cf29f4e17a36e9bd0ea72d45ba03908e"
integrity sha512-CBgLNk4o3XMnqMc0rhb6lc77IwShMEglz05deDcn2lQxyXEZivfwgYJu7SMha9V5XcrP6qZuevTHV/QrN2vjKQ==
"@types/passport@0.4.6":
version "0.4.6"