From d3550357d10966efc2d12d8125dd8c6fa1eb27b3 Mon Sep 17 00:00:00 2001 From: Edvard Date: Mon, 21 Jan 2019 11:17:39 +0100 Subject: [PATCH 1/8] Add initial dockerfile and .dockerignore --- .dockerignore | 18 ++++++++++++++++++ Dockerfile | 14 ++++++++++++++ 2 files changed, 32 insertions(+) create mode 100644 .dockerignore create mode 100644 Dockerfile diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..01c7726c --- /dev/null +++ b/.dockerignore @@ -0,0 +1,18 @@ +.git +.travis.yml +.idea +bin +db +.gitignore +environments +integration_test +LICENSE +postgraphile +.private_blockchain_password +README.md +scripts +Supfile +test_config +.travis.yml +vulcanizedb.log + diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..f749245e --- /dev/null +++ b/Dockerfile @@ -0,0 +1,14 @@ +FROM golang:alpine as builder +RUN apk --update --no-cache add make git g++ + +# Build statically linked vDB binary (wonky path because of Dep) +RUN mkdir -p /go/src/github.com/vulcanize/vulcanizedb +ADD . /go/src/github.com/vulcanize/vulcanizedb +WORKDIR /go/src/github.com/vulcanize/vulcanizedb +RUN GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' . + +# Second stage +FROM scratch +COPY --from=builder /go/src/github.com/vulcanize/vulcanizedb/vulcanizedb /app/vulcanizedb +WORKDIR /app +CMD ["./vulcanizedb", "--help"] From 1602100ed464b178b9d10902d6d7e33eb7fb612b Mon Sep 17 00:00:00 2001 From: Edvard Date: Thu, 24 Jan 2019 17:13:55 +0100 Subject: [PATCH 2/8] Dockerize migrations --- .dockerignore | 2 +- Dockerfile | 11 +++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/.dockerignore b/.dockerignore index 01c7726c..47e3a24c 100644 --- a/.dockerignore +++ b/.dockerignore @@ -2,7 +2,6 @@ .travis.yml .idea bin -db .gitignore environments integration_test @@ -15,4 +14,5 @@ Supfile test_config .travis.yml vulcanizedb.log +Dockerfile diff --git a/Dockerfile b/Dockerfile index f749245e..ed79055c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,8 +7,15 @@ ADD . /go/src/github.com/vulcanize/vulcanizedb WORKDIR /go/src/github.com/vulcanize/vulcanizedb RUN GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' . +# Build migration tool +RUN go get -u -d github.com/pressly/goose/cmd/goose +WORKDIR /go/src/github.com/pressly/goose/cmd/goose +RUN GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -tags='no_mysql no_sqlite' -o goose + # Second stage -FROM scratch +FROM alpine COPY --from=builder /go/src/github.com/vulcanize/vulcanizedb/vulcanizedb /app/vulcanizedb +COPY --from=builder /go/src/github.com/pressly/goose/cmd/goose/goose /app/goose +COPY --from=builder /go/src/github.com/vulcanize/vulcanizedb/db/migrations/* /app/ WORKDIR /app -CMD ["./vulcanizedb", "--help"] +CMD ["./goose"] From e4968c4a75fa65b9e0910d1cad3d6f11267330eb Mon Sep 17 00:00:00 2001 From: Edvard Date: Tue, 29 Jan 2019 17:09:08 +0100 Subject: [PATCH 3/8] Add back override with env variables --- cmd/root.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cmd/root.go b/cmd/root.go index c0d7e7ef..248af4e1 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -111,6 +111,9 @@ func initConfig() { os.Exit(1) } + // Config values from environment overrides file + viper.AutomaticEnv() + if err := viper.ReadInConfig(); err == nil { log.Printf("Using config file: %s\n\n", viper.ConfigFileUsed()) } else { From e6e66d58dd8519161a7a9f2be7792ef76dce0f63 Mon Sep 17 00:00:00 2001 From: Edvard Date: Wed, 30 Jan 2019 13:18:24 +0100 Subject: [PATCH 4/8] Add container service files and docker README --- .dockerignore | 2 -- Dockerfile | 10 +++++++- dockerfiles/README.md | 33 +++++++++++++++++++++++++++ dockerfiles/continuousLogSync-service | 6 +++++ dockerfiles/lightSync-service | 6 +++++ dockerfiles/startup_script.sh | 29 +++++++++++++++++++++++ pkg/transformers/frob/converter.go | 3 --- 7 files changed, 83 insertions(+), 6 deletions(-) create mode 100644 dockerfiles/README.md create mode 100755 dockerfiles/continuousLogSync-service create mode 100755 dockerfiles/lightSync-service create mode 100755 dockerfiles/startup_script.sh diff --git a/.dockerignore b/.dockerignore index 47e3a24c..189c28b0 100644 --- a/.dockerignore +++ b/.dockerignore @@ -3,7 +3,6 @@ .idea bin .gitignore -environments integration_test LICENSE postgraphile @@ -15,4 +14,3 @@ test_config .travis.yml vulcanizedb.log Dockerfile - diff --git a/Dockerfile b/Dockerfile index ed79055c..056f60db 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,8 +14,16 @@ RUN GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflag # Second stage FROM alpine +RUN apk --no-cache --update add openrc +RUN mkdir -p /run/openrc/ && touch /run/openrc/softlevel COPY --from=builder /go/src/github.com/vulcanize/vulcanizedb/vulcanizedb /app/vulcanizedb -COPY --from=builder /go/src/github.com/pressly/goose/cmd/goose/goose /app/goose +COPY --from=builder /go/src/github.com/vulcanize/vulcanizedb/environments/staging.toml /app/environments/ +COPY --from=builder /go/src/github.com/vulcanize/vulcanizedb/dockerfiles/lightSync-service /etc/init.d/lightSync +COPY --from=builder /go/src/github.com/vulcanize/vulcanizedb/dockerfiles/continuousLogSync-service /etc/init.d/continuousLogSync +COPY --from=builder /go/src/github.com/vulcanize/vulcanizedb/dockerfiles/startup_script.sh /app/ COPY --from=builder /go/src/github.com/vulcanize/vulcanizedb/db/migrations/* /app/ +COPY --from=builder /go/src/github.com/pressly/goose/cmd/goose/goose /app/goose + WORKDIR /app +RUN rc-status CMD ["./goose"] diff --git a/dockerfiles/README.md b/dockerfiles/README.md new file mode 100644 index 00000000..d5e8fca3 --- /dev/null +++ b/dockerfiles/README.md @@ -0,0 +1,33 @@ +S +`Dockerfile` will build an alpine image containing: +- vDB as a binary with runtime deps statically linked: `/app/vulcanizedb` +- The migration tool goose: `/app/goose` +- Two services for running `lightSync` and `continuousLogSync`, started with the default configuration `environments/staging.toml`. + +By default, vDB is configured towards the Kovan deploy. The configuration values can be overridden using environment variables, using the same hierarchical naming pattern but in CAPS and using underscores. For example, the contract address for the `Pit` can be set with the variable `CONTRACT_ADDRESS_PIT="0x123..."`. + +## To use the container: +1. Setup a postgres database with owner `vulcanize` + * `vulcanize` does not _have_ to be owner, but otherwise permissions for + connection, tables, and sequences need to be added for the `public` and + `maker` schemas manually when the migrations have been run. +2. Set the env variables `DATABASE_NAME`, `DATABASE_HOSTNAME`, + `DATABASE_PORT`, `DATABASE_USER` & `DATABASE_PASSWORD` +3. Run the DB migrations: + * `./goose postgres "postgresql://$(DATABASE_USER):$(DATABASE_PASSWORD)@$(DATABASE_HOSTNAME):$(DATABASE_PORT)/$(DATABASE_NAME)?sslmode=disable" +e` +4. Set `CLIENT_IPCPATH` to a node endpoint +5. Set the contract variables: + * `CONTRACT_ADDRESS_[CONTRACT NAME]=0x123...` + * `CONTRACT_ABI_[CONTRACT NAME]="ABI STRING"` + * `CONTRACT_DEPLOYMENT-BLOCK_[CONTRACT NAME]=0` (doesn't really matter on a short chain, just avoids long unnecessary searching) +6. Start the `lightSync` and `continuousLogSync` services: + * `rc-service lightSync start` + * `rc-service continuousLogSync start` + +### Automated +The steps above have been rolled into a script: `/app/startup_script.sh`, which just assumes the DB env variables have been set, and defaults the rest to Kovan according to `environments/staging.toml`. + +### Logging +When running, vDB services log to `/vulcanizedb.log`. + diff --git a/dockerfiles/continuousLogSync-service b/dockerfiles/continuousLogSync-service new file mode 100755 index 00000000..9c3adfd5 --- /dev/null +++ b/dockerfiles/continuousLogSync-service @@ -0,0 +1,6 @@ +#!/sbin/openrc-run +command=/app/vulcanizedb +command_args="continuousLogSync --config /app/environments/staging.toml &" +pidfile= +name="continuousLogSync" +description="runs vDB in continuousLogSync mode using staging.toml" diff --git a/dockerfiles/lightSync-service b/dockerfiles/lightSync-service new file mode 100755 index 00000000..a9143de1 --- /dev/null +++ b/dockerfiles/lightSync-service @@ -0,0 +1,6 @@ +#!/sbin/openrc-run +command=/app/vulcanizedb +command_args="lightSync --config /app/environments/staging.toml &" +pidfile= +name="lightSync" +description="runs vDB in lightSync mode using staging.toml" diff --git a/dockerfiles/startup_script.sh b/dockerfiles/startup_script.sh new file mode 100755 index 00000000..47547d99 --- /dev/null +++ b/dockerfiles/startup_script.sh @@ -0,0 +1,29 @@ +#!/bin/sh +# Runs the migrations and starts the lightSync and continuousLogSync services + +# Exit if the variable tests fail +set -e + +# Check the database variables are set +test $DATABASE_NAME +test $DATABASE_HOSTNAME +test $DATABASE_PORT +test $DATABASE_USER +test $DATABASE_PASSWORD + +# Construct the connection string for postgres +CONNECT_STRING=postgresql://$DATABASE_USER:$DATABASE_PASSWORD@$DATABASE_HOSTNAME:$DATABASE_PORT/$DATABASE_NAME?sslmode=disable +echo "Connecting with: $CONNECT_STRING" + +set +e + +# Run the DB migrations +./goose postgres "$CONNECT_STRING" up +if [ $? -eq 0 ]; then + # Fire up the services + rc-service lightSync start + rc-service continuousLogSync start +else + echo "Could not run migrations. Are the database details correct?" +fi + diff --git a/pkg/transformers/frob/converter.go b/pkg/transformers/frob/converter.go index 267626af..9d031ca9 100644 --- a/pkg/transformers/frob/converter.go +++ b/pkg/transformers/frob/converter.go @@ -19,8 +19,6 @@ package frob import ( "encoding/json" "fmt" - log "github.com/sirupsen/logrus" - "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" @@ -32,7 +30,6 @@ type FrobConverter struct{} func (FrobConverter) ToEntities(contractAbi string, ethLogs []types.Log) ([]interface{}, error) { var entities []interface{} - log.Info("blah") for _, ethLog := range ethLogs { entity := FrobEntity{} address := ethLog.Address From 85cec480dca98a802d3d9f22d92dbe8a3b41acbe Mon Sep 17 00:00:00 2001 From: Taka Goto Date: Thu, 31 Jan 2019 21:27:23 -0600 Subject: [PATCH 5/8] Dockerize postgraphile --- dockerfiles/README.md | 9 +++---- postgraphile/.dockerignore | 7 ++++++ postgraphile/Dockerfile | 9 +++++++ postgraphile/README.md | 14 ++++++++++- postgraphile/package.json | 10 ++++---- postgraphile/src/config/parse.ts | 43 +++++++++++++++++++++----------- postgraphile/yarn.lock | 7 +++--- 7 files changed, 71 insertions(+), 28 deletions(-) create mode 100644 postgraphile/.dockerignore create mode 100644 postgraphile/Dockerfile diff --git a/dockerfiles/README.md b/dockerfiles/README.md index d5e8fca3..da98e79e 100644 --- a/dockerfiles/README.md +++ b/dockerfiles/README.md @@ -7,10 +7,7 @@ S By default, vDB is configured towards the Kovan deploy. The configuration values can be overridden using environment variables, using the same hierarchical naming pattern but in CAPS and using underscores. For example, the contract address for the `Pit` can be set with the variable `CONTRACT_ADDRESS_PIT="0x123..."`. ## To use the container: -1. Setup a postgres database with owner `vulcanize` - * `vulcanize` does not _have_ to be owner, but otherwise permissions for - connection, tables, and sequences need to be added for the `public` and - `maker` schemas manually when the migrations have been run. +1. Setup a postgres database with superuser `vulcanize` 2. Set the env variables `DATABASE_NAME`, `DATABASE_HOSTNAME`, `DATABASE_PORT`, `DATABASE_USER` & `DATABASE_PASSWORD` 3. Run the DB migrations: @@ -26,7 +23,9 @@ e` * `rc-service continuousLogSync start` ### Automated -The steps above have been rolled into a script: `/app/startup_script.sh`, which just assumes the DB env variables have been set, and defaults the rest to Kovan according to `environments/staging.toml`. +The steps above have been rolled into a script: `/app/startup_script.sh`, which just assumes the DB env variables have been set, and defaults the rest to Kovan according to `environments/staging.toml`. This can be called with something like: + +`docker run -d -e DATABASE_NAME=vulcanize_pubic -e DATABASE_HOSTNAME=localhost -e DATABASE_PORT=5432 -e DATABASE_USER=vulcanize -e DATABASE_PASSWORD=vulcanize ./startup_script.sh` ### Logging When running, vDB services log to `/vulcanizedb.log`. diff --git a/postgraphile/.dockerignore b/postgraphile/.dockerignore new file mode 100644 index 00000000..1e5df4d8 --- /dev/null +++ b/postgraphile/.dockerignore @@ -0,0 +1,7 @@ +package-lock.json +yarn.lock +node_modules +Dockerfile +README.md +spec +.dockerignore diff --git a/postgraphile/Dockerfile b/postgraphile/Dockerfile new file mode 100644 index 00000000..4a98c0e3 --- /dev/null +++ b/postgraphile/Dockerfile @@ -0,0 +1,9 @@ +FROM mhart/alpine-node:10 +RUN apk --update --no-cache add make g++ python findutils postgresql-dev + +WORKDIR /app +COPY . /app +run yarn install +RUN ["./node_modules/typescript/bin/tsc"] +EXPOSE 3000 +CMD ["node", "/app/build/dist/src/index.js"] diff --git a/postgraphile/README.md b/postgraphile/README.md index 684247da..048a1f7a 100644 --- a/postgraphile/README.md +++ b/postgraphile/README.md @@ -2,6 +2,18 @@ This application utilizes Postgraphile to expose GraphQL endpoints for exposure of the varied data that VulcanizeDB tracks. +## Docker use +_Note: currently this image is ~500MB large (unpacked)_ + +Build the docker image in this directory. Start the `GraphiQL` frontend by: +* Setting the env variables for the database connection: `DATABASE_HOST`, + `DATABASE_NAME`, `DATABASE_USER`, `DATABASE_PASSWORD` (and optionally + `DATABASE_PORT` if running on non-standard port). + * The specified user needs to be `superuser` on the vulcanizeDB database +* Run the container (ex. `docker run -e DATABASE_HOST=localhost -e DATABASE_NAME=vulcanize_public -e DATABASE_USER=vulcanize -e DATABASE_PASSWORD=vulcanize -d postgraphile:latest`) +* GraphiQL is available at `:3000/graphiql` + + ## Building *This application assumes the use of the [Yarn package manager](https://yarnpkg.com/en/). The use of npm may produce unexpected results.* @@ -16,4 +28,4 @@ Provide the built bundle to node as a runnable script: `node ./build/dist/vulcan ## Testing -Tests are executed via Jasmine with a console reporter via the `yarn test` task. \ No newline at end of file +Tests are executed via Jasmine with a console reporter via the `yarn test` task. diff --git a/postgraphile/package.json b/postgraphile/package.json index 9c3dc3fa..9809865a 100644 --- a/postgraphile/package.json +++ b/postgraphile/package.json @@ -22,22 +22,22 @@ "homepage": "https://github.com/vulcanize/vulcanizedb", "dependencies": { "express-session": "1.15.6", + "graphql-subscriptions": "0.5.8", "lodash": "4.17.10", "passport": "0.4.0", + "pg": "6.4.2", "pg-native": "3.0.0", "postgraphile": "4.0.0-rc.4", - "graphql-subscriptions": "0.5.8", "subscriptions-transport-ws": "0.9.14", - "toml": "2.3.3", - "pg": "6.4.2" + "toml": "2.3.3" }, "devDependencies": { - "@types/graphql": "^0.13.4", "@types/express": "4.16.0", "@types/express-session": "1.15.10", + "@types/graphql": "^0.13.4", "@types/jasmine": "2.8.8", "@types/lodash": "4.14.116", - "@types/node": "10.9.3", + "@types/node": "^10.12.21", "@types/passport": "0.4.6", "awesome-typescript-loader": "5.2.0", "jasmine": "3.2.0", diff --git a/postgraphile/src/config/parse.ts b/postgraphile/src/config/parse.ts index 762ba18b..513aa4fe 100644 --- a/postgraphile/src/config/parse.ts +++ b/postgraphile/src/config/parse.ts @@ -6,34 +6,49 @@ export const MISSING_PATH_MESSAGE = `No path to config toml file provided, ` + `please check the value of ${CONFIG_PATH_KEY} in your environment`; export const MISSING_HOST_MESSAGE = 'No database host provided in config toml'; -export const MISSING_DATABASE_MESSAGE = 'No database name provided in config ' - + 'toml'; +export const MISSING_USER_MESSAGE = 'No database user & password provided in config toml'; +export const MISSING_DATABASE_MESSAGE = 'No database name provided in config toml'; export function parseConfig( readCallback: ReadFileSyncCallback, tomlParseCallback: TomlParseCallback, configPath?: string ): DatabaseConfig { - if (!configPath || configPath.length < 1) { - throw new Error(MISSING_PATH_MESSAGE); + let host = ''; + let port = ''; + let database = ''; + let user = ''; + let password = ''; + + if (configPath) { + const tomlContents = readCallback(`${configPath}`).toString(); + const parsedToml = tomlParseCallback(tomlContents); + + host = parsedToml['database']['hostname']; + port = parsedToml['database']['port']; + database = parsedToml['database']['name']; + user = parsedToml['database']['user']; + password = parsedToml['database']['password']; } - const tomlContents = readCallback(`${configPath}`).toString(); - const parsedToml = tomlParseCallback(tomlContents); + // Overwrite config values with env. vars if such are set + host = process.env.DATABASE_HOST || host; + port = process.env.DATABASE_PORT || port; + database = process.env.DATABASE_NAME || database; + user = process.env.DATABASE_USER || user; + password = process.env.DATABASE_PASSWORD || password; - const host = parsedToml['database']['hostname']; - const port = parsedToml['database']['port']; - const database = parsedToml['database']['name']; - const user = parsedToml['database']['user'] || ''; - const password = parsedToml['database']['password'] || ''; - - if (!host || host.length < 1) { + if (!host) { throw new Error(MISSING_HOST_MESSAGE); } - if (!database || database.length < 1) { + if (!database) { throw new Error(MISSING_DATABASE_MESSAGE); } + if (!user || !password) { + throw new Error(MISSING_USER_MESSAGE); + } + return { host: `postgres://${user}:${password}@${host}:${port}`, database }; } diff --git a/postgraphile/yarn.lock b/postgraphile/yarn.lock index 825c7c4e..c0e913de 100644 --- a/postgraphile/yarn.lock +++ b/postgraphile/yarn.lock @@ -108,9 +108,10 @@ version "10.9.4" resolved "https://registry.yarnpkg.com/@types/node/-/node-10.9.4.tgz#0f4cb2dc7c1de6096055357f70179043c33e9897" -"@types/node@10.9.3": - version "10.9.3" - resolved "https://registry.yarnpkg.com/@types/node/-/node-10.9.3.tgz#85f288502503ade0b3bfc049fe1777b05d0327d5" +"@types/node@^10.12.21": + version "10.12.21" + resolved "https://registry.yarnpkg.com/@types/node/-/node-10.12.21.tgz#7e8a0c34cf29f4e17a36e9bd0ea72d45ba03908e" + integrity sha512-CBgLNk4o3XMnqMc0rhb6lc77IwShMEglz05deDcn2lQxyXEZivfwgYJu7SMha9V5XcrP6qZuevTHV/QrN2vjKQ== "@types/passport@0.4.6": version "0.4.6" From 553fdbfa093819992bb3d457b9378f4322aaeaac Mon Sep 17 00:00:00 2001 From: Edvard Date: Tue, 5 Feb 2019 20:05:41 +0100 Subject: [PATCH 6/8] Default config starting block to 0 --- dockerfiles/README.md | 2 +- pkg/transformers/shared/constants/external.go | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/dockerfiles/README.md b/dockerfiles/README.md index da98e79e..ea6e19c9 100644 --- a/dockerfiles/README.md +++ b/dockerfiles/README.md @@ -25,7 +25,7 @@ e` ### Automated The steps above have been rolled into a script: `/app/startup_script.sh`, which just assumes the DB env variables have been set, and defaults the rest to Kovan according to `environments/staging.toml`. This can be called with something like: -`docker run -d -e DATABASE_NAME=vulcanize_pubic -e DATABASE_HOSTNAME=localhost -e DATABASE_PORT=5432 -e DATABASE_USER=vulcanize -e DATABASE_PASSWORD=vulcanize ./startup_script.sh` +`docker run -d -e DATABASE_NAME=vulcanize_public -e DATABASE_HOSTNAME=localhost -e DATABASE_PORT=5432 -e DATABASE_USER=vulcanize -e DATABASE_PASSWORD=vulcanize [container name] ./startup_script.sh` ### Logging When running, vDB services log to `/vulcanizedb.log`. diff --git a/pkg/transformers/shared/constants/external.go b/pkg/transformers/shared/constants/external.go index 154f56f8..d18270be 100644 --- a/pkg/transformers/shared/constants/external.go +++ b/pkg/transformers/shared/constants/external.go @@ -30,11 +30,12 @@ func getEnvironmentString(key string) string { return value } +// Returns an int from the environment, defaulting to 0 if it does not exist func getEnvironmentInt64(key string) int64 { initConfig() value := viper.GetInt64(key) if value == -1 { - panic(fmt.Sprintf("No environment configuration variable set for key: \"%v\"", key)) + return 0 } return value } From 0bf8bd4183f6d31b52207e3a6e79829c8c89ad63 Mon Sep 17 00:00:00 2001 From: Edvard Date: Wed, 6 Feb 2019 13:27:30 +0100 Subject: [PATCH 7/8] Add char replacer when searching for env variables --- cmd/root.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index 248af4e1..6b72c593 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -19,6 +19,7 @@ package cmd import ( "fmt" "os" + "strings" "time" "github.com/ethereum/go-ethereum/ethclient" @@ -78,6 +79,9 @@ func database(cmd *cobra.Command, args []string) { func init() { cobra.OnInitialize(initConfig) + // When searching for env variables, replace dots in config keys with underscores + viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) + viper.AutomaticEnv() rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file location") rootCmd.PersistentFlags().String("database-name", "vulcanize_public", "database name") @@ -111,9 +115,6 @@ func initConfig() { os.Exit(1) } - // Config values from environment overrides file - viper.AutomaticEnv() - if err := viper.ReadInConfig(); err == nil { log.Printf("Using config file: %s\n\n", viper.ConfigFileUsed()) } else { From 9b986c4c144210025a9d3a67808f52d9aa4db5ee Mon Sep 17 00:00:00 2001 From: Edvard Date: Wed, 6 Feb 2019 16:35:51 +0100 Subject: [PATCH 8/8] Nuke OpenRC from docker image, update readme's --- Dockerfile | 7 +------ dockerfiles/README.md | 8 ++++---- dockerfiles/continuousLogSync-service | 6 ------ dockerfiles/lightSync-service | 6 ------ dockerfiles/startup_script.sh | 6 +++--- environments/staging.toml | 4 ---- pkg/datastore/postgres/postgres.go | 2 ++ pkg/transformers/shared/constants/external.go | 17 ----------------- postgraphile/Dockerfile | 2 +- postgraphile/README.md | 2 +- 10 files changed, 12 insertions(+), 48 deletions(-) delete mode 100755 dockerfiles/continuousLogSync-service delete mode 100755 dockerfiles/lightSync-service diff --git a/Dockerfile b/Dockerfile index 056f60db..4c853c43 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,16 +14,11 @@ RUN GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflag # Second stage FROM alpine -RUN apk --no-cache --update add openrc -RUN mkdir -p /run/openrc/ && touch /run/openrc/softlevel COPY --from=builder /go/src/github.com/vulcanize/vulcanizedb/vulcanizedb /app/vulcanizedb COPY --from=builder /go/src/github.com/vulcanize/vulcanizedb/environments/staging.toml /app/environments/ -COPY --from=builder /go/src/github.com/vulcanize/vulcanizedb/dockerfiles/lightSync-service /etc/init.d/lightSync -COPY --from=builder /go/src/github.com/vulcanize/vulcanizedb/dockerfiles/continuousLogSync-service /etc/init.d/continuousLogSync COPY --from=builder /go/src/github.com/vulcanize/vulcanizedb/dockerfiles/startup_script.sh /app/ COPY --from=builder /go/src/github.com/vulcanize/vulcanizedb/db/migrations/* /app/ COPY --from=builder /go/src/github.com/pressly/goose/cmd/goose/goose /app/goose WORKDIR /app -RUN rc-status -CMD ["./goose"] +CMD ["./startup_script.sh"] diff --git a/dockerfiles/README.md b/dockerfiles/README.md index ea6e19c9..5bf3c61a 100644 --- a/dockerfiles/README.md +++ b/dockerfiles/README.md @@ -19,14 +19,14 @@ e` * `CONTRACT_ABI_[CONTRACT NAME]="ABI STRING"` * `CONTRACT_DEPLOYMENT-BLOCK_[CONTRACT NAME]=0` (doesn't really matter on a short chain, just avoids long unnecessary searching) 6. Start the `lightSync` and `continuousLogSync` services: - * `rc-service lightSync start` - * `rc-service continuousLogSync start` + * `./vulcanizedb lightSync --config environments/staging.toml` + * `./vulcanizedb continuousLogSync --config environments/staging.toml` ### Automated The steps above have been rolled into a script: `/app/startup_script.sh`, which just assumes the DB env variables have been set, and defaults the rest to Kovan according to `environments/staging.toml`. This can be called with something like: -`docker run -d -e DATABASE_NAME=vulcanize_public -e DATABASE_HOSTNAME=localhost -e DATABASE_PORT=5432 -e DATABASE_USER=vulcanize -e DATABASE_PASSWORD=vulcanize [container name] ./startup_script.sh` +`docker run -d -e DATABASE_NAME=vulcanize_public -e DATABASE_HOSTNAME=localhost -e DATABASE_PORT=5432 -e DATABASE_USER=vulcanize -e DATABASE_PASSWORD=vulcanize m0ar/images:vDB` ### Logging -When running, vDB services log to `/vulcanizedb.log`. +When running, vDB services log to `/app/vulcanizedb.log`. diff --git a/dockerfiles/continuousLogSync-service b/dockerfiles/continuousLogSync-service deleted file mode 100755 index 9c3adfd5..00000000 --- a/dockerfiles/continuousLogSync-service +++ /dev/null @@ -1,6 +0,0 @@ -#!/sbin/openrc-run -command=/app/vulcanizedb -command_args="continuousLogSync --config /app/environments/staging.toml &" -pidfile= -name="continuousLogSync" -description="runs vDB in continuousLogSync mode using staging.toml" diff --git a/dockerfiles/lightSync-service b/dockerfiles/lightSync-service deleted file mode 100755 index a9143de1..00000000 --- a/dockerfiles/lightSync-service +++ /dev/null @@ -1,6 +0,0 @@ -#!/sbin/openrc-run -command=/app/vulcanizedb -command_args="lightSync --config /app/environments/staging.toml &" -pidfile= -name="lightSync" -description="runs vDB in lightSync mode using staging.toml" diff --git a/dockerfiles/startup_script.sh b/dockerfiles/startup_script.sh index 47547d99..8373715a 100755 --- a/dockerfiles/startup_script.sh +++ b/dockerfiles/startup_script.sh @@ -21,9 +21,9 @@ set +e ./goose postgres "$CONNECT_STRING" up if [ $? -eq 0 ]; then # Fire up the services - rc-service lightSync start - rc-service continuousLogSync start + ./vulcanizedb lightSync --config environments/staging.toml & + ./vulcanizedb continuousLogSync --config environments/staging.toml & else echo "Could not run migrations. Are the database details correct?" fi - +wait diff --git a/environments/staging.toml b/environments/staging.toml index 8954a483..ad320f27 100644 --- a/environments/staging.toml +++ b/environments/staging.toml @@ -1,8 +1,4 @@ [database] - name = "vulcanize_public" - hostname = "localhost" - user = "vulcanize" - password = "vulcanize" port = 5432 [client] diff --git a/pkg/datastore/postgres/postgres.go b/pkg/datastore/postgres/postgres.go index 82a1b2ac..dc43e351 100644 --- a/pkg/datastore/postgres/postgres.go +++ b/pkg/datastore/postgres/postgres.go @@ -18,6 +18,7 @@ package postgres import ( "errors" + "github.com/sirupsen/logrus" "github.com/jmoiron/sqlx" _ "github.com/lib/pq" //postgres driver @@ -40,6 +41,7 @@ var ( func NewDB(databaseConfig config.Database, node core.Node) (*DB, error) { connectString := config.DbConnectionString(databaseConfig) + logrus.Info("Using connection string: ", connectString) db, err := sqlx.Connect("postgres", connectString) if err != nil { return &DB{}, ErrDBConnectionFailed diff --git a/pkg/transformers/shared/constants/external.go b/pkg/transformers/shared/constants/external.go index d18270be..5b14d435 100644 --- a/pkg/transformers/shared/constants/external.go +++ b/pkg/transformers/shared/constants/external.go @@ -6,23 +6,7 @@ import ( "github.com/spf13/viper" ) -var initialized = false - -func initConfig() { - if initialized { - return - } - - if err := viper.ReadInConfig(); err == nil { - fmt.Printf("Using config file: %s\n\n", viper.ConfigFileUsed()) - } else { - panic(fmt.Sprintf("Could not find environment file: %v", err)) - } - initialized = true -} - func getEnvironmentString(key string) string { - initConfig() value := viper.GetString(key) if value == "" { panic(fmt.Sprintf("No environment configuration variable set for key: \"%v\"", key)) @@ -32,7 +16,6 @@ func getEnvironmentString(key string) string { // Returns an int from the environment, defaulting to 0 if it does not exist func getEnvironmentInt64(key string) int64 { - initConfig() value := viper.GetInt64(key) if value == -1 { return 0 diff --git a/postgraphile/Dockerfile b/postgraphile/Dockerfile index 4a98c0e3..f2785e36 100644 --- a/postgraphile/Dockerfile +++ b/postgraphile/Dockerfile @@ -6,4 +6,4 @@ COPY . /app run yarn install RUN ["./node_modules/typescript/bin/tsc"] EXPOSE 3000 -CMD ["node", "/app/build/dist/src/index.js"] +CMD ["node", "/app/build/dist/index.js"] diff --git a/postgraphile/README.md b/postgraphile/README.md index 048a1f7a..0bd5958c 100644 --- a/postgraphile/README.md +++ b/postgraphile/README.md @@ -10,7 +10,7 @@ Build the docker image in this directory. Start the `GraphiQL` frontend by: `DATABASE_NAME`, `DATABASE_USER`, `DATABASE_PASSWORD` (and optionally `DATABASE_PORT` if running on non-standard port). * The specified user needs to be `superuser` on the vulcanizeDB database -* Run the container (ex. `docker run -e DATABASE_HOST=localhost -e DATABASE_NAME=vulcanize_public -e DATABASE_USER=vulcanize -e DATABASE_PASSWORD=vulcanize -d postgraphile:latest`) +* Run the container (ex. `docker run -e DATABASE_HOST=localhost -e DATABASE_NAME=vulcanize_public -e DATABASE_USER=vulcanize -e DATABASE_PASSWORD=vulcanize -d m0ar/images:postgraphile-alpine`) * GraphiQL is available at `:3000/graphiql`