From 738aa1907b7243e0c8ae6bd58c402453fa1f4af0 Mon Sep 17 00:00:00 2001 From: Zach Date: Mon, 9 Jan 2023 20:03:09 -0500 Subject: [PATCH 01/10] Update repository-list.txt --- app/data/repository-list.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/data/repository-list.txt b/app/data/repository-list.txt index cdf40b25..43507d6e 100644 --- a/app/data/repository-list.txt +++ b/app/data/repository-list.txt @@ -2,8 +2,8 @@ cerc-io/ipld-eth-db cerc-io/go-ethereum cerc-io/ipld-eth-server cerc-io/eth-statediff-service -vulcanize/eth-statediff-fill-service -vulcanize/ipld-eth-db-validator +cerc-io/eth-statediff-fill-service +cerc-io/ipld-eth-db-validator cerc-io/ipld-eth-beacon-indexer cerc-io/ipld-eth-beacon-db cerc-io/laconicd -- 2.45.2 From ddd2ea27241a31d9ce2ef0cf5505020dbcd50775 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Tue, 10 Jan 2023 10:24:04 -0700 Subject: [PATCH 02/10] Add a simple smoke test and shiv build script --- build-shiv-package.sh | 4 ++++ tests/smoke-test/run-smoke-test.sh | 21 +++++++++++++++++++++ 2 files changed, 25 insertions(+) create mode 100755 build-shiv-package.sh create mode 100755 tests/smoke-test/run-smoke-test.sh diff --git a/build-shiv-package.sh b/build-shiv-package.sh new file mode 100755 index 00000000..f5fa58ac --- /dev/null +++ b/build-shiv-package.sh @@ -0,0 +1,4 @@ +# Builds the shiv "package" for distribution +# TODO: add build version/tag to filename +# TODO: add version info to code for version subcommand +shiv -c laconic-so -o package/laconic-so . diff --git a/tests/smoke-test/run-smoke-test.sh b/tests/smoke-test/run-smoke-test.sh new file mode 100755 index 00000000..ac1fa1d4 --- /dev/null +++ b/tests/smoke-test/run-smoke-test.sh @@ -0,0 +1,21 @@ +# Basic simple test of stack-orchestrator functionality +echo "Running stack-orchestrator smoke test" +TEST_TARGET_SO=package/laconic-so +# Set a non-default repo dir +export CERC_REPO_BASE_DIR=~/stack-orchestrator-test/repo-base-dir +echo "Testing this package: $TEST_TARGET_SO" +echo "Cloning repositories into: $CERC_REPO_BASE_DIR" +rm -rf $CERC_REPO_BASE_DIR +mkdir -p $CERC_REPO_BASE_DIR +# Pull an example small public repo to test we can pull a repo +$TEST_TARGET_SO setup-repositories --include cerc-io/laconic-sdk +# TODO: test building the repo into a container +# Build two example containers +# TODO: +$TEST_TARGET_SO build-containers --include cerc/builder-js,cerc/test-container +# Deploy the test container +$TEST_TARGET_SO deploy-system --include test up +# TODO: test that we can use the deployed container somehow +# Clean up +$TEST_TARGET_SO deploy-system --include test down +echo "Test passed" \ No newline at end of file -- 2.45.2 From b8b83768d87af0535ec45e71e1b73af0f11611b8 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Tue, 10 Jan 2023 10:24:48 -0700 Subject: [PATCH 03/10] Add trailing newline --- tests/smoke-test/run-smoke-test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/smoke-test/run-smoke-test.sh b/tests/smoke-test/run-smoke-test.sh index ac1fa1d4..ffdf8a06 100755 --- a/tests/smoke-test/run-smoke-test.sh +++ b/tests/smoke-test/run-smoke-test.sh @@ -18,4 +18,4 @@ $TEST_TARGET_SO deploy-system --include test up # TODO: test that we can use the deployed container somehow # Clean up $TEST_TARGET_SO deploy-system --include test down -echo "Test passed" \ No newline at end of file +echo "Test passed" -- 2.45.2 From 261b513d6805cd28473fc2a4f9f758d317039363 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Tue, 10 Jan 2023 14:09:38 -0700 Subject: [PATCH 04/10] Initial implementation --- app/data/version.txt | 2 ++ app/version.py | 30 +++++++++++++++++++ cli.py | 2 ++ scripts/build-shiv-package.sh | 3 ++ .../first_time_setup.sh | 0 scripts/update-version-file.sh | 3 ++ 6 files changed, 40 insertions(+) create mode 100644 app/data/version.txt create mode 100644 app/version.py create mode 100755 scripts/build-shiv-package.sh rename first_time_setup.sh => scripts/first_time_setup.sh (100%) create mode 100755 scripts/update-version-file.sh diff --git a/app/data/version.txt b/app/data/version.txt new file mode 100644 index 00000000..81447904 --- /dev/null +++ b/app/data/version.txt @@ -0,0 +1,2 @@ +# This file should be re-generated running: scripts/update-version-file.sh script +v1.0.6-alpha 9278d09 diff --git a/app/version.py b/app/version.py new file mode 100644 index 00000000..0547f0ea --- /dev/null +++ b/app/version.py @@ -0,0 +1,30 @@ +# Copyright © 2023 Cerc + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import click +import importlib + +@click.command() +@click.pass_context +def command(ctx): + '''print tool version''' + + # See: https://stackoverflow.com/a/20885799/1701505 + from . import data + with importlib.resources.open_text(data, "version.txt") as version_file: + # TODO: code better version that skips comment lines + version_string = version_file.read().splitlines()[1] + + print(f"Version: {version_string}") diff --git a/cli.py b/cli.py index 1703a422..e2e94ab2 100644 --- a/cli.py +++ b/cli.py @@ -19,6 +19,7 @@ from app import setup_repositories from app import build_containers from app import build_npms from app import deploy_system +from app import version CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help']) @@ -49,3 +50,4 @@ cli.add_command(setup_repositories.command, "setup-repositories") cli.add_command(build_containers.command, "build-containers") cli.add_command(build_npms.command, "build-npms") cli.add_command(deploy_system.command, "deploy-system") +cli.add_command(version.command, "version") diff --git a/scripts/build-shiv-package.sh b/scripts/build-shiv-package.sh new file mode 100755 index 00000000..98f49f4f --- /dev/null +++ b/scripts/build-shiv-package.sh @@ -0,0 +1,3 @@ +# Builds the shiv "package" for distribution +./scripts/update_version_file.sh +shiv -c laconic-so -o package/laconic-so . diff --git a/first_time_setup.sh b/scripts/first_time_setup.sh similarity index 100% rename from first_time_setup.sh rename to scripts/first_time_setup.sh diff --git a/scripts/update-version-file.sh b/scripts/update-version-file.sh new file mode 100755 index 00000000..fdcdac32 --- /dev/null +++ b/scripts/update-version-file.sh @@ -0,0 +1,3 @@ +version_file_name=./app/data/version.txt +echo "# This file should be re-generated running: scripts/update-version-file.sh script" > $version_file_name +echo $( git describe --tags --abbrev=0 ; git rev-parse --short HEAD ) >> $version_file_name -- 2.45.2 From ae7a17cecd6210ecd5847155f69a148247abeb6f Mon Sep 17 00:00:00 2001 From: David Boreham Date: Tue, 10 Jan 2023 14:10:22 -0700 Subject: [PATCH 05/10] Make script file names consistent --- scripts/{build-shiv-package.sh => build_shiv_package.sh} | 0 scripts/{update-version-file.sh => update_version_file.sh} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename scripts/{build-shiv-package.sh => build_shiv_package.sh} (100%) rename scripts/{update-version-file.sh => update_version_file.sh} (100%) diff --git a/scripts/build-shiv-package.sh b/scripts/build_shiv_package.sh similarity index 100% rename from scripts/build-shiv-package.sh rename to scripts/build_shiv_package.sh diff --git a/scripts/update-version-file.sh b/scripts/update_version_file.sh similarity index 100% rename from scripts/update-version-file.sh rename to scripts/update_version_file.sh -- 2.45.2 From d315941f210e6379eb377318224c1e3fc9614123 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Tue, 10 Jan 2023 14:23:59 -0700 Subject: [PATCH 06/10] Version command implemented --- app/data/version.txt | 2 +- scripts/build_shiv_package.sh | 4 ++-- scripts/update_version_file.sh | 6 +++++- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/app/data/version.txt b/app/data/version.txt index 81447904..0ee3baa4 100644 --- a/app/data/version.txt +++ b/app/data/version.txt @@ -1,2 +1,2 @@ # This file should be re-generated running: scripts/update-version-file.sh script -v1.0.6-alpha 9278d09 +v1.0.6-alpha-ae7a17c diff --git a/scripts/build_shiv_package.sh b/scripts/build_shiv_package.sh index 98f49f4f..2fbd68ad 100755 --- a/scripts/build_shiv_package.sh +++ b/scripts/build_shiv_package.sh @@ -1,3 +1,3 @@ # Builds the shiv "package" for distribution -./scripts/update_version_file.sh -shiv -c laconic-so -o package/laconic-so . +version_string=$( ./scripts/update_version_file.sh) +shiv -c laconic-so -o package/laconic-so-${version_string} . diff --git a/scripts/update_version_file.sh b/scripts/update_version_file.sh index fdcdac32..9880b21b 100755 --- a/scripts/update_version_file.sh +++ b/scripts/update_version_file.sh @@ -1,3 +1,7 @@ version_file_name=./app/data/version.txt echo "# This file should be re-generated running: scripts/update-version-file.sh script" > $version_file_name -echo $( git describe --tags --abbrev=0 ; git rev-parse --short HEAD ) >> $version_file_name +tag_string=$( git describe --tags --abbrev=0 ) +commit_string=$( git rev-parse --short HEAD ) +version_string=${tag_string}-${commit_string} +echo ${version_string} >> $version_file_name +echo ${version_string} -- 2.45.2 From df469786ca650e19a6aaf29ea049d499a80ebd9a Mon Sep 17 00:00:00 2001 From: David Boreham Date: Tue, 10 Jan 2023 14:32:12 -0700 Subject: [PATCH 07/10] Add version to smoke test --- tests/smoke-test/run-smoke-test.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/smoke-test/run-smoke-test.sh b/tests/smoke-test/run-smoke-test.sh index ffdf8a06..4ee2fddc 100755 --- a/tests/smoke-test/run-smoke-test.sh +++ b/tests/smoke-test/run-smoke-test.sh @@ -1,9 +1,13 @@ # Basic simple test of stack-orchestrator functionality echo "Running stack-orchestrator smoke test" -TEST_TARGET_SO=package/laconic-so +# Bit of a hack, test the most recent package +TEST_TARGET_SO=$( ls -t1 ./package/laconic-so* | head -1 ) # Set a non-default repo dir export CERC_REPO_BASE_DIR=~/stack-orchestrator-test/repo-base-dir echo "Testing this package: $TEST_TARGET_SO" +echo "Test version command" +reported_version_string=$( $TEST_TARGET_SO version ) +echo "Version reported is: ${reported_version_string}" echo "Cloning repositories into: $CERC_REPO_BASE_DIR" rm -rf $CERC_REPO_BASE_DIR mkdir -p $CERC_REPO_BASE_DIR -- 2.45.2 From 106cbe33ca45d768e0f76427f6d1160fca21d598 Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Wed, 11 Jan 2023 12:33:06 -0600 Subject: [PATCH 08/10] Add remote debugging support to fixturenet-geth. --- .../compose/docker-compose-fixturenet-eth.yml | 10 +++++++--- .../cerc-fixturenet-eth-geth/Dockerfile | 4 ++++ .../cerc-fixturenet-eth-geth/run-el.sh | 15 ++++++++++----- 3 files changed, 21 insertions(+), 8 deletions(-) diff --git a/app/data/compose/docker-compose-fixturenet-eth.yml b/app/data/compose/docker-compose-fixturenet-eth.yml index df6ae30a..3986ae43 100644 --- a/app/data/compose/docker-compose-fixturenet-eth.yml +++ b/app/data/compose/docker-compose-fixturenet-eth.yml @@ -14,6 +14,12 @@ services: fixturenet-eth-geth-1: hostname: fixturenet-eth-geth-1 + cap_add: + - SYS_PTRACE + environment: + CERC_RUN_STATEDIFF: "detect" + CERC_STATEDIFF_DB_NODE_ID: 1 + REMOTE_DEBUG: "true" env_file: - ../config/fixturenet-eth/fixturenet-eth.env image: cerc/fixturenet-eth-geth:local @@ -27,12 +33,10 @@ services: - fixturenet-eth-bootnode-geth ports: - "8545" + - "40000" fixturenet-eth-geth-2: hostname: fixturenet-eth-geth-2 - environment: - CERC_RUN_STATEDIFF: "detect" - CERC_STATEDIFF_DB_NODE_ID: 2 healthcheck: test: ["CMD", "nc", "-v", "localhost", "8545"] interval: 30s diff --git a/app/data/container-build/cerc-fixturenet-eth-geth/Dockerfile b/app/data/container-build/cerc-fixturenet-eth-geth/Dockerfile index cefe1bef..4e6df84a 100644 --- a/app/data/container-build/cerc-fixturenet-eth-geth/Dockerfile +++ b/app/data/container-build/cerc-fixturenet-eth-geth/Dockerfile @@ -1,8 +1,12 @@ FROM skylenet/ethereum-genesis-generator@sha256:210353ce7c898686bc5092f16c61220a76d357f51eff9c451e9ad1b9ad03d4d3 AS ethgen +FROM golang:1.18-alpine as delve +RUN go install github.com/go-delve/delve/cmd/dlv@latest + FROM cerc/go-ethereum:local RUN apk add --no-cache python3 python3-dev py3-pip curl wget jq build-base gettext libintl openssl bash bind-tools postgresql-client +COPY --from=delve /go/bin/dlv / COPY --from=ethgen /usr/local/bin/eth2-testnet-genesis /usr/local/bin/eth2-testnet-genesis COPY --from=ethgen /usr/local/bin/eth2-val-tools /usr/local/bin/eth2-val-tools COPY --from=ethgen /apps /apps diff --git a/app/data/container-build/cerc-fixturenet-eth-geth/run-el.sh b/app/data/container-build/cerc-fixturenet-eth-geth/run-el.sh index 34e74182..4fcda1fe 100755 --- a/app/data/container-build/cerc-fixturenet-eth-geth/run-el.sh +++ b/app/data/container-build/cerc-fixturenet-eth-geth/run-el.sh @@ -9,13 +9,18 @@ cd /opt/testnet/build/el python3 -m http.server 9898 & cd $HOME_DIR -if [ "true" == "$RUN_BOOTNODE" ]; then - geth \ +START_CMD="geth" +if [ "true" == "$REMOTE_DEBUG" ] && [ -x "/dlv" ]; then + START_CMD="/dlv --listen=:40000 --headless=true --api-version=2 --accept-multiclient exec /usr/local/bin/geth --continue --" +fi + +if [ "true" == "$RUN_BOOTNODE" ]; then + $START_CMD \ --nodekeyhex="${BOOTNODE_KEY}" \ --nodiscover \ --ipcdisable \ --networkid=${NETWORK_ID} \ - --netrestrict="${NETRESTRICT}" 2>&1 | tee /var/log/geth_bootnode.log + --netrestrict="${NETRESTRICT}" else cd /opt/testnet/accounts ./import_keys.sh @@ -60,7 +65,7 @@ else --statediff.writing=true" fi - geth \ + $START_CMD \ --bootnodes="${ENODE}" \ --allow-insecure-unlock \ --http \ @@ -80,5 +85,5 @@ else --mine \ --miner.threads=1 \ --verbosity=5 \ - --miner.etherbase="${ETHERBASE}" ${STATEDIFF_OPTS} 2>&1 | tee /var/log/geth.log + --miner.etherbase="${ETHERBASE}" ${STATEDIFF_OPTS} fi -- 2.45.2 From 1c78cda72983b3c5892c1e74d52bc3e1e2376af9 Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Wed, 11 Jan 2023 12:40:17 -0600 Subject: [PATCH 09/10] Add CERC_ prefix. --- app/data/compose/docker-compose-fixturenet-eth.yml | 2 +- app/data/container-build/cerc-fixturenet-eth-geth/run-el.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/data/compose/docker-compose-fixturenet-eth.yml b/app/data/compose/docker-compose-fixturenet-eth.yml index 3986ae43..f833ca70 100644 --- a/app/data/compose/docker-compose-fixturenet-eth.yml +++ b/app/data/compose/docker-compose-fixturenet-eth.yml @@ -17,9 +17,9 @@ services: cap_add: - SYS_PTRACE environment: + CERC_REMOTE_DEBUG: "true" CERC_RUN_STATEDIFF: "detect" CERC_STATEDIFF_DB_NODE_ID: 1 - REMOTE_DEBUG: "true" env_file: - ../config/fixturenet-eth/fixturenet-eth.env image: cerc/fixturenet-eth-geth:local diff --git a/app/data/container-build/cerc-fixturenet-eth-geth/run-el.sh b/app/data/container-build/cerc-fixturenet-eth-geth/run-el.sh index 4fcda1fe..7216ca0d 100755 --- a/app/data/container-build/cerc-fixturenet-eth-geth/run-el.sh +++ b/app/data/container-build/cerc-fixturenet-eth-geth/run-el.sh @@ -10,7 +10,7 @@ python3 -m http.server 9898 & cd $HOME_DIR START_CMD="geth" -if [ "true" == "$REMOTE_DEBUG" ] && [ -x "/dlv" ]; then +if [ "true" == "$CERC_REMOTE_DEBUG" ] && [ -x "/dlv" ]; then START_CMD="/dlv --listen=:40000 --headless=true --api-version=2 --accept-multiclient exec /usr/local/bin/geth --continue --" fi -- 2.45.2 From ecc0cb20021d7130a26b415a9cc082ce677b5cd8 Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Wed, 11 Jan 2023 12:53:32 -0600 Subject: [PATCH 10/10] Add comment --- app/data/container-build/cerc-fixturenet-eth-geth/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/app/data/container-build/cerc-fixturenet-eth-geth/Dockerfile b/app/data/container-build/cerc-fixturenet-eth-geth/Dockerfile index 4e6df84a..a7671d50 100644 --- a/app/data/container-build/cerc-fixturenet-eth-geth/Dockerfile +++ b/app/data/container-build/cerc-fixturenet-eth-geth/Dockerfile @@ -1,5 +1,6 @@ FROM skylenet/ethereum-genesis-generator@sha256:210353ce7c898686bc5092f16c61220a76d357f51eff9c451e9ad1b9ad03d4d3 AS ethgen +# Using the same golang image as used to build geth: https://github.com/cerc-io/go-ethereum/blob/HEAD/Dockerfile FROM golang:1.18-alpine as delve RUN go install github.com/go-delve/delve/cmd/dlv@latest -- 2.45.2