From f8de0defadaa12ae22163be43fbf66ceea6f0d0b Mon Sep 17 00:00:00 2001 From: Abdul Rabbani Date: Fri, 17 Jun 2022 11:37:50 -0400 Subject: [PATCH 1/5] RE-test release with correct commit hash --- .github/workflows/on-pr-publish.yaml | 4 ++++ .github/workflows/run_unit_test.sh | 2 +- .github/workflows/tests.yaml | 36 +++++++++++++--------------- 3 files changed, 21 insertions(+), 21 deletions(-) diff --git a/.github/workflows/on-pr-publish.yaml b/.github/workflows/on-pr-publish.yaml index b1e28a79..9834c6ff 100644 --- a/.github/workflows/on-pr-publish.yaml +++ b/.github/workflows/on-pr-publish.yaml @@ -27,6 +27,10 @@ jobs: BUILD_HOSTNAME: ${{ secrets.BUILD_HOSTNAME }} BUILD_USERNAME: ${{ secrets.BUILD_USERNAME }} BUILD_KEY: ${{ secrets.BUILD_KEY }} + with: + STACK_ORCHESTRATOR_REF: "382aca8e42bc5e33f301f77cdd2e09cc80602fc3" + GO_ETHEREUM_REF: "d629c99d84cb8d20b89e46c1d9f852c7409829f2" + IPLD_ETH_DB_REF: "48eb594ea95763bda8e51590f105f7a2657ac6d4" build: name: Run docker build runs-on: ubuntu-latest diff --git a/.github/workflows/run_unit_test.sh b/.github/workflows/run_unit_test.sh index d1854407..73b24c81 100755 --- a/.github/workflows/run_unit_test.sh +++ b/.github/workflows/run_unit_test.sh @@ -17,7 +17,7 @@ rm -f /tmp/git_head_ref /tmp/git_repository rm -rf out/docker-tsdb/ # Copy over files to setup TimescaleDB -ID=$(docker create vulcanize/ipld-eth-db:v4.1.1-alpha) +ID=$(docker create vulcanize/ipld-eth-db:v4.1.3-alpha) docker cp $ID:/app/docker-tsdb out/docker-tsdb/ docker rm -v $ID diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 16cf13cb..1597a55e 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -8,6 +8,16 @@ on: required: true BUILD_KEY: required: true + inputs: + STACK_ORCHESTRATOR_REF: + required: true + type: string + GO_ETHEREUM_REF: + required: true + type: string + IPLD_ETH_DB_REF: + required: true + type: string jobs: build: @@ -68,9 +78,6 @@ jobs: integrationtest: name: Run integration tests env: - STACK_ORCHESTRATOR_REF: 418957a1f745c921b21286c13bb033f922a91ae9 - GO_ETHEREUM_REF: "v1.10.18-statediff-4.0.2-alpha" - IPLD_ETH_DB_REF: 91d30b9ea1acecd0a7f4307390a98bf3e289b8d7 GOPATH: /tmp/go DB_WRITE: true ETH_FORWARD_ETH_CALLS: false @@ -89,12 +96,12 @@ jobs: path: "./ipld-eth-server" - uses: actions/checkout@v2 with: - ref: ${{ env.STACK_ORCHESTRATOR_REF }} + ref: ${{ inputs.STACK_ORCHESTRATOR_REF }} path: "./stack-orchestrator/" repository: vulcanize/stack-orchestrator - uses: actions/checkout@v2 with: - ref: ${{ env.GO_ETHEREUM_REF }} + ref: ${{ inputs.GO_ETHEREUM_REF }} repository: vulcanize/go-ethereum path: "./go-ethereum/" - uses: actions/checkout@v2 @@ -112,9 +119,6 @@ jobs: echo eth_forward_eth_calls=$ETH_FORWARD_ETH_CALLS >> ./config.sh echo eth_proxy_on_error=$ETH_PROXY_ON_ERROR >> ./config.sh echo eth_http_path=$ETH_HTTP_PATH >> ./config.sh - echo ipld_eth_server_db_dependency=access-node >> ./config.sh - echo go_ethereum_db_dependency=access-node >> ./config.sh - echo connecting_db_name=vulcanize_testing_v4 >> ./config.sh echo watched_address_gap_filler_enabled=false >> ./config.sh echo watched_address_gap_filler_interval=5 >> ./config.sh cat ./config.sh @@ -127,11 +131,10 @@ jobs: - name: Run docker compose run: | docker-compose \ - -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-db-migration.yml" \ + -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-db-sharding.yml" \ -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-go-ethereum.yml" \ -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-ipld-eth-server.yml" \ -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-contract.yml" \ - -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/latest/docker-compose-timescale-db.yml" \ --env-file "$GITHUB_WORKSPACE/config.sh" \ up -d --build - name: Test @@ -144,9 +147,6 @@ jobs: integrationtest_forwardethcalls: name: Run integration tests for direct proxy fall-through of eth_calls env: - STACK_ORCHESTRATOR_REF: 418957a1f745c921b21286c13bb033f922a91ae9 - GO_ETHEREUM_REF: "v1.10.18-statediff-4.0.2-alpha" - IPLD_ETH_DB_REF: 91d30b9ea1acecd0a7f4307390a98bf3e289b8d7 GOPATH: /tmp/go DB_WRITE: false ETH_FORWARD_ETH_CALLS: true @@ -165,12 +165,12 @@ jobs: path: "./ipld-eth-server" - uses: actions/checkout@v2 with: - ref: ${{ env.STACK_ORCHESTRATOR_REF }} + ref: ${{ inputs.STACK_ORCHESTRATOR_REF }} path: "./stack-orchestrator/" repository: vulcanize/stack-orchestrator - uses: actions/checkout@v2 with: - ref: ${{ env.GO_ETHEREUM_REF }} + ref: ${{ inputs.GO_ETHEREUM_REF }} repository: vulcanize/go-ethereum path: "./go-ethereum/" - uses: actions/checkout@v2 @@ -188,9 +188,6 @@ jobs: echo eth_forward_eth_calls=$ETH_FORWARD_ETH_CALLS >> ./config.sh echo eth_proxy_on_error=$ETH_PROXY_ON_ERROR >> ./config.sh echo eth_http_path=$ETH_HTTP_PATH >> ./config.sh - echo ipld_eth_server_db_dependency=access-node >> ./config.sh - echo go_ethereum_db_dependency=access-node >> ./config.sh - echo connecting_db_name=vulcanize_testing_v4 >> ./config.sh echo watched_address_gap_filler_enabled=false >> ./config.sh echo watched_address_gap_filler_interval=5 >> ./config.sh cat ./config.sh @@ -203,11 +200,10 @@ jobs: - name: Run docker compose run: | docker-compose \ - -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-db-migration.yml" \ + -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-db-sharding.yml" \ -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-go-ethereum.yml" \ -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-ipld-eth-server.yml" \ -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-contract.yml" \ - -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/latest/docker-compose-timescale-db.yml" \ --env-file "$GITHUB_WORKSPACE/config.sh" \ up -d --build - name: Test From b7b4631231c8b9d8aaaecf7b8e15f705c714f6a3 Mon Sep 17 00:00:00 2001 From: Abdul Rabbani Date: Fri, 17 Jun 2022 12:48:50 -0400 Subject: [PATCH 2/5] Update unit test to use stack-orchestrator --- .github/workflows/run_unit_test.sh | 33 +++++++++++++++++------------- .github/workflows/tests.yaml | 8 ++++++-- 2 files changed, 25 insertions(+), 16 deletions(-) diff --git a/.github/workflows/run_unit_test.sh b/.github/workflows/run_unit_test.sh index 73b24c81..c396aa99 100755 --- a/.github/workflows/run_unit_test.sh +++ b/.github/workflows/run_unit_test.sh @@ -8,30 +8,35 @@ temp_dir=$(mktemp -d) cd $temp_dir git clone -b $(cat /tmp/git_head_ref) "https://github.com/$(cat /tmp/git_repository).git" cd ipld-eth-server -mkdir -p out ## Remove the branch and github related info. This way future runs wont be confused. -rm -f /tmp/git_head_ref /tmp/git_repository +#rm -f /tmp/git_head_ref /tmp/git_repository -# Remove existing docker-tsdb directory -rm -rf out/docker-tsdb/ +# Setup the DB +cd $temp_dir +git clone "https://github.com/vulcanize/ipld-eth-db.git"; cd ipld-eth-db; git checkout $(cat /tmp/ipld_eth_db_ref) -# Copy over files to setup TimescaleDB -ID=$(docker create vulcanize/ipld-eth-db:v4.1.3-alpha) -docker cp $ID:/app/docker-tsdb out/docker-tsdb/ -docker rm -v $ID +# Spin Up DB using Stack Orchestrator +cd $temp_dir +git clone "https://github.com/vulcanize/stack-orchestrator.git"; cd stack-orchestrator; git checkout $(cat /tmp/stack_orchestrator_ref) -# Spin up TimescaleDB -docker-compose -f out/docker-tsdb/docker-compose.test.yml -f docker-compose.yml up ipld-eth-db -trap "docker-compose -f out/docker-tsdb/docker-compose.test.yml -f docker-compose.yml down --remove-orphans --volumes; cd $start_dir ; rm -r $temp_dir" SIGINT SIGTERM ERR -sleep 45 +cd ${temp_dir}/stack-orchestrator +echo vulcanize_ipld_eth_db=${temp_dir}/ipld-eth-db > ./config.sh + +## Remove existing containers if they are present +docker-compose -f docker/local/docker-compose-db-sharding.yml --env-file ./config.sh down -v --remove-orphans; + +trap 'cd ${temp_dir}/stack-orchestrator; docker-compose -f docker/local/docker-compose-db-sharding.yml --env-file ./config.sh down -v --remove-orphans; ' SIGINT SIGTERM +docker-compose -f docker/local/docker-compose-db-sharding.yml --env-file ./config.sh up -d # Remove old logs so there's no confusion, then run test rm -f /tmp/test.log /tmp/return_test.txt -PGPASSWORD=password DATABASE_USER=vdbm DATABASE_PORT=8066 DATABASE_PASSWORD=password DATABASE_HOSTNAME=127.0.0.1 DATABASE_NAME=vulcanize_testing_v4 make test > /tmp/test.log +cd ${temp_dir}/ipld-eth-server +PGPASSWORD=password DATABASE_USER=vdbm DATABASE_PORT=8077 DATABASE_PASSWORD=password DATABASE_HOSTNAME=localhost DATABASE_NAME=vulcanize_testing make test > /tmp/test.log echo $? > /tmp/return_test.txt # Clean up -docker-compose -f out/docker-tsdb/docker-compose.test.yml -f docker-compose.yml down --remove-orphans --volumes + +cd ${temp_dir}/stack-orchestrator; docker-compose -f docker/local/docker-compose-db-sharding.yml --env-file ./config.sh down -v --remove-orphans cd $start_dir rm -fr $temp_dir diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 1597a55e..8c0bfb67 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -46,6 +46,8 @@ jobs: - name: Output variables to files run: | echo $GITHUB_REPOSITORY > /tmp/git_repository + echo ${{ inputs.STACK_ORCHESTRATOR_REF }} > /tmp/ipld_eth_db_ref + echo ${{ inputs.IPLD_ETH_DB_REF }} > /tmp/stack_orchestrator_ref [ -z "$GITHUB_HEAD_REF" ] && echo $GITHUB_REF_NAME > /tmp/git_head_ref || echo $GITHUB_HEAD_REF > /tmp/git_head_ref echo "-----BEGIN OPENSSH PRIVATE KEY-----" >> /tmp/key echo ${{ env.BUILD_KEY }} >> /tmp/key @@ -58,6 +60,8 @@ jobs: run: | scp -o 'StrictHostKeyChecking no' -o UserKnownHostsFile=/dev/null -q -i /tmp/key /tmp/git_repository ${{ env.BUILD_USERNAME }}@${{ env.BUILD_HOSTNAME }}:/tmp/git_repository scp -o 'StrictHostKeyChecking no' -o UserKnownHostsFile=/dev/null -q -i /tmp/key /tmp/git_head_ref ${{ env.BUILD_USERNAME }}@${{ env.BUILD_HOSTNAME }}:/tmp/git_head_ref + scp -o 'StrictHostKeyChecking no' -o UserKnownHostsFile=/dev/null -q -i /tmp/key /tmp/stack_orchestrator_ref ${{ env.BUILD_USERNAME }}@${{ env.BUILD_HOSTNAME }}:/tmp/stack_orchestrator_ref + scp -o 'StrictHostKeyChecking no' -o UserKnownHostsFile=/dev/null -q -i /tmp/key /tmp/ipld_eth_db_ref ${{ env.BUILD_USERNAME }}@${{ env.BUILD_HOSTNAME }}:/tmp/ipld_eth_db_ref scp -o 'StrictHostKeyChecking no' -o UserKnownHostsFile=/dev/null -q -i /tmp/key .github/workflows/run_unit_test.sh ${{ env.BUILD_USERNAME }}@${{ env.BUILD_HOSTNAME }}:/tmp/run_unit_test.sh - name: Trigger Unit Test @@ -106,7 +110,7 @@ jobs: path: "./go-ethereum/" - uses: actions/checkout@v2 with: - ref: ${{ env.IPLD_ETH_DB_REF }} + ref: ${{ inputs.IPLD_ETH_DB_REF }} repository: vulcanize/ipld-eth-db path: "./ipld-eth-db/" - name: Create config file @@ -175,7 +179,7 @@ jobs: path: "./go-ethereum/" - uses: actions/checkout@v2 with: - ref: ${{ env.IPLD_ETH_DB_REF }} + ref: ${{ inputs.IPLD_ETH_DB_REF }} repository: vulcanize/ipld-eth-db path: "./ipld-eth-db/" - name: Create config file From 72e903cf68e3374c72844a6d3244c04bd78afb7b Mon Sep 17 00:00:00 2001 From: Abdul Rabbani Date: Fri, 17 Jun 2022 12:49:39 -0400 Subject: [PATCH 3/5] clean up before --- .github/workflows/run_unit_test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run_unit_test.sh b/.github/workflows/run_unit_test.sh index c396aa99..280bb6d3 100755 --- a/.github/workflows/run_unit_test.sh +++ b/.github/workflows/run_unit_test.sh @@ -10,7 +10,7 @@ git clone -b $(cat /tmp/git_head_ref) "https://github.com/$(cat /tmp/git_reposit cd ipld-eth-server ## Remove the branch and github related info. This way future runs wont be confused. -#rm -f /tmp/git_head_ref /tmp/git_repository +rm -f /tmp/git_head_ref /tmp/git_repository # Setup the DB cd $temp_dir From 45c5a25d5424fa086d1e20c19aa9615858decbb3 Mon Sep 17 00:00:00 2001 From: Abdul Rabbani Date: Fri, 17 Jun 2022 12:53:22 -0400 Subject: [PATCH 4/5] Update write to file --- .github/workflows/tests.yaml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 8c0bfb67..213477a5 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -46,15 +46,17 @@ jobs: - name: Output variables to files run: | echo $GITHUB_REPOSITORY > /tmp/git_repository - echo ${{ inputs.STACK_ORCHESTRATOR_REF }} > /tmp/ipld_eth_db_ref - echo ${{ inputs.IPLD_ETH_DB_REF }} > /tmp/stack_orchestrator_ref + echo ${{ inputs.IPLD_ETH_DB_REF }} > /tmp/ipld_eth_db_ref + echo ${{ inputs.STACK_ORCHESTRATOR_REF }} > /tmp/stack_orchestrator_ref [ -z "$GITHUB_HEAD_REF" ] && echo $GITHUB_REF_NAME > /tmp/git_head_ref || echo $GITHUB_HEAD_REF > /tmp/git_head_ref echo "-----BEGIN OPENSSH PRIVATE KEY-----" >> /tmp/key echo ${{ env.BUILD_KEY }} >> /tmp/key echo "-----END OPENSSH PRIVATE KEY-----" >> /tmp/key chmod 400 /tmp/key - cat /tmp/git_repository - cat /tmp/git_head_ref + cat /tmp/git_repository + cat /tmp/git_head_ref + cat /tmp/stack_orchestrator_ref + cat /tmp/ipld_eth_db_ref - name: Raw SCP run: | From a5ac6c19cf9c4009fdb490dc2103bd295c85a3a8 Mon Sep 17 00:00:00 2001 From: Abdul Rabbani Date: Fri, 17 Jun 2022 13:02:00 -0400 Subject: [PATCH 5/5] Update stack-orch retry --- .github/workflows/on-pr-publish.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/on-pr-publish.yaml b/.github/workflows/on-pr-publish.yaml index 9834c6ff..3d492919 100644 --- a/.github/workflows/on-pr-publish.yaml +++ b/.github/workflows/on-pr-publish.yaml @@ -28,7 +28,7 @@ jobs: BUILD_USERNAME: ${{ secrets.BUILD_USERNAME }} BUILD_KEY: ${{ secrets.BUILD_KEY }} with: - STACK_ORCHESTRATOR_REF: "382aca8e42bc5e33f301f77cdd2e09cc80602fc3" + STACK_ORCHESTRATOR_REF: "b3cb26e93b7e387d96417c81f880a3b8699b67db" GO_ETHEREUM_REF: "d629c99d84cb8d20b89e46c1d9f852c7409829f2" IPLD_ETH_DB_REF: "48eb594ea95763bda8e51590f105f7a2657ac6d4" build: