Upgrade to v5 schema
Now uses: * ipld direct_by_leaf StateDB for basic queries * trie_by_cid StateDB for trie slice and proof queries Also: * vulcanize => cerc refactor * Backend method to close dbs * state tests are in multiple packages, to allow separate ginkgo suites * removes gap-filler module * integration tests and github workflows * run stack-orchestrator for testnet * fix various issues with tests, hardhat server, dockerfile * fix cmd flags / env vars * fix flaky tests and clean up code * remove unused code, scripts * remove outdated docs * update version
This commit is contained in:
@@ -1,137 +0,0 @@
|
||||
import os
|
||||
import sys
|
||||
import subprocess
|
||||
import errno
|
||||
from typing import List, Dict
|
||||
|
||||
"""
|
||||
Resolves dependency conflicts between a plugin repository's and the core repository's go.mods
|
||||
|
||||
Usage: python3 gomoderator.py {path_to_core_repository} {path_to_plugin_repository}
|
||||
"""
|
||||
|
||||
ERROR_INVALID_NAME = 123
|
||||
|
||||
|
||||
def is_pathname_valid(pathname: str) -> bool:
|
||||
"""
|
||||
`True` if the passed pathname is a valid pathname for the current OS;
|
||||
`False` otherwise.
|
||||
"""
|
||||
try:
|
||||
if not isinstance(pathname, str) or not pathname:
|
||||
return False
|
||||
_, pathname = os.path.splitdrive(pathname)
|
||||
root_dirname = os.environ.get('HOMEDRIVE', 'C:') \
|
||||
if sys.platform == 'win32' else os.path.sep
|
||||
assert os.path.isdir(root_dirname) # ...Murphy and her ironclad Law
|
||||
root_dirname = root_dirname.rstrip(os.path.sep) + os.path.sep
|
||||
for pathname_part in pathname.split(os.path.sep):
|
||||
try:
|
||||
os.lstat(root_dirname + pathname_part)
|
||||
except OSError as exc:
|
||||
if hasattr(exc, 'winerror'):
|
||||
if exc.winerror == ERROR_INVALID_NAME:
|
||||
return False
|
||||
elif exc.errno in {errno.ENAMETOOLONG, errno.ERANGE}:
|
||||
return False
|
||||
except TypeError as exc:
|
||||
return False
|
||||
else:
|
||||
return True
|
||||
|
||||
|
||||
def map_deps_to_version(deps_arr: List[str]) -> Dict[str, str]:
|
||||
mapping = {}
|
||||
for d in deps_arr:
|
||||
if d.find(' => ') != -1:
|
||||
ds = d.split(' => ')
|
||||
d = ds[1]
|
||||
d = d.replace(" v", "[>v") # might be able to just split on the empty space not _v and skip this :: insertion
|
||||
d_and_v = d.split("[>")
|
||||
mapping[d_and_v[0]] = d_and_v[1]
|
||||
return mapping
|
||||
|
||||
|
||||
# argument checks
|
||||
assert len(sys.argv) == 3, "need core repository and plugin repository path arguments"
|
||||
core_repository_path = sys.argv[1]
|
||||
plugin_repository_path = sys.argv[2]
|
||||
assert is_pathname_valid(core_repository_path), "core repository path argument is not valid"
|
||||
assert is_pathname_valid(plugin_repository_path), "plugin repository path argument is not valid"
|
||||
|
||||
# collect `go list -m all` output from both repositories; remain in the plugin repository
|
||||
os.chdir(core_repository_path)
|
||||
core_deps_b = subprocess.check_output(["go", "list", "-m", "all"])
|
||||
os.chdir(plugin_repository_path)
|
||||
plugin_deps_b = subprocess.check_output(["go", "list", "-m", "all"])
|
||||
core_deps = core_deps_b.decode("utf-8")
|
||||
core_deps_arr = core_deps.splitlines()
|
||||
del core_deps_arr[0] # first line is the project repo itself
|
||||
plugin_deps = plugin_deps_b.decode("utf-8")
|
||||
plugin_deps_arr = plugin_deps.splitlines()
|
||||
del plugin_deps_arr[0]
|
||||
core_deps_mapping = map_deps_to_version(core_deps_arr)
|
||||
plugin_deps_mapping = map_deps_to_version(plugin_deps_arr)
|
||||
|
||||
# iterate over dependency maps for both repos and find version conflicts
|
||||
# attempt to resolve conflicts by adding adding a `require` for the core version to the plugin's go.mod file
|
||||
none = True
|
||||
for dep, core_version in core_deps_mapping.items():
|
||||
if dep in plugin_deps_mapping.keys():
|
||||
plugin_version = plugin_deps_mapping[dep]
|
||||
if core_version != plugin_version:
|
||||
print(f'{dep} has a conflict: core is using version {core_version} '
|
||||
f'but the plugin is using version {plugin_version}')
|
||||
fixed_dep = f'{dep}@{core_version}'
|
||||
print(f'attempting fix by `go mod edit -require={fixed_dep}')
|
||||
subprocess.check_call(["go", "mod", "edit", f'-require={fixed_dep}'])
|
||||
none = False
|
||||
|
||||
if none:
|
||||
print("no conflicts to resolve")
|
||||
quit()
|
||||
|
||||
# the above process does not work for all dep conflicts e.g. golang.org/x/text v0.3.0 will not stick this way
|
||||
# so we will try the `go get {dep}` route for any remaining conflicts
|
||||
updated_plugin_deps_b = subprocess.check_output(["go", "list", "-m", "all"])
|
||||
updated_plugin_deps = updated_plugin_deps_b.decode("utf-8")
|
||||
updated_plugin_deps_arr = updated_plugin_deps.splitlines()
|
||||
del updated_plugin_deps_arr[0]
|
||||
updated_plugin_deps_mapping = map_deps_to_version(updated_plugin_deps_arr)
|
||||
none = True
|
||||
for dep, core_version in core_deps_mapping.items():
|
||||
if dep in updated_plugin_deps_mapping.keys():
|
||||
updated_plugin_version = updated_plugin_deps_mapping[dep]
|
||||
if core_version != updated_plugin_version:
|
||||
print(f'{dep} still has a conflict: core is using version {core_version} '
|
||||
f'but the plugin is using version {updated_plugin_version}')
|
||||
fixed_dep = f'{dep}@{core_version}'
|
||||
print(f'attempting fix by `go get {fixed_dep}')
|
||||
subprocess.check_call(["go", "get", fixed_dep])
|
||||
none = False
|
||||
|
||||
if none:
|
||||
print("all conflicts have been resolved")
|
||||
quit()
|
||||
|
||||
# iterate over plugins `go list -m all` output one more time and inform whether or not the above has worked
|
||||
final_plugin_deps_b = subprocess.check_output(["go", "list", "-m", "all"])
|
||||
final_plugin_deps = final_plugin_deps_b.decode("utf-8")
|
||||
final_plugin_deps_arr = final_plugin_deps.splitlines()
|
||||
del final_plugin_deps_arr[0]
|
||||
final_plugin_deps_mapping = map_deps_to_version(final_plugin_deps_arr)
|
||||
none = True
|
||||
for dep, core_version in core_deps_mapping.items():
|
||||
if dep in final_plugin_deps_mapping.keys():
|
||||
final_plugin_version = final_plugin_deps_mapping[dep]
|
||||
if core_version != final_plugin_version:
|
||||
print(f'{dep} STILL has a conflict: core is using version {core_version} '
|
||||
f'but the plugin is using version {final_plugin_version}')
|
||||
none = False
|
||||
|
||||
if none:
|
||||
print("all conflicts have been resolved")
|
||||
quit()
|
||||
|
||||
print("failed to resolve all conflicts")
|
||||
@@ -1,15 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -ex
|
||||
|
||||
echo "Installing Postgres 11"
|
||||
sudo service postgresql stop
|
||||
sudo apt-get remove -q 'postgresql-*'
|
||||
sudo apt-get update -q
|
||||
sudo apt-get install -q postgresql-11 postgresql-client-11
|
||||
sudo cp /etc/postgresql/{9.6,11}/main/pg_hba.conf
|
||||
|
||||
echo "Restarting Postgres 11"
|
||||
sudo service postgresql restart
|
||||
|
||||
sudo psql -c 'CREATE ROLE travis SUPERUSER LOGIN CREATEDB;' -U postgres
|
||||
Executable
+58
@@ -0,0 +1,58 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
|
||||
# Prevent conflicting tty output
|
||||
export BUILDKIT_PROGRESS=plain
|
||||
# By default assume we are running in the project root
|
||||
export CERC_REPO_BASE_DIR="${CERC_REPO_BASE_DIR:-..}"
|
||||
|
||||
CONFIG_DIR=$(readlink -f "${CONFIG_DIR:-$(mktemp -d)}")
|
||||
|
||||
# Pass this in so we can run eth_call forwarding tests, which expect no IPLD DB
|
||||
echo "CERC_RUN_STATEDIFF=${CERC_RUN_STATEDIFF:-true}" >> $CONFIG_DIR/stack.env
|
||||
|
||||
laconic_so="${LACONIC_SO:-laconic-so} --verbose --stack fixturenet-eth-loaded"
|
||||
|
||||
set -x
|
||||
|
||||
# # Build and deploy a cluster with only what we need from the stack
|
||||
# $laconic_so setup-repositories \
|
||||
# --exclude cerc-io/ipld-eth-server,cerc-io/tx-spammer \
|
||||
# --branches-file ./test/stack-refs.yml
|
||||
|
||||
# $laconic_so build-containers \
|
||||
# --exclude cerc/ipld-eth-server,cerc/keycloak,cerc/tx-spammer
|
||||
|
||||
IMAGE_IPLD_ETH_DB=git.vdb.to/cerc-io/ipld-eth-db/ipld-eth-db:v5.0.2-alpha
|
||||
IMAGE_GETH=git.vdb.to/cerc-io/go-ethereum/go-ethereum:v1.11.5-statediff-5.0.5-alpha
|
||||
|
||||
docker pull $IMAGE_IPLD_ETH_DB
|
||||
docker pull $IMAGE_GETH
|
||||
docker tag $IMAGE_IPLD_ETH_DB cerc/ipld-eth-db:local
|
||||
docker tag $IMAGE_GETH cerc/go-ethereum:local
|
||||
|
||||
$laconic_so build-containers \
|
||||
--exclude cerc/ipld-eth-server,cerc/keycloak,cerc/tx-spammer,cerc/go-ethereum,cerc/ipld-eth-db
|
||||
|
||||
$laconic_so deploy \
|
||||
--include fixturenet-eth,ipld-eth-db \
|
||||
--env-file $CONFIG_DIR/stack.env \
|
||||
--cluster test up
|
||||
|
||||
# set +x
|
||||
|
||||
# Get IPv4 endpoint of geth file server
|
||||
bootnode_endpoint=$(docker port test-fixturenet-eth-bootnode-geth-1 9898 | head -1)
|
||||
|
||||
# Extract the chain config and ID from genesis file
|
||||
curl -s $bootnode_endpoint/geth.json | jq '.config' > $CONFIG_DIR/chain.json
|
||||
|
||||
# Output vars if we are running on Github
|
||||
if [[ -n "$GITHUB_ENV" ]]; then
|
||||
echo ETH_CHAIN_ID="$(jq '.chainId' $CONFIG_DIR/chain.json)" >> "$GITHUB_ENV"
|
||||
echo ETH_CHAIN_CONFIG="$CONFIG_DIR/chain.json" >> "$GITHUB_ENV"
|
||||
echo ETH_HTTP_PATH="$(docker port test-fixturenet-eth-geth-1-1 8545 | head -1)" >> "$GITHUB_ENV"
|
||||
# Read a private key so we can send from a funded account
|
||||
echo DEPLOYER_PRIVATE_KEY="$(curl -s $bootnode_endpoint/accounts.csv | head -1 | cut -d',' -f3)" >> "$GITHUB_ENV"
|
||||
fi
|
||||
@@ -1,27 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Provide me with a postgres database name, and I will:
|
||||
# - Drop the database
|
||||
# - Recreate the database
|
||||
# - Run the vulcanizedb migration
|
||||
|
||||
if [ "$1" = "" ]; then
|
||||
echo "Provide a database name to reset"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
db=$1
|
||||
dir=$(basename "$(pwd)")
|
||||
if [ $dir != "ipld-eth-server" ]
|
||||
then
|
||||
echo "Run me from the ipld-eth-server root dir"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
user=$(whoami)
|
||||
psql -c "DROP DATABASE $db" postgres
|
||||
if [ $? -eq 0 ]; then
|
||||
psql -c "CREATE DATABASE $db WITH OWNER $user" postgres
|
||||
make migrate HOST_NAME=localhost NAME=$db PORT=5432
|
||||
else
|
||||
echo "Couldnt drop the database. Are you connected? Does it exist?"
|
||||
fi
|
||||
@@ -1,17 +0,0 @@
|
||||
set -e
|
||||
set -o xtrace
|
||||
|
||||
export ETH_FORWARD_ETH_CALLS=false
|
||||
export DB_WRITE=true
|
||||
export ETH_PROXY_ON_ERROR=false
|
||||
|
||||
export PGPASSWORD=password
|
||||
export DATABASE_USER=vdbm
|
||||
export DATABASE_PORT=8077
|
||||
export DATABASE_PASSWORD=password
|
||||
export DATABASE_HOSTNAME=127.0.0.1
|
||||
|
||||
# Wait for containers to be up and execute the integration test.
|
||||
while [ "$(curl -s -o /dev/null -w ''%{http_code}'' localhost:8081)" != "200" ]; do echo "waiting for ipld-eth-server..." && sleep 5; done && \
|
||||
while [ "$(curl -s -o /dev/null -w ''%{http_code}'' localhost:8545)" != "200" ]; do echo "waiting for geth-statediff..." && sleep 5; done && \
|
||||
make integrationtest
|
||||
@@ -1,17 +0,0 @@
|
||||
set -e
|
||||
set -o xtrace
|
||||
|
||||
export ETH_FORWARD_ETH_CALLS=true
|
||||
export DB_WRITE=false
|
||||
export ETH_PROXY_ON_ERROR=false
|
||||
|
||||
export PGPASSWORD=password
|
||||
export DATABASE_USER=vdbm
|
||||
export DATABASE_PORT=8077
|
||||
export DATABASE_PASSWORD=password
|
||||
export DATABASE_HOSTNAME=127.0.0.1
|
||||
|
||||
# Wait for containers to be up and execute the integration test.
|
||||
while [ "$(curl -s -o /dev/null -w ''%{http_code}'' localhost:8081)" != "200" ]; do echo "waiting for ipld-eth-server..." && sleep 5; done && \
|
||||
while [ "$(curl -s -o /dev/null -w ''%{http_code}'' localhost:8545)" != "200" ]; do echo "waiting for geth-statediff..." && sleep 5; done && \
|
||||
make integrationtest
|
||||
Reference in New Issue
Block a user