From ec6b5439f4b6381c80ed56f640be14c4a4de2aa0 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Thu, 25 May 2023 07:19:21 +0800 Subject: [PATCH] Support for git hosts other than github (#407) * Update repository list file * Add host part to repo name * Allow git hosts other than github --- app/data/repository-list.txt | 71 ++++++++++--------- app/data/stacks/azimuth/stack.yml | 2 +- app/data/stacks/chain-chunker/stack.yml | 8 +-- app/data/stacks/erc20/stack.yml | 10 +-- app/data/stacks/erc721/stack.yml | 8 +-- .../stacks/fixturenet-eth-loaded/stack.yml | 10 +-- app/data/stacks/fixturenet-eth-tx/stack.yml | 4 +- app/data/stacks/fixturenet-eth/README.md | 2 +- app/data/stacks/fixturenet-eth/stack.yml | 4 +- .../fixturenet-laconic-loaded/stack.yml | 16 ++--- app/data/stacks/fixturenet-laconicd/stack.yml | 6 +- app/data/stacks/fixturenet-lotus/stack.yml | 2 +- .../stacks/fixturenet-optimism/l2-only.md | 2 +- app/data/stacks/fixturenet-optimism/stack.yml | 8 +-- app/data/stacks/fixturenet-pocket/stack.yml | 6 +- app/data/stacks/gelato/stack.yml | 2 +- app/data/stacks/mobymask-v2/mobymask-only.md | 2 +- app/data/stacks/mobymask-v2/stack.yml | 14 ++-- .../watcher-p2p-network/watcher.md | 2 +- app/data/stacks/mobymask/README.md | 2 +- app/data/stacks/mobymask/stack.yml | 2 +- app/data/stacks/package-registry/stack.yml | 4 +- app/data/stacks/test/stack.yml | 3 +- app/data/stacks/uniswap-v3/stack.yml | 4 +- app/setup_repositories.py | 21 ++++-- 25 files changed, 114 insertions(+), 101 deletions(-) diff --git a/app/data/repository-list.txt b/app/data/repository-list.txt index 84d11b1a..25f54e42 100644 --- a/app/data/repository-list.txt +++ b/app/data/repository-list.txt @@ -1,35 +1,36 @@ -cerc-io/ipld-eth-db -cerc-io/go-ethereum -cerc-io/ipld-eth-server -cerc-io/eth-statediff-service -cerc-io/eth-statediff-fill-service -cerc-io/ipld-eth-db-validator -cerc-io/ipld-eth-beacon-indexer -cerc-io/ipld-eth-beacon-db -cerc-io/laconicd -cerc-io/laconic-sdk -cerc-io/laconic-registry-cli -cerc-io/laconic-console -cerc-io/mobymask-watcher -cerc-io/watcher-ts -cerc-io/mobymask-v2-watcher-ts -cerc-io/MobyMask -vulcanize/uniswap-watcher-ts -vulcanize/uniswap-v3-info -vulcanize/assemblyscript -cerc-io/eth-probe -cerc-io/tx-spammer -dboreham/foundry -lirewine/gem -lirewine/debug -lirewine/crypto -lirewine/sdk -telackey/act_runner -ethereum-optimism/op-geth -ethereum-optimism/optimism -pokt-network/pocket-core -pokt-network/pocket-core-deployments -cerc-io/azimuth-watcher-ts -cerc-io/ipld-eth-state-snapshot -cerc-io/gelato-watcher-ts -filecoin-project/lotus +github.com/cerc-io/ipld-eth-db +github.com/cerc-io/go-ethereum +github.com/cerc-io/ipld-eth-server +github.com/cerc-io/eth-statediff-service +github.com/cerc-io/eth-statediff-fill-service +github.com/cerc-io/ipld-eth-db-validator +github.com/cerc-io/ipld-eth-beacon-indexer +github.com/cerc-io/ipld-eth-beacon-db +github.com/cerc-io/laconicd +github.com/cerc-io/laconic-sdk +github.com/cerc-io/laconic-registry-cli +github.com/cerc-io/laconic-console +github.com/cerc-io/mobymask-watcher +github.com/cerc-io/watcher-ts +github.com/cerc-io/mobymask-v2-watcher-ts +github.com/cerc-io/MobyMask +github.com/vulcanize/uniswap-watcher-ts +github.com/vulcanize/uniswap-v3-info +github.com/vulcanize/assemblyscript +github.com/cerc-io/eth-probe +github.com/cerc-io/tx-spammer +github.com/dboreham/foundry +github.com/lirewine/gem +github.com/lirewine/debug +github.com/lirewine/crypto +github.com/lirewine/sdk +github.com/telackey/act_runner +github.com/ethereum-optimism/op-geth +github.com/ethereum-optimism/optimism +github.com/pokt-network/pocket-core +github.com/pokt-network/pocket-core-deployments +github.com/cerc-io/azimuth-watcher-ts +github.com/cerc-io/ipld-eth-state-snapshot +github.com/cerc-io/gelato-watcher-ts +github.com/filecoin-project/lotus +git.vdb.to/cerc-io/test-project diff --git a/app/data/stacks/azimuth/stack.yml b/app/data/stacks/azimuth/stack.yml index fc3c8473..bb7f2a88 100644 --- a/app/data/stacks/azimuth/stack.yml +++ b/app/data/stacks/azimuth/stack.yml @@ -1,7 +1,7 @@ version: "1.0" name: azimuth repos: - - cerc-io/azimuth-watcher-ts + - github.com/cerc-io/azimuth-watcher-ts containers: - cerc/watcher-azimuth pods: diff --git a/app/data/stacks/chain-chunker/stack.yml b/app/data/stacks/chain-chunker/stack.yml index f7676d34..671859c4 100644 --- a/app/data/stacks/chain-chunker/stack.yml +++ b/app/data/stacks/chain-chunker/stack.yml @@ -2,10 +2,10 @@ version: "1.0" name: chain-chunker decription: "Stack to build containers for chain-chunker" repos: - - cerc-io/ipld-eth-state-snapshot - - cerc-io/eth-statediff-service - - cerc-io/ipld-eth-db - - cerc-io/ipld-eth-server + - github.com/cerc-io/ipld-eth-state-snapshot + - github.com/cerc-io/eth-statediff-service + - github.com/cerc-io/ipld-eth-db + - github.com/cerc-io/ipld-eth-server containers: - cerc/ipld-eth-state-snapshot - cerc/eth-statediff-service diff --git a/app/data/stacks/erc20/stack.yml b/app/data/stacks/erc20/stack.yml index 57baa9eb..ef32f9c2 100644 --- a/app/data/stacks/erc20/stack.yml +++ b/app/data/stacks/erc20/stack.yml @@ -1,11 +1,11 @@ version: "1.0" name: erc20-watcher repos: - - cerc-io/go-ethereum - - cerc-io/ipld-eth-db - - cerc-io/ipld-eth-server - - cerc-io/watcher-ts - - dboreham/foundry + - github.com/cerc-io/go-ethereum + - github.com/cerc-io/ipld-eth-db + - github.com/cerc-io/ipld-eth-server + - github.com/cerc-io/watcher-ts + - github.com/dboreham/foundry containers: - cerc/foundry - cerc/go-ethereum diff --git a/app/data/stacks/erc721/stack.yml b/app/data/stacks/erc721/stack.yml index 28791473..1289059e 100644 --- a/app/data/stacks/erc721/stack.yml +++ b/app/data/stacks/erc721/stack.yml @@ -1,10 +1,10 @@ version: "1.0" name: erc721-watcher repos: - - cerc-io/go-ethereum - - cerc-io/ipld-eth-db - - cerc-io/ipld-eth-server - - cerc-io/watcher-ts + - github.com/cerc-io/go-ethereum + - github.com/cerc-io/ipld-eth-db + - github.com/cerc-io/ipld-eth-server + - github.com/cerc-io/watcher-ts containers: - cerc/go-ethereum - cerc/go-ethereum-foundry diff --git a/app/data/stacks/fixturenet-eth-loaded/stack.yml b/app/data/stacks/fixturenet-eth-loaded/stack.yml index ced862f9..55a05248 100644 --- a/app/data/stacks/fixturenet-eth-loaded/stack.yml +++ b/app/data/stacks/fixturenet-eth-loaded/stack.yml @@ -2,11 +2,11 @@ version: "1.0" name: fixturenet-eth-loaded decription: "Loaded Ethereum Fixturenet" repos: - - cerc-io/go-ethereum - - cerc-io/tx-spammer - - cerc-io/ipld-eth-server - - cerc-io/ipld-eth-db - - cerc/go-ethereum + - github.com/cerc-io/go-ethereum + - github.com/cerc-io/tx-spammer + - github.com/cerc-io/ipld-eth-server + - github.com/cerc-io/ipld-eth-db + - github.com/cerc-io/go-ethereum containers: - cerc/lighthouse - cerc/fixturenet-eth-geth diff --git a/app/data/stacks/fixturenet-eth-tx/stack.yml b/app/data/stacks/fixturenet-eth-tx/stack.yml index 0a0a6bd6..0281b262 100644 --- a/app/data/stacks/fixturenet-eth-tx/stack.yml +++ b/app/data/stacks/fixturenet-eth-tx/stack.yml @@ -2,8 +2,8 @@ version: "1.2" name: fixturenet-eth-tx decription: "Ethereum Fixturenet w/ tx-spammer" repos: - - cerc-io/go-ethereum - - cerc-io/tx-spammer + - github.com/cerc-io/go-ethereum + - github.com/cerc-io/tx-spammer - dboreham/foundry containers: - cerc/go-ethereum diff --git a/app/data/stacks/fixturenet-eth/README.md b/app/data/stacks/fixturenet-eth/README.md index 7bd96506..9f01417a 100644 --- a/app/data/stacks/fixturenet-eth/README.md +++ b/app/data/stacks/fixturenet-eth/README.md @@ -66,7 +66,7 @@ It is not necessary to use them all at once, but a complete example follows: ``` # Setup -$ laconic-so setup-repositories --include cerc-io/go-ethereum,cerc-io/ipld-eth-db,cerc-io/ipld-eth-server,cerc-io/ipld-eth-beacon-db,cerc-io/ipld-eth-beacon-indexer,cerc-io/eth-probe,cerc-io/tx-spammer +$ laconic-so setup-repositories --include github.com/cerc-io/go-ethereum,github.com/cerc-io/ipld-eth-db,github.com/cerc-io/ipld-eth-server,github.com/cerc-io/ipld-eth-beacon-db,github.com/cerc-io/ipld-eth-beacon-indexer,github.com/cerc-io/eth-probe,github.com/cerc-io/tx-spammer # Build $ laconic-so build-containers --include cerc/go-ethereum,cerc/lighthouse,cerc/fixturenet-eth-geth,cerc/fixturenet-eth-lighthouse,cerc/ipld-eth-db,cerc/ipld-eth-server,cerc/ipld-eth-beacon-db,cerc/ipld-eth-beacon-indexer,cerc/eth-probe,cerc/keycloak,cerc/tx-spammer diff --git a/app/data/stacks/fixturenet-eth/stack.yml b/app/data/stacks/fixturenet-eth/stack.yml index a44f086e..ccd2526a 100644 --- a/app/data/stacks/fixturenet-eth/stack.yml +++ b/app/data/stacks/fixturenet-eth/stack.yml @@ -2,8 +2,8 @@ version: "1.1" name: fixturenet-eth decription: "Ethereum Fixturenet" repos: - - cerc-io/go-ethereum - - dboreham/foundry + - github.com/cerc-io/go-ethereum + - github.com/dboreham/foundry containers: - cerc/go-ethereum - cerc/lighthouse diff --git a/app/data/stacks/fixturenet-laconic-loaded/stack.yml b/app/data/stacks/fixturenet-laconic-loaded/stack.yml index 009212f0..3102978a 100644 --- a/app/data/stacks/fixturenet-laconic-loaded/stack.yml +++ b/app/data/stacks/fixturenet-laconic-loaded/stack.yml @@ -2,14 +2,14 @@ version: "1.1" name: fixturenet-laconic-loaded description: "A full featured laconic fixturenet" repos: - - cerc-io/laconicd - - lirewine/debug - - lirewine/crypto - - lirewine/gem - - lirewine/sdk - - cerc-io/laconic-sdk - - cerc-io/laconic-registry-cli - - cerc-io/laconic-console + - github.com/cerc-io/laconicd + - github.com/lirewine/debug + - github.com/lirewine/crypto + - github.com/lirewine/gem + - github.com/lirewine/sdk + - github.com/cerc-io/laconic-sdk + - github.com/cerc-io/laconic-registry-cli + - github.com/cerc-io/laconic-console npms: - laconic-sdk - laconic-registry-cli diff --git a/app/data/stacks/fixturenet-laconicd/stack.yml b/app/data/stacks/fixturenet-laconicd/stack.yml index 54c5dba6..ce0b1946 100644 --- a/app/data/stacks/fixturenet-laconicd/stack.yml +++ b/app/data/stacks/fixturenet-laconicd/stack.yml @@ -2,9 +2,9 @@ version: "1.0" name: fixturenet-laconicd description: "A laconicd fixturenet" repos: - - cerc-io/laconicd - - cerc-io/laconic-sdk - - cerc-io/laconic-registry-cli + - github.com/cerc-io/laconicd + - github.com/cerc-io/laconic-sdk + - github.com/cerc-io/laconic-registry-cli npms: - laconic-sdk - laconic-registry-cli diff --git a/app/data/stacks/fixturenet-lotus/stack.yml b/app/data/stacks/fixturenet-lotus/stack.yml index 35617c4b..25f0f9d8 100644 --- a/app/data/stacks/fixturenet-lotus/stack.yml +++ b/app/data/stacks/fixturenet-lotus/stack.yml @@ -2,7 +2,7 @@ version: "1.0" name: fixturenet-lotus description: "A lotus fixturenet" repos: - - filecoin-project/lotus + - github.com/filecoin-project/lotus containers: - cerc/lotus pods: diff --git a/app/data/stacks/fixturenet-optimism/l2-only.md b/app/data/stacks/fixturenet-optimism/l2-only.md index 8cac9ce1..31397eee 100644 --- a/app/data/stacks/fixturenet-optimism/l2-only.md +++ b/app/data/stacks/fixturenet-optimism/l2-only.md @@ -9,7 +9,7 @@ Prerequisite: An L1 Ethereum RPC endpoint Clone required repositories: ```bash -laconic-so --stack fixturenet-optimism setup-repositories --exclude cerc-io/go-ethereum +laconic-so --stack fixturenet-optimism setup-repositories --exclude github.com/cerc-io/go-ethereum # If this throws an error as a result of being already checked out to a branch/tag in a repo, remove the repositories mentioned below and re-run the command ``` diff --git a/app/data/stacks/fixturenet-optimism/stack.yml b/app/data/stacks/fixturenet-optimism/stack.yml index 45bb1709..61570e42 100644 --- a/app/data/stacks/fixturenet-optimism/stack.yml +++ b/app/data/stacks/fixturenet-optimism/stack.yml @@ -2,10 +2,10 @@ version: "1.0" name: fixturenet-optimism decription: "Optimism Fixturenet" repos: - - cerc-io/go-ethereum - - dboreham/foundry - - ethereum-optimism/optimism - - ethereum-optimism/op-geth + - github.com/cerc-io/go-ethereum + - github.com/dboreham/foundry + - github.com/ethereum-optimism/optimism + - github.com/ethereum-optimism/op-geth containers: - cerc/go-ethereum - cerc/lighthouse diff --git a/app/data/stacks/fixturenet-pocket/stack.yml b/app/data/stacks/fixturenet-pocket/stack.yml index c5491f9a..2f96c53b 100644 --- a/app/data/stacks/fixturenet-pocket/stack.yml +++ b/app/data/stacks/fixturenet-pocket/stack.yml @@ -2,9 +2,9 @@ version: "1.0" name: fixturenet-pocket description: "A single node pocket chain that can serve relays from the geth-1 node in eth-fixturenet" repos: - - cerc-io/go-ethereum - - pokt-network/pocket-core - - pokt-network/pocket-core-deployments # contains the dockerfile + - github.com/cerc-io/go-ethereum + - github.com/pokt-network/pocket-core + - github.com/pokt-network/pocket-core-deployments # contains the dockerfile containers: - cerc/go-ethereum - cerc/lighthouse diff --git a/app/data/stacks/gelato/stack.yml b/app/data/stacks/gelato/stack.yml index 51a8d64f..3118acd3 100644 --- a/app/data/stacks/gelato/stack.yml +++ b/app/data/stacks/gelato/stack.yml @@ -1,7 +1,7 @@ version: "1.0" name: gelato repos: - - cerc-io/gelato-watcher-ts + - github.com/cerc-io/gelato-watcher-ts containers: - cerc/watcher-gelato pods: diff --git a/app/data/stacks/mobymask-v2/mobymask-only.md b/app/data/stacks/mobymask-v2/mobymask-only.md index 01ce0754..3dacef7a 100644 --- a/app/data/stacks/mobymask-v2/mobymask-only.md +++ b/app/data/stacks/mobymask-v2/mobymask-only.md @@ -9,7 +9,7 @@ Prerequisite: L2 Optimism Geth and Node RPC endpoints Clone required repositories: ```bash -laconic-so --stack mobymask-v2 setup-repositories --include cerc-io/MobyMask,cerc-io/watcher-ts,cerc-io/mobymask-v2-watcher-ts +laconic-so --stack mobymask-v2 setup-repositories --include github.com/cerc-io/MobyMask,github.com/cerc-io/watcher-ts,github.com/cerc-io/mobymask-v2-watcher-ts # If this throws an error as a result of being already checked out to a branch/tag in a repo, remove the repositories mentioned below and re-run the command ``` diff --git a/app/data/stacks/mobymask-v2/stack.yml b/app/data/stacks/mobymask-v2/stack.yml index b91e4461..9ce16ca4 100644 --- a/app/data/stacks/mobymask-v2/stack.yml +++ b/app/data/stacks/mobymask-v2/stack.yml @@ -1,13 +1,13 @@ version: "1.0" name: mobymask-v2 repos: - - cerc-io/go-ethereum - - dboreham/foundry - - ethereum-optimism/optimism - - ethereum-optimism/op-geth - - cerc-io/watcher-ts - - cerc-io/mobymask-v2-watcher-ts - - cerc-io/MobyMask + - github.com/cerc-io/go-ethereum + - github.com/dboreham/foundry + - github.com/ethereum-optimism/optimism + - github.com/ethereum-optimism/op-geth + - github.com/cerc-io/watcher-ts + - github.com/cerc-io/mobymask-v2-watcher-ts + - github.com/cerc-io/MobyMask containers: - cerc/go-ethereum - cerc/lighthouse diff --git a/app/data/stacks/mobymask-v2/watcher-p2p-network/watcher.md b/app/data/stacks/mobymask-v2/watcher-p2p-network/watcher.md index 4641726e..d5d4d53f 100644 --- a/app/data/stacks/mobymask-v2/watcher-p2p-network/watcher.md +++ b/app/data/stacks/mobymask-v2/watcher-p2p-network/watcher.md @@ -14,7 +14,7 @@ This demo has been tested on a `Ubuntu 22.04 LTS` machine with `8GB` of RAM Clone required repositories: ```bash - laconic-so --stack mobymask-v2 setup-repositories --include cerc-io/MobyMask,cerc-io/watcher-ts,cerc-io/mobymask-v2-watcher-ts + laconic-so --stack mobymask-v2 setup-repositories --include github.com/cerc-io/MobyMask,github.com/cerc-io/watcher-ts,github.com/cerc-io/mobymask-v2-watcher-ts # This will clone the required repositories at ~/cerc # If this throws an error as a result of being already checked out to a branch/tag in a repo, remove the repositories mentioned in the next step and re-run the command diff --git a/app/data/stacks/mobymask/README.md b/app/data/stacks/mobymask/README.md index d048ba38..28d55ea1 100644 --- a/app/data/stacks/mobymask/README.md +++ b/app/data/stacks/mobymask/README.md @@ -11,7 +11,7 @@ This deployment expects that ipld-eth-server's endpoints are available on the lo ## Clone required repositories ``` -$ laconic-so setup-repositories --include cerc-io/watcher-ts +$ laconic-so setup-repositories --include github.com/cerc-io/watcher-ts ``` ## Build the watcher container diff --git a/app/data/stacks/mobymask/stack.yml b/app/data/stacks/mobymask/stack.yml index 794ec4ab..12799821 100644 --- a/app/data/stacks/mobymask/stack.yml +++ b/app/data/stacks/mobymask/stack.yml @@ -1,7 +1,7 @@ version: "1.0" name: mobymask-watcher repos: - - cerc-io/watcher-ts/v0.2.19 + - github.com/cerc-io/watcher-ts/v0.2.19 containers: - cerc/watcher-mobymask pods: diff --git a/app/data/stacks/package-registry/stack.yml b/app/data/stacks/package-registry/stack.yml index 596f6e72..5eb5f8ff 100644 --- a/app/data/stacks/package-registry/stack.yml +++ b/app/data/stacks/package-registry/stack.yml @@ -2,8 +2,8 @@ version: "1.1" name: package-registry decription: "Local Package Registry" repos: - - cerc-io/hosting - - telackey/act_runner + - github.com/cerc-io/hosting + - github.com/telackey/act_runner containers: - cerc/act-runner - cerc/act-runner-task-executor diff --git a/app/data/stacks/test/stack.yml b/app/data/stacks/test/stack.yml index 44ac8604..b564ccaa 100644 --- a/app/data/stacks/test/stack.yml +++ b/app/data/stacks/test/stack.yml @@ -2,7 +2,8 @@ version: "1.0" name: test description: "A test stack" repos: - - cerc-io/laconicd + - github.com/cerc-io/laconicd + - git.vdb.to/cerc-io/test-project containers: - cerc/test-container pods: diff --git a/app/data/stacks/uniswap-v3/stack.yml b/app/data/stacks/uniswap-v3/stack.yml index 8eec8f87..5073a765 100644 --- a/app/data/stacks/uniswap-v3/stack.yml +++ b/app/data/stacks/uniswap-v3/stack.yml @@ -1,8 +1,8 @@ version: "1.0" name: uniswap-v3 repos: - - vulcanize/uniswap-watcher-ts - - vulcanize/uniswap-v3-info + - github.com/vulcanize/uniswap-watcher-ts + - github.com/vulcanize/uniswap-v3-info containers: - cerc/watcher-uniswap-v3 - cerc/uniswap-v3-info diff --git a/app/setup_repositories.py b/app/setup_repositories.py index e8c08916..770af6d9 100644 --- a/app/setup_repositories.py +++ b/app/setup_repositories.py @@ -125,11 +125,22 @@ def command(ctx, include, exclude, git_ssh, check_only, pull, branches_file): if verbose: print(f"Excluding: {repo}") - def process_repo(repo): - git_ssh_prefix = "git@github.com:" - git_http_prefix = "https://github.com/" - full_github_repo_path = f"{git_ssh_prefix if git_ssh else git_http_prefix}{repo}" - repoName = repo.split("/")[-1] + def host_and_path_for_repo(fully_qualified_repo): + repo_split = fully_qualified_repo.split("/") + # Legacy unqualified repo means github + if len(repo_split) == 2: + return "github.com", "/".join(repo_split) + else: + if len(repo_split) == 3: + # First part is the host + return repo_split[0], "/".join(repo_split[1:]) + + def process_repo(fully_qualified_repo): + repo_host, repo_path = host_and_path_for_repo(fully_qualified_repo) + git_ssh_prefix = f"git@{repo_host}:" + git_http_prefix = f"https://{repo_host}/" + full_github_repo_path = f"{git_ssh_prefix if git_ssh else git_http_prefix}{repo_path}" + repoName = repo_path.split("/")[-1] full_filesystem_repo_path = os.path.join(dev_root_path, repoName) is_present = os.path.isdir(full_filesystem_repo_path) if not quiet: