Static testnet configs (#1603)
## Issue Addressed #1431 ## Proposed Changes Added an archived zip file with required files manually ## Additional Info 1) Used zip, instead of tar.gz to add a single dependency instead of two. 2) I left the download from github code for now, waiting to hear if you'd like it cleaned up or left to be used for some tooling needs.
This commit is contained in:
Daniel Schonfeld
parent
0525876882
commit
810de2f8b7
135
Cargo.lock
generated
135
Cargo.lock
generated
@ -623,6 +623,27 @@ version = "0.5.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0e4cec68f03f32e44924783795810fa50a7035d8c8ebe78580ad7e6c703fba38"
|
||||
|
||||
[[package]]
|
||||
name = "bzip2"
|
||||
version = "0.3.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "42b7c3cbf0fa9c1b82308d57191728ca0256cb821220f4e2fd410a72ade26e3b"
|
||||
dependencies = [
|
||||
"bzip2-sys",
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bzip2-sys"
|
||||
version = "0.1.9+1.0.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ad3b39a260062fca31f7b0b12f207e8f2590a67d32ec7d59c20484b07ea7285e"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"libc",
|
||||
"pkg-config",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "c_linked_list"
|
||||
version = "1.1.1"
|
||||
@ -1579,13 +1600,11 @@ dependencies = [
|
||||
"enr",
|
||||
"eth2_config",
|
||||
"eth2_ssz",
|
||||
"handlebars",
|
||||
"reqwest",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"serde_yaml",
|
||||
"tempdir",
|
||||
"types",
|
||||
"zip",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -1997,9 +2016,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "getrandom"
|
||||
version = "0.1.14"
|
||||
version = "0.1.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7abc8dd8451921606d809ba32e95b6111925cd2906060d2dcc29c070220503eb"
|
||||
checksum = "fc587bc0ec293155d5bfa6b9891ec18a1e330c234f896ea47fbada4cadbe47e6"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"libc",
|
||||
@ -2092,20 +2111,6 @@ version = "1.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d36fab90f82edc3c747f9d438e06cf0a491055896f2a279638bb5beed6c40177"
|
||||
|
||||
[[package]]
|
||||
name = "handlebars"
|
||||
version = "3.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5deefd4816fb852b1ff3cb48f6c41da67be2d0e1d20b26a7a3b076da11f064b1"
|
||||
dependencies = [
|
||||
"log 0.4.11",
|
||||
"pest",
|
||||
"pest_derive",
|
||||
"quick-error 2.0.0",
|
||||
"serde",
|
||||
"serde_json",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hashbrown"
|
||||
version = "0.6.3"
|
||||
@ -2263,7 +2268,7 @@ version = "1.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "df004cfca50ef23c36850aaaa59ad52cc70d0e90243c3c7737a4dd32dc7a3c4f"
|
||||
dependencies = [
|
||||
"quick-error 1.2.3",
|
||||
"quick-error",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -2441,9 +2446,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "integer-sqrt"
|
||||
version = "0.1.4"
|
||||
version = "0.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1d3038ef3da477fd70e6161006ef37f919c24bfe7244062f67a69defe72c02aa"
|
||||
checksum = "276ec31bcb4a9ee45f58bec6f9ec700ae4cf4f4f8f2fa7e06cb406bd5ffdd770"
|
||||
dependencies = [
|
||||
"num-traits",
|
||||
]
|
||||
@ -2601,9 +2606,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "libc"
|
||||
version = "0.2.76"
|
||||
version = "0.2.77"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "755456fae044e6fa1ebbbd1b3e902ae19e73097ed4ed87bb79934a867c007bc3"
|
||||
checksum = "f2f96b10ec2560088a8e76961b00d47107b3a625fecb76dedb29ee7ccbf98235"
|
||||
|
||||
[[package]]
|
||||
name = "libflate"
|
||||
@ -3676,49 +3681,6 @@ version = "2.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e"
|
||||
|
||||
[[package]]
|
||||
name = "pest"
|
||||
version = "2.1.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "10f4872ae94d7b90ae48754df22fd42ad52ce740b8f370b03da4835417403e53"
|
||||
dependencies = [
|
||||
"ucd-trie",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pest_derive"
|
||||
version = "2.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "833d1ae558dc601e9a60366421196a8d94bc0ac980476d0b67e1d0988d72b2d0"
|
||||
dependencies = [
|
||||
"pest",
|
||||
"pest_generator",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pest_generator"
|
||||
version = "2.1.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "99b8db626e31e5b81787b9783425769681b347011cc59471e33ea46d2ea0cf55"
|
||||
dependencies = [
|
||||
"pest",
|
||||
"pest_meta",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pest_meta"
|
||||
version = "2.1.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "54be6e404f5317079812fc8f9f5279de376d8856929e21c184ecf6bbd692a11d"
|
||||
dependencies = [
|
||||
"maplit",
|
||||
"pest",
|
||||
"sha-1 0.8.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "petgraph"
|
||||
version = "0.5.1"
|
||||
@ -3785,6 +3747,12 @@ dependencies = [
|
||||
"web-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "podio"
|
||||
version = "0.1.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b18befed8bc2b61abc79a457295e7e838417326da1586050b919414073977f19"
|
||||
|
||||
[[package]]
|
||||
name = "poly1305"
|
||||
version = "0.6.0"
|
||||
@ -3837,9 +3805,9 @@ checksum = "eba180dafb9038b050a4c280019bbedf9f2467b61e5d892dcad585bb57aadc5a"
|
||||
|
||||
[[package]]
|
||||
name = "proc-macro2"
|
||||
version = "1.0.20"
|
||||
version = "1.0.21"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "175c513d55719db99da20232b06cda8bab6b83ec2d04e3283edf0213c37c1a29"
|
||||
checksum = "36e28516df94f3dd551a587da5357459d9b36d945a7c37c3557928c1c2ff2a2c"
|
||||
dependencies = [
|
||||
"unicode-xid",
|
||||
]
|
||||
@ -3964,12 +3932,6 @@ version = "1.2.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0"
|
||||
|
||||
[[package]]
|
||||
name = "quick-error"
|
||||
version = "2.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3ac73b1112776fc109b2e61909bc46c7e1bf0d7f690ffb1676553acce16d5cda"
|
||||
|
||||
[[package]]
|
||||
name = "quickcheck"
|
||||
version = "0.9.2"
|
||||
@ -4997,9 +4959,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "socket2"
|
||||
version = "0.3.12"
|
||||
version = "0.3.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "03088793f677dce356f3ccc2edb1b314ad191ab702a5de3faf49304f7e104918"
|
||||
checksum = "b1fa70dc5c8104ec096f4fe7ede7a221d35ae13dcd19ba1ad9a81d2cab9a1c44"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"libc",
|
||||
@ -5905,12 +5867,6 @@ dependencies = [
|
||||
"tree_hash_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ucd-trie"
|
||||
version = "0.1.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "56dee185309b50d1f11bfedef0fe6d036842e3fb77413abef29f8f8d1c5d4c1c"
|
||||
|
||||
[[package]]
|
||||
name = "uhttp_sse"
|
||||
version = "0.5.1"
|
||||
@ -6542,3 +6498,16 @@ dependencies = [
|
||||
"syn",
|
||||
"synstructure",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zip"
|
||||
version = "0.5.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "58287c28d78507f5f91f2a4cf1e8310e2c76fd4c6932f93ac60fd1ceb402db7d"
|
||||
dependencies = [
|
||||
"bzip2",
|
||||
"crc32fast",
|
||||
"flate2",
|
||||
"podio",
|
||||
"time 0.1.44",
|
||||
]
|
||||
|
||||
@ -48,22 +48,29 @@ impl Eth2Config {
|
||||
/// Used by the `eth2_testnet_config` crate to initialize testnet directories during build and
|
||||
/// access them at runtime.
|
||||
#[derive(Copy, Clone, Debug, PartialEq)]
|
||||
pub struct Eth2NetDirectory<'a> {
|
||||
pub struct Eth2NetArchiveAndDirectory<'a> {
|
||||
pub name: &'a str,
|
||||
pub unique_id: &'a str,
|
||||
pub archive_name: &'a str,
|
||||
pub commit: &'a str,
|
||||
pub url_template: &'a str,
|
||||
pub genesis_is_known: bool,
|
||||
}
|
||||
|
||||
impl<'a> Eth2NetDirectory<'a> {
|
||||
impl<'a> Eth2NetArchiveAndDirectory<'a> {
|
||||
/// The directory that should be used to store files downloaded for this net.
|
||||
pub fn dir(&self) -> PathBuf {
|
||||
fn pwd(&self) -> PathBuf {
|
||||
env::var("CARGO_MANIFEST_DIR")
|
||||
.expect("should know manifest dir")
|
||||
.parse::<PathBuf>()
|
||||
.expect("should parse manifest dir as path")
|
||||
.join(self.unique_id)
|
||||
}
|
||||
|
||||
pub fn dir(&self) -> PathBuf {
|
||||
self.pwd().join(self.unique_id)
|
||||
}
|
||||
|
||||
pub fn archive_fullpath(&self) -> PathBuf {
|
||||
self.pwd().join(self.archive_name)
|
||||
}
|
||||
}
|
||||
|
||||
@ -72,19 +79,23 @@ macro_rules! unique_id {
|
||||
($name: tt, $commit: tt, $genesis_is_known: tt) => {
|
||||
concat!("testnet_", $name, "_", $commit, "_", $genesis_is_known);
|
||||
};
|
||||
|
||||
($name: tt, $commit: tt) => {
|
||||
concat!("testnet_", $name, "_", $commit, ".zip");
|
||||
};
|
||||
}
|
||||
|
||||
macro_rules! define_net {
|
||||
($title: ident, $macro_title: tt, $name: tt, $commit: tt, $url_template: tt, $genesis_is_known: tt) => {
|
||||
($title: ident, $macro_title: tt, $name: tt, $commit: tt, $genesis_is_known: tt) => {
|
||||
#[macro_use]
|
||||
pub mod $title {
|
||||
use super::*;
|
||||
|
||||
pub const ETH2_NET_DIR: Eth2NetDirectory = Eth2NetDirectory {
|
||||
pub const ETH2_NET_DIR: Eth2NetArchiveAndDirectory = Eth2NetArchiveAndDirectory {
|
||||
name: $name,
|
||||
unique_id: unique_id!($name, $commit, $genesis_is_known),
|
||||
archive_name: unique_id!($name, $commit),
|
||||
commit: $commit,
|
||||
url_template: $url_template,
|
||||
genesis_is_known: $genesis_is_known,
|
||||
};
|
||||
|
||||
@ -110,7 +121,6 @@ define_net!(
|
||||
include_altona_file,
|
||||
"altona",
|
||||
"a94e00c1a03df851f960fcf44a79f2a6b1d29af1",
|
||||
"https://raw.githubusercontent.com/sigp/witti/{{ commit }}/altona/lighthouse/{{ file }}",
|
||||
true
|
||||
);
|
||||
|
||||
@ -119,7 +129,6 @@ define_net!(
|
||||
include_medalla_file,
|
||||
"medalla",
|
||||
"09bbf2c9d108944ac934f94ec6a1d0684ca062a5",
|
||||
"https://raw.githubusercontent.com/sigp/witti/{{ commit }}/medalla/{{ file }}",
|
||||
true
|
||||
);
|
||||
|
||||
|
||||
2
common/eth2_testnet_config/.gitignore
vendored
2
common/eth2_testnet_config/.gitignore
vendored
@ -1,4 +1,4 @@
|
||||
testnet*
|
||||
testnet*/*
|
||||
schlesi-*
|
||||
witti-*
|
||||
altona*
|
||||
|
||||
@ -7,10 +7,8 @@ edition = "2018"
|
||||
build = "build.rs"
|
||||
|
||||
[build-dependencies]
|
||||
reqwest = { version = "0.10.4", features = ["blocking", "native-tls-vendored"] }
|
||||
zip = "0.5"
|
||||
eth2_config = { path = "../eth2_config"}
|
||||
handlebars = "3.3.0"
|
||||
serde_json = "1.0.56"
|
||||
|
||||
[dev-dependencies]
|
||||
tempdir = "0.3.7"
|
||||
@ -21,4 +19,4 @@ serde_yaml = "0.8.11"
|
||||
types = { path = "../../consensus/types"}
|
||||
enr = { version = "0.1.0", features = ["libsecp256k1", "ed25519"] }
|
||||
eth2_ssz = "0.1.2"
|
||||
eth2_config = { path = "../eth2_config"}
|
||||
eth2_config = { path = "../eth2_config"}
|
||||
@ -1,80 +1,68 @@
|
||||
//! Downloads a testnet configuration from Github.
|
||||
|
||||
use eth2_config::{altona, medalla, Eth2NetDirectory};
|
||||
use handlebars::Handlebars;
|
||||
use serde_json::json;
|
||||
use eth2_config::{altona, medalla, Eth2NetArchiveAndDirectory};
|
||||
use std::fs;
|
||||
use std::fs::File;
|
||||
use std::io::Write;
|
||||
use std::io;
|
||||
use zip::ZipArchive;
|
||||
|
||||
const ETH2_NET_DIRS: &[Eth2NetDirectory<'static>] = &[altona::ETH2_NET_DIR, medalla::ETH2_NET_DIR];
|
||||
const ETH2_NET_DIRS: &[Eth2NetArchiveAndDirectory<'static>] =
|
||||
&[altona::ETH2_NET_DIR, medalla::ETH2_NET_DIR];
|
||||
|
||||
fn main() {
|
||||
for testnet in ETH2_NET_DIRS {
|
||||
let testnet_dir = testnet.dir();
|
||||
|
||||
let archive_fullpath = testnet.archive_fullpath();
|
||||
//no need to do anything if archives have already been uncompressed before
|
||||
if !testnet_dir.exists() {
|
||||
std::fs::create_dir_all(&testnet_dir)
|
||||
.unwrap_or_else(|_| panic!("Unable to create {:?}", testnet_dir));
|
||||
if archive_fullpath.exists() {
|
||||
//uncompress archive and continue
|
||||
let archive_file = match File::open(&archive_fullpath) {
|
||||
Ok(f) => f,
|
||||
Err(e) => panic!("Problem opening archive file: {}", e),
|
||||
};
|
||||
|
||||
match get_all_files(testnet) {
|
||||
Ok(()) => (),
|
||||
Err(e) => {
|
||||
std::fs::remove_dir_all(&testnet_dir).unwrap_or_else(|_| panic!(
|
||||
"{}. Failed to remove {:?}, please remove the directory manually because it may contains incomplete testnet data.",
|
||||
e,
|
||||
testnet_dir,
|
||||
));
|
||||
panic!(e);
|
||||
}
|
||||
match uncompress(archive_file) {
|
||||
Ok(_) => (),
|
||||
Err(e) => panic!(e),
|
||||
};
|
||||
} else {
|
||||
panic!(
|
||||
"Couldn't find testnet archive at this location: {:?}",
|
||||
archive_fullpath
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn get_all_files(testnet: &Eth2NetDirectory<'static>) -> Result<(), String> {
|
||||
get_file(testnet, "boot_enr.yaml")?;
|
||||
get_file(testnet, "config.yaml")?;
|
||||
get_file(testnet, "deploy_block.txt")?;
|
||||
get_file(testnet, "deposit_contract.txt")?;
|
||||
fn uncompress(archive_file: File) -> Result<(), String> {
|
||||
let mut archive =
|
||||
ZipArchive::new(archive_file).map_err(|e| format!("Error with zip file: {}", e))?;
|
||||
for i in 0..archive.len() {
|
||||
let mut file = archive
|
||||
.by_index(i)
|
||||
.map_err(|e| format!("Error retrieving file {} inside zip: {}", i, e))?;
|
||||
|
||||
if testnet.genesis_is_known {
|
||||
get_file(testnet, "genesis.ssz")?;
|
||||
} else {
|
||||
File::create(testnet.dir().join("genesis.ssz")).unwrap();
|
||||
let outpath = file.sanitized_name();
|
||||
|
||||
if file.name().ends_with('/') {
|
||||
fs::create_dir_all(&outpath)
|
||||
.map_err(|e| format!("Error creating testnet directories: {}", e))?;
|
||||
} else {
|
||||
if let Some(p) = outpath.parent() {
|
||||
if !p.exists() {
|
||||
fs::create_dir_all(&p)
|
||||
.map_err(|e| format!("Error creating testnet directories: {}", e))?;
|
||||
}
|
||||
}
|
||||
|
||||
let mut outfile = File::create(&outpath)
|
||||
.map_err(|e| format!("Error while creating file {:?}: {}", outpath, e))?;
|
||||
io::copy(&mut file, &mut outfile)
|
||||
.map_err(|e| format!("Error writing file {:?}: {}", outpath, e))?;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn get_file(testnet: &Eth2NetDirectory, filename: &str) -> Result<(), String> {
|
||||
let url = Handlebars::new()
|
||||
.render_template(
|
||||
testnet.url_template,
|
||||
&json!({"commit": testnet.commit, "file": filename}),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let path = testnet.dir().join(filename);
|
||||
|
||||
let mut file =
|
||||
File::create(path).map_err(|e| format!("Failed to create {}: {:?}", filename, e))?;
|
||||
|
||||
let request = reqwest::blocking::Client::builder()
|
||||
.build()
|
||||
.map_err(|_| "Could not build request client".to_string())?
|
||||
.get(&url)
|
||||
.timeout(std::time::Duration::from_secs(120));
|
||||
|
||||
let contents = request
|
||||
.send()
|
||||
.map_err(|e| format!("Failed to download {}: {}", filename, e))?
|
||||
.error_for_status()
|
||||
.map_err(|e| format!("Error downloading {}: {}", filename, e))?
|
||||
.bytes()
|
||||
.map_err(|e| format!("Failed to read {} response bytes: {}", filename, e))?;
|
||||
|
||||
file.write(&contents)
|
||||
.map_err(|e| format!("Failed to write to {}: {:?}", filename, e))?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
Binary file not shown.
Binary file not shown.
Loading…
Reference in New Issue
Block a user