Run EF tests on GitLab (#562)

This commit is contained in:
Michael Sproul 2019-10-22 12:35:35 +09:00 committed by GitHub
parent 96a327fe50
commit 6484d4197d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 65 additions and 30 deletions

1
.gitignore vendored
View File

@ -6,3 +6,4 @@ Cargo.lock
*.raw_keypairs
flamegraph.svg
perf.data*
*.tar.gz

View File

@ -1,6 +1,10 @@
#Adapted from https://users.rust-lang.org/t/my-gitlab-config-docs-tests/16396
image: 'sigp/lighthouse:latest'
default:
image: 'sigp/lighthouse:latest'
cache:
paths:
- tests/ef_tests/*-v0.8.3.tar.gz
stages:
- test
@ -29,12 +33,21 @@ test-release:
script:
- cargo test --verbose --all --release
test-fake-crypto:
test-ef:
stage: test
variables:
GIT_SUBMODULE_STRATEGY: normal
script:
- cargo test --manifest-path tests/ef_tests/Cargo.toml --release --features fake_crypto
- make make-ef-tests
- cargo test --manifest-path tests/ef_tests/Cargo.toml --release --features ef_tests
test-ef-fake-crypto:
stage: test
variables:
GIT_SUBMODULE_STRATEGY: normal
script:
- make make-ef-tests
- cargo test --manifest-path tests/ef_tests/Cargo.toml --release --features ef_tests fake_crypto
documentation:
stage: document

View File

@ -5,6 +5,8 @@ authors = ["Paul Hauner <paul@paulhauner.com>"]
edition = "2018"
[features]
# `ef_tests` feature must be enabled to actually run the tests
ef_tests = []
fake_crypto = ["bls/fake_crypto"]
[dependencies]

View File

@ -1,18 +1,28 @@
# Bump the test tag here and in .gitlab-ci.yml and CI will take care of updating the cached tarballs
TESTS_TAG := v0.8.3
TESTS = general minimal mainnet
TARBALLS = $(patsubst %,%-$(TESTS_TAG).tar.gz,$(TESTS))
REPO_NAME := eth2.0-spec-tests
OUTPUT_DIR := ./$(REPO_NAME)
BASE_URL := https://github.com/ethereum/$(REPO_NAME)/releases/download/$(TESTS_TAG)
eth2.0-spec-tests:
$(OUTPUT_DIR): $(TARBALLS)
mkdir $(OUTPUT_DIR)
for test in $(TESTS); do \
wget $(BASE_URL)/$$test.tar.gz; \
tar -xzvf $$test.tar.gz -C $(OUTPUT_DIR);\
rm $$test.tar.gz;\
for test_tarball in $^; do \
tar -xzf $$test_tarball -C $(OUTPUT_DIR);\
done
clean:
rm -r $(OUTPUT_DIR)
%-$(TESTS_TAG).tar.gz:
wget $(BASE_URL)/$*.tar.gz -O $@
clean-test-files:
rm -rf $(OUTPUT_DIR)
clean-archives:
rm -f $(TARBALLS)
clean: clean-test-files clean-archives
.PHONY: clean clean-archives clean-test-files

View File

@ -2,22 +2,12 @@
This crate parses and executes the test vectors at [ethereum/eth2.0-spec-tests](https://github.com/ethereum/eth2.0-spec-tests).
Functionality is achieved only via the `$ cargo test` command.
Functionality is achieved only via the `$ cargo test --features ef_tests` command.
## Tests
## Running the Tests
Because the test vectors are very large, we do not download the
tests vectors or require that the tests pass by default. Specifically;
- If the `tests/ef_tests/eth2.0-spec-tests` directory is not present, all tests
indicate a `pass` when they did not actually run.
- If that directory _is_ present, the tests are executed faithfully, failing if
a discrepancy is found.
## Downloading Test Vectors
The `eth2.0-spec-tests` directory is not present by default. To
obtain it, use the Makefile:
Because the test vectors are very large, we do not download or run them by default.
To download them, run (in this directory):
```
$ make
@ -27,4 +17,26 @@ _Note: this may download hundreds of MB of compressed archives from the
[ethereum/eth2.0-spec-tests](https://github.com/ethereum/eth2.0-spec-tests/),
which may expand into several GB of files._
Remove the tests to save space or update to a new version with `$ make clean`.
If successful, you should now have the extracted tests in `./eth2.0-spec-tests`.
Run them with:
```
$ cargo test --features ef_tests
```
The tests won't run without the `ef_tests` feature enabled (this is to ensure that a top-level
`cargo test --all` won't fail on missing files).
## Saving Space
When you download the tests, the downloaded archives will be kept in addition to the extracted
files. You have several options for saving space:
1. Delete the archives (`make clean-archives`), and keep the extracted files. Suitable for everyday
use, just don't re-run `make` or it will redownload the archives.
2. Delete the extracted files (`make clean-test-files`), and keep the archives. Suitable for CI, or
temporarily saving space. If you re-run `make` it will extract the archives rather than
redownloading them.
3. Delete everything (`make clean`). Good for updating to a new version, or if you no longer wish to
run the EF tests.

View File

@ -31,11 +31,6 @@ pub trait Handler {
.join(Self::runner_name())
.join(Self::handler_name());
// If the directory containing the tests does not exist, just let all tests pass.
if !handler_path.exists() {
return;
}
// Iterate through test suites
let test_cases = fs::read_dir(&handler_path)
.expect("handler dir exists")

View File

@ -1,3 +1,5 @@
#![cfg(feature = "ef_tests")]
use ef_tests::*;
use types::*;