Run EF tests on GitLab (#562)

.gitignore

@@ -6,3 +6,4 @@ Cargo.lock
 *.raw_keypairs
 flamegraph.svg
 perf.data*
+*.tar.gz

.gitlab-ci.yml

@@ -1,6 +1,10 @@
 #Adapted from https://users.rust-lang.org/t/my-gitlab-config-docs-tests/16396
 
-image: 'sigp/lighthouse:latest'
+default:
+  image: 'sigp/lighthouse:latest'
+  cache:
+    paths:
+      - tests/ef_tests/*-v0.8.3.tar.gz
 
 stages:
   - test
@@ -29,12 +33,21 @@ test-release:
   script:
     - cargo test --verbose --all --release
 
-test-fake-crypto:
+test-ef:
   stage: test
   variables:
     GIT_SUBMODULE_STRATEGY: normal
   script:
-    - cargo test --manifest-path tests/ef_tests/Cargo.toml --release --features fake_crypto
+    - make make-ef-tests
+    - cargo test --manifest-path tests/ef_tests/Cargo.toml --release --features ef_tests
+
+test-ef-fake-crypto:
+  stage: test
+  variables:
+    GIT_SUBMODULE_STRATEGY: normal
+  script:
+    - make make-ef-tests
+    - cargo test --manifest-path tests/ef_tests/Cargo.toml --release --features ef_tests fake_crypto
 
 documentation:
   stage: document

tests/ef_tests/Cargo.toml

@@ -5,6 +5,8 @@ authors = ["Paul Hauner <paul@paulhauner.com>"]
 edition = "2018"
 
 [features]
+# `ef_tests` feature must be enabled to actually run the tests
+ef_tests = []
 fake_crypto = ["bls/fake_crypto"]
 
 [dependencies]

tests/ef_tests/Makefile

@@ -1,18 +1,28 @@
+# Bump the test tag here and in .gitlab-ci.yml and CI will take care of updating the cached tarballs
 TESTS_TAG := v0.8.3
 TESTS = general minimal mainnet
+TARBALLS = $(patsubst %,%-$(TESTS_TAG).tar.gz,$(TESTS))
 
 REPO_NAME := eth2.0-spec-tests
 OUTPUT_DIR := ./$(REPO_NAME)
 
 BASE_URL := https://github.com/ethereum/$(REPO_NAME)/releases/download/$(TESTS_TAG)
 
-eth2.0-spec-tests:
+$(OUTPUT_DIR): $(TARBALLS)
 	mkdir $(OUTPUT_DIR)
-	for test in $(TESTS); do \
+	for test_tarball in $^; do \
-		wget $(BASE_URL)/$$test.tar.gz; \
+		tar -xzf $$test_tarball -C $(OUTPUT_DIR);\
-		tar -xzvf $$test.tar.gz -C $(OUTPUT_DIR);\
-		rm $$test.tar.gz;\
 	done
 
-clean:
+%-$(TESTS_TAG).tar.gz:
-	rm -r $(OUTPUT_DIR)
+	wget $(BASE_URL)/$*.tar.gz -O $@
+
+clean-test-files:
+	rm -rf $(OUTPUT_DIR)
+
+clean-archives:
+	rm -f $(TARBALLS)
+
+clean: clean-test-files clean-archives
+
+.PHONY: clean clean-archives clean-test-files

tests/ef_tests/README.md

@@ -2,22 +2,12 @@
 
 This crate parses and executes the test vectors at [ethereum/eth2.0-spec-tests](https://github.com/ethereum/eth2.0-spec-tests).
 
-Functionality is achieved only via the `$ cargo test` command.
+Functionality is achieved only via the `$ cargo test --features ef_tests` command.
 
-## Tests
+## Running the Tests
 
-Because the test vectors are very large, we do not download the
+Because the test vectors are very large, we do not download or run them by default.
-tests vectors or require that the tests pass by default. Specifically;
+To download them, run (in this directory):
-
-- If the `tests/ef_tests/eth2.0-spec-tests` directory is not present, all tests
-	indicate a `pass` when they did not actually run.
-- If that directory _is_ present, the tests are executed faithfully, failing if
-	a discrepancy is found.
-
-## Downloading Test Vectors
-
-The `eth2.0-spec-tests` directory is not present by default. To
-obtain it, use the Makefile:
 
 ```
 $ make
@@ -27,4 +17,26 @@ _Note: this may download hundreds of MB of compressed archives from the
 [ethereum/eth2.0-spec-tests](https://github.com/ethereum/eth2.0-spec-tests/),
 which may expand into several GB of files._
 
-Remove the tests to save space or update to a new version with `$ make clean`.
+If successful, you should now have the extracted tests in `./eth2.0-spec-tests`.
+
+Run them with:
+
+```
+$ cargo test --features ef_tests
+```
+
+The tests won't run without the `ef_tests` feature enabled (this is to ensure that a top-level
+`cargo test --all` won't fail on missing files).
+
+## Saving Space
+
+When you download the tests, the downloaded archives will be kept in addition to the extracted
+files. You have several options for saving space:
+
+1. Delete the archives (`make clean-archives`), and keep the extracted files. Suitable for everyday
+   use, just don't re-run `make` or it will redownload the archives.
+2. Delete the extracted files (`make clean-test-files`), and keep the archives. Suitable for CI, or
+   temporarily saving space. If you re-run `make` it will extract the archives rather than
+   redownloading them.
+3. Delete everything (`make clean`). Good for updating to a new version, or if you no longer wish to
+   run the EF tests.

tests/ef_tests/src/handler.rs

@@ -31,11 +31,6 @@ pub trait Handler {
             .join(Self::runner_name())
             .join(Self::handler_name());
 
-        // If the directory containing the tests does not exist, just let all tests pass.
-        if !handler_path.exists() {
-            return;
-        }
-
         // Iterate through test suites
         let test_cases = fs::read_dir(&handler_path)
             .expect("handler dir exists")

tests/ef_tests/tests/tests.rs

@@ -1,3 +1,5 @@
+#![cfg(feature = "ef_tests")]
+
 use ef_tests::*;
 use types::*;