Various CI fixes (#3813)

## Issue Addressed Closes #3812 Closes #3750 Closes #3705
2022-12-20 01:34:52 +00:00 · 2022-12-20 01:34:52 +00:00 · 59a7a4703c
commit 59a7a4703c
parent 53aad18da3
5 changed files with 51 additions and 37 deletions
--- a/.github/workflows/local-testnet.yml
+++ b/.github/workflows/local-testnet.yml
@ -21,7 +21,7 @@ jobs:
      - name: Get latest version of stable Rust
        run: rustup update stable
      - name: Install Protoc
-        uses: arduino/setup-protoc@v1
+        uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612
        with:
            repo-token: ${{ secrets.GITHUB_TOKEN }}
      - name: Install ganache
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@ -8,8 +8,8 @@ on:
 env:
    DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
    DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
-    REPO_NAME: sigp/lighthouse
-    IMAGE_NAME: sigp/lighthouse
+    REPO_NAME: ${{ github.repository_owner }}/lighthouse
+    IMAGE_NAME: ${{ github.repository_owner }}/lighthouse

 jobs:
    extract-version:
@ -63,12 +63,8 @@ jobs:
        steps:
            - name: Checkout sources
              uses: actions/checkout@v3
-            - name: Build toolchain
-              uses: actions-rs/toolchain@v1
-              with:
-                  toolchain: stable
-                  profile:  minimal
-                  override: true
+            - name: Get latest version of stable Rust
+              run: rustup update stable

            # ==============================
            # Windows dependencies
@ -88,7 +84,7 @@ jobs:
            # ==============================
            - name: Install Protoc
              if: contains(matrix.arch, 'darwin') || contains(matrix.arch, 'windows')
-              uses: arduino/setup-protoc@v1
+              uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612
              with:
                repo-token: ${{ secrets.GITHUB_TOKEN }}

@ -179,13 +175,13 @@ jobs:
            # =======================================================================

            - name:  Upload artifact
-              uses:  actions/upload-artifact@v2
+              uses:  actions/upload-artifact@v3
              with:
                  name: lighthouse-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.arch }}.tar.gz
                  path: lighthouse-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.arch }}.tar.gz

            - name: Upload signature
-              uses: actions/upload-artifact@v2
+              uses: actions/upload-artifact@v3
              with:
                  name: lighthouse-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.arch }}.tar.gz.asc
                  path: lighthouse-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.arch }}.tar.gz.asc
@ -208,7 +204,7 @@ jobs:
            # ==============================

            - name: Download artifacts
-              uses: actions/download-artifact@v2
+              uses: actions/download-artifact@v3

            # ==============================
            #       Create release draft
@ -216,11 +212,14 @@ jobs:

            - name: Generate Full Changelog
              id: changelog
-              run: echo "CHANGELOG=$(git log --pretty=format:"- %s" $(git describe --tags --abbrev=0 ${{ env.VERSION }}^)..${{ env.VERSION }})" >> $GITHUB_OUTPUT
+              run: |
+                echo "CHANGELOG<<EOF" >> $GITHUB_OUTPUT
+                echo "$(git log --pretty=format:"- %s" $(git describe --tags --abbrev=0 ${{ env.VERSION }}^)..${{ env.VERSION }})" >> $GITHUB_OUTPUT
+                echo "EOF" >> $GITHUB_OUTPUT

            - name: Create Release Draft
              env:
-                  GITHUB_USER: sigp
+                  GITHUB_USER: ${{ github.repository_owner }}
                  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

              # The formatting here is borrowed from OpenEthereum: https://github.com/openethereum/openethereum/blob/main/.github/workflows/build.yml
--- a/.github/workflows/test-suite.yml
+++ b/.github/workflows/test-suite.yml
@ -13,6 +13,8 @@ env:
  RUSTFLAGS: "-D warnings"
  # The Nightly version used for cargo-udeps, might need updating from time to time.
  PINNED_NIGHTLY: nightly-2022-05-20
+  # Prevent Github API rate limiting.
+  LIGHTHOUSE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 jobs:
  target-branch-check:
    name: target-branch-check
@ -51,7 +53,7 @@ jobs:
    - name: Get latest version of stable Rust
      run: rustup update stable
    - name: Install Protoc
-      uses: arduino/setup-protoc@v1
+      uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612
      with:
        repo-token: ${{ secrets.GITHUB_TOKEN }}
    - name: Install ganache
@ -95,7 +97,7 @@ jobs:
    - name: Get latest version of stable Rust
      run: rustup update stable
    - name: Install Protoc
-      uses: arduino/setup-protoc@v1
+      uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612
      with:
        repo-token: ${{ secrets.GITHUB_TOKEN }}
    - name: Run beacon_chain tests for all known forks
@ -109,7 +111,7 @@ jobs:
    - name: Get latest version of stable Rust
      run: rustup update stable
    - name: Install Protoc
-      uses: arduino/setup-protoc@v1
+      uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612
      with:
        repo-token: ${{ secrets.GITHUB_TOKEN }}
    - name: Run operation_pool tests for all known forks
@ -133,7 +135,7 @@ jobs:
    - name: Get latest version of stable Rust
      run: rustup update stable
    - name: Install Protoc
-      uses: arduino/setup-protoc@v1
+      uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612
      with:
        repo-token: ${{ secrets.GITHUB_TOKEN }}
    - name: Install ganache
@ -149,7 +151,7 @@ jobs:
    - name: Get latest version of stable Rust
      run: rustup update stable
    - name: Install Protoc
-      uses: arduino/setup-protoc@v1
+      uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612
      with:
        repo-token: ${{ secrets.GITHUB_TOKEN }}
    - name: Run state_transition_vectors in release.
@ -163,7 +165,7 @@ jobs:
    - name: Get latest version of stable Rust
      run: rustup update stable
    - name: Install Protoc
-      uses: arduino/setup-protoc@v1
+      uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612
      with:
        repo-token: ${{ secrets.GITHUB_TOKEN }}
    - name: Run consensus-spec-tests with blst, milagro and fake_crypto
@ -189,7 +191,7 @@ jobs:
    - name: Get latest version of stable Rust
      run: rustup update stable
    - name: Install Protoc
-      uses: arduino/setup-protoc@v1
+      uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612
      with:
        repo-token: ${{ secrets.GITHUB_TOKEN }}
    - name: Install ganache
@ -205,7 +207,7 @@ jobs:
    - name: Get latest version of stable Rust
      run: rustup update stable
    - name: Install Protoc
-      uses: arduino/setup-protoc@v1
+      uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612
      with:
        repo-token: ${{ secrets.GITHUB_TOKEN }}
    - name: Install ganache
@ -221,7 +223,7 @@ jobs:
    - name: Get latest version of stable Rust
      run: rustup update stable
    - name: Install Protoc
-      uses: arduino/setup-protoc@v1
+      uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612
      with:
        repo-token: ${{ secrets.GITHUB_TOKEN }}
    - name: Install ganache
@ -237,7 +239,7 @@ jobs:
    - name: Get latest version of stable Rust
      run: rustup update stable
    - name: Install Protoc
-      uses: arduino/setup-protoc@v1
+      uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612
      with:
        repo-token: ${{ secrets.GITHUB_TOKEN }}
    - name: Install ganache
@ -253,7 +255,7 @@ jobs:
    - name: Get latest version of stable Rust
      run: rustup update stable
    - name: Install Protoc
-      uses: arduino/setup-protoc@v1
+      uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612
      with:
        repo-token: ${{ secrets.GITHUB_TOKEN }}
    - name: Install ganache
@ -285,7 +287,7 @@ jobs:
    - name: Get latest version of stable Rust
      run: rustup update stable
    - name: Install Protoc
-      uses: arduino/setup-protoc@v1
+      uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612
      with:
        repo-token: ${{ secrets.GITHUB_TOKEN }}
    - name: Run exec engine integration tests in release
@ -299,7 +301,7 @@ jobs:
    - name: Get latest version of stable Rust
      run: rustup update stable
    - name: Install Protoc
-      uses: arduino/setup-protoc@v1
+      uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612
      with:
        repo-token: ${{ secrets.GITHUB_TOKEN }}
    - name: Typecheck benchmark code without running it
@ -323,7 +325,7 @@ jobs:
    - name: Get latest version of stable Rust
      run: rustup update stable
    - name: Install Protoc
-      uses: arduino/setup-protoc@v1
+      uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612
      with:
        repo-token: ${{ secrets.GITHUB_TOKEN }}
    - name: Lint code for quality and style with Clippy
@ -346,7 +348,7 @@ jobs:
        cargo build --release --bin cargo-clippy --bin clippy-driver
        cargo build --release --bin cargo-clippy --bin clippy-driver -Zunstable-options --out-dir $(rustc --print=sysroot)/bin
    - name: Install Protoc
-      uses: arduino/setup-protoc@v1
+      uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612
      with:
        repo-token: ${{ secrets.GITHUB_TOKEN }}
    - name: Run Clippy with the disallowed-from-async lint
@ -360,7 +362,7 @@ jobs:
    - name: Install Rust @ MSRV (${{ needs.extract-msrv.outputs.MSRV }})
      run: rustup override set ${{ needs.extract-msrv.outputs.MSRV }}
    - name: Install Protoc
-      uses: arduino/setup-protoc@v1
+      uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612
      with:
        repo-token: ${{ secrets.GITHUB_TOKEN }}
    - name: Run cargo check
@ -404,7 +406,7 @@ jobs:
    # NOTE: cargo-udeps version is pinned until this issue is resolved:
    # https://github.com/est31/cargo-udeps/issues/135
    - name: Install Protoc
-      uses: arduino/setup-protoc@v1
+      uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612
      with:
        repo-token: ${{ secrets.GITHUB_TOKEN }}
    - name: Install cargo-udeps
--- a/testing/ef_tests/Makefile
+++ b/testing/ef_tests/Makefile
@ -13,6 +13,8 @@ BLS_TARBALL = $(patsubst %,%-$(BLS_TEST_TAG).tar.gz,$(BLS_TEST))
 BLS_OUTPUT_DIR := $(OUTPUT_DIR)/$(BLS_TEST_REPO_NAME)
 BLS_BASE_URL := https://github.com/ethereum/$(BLS_TEST_REPO_NAME)/releases/download/$(BLS_TEST_TAG)

+WGET := $(if $(LIGHTHOUSE_GITHUB_TOKEN),wget --header="Authorization: $(LIGHTHOUSE_GITHUB_TOKEN)",wget)
+
 all:
 	make $(OUTPUT_DIR)
 	make $(BLS_OUTPUT_DIR)
@ -25,11 +27,11 @@ $(OUTPUT_DIR): $(TARBALLS)

 $(BLS_OUTPUT_DIR):
 	mkdir $(BLS_OUTPUT_DIR)
-	wget $(BLS_BASE_URL)/$(BLS_TEST).tar.gz -O $(BLS_TARBALL)
+	$(WGET) $(BLS_BASE_URL)/$(BLS_TEST).tar.gz -O $(BLS_TARBALL)
 	tar -xzf $(BLS_TARBALL) -C $(BLS_OUTPUT_DIR)

 %-$(TESTS_TAG).tar.gz:
-	wget $(BASE_URL)/$*.tar.gz -O $@
+	$(WGET) $(BASE_URL)/$*.tar.gz -O $@

 clean-test-files:
 	rm -rf $(OUTPUT_DIR) $(BLS_OUTPUT_DIR)
--- a/testing/web3signer_tests/build.rs
+++ b/testing/web3signer_tests/build.rs
@ -1,7 +1,10 @@
 //! This build script downloads the latest Web3Signer release and places it in the `OUT_DIR` so it
 //! can be used for integration testing.

-use reqwest::Client;
+use reqwest::{
+    header::{self, HeaderValue},
+    Client,
+};
 use serde_json::Value;
 use std::env;
 use std::fs;
@ -15,10 +18,15 @@ const FIXED_VERSION_STRING: Option<&str> = None;
 #[tokio::main]
 async fn main() {
    let out_dir = env::var("OUT_DIR").unwrap();
-    download_binary(out_dir.into()).await;
+
+    // Read a Github API token from the environment. This is intended to prevent rate-limits on CI.
+    // We use a name that is unlikely to accidentally collide with anything the user has configured.
+    let github_token = env::var("LIGHTHOUSE_GITHUB_TOKEN");
+
+    download_binary(out_dir.into(), github_token.as_deref().unwrap_or("")).await;
 }

-pub async fn download_binary(dest_dir: PathBuf) {
+pub async fn download_binary(dest_dir: PathBuf, github_token: &str) {
    let version_file = dest_dir.join("version");

    let client = Client::builder()
@ -33,8 +41,11 @@ pub async fn download_binary(dest_dir: PathBuf) {
        env_version
    } else {
        // Get the latest release of the web3 signer repo.
+        let mut token_header_value = HeaderValue::from_str(github_token).unwrap();
+        token_header_value.set_sensitive(true);
        let latest_response: Value = client
            .get("https://api.github.com/repos/ConsenSys/web3signer/releases/latest")
+            .header(header::AUTHORIZATION, token_header_value)
            .send()
            .await
            .unwrap()