diff --git a/.circleci/config.yml b/.circleci/config.yml index c3e1c588b..7ae05a7aa 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -179,47 +179,199 @@ commands: paths: - << parameters.install_path >> -defaults: + # -------------------------------------------------------------------------- + # Build Commands + + setup_prerelease_commit_hash: + steps: + - run: + name: Store commit hash and prerelease + command: | + if [[ $CIRCLE_BRANCH == release || -n $CIRCLE_TAG ]]; then + echo -n > prerelease.txt; + else + date -u +"nightly.%Y.%-m.%-d" > prerelease.txt; + fi + echo -n "$CIRCLE_SHA1" > commit_hash.txt + + run_build: + steps: + - run: + name: Build + command: scripts/ci/build.sh + + run_build_ossfuzz: + steps: + - run: + name: Build_ossfuzz + command: scripts/ci/build_ossfuzz.sh + + run_proofs: + steps: + - run: + name: Correctness proofs for optimization rules + command: scripts/run_proofs.sh + + run_soltest: + steps: + - run: + name: soltest + no_output_timeout: 30m + command: .circleci/soltest.sh + + run_soltest_all: + steps: + - run: + name: soltest_all + no_output_timeout: 30m + command: .circleci/soltest_all.sh + + run_cmdline_tests: + steps: + - run: + name: command line tests + no_output_timeout: 30m + command: .circleci/parallel_cli_tests.py + + run_docs_pragma_min_version: + steps: + - run: + name: docs pragma version check + command: scripts/docs_version_pragma_check.sh # -------------------------------------------------------------------------- - # Build Templates + # Artifact Commands - - setup_prerelease_commit_hash: &setup_prerelease_commit_hash - name: Store commit hash and prerelease - command: | - if [ "$CIRCLE_BRANCH" = release -o -n "$CIRCLE_TAG" ]; then echo -n > prerelease.txt; else date -u +"nightly.%Y.%-m.%-d" > prerelease.txt; fi - echo -n "$CIRCLE_SHA1" > commit_hash.txt + store_artifacts_solc: + description: Store compiled solc executable as artifact + steps: + - store_artifacts: + path: build/solc/solc + destination: solc - - run_build: &run_build - name: Build - command: scripts/ci/build.sh + store_artifacts_yul_phaser: + steps: + - store_artifacts: + path: build/tools/yul-phaser + destination: yul-phaser - - run_build_ossfuzz: &run_build_ossfuzz - name: Build_ossfuzz - command: scripts/ci/build_ossfuzz.sh + persist_executables_to_workspace: + description: Persist compiled target executables to workspace + steps: + - persist_to_workspace: + root: build + paths: + - solc/solc + - test/soltest + - test/tools/solfuzzer - - run_proofs: &run_proofs - name: Correctness proofs for optimization rules - command: scripts/run_proofs.sh + persist_executables_to_workspace_osx: + description: Persist compiled target executables to workspace on macOS + steps: + - persist_to_workspace: + root: . + paths: + - build/solc/solc + - build/test/soltest + - build/test/tools/solfuzzer - - run_soltest: &run_soltest - name: soltest - no_output_timeout: 30m - command: ./.circleci/soltest.sh + persist_ossfuzz_executables_to_workspace: + description: Persist compiled OSSFUZZ executables to workspace + steps: + - persist_to_workspace: + root: build + paths: + - test/tools/ossfuzz/abiv2_proto_ossfuzz + - test/tools/ossfuzz/abiv2_isabelle_ossfuzz + - test/tools/ossfuzz/const_opt_ossfuzz + - test/tools/ossfuzz/solc_mutator_ossfuzz + - test/tools/ossfuzz/solc_ossfuzz + - test/tools/ossfuzz/stack_reuse_codegen_ossfuzz + - test/tools/ossfuzz/strictasm_assembly_ossfuzz + - test/tools/ossfuzz/strictasm_diff_ossfuzz + - test/tools/ossfuzz/strictasm_opt_ossfuzz + - test/tools/ossfuzz/yul_proto_diff_ossfuzz + - test/tools/ossfuzz/yul_proto_diff_custom_mutate_ossfuzz + - test/tools/ossfuzz/yul_proto_ossfuzz + - test/tools/ossfuzz/sol_proto_ossfuzz - - run_soltest_all: &run_soltest_all - name: soltest_all - no_output_timeout: 30m - command: ./.circleci/soltest_all.sh + store_artifacts_test_results: + description: Store test output dir as artifact + steps: + - store_artifacts: + path: test_results/ + destination: test_results/ - - run_cmdline_tests: &run_cmdline_tests - name: command line tests - no_output_timeout: 30m - command: .circleci/parallel_cli_tests.py + # -------------------------------------------------------------------------- + # Complex Build Commands - - run_docs_pragma_min_version: &run_docs_pragma_min_version - name: docs pragma version check - command: ./scripts/docs_version_pragma_check.sh + soltest: + steps: + - checkout + - attach_workspace: + at: build + # NOTE: Different build jobs produce different soltest executables (release/debug, + # clang/gcc, windows/linux/macos, etc.). The executable used by these steps comes from the + # attached workspace and we only see the items added to the workspace by jobs we depend on. + - run_soltest + - store_test_results: + path: test_results/ + - store_artifacts_test_results + - matrix_notify_failure_unless_pr + + test_lsp: + steps: + - checkout + - attach_workspace: + at: build + - run: + name: Install dependencies + command: pip install --user deepdiff colorama + - run: + name: Executing solc LSP test suite + command: test/lsp.py build/solc/solc --non-interactive + - matrix_notify_failure_unless_pr + + build: + steps: + - checkout + - run_build + - store_artifacts_solc + - store_artifacts_yul_phaser + - persist_executables_to_workspace + - matrix_notify_failure_unless_pr + + soltest_all: + steps: + - checkout + - attach_workspace: + at: build + - run_soltest_all + - store_test_results: + path: test_results/ + - store_artifacts_test_results + - matrix_notify_failure_unless_pr + + cmdline_tests: + steps: + - checkout + - attach_workspace: + at: build + - run_cmdline_tests + - store_test_results: + path: test_results/ + - store_artifacts_test_results + - matrix_notify_failure_unless_pr + + install_dependencies_osx: + steps: + # FIXME: We used to cache dependencies on macOS but now it takes longer than just installing + # them each time. See https://github.com/ethereum/solidity/issues/12925. + - run: + name: Install build dependencies + command: .circleci/osx_install_dependencies.sh + +defaults: # -------------------------------------------------------------------------- # Matrix templates @@ -241,122 +393,6 @@ defaults: - via-ir-no-optimize # -------------------------------------------------------------------------- - # Artifacts Templates - - # compiled solc executable target - - artifacts_solc: &artifacts_solc - path: build/solc/solc - destination: solc - - # windows artifacts - - artifact_solc_windows: &artifact_solc_windows - path: upload/ - - - artifact_yul_phaser: &artifact_yul_phaser - path: build/tools/yul-phaser - destination: yul-phaser - - # compiled executable targets - - artifacts_executables: &artifacts_executables - root: build - paths: - - solc/solc - - test/soltest - - test/tools/solfuzzer - - # compiled OSSFUZZ targets - - artifacts_executables_ossfuzz: &artifacts_executables_ossfuzz - root: build - paths: - - test/tools/ossfuzz/abiv2_proto_ossfuzz - - test/tools/ossfuzz/abiv2_isabelle_ossfuzz - - test/tools/ossfuzz/const_opt_ossfuzz - - test/tools/ossfuzz/solc_mutator_ossfuzz - - test/tools/ossfuzz/solc_ossfuzz - - test/tools/ossfuzz/stack_reuse_codegen_ossfuzz - - test/tools/ossfuzz/strictasm_assembly_ossfuzz - - test/tools/ossfuzz/strictasm_diff_ossfuzz - - test/tools/ossfuzz/strictasm_opt_ossfuzz - - test/tools/ossfuzz/yul_proto_diff_ossfuzz - - test/tools/ossfuzz/yul_proto_diff_custom_mutate_ossfuzz - - test/tools/ossfuzz/yul_proto_ossfuzz - - test/tools/ossfuzz/sol_proto_ossfuzz - - # test result output directory - - artifacts_test_results: &artifacts_test_results - path: test_results/ - destination: test_results/ - - # -------------------------------------------------------------------------- - # Step Templates - - # store_test_results helper - - store_test_results: &store_test_results - path: test_results/ - - - steps_soltest: &steps_soltest - steps: - - checkout - - attach_workspace: - at: build - # NOTE: Different build jobs produce different soltest executables (release/debug, - # clang/gcc, windows/linux/macos, etc.). The executable used by these steps comes from the - # attached workspace and we only see the items added to the workspace by jobs we depend on. - - run: *run_soltest - - store_test_results: *store_test_results - - store_artifacts: *artifacts_test_results - - matrix_notify_failure_unless_pr - - - steps_test_lsp: &steps_test_lsp - steps: - - checkout - - attach_workspace: - at: build - - run: - name: Install dependencies - command: pip install --user deepdiff colorama - - run: - name: Executing solc LSP test suite - command: ./test/lsp.py ./build/solc/solc --non-interactive - - matrix_notify_failure_unless_pr - - - steps_build: &steps_build - steps: - - checkout - - run: *run_build - - store_artifacts: *artifacts_solc - - store_artifacts: *artifact_yul_phaser - - persist_to_workspace: *artifacts_executables - - matrix_notify_failure_unless_pr - - - steps_soltest_all: &steps_soltest_all - steps: - - checkout - - attach_workspace: - at: build - - run: *run_soltest_all - - store_test_results: *store_test_results - - store_artifacts: *artifacts_test_results - - matrix_notify_failure_unless_pr - - - steps_cmdline_tests: &steps_cmdline_tests - steps: - - checkout - - attach_workspace: - at: build - - run: *run_cmdline_tests - - store_test_results: *store_test_results - - store_artifacts: *artifacts_test_results - - matrix_notify_failure_unless_pr - - - steps_install_dependencies_osx: &steps_install_dependencies_osx - steps: - # FIXME: We used to cache dependencies on macOS but now it takes longer than just installing - # them each time. See https://github.com/ethereum/solidity/issues/12925. - - run: - name: Install build dependencies - command: ./.circleci/osx_install_dependencies.sh - # -------------------------------------------------------------------------- # Base Image Templates @@ -767,7 +803,11 @@ jobs: pip install --user codespell - run: name: Check spelling - command: ~/.local/bin/codespell --skip "*.enc,.git,Dockerfile*,LICENSE,codespell_whitelist.txt,codespell_ignored_lines.txt" --ignore-words ./scripts/codespell_whitelist.txt --exclude-file ./scripts/codespell_ignored_lines.txt + command: | + ~/.local/bin/codespell \ + --skip "*.enc,.git,Dockerfile*,LICENSE,codespell_whitelist.txt,codespell_ignored_lines.txt" \ + --ignore-words scripts/codespell_whitelist.txt \ + --exclude-file scripts/codespell_ignored_lines.txt - matrix_notify_failure_unless_pr chk_docs_examples: @@ -781,7 +821,7 @@ jobs: command: sudo npm install -g solhint - run: name: Test Docs examples - command: ./test/docsCodeStyle.sh + command: test/docsCodeStyle.sh - matrix_notify_failure_unless_pr chk_coding_style: @@ -795,13 +835,13 @@ jobs: sudo apt install -y shellcheck - run: name: Check for C++ coding style - command: ./scripts/check_style.sh + command: scripts/check_style.sh - run: name: checking shell scripts - command: ./scripts/chk_shellscripts/chk_shellscripts.sh + command: scripts/chk_shellscripts/chk_shellscripts.sh - run: name: Check for broken symlinks - command: ./scripts/check_symlinks.sh + command: scripts/check_symlinks.sh - matrix_notify_failure_unless_pr chk_errorcodes: @@ -810,7 +850,7 @@ jobs: - checkout - run: name: Check for error codes - command: ./scripts/error_codes.py --check + command: scripts/error_codes.py --check - matrix_notify_failure_unless_pr chk_pylint: @@ -830,7 +870,7 @@ jobs: - run: pylint --version - run: name: Linting Python Scripts - command: ./scripts/pylint_all.py + command: scripts/pylint_all.py - matrix_notify_failure_unless_pr chk_antlr_grammar: @@ -844,7 +884,7 @@ jobs: sudo apt install -y openjdk-17-jdk - run: name: Run tests - command: ./scripts/test_antlr_grammar.sh + command: scripts/test_antlr_grammar.sh - matrix_notify_failure_unless_pr chk_buglist: @@ -859,7 +899,7 @@ jobs: npm install mktemp - run: name: Test buglist - command: ./test/buglistTests.js + command: test/buglistTests.js - matrix_notify_failure_unless_pr chk_proofs: @@ -868,14 +908,14 @@ jobs: - checkout - install_python3: packages: z3-solver - - run: *run_proofs + - run_proofs - matrix_notify_failure_unless_pr chk_docs_pragma_min_version: <<: *base_ubuntu2204_small steps: - checkout - - run: *run_docs_pragma_min_version + - run_docs_pragma_min_version - matrix_notify_failure_unless_pr t_ubu_pyscripts: @@ -901,7 +941,8 @@ jobs: # this runs 2x faster on xlarge but takes 4x more resources (compared to medium). # Enough other jobs depend on it that it's worth it though. <<: *base_ubuntu2204_xlarge - <<: *steps_build + steps: + - build # x64 ASAN build, for testing for memory related bugs b_ubu_asan: &b_ubu_asan @@ -911,14 +952,16 @@ jobs: <<: *base_ubuntu2204_env CMAKE_OPTIONS: -DSANITIZE=address CMAKE_BUILD_TYPE: Release - <<: *steps_build + steps: + - build b_ubu_clang: &b_ubu_clang <<: *base_ubuntu2204_clang_large environment: <<: *base_ubuntu2204_clang_large_env MAKEFLAGS: -j 10 - <<: *steps_build + steps: + - build b_ubu_san_clang: # This runs a bit faster on large and xlarge but on nightly efficiency matters more. @@ -929,7 +972,8 @@ jobs: environment: <<: *base_ubuntu2204_clang_env CMAKE_OPTIONS: << parameters.cmake_options >> - <<: *steps_build + steps: + - build b_ubu_force_release: &b_ubu_force_release <<: *b_ubu @@ -948,7 +992,7 @@ jobs: CMAKE_OPTIONS: -DCMAKE_BUILD_TYPE=Release -DUSE_Z3_DLOPEN=ON -DUSE_CVC4=OFF -DSOLC_STATIC_STDLIBS=ON steps: - checkout - - run: *run_build + - run_build - run: name: strip binary command: strip build/solc/solc @@ -971,8 +1015,8 @@ jobs: CMAKE_BUILD_TYPE: Debug steps: - checkout - - run: *run_build - - persist_to_workspace: *artifacts_executables + - run_build + - persist_executables_to_workspace - matrix_notify_failure_unless_pr t_ubu_codecov: @@ -991,11 +1035,11 @@ jobs: - run: name: "Code Coverage: Syntax Tests" command: codecov --flags syntax --gcov-root build - - run: *run_soltest + - run_soltest - run: name: "Coverage: All" command: codecov --flags all --gcov-root build - - store_artifacts: *artifacts_test_results + - store_artifacts_test_results - matrix_notify_failure_unless_pr # Builds in C++20 mode and uses debug build in order to speed up. @@ -1009,16 +1053,16 @@ jobs: MAKEFLAGS: -j 10 steps: - checkout - - run: *run_build + - run_build - matrix_notify_failure_unless_pr b_ubu_ossfuzz: &b_ubu_ossfuzz <<: *base_ubuntu_clang steps: - checkout - - run: *setup_prerelease_commit_hash - - run: *run_build_ossfuzz - - persist_to_workspace: *artifacts_executables_ossfuzz + - setup_prerelease_commit_hash + - run_build_ossfuzz + - persist_ossfuzz_executables_to_workspace - matrix_notify_failure_unless_pr t_ubu_ossfuzz: &t_ubu_ossfuzz @@ -1033,8 +1077,9 @@ jobs: git clone https://github.com/ethereum/solidity-fuzzing-corpus /tmp/solidity-fuzzing-corpus mkdir -p test_results scripts/regressions.py -o test_results - - store_test_results: *store_test_results - - store_artifacts: *artifacts_test_results + - store_test_results: + path: test_results/ + - store_artifacts_test_results b_archlinux: <<: *base_archlinux_large @@ -1048,9 +1093,9 @@ jobs: command: | pacman --noconfirm -Syu --noprogressbar --needed base-devel boost cmake cvc4 git openssh tar - checkout - - run: *run_build - - store_artifacts: *artifacts_solc - - persist_to_workspace: *artifacts_executables + - run_build + - store_artifacts_solc + - persist_executables_to_workspace - matrix_notify_failure_unless_pr b_osx: @@ -1060,18 +1105,11 @@ jobs: CMAKE_BUILD_TYPE: Release steps: - checkout - - when: - condition: true - <<: *steps_install_dependencies_osx - - run: *run_build - - store_artifacts: *artifacts_solc - - store_artifacts: *artifact_yul_phaser - - persist_to_workspace: - root: . - paths: - - build/solc/solc - - build/test/soltest - - build/test/tools/solfuzzer + - install_dependencies_osx + - run_build + - store_artifacts_solc + - store_artifacts_yul_phaser + - persist_executables_to_workspace_osx - matrix_notify_failure_unless_pr t_osx_soltest: &t_osx_soltest @@ -1082,14 +1120,13 @@ jobs: OPTIMIZE: 0 steps: - checkout - - when: - condition: true - <<: *steps_install_dependencies_osx + - install_dependencies_osx - attach_workspace: at: . - - run: *run_soltest - - store_test_results: *store_test_results - - store_artifacts: *artifacts_test_results + - run_soltest + - store_test_results: + path: test_results/ + - store_artifacts_test_results - matrix_notify_failure_unless_pr t_osx_cli: @@ -1097,13 +1134,11 @@ jobs: parallelism: 7 # Should match number of tests in .circleci/cli.sh steps: - checkout - - when: - condition: true - <<: *steps_install_dependencies_osx + - install_dependencies_osx - attach_workspace: at: . - - run: *run_cmdline_tests - - store_artifacts: *artifacts_test_results + - run_cmdline_tests + - store_artifacts_test_results - matrix_notify_failure_unless_pr b_ems: @@ -1134,10 +1169,10 @@ jobs: <<: *base_ubuntu2204_small steps: - checkout - - run: *setup_prerelease_commit_hash + - setup_prerelease_commit_hash - run: name: Build documentation - command: ./docs/docs.sh + command: docs/docs.sh - store_artifacts: path: docs/_build/html/ destination: docs-html @@ -1146,11 +1181,13 @@ jobs: t_ubu_soltest_all: &t_ubu_soltest_all <<: *base_ubuntu2204_large parallelism: 50 - <<: *steps_soltest_all + steps: + - soltest_all t_ubu_lsp: &t_ubu_lsp <<: *base_ubuntu2204_small - <<: *steps_test_lsp + steps: + - test_lsp t_archlinux_soltest: &t_archlinux_soltest <<: *base_archlinux @@ -1167,9 +1204,7 @@ jobs: name: Install runtime dependencies command: | pacman --noconfirm -Syu --noprogressbar --needed base-devel boost cmake z3 cvc4 git openssh tar - - when: - condition: true - <<: *steps_soltest + - soltest t_ubu_clang_soltest: &t_ubu_clang_soltest <<: *base_ubuntu2204_clang @@ -1181,7 +1216,8 @@ jobs: # The high parallelism in this job is causing the SMT tests to run out of memory, # so disabling for now. SOLTEST_FLAGS: --no-smt - <<: *steps_soltest + steps: + - soltest t_ubu_force_release_soltest_all: &t_ubu_force_release_soltest_all # NOTE: This definition is identical to t_ubu_soltest_all but in the workflow we make it depend on @@ -1191,7 +1227,8 @@ jobs: t_ubu_cli: &t_ubu_cli <<: *base_ubuntu2204_small parallelism: 7 # Should match number of tests in .circleci/cli.sh - <<: *steps_cmdline_tests + steps: + - cmdline_tests t_ubu_force_release_cli: &t_ubu_force_release_cli <<: *t_ubu_cli @@ -1215,7 +1252,8 @@ jobs: # Suppress CLN memory leak. # See: https://github.com/ethereum/solidity/issues/13891 for details. LSAN_OPTIONS: suppressions=/root/project/.circleci/cln-asan.supp:print_suppressions=0 - <<: *steps_cmdline_tests + steps: + - cmdline_tests t_ubu_asan_soltest: <<: *base_ubuntu2204 @@ -1229,7 +1267,8 @@ jobs: # Suppress CLN memory leak. # See: https://github.com/ethereum/solidity/issues/13891 for details. LSAN_OPTIONS: suppressions=/root/project/.circleci/cln-asan.supp - <<: *steps_soltest + steps: + - soltest t_ubu_asan_clang_soltest: <<: *base_ubuntu2204_clang @@ -1240,7 +1279,8 @@ jobs: OPTIMIZE: 0 SOLTEST_FLAGS: --no-smt ASAN_OPTIONS: check_initialization_order=true:detect_stack_use_after_return=true:strict_init_order=true:strict_string_checks=true:detect_invalid_pointer_pairs=2 - <<: *steps_soltest + steps: + - soltest t_ubu_ubsan_clang_soltest: <<: *base_ubuntu2204_clang @@ -1249,12 +1289,14 @@ jobs: <<: *base_ubuntu2204_clang_env EVM: << pipeline.parameters.evm-version >> SOLTEST_FLAGS: --no-smt - <<: *steps_soltest + steps: + - soltest t_ubu_ubsan_clang_cli: <<: *base_ubuntu2204_clang parallelism: 7 # Should match number of tests in .circleci/cli.sh - <<: *steps_cmdline_tests + steps: + - cmdline_tests t_ems_solcjs: # Unlike other t_ems jobs this one actually runs 2x faster on medium (compared to small). @@ -1478,7 +1520,8 @@ jobs: name: "Run solc.exe to make sure build was successful." command: .\build\solc\Release\solc.exe --version shell: powershell.exe - - store_artifacts: *artifact_solc_windows + - store_artifacts: + path: upload/ - persist_to_workspace: root: build paths: @@ -1507,10 +1550,11 @@ jobs: command: python -m pip install --user deepdiff colorama - run: name: Executing solc LSP test suite - command: python ./test/lsp.py .\build\solc\Release\solc.exe --non-interactive + command: python test/lsp.py build\solc\Release\solc.exe --non-interactive shell: powershell.exe - - store_test_results: *store_test_results - - store_artifacts: *artifacts_test_results + - store_test_results: + path: test_results/ + - store_artifacts_test_results - matrix_notify_failure_unless_pr # Note: b_bytecode_ubu_static is required because b_ubu_static and b_ubu