diff --git a/.github/actions/start-yugabytedb/action.yml b/.github/actions/start-yugabytedb/action.yml new file mode 100644 index 000000000..13c480c66 --- /dev/null +++ b/.github/actions/start-yugabytedb/action.yml @@ -0,0 +1,16 @@ +name: Start YugabyteDB +description: Install Yugabyte Database for Filecoin Lotus + +runs: + using: composite + steps: + - run: docker run --rm --name yugabyte -d -p 5433:5433 yugabytedb/yugabyte:2.18.0.0-b65 bin/yugabyted start --daemon=false + shell: bash + - run: | + while true; do + status=$(docker exec yugabyte bin/yugabyted status); + echo $status; + echo $status | grep Running && break; + sleep 1; + done + shell: bash diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 000000000..491a69d8e --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,321 @@ +name: Test + +on: + pull_request: + push: + branches: + - master + - release/* + workflow_dispatch: + +defaults: + run: + shell: bash + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + discover: + name: Discover Test Groups + runs-on: ubuntu-latest + outputs: + groups: ${{ steps.test.outputs.groups }} + steps: + - uses: actions/checkout@v4 + with: + submodules: 'recursive' + - id: test + env: + # Unit test groups other than unit-rest + utests: | + [ + {"name": "unit-cli", "packages": ["./cli/...", "./cmd/...", "./api/..."]}, + {"name": "unit-storage", "packages": ["./storage/...", "./extern/..."]}, + {"name": "unit-node", "packages": ["./node/..."]} + ] + # Other tests that require special configuration + otests: | + [ + { + "name": "multicore-sdr", + "packages": ["./storage/sealer/ffiwrapper"], + "go_test_flags": "-run=TestMulticoreSDR", + "test_rustproofs_logs": "1" + }, { + "name": "conformance", + "packages": ["./conformance"], + "go_test_flags": "-run=TestConformance", + "skip_conformance": "0" + } + ] + # Mapping from test group names to custom runner labels + # The jobs default to running on the default hosted runners (4 CPU, 16 RAM). + # We use self-hosted xlarge (4 CPU, 8 RAM; and large - 2 CPU, 4 RAM) runners + # to extend the available runner capacity (60 default hosted runners). + # We use self-hosted 4xlarge (16 CPU, 32 RAM; and 2xlarge - 8 CPU, 16 RAM) self-hosted + # to support resource intensive jobs. + # In CircleCI, the jobs defaulted to running on medium+ runners (3 CPU, 6 RAM). + # The following jobs were scheduled to run on 2xlarge runners (16 CPU, 32 RAM): + # - itest-deals_concurrent (✅) + # - itest-sector_pledge (✅) + # - itest-wdpost_worker_config (❌) + # - itest-worker (✅) + # - unit-cli (❌) + # - unit-rest (❌) + runners: | + { + "itest-deals_concurrent": ["self-hosted", "linux", "x64", "4xlarge"], + "itest-sector_pledge": ["self-hosted", "linux", "x64", "4xlarge"], + "itest-worker": ["self-hosted", "linux", "x64", "4xlarge"], + + "itest-gateway": ["self-hosted", "linux", "x64", "2xlarge"], + "itest-sector_import_full": ["self-hosted", "linux", "x64", "2xlarge"], + "itest-sector_import_simple": ["self-hosted", "linux", "x64", "2xlarge"], + "itest-wdpost": ["self-hosted", "linux", "x64", "2xlarge"], + "unit-storage": ["self-hosted", "linux", "x64", "2xlarge"], + + "itest-batch_deal": ["self-hosted", "linux", "x64", "xlarge"], + "itest-cli": ["self-hosted", "linux", "x64", "xlarge"], + "itest-deals_512mb": ["self-hosted", "linux", "x64", "xlarge"], + "itest-deals_anycid": ["self-hosted", "linux", "x64", "xlarge"], + "itest-deals_invalid_utf8_label": ["self-hosted", "linux", "x64", "xlarge"], + "itest-deals_max_staging_deals": ["self-hosted", "linux", "x64", "xlarge"], + "itest-deals_partial_retrieval": ["self-hosted", "linux", "x64", "xlarge"], + "itest-deals_publish": ["self-hosted", "linux", "x64", "xlarge"], + "itest-deals_remote_retrieval": ["self-hosted", "linux", "x64", "xlarge"], + "itest-decode_params": ["self-hosted", "linux", "x64", "xlarge"], + "itest-dup_mpool_messages": ["self-hosted", "linux", "x64", "xlarge"], + "itest-eth_account_abstraction": ["self-hosted", "linux", "x64", "xlarge"], + "itest-eth_api": ["self-hosted", "linux", "x64", "xlarge"], + "itest-eth_balance": ["self-hosted", "linux", "x64", "xlarge"], + "itest-eth_bytecode": ["self-hosted", "linux", "x64", "xlarge"], + "itest-eth_config": ["self-hosted", "linux", "x64", "xlarge"], + "itest-eth_conformance": ["self-hosted", "linux", "x64", "xlarge"], + "itest-eth_deploy": ["self-hosted", "linux", "x64", "xlarge"], + "itest-eth_fee_history": ["self-hosted", "linux", "x64", "xlarge"], + "itest-eth_transactions": ["self-hosted", "linux", "x64", "xlarge"], + "itest-fevm_address": ["self-hosted", "linux", "x64", "xlarge"], + "itest-fevm_events": ["self-hosted", "linux", "x64", "xlarge"], + "itest-gas_estimation": ["self-hosted", "linux", "x64", "xlarge"], + "itest-get_messages_in_ts": ["self-hosted", "linux", "x64", "xlarge"], + "itest-lite_migration": ["self-hosted", "linux", "x64", "xlarge"], + "itest-lookup_robust_address": ["self-hosted", "linux", "x64", "xlarge"], + "itest-mempool": ["self-hosted", "linux", "x64", "xlarge"], + "itest-mpool_msg_uuid": ["self-hosted", "linux", "x64", "xlarge"], + "itest-mpool_push_with_uuid": ["self-hosted", "linux", "x64", "xlarge"], + "itest-msgindex": ["self-hosted", "linux", "x64", "xlarge"], + "itest-multisig": ["self-hosted", "linux", "x64", "xlarge"], + "itest-net": ["self-hosted", "linux", "x64", "xlarge"], + "itest-nonce": ["self-hosted", "linux", "x64", "xlarge"], + "itest-path_detach_redeclare": ["self-hosted", "linux", "x64", "xlarge"], + "itest-pending_deal_allocation": ["self-hosted", "linux", "x64", "xlarge"], + "itest-remove_verifreg_datacap": ["self-hosted", "linux", "x64", "xlarge"], + "itest-sector_miner_collateral": ["self-hosted", "linux", "x64", "xlarge"], + "itest-sector_numassign": ["self-hosted", "linux", "x64", "xlarge"], + "itest-self_sent_txn": ["self-hosted", "linux", "x64", "xlarge"], + "itest-verifreg": ["self-hosted", "linux", "x64", "xlarge"], + "multicore-sdr": ["self-hosted", "linux", "x64", "xlarge"], + "unit-node": ["self-hosted", "linux", "x64", "xlarge"] + } + # A list of test groups that require YugabyteDB to be running + # In CircleCI, all jobs had yugabytedb running as a sidecar. + yugabytedb: | + ["itest-harmonydb", "itest-harmonytask"] + # A list of test groups that require Proof Parameters to be fetched + # In CircleCI, only the following jobs had get-params set: + # - unit-cli (✅) + # - unit-storage (✅) + # - itest-sector_pledge (✅) + # - itest-wdpost (✅) + parameters: | + [ + "conformance", + "itest-api", + "itest-deals_offline", + "itest-deals_padding", + "itest-deals_partial_retrieval_dm-level", + "itest-deals_pricing", + "itest-deals", + "itest-direct_data_onboard_verified", + "itest-direct_data_onboard", + "itest-net", + "itest-path_detach_redeclare", + "itest-path_type_filters", + "itest-sealing_resources", + "itest-sector_finalize_early", + "itest-sector_import_full", + "itest-sector_import_simple", + "itest-sector_pledge", + "itest-sector_unseal", + "itest-wdpost_no_miner_storage", + "itest-wdpost_worker_config", + "itest-wdpost", + "itest-worker_upgrade", + "itest-worker", + "multicore-sdr", + "unit-cli", + "unit-storage" + ] + run: | + # Create a list of integration test groups + itests="$( + find ./itests -name "*_test.go" | \ + jq -R '{ + "name": "itest-\(. | split("/") | .[2] | sub("_test.go$";""))", + "packages": [.] + }' | jq -s + )" + + # Create a list of packages that are covered by the integration and unit tests + packages="$(jq -n --argjson utests "$utests" '$utests | map(.packages) | flatten | . + ["./itests/..."]')" + + # Create a new group for the unit tests that are not yet covered + rest="$( + find . -name "*_test.go" | cut -d/ -f2 | sort | uniq | \ + jq -R '"./\(.)/..."' | \ + jq -s --argjson p "$packages" '{"name": "unit-rest", "packages": (. - $p)}' + )" + + # Combine the groups for integration tests, unit tests, the new unit-rest group, and the other tests + groups="$(jq -n --argjson i "$itests" --argjson u "$utests" --argjson r "$rest" --argjson o "$otests" '$i + $u + [$r] + $o')" + + # Apply custom runner labels to the groups + groups="$(jq -n --argjson g "$groups" --argjson r "$runners" '$g | map(. + {"runner": (.name as $n | $r | .[$n]) })')" + + # Apply the needs_yugabytedb flag to the groups + groups="$(jq -n --argjson g "$groups" --argjson y "$yugabytedb" '$g | map(. + {"needs_yugabytedb": ([.name] | inside($y)) })')" + + # Apply the needs_parameters flag to the groups + groups="$(jq -n --argjson g "$groups" --argjson p "$parameters" '$g | map(. + {"needs_parameters": ([.name] | inside($p)) })')" + + # Output the groups + echo "groups=$groups" + echo "groups=$(jq -nc --argjson g "$groups" '$g')" >> $GITHUB_OUTPUT + cache: + name: Cache Dependencies + runs-on: ubuntu-latest + outputs: + fetch_params_key: ${{ steps.fetch_params.outputs.key }} + fetch_params_path: ${{ steps.fetch_params.outputs.path }} + make_deps_key: ${{ steps.make_deps.outputs.key }} + make_deps_path: ${{ steps.make_deps.outputs.path }} + steps: + - uses: actions/checkout@v4 + with: + submodules: 'recursive' + - id: fetch_params + env: + CACHE_KEY: fetch-params-${{ hashFiles('./extern/filecoin-ffi/parameters.json') }} + CACHE_PATH: | + /var/tmp/filecoin-proof-parameters/ + run: | + echo -e "key=$CACHE_KEY" | tee -a $GITHUB_OUTPUT + echo -e "path<