From cd6aa9ad77fc5ffe22c923bfc32eecf38858347c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 28 Aug 2019 18:14:32 +0200 Subject: [PATCH] Import proofs for paramfetch --- .gitmodules | 3 ++ Makefile | 21 +++++++++ extern/proofs/Makefile | 14 ++++++ extern/proofs/bin/.keep | 0 extern/proofs/install-proofs | 31 +++++++++++++ extern/proofs/install-shared.sh | 82 +++++++++++++++++++++++++++++++++ extern/proofs/misc/.keep | 0 extern/proofs/rust-fil-proofs | 1 + 8 files changed, 152 insertions(+) create mode 100644 extern/proofs/Makefile create mode 100644 extern/proofs/bin/.keep create mode 100644 extern/proofs/install-proofs create mode 100644 extern/proofs/install-shared.sh create mode 100644 extern/proofs/misc/.keep create mode 160000 extern/proofs/rust-fil-proofs diff --git a/.gitmodules b/.gitmodules index d8e0ccfa6..a5162bd71 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,3 +4,6 @@ [submodule "extern/go-sectorbuilder"] path = extern/go-sectorbuilder url = https://github.com/filecoin-project/go-sectorbuilder +[submodule "extern/proofs/rust-fil-proofs"] + path = extern/proofs/rust-fil-proofs + url = git@github.com:filecoin-project/rust-fil-proofs.git diff --git a/Makefile b/Makefile index f26794f9b..5a087c129 100644 --- a/Makefile +++ b/Makefile @@ -7,6 +7,8 @@ MODULES:= CLEAN:= +## BLS + BLS_PATH:=extern/go-bls-sigs/ BLS_DEPS:=libbls_signatures.a libbls_signatures.pc libbls_signatures.h BLS_DEPS:=$(addprefix $(BLS_PATH),$(BLS_DEPS)) @@ -21,6 +23,8 @@ MODULES+=$(BLS_PATH) BUILD_DEPS+=build/.bls-install CLEAN+=build/.bls-install +## SECTOR BUILDER + SECTOR_BUILDER_PATH:=extern/go-sectorbuilder/ SECTOR_BUILDER_DEPS:=libsector_builder_ffi.a sector_builder_ffi.pc sector_builder_ffi.h SECTOR_BUILDER_DEPS:=$(addprefix $(SECTOR_BUILDER_PATH),$(SECTOR_BUILDER_DEPS)) @@ -35,6 +39,23 @@ MODULES+=$(SECTOR_BUILDER_PATH) BUILD_DEPS+=build/.sector-builder-install CLEAN+=build/.sector-builder-install +## PROOFS + +PROOFS_PATH:=extern/proofs/ +PROOFS_DEPS:=bin/paramcache bin/paramfetch misc/parameters.json +PROOFS_DEPS:=$(addprefix $(SECTOR_BUILDER_PATH),$(SECTOR_BUILDER_DEPS)) + +$(PROOFS_DEPS): build/.proofs-install ; + +build/.proofs-install: $(PROOFS_PATH) + $(MAKE) -C $(PROOFS_PATH) $(PROOFS_DEPS:$(PROOFS_PATH)%=%) + @touch $@ + +MODULES+=$(PROOFS_PATH) +BUILD_DEPS+=build/.proofs-install +CLEAN+=build/.proofs-install + +# end git modules $(MODULES): build/.update-modules ; diff --git a/extern/proofs/Makefile b/extern/proofs/Makefile new file mode 100644 index 000000000..33bc829f7 --- /dev/null +++ b/extern/proofs/Makefile @@ -0,0 +1,14 @@ +DEPS:=bin/paramcache bin/paramfetch misc/parameters.json + +all: $(DEPS) +.PHONY: all + +$(DEPS): .install-fil-proofs ; + +.install-fil-proofs: bls-signatures + ./install-proofs + @touch $@ + +clean: + rm -rf $(DEPS) .install-bls +.PHONY: clean diff --git a/extern/proofs/bin/.keep b/extern/proofs/bin/.keep new file mode 100644 index 000000000..e69de29bb diff --git a/extern/proofs/install-proofs b/extern/proofs/install-proofs new file mode 100644 index 000000000..f1de838ad --- /dev/null +++ b/extern/proofs/install-proofs @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +set -Eeo pipefail + +# shellcheck source=install-shared.bash +source "$(dirname "${BASH_SOURCE[0]}")/install-shared.bash" + +subm_dir="rust-fil-proofs" + +git submodule update --init --recursive ${subm_dir} + +if download_release_tarball tarball_path "${subm_dir}"; then + tmp_dir=$(mktemp -d) + tar -C "$tmp_dir" -xzf "$tarball_path" + + mkdir -p bin + mkdir -p misc + + cp -R "${tmp_dir}/bin/*" bin + cp -R "${tmp_dir}/misc/*" misc +else + echo "failed to find or obtain precompiled assets for ${subm_dir}, falling back to local build" + build_from_source "${subm_dir}" + + mkdir -p bin + mkdir -p misc + + find "${subm_dir}" -type f -name parameters.json -exec mv -- "{}" ./misc/ \; + find "${subm_dir}" -type f -name paramcache -exec cp -- "{}" ./bin/ \; + find "${subm_dir}" -type f -name paramfetch -exec cp -- "{}" ./bin/ \; +fi \ No newline at end of file diff --git a/extern/proofs/install-shared.sh b/extern/proofs/install-shared.sh new file mode 100644 index 000000000..3c026bc81 --- /dev/null +++ b/extern/proofs/install-shared.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env bash + +download_release_tarball() { + __resultvar=$1 + __submodule_path=$2 + __repo_name=$(echo "$__submodule_path" | cut -d '/' -f 2) + __release_name="${__repo_name}-$(uname)" + __release_sha1=$(git rev-parse @:"${__submodule_path}") + __release_tag="${__release_sha1:0:16}" + __release_tag_url="https://api.github.com/repos/filecoin-project/${__repo_name}/releases/tags/${__release_tag}" + + echo "acquiring release @ ${__release_tag}" + + __release_response=$(curl \ + --retry 3 \ + --location "$__release_tag_url") + + __release_url=$(echo "$__release_response" | jq -r ".assets[] | select(.name | contains(\"${__release_name}\")) | .url") + + if [[ -z "$__release_url" ]]; then + (>&2 echo "failed to download release (tag URL: ${__release_tag_url}, response: ${__release_response})") + return 1 + fi + + __tar_path="/tmp/${__release_name}_$(basename "${__release_url}").tar.gz" + + if [[ ! -f "${__tar_path}" ]]; then + __asset_url=$(curl \ + --head \ + --retry 3 \ + --header "Accept:application/octet-stream" \ + --location \ + --output /dev/null \ + -w "%{url_effective}" \ + "$__release_url") + + curl --retry 3 --output "${__tar_path}" "$__asset_url" + if [[ $? -ne "0" ]]; then + (>&2 echo "failed to download release asset (tag URL: ${__release_tag_url}, asset URL: ${__asset_url})") + return 1 + fi + fi + + eval $__resultvar="'$__tar_path'" +} + +build_from_source() { + __submodule_path=$1 + __submodule_sha1=$(git rev-parse @:"${__submodule_path}") + __submodule_sha1_truncated="${__submodule_sha1:0:16}" + + echo "building from source @ ${__submodule_sha1_truncated}" + + if ! [ -x "$(command -v cargo)" ]; then + (>&2 echo 'Error: cargo is not installed.') + (>&2 echo 'Install Rust toolchain to resolve this problem.') + exit 1 + fi + + if ! [ -x "$(command -v rustup)" ]; then + (>&2 echo 'Error: rustup is not installed.') + (>&2 echo 'Install Rust toolchain installer to resolve this problem.') + exit 1 + fi + + if ! pushd "$__submodule_path"; then + (>&2 echo 'Error: could not cd into submodule') + exit 1 + fi + + + cargo --version + cargo update + + if [[ -f "./build-release.sh" ]]; then + ./build-release.sh "$(cat rust-toolchain)" + else + cargo build --release --all + fi + + popd || exit 1 +} diff --git a/extern/proofs/misc/.keep b/extern/proofs/misc/.keep new file mode 100644 index 000000000..e69de29bb diff --git a/extern/proofs/rust-fil-proofs b/extern/proofs/rust-fil-proofs new file mode 160000 index 000000000..8ef99ff86 --- /dev/null +++ b/extern/proofs/rust-fil-proofs @@ -0,0 +1 @@ +Subproject commit 8ef99ff867bd0c60acbaae6b4fe4214066936c78