cerc-io/lotus
16
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

Merge pull request #184 from filecoin-project/feat/cache-params

Browse Source 
Cache params
This commit is contained in:
Jakub Sztandera 2019-09-10 18:30:22 +02:00 committed by GitHub
commit adb0910162
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 37 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
.circleci/config.yml
View File

@ -13,6 +13,24 @@ commands:
- checkout - checkout
- run: git submodule sync - run: git submodule sync
- run: git submodule update --init - run: git submodule update --init
download-params:
steps:
- restore_cache:
name: Restore parameters cache
keys:
- 'v2-lotus-params-{{ checksum "build/proof-params/parameters.json" }}'
- 'v2-lotus-params-'
paths:
- /var/tmp/filecoin-proof-parameters/
- run:
command: GO111MODULE=on go get github.com/ipfs/ipget@c0cbd7d9d1925965a5aa0895d5ff32dbdb4a009e
working_directory: "~"
- run: make build/.params-1024
- save_cache:
name: Save parameters cache
key: 'v2-lotus-params-{{ checksum "build/proof-params/parameters.json" }}'
paths:
- /var/tmp/filecoin-proof-parameters/
jobs: jobs:
@ -62,10 +80,8 @@ jobs:
- install-deps - install-deps
- prepare - prepare
- go/mod-download - go/mod-download
- run:
command: GO111MODULE=on go get github.com/ipfs/ipget@c0cbd7d9d1925965a5aa0895d5ff32dbdb4a009e
working_directory: "~"
- run: echo 'export PATH=$HOME/.cargo/bin:$PATH' >> $BASH_ENV - run: echo 'export PATH=$HOME/.cargo/bin:$PATH' >> $BASH_ENV
- download-params
- run: - run:
command: make deps command: make deps
no_output_timeout: 30m no_output_timeout: 30m
@ -120,9 +136,7 @@ jobs:
- install-deps - install-deps
- prepare - prepare
- go/mod-download - go/mod-download
- run: - download-params
command: GO111MODULE=on go get github.com/ipfs/ipget@c0cbd7d9d1925965a5aa0895d5ff32dbdb4a009e
working_directory: "~"
- run: - run:
command: make deps command: make deps
no_output_timeout: 30m no_output_timeout: 30m

18
build/proof-params/paramfetch.sh
View File

@ -1,8 +1,24 @@
#!/usr/bin/env bash #!/usr/bin/env bash
set -euo pipefail
IPGET_PARAMS="--node=spawn -p=/ip4/138.201.67.219/tcp/4002/ws/ipfs/QmUd6zHcbkbcs7SMxwLs48qZVX3vpcM8errYS7xEczwRMA -p=/ip4/138.201.67.218/tcp/4002/ws/ipfs/QmbVWZQhCGrS7DhgLqWbgvdmKN7JueKCREVanfnVpgyq8x -p=/ip4/94.130.135.167/tcp/4002/ws/ipfs/QmUEMvxS2e7iDrereVYc5SWPauXPyNwxcy9BXZrC1QTcHE -p=/ip4/138.201.68.74/tcp/4001/ipfs/QmdnXwLrC8p1ueiq2Qya8joNvk3TVVDAut7PrikmZwubtR -p=/ip4/138.201.67.220/tcp/4001/ipfs/QmNSYxZAiJHeLdkBg38roksAR9So7Y5eojks1yjEcUtZ7i" IPGET_PARAMS="--node=spawn -p=/ip4/138.201.67.219/tcp/4002/ws/ipfs/QmUd6zHcbkbcs7SMxwLs48qZVX3vpcM8errYS7xEczwRMA -p=/ip4/138.201.67.218/tcp/4002/ws/ipfs/QmbVWZQhCGrS7DhgLqWbgvdmKN7JueKCREVanfnVpgyq8x -p=/ip4/94.130.135.167/tcp/4002/ws/ipfs/QmUEMvxS2e7iDrereVYc5SWPauXPyNwxcy9BXZrC1QTcHE -p=/ip4/138.201.68.74/tcp/4001/ipfs/QmdnXwLrC8p1ueiq2Qya8joNvk3TVVDAut7PrikmZwubtR -p=/ip4/138.201.67.220/tcp/4001/ipfs/QmNSYxZAiJHeLdkBg38roksAR9So7Y5eojks1yjEcUtZ7i"
OUT_DIR="/var/tmp/filecoin-proof-parameters" OUT_DIR="/var/tmp/filecoin-proof-parameters"
PARAMS="build/proof-params/parameters.json" PARAMS="build/proof-params/parameters.json"
mkdir -p $OUT_DIR mkdir -p $OUT_DIR
jq '. | to_entries | map("-o '$OUT_DIR'/\(.key) \(.value.cid)") | .[]' --raw-output $PARAMS | xargs -t -L1 ipget --progress $IPGET_PARAMS jq '. | to_entries | map("'$OUT_DIR'/\(.key) \(.value.cid) \(.value.digest)") | .[]' --raw-output $PARAMS | \
while read -r dest cid digest; do
if [[ -f "$dest" ]]; then
b2=$(b2sum "$dest" | head -c 32)
if [[ "$digest" == "$b2" ]]; then
echo "$dest exists and has correct hash"
continue
else
echo "$dest has incorrect hash"
rm -f "$dest"
fi
fi
echo "downloading $dest"
ipget --progress $IPGET_PARAMS -o "$dest" "$cid"
done