Merge pull request #184 from filecoin-project/feat/cache-params

Cache params

.circleci/config.yml

@@ -13,6 +13,24 @@ commands:
       - checkout
       - run: git submodule sync
       - run: git submodule update --init
+  download-params:
+    steps:
+      - restore_cache:
+          name: Restore parameters cache
+          keys: 
+            - 'v2-lotus-params-{{ checksum "build/proof-params/parameters.json" }}'
+            - 'v2-lotus-params-'
+          paths:
+            - /var/tmp/filecoin-proof-parameters/
+      - run:
+          command: GO111MODULE=on go get github.com/ipfs/ipget@c0cbd7d9d1925965a5aa0895d5ff32dbdb4a009e
+          working_directory: "~"
+      - run:  make build/.params-1024
+      - save_cache:
+          name: Save parameters cache
+          key: 'v2-lotus-params-{{ checksum "build/proof-params/parameters.json" }}'
+          paths:
+            - /var/tmp/filecoin-proof-parameters/
 
 
 jobs:
@@ -62,10 +80,8 @@ jobs:
       - install-deps
       - prepare
       - go/mod-download
-      - run:
-          command: GO111MODULE=on go get github.com/ipfs/ipget@c0cbd7d9d1925965a5aa0895d5ff32dbdb4a009e
-          working_directory: "~"
       - run: echo 'export PATH=$HOME/.cargo/bin:$PATH' >> $BASH_ENV
+      - download-params
       - run:
           command: make deps
           no_output_timeout: 30m
@@ -120,9 +136,7 @@ jobs:
       - install-deps
       - prepare
       - go/mod-download
-      - run:
-          command: GO111MODULE=on go get github.com/ipfs/ipget@c0cbd7d9d1925965a5aa0895d5ff32dbdb4a009e
-          working_directory: "~"
+      - download-params
       - run:
           command: make deps
           no_output_timeout: 30m

build/proof-params/paramfetch.sh

@@ -1,8 +1,24 @@
 #!/usr/bin/env bash
 
+set -euo pipefail
+
 IPGET_PARAMS="--node=spawn -p=/ip4/138.201.67.219/tcp/4002/ws/ipfs/QmUd6zHcbkbcs7SMxwLs48qZVX3vpcM8errYS7xEczwRMA -p=/ip4/138.201.67.218/tcp/4002/ws/ipfs/QmbVWZQhCGrS7DhgLqWbgvdmKN7JueKCREVanfnVpgyq8x -p=/ip4/94.130.135.167/tcp/4002/ws/ipfs/QmUEMvxS2e7iDrereVYc5SWPauXPyNwxcy9BXZrC1QTcHE -p=/ip4/138.201.68.74/tcp/4001/ipfs/QmdnXwLrC8p1ueiq2Qya8joNvk3TVVDAut7PrikmZwubtR -p=/ip4/138.201.67.220/tcp/4001/ipfs/QmNSYxZAiJHeLdkBg38roksAR9So7Y5eojks1yjEcUtZ7i"
 OUT_DIR="/var/tmp/filecoin-proof-parameters"
 PARAMS="build/proof-params/parameters.json"
 
 mkdir -p $OUT_DIR
-jq '. | to_entries | map("-o '$OUT_DIR'/\(.key) \(.value.cid)") | .[]' --raw-output $PARAMS | xargs -t -L1 ipget --progress $IPGET_PARAMS
+jq '. | to_entries | map("'$OUT_DIR'/\(.key) \(.value.cid) \(.value.digest)") | .[]' --raw-output $PARAMS | \
+	while read -r dest cid digest; do
+		if [[ -f "$dest" ]]; then
+			b2=$(b2sum "$dest" | head -c 32)
+			if [[ "$digest" == "$b2" ]]; then
+				echo "$dest exists and has correct hash"
+				continue
+			else
+				echo "$dest has incorrect hash"
+				rm -f "$dest"
+			fi
+		fi
+		echo "downloading $dest"
+		ipget --progress $IPGET_PARAMS -o "$dest" "$cid"
+	done