From feb97789c5a2e7a7b0e88fd0dba1c28713b8b5d4 Mon Sep 17 00:00:00 2001 From: laser Date: Wed, 1 Apr 2020 14:58:23 -0700 Subject: [PATCH 01/10] add README.md --- README.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 000000000..23304ec2c --- /dev/null +++ b/README.md @@ -0,0 +1,15 @@ +# sector-storage + +[![](https://img.shields.io/badge/made%20by-Protocol%20Labs-blue.svg?style=flat-square)](http://ipn.io) +[![standard-readme compliant](https://img.shields.io/badge/standard--readme-OK-green.svg?style=flat-square)](https://github.com/RichardLitt/standard-readme) + +> a concrete implementation of the [specs-storage](https://github.com/filecoin-project/specs-storage) interface + +The sector-storage project provides a implementation-nonspecific reference implementation of the [specs-storage](https://github.com/filecoin-project/specs-storage) interface. + +## License + +The Filecoin Project is dual-licensed under Apache 2.0 and MIT terms: + +- Apache License, Version 2.0, ([LICENSE-APACHE](https://github.com/filecoin-project/sector-storage/blob/master/LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0) +- MIT license ([LICENSE-MIT](https://github.com/filecoin-project/sector-storage/blob/master/LICENSE-MIT) or http://opensource.org/licenses/MIT) From ba291a92542289050602b61c8c9749393d6e8819 Mon Sep 17 00:00:00 2001 From: laser Date: Wed, 1 Apr 2020 15:20:17 -0700 Subject: [PATCH 02/10] add CircleCI badge --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 23304ec2c..28eaffe05 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@ # sector-storage [![](https://img.shields.io/badge/made%20by-Protocol%20Labs-blue.svg?style=flat-square)](http://ipn.io) +[![CircleCI](https://circleci.com/gh/filecoin-project/sector-storage.svg?style=svg)](https://circleci.com/gh/filecoin-project/sector-storage) [![standard-readme compliant](https://img.shields.io/badge/standard--readme-OK-green.svg?style=flat-square)](https://github.com/RichardLitt/standard-readme) > a concrete implementation of the [specs-storage](https://github.com/filecoin-project/specs-storage) interface From fcaaae705d232c5f6462dd0282f689d409aec41c Mon Sep 17 00:00:00 2001 From: laser Date: Wed, 1 Apr 2020 15:38:29 -0700 Subject: [PATCH 03/10] circleci: add shell config - download groth parameters and keys - perform go mod tidy check --- .circleci/config.yml | 49 ++++++++++++++++++++++++++ parameters.json | 82 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 131 insertions(+) create mode 100644 .circleci/config.yml create mode 100644 parameters.json diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 000000000..1b0696247 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,49 @@ +version: 2.1 +orbs: + go: gotest/tools@0.0.9 +executors: + golang: + docker: + - image: circleci/golang:1.13 + resource_class: 2xlarge +commands: + install-build-dependencies: + steps: + - checkout + - run: sudo apt-get update + - run: sudo apt-get install -y jq ocl-icd-opencl-dev + - run: git submodule sync + - run: git submodule update --init --recursive + - run: ./extern/filecoin-ffi/install-filcrypto + download-groth-params-and-verifying-keys: + steps: + - restore_cache: + name: Restore parameters cache + keys: + - 'v24-2k-lotus-params' + paths: + - /var/tmp/filecoin-proof-parameters/ + - run: | + DIR=$(pwd) + cd $(mktemp -d) + go get github.com/filecoin-project/go-paramfetch/paramfetch + go build -o go-paramfetch github.com/filecoin-project/go-paramfetch/paramfetch + ./go-paramfetch 2048 "${DIR}/parameters.json" + - save_cache: + name: Save parameters cache + key: 'v24-2k-lotus-params' + paths: + - /var/tmp/filecoin-proof-parameters/ +jobs: + mod-tidy-check: + executor: golang + steps: + - install-build-dependencies + - download-groth-params-and-verifying-keys + - go/mod-download + - go/mod-tidy-check +workflows: + version: 2.1 + build_and_test: + jobs: + - mod-tidy-check diff --git a/parameters.json b/parameters.json new file mode 100644 index 000000000..8591c1218 --- /dev/null +++ b/parameters.json @@ -0,0 +1,82 @@ +{ + "v24-proof-of-spacetime-election-PoseidonHasher-0b0b9781bcb153efbb3cab4be3a792c4f555d4ab6f8dd62b27e1dcad08a34f22.params": { + "cid": "QmUonpeUaLD6G4byFdZAMzwXorD4Qs1XDjmdXFbWYCgvjW", + "digest": "19e50903e53c826ff66f360283f324c1", + "sector_size": 34359738368 + }, + "v24-proof-of-spacetime-election-PoseidonHasher-0b0b9781bcb153efbb3cab4be3a792c4f555d4ab6f8dd62b27e1dcad08a34f22.vk": { + "cid": "QmVXv4Q1T3FbiY5AUgWER11Lsrby9aUVJy2mgWDWrndFbq", + "digest": "223dd87c6161c45daf448ca9eda28298", + "sector_size": 34359738368 + }, + "v24-proof-of-spacetime-election-PoseidonHasher-0b499a953f1a9dcab420b3ba1e6b1f3952dc7f17cf67ed10406ae9a43e2b8ec5.params": { + "cid": "Qmea7VsrYnkrpdMnutkGKppX5finoDwCA2fP5Zg5bDuBQw", + "digest": "3de5b8738a2cd933c214fa2023e30909", + "sector_size": 8388608 + }, + "v24-proof-of-spacetime-election-PoseidonHasher-0b499a953f1a9dcab420b3ba1e6b1f3952dc7f17cf67ed10406ae9a43e2b8ec5.vk": { + "cid": "QmavFXmf3jeacHKB6HoJH3gUqzmKnsDn5F5HSYfwPbDHRu", + "digest": "485b7eab4f70031fdda4eaeccfe4f26e", + "sector_size": 8388608 + }, + "v24-proof-of-spacetime-election-PoseidonHasher-27a7fc680a47e4821f40cf1676fb80b9888820ef6867a71a175b4c9ae068ad3f.params": { + "cid": "QmQrUjB9NSMuThe1JHoHfC7u1xdoLS6WLu15waWcnZ3tQT", + "digest": "7e6adc7cbf73db8c95a54e3c23bea1ae", + "sector_size": 536870912 + }, + "v24-proof-of-spacetime-election-PoseidonHasher-27a7fc680a47e4821f40cf1676fb80b9888820ef6867a71a175b4c9ae068ad3f.vk": { + "cid": "QmVPPk4fBcEero2GHsYuBoh97yhugTBWUp9yWSPPWjRWQ7", + "digest": "952b352d694d650e912b3b92ad63f7c9", + "sector_size": 536870912 + }, + "v24-proof-of-spacetime-election-PoseidonHasher-5916054ae98e28fc2f0470d1fb58eb875a6865be86f0b8c4e302d55f13217fef.params": { + "cid": "QmSXMF85mdGLQfAY98zVL4dUBpGPFFUPDmFzdc1NZrVFdh", + "digest": "a93de0f8cfb04af5d21f66ef48ee59a8", + "sector_size": 2048 + }, + "v24-proof-of-spacetime-election-PoseidonHasher-5916054ae98e28fc2f0470d1fb58eb875a6865be86f0b8c4e302d55f13217fef.vk": { + "cid": "QmaTsAmbdnQtJoSpkWsXmvHPpMJinzFYTe6t5LLm7w5RtQ", + "digest": "e4d0575f119e3e7b42bc3e5b6bb35a0b", + "sector_size": 2048 + }, + "v24-stacked-proof-of-replication-PoseidonHasher-Sha256Hasher-49442c8ce7545579cbd689d578301d0cc1e46e94e2499a0ec36de7ff4f4694a2.params": { + "cid": "QmYCFrU4G2LakPngFXayX7afyondQbB9hfnVRz1ffWD9MS", + "digest": "d64e5d1bbb9120bea4c0cd8cdcdfb834", + "sector_size": 8388608 + }, + "v24-stacked-proof-of-replication-PoseidonHasher-Sha256Hasher-49442c8ce7545579cbd689d578301d0cc1e46e94e2499a0ec36de7ff4f4694a2.vk": { + "cid": "QmfXAPtHKU2MJVJDwLTUCM4W2tYQ8biGq9cZaAnjtaZidZ", + "digest": "572536e8684454a5cd80361e5c952b38", + "sector_size": 8388608 + }, + "v24-stacked-proof-of-replication-PoseidonHasher-Sha256Hasher-d84aa4581c74190f845596893ebe5b71da32ecf16e1d151b9fff74ee8f94d77c.params": { + "cid": "QmdXtQsLbBFmVxrd6kWKr2FYbQfhEdR6PinwrGBXhHmLdT", + "digest": "77cfafee088bd59411d766621df6de42", + "sector_size": 536870912 + }, + "v24-stacked-proof-of-replication-PoseidonHasher-Sha256Hasher-d84aa4581c74190f845596893ebe5b71da32ecf16e1d151b9fff74ee8f94d77c.vk": { + "cid": "QmdE8oZJofaenThLi2TWXJPk9cExZgTA36TjrHeAC65BGA", + "digest": "30586a2396ef6b60b122ac5a2ba87681", + "sector_size": 536870912 + }, + "v24-stacked-proof-of-replication-PoseidonHasher-Sha256Hasher-fc32be6028c2398175466f36fa36810842ae8948fae15c84454af5b61ca99e15.params": { + "cid": "QmNqcqGxf7pJjipHNwcH44D5KgiTUNo3mK5HiSxBwYcjkx", + "digest": "25ea39db2a003c817113f6f2ea936b3d", + "sector_size": 34359738368 + }, + "v24-stacked-proof-of-replication-PoseidonHasher-Sha256Hasher-fc32be6028c2398175466f36fa36810842ae8948fae15c84454af5b61ca99e15.vk": { + "cid": "QmWiaqy8hWshv2FsLDoZAtpJKZng5QN3x2X5C7xsPvSbFb", + "digest": "ab1239c802c480cf12f63d13fb2f620a", + "sector_size": 34359738368 + }, + "v24-stacked-proof-of-replication-PoseidonHasher-Sha256Hasher-fe437922fe766f61b112750506d6be0e4ad5daa85ff9ce96549d99253ba61cbe.params": { + "cid": "QmbPk3fKKLjkm6pD1CzwGyTnMwNSSZVxVSMWEceqSv6LDW", + "digest": "76bd3702312cfe0d69bb5e0891c52615", + "sector_size": 2048 + }, + "v24-stacked-proof-of-replication-PoseidonHasher-Sha256Hasher-fe437922fe766f61b112750506d6be0e4ad5daa85ff9ce96549d99253ba61cbe.vk": { + "cid": "QmPZ9bGSVs5GHQRRAtC1qv9eQ7GPoH8FWukjxAXtXXcTxg", + "digest": "4edb21b7b6d5787b646f3e336e06303e", + "sector_size": 2048 + } +} From fdc2e4a67a61f84994debb9d06786754e81fed5e Mon Sep 17 00:00:00 2001 From: laser Date: Wed, 1 Apr 2020 16:07:34 -0700 Subject: [PATCH 04/10] ci: run tests on build --- .circleci/config.yml | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1b0696247..0bc957588 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -35,15 +35,39 @@ commands: paths: - /var/tmp/filecoin-proof-parameters/ jobs: - mod-tidy-check: + test: executor: golang + environment: + RUST_LOG: info steps: - install-build-dependencies - download-groth-params-and-verifying-keys + - run: go test -v -timeout 10m ./... + mod-tidy-check: + executor: golang + steps: - go/mod-download - go/mod-tidy-check + gofmt-check: + executor: golang + steps: + - go/mod-download + - run: + command: "! go fmt ./... 2>&1 | read" + lint-check: + executor: golang + steps: + - go/mod-download + - go/install-golangci-lint: + gobin: $HOME/.local/bin + version: 1.23.8 + - run: + command: $HOME/.local/bin/golangci-lint run -v --concurrency 2 workflows: version: 2.1 build_and_test: jobs: - mod-tidy-check + - lint-check + - gofmt-check + - test From c694853ddbedddbfaed5fd6edeb47903f37dbc9f Mon Sep 17 00:00:00 2001 From: laser Date: Wed, 1 Apr 2020 16:08:59 -0700 Subject: [PATCH 05/10] ci: check out code before running linters --- .circleci/config.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0bc957588..3ea597993 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -9,7 +9,6 @@ executors: commands: install-build-dependencies: steps: - - checkout - run: sudo apt-get update - run: sudo apt-get install -y jq ocl-icd-opencl-dev - run: git submodule sync @@ -40,23 +39,27 @@ jobs: environment: RUST_LOG: info steps: + - checkout - install-build-dependencies - download-groth-params-and-verifying-keys - run: go test -v -timeout 10m ./... mod-tidy-check: executor: golang steps: + - checkout - go/mod-download - go/mod-tidy-check gofmt-check: executor: golang steps: + - checkout - go/mod-download - run: command: "! go fmt ./... 2>&1 | read" lint-check: executor: golang steps: + - checkout - go/mod-download - go/install-golangci-lint: gobin: $HOME/.local/bin From 5874d90719a2bd617d716ec5d995db4d15f3d54c Mon Sep 17 00:00:00 2001 From: laser Date: Wed, 1 Apr 2020 16:11:32 -0700 Subject: [PATCH 06/10] ensure submodules are initialized --- .circleci/config.yml | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3ea597993..93b4b70d1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -7,12 +7,15 @@ executors: - image: circleci/golang:1.13 resource_class: 2xlarge commands: + prepare-git-checkout: + steps: + - checkout + - run: git submodule sync + - run: git submodule update --init --recursive install-build-dependencies: steps: - run: sudo apt-get update - run: sudo apt-get install -y jq ocl-icd-opencl-dev - - run: git submodule sync - - run: git submodule update --init --recursive - run: ./extern/filecoin-ffi/install-filcrypto download-groth-params-and-verifying-keys: steps: @@ -39,27 +42,27 @@ jobs: environment: RUST_LOG: info steps: - - checkout + - prepare-git-checkout - install-build-dependencies - download-groth-params-and-verifying-keys - run: go test -v -timeout 10m ./... mod-tidy-check: executor: golang steps: - - checkout + - prepare-git-checkout - go/mod-download - go/mod-tidy-check gofmt-check: executor: golang steps: - - checkout + - prepare-git-checkout - go/mod-download - run: command: "! go fmt ./... 2>&1 | read" lint-check: executor: golang steps: - - checkout + - prepare-git-checkout - go/mod-download - go/install-golangci-lint: gobin: $HOME/.local/bin From 58922b7e764164365bbe3d7fafcc1b65f661eca9 Mon Sep 17 00:00:00 2001 From: laser Date: Wed, 1 Apr 2020 16:13:20 -0700 Subject: [PATCH 07/10] golangci-lint needs libfilcrypto --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 93b4b70d1..0739cb286 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -63,6 +63,7 @@ jobs: executor: golang steps: - prepare-git-checkout + - install-build-dependencies - go/mod-download - go/install-golangci-lint: gobin: $HOME/.local/bin From ffc966a9ada89f68c8d29b4c55a2ac2252443e85 Mon Sep 17 00:00:00 2001 From: laser Date: Wed, 1 Apr 2020 16:14:04 -0700 Subject: [PATCH 08/10] fix gofmt issue which ci caught --- resources.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/resources.go b/resources.go index 310373bfe..4bf3fbbb2 100644 --- a/resources.go +++ b/resources.go @@ -192,7 +192,7 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredProof]Resources{ MinMemory: 1 << 11, MultiThread: false, - CanGPU: true, + CanGPU: true, BaseMinMemory: 1 << 11, }, @@ -201,7 +201,7 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredProof]Resources{ MinMemory: 1 << 23, MultiThread: false, - CanGPU: true, + CanGPU: true, BaseMinMemory: 1 << 23, }, From 431a15c14ac8345df74a97a16960e5a47fdc3648 Mon Sep 17 00:00:00 2001 From: laser Date: Wed, 1 Apr 2020 16:18:20 -0700 Subject: [PATCH 09/10] nolint the existing linting errors --- mock/mock.go | 2 +- tarutil/systar.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mock/mock.go b/mock/mock.go index 1ada002d5..6473fe3c6 100644 --- a/mock/mock.go +++ b/mock/mock.go @@ -53,7 +53,7 @@ func NewMockSectorMgr(threads int, ssize abi.SectorSize) *SectorMgr { const ( statePacking = iota statePreCommit - stateCommit + stateCommit // nolint ) type sectorState struct { diff --git a/tarutil/systar.go b/tarutil/systar.go index a94354731..94de58ea8 100644 --- a/tarutil/systar.go +++ b/tarutil/systar.go @@ -11,7 +11,7 @@ import ( logging "github.com/ipfs/go-log/v2" ) -var log = logging.Logger("tarutil") +var log = logging.Logger("tarutil") // nolint func ExtractTar(body io.Reader, dir string) error { if err := os.MkdirAll(dir, 0755); err != nil { @@ -73,7 +73,7 @@ func writeTarDirectory(dir string, w io.Writer) error { return xerrors.Errorf("wiritng header for file %s: %w", file.Name(), err) } - f, err := os.OpenFile(filepath.Join(dir, file.Name()), os.O_RDONLY, 644) + f, err := os.OpenFile(filepath.Join(dir, file.Name()), os.O_RDONLY, 644) // nolint if err != nil { return xerrors.Errorf("opening %s for reading: %w", file.Name(), err) } From b17b2262205cab5874c33110f51268898da9705b Mon Sep 17 00:00:00 2001 From: laser Date: Wed, 1 Apr 2020 16:22:32 -0700 Subject: [PATCH 10/10] one line gofmt --- .circleci/config.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0739cb286..a7cb9a24b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -57,8 +57,7 @@ jobs: steps: - prepare-git-checkout - go/mod-download - - run: - command: "! go fmt ./... 2>&1 | read" + - run: "! go fmt ./... 2>&1 | read" lint-check: executor: golang steps: