From 3daf9103a8b1303ae11f38c1b82477978355c6f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 4 Dec 2019 20:44:15 +0100 Subject: [PATCH 01/11] paramfetch: Only pull necessary params --- build/paramfetch.go | 11 ++++------- cli/params.go | 12 ++++-------- cmd/lotus-bench/main.go | 2 +- cmd/lotus-seed/seed/seed.go | 2 +- cmd/lotus-storage-miner/init.go | 8 +++++--- cmd/lotus-storage-miner/run.go | 4 ---- cmd/lotus/daemon.go | 2 +- lib/sectorbuilder/sectorbuilder_test.go | 6 +++--- node/builder.go | 2 ++ node/modules/storageminer.go | 8 ++++++++ 10 files changed, 29 insertions(+), 28 deletions(-) diff --git a/build/paramfetch.go b/build/paramfetch.go index b167f1ca5..1bb0f4b25 100644 --- a/build/paramfetch.go +++ b/build/paramfetch.go @@ -39,7 +39,7 @@ type fetch struct { errs []error } -func GetParams(storage bool, tests bool) error { +func GetParams(storageSize uint64) error { if err := os.Mkdir(paramdir, 0755); err != nil && !os.IsExist(err) { return err } @@ -54,10 +54,7 @@ func GetParams(storage bool, tests bool) error { ft := &fetch{} for name, info := range params { - if !(SupportedSectorSize(info.SectorSize) || (tests && info.SectorSize == 1<<10)) { - continue - } - if !storage && strings.HasSuffix(name, ".params") { + if storageSize != info.SectorSize && strings.HasSuffix(name, ".params") { continue } @@ -83,8 +80,8 @@ func (ft *fetch) maybeFetchAsync(name string, info paramFile) { return } - ft.fetchLk.Lock() - defer ft.fetchLk.Unlock() + /*ft.fetchLk.Lock() + defer ft.fetchLk.Unlock()*/ if err := doFetch(path, info); err != nil { ft.errs = append(ft.errs, xerrors.Errorf("fetching file %s failed: %w", path, err)) diff --git a/cli/params.go b/cli/params.go index eabee8d5e..df7e240ec 100644 --- a/cli/params.go +++ b/cli/params.go @@ -10,17 +10,13 @@ var fetchParamCmd = &cli.Command{ Name: "fetch-params", Usage: "Fetch proving parameters", Flags: []cli.Flag{ - &cli.BoolFlag{ - Name: "only-verify-keys", - Usage: "only download the verify keys", - }, - &cli.BoolFlag{ - Name: "include-test-params", - Usage: "download params used for tests", + &cli.Uint64Flag{ + Name: "proving-params", + Usage: "download params used creating proofs for given size", }, }, Action: func(cctx *cli.Context) error { - err := build.GetParams(!cctx.Bool("only-verify-keys"), cctx.Bool("include-test-params")) + err := build.GetParams(cctx.Uint64("proving-params")) if err != nil { return xerrors.Errorf("fetching proof parameters: %w", err) } diff --git a/cmd/lotus-bench/main.go b/cmd/lotus-bench/main.go index 33fdfcbf2..7964186f2 100644 --- a/cmd/lotus-bench/main.go +++ b/cmd/lotus-bench/main.go @@ -113,7 +113,7 @@ func main() { } } - if err := build.GetParams(true, false); err != nil { + if err := build.GetParams(sectorSize); err != nil { return xerrors.Errorf("getting params: %w", err) } sb, err := sectorbuilder.New(cfg, mds) diff --git a/cmd/lotus-seed/seed/seed.go b/cmd/lotus-seed/seed/seed.go index d52c1b32c..88b79ec3d 100644 --- a/cmd/lotus-seed/seed/seed.go +++ b/cmd/lotus-seed/seed/seed.go @@ -48,7 +48,7 @@ func PreSeal(maddr address.Address, ssize uint64, sectors int, sbroot string, pr return nil, err } - if err := build.GetParams(true, false); err != nil { + if err := build.GetParams(ssize); err != nil { return nil, xerrors.Errorf("getting params: %w", err) } diff --git a/cmd/lotus-storage-miner/init.go b/cmd/lotus-storage-miner/init.go index 05889b012..7ae748c5f 100644 --- a/cmd/lotus-storage-miner/init.go +++ b/cmd/lotus-storage-miner/init.go @@ -75,8 +75,10 @@ var initCmd = &cli.Command{ Action: func(cctx *cli.Context) error { log.Info("Initializing lotus storage miner") + ssize := cctx.Uint64("sector-size") + log.Info("Checking proof parameters") - if err := build.GetParams(true, false); err != nil { + if err := build.GetParams(ssize); err != nil { return xerrors.Errorf("fetching proof parameters: %w", err) } @@ -152,7 +154,7 @@ var initCmd = &cli.Command{ } oldsb, err := sectorbuilder.New(§orbuilder.Config{ - SectorSize: 1024, + SectorSize: ssize, WorkerThreads: 2, SealedDir: filepath.Join(pssb, "sealed"), CacheDir: filepath.Join(pssb, "cache"), @@ -164,7 +166,7 @@ var initCmd = &cli.Command{ } nsb, err := sectorbuilder.New(§orbuilder.Config{ - SectorSize: 1024, + SectorSize: ssize, WorkerThreads: 2, SealedDir: filepath.Join(lr.Path(), "sealed"), CacheDir: filepath.Join(lr.Path(), "cache"), diff --git a/cmd/lotus-storage-miner/run.go b/cmd/lotus-storage-miner/run.go index 2498fe419..3d833290d 100644 --- a/cmd/lotus-storage-miner/run.go +++ b/cmd/lotus-storage-miner/run.go @@ -41,10 +41,6 @@ var runCmd = &cli.Command{ }, }, Action: func(cctx *cli.Context) error { - if err := build.GetParams(true, false); err != nil { - return xerrors.Errorf("fetching proof parameters: %w", err) - } - if !cctx.Bool("enable-gpu-proving") { os.Setenv("BELLMAN_NO_GPU", "true") } diff --git a/cmd/lotus/daemon.go b/cmd/lotus/daemon.go index bbf036275..ce2cc64b9 100644 --- a/cmd/lotus/daemon.go +++ b/cmd/lotus/daemon.go @@ -63,7 +63,7 @@ var DaemonCmd = &cli.Command{ return xerrors.Errorf("repo init error: %w", err) } - if err := build.GetParams(false, false); err != nil { + if err := build.GetParams(0); err != nil { return xerrors.Errorf("fetching proof parameters: %w", err) } diff --git a/lib/sectorbuilder/sectorbuilder_test.go b/lib/sectorbuilder/sectorbuilder_test.go index 6f97f0a34..cc8cb9563 100644 --- a/lib/sectorbuilder/sectorbuilder_test.go +++ b/lib/sectorbuilder/sectorbuilder_test.go @@ -126,7 +126,7 @@ func TestSealAndVerify(t *testing.T) { build.SectorSizes = []uint64{sectorSize} - if err := build.GetParams(true, true); err != nil { + if err := build.GetParams(sectorSize); err != nil { t.Fatalf("%+v", err) } @@ -195,7 +195,7 @@ func TestSealPoStNoCommit(t *testing.T) { build.SectorSizes = []uint64{sectorSize} - if err := build.GetParams(true, true); err != nil { + if err := build.GetParams(sectorSize); err != nil { t.Fatalf("%+v", err) } @@ -257,7 +257,7 @@ func TestSealAndVerify2(t *testing.T) { build.SectorSizes = []uint64{sectorSize} - if err := build.GetParams(true, true); err != nil { + if err := build.GetParams(sectorSize); err != nil { t.Fatalf("%+v", err) } diff --git a/node/builder.go b/node/builder.go index bc8dbea59..eedacadaf 100644 --- a/node/builder.go +++ b/node/builder.go @@ -91,6 +91,7 @@ const ( RegisterClientValidatorKey // storage miner + GetParamsKey HandleDealsKey HandleRetrievalKey RunSectorServiceKey @@ -250,6 +251,7 @@ func Online() Option { Override(new(*deals.Provider), deals.NewProvider), Override(RegisterProviderValidatorKey, modules.RegisterProviderValidator), Override(HandleRetrievalKey, modules.HandleRetrieval), + Override(GetParamsKey, modules.GetParams), Override(HandleDealsKey, modules.HandleDeals), Override(new(gen.ElectionPoStProver), storage.NewElectionPoStProver), Override(new(*miner.Miner), modules.SetupBlockProducer), diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index eaa2a837a..20f06ace0 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -44,6 +44,14 @@ func minerAddrFromDS(ds dtypes.MetadataDS) (address.Address, error) { return address.NewFromBytes(maddrb) } +func GetParams(sbc *sectorbuilder.Config) error { + if err := build.GetParams(sbc.SectorSize); err != nil { + return xerrors.Errorf("fetching proof parameters: %w", err) + } + + return nil +} + func SectorBuilderConfig(storagePath string, threads uint) func(dtypes.MetadataDS, api.FullNode) (*sectorbuilder.Config, error) { return func(ds dtypes.MetadataDS, api api.FullNode) (*sectorbuilder.Config, error) { minerAddr, err := minerAddrFromDS(ds) From ab7bd8e2f0737b74f2e5717c4ad4aa71d6d69888 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 4 Dec 2019 20:44:51 +0100 Subject: [PATCH 02/11] v20 parameters --- .circleci/config.yml | 6 +- build/proof-params/parameters.json | 128 +++++++++++++++++------------ 2 files changed, 77 insertions(+), 57 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index cfc6e995a..3ae25e2f9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -37,14 +37,14 @@ commands: - restore_cache: name: Restore parameters cache keys: - - 'v19-lotus-params-{{ checksum "build/proof-params/parameters.json" }}-{{ checksum "build/paramfetch.go" }}' - - 'v19-lotus-params-{{ checksum "build/proof-params/parameters.json" }}-' + - 'v20-lotus-params-{{ checksum "build/proof-params/parameters.json" }}-{{ checksum "build/paramfetch.go" }}' + - 'v20-lotus-params-{{ checksum "build/proof-params/parameters.json" }}-' paths: - /var/tmp/filecoin-proof-parameters/ - run: ./lotus fetch-params --include-test-params - save_cache: name: Save parameters cache - key: 'v19-lotus-params-{{ checksum "build/proof-params/parameters.json" }}-{{ checksum "build/paramfetch.go" }}' + key: 'v20-lotus-params-{{ checksum "build/proof-params/parameters.json" }}-{{ checksum "build/paramfetch.go" }}' paths: - /var/tmp/filecoin-proof-parameters/ diff --git a/build/proof-params/parameters.json b/build/proof-params/parameters.json index 0fc8abffb..9093cf15e 100644 --- a/build/proof-params/parameters.json +++ b/build/proof-params/parameters.json @@ -1,82 +1,102 @@ { - "v19-proof-of-spacetime-election-09ae025de08399327e14f0cb6b4c907b6fe1e8b77046e31de8921bde588de900.params": { - "cid": "QmZEKhzKbC7SPngjL85ghyuxabPmiEuUh4fpkC7CDK1J5q", - "digest": "1c81338b8afeaae514fd5d6c08c9c6e5", + "v20-proof-of-spacetime-election-09ae025de08399327e14f0cb6b4c907b6fe1e8b77046e31de8921bde588de900.params": { + "cid": "QmbbP455Dmu4GvEYgJTJjDvN8pj98cAKBQWgo4wmzUQhHM", + "digest": "34ecd0781d730a78f905c93dc35f107b", "sector_size": 268435456 }, - "v19-proof-of-spacetime-election-09ae025de08399327e14f0cb6b4c907b6fe1e8b77046e31de8921bde588de900.vk": { - "cid": "Qmf5XQuM58jNmxudXeZMuZtGGiNzcd56Fiyn9h76wpX5dN", - "digest": "bb0f07b6071cd28e9348223c6a9c46d1", + "v20-proof-of-spacetime-election-09ae025de08399327e14f0cb6b4c907b6fe1e8b77046e31de8921bde588de900.vk": { + "cid": "QmUEsQPP46mjiZkFoVdgJMBt8yhDxrzGTdqExWHAMA2eAX", + "digest": "cb5096105e25070e4894f682a0e7ce7e", "sector_size": 268435456 }, - "v19-proof-of-spacetime-election-4a2342062706429612fac099694f77294e355c6c9265b80feaff12a0268b0a92.params": { - "cid": "QmaHs5CHcSD6QhaGAp4ysJP4HTZaxPxybCGmUVDk3TNocA", - "digest": "727fb2896a5668d04ba6e0ce71eb50d7", + "v20-proof-of-spacetime-election-4a2342062706429612fac099694f77294e355c6c9265b80feaff12a0268b0a92.params": { + "cid": "QmNegxzrsuFT9Z44B1zyxYGsxjUn5zVTCAqyBU6fp4X1Lw", + "digest": "1d3e2add91e31a4c7ad7f985d758e844", "sector_size": 1024 }, - "v19-proof-of-spacetime-election-4a2342062706429612fac099694f77294e355c6c9265b80feaff12a0268b0a92.vk": { - "cid": "QmVg8mUXMb6MiZQseAyCmHzEgNkPbV72xQoRmdFr1yJA4w", - "digest": "3fdf4e65a7baf1a2bab5b8a717f3379a", + "v20-proof-of-spacetime-election-4a2342062706429612fac099694f77294e355c6c9265b80feaff12a0268b0a92.vk": { + "cid": "QmSDyi3BQrYXTQp4vSVAjZSF2RfU5iZJzzJ8jM5WSMTgjd", + "digest": "21f6b4c8fb316c41a7b2ffd0e2bf39ca", "sector_size": 1024 }, - "v19-proof-of-spacetime-election-512f5e6dc00a37fa13c8b0e468188f85957b7bf1ab36d17fb9fe9ed49ae8d657.params": { - "cid": "QmYf1532WoeXhy8AoduWNxpBuwn5DEkpU6YFDGCXh1mqBX", - "digest": "983e641f9df01799bc33d5fb3c3020b2", + "v20-proof-of-spacetime-election-512f5e6dc00a37fa13c8b0e468188f85957b7bf1ab36d17fb9fe9ed49ae8d657.params": { + "cid": "QmQQhB1g98j3bPQLTnvUcMLzqhAaFjpE6giwMZp4BkLSkj", + "digest": "54c4ef0dffee9b3f52319ae0a13e0ed0", "sector_size": 1073741824 }, - "v19-proof-of-spacetime-election-512f5e6dc00a37fa13c8b0e468188f85957b7bf1ab36d17fb9fe9ed49ae8d657.vk": { - "cid": "QmVig7LUpNSXUcfjkSxUKsFaxqYxEZdaEARWvi14sbihJ4", - "digest": "e1f7a46b60217f1dddf56671dd86e6a7", + "v20-proof-of-spacetime-election-512f5e6dc00a37fa13c8b0e468188f85957b7bf1ab36d17fb9fe9ed49ae8d657.vk": { + "cid": "QmcDgvf8B7LNNG1UD9JQ9srbmGCqWQ3SpYnSt11Q7M6N6o", + "digest": "f3cd7afac7b4635c7d90ce7108539649", "sector_size": 1073741824 }, - "v19-proof-of-spacetime-election-6c7cbfe7eed40b6c0b23a213a70648770aed65d9ca03ae85451573c18532304b.params": { - "cid": "QmSTF7C6vQbV6qjEQknXpBDuixBkxYeMshX25NonjJxjbi", - "digest": "b434ece6a37e588ca11ed117f1c14325", + "v20-proof-of-spacetime-election-6c7cbfe7eed40b6c0b23a213a70648770aed65d9ca03ae85451573c18532304b.params": { + "cid": "QmX2Tfu49qnSUbz42s9jPf7WTVjmAPNAXFzqroAAWXN4ms", + "digest": "5078bde58986f57f4393e96be9623536", "sector_size": 16777216 }, - "v19-proof-of-spacetime-election-6c7cbfe7eed40b6c0b23a213a70648770aed65d9ca03ae85451573c18532304b.vk": { - "cid": "QmaFV9n5scuYxKc9QpJnbBUf4sjhkUzpZ7QkoTqL8XsNQz", - "digest": "9c2e40b6635470d3674324b01c9a3976", + "v20-proof-of-spacetime-election-6c7cbfe7eed40b6c0b23a213a70648770aed65d9ca03ae85451573c18532304b.vk": { + "cid": "QmaTM9V7AguRAHbjsadVKtHdVvXS8k18qt8rh1S1JPPyJT", + "digest": "2a1d0da743c8162a937ca21e15424d3f", "sector_size": 16777216 }, - "v19-stacked-proof-of-replication-5a438611c880423c4f5199787cd531b197846ef46af40af69222467ab9073226.params": { - "cid": "QmQb5mc3ksZh2K5GsNwJrkwuHoWXh8adBdJV3qTbFaRvGe", - "digest": "a187287b1d03cd7ec4f1fccd57f3f3d1", - "sector_size": 1024 + "v20-proof-of-spacetime-election-7e98e29a3b6fd661ce53507e168a8194bc7c8a29aa069b5c057d95462a8fcf9f.params": { + "cid": "QmRkidQgNyNiG39ikHrQ4pWv6VzVuSNT8f6JG5i6W2BfJG", + "digest": "81906993f9e4525345074c88c6238f61", + "sector_size": 34359738368 }, - "v19-stacked-proof-of-replication-5a438611c880423c4f5199787cd531b197846ef46af40af69222467ab9073226.vk": { - "cid": "QmYUjFHzeX22dfzv9wQxF4Qn8wS67bqBwr1Wcz2rga9rAZ", - "digest": "747792363f08d7b53be1a2f51f91582a", - "sector_size": 1024 + "v20-proof-of-spacetime-election-7e98e29a3b6fd661ce53507e168a8194bc7c8a29aa069b5c057d95462a8fcf9f.vk": { + "cid": "QmTcr7okdprbf2rak2t9S7DZEN4i6sKzJSiSigL36d7Qb5", + "digest": "95638189775b411a489980cb3839cf3d", + "sector_size": 34359738368 }, - "v19-stacked-proof-of-replication-6ae8ae8998ef393ffd171487bc5141fa3642e9fd39e3a7dbada4f6e7bacffb9b.params": { - "cid": "QmRUZqCwYdcVfQ49Z97g2xkJnaSh4b9SHWwfB3kgZiPo9L", - "digest": "b9494e0ae432a0ebde9c8c877c914583", - "sector_size": 16777216 - }, - "v19-stacked-proof-of-replication-6ae8ae8998ef393ffd171487bc5141fa3642e9fd39e3a7dbada4f6e7bacffb9b.vk": { - "cid": "QmXngrBy74h8LYhYrbBpefXsXQFWLU3WX3LTXnVQnu1Sdc", - "digest": "af2c0d6834fa581b6f507f8266244dfb", - "sector_size": 16777216 - }, - "v19-stacked-proof-of-replication-d2ca0f634aebcecba88904612ff82f2349b080b1290879f3fba73c1d9a13d84e.params": { - "cid": "QmfNstuJFKnBt4yJHsNfoKahn1LafBdpJju23U5UNZd9Xy", - "digest": "3911d84ca2b86f491bc7c6372d7d9285", + "v20-stacked-proof-of-replication-117839dacd1ef31e5968a6fd13bcd6fa86638d85c40c9241a1d07c2a954eb89b.params": { + "cid": "QmQZe8eLo2xXbhSDxtyYZNqEjqjdcWGdADywECRvNEZQdX", + "digest": "fcd50e2e08a8560a6bb3418e883567ed", "sector_size": 268435456 }, - "v19-stacked-proof-of-replication-d2ca0f634aebcecba88904612ff82f2349b080b1290879f3fba73c1d9a13d84e.vk": { - "cid": "QmQgSRQBbp7udHDp5pNA3GSCjSyXktUHBw15wx9meL4wgc", - "digest": "ecd2a9bdd178b0ebc9110f568fd70d07", + "v20-stacked-proof-of-replication-117839dacd1ef31e5968a6fd13bcd6fa86638d85c40c9241a1d07c2a954eb89b.vk": { + "cid": "Qme1hn6QT1covfoUFGDZkqoE1pMTax9FNW3nWWmTNqFe7y", + "digest": "872e244d86499fd659082e3bcf3f13e7", "sector_size": 268435456 }, - "v19-stacked-proof-of-replication-f7b95455d6b7a5e967388a97c2ddc01807eff4c1736e84be4554853bf7783105.params": { - "cid": "QmV5mAkhohUHPRWoNtS3Uo4yvmF23CR2u8JxeeCqthMCdX", - "digest": "9306d91c3518b87016d5efc19428b25e", + "v20-stacked-proof-of-replication-b46f3a1051afbb67f70aae7082da95def62eee943662f3e1bf69837fb08aaae4.params": { + "cid": "QmSfrPDC9jwY4MKrjzhCqDBBAG44wSDM8oE5NuDwWSh2xN", + "digest": "0a338b941c5f17946340de5fc95cab30", + "sector_size": 34359738368 + }, + "v20-stacked-proof-of-replication-b46f3a1051afbb67f70aae7082da95def62eee943662f3e1bf69837fb08aaae4.vk": { + "cid": "QmTDGynCmnbaZNBP3Bv3F3duC3ecKRubCKeMUiQQZYbGpF", + "digest": "c752e070a6b7aa8b79aa661a6b600b55", + "sector_size": 34359738368 + }, + "v20-stacked-proof-of-replication-e71093863cadc71de61f38311ee45816633973bbf34849316b147f8d2e66f199.params": { + "cid": "QmXjSSnMUnc7EjQBYtTHhvLU3kXJTbUyhVhJRSTRehh186", + "digest": "efa407fd09202dffd15799a8518e73d3", + "sector_size": 1024 + }, + "v20-stacked-proof-of-replication-e71093863cadc71de61f38311ee45816633973bbf34849316b147f8d2e66f199.vk": { + "cid": "QmYHW3zhQouDP4okFbXSsRMcZ8bokKGvzxqbv7ZrunPMiG", + "digest": "b2f09a0ccb62da28c890d5b881c8dcd2", + "sector_size": 1024 + }, + "v20-stacked-proof-of-replication-e99a585174b6a45b254ba4780d72c89ad808c305c6d11711009ade4f39dba8e9.params": { + "cid": "QmUhyfNeLb32LfSkjsUwTFYLXQGMj6JQ8daff4DdVMt79q", + "digest": "b53c1916a63839ec345aa2224e9198b7", "sector_size": 1073741824 }, - "v19-stacked-proof-of-replication-f7b95455d6b7a5e967388a97c2ddc01807eff4c1736e84be4554853bf7783105.vk": { - "cid": "QmPjwPdUQJmqp2rQuJX2dy57AC9YCwYtMHdpZzW5BG9JWh", - "digest": "a2004fca043da423df51f6c4bb65788c", + "v20-stacked-proof-of-replication-e99a585174b6a45b254ba4780d72c89ad808c305c6d11711009ade4f39dba8e9.vk": { + "cid": "QmWReGfbuoozNErbskmFvqV4q36BY6F2WWb4cVFc3zoYkA", + "digest": "20d58a3fae7343481f8298a2dd493dd7", "sector_size": 1073741824 + }, + "v20-stacked-proof-of-replication-f571ee2386f4c65a68e802747f2d78691006fc81a67971c4d9641403fffece16.params": { + "cid": "QmSAHu14Pe8iav6BYCt9XkpHJ73XM7tcpY4d9JK9BST9HU", + "digest": "7698426202c7e07b26ef056d31485b3a", + "sector_size": 16777216 + }, + "v20-stacked-proof-of-replication-f571ee2386f4c65a68e802747f2d78691006fc81a67971c4d9641403fffece16.vk": { + "cid": "QmaKtFLShnhMGVn7P9UsHjkgqtqRFSwCStqqykBN7u8dax", + "digest": "834408e5c3fce6ec5d1bf64e64cee94e", + "sector_size": 16777216 } } \ No newline at end of file From 816a2f4ca8ebc17d19bce4748ad23119fde2c4b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 4 Dec 2019 20:49:36 +0100 Subject: [PATCH 03/11] circle: Fix fetch-params call --- .circleci/config.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3ae25e2f9..af38163be 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -37,14 +37,14 @@ commands: - restore_cache: name: Restore parameters cache keys: - - 'v20-lotus-params-{{ checksum "build/proof-params/parameters.json" }}-{{ checksum "build/paramfetch.go" }}' - - 'v20-lotus-params-{{ checksum "build/proof-params/parameters.json" }}-' + - 'v20-1k-lotus-params-{{ checksum "build/proof-params/parameters.json" }}-{{ checksum "build/paramfetch.go" }}' + - 'v20-1k-lotus-params-{{ checksum "build/proof-params/parameters.json" }}-' paths: - /var/tmp/filecoin-proof-parameters/ - - run: ./lotus fetch-params --include-test-params + - run: ./lotus fetch-params --proving-params 1024 - save_cache: name: Save parameters cache - key: 'v20-lotus-params-{{ checksum "build/proof-params/parameters.json" }}-{{ checksum "build/paramfetch.go" }}' + key: 'v20-1k-lotus-params-{{ checksum "build/proof-params/parameters.json" }}-{{ checksum "build/paramfetch.go" }}' paths: - /var/tmp/filecoin-proof-parameters/ From 9adb1c004b065313c438ba89183ac10b46072efa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 5 Dec 2019 10:58:57 +0100 Subject: [PATCH 04/11] Update filecoin-ffi --- build/paramfetch.go | 4 ++-- extern/filecoin-ffi | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/build/paramfetch.go b/build/paramfetch.go index 1bb0f4b25..b647f675b 100644 --- a/build/paramfetch.go +++ b/build/paramfetch.go @@ -80,8 +80,8 @@ func (ft *fetch) maybeFetchAsync(name string, info paramFile) { return } - /*ft.fetchLk.Lock() - defer ft.fetchLk.Unlock()*/ + ft.fetchLk.Lock() + defer ft.fetchLk.Unlock() if err := doFetch(path, info); err != nil { ft.errs = append(ft.errs, xerrors.Errorf("fetching file %s failed: %w", path, err)) diff --git a/extern/filecoin-ffi b/extern/filecoin-ffi index ebb3e13ad..6d9e80001 160000 --- a/extern/filecoin-ffi +++ b/extern/filecoin-ffi @@ -1 +1 @@ -Subproject commit ebb3e13addf13059658ba92e84c9ce4300fbdf25 +Subproject commit 6d9e80001bfa2d80eec4e157da46d783038d9b42 From 291fab74c1cf26987b5b10e64c6c8dbdf85f2441 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 5 Dec 2019 12:52:13 +0100 Subject: [PATCH 05/11] sectorbuilder: update PoRepProofPartitions --- lib/sectorbuilder/sectorbuilder.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/sectorbuilder/sectorbuilder.go b/lib/sectorbuilder/sectorbuilder.go index c8f0a1e1c..04c5bc0a4 100644 --- a/lib/sectorbuilder/sectorbuilder.go +++ b/lib/sectorbuilder/sectorbuilder.go @@ -21,7 +21,7 @@ import ( ) const PoStReservedWorkers = 1 -const PoRepProofPartitions = 2 +const PoRepProofPartitions = 10 var lastSectorIdKey = datastore.NewKey("/sectorbuilder/last") From 2960c256e01778db5ca243a59933d529159d44a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 5 Dec 2019 13:04:34 +0100 Subject: [PATCH 06/11] circle: Increase timoeut on test --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index af38163be..e3447a15e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -134,6 +134,7 @@ jobs: << parameters.coverage >> \ << parameters.go-test-flags >> \ << parameters.packages >> + no_output_timeout: 30m - store_test_results: path: /tmp/test-reports - when: From 5aa7bf612394a4631bdbdf6313ad2ed921fef2da Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Tue, 3 Dec 2019 13:57:17 -0800 Subject: [PATCH 07/11] there is no real correlation between challenge count and len(winners) --- lib/sectorbuilder/simple.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/lib/sectorbuilder/simple.go b/lib/sectorbuilder/simple.go index f20395ddd..0c9231092 100644 --- a/lib/sectorbuilder/simple.go +++ b/lib/sectorbuilder/simple.go @@ -46,11 +46,6 @@ func VerifyFallbackPost(ctx context.Context, sectorSize uint64, sectorInfo Sorte } func verifyPost(ctx context.Context, sectorSize uint64, sectorInfo SortedPublicSectorInfo, challengeCount uint64, challengeSeed []byte, proof []byte, candidates []EPostCandidate, proverID address.Address) (bool, error) { - if challengeCount != uint64(len(candidates)) { - log.Warnf("verifyPost with wrong candidate count: expected %d, got %d", challengeCount, len(candidates)) - return false, nil // user input, dont't error - } - var challengeSeeda [CommLen]byte copy(challengeSeeda[:], challengeSeed) From 64bbcae862093956394ed4bfd8c26fec9937849c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 5 Dec 2019 13:28:44 +0100 Subject: [PATCH 08/11] Set 30m timeout on gotest as well --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e3447a15e..91f1a5b67 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -85,7 +85,7 @@ jobs: default: golang go-test-flags: type: string - default: "" + default: "-timeout 30m" description: Flags passed to go test. packages: type: string From 311cf832d6ae6241d2c43bc3268051f8a6c256ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 5 Dec 2019 14:08:58 +0100 Subject: [PATCH 09/11] mod tidy --- go.mod | 2 -- go.sum | 6 ------ 2 files changed, 8 deletions(-) diff --git a/go.mod b/go.mod index 12319e951..795e4eb39 100644 --- a/go.mod +++ b/go.mod @@ -51,13 +51,11 @@ require ( github.com/libp2p/go-libp2p-connmgr v0.1.0 github.com/libp2p/go-libp2p-core v0.2.4 github.com/libp2p/go-libp2p-discovery v0.2.0 - github.com/libp2p/go-libp2p-host v0.1.0 github.com/libp2p/go-libp2p-kad-dht v0.1.1 github.com/libp2p/go-libp2p-mplex v0.2.1 github.com/libp2p/go-libp2p-peer v0.2.0 github.com/libp2p/go-libp2p-peerstore v0.1.3 github.com/libp2p/go-libp2p-pnet v0.1.0 - github.com/libp2p/go-libp2p-protocol v0.1.0 github.com/libp2p/go-libp2p-pubsub v0.2.3 github.com/libp2p/go-libp2p-quic-transport v0.1.1 github.com/libp2p/go-libp2p-record v0.1.1 diff --git a/go.sum b/go.sum index 3b69e1714..e06831208 100644 --- a/go.sum +++ b/go.sum @@ -78,8 +78,6 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv github.com/fd/go-nat v1.0.0/go.mod h1:BTBu/CKvMmOMUPkKVef1pngt2WFH/lg7E6yQnulfp6E= github.com/filecoin-project/chain-validation v0.0.3 h1:luT/8kJ0WdMIqQ9Bm31W4JkuYCW0wUb26AvnD4WK59M= github.com/filecoin-project/chain-validation v0.0.3/go.mod h1:NCEGFjcWRjb8akWFSOXvU6n2efkWIqAeOKU6o5WBGQw= -github.com/filecoin-project/go-amt-ipld v0.0.0-20191203073133-f941215342ed h1:Wt4+eF3fda6MKLjK0/zzBOxs5cUwGyucJSAfO4LnX/w= -github.com/filecoin-project/go-amt-ipld v0.0.0-20191203073133-f941215342ed/go.mod h1:KsFPWjF+UUYl6n9A+qbg4bjFgAOneicFZtDH/LQEX2U= github.com/filecoin-project/go-amt-ipld v0.0.0-20191205011053-79efc22d6cdc h1:cODZD2YzpTUtrOSxbEnWFcQHidNRZiRdvLxySjGvG/M= github.com/filecoin-project/go-amt-ipld v0.0.0-20191205011053-79efc22d6cdc/go.mod h1:KsFPWjF+UUYl6n9A+qbg4bjFgAOneicFZtDH/LQEX2U= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= @@ -331,8 +329,6 @@ github.com/libp2p/go-libp2p-discovery v0.1.0/go.mod h1:4F/x+aldVHjHDHuX85x1zWoFT github.com/libp2p/go-libp2p-discovery v0.2.0 h1:1p3YSOq7VsgaL+xVHPi8XAmtGyas6D2J6rWBEfz/aiY= github.com/libp2p/go-libp2p-discovery v0.2.0/go.mod h1:s4VGaxYMbw4+4+tsoQTqh7wfxg97AEdo4GYBt6BadWg= github.com/libp2p/go-libp2p-host v0.0.1/go.mod h1:qWd+H1yuU0m5CwzAkvbSjqKairayEHdR5MMl7Cwa7Go= -github.com/libp2p/go-libp2p-host v0.1.0 h1:OZwENiFm6JOK3YR5PZJxkXlJE8a5u8g4YvAUrEV2MjM= -github.com/libp2p/go-libp2p-host v0.1.0/go.mod h1:5+fWuLbDn8OxoxPN3CV0vsLe1hAKScSMbT84qRfxum8= github.com/libp2p/go-libp2p-interface-connmgr v0.0.1/go.mod h1:GarlRLH0LdeWcLnYM/SaBykKFl9U5JFnbBGruAk/D5k= github.com/libp2p/go-libp2p-interface-pnet v0.0.1/go.mod h1:el9jHpQAXK5dnTpKA4yfCNBZXvrzdOU75zz+C6ryp3k= github.com/libp2p/go-libp2p-kad-dht v0.1.1 h1:IH6NQuoUv5w5e1O8Jc3KyVDtr0rNd0G9aaADpLI1xVo= @@ -360,8 +356,6 @@ github.com/libp2p/go-libp2p-peerstore v0.1.3/go.mod h1:BJ9sHlm59/80oSkpWgr1MyY1c github.com/libp2p/go-libp2p-pnet v0.1.0 h1:kRUES28dktfnHNIRW4Ro78F7rKBHBiw5MJpl0ikrLIA= github.com/libp2p/go-libp2p-pnet v0.1.0/go.mod h1:ZkyZw3d0ZFOex71halXRihWf9WH/j3OevcJdTmD0lyE= github.com/libp2p/go-libp2p-protocol v0.0.1/go.mod h1:Af9n4PiruirSDjHycM1QuiMi/1VZNHYcK8cLgFJLZ4s= -github.com/libp2p/go-libp2p-protocol v0.1.0 h1:HdqhEyhg0ToCaxgMhnOmUO8snQtt/kQlcjVk3UoJU3c= -github.com/libp2p/go-libp2p-protocol v0.1.0/go.mod h1:KQPHpAabB57XQxGrXCNvbL6UEXfQqUgC/1adR2Xtflk= github.com/libp2p/go-libp2p-pubsub v0.2.3 h1:qJRnRnM7Z4xnHb4i6EBb3DKQXRPgtFWlKP4AmfJudLQ= github.com/libp2p/go-libp2p-pubsub v0.2.3/go.mod h1:Jscj3fk23R5mCrOwb625xjVs5ZEyTZcx/OlTwMDqU+g= github.com/libp2p/go-libp2p-quic-transport v0.1.1 h1:MFMJzvsxIEDEVKzO89BnB/FgvMj9WI4GDGUW2ArDPUA= From 5dd6d52894a6b6dc5489b82d4b6badbac0d2a6a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 5 Dec 2019 14:41:10 +0100 Subject: [PATCH 10/11] Bump some numbers --- build/version.go | 4 ++-- cmd/lotus-fountain/site/miner.html | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/build/version.go b/build/version.go index abc715b48..9c768135f 100644 --- a/build/version.go +++ b/build/version.go @@ -1,7 +1,7 @@ package build // Version is the local build version, set by build system -const Version = "0.7.0" +const Version = "0.10.0" // APIVersion is a hex semver version of the rpc api exposed // @@ -12,7 +12,7 @@ const Version = "0.7.0" // R R H // |\vv/| // vv vv -const APIVersion = 0x000701 +const APIVersion = 0x000a01 const ( MajorMask = 0xff0000 diff --git a/cmd/lotus-fountain/site/miner.html b/cmd/lotus-fountain/site/miner.html index 80b087712..124809c72 100644 --- a/cmd/lotus-fountain/site/miner.html +++ b/cmd/lotus-fountain/site/miner.html @@ -16,6 +16,7 @@ From 7d746faaad61c2c883037a60aacce58a8c9fae45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 5 Dec 2019 15:05:17 +0100 Subject: [PATCH 11/11] Devnet 10 --- build/bootstrap/bootstrappers.pi | 4 ++-- build/genesis/devnet.car | Bin 3055 -> 3055 bytes scripts/deploy-bootstrapper.sh | 34 +++++++++++++++++++++++++++++++ scripts/deploy-devnet.sh | 27 +----------------------- 4 files changed, 37 insertions(+), 28 deletions(-) create mode 100755 scripts/deploy-bootstrapper.sh diff --git a/build/bootstrap/bootstrappers.pi b/build/bootstrap/bootstrappers.pi index 98e1010e6..e57d5490f 100644 --- a/build/bootstrap/bootstrappers.pi +++ b/build/bootstrap/bootstrappers.pi @@ -1,2 +1,2 @@ -/ip4/147.75.80.17/tcp/1347/p2p/12D3KooWPWCCqUN3gPEaFAMpAwfh5a6SryBEsFt5R2oK8oW86a4C -/ip4/147.75.80.29/tcp/1347/p2p/12D3KooWNL1fJPBArhsoqwg2wbXgCDTByMyg4ZGp6HjgWr9bgnaJ +/ip4/147.75.80.29/tcp/1347/p2p/12D3KooWQYHfhaqRCEAcZKFEVZnK7kgPh1FHwYNwLgCZkoxmmARu +/ip4/147.75.80.17/tcp/1347/p2p/12D3KooWQg7XTDcU7ySChvLSMZ3L4cywXPsAcv1uyMNw8n5hv9KS diff --git a/build/genesis/devnet.car b/build/genesis/devnet.car index ab806416a9e8163724cc699d24aca782b1145440..2abd987082fc5c06e0f9267545f6c33c013daaab 100644 GIT binary patch delta 1111 zcmaDa{$6~7NMLmHVaGm}`1IS`4}KA>+EQZGa6EXn!`3%Ri>}tpiWGjDUY1%^oSC1; zxRQynaKRHM1wtAorlu)eVNKd(xiz@`>rx(O75CV0FWj^@)@Rjw`TDB+A1(+iwq3m6ZlQIU zmDJo9XWzs(bckNMAhghT*JKB#GYVJy{tKNfIdkd$Obzd^e70_0%xk{8sKvau+WX?k zd#U6LlXaL)6;?;`yI@JMq=@OW`2Wf~!6@ju{dA(eqVnB^jRmhytZQtyPhQcyXO|@Ni|LHSDQyaO zU;qIxFbN@8z*N)1M8n3W3Pa}h2h*3TB>xJYIbo)FdX;;C|B2t#X~$*dJa>An<~~zj zVpFu}yO(D*c>WX_x^YyZz}dmGs- zyvX{9YOCV{)1$|XUhj-qvQ5@U~gp^Kh4qNBd%WgU7_R~Uq$87do?@6}5-t$FQd44L<Gtr@EOr>?eq zoz8tg`<*e5<&&8&uPo_1b*lbj-mcRck2mvm^IImh37mhP(eQEk{iKT%U72MUMl+Z3 L?%KSE)0hbWR2B!i delta 1111 zcmaDa{$6~7NZ^egw&kqXr%pc}nDK4x9N+9MulApLn#fc0=T~&2{o;eO)5}teiZk=` z7*{ee7A|VcI44J5R(P13AzX9no|Jl_V)4D->s@=Ovbu7Nr8g4pxXYgp8OxfpLPujw_ehFBWb1_WR1axQ(9< z%O-C55hAu|zMeY|Q-|Eqvc$;_OlK6td-CSTyw$kQdu`qERemnfcatXQbC}Ol>5&hV zel$bo*<>ANQw6CryDPYFGbpovf2#48Yjwh2U-KWkPEk7278}To9+&*dd-DPoM$@dtX-9ogOYR}9*-4!m${9-yIaY~!Q z9T-5s3rr#i2ZpAFiH40$6^6{6KVL~Nx^3lk&MYxo>vn10$*$vEJ63+~`f#Hw(q#Y3 znvcw{u7ADo%~4#x{KtdbeNvNsSUU-a%vaWA1@+A#C%nD?9dGAIn5aJK-AUbM=R5gw zO&2*%-!A+4U*CLk1zTr5Aw?|_q&txWEgW!fb+8)~jLXTl+0B(6D=oi$*5syYu*-7Q zV;a+S%#1$;@I8IeA_tp&tH1sT^13d*pguh8Sea~wcYa;q*CkOVRTF;r zo@T8w-d_5s>|feFnXP7O?Y~78t zD}i=w6m3_|T)#^{gQfLIz|8=qgQs^bu1(viRdY78OCyhy-EYd=%;o875rOCRRy%Oa ze#GWBS#VAno7lu-e8-r&?Q0d?z6g0=Y|45Rr8Rk4NPt#H+6$hysf8VmTdTeaGC#dP KY4aXVV Deploying bootstrap node $host" +log "Stopping lotus daemon" + +ssh "$host" 'systemctl stop lotus-daemon' & +ssh "$host" 'systemctl stop lotus-storage-miner' & + +wait + +ssh "$host" 'rm -rf .lotus' & +ssh "$host" 'rm -rf .lotusstorage' & + +scp -C lotus "${host}":/usr/local/bin/lotus & +scp -C lotus-storage-miner "${host}":/usr/local/bin/lotus-storage-miner & + +wait + +log 'Initializing repo' + +ssh "$host" 'systemctl start lotus-daemon' +scp scripts/bootstrap.toml "${host}:.lotus/config.toml" +ssh "$host" "echo -e '[Metrics]\nNickname=\"Boot-$host\"' >> .lotus/config.toml" +ssh "$host" 'systemctl restart lotus-daemon' + +log 'Extracting addr info' + +ssh "$host" 'lotus net listen' | grep -v '/10' | grep -v '/127' >> build/bootstrap/bootstrappers.pi diff --git a/scripts/deploy-devnet.sh b/scripts/deploy-devnet.sh index dd4409b01..c1b82933c 100755 --- a/scripts/deploy-devnet.sh +++ b/scripts/deploy-devnet.sh @@ -91,32 +91,7 @@ make for host in "${BOOTSTRAPPERS[@]}" do - log "> Deploying bootstrap node $host" - log "Stopping lotus daemon" - - ssh "$host" 'systemctl stop lotus-daemon' & - ssh "$host" 'systemctl stop lotus-storage-miner' & - - wait - - ssh "$host" 'rm -rf .lotus' & - ssh "$host" 'rm -rf .lotusstorage' & - - scp -C lotus "${host}":/usr/local/bin/lotus & - scp -C lotus-storage-miner "${host}":/usr/local/bin/lotus-storage-miner & - - wait - - log 'Initializing repo' - - ssh "$host" 'systemctl start lotus-daemon' - scp scripts/bootstrap.toml "${host}:.lotus/config.toml" - ssh "$host" "echo -e '[Metrics]\nNickname=\"Boot-$host\"' >> .lotus/config.toml" - ssh "$host" 'systemctl restart lotus-daemon' - - log 'Extracting addr info' - - ssh "$host" 'lotus net listen' | grep -v '/10' | grep -v '/127' >> build/bootstrap/bootstrappers.pi + ./scripts/deploy-bootstrapper.sh $host done log 'Updating genesis node with bootstrapable binaries'