From e51fa5c9ac472938b77a2a9dabdbcd2d1ea68cfa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 6 Mar 2020 19:50:41 +0100 Subject: [PATCH 1/4] libs: Use zerocomm from sectorbuilder --- chain/vm/syscalls.go | 24 -------- lib/zerocomm/zerocomm.go | 55 ------------------ lib/zerocomm/zerocomm_test.go | 106 ---------------------------------- storage/sbmock/preseal.go | 4 +- storage/sealing/checks.go | 4 +- 5 files changed, 4 insertions(+), 189 deletions(-) delete mode 100644 lib/zerocomm/zerocomm.go delete mode 100644 lib/zerocomm/zerocomm_test.go diff --git a/chain/vm/syscalls.go b/chain/vm/syscalls.go index 45c83ec0b..837d010d0 100644 --- a/chain/vm/syscalls.go +++ b/chain/vm/syscalls.go @@ -3,11 +3,9 @@ package vm import ( "context" "fmt" - "math/bits" "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-sectorbuilder" - "github.com/filecoin-project/lotus/lib/zerocomm" "github.com/filecoin-project/specs-actors/actors/abi" "github.com/filecoin-project/specs-actors/actors/crypto" "github.com/filecoin-project/specs-actors/actors/runtime" @@ -36,28 +34,6 @@ func (ss *syscallShim) ComputeUnsealedSectorCID(st abi.RegisteredProof, pieces [ sum += p.Size } - ssize, err := st.SectorSize() - if err != nil { - return cid.Undef, err - } - - { - // pad remaining space with 0 CommPs - toFill := uint64(abi.PaddedPieceSize(ssize) - sum) - n := bits.OnesCount64(toFill) - for i := 0; i < n; i++ { - next := bits.TrailingZeros64(toFill) - psize := uint64(1) << next - toFill ^= psize - - unpadded := abi.PaddedPieceSize(psize).Unpadded() - pieces = append(pieces, abi.PieceInfo{ - Size: unpadded.Padded(), - PieceCID: zerocomm.ForSize(unpadded), - }) - } - } - commd, err := sectorbuilder.GenerateUnsealedCID(st, pieces) if err != nil { log.Errorf("generate data commitment failed: %s", err) diff --git a/lib/zerocomm/zerocomm.go b/lib/zerocomm/zerocomm.go deleted file mode 100644 index 92afbfeae..000000000 --- a/lib/zerocomm/zerocomm.go +++ /dev/null @@ -1,55 +0,0 @@ -package zerocomm - -import ( - "math/bits" - - commcid "github.com/filecoin-project/go-fil-commcid" - "github.com/filecoin-project/specs-actors/actors/abi" - "github.com/ipfs/go-cid" -) - -const levels = 37 -const skip = 2 // can't generate for 32, 64b - -var pieceComms = [levels - skip][32]byte{ - {0x37, 0x31, 0xbb, 0x99, 0xac, 0x68, 0x9f, 0x66, 0xee, 0xf5, 0x97, 0x3e, 0x4a, 0x94, 0xda, 0x18, 0x8f, 0x4d, 0xdc, 0xae, 0x58, 0x7, 0x24, 0xfc, 0x6f, 0x3f, 0xd6, 0xd, 0xfd, 0x48, 0x83, 0x33}, - {0x64, 0x2a, 0x60, 0x7e, 0xf8, 0x86, 0xb0, 0x4, 0xbf, 0x2c, 0x19, 0x78, 0x46, 0x3a, 0xe1, 0xd4, 0x69, 0x3a, 0xc0, 0xf4, 0x10, 0xeb, 0x2d, 0x1b, 0x7a, 0x47, 0xfe, 0x20, 0x5e, 0x5e, 0x75, 0xf}, - {0x57, 0xa2, 0x38, 0x1a, 0x28, 0x65, 0x2b, 0xf4, 0x7f, 0x6b, 0xef, 0x7a, 0xca, 0x67, 0x9b, 0xe4, 0xae, 0xde, 0x58, 0x71, 0xab, 0x5c, 0xf3, 0xeb, 0x2c, 0x8, 0x11, 0x44, 0x88, 0xcb, 0x85, 0x26}, - {0x1f, 0x7a, 0xc9, 0x59, 0x55, 0x10, 0xe0, 0x9e, 0xa4, 0x1c, 0x46, 0xb, 0x17, 0x64, 0x30, 0xbb, 0x32, 0x2c, 0xd6, 0xfb, 0x41, 0x2e, 0xc5, 0x7c, 0xb1, 0x7d, 0x98, 0x9a, 0x43, 0x10, 0x37, 0x2f}, - {0xfc, 0x7e, 0x92, 0x82, 0x96, 0xe5, 0x16, 0xfa, 0xad, 0xe9, 0x86, 0xb2, 0x8f, 0x92, 0xd4, 0x4a, 0x4f, 0x24, 0xb9, 0x35, 0x48, 0x52, 0x23, 0x37, 0x6a, 0x79, 0x90, 0x27, 0xbc, 0x18, 0xf8, 0x33}, - {0x8, 0xc4, 0x7b, 0x38, 0xee, 0x13, 0xbc, 0x43, 0xf4, 0x1b, 0x91, 0x5c, 0xe, 0xed, 0x99, 0x11, 0xa2, 0x60, 0x86, 0xb3, 0xed, 0x62, 0x40, 0x1b, 0xf9, 0xd5, 0x8b, 0x8d, 0x19, 0xdf, 0xf6, 0x24}, - {0xb2, 0xe4, 0x7b, 0xfb, 0x11, 0xfa, 0xcd, 0x94, 0x1f, 0x62, 0xaf, 0x5c, 0x75, 0xf, 0x3e, 0xa5, 0xcc, 0x4d, 0xf5, 0x17, 0xd5, 0xc4, 0xf1, 0x6d, 0xb2, 0xb4, 0xd7, 0x7b, 0xae, 0xc1, 0xa3, 0x2f}, - {0xf9, 0x22, 0x61, 0x60, 0xc8, 0xf9, 0x27, 0xbf, 0xdc, 0xc4, 0x18, 0xcd, 0xf2, 0x3, 0x49, 0x31, 0x46, 0x0, 0x8e, 0xae, 0xfb, 0x7d, 0x2, 0x19, 0x4d, 0x5e, 0x54, 0x81, 0x89, 0x0, 0x51, 0x8}, - {0x2c, 0x1a, 0x96, 0x4b, 0xb9, 0xb, 0x59, 0xeb, 0xfe, 0xf, 0x6d, 0xa2, 0x9a, 0xd6, 0x5a, 0xe3, 0xe4, 0x17, 0x72, 0x4a, 0x8f, 0x7c, 0x11, 0x74, 0x5a, 0x40, 0xca, 0xc1, 0xe5, 0xe7, 0x40, 0x11}, - {0xfe, 0xe3, 0x78, 0xce, 0xf1, 0x64, 0x4, 0xb1, 0x99, 0xed, 0xe0, 0xb1, 0x3e, 0x11, 0xb6, 0x24, 0xff, 0x9d, 0x78, 0x4f, 0xbb, 0xed, 0x87, 0x8d, 0x83, 0x29, 0x7e, 0x79, 0x5e, 0x2, 0x4f, 0x2}, - {0x8e, 0x9e, 0x24, 0x3, 0xfa, 0x88, 0x4c, 0xf6, 0x23, 0x7f, 0x60, 0xdf, 0x25, 0xf8, 0x3e, 0xe4, 0xd, 0xca, 0x9e, 0xd8, 0x79, 0xeb, 0x6f, 0x63, 0x52, 0xd1, 0x50, 0x84, 0xf5, 0xad, 0xd, 0x3f}, - {0x75, 0x2d, 0x96, 0x93, 0xfa, 0x16, 0x75, 0x24, 0x39, 0x54, 0x76, 0xe3, 0x17, 0xa9, 0x85, 0x80, 0xf0, 0x9, 0x47, 0xaf, 0xb7, 0xa3, 0x5, 0x40, 0xd6, 0x25, 0xa9, 0x29, 0x1c, 0xc1, 0x2a, 0x7}, - {0x70, 0x22, 0xf6, 0xf, 0x7e, 0xf6, 0xad, 0xfa, 0x17, 0x11, 0x7a, 0x52, 0x61, 0x9e, 0x30, 0xce, 0xa8, 0x2c, 0x68, 0x7, 0x5a, 0xdf, 0x1c, 0x66, 0x77, 0x86, 0xec, 0x50, 0x6e, 0xef, 0x2d, 0x19}, - {0xd9, 0x98, 0x87, 0xb9, 0x73, 0x57, 0x3a, 0x96, 0xe1, 0x13, 0x93, 0x64, 0x52, 0x36, 0xc1, 0x7b, 0x1f, 0x4c, 0x70, 0x34, 0xd7, 0x23, 0xc7, 0xa9, 0x9f, 0x70, 0x9b, 0xb4, 0xda, 0x61, 0x16, 0x2b}, - {0xd0, 0xb5, 0x30, 0xdb, 0xb0, 0xb4, 0xf2, 0x5c, 0x5d, 0x2f, 0x2a, 0x28, 0xdf, 0xee, 0x80, 0x8b, 0x53, 0x41, 0x2a, 0x2, 0x93, 0x1f, 0x18, 0xc4, 0x99, 0xf5, 0xa2, 0x54, 0x8, 0x6b, 0x13, 0x26}, - {0x84, 0xc0, 0x42, 0x1b, 0xa0, 0x68, 0x5a, 0x1, 0xbf, 0x79, 0x5a, 0x23, 0x44, 0x6, 0x4f, 0xe4, 0x24, 0xbd, 0x52, 0xa9, 0xd2, 0x43, 0x77, 0xb3, 0x94, 0xff, 0x4c, 0x4b, 0x45, 0x68, 0xe8, 0x11}, - {0x65, 0xf2, 0x9e, 0x5d, 0x98, 0xd2, 0x46, 0xc3, 0x8b, 0x38, 0x8c, 0xfc, 0x6, 0xdb, 0x1f, 0x6b, 0x2, 0x13, 0x3, 0xc5, 0xa2, 0x89, 0x0, 0xb, 0xdc, 0xe8, 0x32, 0xa9, 0xc3, 0xec, 0x42, 0x1c}, - {0xa2, 0x24, 0x75, 0x8, 0x28, 0x58, 0x50, 0x96, 0x5b, 0x7e, 0x33, 0x4b, 0x31, 0x27, 0xb0, 0xc0, 0x42, 0xb1, 0xd0, 0x46, 0xdc, 0x54, 0x40, 0x21, 0x37, 0x62, 0x7c, 0xd8, 0x79, 0x9c, 0xe1, 0x3a}, - {0xda, 0xfd, 0xab, 0x6d, 0xa9, 0x36, 0x44, 0x53, 0xc2, 0x6d, 0x33, 0x72, 0x6b, 0x9f, 0xef, 0xe3, 0x43, 0xbe, 0x8f, 0x81, 0x64, 0x9e, 0xc0, 0x9, 0xaa, 0xd3, 0xfa, 0xff, 0x50, 0x61, 0x75, 0x8}, - {0xd9, 0x41, 0xd5, 0xe0, 0xd6, 0x31, 0x4a, 0x99, 0x5c, 0x33, 0xff, 0xbd, 0x4f, 0xbe, 0x69, 0x11, 0x8d, 0x73, 0xd4, 0xe5, 0xfd, 0x2c, 0xd3, 0x1f, 0xf, 0x7c, 0x86, 0xeb, 0xdd, 0x14, 0xe7, 0x6}, - {0x51, 0x4c, 0x43, 0x5c, 0x3d, 0x4, 0xd3, 0x49, 0xa5, 0x36, 0x5f, 0xbd, 0x59, 0xff, 0xc7, 0x13, 0x62, 0x91, 0x11, 0x78, 0x59, 0x91, 0xc1, 0xa3, 0xc5, 0x3a, 0xf2, 0x20, 0x79, 0x74, 0x1a, 0x2f}, - {0xad, 0x6, 0x85, 0x39, 0x69, 0xd3, 0x7d, 0x34, 0xff, 0x8, 0xe0, 0x9f, 0x56, 0x93, 0xa, 0x4a, 0xd1, 0x9a, 0x89, 0xde, 0xf6, 0xc, 0xbf, 0xee, 0x7e, 0x1d, 0x33, 0x81, 0xc1, 0xe7, 0x1c, 0x37}, - {0x39, 0x56, 0xe, 0x7b, 0x13, 0xa9, 0x3b, 0x7, 0xa2, 0x43, 0xfd, 0x27, 0x20, 0xff, 0xa7, 0xcb, 0x3e, 0x1d, 0x2e, 0x50, 0x5a, 0xb3, 0x62, 0x9e, 0x79, 0xf4, 0x63, 0x13, 0x51, 0x2c, 0xda, 0x6}, - {0xcc, 0xc3, 0xc0, 0x12, 0xf5, 0xb0, 0x5e, 0x81, 0x1a, 0x2b, 0xbf, 0xdd, 0xf, 0x68, 0x33, 0xb8, 0x42, 0x75, 0xb4, 0x7b, 0xf2, 0x29, 0xc0, 0x5, 0x2a, 0x82, 0x48, 0x4f, 0x3c, 0x1a, 0x5b, 0x3d}, - {0x7d, 0xf2, 0x9b, 0x69, 0x77, 0x31, 0x99, 0xe8, 0xf2, 0xb4, 0xb, 0x77, 0x91, 0x9d, 0x4, 0x85, 0x9, 0xee, 0xd7, 0x68, 0xe2, 0xc7, 0x29, 0x7b, 0x1f, 0x14, 0x37, 0x3, 0x4f, 0xc3, 0xc6, 0x2c}, - {0x66, 0xce, 0x5, 0xa3, 0x66, 0x75, 0x52, 0xcf, 0x45, 0xc0, 0x2b, 0xcc, 0x4e, 0x83, 0x92, 0x91, 0x9b, 0xde, 0xac, 0x35, 0xde, 0x2f, 0xf5, 0x62, 0x71, 0x84, 0x8e, 0x9f, 0x7b, 0x67, 0x51, 0x7}, - {0xd8, 0x61, 0x2, 0x18, 0x42, 0x5a, 0xb5, 0xe9, 0x5b, 0x1c, 0xa6, 0x23, 0x9d, 0x29, 0xa2, 0xe4, 0x20, 0xd7, 0x6, 0xa9, 0x6f, 0x37, 0x3e, 0x2f, 0x9c, 0x9a, 0x91, 0xd7, 0x59, 0xd1, 0x9b, 0x1}, - {0x6d, 0x36, 0x4b, 0x1e, 0xf8, 0x46, 0x44, 0x1a, 0x5a, 0x4a, 0x68, 0x86, 0x23, 0x14, 0xac, 0xc0, 0xa4, 0x6f, 0x1, 0x67, 0x17, 0xe5, 0x34, 0x43, 0xe8, 0x39, 0xee, 0xdf, 0x83, 0xc2, 0x85, 0x3c}, - {0x7, 0x7e, 0x5f, 0xde, 0x35, 0xc5, 0xa, 0x93, 0x3, 0xa5, 0x50, 0x9, 0xe3, 0x49, 0x8a, 0x4e, 0xbe, 0xdf, 0xf3, 0x9c, 0x42, 0xb7, 0x10, 0xb7, 0x30, 0xd8, 0xec, 0x7a, 0xc7, 0xaf, 0xa6, 0x3e}, - {0xe6, 0x40, 0x5, 0xa6, 0xbf, 0xe3, 0x77, 0x79, 0x53, 0xb8, 0xad, 0x6e, 0xf9, 0x3f, 0xf, 0xca, 0x10, 0x49, 0xb2, 0x4, 0x16, 0x54, 0xf2, 0xa4, 0x11, 0xf7, 0x70, 0x27, 0x99, 0xce, 0xce, 0x2}, - {0x25, 0x9d, 0x3d, 0x6b, 0x1f, 0x4d, 0x87, 0x6d, 0x11, 0x85, 0xe1, 0x12, 0x3a, 0xf6, 0xf5, 0x50, 0x1a, 0xf0, 0xf6, 0x7c, 0xf1, 0x5b, 0x52, 0x16, 0x25, 0x5b, 0x7b, 0x17, 0x8d, 0x12, 0x5, 0x1d}, - {0x3f, 0x9a, 0x4d, 0x41, 0x1d, 0xa4, 0xef, 0x1b, 0x36, 0xf3, 0x5f, 0xf0, 0xa1, 0x95, 0xae, 0x39, 0x2a, 0xb2, 0x3f, 0xee, 0x79, 0x67, 0xb7, 0xc4, 0x1b, 0x3, 0xd1, 0x61, 0x3f, 0xc2, 0x92, 0x39}, - {0xfe, 0x4e, 0xf3, 0x28, 0xc6, 0x1a, 0xa3, 0x9c, 0xfd, 0xb2, 0x48, 0x4e, 0xaa, 0x32, 0xa1, 0x51, 0xb1, 0xfe, 0x3d, 0xfd, 0x1f, 0x96, 0xdd, 0x8c, 0x97, 0x11, 0xfd, 0x86, 0xd6, 0xc5, 0x81, 0x13}, - {0xf5, 0x5d, 0x68, 0x90, 0xe, 0x2d, 0x83, 0x81, 0xec, 0xcb, 0x81, 0x64, 0xcb, 0x99, 0x76, 0xf2, 0x4b, 0x2d, 0xe0, 0xdd, 0x61, 0xa3, 0x1b, 0x97, 0xce, 0x6e, 0xb2, 0x38, 0x50, 0xd5, 0xe8, 0x19}, - {0xaa, 0xaa, 0x8c, 0x4c, 0xb4, 0xa, 0xac, 0xee, 0x1e, 0x2, 0xdc, 0x65, 0x42, 0x4b, 0x2a, 0x6c, 0x8e, 0x99, 0xf8, 0x3, 0xb7, 0x2f, 0x79, 0x29, 0xc4, 0x10, 0x1d, 0x7f, 0xae, 0x6b, 0xff, 0x32}, -} - -func ForSize(sz abi.UnpaddedPieceSize) cid.Cid { - level := bits.TrailingZeros64(uint64(sz.Padded())) - skip - 5 // 2^5 = 32 - return commcid.DataCommitmentV1ToCID(pieceComms[level][:]) -} diff --git a/lib/zerocomm/zerocomm_test.go b/lib/zerocomm/zerocomm_test.go deleted file mode 100644 index a7f745d6d..000000000 --- a/lib/zerocomm/zerocomm_test.go +++ /dev/null @@ -1,106 +0,0 @@ -package zerocomm - -import ( - "bytes" - "fmt" - "io" - "testing" - - commcid "github.com/filecoin-project/go-fil-commcid" - "github.com/filecoin-project/go-sectorbuilder" - abi "github.com/filecoin-project/specs-actors/actors/abi" - "github.com/ipfs/go-cid" - - "github.com/filecoin-project/lotus/lib/nullreader" -) - -func TestComms(t *testing.T) { - t.Skip("don't have enough ram") // no, but seriously, currently this needs like 3tb of /tmp - - var expPieceComms [levels - skip]cid.Cid - - { - l2, err := sectorbuilder.GeneratePieceCIDFromFile(abi.RegisteredProof_StackedDRG2KiBPoSt, bytes.NewReader(make([]byte, 127)), 127) - if err != nil { - t.Fatal(err) - } - expPieceComms[0] = l2 - } - - for i := 1; i < levels-2; i++ { - var err error - sz := abi.UnpaddedPieceSize(127 << i) - fmt.Println(i, sz) - r := io.LimitReader(&nullreader.Reader{}, int64(sz)) - - expPieceComms[i], err = sectorbuilder.GeneratePieceCIDFromFile(abi.RegisteredProof_StackedDRG2KiBPoSt, r, sz) - if err != nil { - t.Fatal(err) - } - } - - for i, comm := range expPieceComms { - c, err := commcid.CIDToPieceCommitmentV1(comm) - if err != nil { - t.Fatal(err) - } - if string(c) != string(pieceComms[i][:]) { - t.Errorf("zero commitment %d didn't match", i) - } - } - - for _, comm := range expPieceComms { // Could do codegen, but this is good enough - fmt.Printf("%#v,\n", comm) - } -} - -func TestCommsSmall(t *testing.T) { - var expPieceComms [8]cid.Cid - lvls := len(expPieceComms) + skip - - { - l2, err := sectorbuilder.GeneratePieceCIDFromFile(abi.RegisteredProof_StackedDRG2KiBPoSt, bytes.NewReader(make([]byte, 127)), 127) - if err != nil { - t.Fatal(err) - } - expPieceComms[0] = l2 - } - - for i := 1; i < lvls-2; i++ { - var err error - sz := abi.UnpaddedPieceSize(127 << i) - fmt.Println(i, sz) - r := io.LimitReader(&nullreader.Reader{}, int64(sz)) - - expPieceComms[i], err = sectorbuilder.GeneratePieceCIDFromFile(abi.RegisteredProof_StackedDRG2KiBPoSt, r, sz) - if err != nil { - t.Fatal(err) - } - } - - for i, comm := range expPieceComms { - c, err := commcid.CIDToPieceCommitmentV1(comm) - if err != nil { - t.Fatal(err) - } - if string(c) != string(pieceComms[i][:]) { - t.Errorf("zero commitment %d didn't match", i) - } - } - - for _, comm := range expPieceComms { // Could do codegen, but this is good enough - fmt.Printf("%#v,\n", comm) - } -} - -func TestForSise(t *testing.T) { - exp, err := sectorbuilder.GeneratePieceCIDFromFile(abi.RegisteredProof_StackedDRG2KiBPoSt, bytes.NewReader(make([]byte, 1016)), 1016) - if err != nil { - return - } - - actual := ForSize(1016) - if !exp.Equals(actual) { - t.Errorf("zero commitment didn't match") - } -} diff --git a/storage/sbmock/preseal.go b/storage/sbmock/preseal.go index 7d7e1d741..4cc790c66 100644 --- a/storage/sbmock/preseal.go +++ b/storage/sbmock/preseal.go @@ -3,6 +3,7 @@ package sbmock import ( "github.com/filecoin-project/go-address" commcid "github.com/filecoin-project/go-fil-commcid" + "github.com/filecoin-project/go-sectorbuilder" "github.com/filecoin-project/specs-actors/actors/abi" "github.com/filecoin-project/specs-actors/actors/abi/big" "github.com/filecoin-project/specs-actors/actors/builtin/market" @@ -12,7 +13,6 @@ import ( "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/wallet" "github.com/filecoin-project/lotus/genesis" - "github.com/filecoin-project/lotus/lib/zerocomm" ) func PreSeal(ssize abi.SectorSize, maddr address.Address, sectors int) (*genesis.Miner, *types.KeyInfo, error) { @@ -39,7 +39,7 @@ func PreSeal(ssize abi.SectorSize, maddr address.Address, sectors int) (*genesis preseal := &genesis.PreSeal{} preseal.ProofType = st - preseal.CommD = zerocomm.ForSize(abi.PaddedPieceSize(ssize).Unpadded()) + preseal.CommD = sectorbuilder.ZeroPieceCommitment(abi.PaddedPieceSize(ssize).Unpadded()) d, _ := commcid.CIDToPieceCommitmentV1(preseal.CommD) r := commDR(d) preseal.CommR = commcid.ReplicaCommitmentV1ToCID(r[:]) diff --git a/storage/sealing/checks.go b/storage/sealing/checks.go index cbede661f..381be0196 100644 --- a/storage/sealing/checks.go +++ b/storage/sealing/checks.go @@ -4,6 +4,7 @@ import ( "bytes" "context" + "github.com/filecoin-project/go-sectorbuilder" "github.com/ipfs/go-cid" cbg "github.com/whyrusleeping/cbor-gen" "golang.org/x/xerrors" @@ -15,7 +16,6 @@ import ( "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/lib/zerocomm" ) // TODO: For now we handle this by halting state execution, when we get jsonrpc reconnecting @@ -42,7 +42,7 @@ func checkPieces(ctx context.Context, si SectorInfo, api sealingApi) error { for i, piece := range si.Pieces { if piece.DealID == nil { - exp := zerocomm.ForSize(piece.Size) + exp := sectorbuilder.ZeroPieceCommitment(piece.Size) if piece.CommP != exp { return &ErrInvalidPiece{xerrors.Errorf("deal %d piece %d had non-zero CommP %+v", piece.DealID, i, piece.CommP)} } From 79237a9309779082c02a44ab06d9f42551ad0010 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 6 Mar 2020 19:52:38 +0100 Subject: [PATCH 2/4] libs: Remove rlepluslazy (using specs-actors everywhere) --- lib/rlepluslazy/bits.go | 160 ----------------------- lib/rlepluslazy/bits_test.go | 27 ---- lib/rlepluslazy/bitvec.go | 114 ----------------- lib/rlepluslazy/bitvec_test.go | 21 --- lib/rlepluslazy/interface.go | 24 ---- lib/rlepluslazy/rleplus.go | 47 ------- lib/rlepluslazy/rleplus_golden_test.go | 3 - lib/rlepluslazy/rleplus_reader.go | 77 ----------- lib/rlepluslazy/rleplus_test.go | 170 ------------------------- lib/rlepluslazy/rleplus_writer.go | 51 -------- lib/rlepluslazy/runs.go | 120 ----------------- lib/rlepluslazy/runs_test.go | 94 -------------- 12 files changed, 908 deletions(-) delete mode 100644 lib/rlepluslazy/bits.go delete mode 100644 lib/rlepluslazy/bits_test.go delete mode 100644 lib/rlepluslazy/bitvec.go delete mode 100644 lib/rlepluslazy/bitvec_test.go delete mode 100644 lib/rlepluslazy/interface.go delete mode 100644 lib/rlepluslazy/rleplus.go delete mode 100644 lib/rlepluslazy/rleplus_golden_test.go delete mode 100644 lib/rlepluslazy/rleplus_reader.go delete mode 100644 lib/rlepluslazy/rleplus_test.go delete mode 100644 lib/rlepluslazy/rleplus_writer.go delete mode 100644 lib/rlepluslazy/runs.go delete mode 100644 lib/rlepluslazy/runs_test.go diff --git a/lib/rlepluslazy/bits.go b/lib/rlepluslazy/bits.go deleted file mode 100644 index 6cf57e27b..000000000 --- a/lib/rlepluslazy/bits.go +++ /dev/null @@ -1,160 +0,0 @@ -package rlepluslazy - -import ( - "sort" -) - -type it2b struct { - source RunIterator - curIdx uint64 - - run Run -} - -func (it *it2b) HasNext() bool { - return it.run.Valid() -} - -func (it *it2b) Next() (uint64, error) { - it.run.Len-- - res := it.curIdx - it.curIdx++ - return res, it.prep() -} - -func (it *it2b) prep() error { - for !it.run.Valid() && it.source.HasNext() { - var err error - it.run, err = it.source.NextRun() - if err != nil { - return err - } - - if !it.run.Val { - it.curIdx += it.run.Len - it.run.Len = 0 - } - } - return nil -} - -func BitsFromRuns(source RunIterator) (BitIterator, error) { - it := &it2b{source: source} - if err := it.prep(); err != nil { - return nil, err - } - return it, nil -} - -type sliceIt struct { - s []uint64 -} - -func (it sliceIt) HasNext() bool { - return len(it.s) != 0 -} - -func (it *sliceIt) Next() (uint64, error) { - res := it.s[0] - it.s = it.s[1:] - return res, nil -} - -func BitsFromSlice(slice []uint64) BitIterator { - sort.Slice(slice, func(i, j int) bool { return slice[i] < slice[j] }) - return &sliceIt{slice} -} - -type it2r struct { - source BitIterator - - runIdx uint64 - run [2]Run -} - -func (it *it2r) HasNext() bool { - return it.run[0].Valid() -} - -func (it *it2r) NextRun() (Run, error) { - res := it.run[0] - it.runIdx = it.runIdx + res.Len - it.run[0], it.run[1] = it.run[1], Run{} - return res, it.prep() -} - -func (it *it2r) prep() error { - if !it.HasNext() { - return nil - } - if it.run[0].Val == false { - it.run[1].Val = true - it.run[1].Len = 1 - return nil - } - - for it.source.HasNext() && !it.run[1].Valid() { - nB, err := it.source.Next() - if err != nil { - return err - } - - //fmt.Printf("runIdx: %d, run[0].Len: %d, nB: %d\n", it.runIdx, it.run[0].Len, nB) - if it.runIdx+it.run[0].Len == nB { - it.run[0].Len++ - } else { - it.run[1].Len = nB - it.runIdx - it.run[0].Len - it.run[1].Val = false - } - } - return nil -} - -func (it *it2r) init() error { - if it.source.HasNext() { - nB, err := it.source.Next() - if err != nil { - return err - } - it.run[0].Len = nB - it.run[0].Val = false - it.run[1].Len = 1 - it.run[1].Val = true - } - - if !it.run[0].Valid() { - it.run[0], it.run[1] = it.run[1], Run{} - return it.prep() - } - return nil -} - -func SliceFromRuns(source RunIterator) ([]uint64, error) { - rit, err := BitsFromRuns(source) - if err != nil { - return nil, err - } - - res := make([]uint64, 0) - for rit.HasNext() { - bit, err := rit.Next() - if err != nil { - return nil, err - } - res = append(res, bit) - } - return res, nil -} - -func RunsFromBits(source BitIterator) (RunIterator, error) { - it := &it2r{source: source} - - if err := it.init(); err != nil { - return nil, err - } - return it, nil -} - -func RunsFromSlice(slice []uint64) (RunIterator, error) { - return RunsFromBits(BitsFromSlice(slice)) -} diff --git a/lib/rlepluslazy/bits_test.go b/lib/rlepluslazy/bits_test.go deleted file mode 100644 index 96eac25fa..000000000 --- a/lib/rlepluslazy/bits_test.go +++ /dev/null @@ -1,27 +0,0 @@ -package rlepluslazy - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestRunsFromBits(t *testing.T) { - expected := []Run{Run{Val: false, Len: 0x1}, - {Val: true, Len: 0x3}, - {Val: false, Len: 0x2}, - {Val: true, Len: 0x3}, - } - rit, err := RunsFromBits(BitsFromSlice([]uint64{1, 2, 3, 6, 7, 8})) - assert.NoError(t, err) - i := 10 - output := make([]Run, 0, 4) - for rit.HasNext() && i > 0 { - run, err := rit.NextRun() - assert.NoError(t, err) - i-- - output = append(output, run) - } - assert.NotEqual(t, 0, i, "too many iterations") - assert.Equal(t, expected, output) -} diff --git a/lib/rlepluslazy/bitvec.go b/lib/rlepluslazy/bitvec.go deleted file mode 100644 index e442c09c4..000000000 --- a/lib/rlepluslazy/bitvec.go +++ /dev/null @@ -1,114 +0,0 @@ -package rlepluslazy - -type rbitvec struct { - index int - - bits uint16 - bitCap byte - - vec []byte -} - -func readBitvec(vec []byte) *rbitvec { - bv := &rbitvec{ - vec: vec, - index: 1, - bitCap: 8, - } - if len(vec) > 0 { - bv.bits = uint16(bv.vec[0]) - } - return bv -} - -// bitMasks is a mask for selecting N first bits out of a byte -var bitMasks = [9]byte{ - 0x0, - 0x1, - 0x3, - 0x7, - 0xF, - 0x1F, - 0x3F, - 0x7F, - 0xFF, -} - -func (bv *rbitvec) Get(count byte) byte { - res := byte(bv.bits) & bitMasks[count] // select count bits - bv.bits = bv.bits >> count // remove those bits from storage - bv.bitCap = bv.bitCap - count // decrease nuber of stored bits - - if bv.index < len(bv.vec) { // if vector allows - // add bits onto the end of temporary storage - bv.bits = bv.bits | uint16(bv.vec[bv.index])<> 7 // inc == 1 iff bitcap<8 (+10% perf) - bv.index = bv.index + int(inc) // increase index if we need more bits - bv.bitCap = bv.bitCap + inc*8 // increase bitCap by 8 - - return res -} - -func writeBitvec(buf []byte) *wbitvec { - // reslice to 0 length for consistent input but to keep capacity - return &wbitvec{buf: buf[:0]} -} - -type wbitvec struct { - buf []byte // buffer we will be saving to - index int // index of at which the next byte will be saved - - bits uint16 // temporary storage for bits - bitCap byte // number of bits stored in temporary storage -} - -func (bv *wbitvec) Out() []byte { - if bv.bitCap != 0 { - // if there are some bits in temporary storage we need to save them - bv.buf = append(bv.buf, 0)[:bv.index+1] - bv.buf[bv.index] = byte(bv.bits) - } - if bv.bitCap > 8 { - // if we store some needed bits in second byte, save them also - bv.buf = append(bv.buf, byte(bv.bitCap>>8)) - bv.index++ - bv.bits = bv.bits - 8 - } - return bv.buf -} - -func (bv *wbitvec) Put(val byte, count byte) { - // put val into its place in bv.bits - bv.bits = bv.bits | uint16(val)< cap(bv.buf) { - bv.buf = append(bv.buf, 0) - } - bv.buf = bv.buf[:bv.index+1] - // save the bits - bv.buf[bv.index] = byte(bv.bits) - - // Warning, dragons again - // if bitCap is greater than 7 it underflows, same thing as in Put - inc := (7 - bv.bitCap) >> 7 // inc == 1 iff bitcap>=8 - bv.index = bv.index + int(inc) // increase index for the next save - bv.bitCap = bv.bitCap - inc*8 // we store less bits now in temporary buffer - bv.bits = bv.bits >> (inc * 8) // we can discard those bits as they were saved -} diff --git a/lib/rlepluslazy/bitvec_test.go b/lib/rlepluslazy/bitvec_test.go deleted file mode 100644 index 6938bd793..000000000 --- a/lib/rlepluslazy/bitvec_test.go +++ /dev/null @@ -1,21 +0,0 @@ -package rlepluslazy - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestReadBitVec(t *testing.T) { - buf := []byte{0x0, 0xff} - bv := readBitvec(buf) - - o := bv.Get(1) - assert.EqualValues(t, 0, o) - - o = bv.Get(8) - assert.EqualValues(t, 0x80, o) - - o = bv.Get(7) - assert.EqualValues(t, 0x7f, o) -} diff --git a/lib/rlepluslazy/interface.go b/lib/rlepluslazy/interface.go deleted file mode 100644 index e9f45910f..000000000 --- a/lib/rlepluslazy/interface.go +++ /dev/null @@ -1,24 +0,0 @@ -package rlepluslazy - -type Run struct { - Val bool - Len uint64 -} - -func (r Run) Valid() bool { - return r.Len != 0 -} - -type RunIterator interface { - NextRun() (Run, error) - HasNext() bool -} - -type RunIterable interface { - RunIterator() (RunIterator, error) -} - -type BitIterator interface { - Next() (uint64, error) - HasNext() bool -} diff --git a/lib/rlepluslazy/rleplus.go b/lib/rlepluslazy/rleplus.go deleted file mode 100644 index 100007d65..000000000 --- a/lib/rlepluslazy/rleplus.go +++ /dev/null @@ -1,47 +0,0 @@ -package rlepluslazy - -import ( - "errors" - "fmt" - - "golang.org/x/xerrors" -) - -const Version = 0 - -var ( - ErrWrongVersion = errors.New("invalid RLE+ version") - ErrDecode = fmt.Errorf("invalid encoding for RLE+ version %d", Version) -) - -type RLE struct { - buf []byte -} - -func FromBuf(buf []byte) (RLE, error) { - rle := RLE{buf: buf} - - if len(buf) > 0 && buf[0]&3 != Version { - return RLE{}, xerrors.Errorf("could not create RLE+ for a buffer: %w", ErrWrongVersion) - } - - _, err := rle.Count() - if err != nil { - return RLE{}, err - } - - return rle, nil -} - -func (rle *RLE) RunIterator() (RunIterator, error) { - source, err := DecodeRLE(rle.buf) - return source, err -} - -func (rle *RLE) Count() (uint64, error) { - it, err := rle.RunIterator() - if err != nil { - return 0, err - } - return Count(it) -} diff --git a/lib/rlepluslazy/rleplus_golden_test.go b/lib/rlepluslazy/rleplus_golden_test.go deleted file mode 100644 index 091d9826c..000000000 --- a/lib/rlepluslazy/rleplus_golden_test.go +++ /dev/null @@ -1,3 +0,0 @@ -package rlepluslazy - -var goldenRLE = []byte{0x20, 0xfc, 0x40, 0xc2, 0xcc, 0xe5, 0xd8, 0xc1, 0xe1, 0x1e, 0x23, 0xd3, 0x2, 0x2e, 0xcd, 0x3, 0x3e, 0x83, 0x16, 0x26, 0x3a, 0x5c, 0x30, 0x8e, 0x0, 0x5, 0xcc, 0x24, 0xe, 0x96, 0x15, 0x48, 0xa0, 0x2a, 0x40, 0x4, 0x92, 0xe, 0x94, 0xc0, 0x48, 0x91, 0xee, 0x5, 0x28, 0x98, 0x55, 0x90, 0xa0, 0xa4, 0x50, 0x98, 0x14, 0x40, 0x4, 0x59, 0xa, 0x22, 0x0, 0x74, 0xb0, 0x40, 0x66, 0x30, 0xf9, 0x66, 0x90, 0x51, 0xc7, 0x70, 0x74, 0x40, 0x48, 0x7f, 0xf0, 0x80, 0x24, 0x20, 0x85, 0x58, 0x6, 0x7, 0x66, 0x4, 0x87, 0xe5, 0x5, 0x28, 0x0, 0xa4, 0xf0, 0x61, 0x2e, 0x90, 0x8, 0x44, 0x70, 0x38, 0x34, 0xf0, 0x8, 0x4f, 0x70, 0x68, 0xad, 0xd0, 0x90, 0x1e, 0x90, 0x38, 0xc1, 0x85, 0x76, 0x4, 0x15, 0x7c, 0x4, 0x28, 0x28, 0x17, 0x70, 0xe0, 0x15, 0x0, 0x82, 0xfb, 0x11, 0xb, 0x76, 0x9, 0x22, 0xb8, 0x2f, 0x90, 0x20, 0x5f, 0x80, 0x84, 0xc9, 0x10, 0x85, 0x66, 0x9, 0x5, 0xc9, 0x3, 0x2d, 0x19, 0xa4, 0x5a, 0x70, 0x45, 0xa5, 0x40, 0xc2, 0x5, 0x6c, 0x4a, 0xf, 0x64, 0xf4, 0x19, 0xb0, 0x40, 0x28, 0x8b, 0x2, 0x6c, 0x20, 0x3e, 0x90, 0x40, 0x1, 0x19, 0xea, 0x9, 0x20, 0x60, 0x2f, 0x50, 0x60, 0x15, 0x0, 0x4, 0x69, 0x6, 0xb1, 0x8c, 0xa9, 0x85, 0xc6, 0x1f, 0x13, 0x54, 0x3e, 0x58, 0x40, 0x17, 0x60, 0x41, 0x2e, 0x8a, 0x42, 0x3c, 0x8b, 0xb, 0x3f, 0x8, 0x10, 0x5c, 0x32, 0x38, 0xd4, 0x1e, 0x68, 0x18, 0x5b, 0x70, 0xc1, 0x2c, 0x6, 0x17, 0x6c, 0x17, 0xf8, 0x74, 0x36, 0x28, 0x9c, 0xe, 0x20, 0x2, 0xa1, 0x84, 0xf, 0xa2, 0x82, 0xe, 0xbf, 0x82, 0x45, 0x29, 0x84, 0x7, 0x16, 0x25, 0x10, 0x12, 0x38, 0x8c, 0x13, 0x50, 0x41, 0xff, 0x3, 0x13, 0x70, 0x16, 0x80, 0x81, 0x63, 0x85, 0x8f, 0xa0, 0x81, 0xa, 0xe1, 0x80, 0x84, 0xa3, 0x81, 0x89, 0xff, 0x84, 0xc7, 0x3c, 0x87, 0x49, 0xbb, 0x81, 0x4b, 0x21, 0x82, 0x3, 0x2f, 0x83, 0x84, 0x29, 0x83, 0x8c, 0xe0, 0x0, 0x40, 0x8c, 0xda, 0x20, 0x42, 0x6, 0x13, 0x64, 0x7, 0x24, 0x85, 0xc, 0xa6, 0x21, 0x24, 0x6c, 0x7, 0x74, 0x20, 0x13, 0x3c, 0x2, 0x13, 0x54, 0xba, 0x1d, 0x14, 0x9e, 0xb, 0xe0, 0x0, 0x39, 0x82, 0xa, 0x72, 0xb, 0x80, 0x60, 0x53, 0x20, 0xd5, 0xe, 0x32, 0xd5, 0xe, 0x12, 0xb9, 0x1, 0x44, 0x90, 0x27, 0x51, 0x59, 0xe0, 0x12, 0xc3, 0x4, 0xe8, 0x1, 0x78, 0x40, 0xd8, 0xe0, 0x3, 0x7b, 0x60, 0x93, 0x6b, 0x20, 0x72, 0xf8, 0x20, 0xf1, 0xe1, 0xe0, 0x4d, 0x90, 0xdf, 0x2, 0x1f, 0x69, 0x3, 0x36, 0x30, 0x8d, 0x20, 0x82, 0x58, 0xa, 0x8f, 0xec, 0x2, 0x8d, 0xe8, 0x0, 0x11, 0x55, 0x81, 0x58, 0x17, 0xa0, 0x82, 0xa5, 0x81, 0x88, 0xb6, 0x83, 0x88, 0xed, 0x86, 0x88, 0xe0, 0x86, 0x4b, 0x6e, 0x82, 0x45, 0xe1, 0x89, 0xc8, 0x2c, 0x41, 0x78, 0x9, 0x12, 0xbd, 0x91, 0x50, 0x1, 0x42, 0x90, 0x85, 0x70, 0x69, 0xb5, 0x20, 0x83, 0x5e, 0x2, 0x8d, 0xfb, 0x2, 0xf, 0xfb, 0xa, 0x9, 0xcc, 0x4, 0x8b, 0xe4, 0x5, 0x87, 0xc9, 0x2, 0xf, 0xfc, 0x1, 0x5c, 0x8, 0xf, 0xd0, 0x1, 0x4e, 0xd0, 0x60, 0x4d, 0x50, 0xa1, 0x3c, 0xf0, 0x20, 0x6c, 0x88, 0xe1, 0x41, 0x5c, 0x11, 0x1c, 0xc8, 0x25, 0x4c, 0xc0, 0x1d, 0x1c, 0x4e, 0x61, 0x54, 0x2a, 0x5, 0x14, 0xf6, 0x54, 0x3c, 0xb, 0x44, 0xe4, 0x5, 0x64, 0x42, 0x1f, 0x64, 0x32, 0x23, 0xa0, 0x20, 0xf9, 0x1, 0xc, 0x40, 0x27, 0xf8, 0x40, 0x3c, 0xc0, 0x82, 0x72, 0x1, 0xd, 0xd1, 0x43, 0x11, 0x1, 0xd, 0xaa, 0x35, 0xc8, 0xe0, 0x1e, 0x42, 0x41, 0x5b, 0x20, 0x8b, 0x4, 0xe, 0xc9, 0x3, 0xa, 0xea, 0x1, 0x3e, 0x5f, 0x12, 0xab, 0x7, 0x40, 0xf0, 0xad, 0x60, 0x42, 0x6f, 0xa1, 0xd2, 0x39, 0x60, 0x91, 0x64, 0xf1, 0xe4, 0x10, 0x3f, 0x60, 0xc2, 0x5c, 0x94, 0x62, 0x7, 0x1f, 0x67, 0x5, 0x87, 0xcc, 0x7, 0x9f, 0xee, 0xe, 0x13, 0xd8, 0x2, 0x38, 0xa0, 0xb5, 0x20, 0x2, 0x4e, 0x1, 0x85, 0x7c, 0xc, 0x4e, 0xb0, 0xde, 0x90, 0x80, 0x2c, 0x50, 0xd8, 0x77, 0x88, 0xf6, 0x80, 0x4, 0x22, 0x85, 0xce, 0xf2, 0x82, 0xcb, 0xb4, 0x5, 0x14, 0x68, 0xa9, 0x70, 0x77, 0xb8, 0x34, 0x1b, 0xf0, 0x1, 0xec, 0x1, 0x13, 0x58, 0x1a, 0x78, 0x88, 0xb, 0x58, 0x80, 0x27, 0x98, 0x20, 0x4b, 0x50, 0x41, 0x69, 0x84, 0xc9, 0x63, 0x81, 0x8, 0x7b, 0x6, 0x10, 0xd8, 0x1e, 0x38, 0x1c, 0x17, 0x68, 0x0, 0x2e, 0xa8, 0xe4, 0x23, 0x78, 0xdc, 0xa, 0x28, 0x38, 0xb, 0x28, 0xcc, 0x1f, 0x40, 0x1, 0x6f, 0x83, 0x88, 0x75, 0x82, 0xc7, 0x8c, 0x4, 0xec, 0x8, 0x28, 0xcc, 0x7b, 0xe4, 0xb8, 0x0, 0xe, 0x46, 0x35, 0x2c, 0xc2, 0x2d, 0xb8, 0xe0, 0xb5, 0x42, 0xa2, 0x75, 0x40, 0x2, 0xfc, 0xc4, 0x43, 0x98, 0x41, 0xa1, 0x56, 0x40, 0xc4, 0x5a, 0x1, 0x13, 0xda, 0x33, 0x5c, 0xf2, 0x7, 0x44, 0x74, 0x3f, 0x5c, 0x5a, 0x49, 0x24, 0x64, 0x3, 0x2c, 0xd8, 0x23, 0x34, 0x78, 0x19, 0xb8, 0x61, 0x15, 0x42, 0xa3, 0xc6, 0x46, 0x73, 0x80, 0xa, 0x52, 0x17, 0x0, 0xa1, 0x7d, 0x40, 0x26, 0x19, 0xc1, 0x6, 0xf0, 0x40, 0x45, 0x90, 0xc2, 0x81, 0xf6, 0xc0, 0x22, 0xf3, 0x80, 0x15, 0x2, 0x7, 0x2c, 0x9a, 0x3, 0xd8, 0xc0, 0x95, 0x21, 0xbb, 0x4, 0x80, 0x0, 0x99, 0xe0, 0x90, 0x19, 0x21, 0x93, 0xfe, 0x21, 0x43, 0x1a, 0x90, 0x64, 0x2, 0x2e, 0x20, 0x65, 0xc8, 0x79, 0x86, 0xc4, 0x78, 0x4, 0x2a, 0x38, 0x9e, 0x50, 0x82, 0xaf, 0x5, 0x17, 0x50, 0x33, 0xa0, 0x42, 0x21, 0x5, 0x24, 0xc0, 0xf, 0x64, 0xb0, 0x40, 0xe5, 0x3a, 0x42, 0xc1, 0xfd, 0x2, 0x3c, 0x10, 0x9, 0x88, 0x40, 0x5a, 0x40, 0x84, 0x11, 0x41, 0x83, 0x7f, 0x42, 0x82, 0x10, 0x3, 0x8, 0x8a, 0x1b, 0xc8, 0xc0, 0xb2, 0x81, 0x16, 0x6e, 0x68, 0x21, 0x1a, 0x2, 0x11, 0x30, 0x0, 0x42, 0x71, 0xc1, 0xe7, 0x89, 0x18, 0x60, 0x5, 0x3c, 0x70, 0x3c, 0xa2, 0x35, 0x14, 0xe6, 0x25, 0x98, 0x20, 0x97, 0x20, 0xa8, 0x2, 0x26, 0x8e, 0x12, 0x6c, 0x30, 0x2a, 0xa0, 0x22, 0x5e, 0xc2, 0x5, 0x8f, 0x8, 0xa, 0xea, 0x4, 0xa, 0x9b, 0x4, 0x3e, 0xf4, 0x3, 0x2e, 0x92, 0x5, 0x26, 0x65, 0xe0, 0x28, 0xe, 0x38, 0x50, 0xb0, 0x2f, 0x61, 0xe1, 0x6a, 0xa0, 0xf3, 0x2c, 0x12, 0x49, 0x6, 0x26, 0x20, 0x84, 0x30, 0x51, 0x5d, 0x90, 0xb0, 0x57, 0x70, 0x11, 0x3d, 0x70, 0x10, 0x53, 0x1c, 0x98, 0xf8, 0x54, 0x0, 0x2, 0xc3, 0x10, 0x58, 0xb0, 0x4d, 0xb0, 0x20, 0xd, 0x71, 0x40, 0x4d, 0x20, 0x4, 0x79, 0x5, 0x1f, 0x7b, 0x1, 0x89, 0x69, 0x2, 0x28, 0xc8, 0x17, 0x30, 0x81, 0x15, 0xd0, 0xc0, 0x5f, 0x50, 0x19, 0x1e, 0x72, 0x4d, 0x8d, 0xc1, 0x1, 0x91, 0x7e, 0x5, 0x34, 0x30, 0xe, 0x91, 0xc9, 0x74, 0x20, 0x84, 0xd5, 0x13, 0xb, 0x61, 0xd, 0x8f, 0xd8, 0x0, 0x32, 0x88, 0x4e, 0x20, 0x4, 0xd7, 0x84, 0xbc, 0x6, 0x38, 0x6c, 0x1f, 0xc8, 0xe0, 0x12, 0x68, 0xb4, 0x16, 0xa0, 0x41, 0x3a, 0x7, 0x27, 0x90, 0x1a, 0x88, 0x18, 0xa, 0xa5, 0x71, 0x25, 0xf1, 0x9, 0xa8, 0x50, 0x9d, 0xe1, 0x82, 0xbc, 0x81, 0x4, 0xc9, 0x9, 0x7c, 0x10, 0x3e, 0x40, 0x5, 0xe9, 0x2, 0x3a, 0xce, 0xd, 0x5c, 0x70, 0xfa, 0x60, 0x91, 0x41, 0x9, 0xc7, 0x8, 0xa, 0x93, 0x3, 0x7c, 0xa0, 0x3f, 0xe0, 0x53, 0x4b, 0x40, 0x6, 0xea, 0xd, 0xa, 0xcd, 0x18, 0xa, 0x83, 0x5, 0x2a, 0x29, 0xe, 0xc3, 0x1, 0x54, 0xb0, 0x7c, 0x41, 0x4, 0xaf, 0x2, 0x54, 0x40, 0xbe, 0x20, 0x41, 0xa7, 0x21, 0x29, 0xe1, 0x62, 0x6a, 0x81, 0x8, 0xb8, 0x4, 0x64, 0xb0, 0x29, 0xa0, 0xb0, 0x7f, 0xe0, 0x0, 0x3a, 0xc0, 0x5, 0xcb, 0x2, 0xb8, 0xe0, 0x6e, 0xc0, 0x8, 0xda, 0x13, 0x78, 0xe0, 0x1a, 0x0, 0x4, 0xe2, 0x8, 0xa, 0x1b, 0x16, 0x9d, 0x35, 0x88, 0x0, 0xa, 0x61, 0xe3, 0x1d, 0x61, 0xa1, 0x8f, 0xe0, 0xe3, 0x7d, 0x0, 0x8, 0x56, 0x16, 0xb9, 0x5, 0x7c, 0x50, 0x5a, 0xe2, 0xd1, 0xc9, 0x40, 0x8, 0x93, 0xa, 0x36, 0xc4, 0x2, 0xe, 0xf7, 0x15, 0xa, 0xae, 0x1b, 0xa, 0x83, 0xd, 0x36, 0xe8, 0xa, 0x16, 0xde, 0x11, 0x3e, 0x89, 0xb, 0x50, 0x70, 0xd9, 0x10, 0xca, 0x0, 0x38, 0xc0, 0x8f, 0xc1, 0x83, 0x77, 0x86, 0xc, 0x1b, 0xe8, 0x0, 0x9e, 0x78, 0x8, 0xb, 0xa0, 0xc3, 0xb7, 0x84, 0x82, 0x18, 0x4b, 0x54, 0x3b, 0x70, 0x81, 0xbe, 0x41, 0xcc, 0xd, 0x8, 0x61, 0x8e, 0x17, 0x0, 0xd, 0xd8, 0x40, 0x34, 0x20, 0xbd, 0x12, 0xe, 0x83, 0x16, 0x3a, 0x8a, 0x16, 0x48, 0xd0, 0x59, 0xe0, 0x21, 0x4c, 0x60, 0x42, 0xaa, 0xa0, 0x60, 0x69, 0x0, 0x5, 0xb9, 0x2, 0x12, 0x94, 0xf, 0x48, 0x60, 0xfe, 0x90, 0x66, 0x3, 0x4e, 0xf8, 0x4c, 0x50, 0xd8, 0x5f, 0x70, 0x99, 0x1f, 0xd0, 0xf0, 0x55, 0x88, 0x74, 0x81, 0xd, 0x6b, 0x80, 0x84, 0xb9, 0x8d, 0xc7, 0x34, 0x81, 0x4e, 0xa4, 0x8a, 0x82, 0xa1, 0x2, 0x15, 0x40, 0x7, 0x70, 0x81, 0x7f, 0x8d, 0x3, 0x61, 0x2, 0x11, 0x1c, 0x1e, 0x78, 0xb8, 0x1a, 0x80, 0x2, 0xa3, 0x80, 0x2, 0x69, 0x80, 0xc7, 0xf3, 0x1, 0x2c, 0xdc, 0x13, 0x90, 0x81, 0xad, 0x81, 0x88, 0xb8, 0x81, 0x82, 0x78, 0x84, 0x4e, 0xb6, 0x83, 0x45, 0xb3, 0x6, 0x37, 0x90, 0x5e, 0x68, 0x48, 0x82, 0x28, 0x0, 0xc1, 0x1, 0xe5, 0x80, 0x86, 0xf2, 0x81, 0xcc, 0xb9, 0x81, 0x88, 0xeb, 0x80, 0x3, 0xb7, 0x87, 0x8d, 0x69, 0x83, 0x45, 0x74, 0x5, 0x22, 0x84, 0x43, 0x48, 0x48, 0x9f, 0xa4, 0xf3, 0x40, 0xa4, 0x78, 0x80, 0xb, 0xec, 0xb, 0x2c, 0x8, 0x7, 0x74, 0x88, 0x15, 0x34, 0xc2, 0xa2, 0xa1, 0x33, 0xd, 0xc1, 0x8, 0x91, 0x18, 0xf0, 0x60, 0x38, 0x40, 0x7, 0x83, 0x1, 0x8c, 0x10, 0x19, 0x40, 0x4, 0xbb, 0x1a, 0x68, 0x30, 0x2b, 0x60, 0x92, 0xaa, 0x80, 0x8, 0xef, 0x2, 0x54, 0x50, 0x7f, 0x80, 0xa, 0x89, 0x2, 0x32, 0x94, 0x17, 0x1a, 0xe1, 0x4, 0xe, 0xa6, 0x1, 0xa, 0x97, 0xb, 0x16, 0xe4, 0xa, 0x91, 0x3c, 0x90, 0xf9, 0x1e, 0x10, 0xf1, 0x25, 0x70, 0x29, 0x1d, 0xa0, 0x82, 0xf0, 0x4, 0x40, 0xe8, 0x56, 0x40, 0x5, 0x54, 0xf, 0x1b, 0xc6, 0x1, 0x2a, 0x98, 0x35, 0xf0, 0x30, 0xbd, 0x80, 0x7, 0xd9, 0x6, 0x20, 0x90, 0x5e, 0x9, 0xf9, 0xb, 0x1c, 0xac, 0x22, 0x20, 0xc1, 0x96, 0x4f, 0x70, 0x80, 0x4b, 0xae, 0x81, 0x47, 0xb9, 0x8d, 0x8d, 0x3d, 0x82, 0x45, 0x65, 0x80, 0xca, 0xb9, 0x5, 0x18, 0x6c, 0x26, 0xa4, 0x53, 0x40, 0xc7, 0xb2, 0xc2, 0x6, 0xdf, 0x80, 0xa, 0xa0, 0x29, 0x30, 0x23, 0x32, 0x81, 0x8, 0x7c, 0x3c, 0x3e, 0x37, 0xb8, 0x20, 0xd7, 0x80, 0xb, 0xfc, 0x25, 0x24, 0xe8, 0x1d, 0x2c, 0x50, 0xd, 0x3c, 0xe0, 0x88, 0x60, 0x3a, 0x83, 0xc, 0xa6, 0x11, 0xb2, 0x5c, 0xc1, 0x7, 0xf2, 0x1, 0x2a, 0xce, 0x1, 0x12, 0xd7, 0x64, 0xe, 0xb2, 0x7, 0xa9, 0x6e, 0xd0, 0x71, 0x1d, 0x70, 0x80, 0xe5, 0x8, 0x6d, 0x83, 0x84, 0x6a, 0x0, 0x19, 0x78, 0xe, 0x48, 0xc4, 0x7, 0x80, 0x81, 0xae, 0x80, 0x44, 0x3f, 0x1, 0x13, 0x4c, 0x1f, 0x58, 0xe0, 0x4a, 0x48, 0x8c, 0x2e, 0x4, 0x78, 0x81, 0x16, 0x7a, 0x5, 0x90, 0xa0, 0xdd, 0xc3, 0xa7, 0xd8, 0xc2, 0x44, 0xfc, 0x40, 0x2, 0xda, 0x0, 0xc, 0x54, 0x11, 0xe0, 0xc0, 0xba, 0x80, 0x9, 0x7a, 0x9, 0xa0, 0x0, 0x52, 0x40, 0x61, 0x51, 0x44, 0x67, 0x12, 0x3, 0xc, 0x0, 0xf, 0x70, 0xe1, 0x31, 0xc0, 0xe1, 0xb6, 0xc2, 0xa1, 0xb9, 0xc1, 0x46, 0x50, 0x40, 0x1, 0x9b, 0x1, 0x1a, 0xc2, 0x11, 0x72, 0x4c, 0xa1, 0x2, 0x5b, 0x20, 0xb1, 0x28, 0xc0, 0x4, 0xc7, 0x12, 0x40, 0x0, 0xee, 0x0, 0x8, 0xd1, 0x2a, 0x32, 0xd0, 0x11, 0x81, 0x7f, 0xa0, 0x84, 0x77, 0xa, 0x93, 0xc8, 0x0, 0x97, 0xe7, 0x1, 0x9, 0x50, 0x1, 0x5, 0xf5, 0x0, 0x85, 0xee, 0x3, 0x93, 0x59, 0x82, 0x3c, 0x59, 0x14, 0x52, 0x28, 0xa4, 0xb, 0x28, 0x30, 0xa, 0x84, 0x4d, 0x81, 0x7f, 0x43, 0xa7, 0x56, 0x80, 0xb, 0x92, 0xf, 0x3c, 0x98, 0x47, 0xa8, 0x0, 0x35, 0x20, 0xde, 0x3, 0x2a, 0x8b, 0x2, 0x2e, 0x8f, 0x6, 0x58, 0x40, 0x58, 0xe0, 0x33, 0x4d, 0xc1, 0xa, 0xc0, 0x6, 0xc0, 0xc0, 0x4, 0x5, 0xf5, 0x4, 0x16, 0xfa, 0x26, 0x60, 0x60, 0x9c, 0xa0, 0x63, 0x2f, 0x20, 0x13, 0xbe, 0x81, 0xc, 0x97, 0xd, 0x32, 0xd0, 0x1, 0x22, 0x3e, 0x80, 0x20, 0x49, 0xe0, 0x50, 0x18, 0x21, 0x2, 0x19, 0xe0, 0x20, 0xbd, 0x10, 0xc8, 0x4, 0x9b, 0x40, 0xd, 0x24, 0xe0, 0xcd, 0x90, 0xe9, 0x3, 0x82, 0x16, 0x11, 0x28, 0x2a, 0xd0, 0xb5, 0x0, 0x83, 0x73, 0xb, 0x19, 0xd6, 0x7, 0x1b, 0x71, 0x5, 0x26, 0x28, 0x6d, 0x0, 0x82, 0x7f, 0x5, 0x2a, 0xb8, 0x56, 0x80, 0x84, 0x79, 0x4, 0x86, 0xa0, 0x7e, 0x50, 0x20, 0xe3, 0x3, 0x20, 0x36, 0x88, 0xaf, 0x90, 0x91, 0xd5, 0x60, 0x82, 0xe4, 0x1, 0x93, 0xf4, 0x8, 0x95, 0xc0, 0x5, 0x1b, 0x61, 0x6, 0x87, 0xc3, 0x1, 0x47, 0xb5, 0x41, 0x51, 0xc9, 0x26, 0x0, 0x3, 0xc1, 0x0, 0x87, 0x46, 0x5, 0x8f, 0x5a, 0x1, 0x95, 0xef, 0x9, 0x89, 0xca, 0x0, 0x8b, 0xf7, 0x2, 0x19, 0x5b, 0xb, 0x32, 0x8, 0x82, 0x2, 0xec, 0x0, 0x13, 0x23, 0x11, 0x40, 0x4, 0x20, 0xe0, 0x26, 0x51, 0x89, 0x85, 0x40, 0x84, 0xc2, 0x1, 0xf, 0x51, 0x1, 0xd, 0x8, 0x6e, 0xf8, 0x6e, 0x0, 0x82, 0x6a, 0x1, 0xf, 0x72, 0x4, 0x2c, 0x40, 0x4d, 0x50, 0x41, 0x16, 0x8, 0x67, 0x46, 0x4a, 0xd, 0x90, 0xc0, 0xfe, 0xc1, 0x42, 0x30, 0x1, 0x9, 0x50, 0xd, 0x5c, 0x16, 0x9, 0xc0, 0x0, 0xf6, 0x41, 0x27, 0x92, 0xc0, 0x63, 0x7a, 0xc2, 0xa1, 0x9c, 0x40, 0x21, 0x27, 0xec, 0x25, 0x4c, 0x80, 0x1b, 0x20, 0x81, 0x2b, 0xa0, 0x0, 0x18, 0xe0, 0xe1, 0x6a, 0x21, 0x2, 0xa3, 0xd2, 0xaf, 0xe0, 0x3, 0x7a, 0x0, 0x4, 0x99, 0x26, 0x50, 0x70, 0xcf, 0xa0, 0xe1, 0x58, 0xa1, 0xd3, 0x38, 0xa0, 0x30, 0x79, 0x20, 0xd2, 0x3a, 0x20, 0xb3, 0x5e, 0x60, 0xa1, 0xeb, 0x80, 0x5, 0xc8, 0x1, 0x4c, 0xc0, 0x5b, 0xa0, 0x60, 0xc8, 0x41, 0x4, 0x6d, 0xc8, 0x20, 0xab, 0x10, 0xe3, 0x7, 0x1f, 0x4b, 0x7, 0x28, 0x70, 0x17, 0x80, 0x84, 0xf9, 0x6, 0x46, 0x90, 0x44, 0xf0, 0x40, 0xf6, 0x60, 0x5, 0xca, 0x7, 0x1b, 0xcf, 0x2, 0xb, 0x61, 0x12, 0x15} diff --git a/lib/rlepluslazy/rleplus_reader.go b/lib/rlepluslazy/rleplus_reader.go deleted file mode 100644 index 0b3a77da0..000000000 --- a/lib/rlepluslazy/rleplus_reader.go +++ /dev/null @@ -1,77 +0,0 @@ -package rlepluslazy - -import ( - "github.com/multiformats/go-varint" - "golang.org/x/xerrors" -) - -func DecodeRLE(buf []byte) (RunIterator, error) { - bv := readBitvec(buf) - - ver := bv.Get(2) // Read version - if ver != Version { - return nil, ErrWrongVersion - } - - it := &rleIterator{bv: bv} - - // next run is previous in relation to prep - // so we invert the value - it.nextRun.Val = bv.Get(1) != 1 - if err := it.prep(); err != nil { - return nil, err - } - return it, nil -} - -type rleIterator struct { - bv *rbitvec - - nextRun Run -} - -func (it *rleIterator) HasNext() bool { - return it.nextRun.Valid() -} - -func (it *rleIterator) NextRun() (Run, error) { - ret := it.nextRun - return ret, it.prep() -} - -func (it *rleIterator) prep() error { - x := it.bv.Get(1) - - switch x { - case 1: - it.nextRun.Len = 1 - - case 0: - y := it.bv.Get(1) - switch y { - case 1: - it.nextRun.Len = uint64(it.bv.Get(4)) - case 0: - var buf = make([]byte, 0, 10) - for { - b := it.bv.Get(8) - buf = append(buf, b) - if b&0x80 == 0 { - break - } - if len(buf) > 10 { - return xerrors.Errorf("run too long: %w", ErrDecode) - } - } - var err error - it.nextRun.Len, _, err = varint.FromUvarint(buf) - if err != nil { - return err - } - - } - } - - it.nextRun.Val = !it.nextRun.Val - return nil -} diff --git a/lib/rlepluslazy/rleplus_test.go b/lib/rlepluslazy/rleplus_test.go deleted file mode 100644 index 3b47d02c2..000000000 --- a/lib/rlepluslazy/rleplus_test.go +++ /dev/null @@ -1,170 +0,0 @@ -package rlepluslazy - -import ( - "math/rand" - "testing" - - "github.com/filecoin-project/lotus/extern/rleplus" - "github.com/stretchr/testify/assert" -) - -func TestDecode(t *testing.T) { - // Encoding bitvec![LittleEndian; 1, 0, 1, 0, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] - // in the Rust reference implementation gives an encoding of [223, 145, 136, 0] (without version field) - // The bit vector is equivalent to the integer set { 0, 2, 4, 5, 6, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27 } - - // This is the above reference output with a version header "00" manually added - referenceEncoding := []byte{124, 71, 34, 2} - - expectedNumbers := []uint64{0, 2, 4, 5, 6, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27} - - runs, err := RunsFromBits(BitsFromSlice(expectedNumbers)) - assert.NoError(t, err) - encoded, err := EncodeRuns(runs, []byte{}) - assert.NoError(t, err) - - // Our encoded bytes are the same as the ref bytes - assert.Equal(t, len(referenceEncoding), len(encoded)) - assert.Equal(t, referenceEncoding, encoded) - - rle, err := FromBuf(encoded) - assert.NoError(t, err) - decoded := make([]uint64, 0, len(expectedNumbers)) - - rit, err := rle.RunIterator() - assert.NoError(t, err) - - it, err := BitsFromRuns(rit) - assert.NoError(t, err) - for it.HasNext() { - bit, err := it.Next() - assert.NoError(t, err) - decoded = append(decoded, bit) - } - - // Our decoded integers are the same as expected - assert.Equal(t, expectedNumbers, decoded) -} - -func TestGoldenGen(t *testing.T) { - t.SkipNow() - N := 10000 - mod := uint32(1) << 20 - runExProp := float32(0.93) - - bits := make([]uint64, N) - - for i := 0; i < N; i++ { - x := rand.Uint32() % mod - bits[i] = uint64(x) - for rand.Float32() < runExProp && i+1 < N { - i++ - x = (x + 1) % mod - bits[i] = uint64(x) - } - } - - out, _, err := rleplus.Encode(bits) - assert.NoError(t, err) - t.Logf("%#v", out) - _, runs := rleplus.RunLengths(bits) - t.Logf("runs: %v", runs) - t.Logf("len: %d", len(out)) -} - -func TestGolden(t *testing.T) { - expected, _ := rleplus.Decode(goldenRLE) - res := make([]uint64, 0, len(expected)) - - rle, err := FromBuf(goldenRLE) - assert.NoError(t, err) - rit, err := rle.RunIterator() - assert.NoError(t, err) - it, err := BitsFromRuns(rit) - assert.NoError(t, err) - for it.HasNext() { - bit, err := it.Next() - assert.NoError(t, err) - res = append(res, bit) - } - assert.Equal(t, expected, res) -} - -func TestGoldenLoop(t *testing.T) { - rle, err := FromBuf(goldenRLE) - assert.NoError(t, err) - - rit, err := rle.RunIterator() - assert.NoError(t, err) - - buf, err := EncodeRuns(rit, nil) - assert.NoError(t, err) - - assert.Equal(t, goldenRLE, buf) -} - -var Res uint64 = 0 - -func BenchmarkRunIterator(b *testing.B) { - b.ReportAllocs() - var r uint64 - for i := 0; i < b.N; i++ { - rle, _ := FromBuf(goldenRLE) - rit, _ := rle.RunIterator() - for rit.HasNext() { - run, _ := rit.NextRun() - if run.Val { - r = r + run.Len - } - } - } - Res = Res + r -} - -func BenchmarkRunsToBits(b *testing.B) { - b.ReportAllocs() - var r uint64 - for i := 0; i < b.N; i++ { - rle, _ := FromBuf(goldenRLE) - rit, _ := rle.RunIterator() - it, _ := BitsFromRuns(rit) - for it.HasNext() { - bit, _ := it.Next() - if bit < 1<<63 { - r++ - } - } - } - Res = Res + r -} - -func BenchmarkOldRLE(b *testing.B) { - b.ReportAllocs() - var r uint64 - for i := 0; i < b.N; i++ { - rle, _ := rleplus.Decode(goldenRLE) - r = r + uint64(len(rle)) - } - Res = Res + r -} - -func BenchmarkDecodeEncode(b *testing.B) { - b.ReportAllocs() - var r uint64 - out := make([]byte, 0, len(goldenRLE)) - for i := 0; i < b.N; i++ { - rle, _ := FromBuf(goldenRLE) - rit, _ := rle.RunIterator() - out, _ = EncodeRuns(rit, out) - r = r + uint64(len(out)) - } - - /* - for i := 0; i < b.N; i++ { - rle, _ := rleplus.Decode(goldenRLE) - out, _, _ := rleplus.Encode(rle) - r = r + uint64(len(out)) - } - */ - Res = Res + r -} diff --git a/lib/rlepluslazy/rleplus_writer.go b/lib/rlepluslazy/rleplus_writer.go deleted file mode 100644 index 306da18d3..000000000 --- a/lib/rlepluslazy/rleplus_writer.go +++ /dev/null @@ -1,51 +0,0 @@ -package rlepluslazy - -import ( - "encoding/binary" -) - -func EncodeRuns(rit RunIterator, buf []byte) ([]byte, error) { - bv := writeBitvec(buf) - bv.Put(0, 2) - - first := true - varBuf := make([]byte, binary.MaxVarintLen64) - - for rit.HasNext() { - run, err := rit.NextRun() - if err != nil { - return nil, err - } - - if first { - if run.Val { - bv.Put(1, 1) - } else { - bv.Put(0, 1) - } - first = false - } - - switch { - case run.Len == 1: - bv.Put(1, 1) - case run.Len < 16: - bv.Put(2, 2) - bv.Put(byte(run.Len), 4) - case run.Len >= 16: - bv.Put(0, 2) - numBytes := binary.PutUvarint(varBuf, run.Len) - for i := 0; i < numBytes; i++ { - bv.Put(varBuf[i], 8) - } - } - - } - - if first { - bv.Put(0, 1) - } - - return bv.Out(), nil - -} diff --git a/lib/rlepluslazy/runs.go b/lib/rlepluslazy/runs.go deleted file mode 100644 index 034a449c9..000000000 --- a/lib/rlepluslazy/runs.go +++ /dev/null @@ -1,120 +0,0 @@ -package rlepluslazy - -import ( - "math" - - "golang.org/x/xerrors" -) - -func Sum(a, b RunIterator) (RunIterator, error) { - it := addIt{a: a, b: b} - it.prep() - return &it, nil -} - -type addIt struct { - a RunIterator - b RunIterator - - next Run - - arun Run - brun Run -} - -func (it *addIt) prep() error { - var err error - - fetch := func() error { - if !it.arun.Valid() && it.a.HasNext() { - it.arun, err = it.a.NextRun() - if err != nil { - return err - } - } - - if !it.brun.Valid() && it.b.HasNext() { - it.brun, err = it.b.NextRun() - if err != nil { - return err - } - } - return nil - } - - if err := fetch(); err != nil { - return err - } - - // one is not valid - if !it.arun.Valid() { - it.next = it.brun - it.brun.Len = 0 - return nil - } - - if !it.brun.Valid() { - it.next = it.arun - it.arun.Len = 0 - return nil - } - - if !(it.arun.Val || it.brun.Val) { - min := it.arun.Len - if it.brun.Len < min { - min = it.brun.Len - } - it.next = Run{Val: it.arun.Val, Len: min} - it.arun.Len -= it.next.Len - it.brun.Len -= it.next.Len - return nil - } - - it.next = Run{Val: true} - // different vals, 'true' wins - for (it.arun.Val && it.arun.Valid()) || (it.brun.Val && it.brun.Valid()) { - min := it.arun.Len - if it.brun.Len < min && it.brun.Valid() || !it.arun.Valid() { - min = it.brun.Len - } - it.next.Len += min - if it.arun.Valid() { - it.arun.Len -= min - } - if it.brun.Valid() { - it.brun.Len -= min - } - if err := fetch(); err != nil { - return err - } - } - - return nil -} - -func (it *addIt) HasNext() bool { - return it.next.Valid() -} - -func (it *addIt) NextRun() (Run, error) { - next := it.next - return next, it.prep() -} - -func Count(ri RunIterator) (uint64, error) { - var count uint64 - - for ri.HasNext() { - r, err := ri.NextRun() - if err != nil { - return 0, err - } - if r.Val { - if math.MaxUint64-r.Len < count { - return 0, xerrors.New("RLE+ overflows") - } - count += r.Len - } - } - return count, nil -} diff --git a/lib/rlepluslazy/runs_test.go b/lib/rlepluslazy/runs_test.go deleted file mode 100644 index aefeb2c70..000000000 --- a/lib/rlepluslazy/runs_test.go +++ /dev/null @@ -1,94 +0,0 @@ -package rlepluslazy - -import ( - "math/rand" - "sort" - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestSumRuns(t *testing.T) { - { - a, err := RunsFromSlice([]uint64{0, 1, 2, 3, 4, 5, 6, 7, 8, 11, 12, 13, 14}) - assert.NoError(t, err) - b, err := RunsFromSlice([]uint64{0, 1, 2, 3, 9, 10, 16, 17, 18, 50, 51, 70}) - assert.NoError(t, err) - - s, err := Sum(a, b) - assert.NoError(t, err) - bis, err := SliceFromRuns(s) - assert.Equal(t, []uint64{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 17, 18, 50, 51, 70}, bis) - assert.NoError(t, err) - } - - { - a, err := RunsFromSlice([]uint64{0, 1, 2, 3, 4, 5, 6, 7, 8, 11, 12, 13, 14}) - assert.NoError(t, err) - b, err := RunsFromSlice([]uint64{0, 1, 2, 3, 9, 10, 16, 17, 18, 50, 51, 70}) - assert.NoError(t, err) - - s, err := Sum(b, a) - assert.NoError(t, err) - bis, err := SliceFromRuns(s) - assert.Equal(t, []uint64{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 17, 18, 50, 51, 70}, bis) - assert.NoError(t, err) - } -} - -func randomBits(N int, max uint64) []uint64 { - all := make(map[uint64]struct{}) - for len(all) <= N { - x := rand.Uint64() % max - if _, has := all[x]; has { - continue - } - all[x] = struct{}{} - } - - res := make([]uint64, 0, N) - for x := range all { - res = append(res, x) - } - sort.Slice(res, func(i, j int) bool { return res[i] < res[j] }) - - return res -} - -func sum(a, b []uint64) []uint64 { - all := make(map[uint64]struct{}) - for _, x := range a { - all[x] = struct{}{} - } - for _, x := range b { - all[x] = struct{}{} - } - res := make([]uint64, 0, len(all)) - for x := range all { - res = append(res, x) - } - sort.Slice(res, func(i, j int) bool { return res[i] < res[j] }) - - return res -} - -func TestSumRandom(t *testing.T) { - N := 100 - - for i := 0; i < N; i++ { - abits := randomBits(1000, 1500) - bbits := randomBits(1000, 1500) - sumbits := sum(abits, bbits) - - a, err := RunsFromSlice(abits) - assert.NoError(t, err) - b, err := RunsFromSlice(bbits) - assert.NoError(t, err) - - s, err := Sum(b, a) - assert.NoError(t, err) - bis, err := SliceFromRuns(s) - assert.NoError(t, err) - assert.Equal(t, sumbits, bis) - } -} From 3d1a5f4bf3edd4c158da3b2f1eaa700eb0f2d5f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 6 Mar 2020 19:59:08 +0100 Subject: [PATCH 3/4] libs: Use go-statemachine --- gen/main.go | 10 -- go.mod | 1 + lib/statemachine/cbor_gen.go | 241 ------------------------------- lib/statemachine/context.go | 16 -- lib/statemachine/group.go | 116 --------------- lib/statemachine/machine.go | 127 ---------------- lib/statemachine/machine_test.go | 105 -------------- lib/statemachine/testing.go | 11 -- storage/sealing/fsm.go | 11 +- storage/sealing/fsm_test.go | 3 +- storage/sealing/sealing.go | 2 +- storage/sealing/states.go | 2 +- storage/sealing/states_failed.go | 2 +- 13 files changed, 12 insertions(+), 635 deletions(-) delete mode 100644 lib/statemachine/cbor_gen.go delete mode 100644 lib/statemachine/context.go delete mode 100644 lib/statemachine/group.go delete mode 100644 lib/statemachine/machine.go delete mode 100644 lib/statemachine/machine_test.go delete mode 100644 lib/statemachine/testing.go diff --git a/gen/main.go b/gen/main.go index 03b82ad5b..6e464e79a 100644 --- a/gen/main.go +++ b/gen/main.go @@ -9,7 +9,6 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/blocksync" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/lib/statemachine" "github.com/filecoin-project/lotus/node/hello" "github.com/filecoin-project/lotus/paychmgr" "github.com/filecoin-project/lotus/storage/sealing" @@ -83,13 +82,4 @@ func main() { fmt.Println(err) os.Exit(1) } - - err = gen.WriteMapEncodersToFile("./lib/statemachine/cbor_gen.go", "statemachine", - statemachine.TestState{}, - statemachine.TestEvent{}, - ) - if err != nil { - fmt.Printf("%+v\n", err) - os.Exit(1) - } } diff --git a/go.mod b/go.mod index c22687f74..1e7953a48 100644 --- a/go.mod +++ b/go.mod @@ -22,6 +22,7 @@ require ( github.com/filecoin-project/go-padreader v0.0.0-20200210211231-548257017ca6 github.com/filecoin-project/go-paramfetch v0.0.2-0.20200218225740-47c639bab663 github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200306043753-5cdbe369b47d + github.com/filecoin-project/go-statemachine v0.0.0-20200226041606-2074af6d51d9 github.com/filecoin-project/go-statestore v0.1.0 github.com/filecoin-project/specs-actors v0.0.0-20200306043603-709a3ce21094 github.com/filecoin-project/specs-storage v0.0.0-20200303233430-1a5a408f7513 diff --git a/lib/statemachine/cbor_gen.go b/lib/statemachine/cbor_gen.go deleted file mode 100644 index 4bebe01d0..000000000 --- a/lib/statemachine/cbor_gen.go +++ /dev/null @@ -1,241 +0,0 @@ -// Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT. - -package statemachine - -import ( - "fmt" - "io" - - cbg "github.com/whyrusleeping/cbor-gen" - xerrors "golang.org/x/xerrors" -) - -var _ = xerrors.Errorf - -func (t *TestState) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - if _, err := w.Write([]byte{162}); err != nil { - return err - } - - // t.A (uint64) (uint64) - if len("A") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"A\" was too long") - } - - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("A")))); err != nil { - return err - } - if _, err := w.Write([]byte("A")); err != nil { - return err - } - - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.A))); err != nil { - return err - } - - // t.B (uint64) (uint64) - if len("B") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"B\" was too long") - } - - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("B")))); err != nil { - return err - } - if _, err := w.Write([]byte("B")); err != nil { - return err - } - - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.B))); err != nil { - return err - } - - return nil -} - -func (t *TestState) UnmarshalCBOR(r io.Reader) error { - br := cbg.GetPeeker(r) - - maj, extra, err := cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajMap { - return fmt.Errorf("cbor input should be of type map") - } - - if extra > cbg.MaxLength { - return fmt.Errorf("TestState: map struct too large (%d)", extra) - } - - var name string - n := extra - - for i := uint64(0); i < n; i++ { - - { - sval, err := cbg.ReadString(br) - if err != nil { - return err - } - - name = string(sval) - } - - switch name { - // t.A (uint64) (uint64) - case "A": - - { - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.A = uint64(extra) - - } - // t.B (uint64) (uint64) - case "B": - - { - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.B = uint64(extra) - - } - - default: - return fmt.Errorf("unknown struct field %d: '%s'", i, name) - } - } - - return nil -} -func (t *TestEvent) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - if _, err := w.Write([]byte{162}); err != nil { - return err - } - - // t.A (string) (string) - if len("A") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"A\" was too long") - } - - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("A")))); err != nil { - return err - } - if _, err := w.Write([]byte("A")); err != nil { - return err - } - - if len(t.A) > cbg.MaxLength { - return xerrors.Errorf("Value in field t.A was too long") - } - - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(t.A)))); err != nil { - return err - } - if _, err := w.Write([]byte(t.A)); err != nil { - return err - } - - // t.Val (uint64) (uint64) - if len("Val") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"Val\" was too long") - } - - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("Val")))); err != nil { - return err - } - if _, err := w.Write([]byte("Val")); err != nil { - return err - } - - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Val))); err != nil { - return err - } - - return nil -} - -func (t *TestEvent) UnmarshalCBOR(r io.Reader) error { - br := cbg.GetPeeker(r) - - maj, extra, err := cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajMap { - return fmt.Errorf("cbor input should be of type map") - } - - if extra > cbg.MaxLength { - return fmt.Errorf("TestEvent: map struct too large (%d)", extra) - } - - var name string - n := extra - - for i := uint64(0); i < n; i++ { - - { - sval, err := cbg.ReadString(br) - if err != nil { - return err - } - - name = string(sval) - } - - switch name { - // t.A (string) (string) - case "A": - - { - sval, err := cbg.ReadString(br) - if err != nil { - return err - } - - t.A = string(sval) - } - // t.Val (uint64) (uint64) - case "Val": - - { - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.Val = uint64(extra) - - } - - default: - return fmt.Errorf("unknown struct field %d: '%s'", i, name) - } - } - - return nil -} diff --git a/lib/statemachine/context.go b/lib/statemachine/context.go deleted file mode 100644 index 2fb9b8f00..000000000 --- a/lib/statemachine/context.go +++ /dev/null @@ -1,16 +0,0 @@ -package statemachine - -import "context" - -type Context struct { - ctx context.Context - send func(evt interface{}) error -} - -func (ctx *Context) Context() context.Context { - return ctx.ctx -} - -func (ctx *Context) Send(evt interface{}) error { - return ctx.send(evt) -} diff --git a/lib/statemachine/group.go b/lib/statemachine/group.go deleted file mode 100644 index 2c8562955..000000000 --- a/lib/statemachine/group.go +++ /dev/null @@ -1,116 +0,0 @@ -package statemachine - -import ( - "context" - "reflect" - "sync" - - "github.com/filecoin-project/go-statestore" - "github.com/ipfs/go-datastore" - "golang.org/x/xerrors" -) - -type StateHandler interface { - // returns - Plan(events []Event, user interface{}) (interface{}, error) -} - -// StateGroup manages a group of state machines sharing the same logic -type StateGroup struct { - sts *statestore.StateStore - hnd StateHandler - stateType reflect.Type - - lk sync.Mutex - sms map[datastore.Key]*StateMachine -} - -// stateType: T - (MyStateStruct{}) -func New(ds datastore.Datastore, hnd StateHandler, stateType interface{}) *StateGroup { - return &StateGroup{ - sts: statestore.New(ds), - hnd: hnd, - stateType: reflect.TypeOf(stateType), - - sms: map[datastore.Key]*StateMachine{}, - } -} - -// Send sends an event to machine identified by `id`. -// `evt` is going to be passed into StateHandler.Planner, in the events[].User param -// -// If a state machine with the specified id doesn't exits, it's created, and it's -// state is set to zero-value of stateType provided in group constructor -func (s *StateGroup) Send(id interface{}, evt interface{}) (err error) { - s.lk.Lock() - defer s.lk.Unlock() - - sm, exist := s.sms[statestore.ToKey(id)] - if !exist { - sm, err = s.loadOrCreate(id) - if err != nil { - return xerrors.Errorf("loadOrCreate state: %w", err) - } - s.sms[statestore.ToKey(id)] = sm - } - - return sm.send(Event{User: evt}) -} - -func (s *StateGroup) loadOrCreate(name interface{}) (*StateMachine, error) { - exists, err := s.sts.Has(name) - if err != nil { - return nil, xerrors.Errorf("failed to check if state for %v exists: %w", name, err) - } - - if !exists { - userState := reflect.New(s.stateType).Interface() - - err = s.sts.Begin(name, userState) - if err != nil { - return nil, xerrors.Errorf("saving initial state: %w", err) - } - } - - res := &StateMachine{ - planner: s.hnd.Plan, - eventsIn: make(chan Event), - - name: name, - st: s.sts.Get(name), - stateType: s.stateType, - - stageDone: make(chan struct{}), - closing: make(chan struct{}), - closed: make(chan struct{}), - } - - go res.run() - - return res, nil -} - -// Stop stops all state machines in this group -func (s *StateGroup) Stop(ctx context.Context) error { - s.lk.Lock() - defer s.lk.Unlock() - - for _, sm := range s.sms { - if err := sm.stop(ctx); err != nil { - return err - } - } - - return nil -} - -// List outputs states of all state machines in this group -// out: *[]StateT -func (s *StateGroup) List(out interface{}) error { - return s.sts.List(out) -} - -// Get gets state for a single state machine -func (s *StateGroup) Get(id interface{}) *statestore.StoredState { - return s.sts.Get(id) -} diff --git a/lib/statemachine/machine.go b/lib/statemachine/machine.go deleted file mode 100644 index d30c9e7d4..000000000 --- a/lib/statemachine/machine.go +++ /dev/null @@ -1,127 +0,0 @@ -package statemachine - -import ( - "context" - "reflect" - "sync/atomic" - - "github.com/filecoin-project/go-statestore" - - logging "github.com/ipfs/go-log" -) - -var log = logging.Logger("evtsm") - -type Event struct { - User interface{} -} - -// TODO: This probably should be returning an int indicating partial event processing -// (or something like errPartial(nEvents)) -// returns func(ctx Context, st ) (func(*), error), where is the typeOf(User) param -type Planner func(events []Event, user interface{}) (interface{}, error) - -type StateMachine struct { - planner Planner - eventsIn chan Event - - name interface{} - st *statestore.StoredState - stateType reflect.Type - - stageDone chan struct{} - closing chan struct{} - closed chan struct{} - - busy int32 -} - -func (fsm *StateMachine) run() { - defer close(fsm.closed) - - var pendingEvents []Event - - for { - // NOTE: This requires at least one event to be sent to trigger a stage - // This means that after restarting the state machine users of this - // code must send a 'restart' event - select { - case evt := <-fsm.eventsIn: - pendingEvents = append(pendingEvents, evt) - case <-fsm.stageDone: - if len(pendingEvents) == 0 { - continue - } - case <-fsm.closing: - return - } - - if atomic.CompareAndSwapInt32(&fsm.busy, 0, 1) { - var nextStep interface{} - var ustate interface{} - - err := fsm.mutateUser(func(user interface{}) (err error) { - nextStep, err = fsm.planner(pendingEvents, user) - ustate = user - return err - }) - if err != nil { - log.Errorf("Executing event planner failed: %+v", err) - return - } - - pendingEvents = nil - - if nextStep == nil { - continue - } - - ctx := Context{ - ctx: context.TODO(), - send: func(evt interface{}) error { - return fsm.send(Event{User: evt}) - }, - } - - go func() { - res := reflect.ValueOf(nextStep).Call([]reflect.Value{reflect.ValueOf(ctx), reflect.ValueOf(ustate).Elem()}) - - if res[0].Interface() != nil { - log.Errorf("executing step: %+v", res[0].Interface().(error)) // TODO: propagate top level - return - } - - atomic.StoreInt32(&fsm.busy, 0) - fsm.stageDone <- struct{}{} - }() - - } - } -} - -func (fsm *StateMachine) mutateUser(cb func(user interface{}) error) error { - mutt := reflect.FuncOf([]reflect.Type{reflect.PtrTo(fsm.stateType)}, []reflect.Type{reflect.TypeOf(new(error)).Elem()}, false) - - mutf := reflect.MakeFunc(mutt, func(args []reflect.Value) (results []reflect.Value) { - err := cb(args[0].Interface()) - return []reflect.Value{reflect.ValueOf(&err).Elem()} - }) - - return fsm.st.Mutate(mutf.Interface()) -} - -func (fsm *StateMachine) send(evt Event) error { - fsm.eventsIn <- evt // TODO: ctx, at least - return nil -} - -func (fsm *StateMachine) stop(ctx context.Context) error { - close(fsm.closing) - - select { - case <-fsm.closed: - return nil - case <-ctx.Done(): - return ctx.Err() - } -} diff --git a/lib/statemachine/machine_test.go b/lib/statemachine/machine_test.go deleted file mode 100644 index 49b8324ce..000000000 --- a/lib/statemachine/machine_test.go +++ /dev/null @@ -1,105 +0,0 @@ -package statemachine - -import ( - "context" - "testing" - - "github.com/ipfs/go-datastore" - logging "github.com/ipfs/go-log" - "gotest.tools/assert" -) - -func init() { - logging.SetLogLevel("*", "INFO") -} - -type testHandler struct { - t *testing.T - proceed chan struct{} - done chan struct{} -} - -func (t *testHandler) Plan(events []Event, state interface{}) (interface{}, error) { - return t.plan(events, state.(*TestState)) -} - -func (t *testHandler) plan(events []Event, state *TestState) (func(Context, TestState) error, error) { - for _, event := range events { - e := event.User.(*TestEvent) - switch e.A { - case "restart": - case "start": - state.A = 1 - case "b": - state.A = 2 - state.B = e.Val - } - } - - switch state.A { - case 1: - return t.step0, nil - case 2: - return t.step1, nil - default: - t.t.Fatal(state.A) - } - panic("how?") -} - -func (t *testHandler) step0(ctx Context, st TestState) error { - ctx.Send(&TestEvent{A: "b", Val: 55}) - <-t.proceed - return nil -} - -func (t *testHandler) step1(ctx Context, st TestState) error { - assert.Equal(t.t, uint64(2), st.A) - - close(t.done) - return nil -} - -func TestBasic(t *testing.T) { - for i := 0; i < 1000; i++ { // run a few times to expose any races - ds := datastore.NewMapDatastore() - - th := &testHandler{t: t, done: make(chan struct{}), proceed: make(chan struct{})} - close(th.proceed) - smm := New(ds, th, TestState{}) - - if err := smm.Send(uint64(2), &TestEvent{A: "start"}); err != nil { - t.Fatalf("%+v", err) - } - - <-th.done - } -} - -func TestPersist(t *testing.T) { - for i := 0; i < 1000; i++ { // run a few times to expose any races - ds := datastore.NewMapDatastore() - - th := &testHandler{t: t, done: make(chan struct{}), proceed: make(chan struct{})} - smm := New(ds, th, TestState{}) - - if err := smm.Send(uint64(2), &TestEvent{A: "start"}); err != nil { - t.Fatalf("%+v", err) - } - - if err := smm.Stop(context.Background()); err != nil { - t.Fatal(err) - return - } - - smm = New(ds, th, TestState{}) - if err := smm.Send(uint64(2), &TestEvent{A: "restart"}); err != nil { - t.Fatalf("%+v", err) - } - close(th.proceed) - - <-th.done - } -} - -var _ StateHandler = &testHandler{} diff --git a/lib/statemachine/testing.go b/lib/statemachine/testing.go deleted file mode 100644 index 8a2c7e8c6..000000000 --- a/lib/statemachine/testing.go +++ /dev/null @@ -1,11 +0,0 @@ -package statemachine - -type TestState struct { - A uint64 - B uint64 -} - -type TestEvent struct { - A string - Val uint64 -} diff --git a/storage/sealing/fsm.go b/storage/sealing/fsm.go index 06ba11354..43be0077c 100644 --- a/storage/sealing/fsm.go +++ b/storage/sealing/fsm.go @@ -6,17 +6,18 @@ import ( "reflect" "time" - "github.com/filecoin-project/specs-actors/actors/abi" "golang.org/x/xerrors" + "github.com/filecoin-project/go-statemachine" + "github.com/filecoin-project/specs-actors/actors/abi" + "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/lib/statemachine" ) -func (m *Sealing) Plan(events []statemachine.Event, user interface{}) (interface{}, error) { +func (m *Sealing) Plan(events []statemachine.Event, user interface{}) (interface{}, uint64, error) { next, err := m.plan(events, user.(*SectorInfo)) if err != nil || next == nil { - return nil, err + return nil, 0, err } return func(ctx statemachine.Context, si SectorInfo) error { @@ -27,7 +28,7 @@ func (m *Sealing) Plan(events []statemachine.Event, user interface{}) (interface } return nil - }, nil + }, uint64(len(events)), nil // TODO: This processed event count is not very correct } var fsmPlanners = []func(events []statemachine.Event, state *SectorInfo) error{ diff --git a/storage/sealing/fsm_test.go b/storage/sealing/fsm_test.go index a4d53d018..41becc4f3 100644 --- a/storage/sealing/fsm_test.go +++ b/storage/sealing/fsm_test.go @@ -6,8 +6,9 @@ import ( logging "github.com/ipfs/go-log/v2" "github.com/stretchr/testify/require" + "github.com/filecoin-project/go-statemachine" + "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/lib/statemachine" ) func init() { diff --git a/storage/sealing/sealing.go b/storage/sealing/sealing.go index a3652485e..89294eb91 100644 --- a/storage/sealing/sealing.go +++ b/storage/sealing/sealing.go @@ -12,6 +12,7 @@ import ( "golang.org/x/xerrors" "github.com/filecoin-project/go-padreader" + "github.com/filecoin-project/go-statemachine" "github.com/filecoin-project/specs-actors/actors/abi" "github.com/filecoin-project/specs-actors/actors/builtin/miner" "github.com/filecoin-project/specs-actors/actors/crypto" @@ -20,7 +21,6 @@ import ( "github.com/filecoin-project/lotus/chain/events" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/lib/statemachine" "github.com/filecoin-project/lotus/storage/sealmgr" ) diff --git a/storage/sealing/states.go b/storage/sealing/states.go index b083096e9..e12860447 100644 --- a/storage/sealing/states.go +++ b/storage/sealing/states.go @@ -5,6 +5,7 @@ import ( "github.com/filecoin-project/specs-actors/actors/crypto" + "github.com/filecoin-project/go-statemachine" "github.com/filecoin-project/specs-actors/actors/abi" "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/filecoin-project/specs-actors/actors/builtin/miner" @@ -14,7 +15,6 @@ import ( "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/lib/statemachine" ) func (m *Sealing) handlePacking(ctx statemachine.Context, sector SectorInfo) error { diff --git a/storage/sealing/states_failed.go b/storage/sealing/states_failed.go index a7dfe36a9..d5667f57e 100644 --- a/storage/sealing/states_failed.go +++ b/storage/sealing/states_failed.go @@ -4,6 +4,7 @@ import ( "bytes" "time" + "github.com/filecoin-project/go-statemachine" "github.com/filecoin-project/specs-actors/actors/builtin/miner" "github.com/filecoin-project/specs-actors/actors/util/adt" "golang.org/x/xerrors" @@ -12,7 +13,6 @@ import ( "github.com/filecoin-project/lotus/api/apibstore" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/lib/statemachine" ) const minRetryTime = 1 * time.Minute From 3cab8bd5d1e203d5be194ef685ef4ce81ee11e96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 6 Mar 2020 20:01:28 +0100 Subject: [PATCH 4/4] gofmt --- chain/validation/keymanager.go | 2 +- cli/client.go | 20 +++++++------- cli/multisig.go | 22 ++++++++-------- cli/paych.go | 28 ++++++++++---------- cli/state.go | 48 +++++++++++++++++----------------- cli/sync.go | 8 +++--- cli/wallet.go | 12 ++++----- cmd/lotus-bench/main.go | 4 +-- cmd/lotus-seed/seed/seed.go | 4 +-- go.mod | 1 - 10 files changed, 74 insertions(+), 75 deletions(-) diff --git a/chain/validation/keymanager.go b/chain/validation/keymanager.go index 3161883b9..76fe8673d 100644 --- a/chain/validation/keymanager.go +++ b/chain/validation/keymanager.go @@ -87,7 +87,7 @@ func (k *KeyManager) newBLSKey() *wallet.Key { //sk := ffi.PrivateKeyGenerate(s.blsSeed) // s.blsSeed++ sk := [32]byte{} - sk[0] = uint8(k.blsSeed+1) // hack to keep gas values determinist + sk[0] = uint8(k.blsSeed + 1) // hack to keep gas values determinist key, err := wallet.NewKey(types.KeyInfo{ Type: wallet.KTBLS, PrivateKey: sk[:], diff --git a/cli/client.go b/cli/client.go index 3150840dc..5581632e1 100644 --- a/cli/client.go +++ b/cli/client.go @@ -35,8 +35,8 @@ var clientCmd = &cli.Command{ } var clientImportCmd = &cli.Command{ - Name: "import", - Usage: "Import data", + Name: "import", + Usage: "Import data", ArgsUsage: "[inputPath]", Action: func(cctx *cli.Context) error { api, closer, err := GetFullNodeAPI(cctx) @@ -82,8 +82,8 @@ var clientLocalCmd = &cli.Command{ } var clientDealCmd = &cli.Command{ - Name: "deal", - Usage: "Initialize storage deal with a miner", + Name: "deal", + Usage: "Initialize storage deal with a miner", ArgsUsage: "[dataCid miner price duration]", Flags: []cli.Flag{ &cli.BoolFlag{ @@ -193,8 +193,8 @@ var clientDealCmd = &cli.Command{ } var clientFindCmd = &cli.Command{ - Name: "find", - Usage: "find data in the network", + Name: "find", + Usage: "find data in the network", ArgsUsage: "[dataCid]", Action: func(cctx *cli.Context) error { if !cctx.Args().Present() { @@ -243,8 +243,8 @@ var clientFindCmd = &cli.Command{ } var clientRetrieveCmd = &cli.Command{ - Name: "retrieve", - Usage: "retrieve data from network", + Name: "retrieve", + Usage: "retrieve data from network", ArgsUsage: "[dataCid outputPath]", Flags: []cli.Flag{ &cli.StringFlag{ @@ -314,8 +314,8 @@ var clientRetrieveCmd = &cli.Command{ } var clientQueryAskCmd = &cli.Command{ - Name: "query-ask", - Usage: "find a miners ask", + Name: "query-ask", + Usage: "find a miners ask", ArgsUsage: "[minerAddress]", Flags: []cli.Flag{ &cli.StringFlag{ diff --git a/cli/multisig.go b/cli/multisig.go index 877fd41a4..9d1123f6e 100644 --- a/cli/multisig.go +++ b/cli/multisig.go @@ -47,8 +47,8 @@ var multisigCmd = &cli.Command{ } var msigCreateCmd = &cli.Command{ - Name: "create", - Usage: "Create a new multisig wallet", + Name: "create", + Usage: "Create a new multisig wallet", ArgsUsage: "[address1 address2 ...]", Flags: []cli.Flag{ &cli.Int64Flag{ @@ -158,10 +158,10 @@ var msigCreateCmd = &cli.Command{ } var msigInspectCmd = &cli.Command{ - Name: "inspect", - Usage: "Inspect a multisig wallet", + Name: "inspect", + Usage: "Inspect a multisig wallet", ArgsUsage: "[address]", - Flags: []cli.Flag{}, + Flags: []cli.Flag{}, Action: func(cctx *cli.Context) error { api, closer, err := GetFullNodeAPI(cctx) if err != nil { @@ -271,10 +271,10 @@ func state(tx *samsig.Transaction) string { } var msigProposeCmd = &cli.Command{ - Name: "propose", - Usage: "Propose a multisig transaction", + Name: "propose", + Usage: "Propose a multisig transaction", ArgsUsage: "[multisigAddress destinationAddress value (optional)]", - Flags: []cli.Flag{}, + Flags: []cli.Flag{}, Action: func(cctx *cli.Context) error { api, closer, err := GetFullNodeAPI(cctx) if err != nil { @@ -385,10 +385,10 @@ var msigProposeCmd = &cli.Command{ } var msigApproveCmd = &cli.Command{ - Name: "approve", - Usage: "Approve a multisig message", + Name: "approve", + Usage: "Approve a multisig message", ArgsUsage: "[multisigAddress messageId]", - Flags: []cli.Flag{}, + Flags: []cli.Flag{}, Action: func(cctx *cli.Context) error { api, closer, err := GetFullNodeAPI(cctx) if err != nil { diff --git a/cli/paych.go b/cli/paych.go index 4d2077e75..e150d6769 100644 --- a/cli/paych.go +++ b/cli/paych.go @@ -23,8 +23,8 @@ var paychCmd = &cli.Command{ } var paychGetCmd = &cli.Command{ - Name: "get", - Usage: "Create a new payment channel or get existing one", + Name: "get", + Usage: "Create a new payment channel or get existing one", ArgsUsage: "[fromAddress toAddress amount]", Action: func(cctx *cli.Context) error { if cctx.Args().Len() != 3 { @@ -102,8 +102,8 @@ var paychVoucherCmd = &cli.Command{ } var paychVoucherCreateCmd = &cli.Command{ - Name: "create", - Usage: "Create a signed payment channel voucher", + Name: "create", + Usage: "Create a signed payment channel voucher", ArgsUsage: "[channelAddress amount]", Flags: []cli.Flag{ &cli.IntFlag{ @@ -153,8 +153,8 @@ var paychVoucherCreateCmd = &cli.Command{ } var paychVoucherCheckCmd = &cli.Command{ - Name: "check", - Usage: "Check validity of payment channel voucher", + Name: "check", + Usage: "Check validity of payment channel voucher", ArgsUsage: "[channelAddress voucher]", Action: func(cctx *cli.Context) error { if cctx.Args().Len() != 2 { @@ -189,8 +189,8 @@ var paychVoucherCheckCmd = &cli.Command{ } var paychVoucherAddCmd = &cli.Command{ - Name: "add", - Usage: "Add payment channel voucher to local datastore", + Name: "add", + Usage: "Add payment channel voucher to local datastore", ArgsUsage: "[channelAddress voucher]", Action: func(cctx *cli.Context) error { if cctx.Args().Len() != 2 { @@ -225,8 +225,8 @@ var paychVoucherAddCmd = &cli.Command{ } var paychVoucherListCmd = &cli.Command{ - Name: "list", - Usage: "List stored vouchers for a given payment channel", + Name: "list", + Usage: "List stored vouchers for a given payment channel", ArgsUsage: "[channelAddress]", Flags: []cli.Flag{ &cli.BoolFlag{ @@ -275,8 +275,8 @@ var paychVoucherListCmd = &cli.Command{ } var paychVoucherBestSpendableCmd = &cli.Command{ - Name: "best-spendable", - Usage: "Print voucher with highest value that is currently spendable", + Name: "best-spendable", + Usage: "Print voucher with highest value that is currently spendable", ArgsUsage: "[channelAddress]", Action: func(cctx *cli.Context) error { if cctx.Args().Len() != 1 { @@ -330,8 +330,8 @@ var paychVoucherBestSpendableCmd = &cli.Command{ } var paychVoucherSubmitCmd = &cli.Command{ - Name: "submit", - Usage: "Submit voucher to chain to update payment channel state", + Name: "submit", + Usage: "Submit voucher to chain to update payment channel state", ArgsUsage: "[channelAddress voucher]", Action: func(cctx *cli.Context) error { if cctx.Args().Len() != 2 { diff --git a/cli/state.go b/cli/state.go index ca6ce888c..b736c91de 100644 --- a/cli/state.go +++ b/cli/state.go @@ -61,8 +61,8 @@ var stateCmd = &cli.Command{ } var stateMinerInfo = &cli.Command{ - Name: "miner-info", - Usage: "Retrieve miner information", + Name: "miner-info", + Usage: "Retrieve miner information", ArgsUsage: "[minerAddress]", Action: func(cctx *cli.Context) error { api, closer, err := GetFullNodeAPI(cctx) @@ -153,8 +153,8 @@ func loadTipSet(ctx context.Context, cctx *cli.Context, api api.FullNode) (*type } var statePowerCmd = &cli.Command{ - Name: "power", - Usage: "Query network or miner power", + Name: "power", + Usage: "Query network or miner power", ArgsUsage: "[ (optional)]", Action: func(cctx *cli.Context) error { api, closer, err := GetFullNodeAPI(cctx) @@ -197,8 +197,8 @@ var statePowerCmd = &cli.Command{ } var stateSectorsCmd = &cli.Command{ - Name: "sectors", - Usage: "Query the sector set of a miner", + Name: "sectors", + Usage: "Query the sector set of a miner", ArgsUsage: "[minerAddress]", Action: func(cctx *cli.Context) error { api, closer, err := GetFullNodeAPI(cctx) @@ -237,8 +237,8 @@ var stateSectorsCmd = &cli.Command{ } var stateProvingSetCmd = &cli.Command{ - Name: "proving", - Usage: "Query the proving set of a miner", + Name: "proving", + Usage: "Query the proving set of a miner", ArgsUsage: "[minerAddress]", Action: func(cctx *cli.Context) error { api, closer, err := GetFullNodeAPI(cctx) @@ -277,8 +277,8 @@ var stateProvingSetCmd = &cli.Command{ } var stateReplaySetCmd = &cli.Command{ - Name: "replay", - Usage: "Replay a particular message within a tipset", + Name: "replay", + Usage: "Replay a particular message within a tipset", ArgsUsage: "[tipsetKey messageCid]", Action: func(cctx *cli.Context) error { if cctx.Args().Len() < 1 { @@ -384,8 +384,8 @@ var statePledgeCollateralCmd = &cli.Command{ } var stateGetDealSetCmd = &cli.Command{ - Name: "get-deal", - Usage: "View on-chain deal info", + Name: "get-deal", + Usage: "View on-chain deal info", ArgsUsage: "[dealId]", Action: func(cctx *cli.Context) error { api, closer, err := GetFullNodeAPI(cctx) @@ -486,8 +486,8 @@ var stateListActorsCmd = &cli.Command{ } var stateGetActorCmd = &cli.Command{ - Name: "get-actor", - Usage: "Print actor information", + Name: "get-actor", + Usage: "Print actor information", ArgsUsage: "[actorrAddress]", Action: func(cctx *cli.Context) error { api, closer, err := GetFullNodeAPI(cctx) @@ -528,8 +528,8 @@ var stateGetActorCmd = &cli.Command{ } var stateLookupIDCmd = &cli.Command{ - Name: "lookup", - Usage: "Find corresponding ID address", + Name: "lookup", + Usage: "Find corresponding ID address", ArgsUsage: "[address]", Action: func(cctx *cli.Context) error { api, closer, err := GetFullNodeAPI(cctx) @@ -566,8 +566,8 @@ var stateLookupIDCmd = &cli.Command{ } var stateSectorSizeCmd = &cli.Command{ - Name: "sector-size", - Usage: "Look up miners sector size", + Name: "sector-size", + Usage: "Look up miners sector size", ArgsUsage: "[minerAddress]", Action: func(cctx *cli.Context) error { api, closer, err := GetFullNodeAPI(cctx) @@ -603,8 +603,8 @@ var stateSectorSizeCmd = &cli.Command{ } var stateReadStateCmd = &cli.Command{ - Name: "read-state", - Usage: "View a json representation of an actors state", + Name: "read-state", + Usage: "View a json representation of an actors state", ArgsUsage: "[actorAddress]", Action: func(cctx *cli.Context) error { api, closer, err := GetFullNodeAPI(cctx) @@ -796,8 +796,8 @@ var stateComputeStateCmd = &cli.Command{ } var stateWaitMsgCmd = &cli.Command{ - Name: "wait-msg", - Usage: "Wait for a message to appear on chain", + Name: "wait-msg", + Usage: "Wait for a message to appear on chain", ArgsUsage: "[messageCid]", Flags: []cli.Flag{ &cli.StringFlag{ @@ -837,8 +837,8 @@ var stateWaitMsgCmd = &cli.Command{ } var stateCallCmd = &cli.Command{ - Name: "call", - Usage: "Invoke a method on an actor locally", + Name: "call", + Usage: "Invoke a method on an actor locally", ArgsUsage: "[toAddress methodId (optional)]", Flags: []cli.Flag{ &cli.StringFlag{ diff --git a/cli/sync.go b/cli/sync.go index 0cde9c97e..891f61da3 100644 --- a/cli/sync.go +++ b/cli/sync.go @@ -94,8 +94,8 @@ var syncWaitCmd = &cli.Command{ } var syncMarkBadCmd = &cli.Command{ - Name: "mark-bad", - Usage: "Mark the given block as bad, will prevent syncing to a chain that contains it", + Name: "mark-bad", + Usage: "Mark the given block as bad, will prevent syncing to a chain that contains it", ArgsUsage: "[blockCid]", Action: func(cctx *cli.Context) error { napi, closer, err := GetFullNodeAPI(cctx) @@ -119,8 +119,8 @@ var syncMarkBadCmd = &cli.Command{ } var syncCheckBadCmd = &cli.Command{ - Name: "check-bad", - Usage: "check if the given block was marked bad, and for what reason", + Name: "check-bad", + Usage: "check if the given block was marked bad, and for what reason", ArgsUsage: "[blockCid]", Action: func(cctx *cli.Context) error { napi, closer, err := GetFullNodeAPI(cctx) diff --git a/cli/wallet.go b/cli/wallet.go index b7123b594..ed80c062c 100644 --- a/cli/wallet.go +++ b/cli/wallet.go @@ -135,8 +135,8 @@ var walletGetDefault = &cli.Command{ } var walletSetDefault = &cli.Command{ - Name: "set-default", - Usage: "Set default wallet address", + Name: "set-default", + Usage: "Set default wallet address", ArgsUsage: "[address]", Action: func(cctx *cli.Context) error { api, closer, err := GetFullNodeAPI(cctx) @@ -160,8 +160,8 @@ var walletSetDefault = &cli.Command{ } var walletExport = &cli.Command{ - Name: "export", - Usage: "export keys", + Name: "export", + Usage: "export keys", ArgsUsage: "[address]", Action: func(cctx *cli.Context) error { api, closer, err := GetFullNodeAPI(cctx) @@ -196,8 +196,8 @@ var walletExport = &cli.Command{ } var walletImport = &cli.Command{ - Name: "import", - Usage: "import keys", + Name: "import", + Usage: "import keys", ArgsUsage: "[ (optional, will read from stdin if omitted)]", Action: func(cctx *cli.Context) error { api, closer, err := GetFullNodeAPI(cctx) diff --git a/cmd/lotus-bench/main.go b/cmd/lotus-bench/main.go index fa6b89a81..3b5927c88 100644 --- a/cmd/lotus-bench/main.go +++ b/cmd/lotus-bench/main.go @@ -179,8 +179,8 @@ func main() { } sbfs := &fs.Basic{ - Miner: maddr, - Root: sbdir, + Miner: maddr, + Root: sbdir, } sb, err := sectorbuilder.New(sbfs, cfg) diff --git a/cmd/lotus-seed/seed/seed.go b/cmd/lotus-seed/seed/seed.go index 59e4200cf..3f5946ccc 100644 --- a/cmd/lotus-seed/seed/seed.go +++ b/cmd/lotus-seed/seed/seed.go @@ -55,8 +55,8 @@ func PreSeal(maddr address.Address, pt abi.RegisteredProof, offset abi.SectorNum next := offset sbfs := &fs.Basic{ - Miner: maddr, - Root: sbroot, + Miner: maddr, + Root: sbroot, } sb, err := sectorbuilder.New(sbfs, cfg) diff --git a/go.mod b/go.mod index 1e7953a48..fe67193e5 100644 --- a/go.mod +++ b/go.mod @@ -85,7 +85,6 @@ require ( github.com/multiformats/go-multiaddr-dns v0.2.0 github.com/multiformats/go-multiaddr-net v0.1.2 github.com/multiformats/go-multihash v0.0.13 - github.com/multiformats/go-varint v0.0.5 github.com/opentracing/opentracing-go v1.1.0 github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829 github.com/prometheus/common v0.4.0