From 84a4766d8f3ce9cb43bd60a3f5754b5806df4be7 Mon Sep 17 00:00:00 2001 From: anorth Date: Fri, 8 Nov 2019 16:36:50 +1100 Subject: [PATCH 1/3] Add TipSetKey as a concatenation of block CIDs, and use if for ChainGetTipSet, ChainGetRandomness. --- api/api_full.go | 4 +- api/struct.go | 32 ++++---- chain/gen/gen.go | 6 +- chain/types/tipset.go | 4 + chain/types/tipset_key.go | 132 +++++++++++++++++++++++++++++++++ chain/types/tipset_key_test.go | 55 ++++++++++++++ cli/chain.go | 2 +- node/impl/full/chain.go | 6 +- node/modules/storageminer.go | 2 +- storage/miner.go | 2 +- storage/post.go | 2 +- storage/sector_states.go | 2 +- 12 files changed, 220 insertions(+), 29 deletions(-) create mode 100644 chain/types/tipset_key.go create mode 100644 chain/types/tipset_key_test.go diff --git a/api/api_full.go b/api/api_full.go index 057dca8cb..f1dc51efc 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -23,9 +23,9 @@ type FullNode interface { // First message is guaranteed to be of len == 1, and type == 'current' ChainNotify(context.Context) (<-chan []*store.HeadChange, error) ChainHead(context.Context) (*types.TipSet, error) - ChainGetRandomness(context.Context, *types.TipSet, []*types.Ticket, int) ([]byte, error) + ChainGetRandomness(context.Context, types.TipSetKey, []*types.Ticket, int) ([]byte, error) ChainGetBlock(context.Context, cid.Cid) (*types.BlockHeader, error) - ChainGetTipSet(context.Context, []cid.Cid) (*types.TipSet, error) + ChainGetTipSet(context.Context, types.TipSetKey) (*types.TipSet, error) ChainGetBlockMessages(context.Context, cid.Cid) (*BlockMessages, error) ChainGetParentReceipts(context.Context, cid.Cid) ([]*types.MessageReceipt, error) ChainGetParentMessages(context.Context, cid.Cid) ([]Message, error) diff --git a/api/struct.go b/api/struct.go index 8d14af1d2..d94cc031f 100644 --- a/api/struct.go +++ b/api/struct.go @@ -37,19 +37,19 @@ type FullNodeStruct struct { CommonStruct Internal struct { - ChainNotify func(context.Context) (<-chan []*store.HeadChange, error) `perm:"read"` - ChainHead func(context.Context) (*types.TipSet, error) `perm:"read"` - ChainGetRandomness func(context.Context, *types.TipSet, []*types.Ticket, int) ([]byte, error) `perm:"read"` - ChainGetBlock func(context.Context, cid.Cid) (*types.BlockHeader, error) `perm:"read"` - ChainGetTipSet func(context.Context, []cid.Cid) (*types.TipSet, error) `perm:"read"` - ChainGetBlockMessages func(context.Context, cid.Cid) (*BlockMessages, error) `perm:"read"` - ChainGetParentReceipts func(context.Context, cid.Cid) ([]*types.MessageReceipt, error) `perm:"read"` - ChainGetParentMessages func(context.Context, cid.Cid) ([]Message, error) `perm:"read"` - ChainGetTipSetByHeight func(context.Context, uint64, *types.TipSet) (*types.TipSet, error) `perm:"read"` - ChainReadObj func(context.Context, cid.Cid) ([]byte, error) `perm:"read"` - ChainSetHead func(context.Context, *types.TipSet) error `perm:"admin"` - ChainGetGenesis func(context.Context) (*types.TipSet, error) `perm:"read"` - ChainTipSetWeight func(context.Context, *types.TipSet) (types.BigInt, error) `perm:"read"` + ChainNotify func(context.Context) (<-chan []*store.HeadChange, error) `perm:"read"` + ChainHead func(context.Context) (*types.TipSet, error) `perm:"read"` + ChainGetRandomness func(context.Context, types.TipSetKey, []*types.Ticket, int) ([]byte, error) `perm:"read"` + ChainGetBlock func(context.Context, cid.Cid) (*types.BlockHeader, error) `perm:"read"` + ChainGetTipSet func(context.Context, types.TipSetKey) (*types.TipSet, error) `perm:"read"` + ChainGetBlockMessages func(context.Context, cid.Cid) (*BlockMessages, error) `perm:"read"` + ChainGetParentReceipts func(context.Context, cid.Cid) ([]*types.MessageReceipt, error) `perm:"read"` + ChainGetParentMessages func(context.Context, cid.Cid) ([]Message, error) `perm:"read"` + ChainGetTipSetByHeight func(context.Context, uint64, *types.TipSet) (*types.TipSet, error) `perm:"read"` + ChainReadObj func(context.Context, cid.Cid) ([]byte, error) `perm:"read"` + ChainSetHead func(context.Context, *types.TipSet) error `perm:"admin"` + ChainGetGenesis func(context.Context) (*types.TipSet, error) `perm:"read"` + ChainTipSetWeight func(context.Context, *types.TipSet) (types.BigInt, error) `perm:"read"` SyncState func(context.Context) (*SyncState, error) `perm:"read"` SyncSubmitBlock func(ctx context.Context, blk *types.BlockMsg) error `perm:"write"` @@ -245,7 +245,7 @@ func (c *FullNodeStruct) ChainHead(ctx context.Context) (*types.TipSet, error) { return c.Internal.ChainHead(ctx) } -func (c *FullNodeStruct) ChainGetRandomness(ctx context.Context, pts *types.TipSet, ticks []*types.Ticket, lb int) ([]byte, error) { +func (c *FullNodeStruct) ChainGetRandomness(ctx context.Context, pts types.TipSetKey, ticks []*types.Ticket, lb int) ([]byte, error) { return c.Internal.ChainGetRandomness(ctx, pts, ticks, lb) } @@ -301,8 +301,8 @@ func (c *FullNodeStruct) ChainGetBlock(ctx context.Context, b cid.Cid) (*types.B return c.Internal.ChainGetBlock(ctx, b) } -func (c *FullNodeStruct) ChainGetTipSet(ctx context.Context, cids []cid.Cid) (*types.TipSet, error) { - return c.Internal.ChainGetTipSet(ctx, cids) +func (c *FullNodeStruct) ChainGetTipSet(ctx context.Context, key types.TipSetKey) (*types.TipSet, error) { + return c.Internal.ChainGetTipSet(ctx, key) } func (c *FullNodeStruct) ChainGetBlockMessages(ctx context.Context, b cid.Cid) (*BlockMessages, error) { diff --git a/chain/gen/gen.go b/chain/gen/gen.go index fd0553f2d..1fa00f47f 100644 --- a/chain/gen/gen.go +++ b/chain/gen/gen.go @@ -354,7 +354,7 @@ func (cg *ChainGen) YieldRepo() (repo.Repo, error) { } type MiningCheckAPI interface { - ChainGetRandomness(context.Context, *types.TipSet, []*types.Ticket, int) ([]byte, error) + ChainGetRandomness(context.Context, types.TipSetKey, []*types.Ticket, int) ([]byte, error) StateMinerPower(context.Context, address.Address, *types.TipSet) (api.MinerPower, error) @@ -368,7 +368,7 @@ type mca struct { sm *stmgr.StateManager } -func (mca mca) ChainGetRandomness(ctx context.Context, pts *types.TipSet, ticks []*types.Ticket, lb int) ([]byte, error) { +func (mca mca) ChainGetRandomness(ctx context.Context, pts types.TipSetKey, ticks []*types.Ticket, lb int) ([]byte, error) { return mca.sm.ChainStore().GetRandomness(ctx, pts.Cids(), ticks, int64(lb)) } @@ -393,7 +393,7 @@ func (mca mca) WalletSign(ctx context.Context, a address.Address, v []byte) (*ty } func IsRoundWinner(ctx context.Context, ts *types.TipSet, ticks []*types.Ticket, miner address.Address, a MiningCheckAPI) (bool, types.ElectionProof, error) { - r, err := a.ChainGetRandomness(ctx, ts, ticks, build.EcRandomnessLookback) + r, err := a.ChainGetRandomness(ctx, ts.Key(), ticks, build.EcRandomnessLookback) if err != nil { return false, nil, xerrors.Errorf("chain get randomness: %w", err) } diff --git a/chain/types/tipset.go b/chain/types/tipset.go index 733e29685..13fb7e582 100644 --- a/chain/types/tipset.go +++ b/chain/types/tipset.go @@ -123,6 +123,10 @@ func (ts *TipSet) Cids() []cid.Cid { return ts.cids } +func (ts *TipSet) Key() TipSetKey { + return NewTipSetKey(ts.cids...) +} + func (ts *TipSet) Height() uint64 { return ts.height } diff --git a/chain/types/tipset_key.go b/chain/types/tipset_key.go new file mode 100644 index 000000000..02633d5de --- /dev/null +++ b/chain/types/tipset_key.go @@ -0,0 +1,132 @@ +package types + +import ( + "bytes" + "encoding/binary" + "strings" + + "github.com/ipfs/go-cid" +) + +// A TipSetKey is an immutable collection of CIDs forming a unique key for a tipset. +// The CIDs are assumed to be distinct and in canonical order. Two keys with the same +// CIDs in a different order are not considered equal. +// TipSetKey is a lightweight value type, and may be compared for equality with ==. +type TipSetKey struct { + // The internal representation is a concatenation of the bytes of the CIDs, each preceded by + // uint32 specifying the length of the CIDs bytes, and the whole preceded by a uint32 + // giving the number of CIDs. + // These gymnastics make the a TipSetKey usable as a map key. + // This representation is slightly larger than strictly necessary: CIDs do carry a prefix + // including their length, but the cid package doesn't quite expose the functions needed to + // safely parse a sequence of concatenated CIDs from a stream, and we probably don't want to + // (re-)implement that here. See https://github.com/ipfs/go-cid/issues/93. + // The empty key has value "" (no encoded-zero prefix). + value string +} + +// NewTipSetKey builds a new key from a slice of CIDs. +// The CIDs are assumed to be ordered correctly. +func NewTipSetKey(cids ...cid.Cid) TipSetKey { + encoded, err := encodeKey(cids) + if err != nil { + panic("failed to encode CIDs: " + err.Error()) + } + return TipSetKey{string(encoded)} +} + +// TipSetKeyFromBytes wraps an encoded key, validating correct decoding. +func TipSetKeyFromBytes(encoded []byte) (TipSetKey, error) { + _, err := decodeKey(encoded) + if err != nil { + return TipSetKey{}, err + } + return TipSetKey{string(encoded)}, nil +} + +// Cids returns a slice of the CIDs comprising this key. +func (k TipSetKey) Cids() []cid.Cid { + cids, err := decodeKey([]byte(k.value)) + if err != nil { + panic("invalid tipset key: " + err.Error()) + } + return cids +} + +// String() returns a human-readable representation of the key. +func (k TipSetKey) String() string { + b := strings.Builder{} + b.WriteString("{") + cids := k.Cids() + for i, c := range cids { + b.WriteString(c.String()) + if i < len(cids)-1 { + b.WriteString(",") + } + } + b.WriteString("}") + return b.String() +} + +// Bytes() returns a binary representation of the key. +func (k TipSetKey) Bytes() []byte { + return []byte(k.value) +} + +func encodeKey(cids []cid.Cid) ([]byte, error) { + length := uint32(len(cids)) + if length == uint32(0) { + return []byte{}, nil + } + buffer := new(bytes.Buffer) + err := binary.Write(buffer, binary.LittleEndian, length) + if err != nil { + return nil, err + } + for _, c := range cids { + b := c.Bytes() + l := uint32(len(b)) + err = binary.Write(buffer, binary.LittleEndian, l) + if err != nil { + return nil, err + } + err = binary.Write(buffer, binary.LittleEndian, c.Bytes()) + if err != nil { + return nil, err + } + } + return buffer.Bytes(), nil +} + +func decodeKey(encoded []byte) ([]cid.Cid, error) { + if len(encoded) == 0 { + return []cid.Cid{}, nil + } + + buffer := bytes.NewReader(encoded) + var length uint32 + err := binary.Read(buffer, binary.LittleEndian, &length) + if err != nil { + return nil, err + } + + var cids []cid.Cid + for idx := uint32(0); idx < length; idx++ { + var l uint32 + err = binary.Read(buffer, binary.LittleEndian, &l) + if err != nil { + return nil, err + } + buf := make([]byte, l) + err = binary.Read(buffer, binary.LittleEndian, &buf) + if err != nil { + return nil, err + } + blockCid, err := cid.Cast(buf) + if err != nil { + return nil, err + } + cids = append(cids, blockCid) + } + return cids, nil +} \ No newline at end of file diff --git a/chain/types/tipset_key_test.go b/chain/types/tipset_key_test.go new file mode 100644 index 000000000..b709ad301 --- /dev/null +++ b/chain/types/tipset_key_test.go @@ -0,0 +1,55 @@ +package types + +import ( + "testing" + + "github.com/ipfs/go-cid" + "github.com/multiformats/go-multihash" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestTipSetKey(t *testing.T) { + cb := cid.V1Builder{Codec: cid.DagCBOR, MhType: multihash.SHA2_256} + c1, _ := cb.Sum([]byte("a")) + c2, _ := cb.Sum([]byte("b")) + c3, _ := cb.Sum([]byte("c")) + + t.Run("zero value", func(t *testing.T) { + assert.Equal(t, TipSetKey{}, NewTipSetKey()) + }) + + t.Run("CID extraction", func(t *testing.T) { + assert.Equal(t, []cid.Cid{}, NewTipSetKey().Cids()) + assert.Equal(t, []cid.Cid{c1}, NewTipSetKey(c1).Cids()) + assert.Equal(t, []cid.Cid{c1, c2, c3}, NewTipSetKey(c1, c2, c3).Cids()) + + // The key doesn't check for duplicates. + assert.Equal(t, []cid.Cid{c1, c1}, NewTipSetKey(c1, c1).Cids()) + }) + + t.Run("equality", func(t *testing.T) { + assert.Equal(t, NewTipSetKey(), NewTipSetKey()) + assert.Equal(t, NewTipSetKey(c1), NewTipSetKey(c1)) + assert.Equal(t, NewTipSetKey(c1, c2, c3), NewTipSetKey(c1, c2, c3)) + + assert.NotEqual(t, NewTipSetKey(), NewTipSetKey(c1)) + assert.NotEqual(t, NewTipSetKey(c2), NewTipSetKey(c1)) + // The key doesn't normalize order. + assert.NotEqual(t, NewTipSetKey(c1, c2), NewTipSetKey(c2, c1)) + }) + + t.Run("encoding", func(t *testing.T) { + keys := []TipSetKey { + NewTipSetKey(), + NewTipSetKey(c1), + NewTipSetKey(c1, c2, c3), + } + + for _, tk := range keys { + roundTrip, err := TipSetKeyFromBytes(tk.Bytes()) + require.NoError(t, err) + assert.Equal(t, tk, roundTrip) + } + }) +} diff --git a/cli/chain.go b/cli/chain.go index 0e948d36c..10c7dd5cc 100644 --- a/cli/chain.go +++ b/cli/chain.go @@ -319,7 +319,7 @@ var chainListCmd = &cli.Command{ break } - head, err = api.ChainGetTipSet(ctx, head.Parents()) + head, err = api.ChainGetTipSet(ctx, types.NewTipSetKey(head.Parents()...)) if err != nil { return err } diff --git a/node/impl/full/chain.go b/node/impl/full/chain.go index fd3a91d66..4285682fc 100644 --- a/node/impl/full/chain.go +++ b/node/impl/full/chain.go @@ -28,7 +28,7 @@ func (a *ChainAPI) ChainHead(context.Context) (*types.TipSet, error) { return a.Chain.GetHeaviestTipSet(), nil } -func (a *ChainAPI) ChainGetRandomness(ctx context.Context, pts *types.TipSet, tickets []*types.Ticket, lb int) ([]byte, error) { +func (a *ChainAPI) ChainGetRandomness(ctx context.Context, pts types.TipSetKey, tickets []*types.Ticket, lb int) ([]byte, error) { return a.Chain.GetRandomness(ctx, pts.Cids(), tickets, int64(lb)) } @@ -36,8 +36,8 @@ func (a *ChainAPI) ChainGetBlock(ctx context.Context, msg cid.Cid) (*types.Block return a.Chain.GetBlock(msg) } -func (a *ChainAPI) ChainGetTipSet(ctx context.Context, cids []cid.Cid) (*types.TipSet, error) { - return a.Chain.LoadTipSet(cids) +func (a *ChainAPI) ChainGetTipSet(ctx context.Context, key types.TipSetKey) (*types.TipSet, error) { + return a.Chain.LoadTipSet(key.Cids()) } func (a *ChainAPI) ChainGetBlockMessages(ctx context.Context, msg cid.Cid) (*api.BlockMessages, error) { diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index 6812a7f35..e5cbe8d62 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -221,7 +221,7 @@ func SealTicketGen(api api.FullNode) storage.TicketFn { return nil, xerrors.Errorf("getting head ts for SealTicket failed: %w", err) } - r, err := api.ChainGetRandomness(ctx, ts, nil, build.SealRandomnessLookback) + r, err := api.ChainGetRandomness(ctx, ts.Key(), nil, build.SealRandomnessLookback) if err != nil { return nil, xerrors.Errorf("getting randomness for SealTicket failed: %w", err) } diff --git a/storage/miner.go b/storage/miner.go index 5cc8d37ed..af8bf5a6d 100644 --- a/storage/miner.go +++ b/storage/miner.go @@ -64,7 +64,7 @@ type storageMinerApi interface { ChainHead(context.Context) (*types.TipSet, error) ChainNotify(context.Context) (<-chan []*store.HeadChange, error) - ChainGetRandomness(context.Context, *types.TipSet, []*types.Ticket, int) ([]byte, error) + ChainGetRandomness(context.Context, types.TipSetKey, []*types.Ticket, int) ([]byte, error) ChainGetTipSetByHeight(context.Context, uint64, *types.TipSet) (*types.TipSet, error) ChainGetBlockMessages(context.Context, cid.Cid) (*api.BlockMessages, error) diff --git a/storage/post.go b/storage/post.go index 98bd798fc..ab94d719a 100644 --- a/storage/post.go +++ b/storage/post.go @@ -156,7 +156,7 @@ func (p *post) preparePost(ctx context.Context) error { // Compute how many blocks back we have to look from the given tipset for the PoSt challenge challengeLookback := int((int64(p.ts.Height()) - int64(p.ppe)) + int64(build.PoStChallangeTime) + int64(build.PoStRandomnessLookback)) - r, err := p.m.api.ChainGetRandomness(ctx, p.ts, nil, challengeLookback) + r, err := p.m.api.ChainGetRandomness(ctx, p.ts.Key(), nil, challengeLookback) if err != nil { return xerrors.Errorf("failed to get chain randomness for post (ts=%d; ppe=%d): %w", p.ts.Height(), p.ppe, err) } diff --git a/storage/sector_states.go b/storage/sector_states.go index d91e3cead..2e69b7ce7 100644 --- a/storage/sector_states.go +++ b/storage/sector_states.go @@ -145,7 +145,7 @@ func (m *Miner) preCommitted(ctx context.Context, sector SectorInfo) (func(*Sect log.Infof("precommit for sector %d made it on chain, will start proof computation at height %d", sector.SectorID, randHeight) err = m.events.ChainAt(func(ctx context.Context, ts *types.TipSet, curH uint64) error { - rand, err := m.api.ChainGetRandomness(ctx, ts, nil, int(ts.Height()-randHeight)) + rand, err := m.api.ChainGetRandomness(ctx, ts.Key(), nil, int(ts.Height()-randHeight)) if err != nil { return xerrors.Errorf("failed to get randomness for computing seal proof: %w", err) } From 36d57385abb35b4ff6c22ccedb340a510219f497 Mon Sep 17 00:00:00 2001 From: anorth Date: Tue, 12 Nov 2019 16:14:10 +1100 Subject: [PATCH 2/3] Upgrade go-cid and drop superfluous envelope bytes in TipSetKey --- chain/types/tipset_key.go | 62 ++++++++-------------------------- chain/types/tipset_key_test.go | 3 ++ go.mod | 6 ++-- go.sum | 8 +++-- 4 files changed, 26 insertions(+), 53 deletions(-) diff --git a/chain/types/tipset_key.go b/chain/types/tipset_key.go index 02633d5de..e73b9c7c7 100644 --- a/chain/types/tipset_key.go +++ b/chain/types/tipset_key.go @@ -13,15 +13,10 @@ import ( // CIDs in a different order are not considered equal. // TipSetKey is a lightweight value type, and may be compared for equality with ==. type TipSetKey struct { - // The internal representation is a concatenation of the bytes of the CIDs, each preceded by - // uint32 specifying the length of the CIDs bytes, and the whole preceded by a uint32 - // giving the number of CIDs. + // The internal representation is a concatenation of the bytes of the CIDs, which are + // self-describing, wrapped as a string. // These gymnastics make the a TipSetKey usable as a map key. - // This representation is slightly larger than strictly necessary: CIDs do carry a prefix - // including their length, but the cid package doesn't quite expose the functions needed to - // safely parse a sequence of concatenated CIDs from a stream, and we probably don't want to - // (re-)implement that here. See https://github.com/ipfs/go-cid/issues/93. - // The empty key has value "" (no encoded-zero prefix). + // The empty key has value "". value string } @@ -74,23 +69,9 @@ func (k TipSetKey) Bytes() []byte { } func encodeKey(cids []cid.Cid) ([]byte, error) { - length := uint32(len(cids)) - if length == uint32(0) { - return []byte{}, nil - } buffer := new(bytes.Buffer) - err := binary.Write(buffer, binary.LittleEndian, length) - if err != nil { - return nil, err - } for _, c := range cids { - b := c.Bytes() - l := uint32(len(b)) - err = binary.Write(buffer, binary.LittleEndian, l) - if err != nil { - return nil, err - } - err = binary.Write(buffer, binary.LittleEndian, c.Bytes()) + err := binary.Write(buffer, binary.LittleEndian, c.Bytes()) if err != nil { return nil, err } @@ -99,34 +80,19 @@ func encodeKey(cids []cid.Cid) ([]byte, error) { } func decodeKey(encoded []byte) ([]cid.Cid, error) { - if len(encoded) == 0 { - return []cid.Cid{}, nil - } - - buffer := bytes.NewReader(encoded) - var length uint32 - err := binary.Read(buffer, binary.LittleEndian, &length) - if err != nil { - return nil, err - } - - var cids []cid.Cid - for idx := uint32(0); idx < length; idx++ { - var l uint32 - err = binary.Read(buffer, binary.LittleEndian, &l) + // Estimate the number of CIDs to be extracted by dividing the encoded length by the shortest + // common CID length (V0 CIDs are 34 bytes). V1 CIDs are longer which means this might + // over-allocate, but avoid reallocation of the underlying array. + estimatedCount := len(encoded) / 34 + cids := make([]cid.Cid, 0, estimatedCount) + nextIdx := 0 + for nextIdx < len(encoded) { + nr, c, err := cid.CidFromBytes(encoded[nextIdx:]) if err != nil { return nil, err } - buf := make([]byte, l) - err = binary.Read(buffer, binary.LittleEndian, &buf) - if err != nil { - return nil, err - } - blockCid, err := cid.Cast(buf) - if err != nil { - return nil, err - } - cids = append(cids, blockCid) + cids = append(cids, c) + nextIdx += nr } return cids, nil } \ No newline at end of file diff --git a/chain/types/tipset_key_test.go b/chain/types/tipset_key_test.go index b709ad301..a316acc7a 100644 --- a/chain/types/tipset_key_test.go +++ b/chain/types/tipset_key_test.go @@ -51,5 +51,8 @@ func TestTipSetKey(t *testing.T) { require.NoError(t, err) assert.Equal(t, tk, roundTrip) } + + _, err := TipSetKeyFromBytes(NewTipSetKey(c1).Bytes()[1:]) + assert.Error(t, err) }) } diff --git a/go.mod b/go.mod index 116765150..8f3830386 100644 --- a/go.mod +++ b/go.mod @@ -27,7 +27,7 @@ require ( github.com/ipfs/go-block-format v0.0.2 github.com/ipfs/go-blockservice v0.1.3-0.20190908200855-f22eea50656c github.com/ipfs/go-car v0.0.2 - github.com/ipfs/go-cid v0.0.3 + github.com/ipfs/go-cid v0.0.4-0.20191112011718-79e75dffeb10 github.com/ipfs/go-datastore v0.1.0 github.com/ipfs/go-ds-badger v0.0.5 github.com/ipfs/go-filestore v0.0.2 @@ -74,13 +74,13 @@ require ( github.com/mattn/go-runewidth v0.0.4 // indirect github.com/miekg/dns v1.1.16 // indirect github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 - github.com/minio/sha256-simd v0.1.0 + github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771 github.com/mitchellh/go-homedir v1.1.0 github.com/multiformats/go-base32 v0.0.3 github.com/multiformats/go-multiaddr v0.0.4 github.com/multiformats/go-multiaddr-dns v0.0.3 github.com/multiformats/go-multiaddr-net v0.0.1 - github.com/multiformats/go-multihash v0.0.7 + github.com/multiformats/go-multihash v0.0.9 github.com/onsi/ginkgo v1.9.0 // indirect github.com/onsi/gomega v1.6.0 // indirect github.com/opentracing/opentracing-go v1.1.0 diff --git a/go.sum b/go.sum index 2ba4fd2b6..6da9b96f6 100644 --- a/go.sum +++ b/go.sum @@ -168,6 +168,8 @@ github.com/ipfs/go-cid v0.0.1/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUP github.com/ipfs/go-cid v0.0.2/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= github.com/ipfs/go-cid v0.0.3 h1:UIAh32wymBpStoe83YCzwVQQ5Oy/H0FdxvUS6DJDzms= github.com/ipfs/go-cid v0.0.3/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= +github.com/ipfs/go-cid v0.0.4-0.20191112011718-79e75dffeb10 h1:5mRf2p8Bv2iKiuPsGrQUrx38rdBm2T/03JCM6VWzoMc= +github.com/ipfs/go-cid v0.0.4-0.20191112011718-79e75dffeb10/go.mod h1:/BYOuUoxkE+0f6tGzlzMvycuN+5l35VOR4Bpg2sCmds= github.com/ipfs/go-datastore v0.0.1/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE= github.com/ipfs/go-datastore v0.0.5/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE= github.com/ipfs/go-datastore v0.1.0 h1:TOxI04l8CmO4zGtesENhzm4PwkFwJXY3rKiYaaMf9fI= @@ -439,6 +441,8 @@ github.com/minio/sha256-simd v0.0.0-20190131020904-2d45a736cd16/go.mod h1:2FMWW+ github.com/minio/sha256-simd v0.0.0-20190328051042-05b4dd3047e5/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U= github.com/minio/sha256-simd v0.1.0 h1:U41/2erhAKcmSI14xh/ZTUdBPOzDOIfS93ibzUSl8KM= github.com/minio/sha256-simd v0.1.0/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U= +github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771 h1:MHkK1uRtFbVqvAgvWxafZe54+5uBxLluGylDiKgdhwo= +github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= @@ -464,8 +468,8 @@ github.com/multiformats/go-multibase v0.0.1 h1:PN9/v21eLywrFWdFNsFKaU04kLJzuYzmr github.com/multiformats/go-multibase v0.0.1/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/gviWFaSteVbWT51qgs= github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U= github.com/multiformats/go-multihash v0.0.5/go.mod h1:lt/HCbqlQwlPBz7lv0sQCdtfcMtlJvakRUn/0Ual8po= -github.com/multiformats/go-multihash v0.0.7 h1:uoqoE03rGJdlQEPq2EAc6UeSbo4L7mZyeAAoqNalf54= -github.com/multiformats/go-multihash v0.0.7/go.mod h1:XuKXPp8VHcTygube3OWZC+aZrA+H1IhmjoCDtJc7PXM= +github.com/multiformats/go-multihash v0.0.9 h1:aoijQXYYl7Xtb2pUUP68R+ys1TlnlR3eX6wmozr0Hp4= +github.com/multiformats/go-multihash v0.0.9/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= github.com/multiformats/go-multistream v0.0.1/go.mod h1:fJTiDfXJVmItycydCnNx4+wSzZ5NwG2FEVAI30fiovg= github.com/multiformats/go-multistream v0.1.0 h1:UpO6jrsjqs46mqAK3n6wKRYFhugss9ArzbyUzU+4wkQ= github.com/multiformats/go-multistream v0.1.0/go.mod h1:fJTiDfXJVmItycydCnNx4+wSzZ5NwG2FEVAI30fiovg= From 14c2ee56f6e588301e244cdd67651921fad574da Mon Sep 17 00:00:00 2001 From: anorth Date: Wed, 13 Nov 2019 15:18:02 +1100 Subject: [PATCH 3/3] Add JSON marshalling and optimize decoding pre-allocation to use expected (rather than min) CID length --- chain/types/tipset_key.go | 51 +++++++++++++++++++++++----------- chain/types/tipset_key_test.go | 26 ++++++++++++++++- 2 files changed, 60 insertions(+), 17 deletions(-) diff --git a/chain/types/tipset_key.go b/chain/types/tipset_key.go index e73b9c7c7..34b1c8f18 100644 --- a/chain/types/tipset_key.go +++ b/chain/types/tipset_key.go @@ -2,12 +2,24 @@ package types import ( "bytes" - "encoding/binary" + "encoding/json" "strings" "github.com/ipfs/go-cid" + "github.com/multiformats/go-multihash" ) +// The length of a block header CID in bytes. +var blockHeaderCIDLen int + +func init() { + c, err := cid.V1Builder{Codec: cid.DagCBOR, MhType: multihash.BLAKE2B_MIN + 31}.Sum([]byte{}) + if err != nil { + panic(err) + } + blockHeaderCIDLen = len(c.Bytes()) +} + // A TipSetKey is an immutable collection of CIDs forming a unique key for a tipset. // The CIDs are assumed to be distinct and in canonical order. Two keys with the same // CIDs in a different order are not considered equal. @@ -23,10 +35,7 @@ type TipSetKey struct { // NewTipSetKey builds a new key from a slice of CIDs. // The CIDs are assumed to be ordered correctly. func NewTipSetKey(cids ...cid.Cid) TipSetKey { - encoded, err := encodeKey(cids) - if err != nil { - panic("failed to encode CIDs: " + err.Error()) - } + encoded := encodeKey(cids) return TipSetKey{string(encoded)} } @@ -68,22 +77,32 @@ func (k TipSetKey) Bytes() []byte { return []byte(k.value) } -func encodeKey(cids []cid.Cid) ([]byte, error) { +func (k *TipSetKey) MarshalJSON() ([]byte, error) { + return json.Marshal(k.Cids()) +} + +func (k *TipSetKey) UnmarshalJSON(b []byte) error { + var cids []cid.Cid + if err := json.Unmarshal(b, &cids); err != nil { + return err + } + k.value = string(encodeKey(cids)) + return nil +} + +func encodeKey(cids []cid.Cid) []byte { buffer := new(bytes.Buffer) for _, c := range cids { - err := binary.Write(buffer, binary.LittleEndian, c.Bytes()) - if err != nil { - return nil, err - } + // bytes.Buffer.Write() err is documented to be always nil. + _, _ = buffer.Write(c.Bytes()) } - return buffer.Bytes(), nil + return buffer.Bytes() } func decodeKey(encoded []byte) ([]cid.Cid, error) { - // Estimate the number of CIDs to be extracted by dividing the encoded length by the shortest - // common CID length (V0 CIDs are 34 bytes). V1 CIDs are longer which means this might - // over-allocate, but avoid reallocation of the underlying array. - estimatedCount := len(encoded) / 34 + // To avoid reallocation of the underlying array, estimate the number of CIDs to be extracted + // by dividing the encoded length by the expected CID length. + estimatedCount := len(encoded) / blockHeaderCIDLen cids := make([]cid.Cid, 0, estimatedCount) nextIdx := 0 for nextIdx < len(encoded) { @@ -95,4 +114,4 @@ func decodeKey(encoded []byte) ([]cid.Cid, error) { nextIdx += nr } return cids, nil -} \ No newline at end of file +} diff --git a/chain/types/tipset_key_test.go b/chain/types/tipset_key_test.go index a316acc7a..c6411a741 100644 --- a/chain/types/tipset_key_test.go +++ b/chain/types/tipset_key_test.go @@ -1,6 +1,7 @@ package types import ( + "fmt" "testing" "github.com/ipfs/go-cid" @@ -10,10 +11,11 @@ import ( ) func TestTipSetKey(t *testing.T) { - cb := cid.V1Builder{Codec: cid.DagCBOR, MhType: multihash.SHA2_256} + cb := cid.V1Builder{Codec: cid.DagCBOR, MhType: multihash.BLAKE2B_MIN+31} c1, _ := cb.Sum([]byte("a")) c2, _ := cb.Sum([]byte("b")) c3, _ := cb.Sum([]byte("c")) + fmt.Println(len(c1.Bytes())) t.Run("zero value", func(t *testing.T) { assert.Equal(t, TipSetKey{}, NewTipSetKey()) @@ -55,4 +57,26 @@ func TestTipSetKey(t *testing.T) { _, err := TipSetKeyFromBytes(NewTipSetKey(c1).Bytes()[1:]) assert.Error(t, err) }) + + t.Run("JSON", func(t *testing.T) { + k0 := NewTipSetKey() + verifyJson(t, "[]", k0) + k3 := NewTipSetKey(c1, c2, c3) + verifyJson(t, `[` + + `{"/":"bafy2bzacecesrkxghscnq7vatble2hqdvwat6ed23vdu4vvo3uuggsoaya7ki"},` + + `{"/":"bafy2bzacebxfyh2fzoxrt6kcgc5dkaodpcstgwxxdizrww225vrhsizsfcg4g"},` + + `{"/":"bafy2bzacedwviarjtjraqakob5pslltmuo5n3xev3nt5zylezofkbbv5jclyu"}` + + `]`, k3) + }) } + +func verifyJson(t *testing.T, expected string, k TipSetKey) { + bytes, err := k.MarshalJSON() + require.NoError(t, err) + assert.Equal(t, expected, string(bytes)) + + var rehydrated TipSetKey + err = rehydrated.UnmarshalJSON(bytes) + require.NoError(t, err) + assert.Equal(t, k, rehydrated) +} \ No newline at end of file