From 36d57385abb35b4ff6c22ccedb340a510219f497 Mon Sep 17 00:00:00 2001 From: anorth Date: Tue, 12 Nov 2019 16:14:10 +1100 Subject: [PATCH] Upgrade go-cid and drop superfluous envelope bytes in TipSetKey --- chain/types/tipset_key.go | 62 ++++++++-------------------------- chain/types/tipset_key_test.go | 3 ++ go.mod | 6 ++-- go.sum | 8 +++-- 4 files changed, 26 insertions(+), 53 deletions(-) diff --git a/chain/types/tipset_key.go b/chain/types/tipset_key.go index 02633d5de..e73b9c7c7 100644 --- a/chain/types/tipset_key.go +++ b/chain/types/tipset_key.go @@ -13,15 +13,10 @@ import ( // CIDs in a different order are not considered equal. // TipSetKey is a lightweight value type, and may be compared for equality with ==. type TipSetKey struct { - // The internal representation is a concatenation of the bytes of the CIDs, each preceded by - // uint32 specifying the length of the CIDs bytes, and the whole preceded by a uint32 - // giving the number of CIDs. + // The internal representation is a concatenation of the bytes of the CIDs, which are + // self-describing, wrapped as a string. // These gymnastics make the a TipSetKey usable as a map key. - // This representation is slightly larger than strictly necessary: CIDs do carry a prefix - // including their length, but the cid package doesn't quite expose the functions needed to - // safely parse a sequence of concatenated CIDs from a stream, and we probably don't want to - // (re-)implement that here. See https://github.com/ipfs/go-cid/issues/93. - // The empty key has value "" (no encoded-zero prefix). + // The empty key has value "". value string } @@ -74,23 +69,9 @@ func (k TipSetKey) Bytes() []byte { } func encodeKey(cids []cid.Cid) ([]byte, error) { - length := uint32(len(cids)) - if length == uint32(0) { - return []byte{}, nil - } buffer := new(bytes.Buffer) - err := binary.Write(buffer, binary.LittleEndian, length) - if err != nil { - return nil, err - } for _, c := range cids { - b := c.Bytes() - l := uint32(len(b)) - err = binary.Write(buffer, binary.LittleEndian, l) - if err != nil { - return nil, err - } - err = binary.Write(buffer, binary.LittleEndian, c.Bytes()) + err := binary.Write(buffer, binary.LittleEndian, c.Bytes()) if err != nil { return nil, err } @@ -99,34 +80,19 @@ func encodeKey(cids []cid.Cid) ([]byte, error) { } func decodeKey(encoded []byte) ([]cid.Cid, error) { - if len(encoded) == 0 { - return []cid.Cid{}, nil - } - - buffer := bytes.NewReader(encoded) - var length uint32 - err := binary.Read(buffer, binary.LittleEndian, &length) - if err != nil { - return nil, err - } - - var cids []cid.Cid - for idx := uint32(0); idx < length; idx++ { - var l uint32 - err = binary.Read(buffer, binary.LittleEndian, &l) + // Estimate the number of CIDs to be extracted by dividing the encoded length by the shortest + // common CID length (V0 CIDs are 34 bytes). V1 CIDs are longer which means this might + // over-allocate, but avoid reallocation of the underlying array. + estimatedCount := len(encoded) / 34 + cids := make([]cid.Cid, 0, estimatedCount) + nextIdx := 0 + for nextIdx < len(encoded) { + nr, c, err := cid.CidFromBytes(encoded[nextIdx:]) if err != nil { return nil, err } - buf := make([]byte, l) - err = binary.Read(buffer, binary.LittleEndian, &buf) - if err != nil { - return nil, err - } - blockCid, err := cid.Cast(buf) - if err != nil { - return nil, err - } - cids = append(cids, blockCid) + cids = append(cids, c) + nextIdx += nr } return cids, nil } \ No newline at end of file diff --git a/chain/types/tipset_key_test.go b/chain/types/tipset_key_test.go index b709ad301..a316acc7a 100644 --- a/chain/types/tipset_key_test.go +++ b/chain/types/tipset_key_test.go @@ -51,5 +51,8 @@ func TestTipSetKey(t *testing.T) { require.NoError(t, err) assert.Equal(t, tk, roundTrip) } + + _, err := TipSetKeyFromBytes(NewTipSetKey(c1).Bytes()[1:]) + assert.Error(t, err) }) } diff --git a/go.mod b/go.mod index 116765150..8f3830386 100644 --- a/go.mod +++ b/go.mod @@ -27,7 +27,7 @@ require ( github.com/ipfs/go-block-format v0.0.2 github.com/ipfs/go-blockservice v0.1.3-0.20190908200855-f22eea50656c github.com/ipfs/go-car v0.0.2 - github.com/ipfs/go-cid v0.0.3 + github.com/ipfs/go-cid v0.0.4-0.20191112011718-79e75dffeb10 github.com/ipfs/go-datastore v0.1.0 github.com/ipfs/go-ds-badger v0.0.5 github.com/ipfs/go-filestore v0.0.2 @@ -74,13 +74,13 @@ require ( github.com/mattn/go-runewidth v0.0.4 // indirect github.com/miekg/dns v1.1.16 // indirect github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 - github.com/minio/sha256-simd v0.1.0 + github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771 github.com/mitchellh/go-homedir v1.1.0 github.com/multiformats/go-base32 v0.0.3 github.com/multiformats/go-multiaddr v0.0.4 github.com/multiformats/go-multiaddr-dns v0.0.3 github.com/multiformats/go-multiaddr-net v0.0.1 - github.com/multiformats/go-multihash v0.0.7 + github.com/multiformats/go-multihash v0.0.9 github.com/onsi/ginkgo v1.9.0 // indirect github.com/onsi/gomega v1.6.0 // indirect github.com/opentracing/opentracing-go v1.1.0 diff --git a/go.sum b/go.sum index 2ba4fd2b6..6da9b96f6 100644 --- a/go.sum +++ b/go.sum @@ -168,6 +168,8 @@ github.com/ipfs/go-cid v0.0.1/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUP github.com/ipfs/go-cid v0.0.2/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= github.com/ipfs/go-cid v0.0.3 h1:UIAh32wymBpStoe83YCzwVQQ5Oy/H0FdxvUS6DJDzms= github.com/ipfs/go-cid v0.0.3/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= +github.com/ipfs/go-cid v0.0.4-0.20191112011718-79e75dffeb10 h1:5mRf2p8Bv2iKiuPsGrQUrx38rdBm2T/03JCM6VWzoMc= +github.com/ipfs/go-cid v0.0.4-0.20191112011718-79e75dffeb10/go.mod h1:/BYOuUoxkE+0f6tGzlzMvycuN+5l35VOR4Bpg2sCmds= github.com/ipfs/go-datastore v0.0.1/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE= github.com/ipfs/go-datastore v0.0.5/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE= github.com/ipfs/go-datastore v0.1.0 h1:TOxI04l8CmO4zGtesENhzm4PwkFwJXY3rKiYaaMf9fI= @@ -439,6 +441,8 @@ github.com/minio/sha256-simd v0.0.0-20190131020904-2d45a736cd16/go.mod h1:2FMWW+ github.com/minio/sha256-simd v0.0.0-20190328051042-05b4dd3047e5/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U= github.com/minio/sha256-simd v0.1.0 h1:U41/2erhAKcmSI14xh/ZTUdBPOzDOIfS93ibzUSl8KM= github.com/minio/sha256-simd v0.1.0/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U= +github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771 h1:MHkK1uRtFbVqvAgvWxafZe54+5uBxLluGylDiKgdhwo= +github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= @@ -464,8 +468,8 @@ github.com/multiformats/go-multibase v0.0.1 h1:PN9/v21eLywrFWdFNsFKaU04kLJzuYzmr github.com/multiformats/go-multibase v0.0.1/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/gviWFaSteVbWT51qgs= github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U= github.com/multiformats/go-multihash v0.0.5/go.mod h1:lt/HCbqlQwlPBz7lv0sQCdtfcMtlJvakRUn/0Ual8po= -github.com/multiformats/go-multihash v0.0.7 h1:uoqoE03rGJdlQEPq2EAc6UeSbo4L7mZyeAAoqNalf54= -github.com/multiformats/go-multihash v0.0.7/go.mod h1:XuKXPp8VHcTygube3OWZC+aZrA+H1IhmjoCDtJc7PXM= +github.com/multiformats/go-multihash v0.0.9 h1:aoijQXYYl7Xtb2pUUP68R+ys1TlnlR3eX6wmozr0Hp4= +github.com/multiformats/go-multihash v0.0.9/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= github.com/multiformats/go-multistream v0.0.1/go.mod h1:fJTiDfXJVmItycydCnNx4+wSzZ5NwG2FEVAI30fiovg= github.com/multiformats/go-multistream v0.1.0 h1:UpO6jrsjqs46mqAK3n6wKRYFhugss9ArzbyUzU+4wkQ= github.com/multiformats/go-multistream v0.1.0/go.mod h1:fJTiDfXJVmItycydCnNx4+wSzZ5NwG2FEVAI30fiovg=