From 137a781beeb2429884f9b15e14b491e2d53d12b1 Mon Sep 17 00:00:00 2001 From: Sai Kumar Date: Mon, 23 May 2022 12:38:46 +0530 Subject: [PATCH 1/3] feat: add `go-ipld-prime` for encode and decode CID --- go.mod | 5 +++-- go.sum | 11 ++++++++++ utils/json.go | 59 +++++++++++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 71 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 111a5f42..c4c67fde 100644 --- a/go.mod +++ b/go.mod @@ -118,9 +118,10 @@ require ( github.com/huin/goupnp v1.0.3-0.20220313090229-ca81a64b4204 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/ipfs/go-block-format v0.0.2 // indirect - github.com/ipfs/go-cid v0.0.3 // indirect + github.com/ipfs/go-cid v0.0.4 // indirect github.com/ipfs/go-ipfs-util v0.0.1 // indirect github.com/ipfs/go-ipld-format v0.0.1 // indirect + github.com/ipld/go-ipld-prime v0.16.0 // indirect github.com/jackpal/go-nat-pmp v1.0.2 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/jmhodges/levigo v1.0.0 // indirect @@ -151,7 +152,7 @@ require ( github.com/pelletier/go-toml v1.9.4 // indirect github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/polydawn/refmt v0.0.0-20190221155625-df39d6c2d992 // indirect + github.com/polydawn/refmt v0.0.0-20201211092308-30ac6d18308e // indirect github.com/prometheus/client_golang v1.12.1 // indirect github.com/prometheus/client_model v0.2.0 // indirect github.com/prometheus/common v0.33.0 // indirect diff --git a/go.sum b/go.sum index c4f6b0e9..fcc2d812 100644 --- a/go.sum +++ b/go.sum @@ -416,6 +416,7 @@ github.com/franela/goblin v0.0.0-20210519012713-85d372ac71e2/go.mod h1:VzmDKDJVZ github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/frankban/quicktest v1.14.0/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09mUdL/ijj8og= +github.com/frankban/quicktest v1.14.2/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI= @@ -785,12 +786,16 @@ github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJ github.com/ipfs/go-cid v0.0.1/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= github.com/ipfs/go-cid v0.0.3 h1:UIAh32wymBpStoe83YCzwVQQ5Oy/H0FdxvUS6DJDzms= github.com/ipfs/go-cid v0.0.3/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= +github.com/ipfs/go-cid v0.0.4 h1:UlfXKrZx1DjZoBhQHmNHLC1fK1dUJDN20Y28A7s+gJ8= +github.com/ipfs/go-cid v0.0.4/go.mod h1:4LLaPOQwmk5z9LBgQnpkivrx8BJjUyGwTXCd5Xfj6+M= github.com/ipfs/go-ipfs-util v0.0.1 h1:Wz9bL2wB2YBJqggkA4dD7oSmqB4cAnpNbGrlHJulv50= github.com/ipfs/go-ipfs-util v0.0.1/go.mod h1:spsl5z8KUnrve+73pOhSVZND1SIxPW5RyBCNzQxlJBc= github.com/ipfs/go-ipld-cbor v0.0.6 h1:pYuWHyvSpIsOOLw4Jy7NbBkCyzLDcl64Bf/LZW7eBQ0= github.com/ipfs/go-ipld-cbor v0.0.6/go.mod h1:ssdxxaLJPXH7OjF5V4NSjBbcfh+evoR4ukuru0oPXMA= github.com/ipfs/go-ipld-format v0.0.1 h1:HCu4eB/Gh+KD/Q0M8u888RFkorTWNIL3da4oc5dwc80= github.com/ipfs/go-ipld-format v0.0.1/go.mod h1:kyJtbkDALmFHv3QR6et67i35QzO3S0dCDnkOJhcZkms= +github.com/ipld/go-ipld-prime v0.16.0 h1:RS5hhjB/mcpeEPJvfyj0qbOj/QL+/j05heZ0qa97dVo= +github.com/ipld/go-ipld-prime v0.16.0/go.mod h1:axSCuOCBPqrH+gvXr2w9uAOulJqBPhHPT2PjoiiU1qA= github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= @@ -1019,6 +1024,7 @@ github.com/multiformats/go-base32 v0.0.3 h1:tw5+NhuwaOjJCC5Pp82QuXbrmLzWg7uxlMFp github.com/multiformats/go-base32 v0.0.3/go.mod h1:pLiuGC8y0QR3Ue4Zug5UzK9LjgbkL8NSQj0zQ5Nz/AA= github.com/multiformats/go-multibase v0.0.1 h1:PN9/v21eLywrFWdFNsFKaU04kLJzuYzmrJR+ubhT9qA= github.com/multiformats/go-multibase v0.0.1/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/gviWFaSteVbWT51qgs= +github.com/multiformats/go-multicodec v0.3.0/go.mod h1:qGGaQmioCDh+TeFOnxrbU0DaIPw8yFgAZgFG0V7p1qQ= github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U= github.com/multiformats/go-multihash v0.0.10/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= github.com/multiformats/go-multihash v0.1.0 h1:CgAgwqk3//SVEw3T+6DqI4mWMyRuDwZtOWcJT0q9+EA= @@ -1145,6 +1151,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/polydawn/refmt v0.0.0-20190221155625-df39d6c2d992 h1:bzMe+2coZJYHnhGgVlcQKuRy4FSny4ds8dLQjw5P1XE= github.com/polydawn/refmt v0.0.0-20190221155625-df39d6c2d992/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o= +github.com/polydawn/refmt v0.0.0-20201211092308-30ac6d18308e h1:ZOcivgkkFRnjfoTcGsDq3UQYiBmekwLA+qg0OjyB/ls= +github.com/polydawn/refmt v0.0.0-20201211092308-30ac6d18308e/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o= github.com/polyfloyd/go-errorlint v0.0.0-20211125173453-6d6d39c5bb8b/go.mod h1:wi9BfjxjF/bwiZ701TzmfKu6UKC357IOAtNr0Td0Lvw= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s= @@ -1413,8 +1421,11 @@ github.com/vulcanize/cosmos-sdk v0.46.0-smt-0.0.3-alpha h1:uo9s4iQ5GFqEPBaWWCxc1 github.com/vulcanize/cosmos-sdk v0.46.0-smt-0.0.3-alpha/go.mod h1:qvWphKMtDelR8tVWRq1+kQRHgMSanu3hfM/ahK7X86Q= github.com/vulcanize/cosmos-sdk/db v1.0.0-beta.1 h1:fLVdRk/JEWN5SGdOERhjHWU/zq3Efxeofu3VmJsrk90= github.com/vulcanize/cosmos-sdk/db v1.0.0-beta.1/go.mod h1:n79xAGrkFZLzudBwLIgQCa62PTHx9mCymwC+lSvoR2s= +github.com/warpfork/go-testmark v0.3.0/go.mod h1:jhEf8FVxd+F17juRubpmut64NEG6I2rgkUhlcqqXwE0= github.com/warpfork/go-wish v0.0.0-20180510122957-5ad1f5abf436 h1:qOpVTI+BrstcjTZLm2Yz/3sOnqkzj3FQoh0g+E5s3Gc= github.com/warpfork/go-wish v0.0.0-20180510122957-5ad1f5abf436/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= +github.com/warpfork/go-wish v0.0.0-20200122115046-b9ea61034e4a h1:G++j5e0OC488te356JvdhaM8YS6nMsjLAYF7JxCv07w= +github.com/warpfork/go-wish v0.0.0-20200122115046-b9ea61034e4a/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= github.com/whyrusleeping/cbor-gen v0.0.0-20200123233031-1cdf64d27158 h1:WXhVOwj2USAXB5oMDwRl3piOux2XMV9TANaYxXHdkoE= github.com/whyrusleeping/cbor-gen v0.0.0-20200123233031-1cdf64d27158/go.mod h1:Xj/M2wWU+QdTdRbu/L/1dIZY8/Wb2K9pAhtroQuxJJI= github.com/willf/bitset v1.1.3/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= diff --git a/utils/json.go b/utils/json.go index b6faafe2..959fe0b5 100644 --- a/utils/json.go +++ b/utils/json.go @@ -7,12 +7,25 @@ package utils import ( "bytes" "errors" + "github.com/ipfs/go-cid" + "github.com/ipld/go-ipld-prime/codec/dagcbor" + "github.com/ipld/go-ipld-prime/fluent" + "github.com/ipld/go-ipld-prime/linking" + cidlink "github.com/ipld/go-ipld-prime/linking/cid" + "github.com/ipld/go-ipld-prime/multicodec" + "github.com/ipld/go-ipld-prime/storage/memstore" canonicalJson "github.com/gibson042/canonicaljson-go" cbor "github.com/ipfs/go-ipld-cbor" + basicnode "github.com/ipld/go-ipld-prime/node/basic" mh "github.com/multiformats/go-multihash" ) +func init() { + multicodec.RegisterEncoder(0x71, dagcbor.Encode) + multicodec.RegisterDecoder(0x71, dagcbor.Decode) +} + // GenerateHash returns the hash of the canonicalized JSON input. func GenerateHash(json map[string]interface{}) (string, []byte, error) { content, err := canonicalJson.Marshal(json) @@ -20,12 +33,13 @@ func GenerateHash(json map[string]interface{}) (string, []byte, error) { return "", nil, err } - cid, err := CIDFromJSONBytes(content) + //cid, err := CIDFromJSONBytes(content) + cidString, err := CIDFromJSONBytesUsingIpldPrime(content) if err != nil { return "", nil, err } - return cid, content, nil + return cidString, content, nil } // CIDFromJSONBytes returns CID (cbor) for json (as bytes). @@ -50,3 +64,44 @@ func GetAttributeAsString(obj map[string]interface{}, attr string) (string, erro return "", errors.New("attribute not found") } + +// CIDFromJSONBytesUsingIpldPrime returns CID (dagcbor) for json (as bytes). +func CIDFromJSONBytesUsingIpldPrime(content []byte) (string, error) { + lsys := cidlink.DefaultLinkSystem() + var store = memstore.Store{} + + // We want to store the serialized data somewhere. + // We'll use an in-memory store for this. (It's a package scoped variable.) + // You can use any kind of storage system here; + // or if you need even more control, you could also write a function that conforms to the linking.BlockWriteOpener interface. + lsys.SetWriteStorage(&store) + // To create any links, first we need a LinkPrototype. + // This gathers together any parameters that might be needed when making a link. + // (For CIDs, the version, the codec, and the multihash type are all parameters we'll need.) + // Often, you can probably make this a constant for your whole application. + lp := cidlink.LinkPrototype{Prefix: cid.Prefix{ + Version: 1, // Usually '1'. + Codec: 0x71, // 0x71 means "dag-cbor" -- See the multicodecs table: https://github.com/multiformats/multicodec/ + MhType: 0x13, // 0x20 means "sha2-512" -- See the multicodecs table: https://github.com/multiformats/multicodec/ + MhLength: 64, // sha2-512 hash has a 64-byte sum. + }} + + // And we need some data to link to! Here's a quick piece of example data: + n, err := fluent.Build(basicnode.Prototype.Any, func(na fluent.NodeAssembler) { + na.AssignBytes(content) + }) + if err != nil { + return "", err + } + + // Now: time to apply the LinkSystem, and do the actual store operation! + lnk, err := lsys.Store( + linking.LinkContext{}, // The zero value is fine. Configure it it you want cancellability or other features. + lp, // The LinkPrototype says what codec and hashing to use. + n, // And here's our data. + ) + if err != nil { + return "", err + } + return lnk.String(), nil +} From 399bf4f69f63ca36826a3930a1dd27ba28e56b31 Mon Sep 17 00:00:00 2001 From: Sai Kumar Date: Mon, 23 May 2022 12:40:27 +0530 Subject: [PATCH 2/3] chore: made `store` as global --- utils/json.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/utils/json.go b/utils/json.go index 959fe0b5..f3a4ed42 100644 --- a/utils/json.go +++ b/utils/json.go @@ -21,6 +21,8 @@ import ( mh "github.com/multiformats/go-multihash" ) +var store = memstore.Store{} + func init() { multicodec.RegisterEncoder(0x71, dagcbor.Encode) multicodec.RegisterDecoder(0x71, dagcbor.Decode) @@ -68,7 +70,6 @@ func GetAttributeAsString(obj map[string]interface{}, attr string) (string, erro // CIDFromJSONBytesUsingIpldPrime returns CID (dagcbor) for json (as bytes). func CIDFromJSONBytesUsingIpldPrime(content []byte) (string, error) { lsys := cidlink.DefaultLinkSystem() - var store = memstore.Store{} // We want to store the serialized data somewhere. // We'll use an in-memory store for this. (It's a package scoped variable.) From dde10385713fcd66aca63f3ecb390e48dcf87f4b Mon Sep 17 00:00:00 2001 From: Sai Kumar Date: Fri, 3 Jun 2022 19:36:06 +0530 Subject: [PATCH 3/3] chroe: change the hashing algo --- utils/json.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/utils/json.go b/utils/json.go index f3a4ed42..a1062e73 100644 --- a/utils/json.go +++ b/utils/json.go @@ -7,6 +7,7 @@ package utils import ( "bytes" "errors" + "github.com/ipfs/go-cid" "github.com/ipld/go-ipld-prime/codec/dagcbor" "github.com/ipld/go-ipld-prime/fluent" @@ -83,8 +84,8 @@ func CIDFromJSONBytesUsingIpldPrime(content []byte) (string, error) { lp := cidlink.LinkPrototype{Prefix: cid.Prefix{ Version: 1, // Usually '1'. Codec: 0x71, // 0x71 means "dag-cbor" -- See the multicodecs table: https://github.com/multiformats/multicodec/ - MhType: 0x13, // 0x20 means "sha2-512" -- See the multicodecs table: https://github.com/multiformats/multicodec/ - MhLength: 64, // sha2-512 hash has a 64-byte sum. + MhType: 0x12, // 0x12 means "sha2-256" -- See the multicodecs table: https://github.com/multiformats/multicodec/ + MhLength: 32, // sha2-256 hash has a 32-byte sum. }} // And we need some data to link to! Here's a quick piece of example data: