From 15d2e2525fd6468787d2897cd75e3a8725cb2294 Mon Sep 17 00:00:00 2001 From: Alexis Sellier Date: Mon, 7 Aug 2017 17:15:16 +0200 Subject: [PATCH 01/22] Start working on proof tests --- client/commands/query/get.go | 26 ++++++--- client/commands/query/query_test.go | 85 +++++++++++++++++++++++++++++ client/commands/query/tx.go | 7 ++- 3 files changed, 108 insertions(+), 10 deletions(-) create mode 100644 client/commands/query/query_test.go diff --git a/client/commands/query/get.go b/client/commands/query/get.go index 91831892d9..f608c71acc 100644 --- a/client/commands/query/get.go +++ b/client/commands/query/get.go @@ -18,6 +18,10 @@ import ( "github.com/tendermint/basecoin/client/commands" ) +type Certifier interface { + Certify(check lc.Checkpoint) error +} + // GetParsed does most of the work of the query commands, but is quite // opinionated, so if you want more control about parsing, call Get // directly. @@ -60,6 +64,16 @@ func Get(key []byte, prove bool) (data.Bytes, uint64, error) { // proof for the key-value pair if it exists, and all checks pass. func GetWithProof(key []byte) (data.Bytes, uint64, *iavl.KeyExistsProof, error) { node := commands.GetNode() + cert, err := commands.GetCertifier() + if err != nil { + return nil, 0, nil, err + } + return CustomGetWithProof(key, node, cert) +} + +// TODO: fix this up alexis +func CustomGetWithProof(key []byte, node client.Client, + cert Certifier) (data.Bytes, uint64, *iavl.KeyExistsProof, error) { resp, err := node.ABCIQuery("/key", key, true) if err != nil { @@ -79,7 +93,7 @@ func GetWithProof(key []byte) (data.Bytes, uint64, *iavl.KeyExistsProof, error) return nil, 0, nil, lc.ErrHeightMismatch(ph, int(resp.Height)) } - check, err := GetCertifiedCheckpoint(ph) + check, err := GetCertifiedCheckpoint(ph, node, cert) if err != nil { return nil, 0, nil, err } @@ -101,14 +115,8 @@ func GetWithProof(key []byte) (data.Bytes, uint64, *iavl.KeyExistsProof, error) // GetCertifiedCheckpoint gets the signed header for a given height // and certifies it. Returns error if unable to get a proven header. -func GetCertifiedCheckpoint(h int) (empty lc.Checkpoint, err error) { - // here is the certifier, root of all trust - node := commands.GetNode() - cert, err := commands.GetCertifier() - if err != nil { - return - } - +func GetCertifiedCheckpoint(h int, node client.Client, + cert Certifier) (empty lc.Checkpoint, err error) { // get the checkpoint for this height // FIXME: cannot use cert.GetByHeight for now, as it also requires diff --git a/client/commands/query/query_test.go b/client/commands/query/query_test.go new file mode 100644 index 0000000000..5915198fb3 --- /dev/null +++ b/client/commands/query/query_test.go @@ -0,0 +1,85 @@ +package query + +import ( + "os" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/tendermint/go-wire" + "github.com/tendermint/light-client/certifiers" + certclient "github.com/tendermint/light-client/certifiers/client" + nm "github.com/tendermint/tendermint/node" + "github.com/tendermint/tendermint/rpc/client" + rpctest "github.com/tendermint/tendermint/rpc/test" + "github.com/tendermint/tmlibs/log" + + "github.com/tendermint/basecoin/app" + "github.com/tendermint/basecoin/modules/etc" +) + +var node *nm.Node + +func TestMain(m *testing.M) { + logger := log.TestingLogger() + store, err := app.NewStore("", 0, logger) + if err != nil { + panic(err) + } + app := app.NewBasecoin(etc.NewHandler(), store, logger) + node = rpctest.StartTendermint(app) + + code := m.Run() + + node.Stop() + node.Wait() + os.Exit(code) +} + +func TestAppProofs(t *testing.T) { + assert, require := assert.New(t), require.New(t) + + cl := client.NewLocal(node) + time.Sleep(200 * time.Millisecond) + + k := []byte("my-key") + v := []byte("my-value") + + tx := etc.SetTx{Key: k, Value: v}.Wrap() + btx := wire.BinaryBytes(tx) + br, err := cl.BroadcastTxCommit(btx) + require.Nil(err, "%+v", err) + require.EqualValues(0, br.CheckTx.Code, "%#v", br.CheckTx) + require.EqualValues(0, br.DeliverTx.Code) + + // this sets up our trust on the node based on some past point. + // maybe this can be cleaned up and made easy to reuse + source := certclient.New(cl) + trusted := certifiers.NewMemStoreProvider() + // let's start with some trust before the query... + seed, err := source.GetByHeight(br.Height - 2) + require.Nil(err, "%+v", err) + cert := certifiers.NewInquiring("my-chain", seed, trusted, source) + + // Test existing key. + + bs, _, proof, err := CustomGetWithProof(k, cl, cert) + require.Nil(err, "%+v", err) + require.NotNil(proof) + + var data etc.Data + err = wire.ReadBinaryBytes(bs, &data) + require.Nil(err, "%+v", err) + assert.EqualValues(v, data.Value) + + // Test non-existing key. + + // TODO: This currently fails. + missing := []byte("my-missing-key") + bs, _, proof, err = CustomGetWithProof(missing, cl, cert) + require.Nil(err, "%+v", err) + require.Nil(bs) + require.NotNil(proof) +} diff --git a/client/commands/query/tx.go b/client/commands/query/tx.go index 873deb8666..1d520f89e5 100644 --- a/client/commands/query/tx.go +++ b/client/commands/query/tx.go @@ -46,7 +46,12 @@ func txQueryCmd(cmd *cobra.Command, args []string) error { return showTx(res.Height, res.Tx) } - check, err := GetCertifiedCheckpoint(res.Height) + cert, err := commands.GetCertifier() + if err != nil { + return err + } + + check, err := GetCertifiedCheckpoint(res.Height, node, cert) if err != nil { return err } From 538824d7366419c9d91e8c9095ada8cd55d51cd8 Mon Sep 17 00:00:00 2001 From: Alexis Sellier Date: Mon, 7 Aug 2017 19:24:58 +0200 Subject: [PATCH 02/22] Change module import etc -> eyes --- client/commands/query/query_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/client/commands/query/query_test.go b/client/commands/query/query_test.go index 5915198fb3..862b4ceb53 100644 --- a/client/commands/query/query_test.go +++ b/client/commands/query/query_test.go @@ -17,7 +17,7 @@ import ( "github.com/tendermint/tmlibs/log" "github.com/tendermint/basecoin/app" - "github.com/tendermint/basecoin/modules/etc" + "github.com/tendermint/basecoin/modules/eyes" ) var node *nm.Node @@ -28,7 +28,7 @@ func TestMain(m *testing.M) { if err != nil { panic(err) } - app := app.NewBasecoin(etc.NewHandler(), store, logger) + app := app.NewBasecoin(eyes.NewHandler(), store, logger) node = rpctest.StartTendermint(app) code := m.Run() @@ -47,7 +47,7 @@ func TestAppProofs(t *testing.T) { k := []byte("my-key") v := []byte("my-value") - tx := etc.SetTx{Key: k, Value: v}.Wrap() + tx := eyes.SetTx{Key: k, Value: v}.Wrap() btx := wire.BinaryBytes(tx) br, err := cl.BroadcastTxCommit(btx) require.Nil(err, "%+v", err) @@ -69,7 +69,7 @@ func TestAppProofs(t *testing.T) { require.Nil(err, "%+v", err) require.NotNil(proof) - var data etc.Data + var data eyes.Data err = wire.ReadBinaryBytes(bs, &data) require.Nil(err, "%+v", err) assert.EqualValues(v, data.Value) From 2b735d8968eeb59bc3d5a5c7a9c71e7190fe288f Mon Sep 17 00:00:00 2001 From: Alexis Sellier Date: Tue, 8 Aug 2017 11:51:15 +0200 Subject: [PATCH 03/22] Support non-existence proof --- client/commands/query/get.go | 53 ++++++++++++++++++++--------- client/commands/query/query_test.go | 10 +++--- 2 files changed, 41 insertions(+), 22 deletions(-) diff --git a/client/commands/query/get.go b/client/commands/query/get.go index f608c71acc..46b6c64c5a 100644 --- a/client/commands/query/get.go +++ b/client/commands/query/get.go @@ -55,62 +55,81 @@ func Get(key []byte, prove bool) (data.Bytes, uint64, error) { resp, err := node.ABCIQuery("/key", key, false) return data.Bytes(resp.Value), resp.Height, err } - val, h, _, err := GetWithProof(key) + val, h, _, _, err := GetWithProof(key) return val, h, err } // GetWithProof returns the values stored under a given key at the named // height as in Get. Additionally, it will return a validated merkle // proof for the key-value pair if it exists, and all checks pass. -func GetWithProof(key []byte) (data.Bytes, uint64, *iavl.KeyExistsProof, error) { +func GetWithProof(key []byte) (data.Bytes, uint64, + *iavl.KeyExistsProof, *iavl.KeyNotExistsProof, error) { + node := commands.GetNode() cert, err := commands.GetCertifier() if err != nil { - return nil, 0, nil, err + return nil, 0, nil, nil, err } return CustomGetWithProof(key, node, cert) } // TODO: fix this up alexis -func CustomGetWithProof(key []byte, node client.Client, - cert Certifier) (data.Bytes, uint64, *iavl.KeyExistsProof, error) { +func CustomGetWithProof(key []byte, node client.Client, cert Certifier) (data.Bytes, uint64, + *iavl.KeyExistsProof, *iavl.KeyNotExistsProof, error) { resp, err := node.ABCIQuery("/key", key, true) if err != nil { - return nil, 0, nil, err + return nil, 0, nil, nil, err } ph := int(resp.Height) // make sure the proof is the proper height if !resp.Code.IsOK() { - return nil, 0, nil, errors.Errorf("Query error %d: %s", resp.Code, resp.Code.String()) + return nil, 0, nil, nil, errors.Errorf("Query error %d: %s", resp.Code, resp.Code.String()) } - // TODO: Handle null proofs - if len(resp.Key) == 0 || len(resp.Value) == 0 || len(resp.Proof) == 0 { - return nil, 0, nil, lc.ErrNoData() + if len(resp.Key) == 0 || len(resp.Proof) == 0 { + return nil, 0, nil, nil, lc.ErrNoData() } if ph != 0 && ph != int(resp.Height) { - return nil, 0, nil, lc.ErrHeightMismatch(ph, int(resp.Height)) + return nil, 0, nil, nil, lc.ErrHeightMismatch(ph, int(resp.Height)) } check, err := GetCertifiedCheckpoint(ph, node, cert) if err != nil { - return nil, 0, nil, err + return nil, 0, nil, nil, err } - proof := new(iavl.KeyExistsProof) + if len(resp.Value) > 0 { + // The key was found, construct a proof of existence. + proof := new(iavl.KeyExistsProof) + err = wire.ReadBinaryBytes(resp.Proof, &proof) + if err != nil { + return nil, 0, nil, nil, err + } + + // validate the proof against the certified header to ensure data integrity + err = proof.Verify(resp.Key, resp.Value, check.Header.AppHash) + if err != nil { + return nil, 0, nil, nil, err + } + + return data.Bytes(resp.Value), resp.Height, proof, nil, nil + } + + // The key wasn't found, construct a proof of non-existence. + proof := new(iavl.KeyNotExistsProof) err = wire.ReadBinaryBytes(resp.Proof, &proof) if err != nil { - return nil, 0, nil, err + return nil, 0, nil, nil, err } // validate the proof against the certified header to ensure data integrity - err = proof.Verify(resp.Key, resp.Value, check.Header.AppHash) + err = proof.Verify(resp.Key, check.Header.AppHash) if err != nil { - return nil, 0, nil, err + return nil, 0, nil, nil, err } - return data.Bytes(resp.Value), resp.Height, proof, nil + return data.Bytes(resp.Value), resp.Height, nil, proof, nil } // GetCertifiedCheckpoint gets the signed header for a given height diff --git a/client/commands/query/query_test.go b/client/commands/query/query_test.go index 862b4ceb53..07c5716326 100644 --- a/client/commands/query/query_test.go +++ b/client/commands/query/query_test.go @@ -65,9 +65,9 @@ func TestAppProofs(t *testing.T) { // Test existing key. - bs, _, proof, err := CustomGetWithProof(k, cl, cert) + bs, _, proofExists, _, err := CustomGetWithProof(k, cl, cert) require.Nil(err, "%+v", err) - require.NotNil(proof) + require.NotNil(proofExists) var data eyes.Data err = wire.ReadBinaryBytes(bs, &data) @@ -76,10 +76,10 @@ func TestAppProofs(t *testing.T) { // Test non-existing key. - // TODO: This currently fails. missing := []byte("my-missing-key") - bs, _, proof, err = CustomGetWithProof(missing, cl, cert) + bs, _, proofExists, proofNotExists, err := CustomGetWithProof(missing, cl, cert) require.Nil(err, "%+v", err) require.Nil(bs) - require.NotNil(proof) + require.Nil(proofExists) + require.NotNil(proofNotExists) } From 225f0e7dbeb80759f594a1828da1fb46d5681633 Mon Sep 17 00:00:00 2001 From: Alexis Sellier Date: Tue, 8 Aug 2017 12:20:21 +0200 Subject: [PATCH 04/22] Rename function --- client/commands/query/get.go | 5 ++--- client/commands/query/query_test.go | 4 ++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/client/commands/query/get.go b/client/commands/query/get.go index 46b6c64c5a..aa0f15f892 100644 --- a/client/commands/query/get.go +++ b/client/commands/query/get.go @@ -70,11 +70,10 @@ func GetWithProof(key []byte) (data.Bytes, uint64, if err != nil { return nil, 0, nil, nil, err } - return CustomGetWithProof(key, node, cert) + return getWithProof(key, node, cert) } -// TODO: fix this up alexis -func CustomGetWithProof(key []byte, node client.Client, cert Certifier) (data.Bytes, uint64, +func getWithProof(key []byte, node client.Client, cert Certifier) (data.Bytes, uint64, *iavl.KeyExistsProof, *iavl.KeyNotExistsProof, error) { resp, err := node.ABCIQuery("/key", key, true) diff --git a/client/commands/query/query_test.go b/client/commands/query/query_test.go index 07c5716326..ecf1e5cef2 100644 --- a/client/commands/query/query_test.go +++ b/client/commands/query/query_test.go @@ -65,7 +65,7 @@ func TestAppProofs(t *testing.T) { // Test existing key. - bs, _, proofExists, _, err := CustomGetWithProof(k, cl, cert) + bs, _, proofExists, _, err := getWithProof(k, cl, cert) require.Nil(err, "%+v", err) require.NotNil(proofExists) @@ -77,7 +77,7 @@ func TestAppProofs(t *testing.T) { // Test non-existing key. missing := []byte("my-missing-key") - bs, _, proofExists, proofNotExists, err := CustomGetWithProof(missing, cl, cert) + bs, _, proofExists, proofNotExists, err := getWithProof(missing, cl, cert) require.Nil(err, "%+v", err) require.Nil(bs) require.Nil(proofExists) From cedb66066cba1367d7e47dcc48aec65485cb27fe Mon Sep 17 00:00:00 2001 From: Alexis Sellier Date: Tue, 8 Aug 2017 12:21:18 +0200 Subject: [PATCH 05/22] Fix return args --- modules/ibc/commands/query.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/ibc/commands/query.go b/modules/ibc/commands/query.go index b320da81ce..e0e3c564c7 100644 --- a/modules/ibc/commands/query.go +++ b/modules/ibc/commands/query.go @@ -197,7 +197,7 @@ func packetQueryCmd(cmd *cobra.Command, args []string) error { } // output queue, create a post packet - bs, height, proof, err := query.GetWithProof(key) + bs, height, proof, _, err := query.GetWithProof(key) if err != nil { return err } From f7d4b7f721f7a9aa3ad4c46d62a73e8d080f215e Mon Sep 17 00:00:00 2001 From: Ethan Frey Date: Tue, 8 Aug 2017 12:27:40 +0200 Subject: [PATCH 06/22] Add tests that the proofs verify --- client/commands/query/query_test.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/client/commands/query/query_test.go b/client/commands/query/query_test.go index ecf1e5cef2..2bce1642fd 100644 --- a/client/commands/query/query_test.go +++ b/client/commands/query/query_test.go @@ -73,6 +73,8 @@ func TestAppProofs(t *testing.T) { err = wire.ReadBinaryBytes(bs, &data) require.Nil(err, "%+v", err) assert.EqualValues(v, data.Value) + err = proofExists.Verify(k, bs, proofExists.RootHash) + assert.Nil(err, "%+v", err) // Test non-existing key. @@ -82,4 +84,8 @@ func TestAppProofs(t *testing.T) { require.Nil(bs) require.Nil(proofExists) require.NotNil(proofNotExists) + err = proofNotExists.Verify(missing, proofNotExists.RootHash) + assert.Nil(err, "%+v", err) + err = proofNotExists.Verify(k, proofNotExists.RootHash) + assert.NotNil(err) } From 7857f2564959ef154639693cca46098f1adb9db0 Mon Sep 17 00:00:00 2001 From: Ethan Frey Date: Tue, 8 Aug 2017 12:46:17 +0200 Subject: [PATCH 07/22] No sleep, just wait for one block to exist --- client/commands/query/query_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client/commands/query/query_test.go b/client/commands/query/query_test.go index 2bce1642fd..87e92f036e 100644 --- a/client/commands/query/query_test.go +++ b/client/commands/query/query_test.go @@ -3,7 +3,6 @@ package query import ( "os" "testing" - "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -42,7 +41,8 @@ func TestAppProofs(t *testing.T) { assert, require := assert.New(t), require.New(t) cl := client.NewLocal(node) - time.Sleep(200 * time.Millisecond) + // make sure one block is created + client.WaitForHeight(cl, 1, nil) k := []byte("my-key") v := []byte("my-value") From 80ef09f1ee22195b35344658ca96bfcbe00dca37 Mon Sep 17 00:00:00 2001 From: Alexis Sellier Date: Tue, 8 Aug 2017 16:19:40 +0200 Subject: [PATCH 08/22] Fix bogus height check --- client/commands/query/get.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/client/commands/query/get.go b/client/commands/query/get.go index aa0f15f892..3047952b97 100644 --- a/client/commands/query/get.go +++ b/client/commands/query/get.go @@ -80,7 +80,6 @@ func getWithProof(key []byte, node client.Client, cert Certifier) (data.Bytes, u if err != nil { return nil, 0, nil, nil, err } - ph := int(resp.Height) // make sure the proof is the proper height if !resp.Code.IsOK() { @@ -89,11 +88,11 @@ func getWithProof(key []byte, node client.Client, cert Certifier) (data.Bytes, u if len(resp.Key) == 0 || len(resp.Proof) == 0 { return nil, 0, nil, nil, lc.ErrNoData() } - if ph != 0 && ph != int(resp.Height) { - return nil, 0, nil, nil, lc.ErrHeightMismatch(ph, int(resp.Height)) + if resp.Height == 0 { + return nil, 0, nil, nil, errors.New("Height returned is zero") } - check, err := GetCertifiedCheckpoint(ph, node, cert) + check, err := GetCertifiedCheckpoint(int(resp.Height), node, cert) if err != nil { return nil, 0, nil, nil, err } From 1882376a345070915eb5e9e55c0dd8ff5c42dbd2 Mon Sep 17 00:00:00 2001 From: Alexis Sellier Date: Tue, 8 Aug 2017 18:06:14 +0200 Subject: [PATCH 09/22] Cleanup tests a bit --- client/commands/query/query_test.go | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/client/commands/query/query_test.go b/client/commands/query/query_test.go index 87e92f036e..f141998dfc 100644 --- a/client/commands/query/query_test.go +++ b/client/commands/query/query_test.go @@ -41,7 +41,6 @@ func TestAppProofs(t *testing.T) { assert, require := assert.New(t), require.New(t) cl := client.NewLocal(node) - // make sure one block is created client.WaitForHeight(cl, 1, nil) k := []byte("my-key") @@ -54,22 +53,19 @@ func TestAppProofs(t *testing.T) { require.EqualValues(0, br.CheckTx.Code, "%#v", br.CheckTx) require.EqualValues(0, br.DeliverTx.Code) - // this sets up our trust on the node based on some past point. - // maybe this can be cleaned up and made easy to reuse + // This sets up our trust on the node based on some past point. source := certclient.New(cl) - trusted := certifiers.NewMemStoreProvider() - // let's start with some trust before the query... seed, err := source.GetByHeight(br.Height - 2) require.Nil(err, "%+v", err) - cert := certifiers.NewInquiring("my-chain", seed, trusted, source) + cert := certifiers.NewStatic("my-chain", seed.Validators) // Test existing key. + var data eyes.Data bs, _, proofExists, _, err := getWithProof(k, cl, cert) require.Nil(err, "%+v", err) require.NotNil(proofExists) - var data eyes.Data err = wire.ReadBinaryBytes(bs, &data) require.Nil(err, "%+v", err) assert.EqualValues(v, data.Value) @@ -77,7 +73,6 @@ func TestAppProofs(t *testing.T) { assert.Nil(err, "%+v", err) // Test non-existing key. - missing := []byte("my-missing-key") bs, _, proofExists, proofNotExists, err := getWithProof(missing, cl, cert) require.Nil(err, "%+v", err) From d1b49da82564363e2876ade84168d934fa1238ae Mon Sep 17 00:00:00 2001 From: Alexis Sellier Date: Tue, 8 Aug 2017 18:52:28 +0200 Subject: [PATCH 10/22] Use new Certifier type from light-client --- client/commands/query/get.go | 9 +++------ glide.lock | 5 +++-- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/client/commands/query/get.go b/client/commands/query/get.go index 3047952b97..ecf230f1af 100644 --- a/client/commands/query/get.go +++ b/client/commands/query/get.go @@ -11,6 +11,7 @@ import ( wire "github.com/tendermint/go-wire" "github.com/tendermint/go-wire/data" lc "github.com/tendermint/light-client" + "github.com/tendermint/light-client/certifiers" "github.com/tendermint/light-client/proofs" "github.com/tendermint/merkleeyes/iavl" "github.com/tendermint/tendermint/rpc/client" @@ -18,10 +19,6 @@ import ( "github.com/tendermint/basecoin/client/commands" ) -type Certifier interface { - Certify(check lc.Checkpoint) error -} - // GetParsed does most of the work of the query commands, but is quite // opinionated, so if you want more control about parsing, call Get // directly. @@ -73,7 +70,7 @@ func GetWithProof(key []byte) (data.Bytes, uint64, return getWithProof(key, node, cert) } -func getWithProof(key []byte, node client.Client, cert Certifier) (data.Bytes, uint64, +func getWithProof(key []byte, node client.Client, cert certifiers.Certifier) (data.Bytes, uint64, *iavl.KeyExistsProof, *iavl.KeyNotExistsProof, error) { resp, err := node.ABCIQuery("/key", key, true) @@ -133,7 +130,7 @@ func getWithProof(key []byte, node client.Client, cert Certifier) (data.Bytes, u // GetCertifiedCheckpoint gets the signed header for a given height // and certifies it. Returns error if unable to get a proven header. func GetCertifiedCheckpoint(h int, node client.Client, - cert Certifier) (empty lc.Checkpoint, err error) { + cert certifiers.Certifier) (empty lc.Checkpoint, err error) { // get the checkpoint for this height // FIXME: cannot use cert.GetByHeight for now, as it also requires diff --git a/glide.lock b/glide.lock index 5d0c74a713..bb4b3cb4c5 100644 --- a/glide.lock +++ b/glide.lock @@ -1,5 +1,5 @@ hash: 2848c30b31fb205f846dd7dfca14ebed8a3249cbc5aaa759066b2bab3e4bbf42 -updated: 2017-08-04T15:38:45.048261895+02:00 +updated: 2017-08-08T18:32:19.025782383+02:00 imports: - name: github.com/bgentry/speakeasy version: 4aabc24848ce5fd31929f7d1e4ea74d3709c14cd @@ -135,7 +135,7 @@ imports: - data - data/base58 - name: github.com/tendermint/light-client - version: fcf4e411583135a1900157b8b0274c41e20ea3a1 + version: b2afece9635d11e77dd404019b9cf3885d34f4e5 subpackages: - certifiers - certifiers/client @@ -166,6 +166,7 @@ imports: - rpc/lib/client - rpc/lib/server - rpc/lib/types + - rpc/test - state - state/txindex - state/txindex/kv From 668eea8628cfd66a383649b92afcb5f4f0fad2e6 Mon Sep 17 00:00:00 2001 From: Alexis Sellier Date: Wed, 9 Aug 2017 13:13:16 +0200 Subject: [PATCH 11/22] Fix return value when nothing is found --- client/commands/query/get.go | 2 +- client/commands/query/query_test.go | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/client/commands/query/get.go b/client/commands/query/get.go index ecf230f1af..d106bb1be9 100644 --- a/client/commands/query/get.go +++ b/client/commands/query/get.go @@ -124,7 +124,7 @@ func getWithProof(key []byte, node client.Client, cert certifiers.Certifier) (da return nil, 0, nil, nil, err } - return data.Bytes(resp.Value), resp.Height, nil, proof, nil + return nil, resp.Height, nil, proof, lc.ErrNoData() } // GetCertifiedCheckpoint gets the signed header for a given height diff --git a/client/commands/query/query_test.go b/client/commands/query/query_test.go index f141998dfc..c16affb7c6 100644 --- a/client/commands/query/query_test.go +++ b/client/commands/query/query_test.go @@ -8,6 +8,7 @@ import ( "github.com/stretchr/testify/require" "github.com/tendermint/go-wire" + lc "github.com/tendermint/light-client" "github.com/tendermint/light-client/certifiers" certclient "github.com/tendermint/light-client/certifiers/client" nm "github.com/tendermint/tendermint/node" @@ -75,7 +76,7 @@ func TestAppProofs(t *testing.T) { // Test non-existing key. missing := []byte("my-missing-key") bs, _, proofExists, proofNotExists, err := getWithProof(missing, cl, cert) - require.Nil(err, "%+v", err) + require.True(lc.IsNoDataErr(err)) require.Nil(bs) require.Nil(proofExists) require.NotNil(proofNotExists) From 54304ba5e6ae90ce82c04697529043077922efdc Mon Sep 17 00:00:00 2001 From: Alexis Sellier Date: Wed, 9 Aug 2017 17:21:11 +0200 Subject: [PATCH 12/22] Update merkleeyes dependency It seems like to force it to update to latest unstable, we have to specify origin/unstable. --- glide.lock | 6 +++--- glide.yaml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/glide.lock b/glide.lock index bb4b3cb4c5..97fee36fd8 100644 --- a/glide.lock +++ b/glide.lock @@ -1,5 +1,5 @@ -hash: 2848c30b31fb205f846dd7dfca14ebed8a3249cbc5aaa759066b2bab3e4bbf42 -updated: 2017-08-08T18:32:19.025782383+02:00 +hash: 246a02006fc46d91294fba71971c51477241b0ace2989df04d728ae6d09f1013 +updated: 2017-08-09T17:20:03.756445376+02:00 imports: - name: github.com/bgentry/speakeasy version: 4aabc24848ce5fd31929f7d1e4ea74d3709c14cd @@ -142,7 +142,7 @@ imports: - certifiers/files - proofs - name: github.com/tendermint/merkleeyes - version: 44c4c64c731db5be4261ff3971b01b7e19729419 + version: 25b700b87a45619cb6bd85330ab4a81b7b7fbb0d subpackages: - client - iavl diff --git a/glide.yaml b/glide.yaml index e10ba05f98..9fde2472a1 100644 --- a/glide.yaml +++ b/glide.yaml @@ -29,7 +29,7 @@ import: - certifiers/client - certifiers/files - package: github.com/tendermint/merkleeyes - version: unstable + version: origin/unstable subpackages: - client - iavl From ee1a27e6fd0c740c0e6f2214959eb7ed16eedbd2 Mon Sep 17 00:00:00 2001 From: Alexis Sellier Date: Wed, 9 Aug 2017 17:21:41 +0200 Subject: [PATCH 13/22] Implement String method on *Bonsai --- state/bonsai.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/state/bonsai.go b/state/bonsai.go index 7eb98568a9..eae0f589fb 100644 --- a/state/bonsai.go +++ b/state/bonsai.go @@ -15,6 +15,10 @@ type Bonsai struct { Tree *iavl.IAVLTree } +func (b *Bonsai) String() string { + return "Bonsai{" + b.Tree.String() + "}" +} + var _ SimpleDB = &Bonsai{} // NewBonsai wraps a merkle tree and tags it to track children From 388f0eece7daba5e052df487917adde6a909111c Mon Sep 17 00:00:00 2001 From: Alexis Sellier Date: Wed, 9 Aug 2017 17:22:16 +0200 Subject: [PATCH 14/22] Wrap error for good measure --- client/commands/query/get.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/commands/query/get.go b/client/commands/query/get.go index d106bb1be9..302267a1ff 100644 --- a/client/commands/query/get.go +++ b/client/commands/query/get.go @@ -121,7 +121,7 @@ func getWithProof(key []byte, node client.Client, cert certifiers.Certifier) (da // validate the proof against the certified header to ensure data integrity err = proof.Verify(resp.Key, check.Header.AppHash) if err != nil { - return nil, 0, nil, nil, err + return nil, 0, nil, proof, errors.Wrap(err, "Couldn't verify proof") } return nil, resp.Height, nil, proof, lc.ErrNoData() From f9c3fce5b43a17b2c7a420ea076553ae878559fa Mon Sep 17 00:00:00 2001 From: Alexis Sellier Date: Thu, 10 Aug 2017 14:25:17 +0200 Subject: [PATCH 15/22] Write tx proof tests --- client/commands/query/query_test.go | 39 +++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/client/commands/query/query_test.go b/client/commands/query/query_test.go index c16affb7c6..25362a4798 100644 --- a/client/commands/query/query_test.go +++ b/client/commands/query/query_test.go @@ -14,6 +14,7 @@ import ( nm "github.com/tendermint/tendermint/node" "github.com/tendermint/tendermint/rpc/client" rpctest "github.com/tendermint/tendermint/rpc/test" + "github.com/tendermint/tendermint/types" "github.com/tendermint/tmlibs/log" "github.com/tendermint/basecoin/app" @@ -85,3 +86,41 @@ func TestAppProofs(t *testing.T) { err = proofNotExists.Verify(k, proofNotExists.RootHash) assert.NotNil(err) } + +func TestTxProofs(t *testing.T) { + assert, require := assert.New(t), require.New(t) + + cl := client.NewLocal(node) + client.WaitForHeight(cl, 1, nil) + + tx := eyes.SetTx{Key: []byte("key-a"), Value: []byte("value-a")}.Wrap() + + btx := types.Tx(wire.BinaryBytes(tx)) + br, err := cl.BroadcastTxCommit(btx) + require.Nil(err, "%+v", err) + require.EqualValues(0, br.CheckTx.Code, "%#v", br.CheckTx) + require.EqualValues(0, br.DeliverTx.Code) + + source := certclient.New(cl) + seed, err := source.GetByHeight(br.Height - 2) + require.Nil(err, "%+v", err) + cert := certifiers.NewStatic("my-chain", seed.Validators) + + // First let's make sure a bogus transaction hash returns a valid non-existence proof. + key := types.Tx([]byte("bogus")).Hash() + bs, _, proofExists, proofNotExists, err := getWithProof(key, cl, cert) + assert.Nil(bs, "value should be nil") + require.True(lc.IsNoDataErr(err), "error should signal 'no data'") + assert.Nil(proofExists, "existence proof should be nil") + require.NotNil(proofNotExists, "non-existence proof shouldn't be nil") + err = proofNotExists.Verify(key, proofNotExists.RootHash) + require.Nil(err, "%+v", err) + + // Now let's check with the real tx hash. + key = btx.Hash() + res, err := cl.Tx(key, true) + require.Nil(err, "%+v", err) + require.NotNil(res) + err = res.Proof.Validate(key) + assert.Nil(err, "%+v", err) +} From 77207a19db9c0d30de5baec256d0685f789f637a Mon Sep 17 00:00:00 2001 From: Alexis Sellier Date: Thu, 10 Aug 2017 15:05:01 +0200 Subject: [PATCH 16/22] Fix some comments --- client/commands/query/get.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/client/commands/query/get.go b/client/commands/query/get.go index 302267a1ff..8a2ff820a8 100644 --- a/client/commands/query/get.go +++ b/client/commands/query/get.go @@ -102,7 +102,7 @@ func getWithProof(key []byte, node client.Client, cert certifiers.Certifier) (da return nil, 0, nil, nil, err } - // validate the proof against the certified header to ensure data integrity + // Validate the proof against the certified header to ensure data integrity. err = proof.Verify(resp.Key, resp.Value, check.Header.AppHash) if err != nil { return nil, 0, nil, nil, err @@ -118,7 +118,7 @@ func getWithProof(key []byte, node client.Client, cert certifiers.Certifier) (da return nil, 0, nil, nil, err } - // validate the proof against the certified header to ensure data integrity + // Validate the proof against the certified header to ensure data integrity. err = proof.Verify(resp.Key, check.Header.AppHash) if err != nil { return nil, 0, nil, proof, errors.Wrap(err, "Couldn't verify proof") @@ -131,7 +131,6 @@ func getWithProof(key []byte, node client.Client, cert certifiers.Certifier) (da // and certifies it. Returns error if unable to get a proven header. func GetCertifiedCheckpoint(h int, node client.Client, cert certifiers.Certifier) (empty lc.Checkpoint, err error) { - // get the checkpoint for this height // FIXME: cannot use cert.GetByHeight for now, as it also requires // Validators and will fail on querying tendermint for non-current height. From eec4b10067e9dde40f7828489d76caa203e3ad5f Mon Sep 17 00:00:00 2001 From: Alexis Sellier Date: Thu, 10 Aug 2017 15:10:21 +0200 Subject: [PATCH 17/22] This doesn't need to be public --- client/commands/query/get.go | 6 +++--- client/commands/query/tx.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/client/commands/query/get.go b/client/commands/query/get.go index 8a2ff820a8..577d9cbe0c 100644 --- a/client/commands/query/get.go +++ b/client/commands/query/get.go @@ -89,7 +89,7 @@ func getWithProof(key []byte, node client.Client, cert certifiers.Certifier) (da return nil, 0, nil, nil, errors.New("Height returned is zero") } - check, err := GetCertifiedCheckpoint(int(resp.Height), node, cert) + check, err := getCertifiedCheckpoint(int(resp.Height), node, cert) if err != nil { return nil, 0, nil, nil, err } @@ -127,9 +127,9 @@ func getWithProof(key []byte, node client.Client, cert certifiers.Certifier) (da return nil, resp.Height, nil, proof, lc.ErrNoData() } -// GetCertifiedCheckpoint gets the signed header for a given height +// getCertifiedCheckpoint gets the signed header for a given height // and certifies it. Returns error if unable to get a proven header. -func GetCertifiedCheckpoint(h int, node client.Client, +func getCertifiedCheckpoint(h int, node client.Client, cert certifiers.Certifier) (empty lc.Checkpoint, err error) { // FIXME: cannot use cert.GetByHeight for now, as it also requires diff --git a/client/commands/query/tx.go b/client/commands/query/tx.go index 1d520f89e5..7b108b950a 100644 --- a/client/commands/query/tx.go +++ b/client/commands/query/tx.go @@ -51,7 +51,7 @@ func txQueryCmd(cmd *cobra.Command, args []string) error { return err } - check, err := GetCertifiedCheckpoint(res.Height, node, cert) + check, err := getCertifiedCheckpoint(res.Height, node, cert) if err != nil { return err } From 3089673a2e57dcd69ff6a421706d693e241e4be0 Mon Sep 17 00:00:00 2001 From: Alexis Sellier Date: Thu, 10 Aug 2017 15:30:45 +0200 Subject: [PATCH 18/22] Remove TODO --- TODO.md | 21 --------------------- 1 file changed, 21 deletions(-) delete mode 100644 TODO.md diff --git a/TODO.md b/TODO.md deleted file mode 100644 index 9a215a0785..0000000000 --- a/TODO.md +++ /dev/null @@ -1,21 +0,0 @@ -Alexis: - -* merkle - proof (non-existence - maybe range) -* intro to light-client and proofs -light-client proofs: -* make this sensible -> very tied to merkle proofs and API -* support new proof types - -* expose more proof types in basecoin.Query - - -* merkle - api cleanup (also Bonsai) -* later: C bindings (to Bonsai?) - - -* crypto-ledger (while ethan gone) - -light-client provider: -* caching checkpoint on Verify -* cleanup (trim old node) - From 3f0c0ad8f5024c192b1ed7cc4208f9bafa5a3a90 Mon Sep 17 00:00:00 2001 From: Alexis Sellier Date: Fri, 11 Aug 2017 13:30:53 +0200 Subject: [PATCH 19/22] Refactor function for readability --- client/commands/query/get.go | 62 ++++++++++++++++++++---------------- 1 file changed, 35 insertions(+), 27 deletions(-) diff --git a/client/commands/query/get.go b/client/commands/query/get.go index 577d9cbe0c..574d679bad 100644 --- a/client/commands/query/get.go +++ b/client/commands/query/get.go @@ -70,61 +70,69 @@ func GetWithProof(key []byte) (data.Bytes, uint64, return getWithProof(key, node, cert) } -func getWithProof(key []byte, node client.Client, cert certifiers.Certifier) (data.Bytes, uint64, - *iavl.KeyExistsProof, *iavl.KeyNotExistsProof, error) { +func getWithProof(key []byte, node client.Client, cert certifiers.Certifier) ( + val data.Bytes, height uint64, eproof *iavl.KeyExistsProof, neproof *iavl.KeyNotExistsProof, err error) { resp, err := node.ABCIQuery("/key", key, true) if err != nil { - return nil, 0, nil, nil, err + return } // make sure the proof is the proper height if !resp.Code.IsOK() { - return nil, 0, nil, nil, errors.Errorf("Query error %d: %s", resp.Code, resp.Code.String()) + err = errors.Errorf("Query error %d: %s", resp.Code, resp.Code.String()) + return } if len(resp.Key) == 0 || len(resp.Proof) == 0 { - return nil, 0, nil, nil, lc.ErrNoData() + err = lc.ErrNoData() + return } if resp.Height == 0 { - return nil, 0, nil, nil, errors.New("Height returned is zero") + err = errors.New("Height returned is zero") + return } check, err := getCertifiedCheckpoint(int(resp.Height), node, cert) if err != nil { - return nil, 0, nil, nil, err + return } if len(resp.Value) > 0 { // The key was found, construct a proof of existence. - proof := new(iavl.KeyExistsProof) - err = wire.ReadBinaryBytes(resp.Proof, &proof) + eproof = new(iavl.KeyExistsProof) + err = wire.ReadBinaryBytes(resp.Proof, &eproof) if err != nil { - return nil, 0, nil, nil, err + err = errors.Wrap(err, "Error reading proof") + return } // Validate the proof against the certified header to ensure data integrity. - err = proof.Verify(resp.Key, resp.Value, check.Header.AppHash) + err = eproof.Verify(resp.Key, resp.Value, check.Header.AppHash) if err != nil { - return nil, 0, nil, nil, err + err = errors.Wrap(err, "Couldn't verify proof") + return + } + val = data.Bytes(resp.Value) + } else { + // The key wasn't found, construct a proof of non-existence. + neproof = new(iavl.KeyNotExistsProof) + err = wire.ReadBinaryBytes(resp.Proof, &neproof) + if err != nil { + err = errors.Wrap(err, "Error reading proof") + return } - return data.Bytes(resp.Value), resp.Height, proof, nil, nil + // Validate the proof against the certified header to ensure data integrity. + err = neproof.Verify(resp.Key, check.Header.AppHash) + if err != nil { + err = errors.Wrap(err, "Couldn't verify proof") + return + } + err = lc.ErrNoData() } - // The key wasn't found, construct a proof of non-existence. - proof := new(iavl.KeyNotExistsProof) - err = wire.ReadBinaryBytes(resp.Proof, &proof) - if err != nil { - return nil, 0, nil, nil, err - } - - // Validate the proof against the certified header to ensure data integrity. - err = proof.Verify(resp.Key, check.Header.AppHash) - if err != nil { - return nil, 0, nil, proof, errors.Wrap(err, "Couldn't verify proof") - } - - return nil, resp.Height, nil, proof, lc.ErrNoData() + height = resp.Height + return } // getCertifiedCheckpoint gets the signed header for a given height From 6a642e33b8ba51fd25be2975731e8ecb60185fc8 Mon Sep 17 00:00:00 2001 From: Alexis Sellier Date: Fri, 11 Aug 2017 13:57:38 +0200 Subject: [PATCH 20/22] Check returned height --- client/commands/query/query_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/client/commands/query/query_test.go b/client/commands/query/query_test.go index 25362a4798..3fcef7781b 100644 --- a/client/commands/query/query_test.go +++ b/client/commands/query/query_test.go @@ -64,9 +64,10 @@ func TestAppProofs(t *testing.T) { // Test existing key. var data eyes.Data - bs, _, proofExists, _, err := getWithProof(k, cl, cert) + bs, height, proofExists, _, err := getWithProof(k, cl, cert) require.Nil(err, "%+v", err) require.NotNil(proofExists) + require.True(uint64(br.Height) < height) err = wire.ReadBinaryBytes(bs, &data) require.Nil(err, "%+v", err) From b69df980d133fd0f0b1a41430ec8cb9f1c469b0b Mon Sep 17 00:00:00 2001 From: Alexis Sellier Date: Fri, 11 Aug 2017 14:10:06 +0200 Subject: [PATCH 21/22] Check proofs with latest commit --- client/commands/query/query_test.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/client/commands/query/query_test.go b/client/commands/query/query_test.go index 3fcef7781b..a958c1def1 100644 --- a/client/commands/query/query_test.go +++ b/client/commands/query/query_test.go @@ -61,18 +61,22 @@ func TestAppProofs(t *testing.T) { require.Nil(err, "%+v", err) cert := certifiers.NewStatic("my-chain", seed.Validators) + latest, err := source.GetLatestCommit() + require.Nil(err, "%+v", err) + rootHash := latest.Header.AppHash + // Test existing key. var data eyes.Data bs, height, proofExists, _, err := getWithProof(k, cl, cert) require.Nil(err, "%+v", err) require.NotNil(proofExists) - require.True(uint64(br.Height) < height) + require.True(height >= uint64(latest.Header.Height)) err = wire.ReadBinaryBytes(bs, &data) require.Nil(err, "%+v", err) assert.EqualValues(v, data.Value) - err = proofExists.Verify(k, bs, proofExists.RootHash) + err = proofExists.Verify(k, bs, rootHash) assert.Nil(err, "%+v", err) // Test non-existing key. @@ -82,9 +86,9 @@ func TestAppProofs(t *testing.T) { require.Nil(bs) require.Nil(proofExists) require.NotNil(proofNotExists) - err = proofNotExists.Verify(missing, proofNotExists.RootHash) + err = proofNotExists.Verify(missing, rootHash) assert.Nil(err, "%+v", err) - err = proofNotExists.Verify(k, proofNotExists.RootHash) + err = proofNotExists.Verify(k, rootHash) assert.NotNil(err) } From d3cdce38500767c194e82cc82fb739df715e5c7a Mon Sep 17 00:00:00 2001 From: Alexis Sellier Date: Wed, 16 Aug 2017 16:55:25 +0200 Subject: [PATCH 22/22] Use more specific assertion functions --- client/commands/query/query_test.go | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/client/commands/query/query_test.go b/client/commands/query/query_test.go index a958c1def1..23ec06f424 100644 --- a/client/commands/query/query_test.go +++ b/client/commands/query/query_test.go @@ -51,33 +51,33 @@ func TestAppProofs(t *testing.T) { tx := eyes.SetTx{Key: k, Value: v}.Wrap() btx := wire.BinaryBytes(tx) br, err := cl.BroadcastTxCommit(btx) - require.Nil(err, "%+v", err) + require.NoError(err, "%+v", err) require.EqualValues(0, br.CheckTx.Code, "%#v", br.CheckTx) require.EqualValues(0, br.DeliverTx.Code) // This sets up our trust on the node based on some past point. source := certclient.New(cl) seed, err := source.GetByHeight(br.Height - 2) - require.Nil(err, "%+v", err) + require.NoError(err, "%+v", err) cert := certifiers.NewStatic("my-chain", seed.Validators) latest, err := source.GetLatestCommit() - require.Nil(err, "%+v", err) + require.NoError(err, "%+v", err) rootHash := latest.Header.AppHash // Test existing key. var data eyes.Data bs, height, proofExists, _, err := getWithProof(k, cl, cert) - require.Nil(err, "%+v", err) + require.NoError(err, "%+v", err) require.NotNil(proofExists) require.True(height >= uint64(latest.Header.Height)) err = wire.ReadBinaryBytes(bs, &data) - require.Nil(err, "%+v", err) + require.NoError(err, "%+v", err) assert.EqualValues(v, data.Value) err = proofExists.Verify(k, bs, rootHash) - assert.Nil(err, "%+v", err) + assert.NoError(err, "%+v", err) // Test non-existing key. missing := []byte("my-missing-key") @@ -87,9 +87,9 @@ func TestAppProofs(t *testing.T) { require.Nil(proofExists) require.NotNil(proofNotExists) err = proofNotExists.Verify(missing, rootHash) - assert.Nil(err, "%+v", err) + assert.NoError(err, "%+v", err) err = proofNotExists.Verify(k, rootHash) - assert.NotNil(err) + assert.Error(err) } func TestTxProofs(t *testing.T) { @@ -102,13 +102,13 @@ func TestTxProofs(t *testing.T) { btx := types.Tx(wire.BinaryBytes(tx)) br, err := cl.BroadcastTxCommit(btx) - require.Nil(err, "%+v", err) + require.NoError(err, "%+v", err) require.EqualValues(0, br.CheckTx.Code, "%#v", br.CheckTx) require.EqualValues(0, br.DeliverTx.Code) source := certclient.New(cl) seed, err := source.GetByHeight(br.Height - 2) - require.Nil(err, "%+v", err) + require.NoError(err, "%+v", err) cert := certifiers.NewStatic("my-chain", seed.Validators) // First let's make sure a bogus transaction hash returns a valid non-existence proof. @@ -119,13 +119,13 @@ func TestTxProofs(t *testing.T) { assert.Nil(proofExists, "existence proof should be nil") require.NotNil(proofNotExists, "non-existence proof shouldn't be nil") err = proofNotExists.Verify(key, proofNotExists.RootHash) - require.Nil(err, "%+v", err) + require.NoError(err, "%+v", err) // Now let's check with the real tx hash. key = btx.Hash() res, err := cl.Tx(key, true) - require.Nil(err, "%+v", err) + require.NoError(err, "%+v", err) require.NotNil(res) err = res.Proof.Validate(key) - assert.Nil(err, "%+v", err) + assert.NoError(err, "%+v", err) }