From c388e7eac0d26f4d299523fd14ec350655f66879 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Fri, 20 Mar 2015 13:36:52 +0100 Subject: [PATCH 01/10] crypto: remove use of common.Value.Encode This seems to be the last remaining use of it. --- crypto/keypair.go | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/crypto/keypair.go b/crypto/keypair.go index 6702e6595..cc17328cb 100644 --- a/crypto/keypair.go +++ b/crypto/keypair.go @@ -3,8 +3,8 @@ package crypto import ( "strings" - "github.com/ethereum/go-ethereum/crypto/secp256k1" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/crypto/secp256k1" ) type KeyPair struct { @@ -48,11 +48,3 @@ func (k *KeyPair) Mnemonic() string { func (k *KeyPair) AsStrings() (string, string, string, string) { return k.Mnemonic(), common.Bytes2Hex(k.Address()), common.Bytes2Hex(k.PrivateKey), common.Bytes2Hex(k.PublicKey) } - -func (k *KeyPair) RlpEncode() []byte { - return k.RlpValue().Encode() -} - -func (k *KeyPair) RlpValue() *common.Value { - return common.NewValue(k.PrivateKey) -} From f7d1d601e92d3498f0fab5e80e5825d71f5d48fb Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Fri, 20 Mar 2015 11:45:45 +0100 Subject: [PATCH 02/10] common: make Value encodable with package rlp Value.{Encode,Decode} are gone. It implements rlp.Encoder and rlp.Decoder instead, so Value can be decoded into directly. --- common/rlp_test.go | 28 +++++++++++--------- common/value.go | 65 +++++++++++++++++++++++++++++++--------------- 2 files changed, 60 insertions(+), 33 deletions(-) diff --git a/common/rlp_test.go b/common/rlp_test.go index 16a3553d7..2a55da928 100644 --- a/common/rlp_test.go +++ b/common/rlp_test.go @@ -5,6 +5,8 @@ import ( "math/big" "reflect" "testing" + + "github.com/ethereum/go-ethereum/rlp" ) func TestNonInterfaceSlice(t *testing.T) { @@ -19,13 +21,16 @@ func TestNonInterfaceSlice(t *testing.T) { func TestRlpValueEncoding(t *testing.T) { val := EmptyValue() - val.AppendList().Append(1).Append(2).Append(3) - val.Append("4").AppendList().Append(5) + val.AppendList().Append(byte(1)).Append(byte(2)).Append(byte(3)) + val.Append("4").AppendList().Append(byte(5)) - res := val.Encode() + res, err := rlp.EncodeToBytes(val) + if err != nil { + t.Fatalf("encode error: %v", err) + } exp := Encode([]interface{}{[]interface{}{1, 2, 3}, "4", []interface{}{5}}) if bytes.Compare(res, exp) != 0 { - t.Errorf("expected %q, got %q", res, exp) + t.Errorf("expected %x, got %x", exp, res) } } @@ -57,9 +62,7 @@ func TestValueSlice(t *testing.T) { func TestLargeData(t *testing.T) { data := make([]byte, 100000) enc := Encode(data) - value := NewValue(enc) - value.Decode() - + value := NewValueFromBytes(enc) if value.Len() != len(data) { t.Error("Expected data to be", len(data), "got", value.Len()) } @@ -133,15 +136,16 @@ func TestEncodeDecodeBigInt(t *testing.T) { } func TestEncodeDecodeBytes(t *testing.T) { - b := NewValue([]interface{}{[]byte{1, 2, 3, 4, 5}, byte(6)}) - val := NewValueFromBytes(b.Encode()) - if !b.Cmp(val) { - t.Errorf("Expected %v, got %v", val, b) + bv := NewValue([]interface{}{[]byte{1, 2, 3, 4, 5}, []byte{6}}) + b, _ := rlp.EncodeToBytes(bv) + val := NewValueFromBytes(b) + if !bv.Cmp(val) { + t.Errorf("Expected %#v, got %#v", bv, val) } } func TestEncodeZero(t *testing.T) { - b := NewValue(0).Encode() + b, _ := rlp.EncodeToBytes(NewValue(0)) exp := []byte{0xc0} if bytes.Compare(b, exp) == 0 { t.Error("Expected", exp, "got", b) diff --git a/common/value.go b/common/value.go index 72a123772..f6150cb82 100644 --- a/common/value.go +++ b/common/value.go @@ -3,14 +3,29 @@ package common import ( "bytes" "fmt" + "io" "math/big" "reflect" "strconv" + + "github.com/ethereum/go-ethereum/rlp" ) -// Data values are returned by the rlp decoder. The data values represents -// one item within the rlp data structure. It's responsible for all the casting -// It always returns something valid +// Value can hold values of certain basic types and provides ways to +// convert between types without bothering to check whether the +// conversion is actually meaningful. +// +// It currently supports the following types: +// +// - int{,8,16,32,64} +// - uint{,8,16,32,64} +// - *big.Int +// - []byte, string +// - []interface{} +// +// Value is useful whenever you feel that Go's types limit your +// ability to express yourself. In these situations, use Value and +// forget about this strong typing nonsense. type Value struct { Val interface{} kind reflect.Value @@ -260,26 +275,34 @@ func (self *Value) DeepCmp(o *Value) bool { return bytes.Compare(self.Bytes(), o.Bytes()) == 0 } -func (val *Value) Encode() []byte { - return Encode(val.Val) -} - -// Assume that the data we have is encoded -func (self *Value) Decode() { - v, _ := Decode(self.Bytes(), 0) - self.Val = v - //self.Val = DecodeWithReader(bytes.NewBuffer(self.Bytes())) -} - -func NewValueFromBytes(data []byte) *Value { - if len(data) != 0 { - value := NewValue(data) - value.Decode() - - return value +func (self *Value) DecodeRLP(s *rlp.Stream) error { + var v interface{} + if err := s.Decode(&v); err != nil { + return err } + self.Val = v + return nil +} - return NewValue(nil) +func (self *Value) EncodeRLP(w io.Writer) error { + if self == nil { + w.Write(rlp.EmptyList) + return nil + } else { + return rlp.Encode(w, self.Val) + } +} + +// NewValueFromBytes decodes RLP data. +// The contained value will be nil if data contains invalid RLP. +func NewValueFromBytes(data []byte) *Value { + v := new(Value) + if len(data) != 0 { + if err := rlp.DecodeBytes(data, v); err != nil { + v.Val = nil + } + } + return v } // Value setters From dcb9614dfec3c8dcaaa47e9fa516528fdc47279b Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Fri, 20 Mar 2015 11:48:40 +0100 Subject: [PATCH 03/10] common: drop unused kind field from Value This makes Value 24 bytes smaller on 64bit systems. --- common/value.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/common/value.go b/common/value.go index f6150cb82..722e641b5 100644 --- a/common/value.go +++ b/common/value.go @@ -26,10 +26,7 @@ import ( // Value is useful whenever you feel that Go's types limit your // ability to express yourself. In these situations, use Value and // forget about this strong typing nonsense. -type Value struct { - Val interface{} - kind reflect.Value -} +type Value struct{ Val interface{} } func (val *Value) String() string { return fmt.Sprintf("%x", val.Val) @@ -53,7 +50,6 @@ func (val *Value) IsNil() bool { } func (val *Value) Len() int { - //return val.kind.Len() if data, ok := val.Val.([]interface{}); ok { return len(data) } From c161d73d429ef421cdb9c75b743c16d72aa8a89a Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Fri, 20 Mar 2015 13:33:11 +0100 Subject: [PATCH 04/10] common: drop accessors for Value.Val I don't see why we would need two different accessors for a public field. --- common/rlp.go | 2 +- common/value.go | 8 -------- common/value_test.go | 2 +- ethdb/memory_database.go | 2 +- 4 files changed, 3 insertions(+), 11 deletions(-) diff --git a/common/rlp.go b/common/rlp.go index 602f13202..06ac410e9 100644 --- a/common/rlp.go +++ b/common/rlp.go @@ -112,7 +112,7 @@ func Encode(object interface{}) []byte { if object != nil { switch t := object.(type) { case *Value: - buff.Write(Encode(t.Raw())) + buff.Write(Encode(t.Val)) case RlpEncodable: buff.Write(Encode(t.RlpData())) // Code dup :-/ diff --git a/common/value.go b/common/value.go index 722e641b5..c3893d565 100644 --- a/common/value.go +++ b/common/value.go @@ -57,14 +57,6 @@ func (val *Value) Len() int { return len(val.Bytes()) } -func (val *Value) Raw() interface{} { - return val.Val -} - -func (val *Value) Interface() interface{} { - return val.Val -} - func (val *Value) Uint() uint64 { if Val, ok := val.Val.(uint8); ok { return uint64(Val) diff --git a/common/value_test.go b/common/value_test.go index 09d37802d..38a0e9843 100644 --- a/common/value_test.go +++ b/common/value_test.go @@ -35,7 +35,7 @@ func (s *ValueSuite) TestValueTypes(c *checker.C) { c.Assert(str.Str(), checker.Equals, strExp) c.Assert(num.Uint(), checker.Equals, numExp) - c.Assert(NewValue(inter.Interface()).Cmp(NewValue(interExp)), checker.Equals, true) + c.Assert(NewValue(inter.Val).Cmp(NewValue(interExp)), checker.Equals, true) c.Assert(byt.Bytes(), checker.DeepEquals, bytExp) c.Assert(bigInt.BigInt(), checker.DeepEquals, bigExp) } diff --git a/ethdb/memory_database.go b/ethdb/memory_database.go index d914f47f8..d4988d0d8 100644 --- a/ethdb/memory_database.go +++ b/ethdb/memory_database.go @@ -49,7 +49,7 @@ func (db *MemDatabase) Print() { for key, val := range db.db { fmt.Printf("%x(%d): ", key, len(key)) node := common.NewValueFromBytes(val) - fmt.Printf("%q\n", node.Interface()) + fmt.Printf("%q\n", node.Val) } } From b41185a68f2ec8c492a7dbec1a6bf65f29b0843a Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Fri, 20 Mar 2015 22:33:40 +0100 Subject: [PATCH 05/10] rlp: fix nil pointer decoding The generic pointer decoder did not advance the input position for empty values. This can lead to strange issues and even infinite loops. --- rlp/decode.go | 7 ++++++- rlp/decode_test.go | 22 +++++++++++++++++++++- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/rlp/decode.go b/rlp/decode.go index 6d7e670c2..0e99d9caa 100644 --- a/rlp/decode.go +++ b/rlp/decode.go @@ -367,7 +367,12 @@ func makePtrDecoder(typ reflect.Type) (decoder, error) { dec := func(s *Stream, val reflect.Value) (err error) { _, size, err := s.Kind() if err != nil || size == 0 && s.byteval == 0 { - val.Set(reflect.Zero(typ)) // set to nil + // rearm s.Kind. This is important because the input + // position must advance to the next value even though + // we don't read anything. + s.kind = -1 + // set the pointer to nil. + val.Set(reflect.Zero(typ)) return err } newval := val diff --git a/rlp/decode_test.go b/rlp/decode_test.go index 9f66840b1..0f034d5d8 100644 --- a/rlp/decode_test.go +++ b/rlp/decode_test.go @@ -39,7 +39,7 @@ func TestStreamKind(t *testing.T) { s := NewStream(bytes.NewReader(unhex(test.input))) kind, len, err := s.Kind() if err != nil { - t.Errorf("test %d: Type returned error: %v", i, err) + t.Errorf("test %d: Kind returned error: %v", i, err) continue } if kind != test.wantKind { @@ -93,6 +93,23 @@ func TestStreamErrors(t *testing.T) { {"C3C2010201", calls{"List", "List", "Uint", "Uint", "ListEnd", "Uint"}, EOL}, {"00", calls{"ListEnd"}, errNotInList}, {"C40102", calls{"List", "Uint", "ListEnd"}, errNotAtEOL}, + + // This test verifies that the input position is advanced + // correctly when calling Bytes for empty strings. Kind can be called + // any number of times in between and doesn't advance. + {"C3808080", calls{ + "List", // enter the list + "Bytes", // past first element + + "Kind", "Kind", "Kind", // this shouldn't advance + + "Bytes", // past second element + + "Kind", "Kind", // can't hurt to try + + "Bytes", // past final element + "Bytes", // this one should fail + }, EOL}, } testfor: @@ -314,6 +331,9 @@ var decodeTests = []decodeTest{ {input: "C109", ptr: new(*[]uint), value: &[]uint{9}}, {input: "C58403030303", ptr: new(*[][]byte), value: &[][]byte{{3, 3, 3, 3}}}, + // check that input position is advanced also empty values. + {input: "C3808005", ptr: new([]*uint), value: []*uint{nil, nil, uintp(5)}}, + // pointer should be reset to nil {input: "05", ptr: sharedPtr, value: uintp(5)}, {input: "80", ptr: sharedPtr, value: (*uint)(nil)}, From b95ff54632d9a31286f5b629556071b6043d274a Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Fri, 20 Mar 2015 22:46:41 +0100 Subject: [PATCH 06/10] cmd/blocktest: delete package Blocktests are now integrated in cmd/ethereum. --- cmd/blocktest/main.go | 213 ------------------------------------------ 1 file changed, 213 deletions(-) delete mode 100644 cmd/blocktest/main.go diff --git a/cmd/blocktest/main.go b/cmd/blocktest/main.go deleted file mode 100644 index acbadee77..000000000 --- a/cmd/blocktest/main.go +++ /dev/null @@ -1,213 +0,0 @@ -/* - This file is part of go-ethereum - - go-ethereum is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - go-ethereum is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with go-ethereum. If not, see . -*/ -/** - * @authors - * Gustav Simonsson - * @date 2015 - * - */ - -package main - -import ( - "bytes" - "encoding/hex" - "encoding/json" - "flag" - "fmt" - "io/ioutil" - "log" - "math/big" - "os" - "runtime" - "strings" - - "github.com/ethereum/go-ethereum/cmd/utils" - "github.com/ethereum/go-ethereum/core" - types "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/event" - "github.com/ethereum/go-ethereum/logger" - "github.com/ethereum/go-ethereum/rlp" -) - -type Account struct { - Balance string - Code string - Nonce string - Storage map[string]string -} - -type BlockHeader struct { - Bloom string - Coinbase string - Difficulty string - ExtraData string - GasLimit string - GasUsed string - MixHash string - Nonce string - Number string - ParentHash string - ReceiptTrie string - SeedHash string - StateRoot string - Timestamp string - TransactionsTrie string - UncleHash string -} - -type Tx struct { - Data string - GasLimit string - GasPrice string - Nonce string - R string - S string - To string - V string - Value string -} - -type Block struct { - BlockHeader BlockHeader - Rlp string - Transactions []Tx - UncleHeaders []string -} - -type Test struct { - Blocks []Block - GenesisBlockHeader BlockHeader - Pre map[string]Account -} - -func main() { - flag.Usage = func() { - fmt.Fprintf(os.Stderr, "%s \n", os.Args[0]) - flag.PrintDefaults() - } - flag.Parse() - - runtime.GOMAXPROCS(runtime.NumCPU()) - logger.AddLogSystem(logger.NewStdLogSystem(os.Stderr, log.LstdFlags, logger.DebugDetailLevel)) - defer func() { logger.Flush() }() - - if len(os.Args) < 2 { - utils.Fatalf("Please specify a test file as the first argument.") - } - blocks, err := loadBlocksFromTestFile(os.Args[1]) - if err != nil { - utils.Fatalf("Could not load blocks: %v", err) - } - - chain := memchain() - chain.ResetWithGenesisBlock(blocks[0]) - if err = chain.InsertChain(types.Blocks{blocks[1]}); err != nil { - utils.Fatalf("Error: %v", err) - } else { - fmt.Println("PASS") - } -} - -func memchain() *core.ChainManager { - blockdb, err := ethdb.NewMemDatabase() - if err != nil { - utils.Fatalf("Could not create in-memory database: %v", err) - } - statedb, err := ethdb.NewMemDatabase() - if err != nil { - utils.Fatalf("Could not create in-memory database: %v", err) - } - return core.NewChainManager(blockdb, statedb, new(event.TypeMux)) -} - -func loadBlocksFromTestFile(filePath string) (blocks types.Blocks, err error) { - fileContent, err := ioutil.ReadFile(filePath) - if err != nil { - return - } - bt := make(map[string]Test) - if err = json.Unmarshal(fileContent, &bt); err != nil { - return - } - - // TODO: support multiple blocks; loop over all blocks - gbh := new(types.Header) - - // Let's use slighlty different namings for the same things, because that's awesome. - gbh.ParentHash, err = hex_decode(bt["SimpleTx"].GenesisBlockHeader.ParentHash) - gbh.UncleHash, err = hex_decode(bt["SimpleTx"].GenesisBlockHeader.UncleHash) - gbh.Coinbase, err = hex_decode(bt["SimpleTx"].GenesisBlockHeader.Coinbase) - gbh.Root, err = hex_decode(bt["SimpleTx"].GenesisBlockHeader.StateRoot) - gbh.TxHash, err = hex_decode(bt["SimpleTx"].GenesisBlockHeader.TransactionsTrie) - gbh.ReceiptHash, err = hex_decode(bt["SimpleTx"].GenesisBlockHeader.ReceiptTrie) - gbh.Bloom, err = hex_decode(bt["SimpleTx"].GenesisBlockHeader.Bloom) - - gbh.MixDigest, err = hex_decode(bt["SimpleTx"].GenesisBlockHeader.MixHash) - //gbh.SeedHash, err = hex_decode(bt["SimpleTx"].GenesisBlockHeader.SeedHash) - - d, _ := new(big.Int).SetString(bt["SimpleTx"].GenesisBlockHeader.Difficulty, 10) - gbh.Difficulty = d - - n, _ := new(big.Int).SetString(bt["SimpleTx"].GenesisBlockHeader.Number, 10) - gbh.Number = n - - gl, _ := new(big.Int).SetString(bt["SimpleTx"].GenesisBlockHeader.GasLimit, 10) - gbh.GasLimit = gl - - gu, _ := new(big.Int).SetString(bt["SimpleTx"].GenesisBlockHeader.GasUsed, 10) - gbh.GasUsed = gu - - ts, _ := new(big.Int).SetString(bt["SimpleTx"].GenesisBlockHeader.Timestamp, 0) - gbh.Time = ts.Uint64() - - extra, err := hex_decode(bt["SimpleTx"].GenesisBlockHeader.ExtraData) - gbh.Extra = string(extra) // TODO: change ExtraData to byte array - - nonce, _ := hex_decode(bt["SimpleTx"].GenesisBlockHeader.Nonce) - gbh.Nonce = nonce - - if err != nil { - return - } - - gb := types.NewBlockWithHeader(gbh) - //gb.uncles = *new([]*types.Header) - //gb.transactions = *new(types.Transactions) - gb.Td = new(big.Int) - gb.Reward = new(big.Int) - - testBlock := new(types.Block) - - rlpBytes, err := hex_decode(bt["SimpleTx"].Blocks[0].Rlp) - err = rlp.Decode(bytes.NewReader(rlpBytes), &testBlock) - if err != nil { - return - } - - blocks = types.Blocks{ - gb, - testBlock, - } - - return -} - -func hex_decode(s string) (res []byte, err error) { - return hex.DecodeString(strings.TrimPrefix(s, "0x")) -} From a829a5658723ff3681f14650818ef050cb0a7fa8 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Sat, 21 Mar 2015 00:49:31 +0100 Subject: [PATCH 07/10] rlp: add Stream.Raw --- rlp/decode.go | 25 +++++++++++++++++++++++++ rlp/decode_test.go | 16 +++++++++++++++- rlp/encode.go | 32 ++++++++++++++++++++------------ 3 files changed, 60 insertions(+), 13 deletions(-) diff --git a/rlp/decode.go b/rlp/decode.go index 0e99d9caa..0fde0a947 100644 --- a/rlp/decode.go +++ b/rlp/decode.go @@ -540,6 +540,31 @@ func (s *Stream) Bytes() ([]byte, error) { } } +// Raw reads a raw encoded value including RLP type information. +func (s *Stream) Raw() ([]byte, error) { + kind, size, err := s.Kind() + if err != nil { + return nil, err + } + if kind == Byte { + s.kind = -1 // rearm Kind + return []byte{s.byteval}, nil + } + // the original header has already been read and is no longer + // available. read content and put a new header in front of it. + start := headsize(size) + buf := make([]byte, uint64(start)+size) + if err := s.readFull(buf[start:]); err != nil { + return nil, err + } + if kind == String { + puthead(buf, 0x80, 0xB8, size) + } else { + puthead(buf, 0xC0, 0xF7, size) + } + return buf, nil +} + var errUintOverflow = errors.New("rlp: uint overflow") // Uint reads an RLP string of up to 8 bytes and returns its contents diff --git a/rlp/decode_test.go b/rlp/decode_test.go index 0f034d5d8..a18ff1d08 100644 --- a/rlp/decode_test.go +++ b/rlp/decode_test.go @@ -165,6 +165,20 @@ func TestStreamList(t *testing.T) { } } +func TestStreamRaw(t *testing.T) { + s := NewStream(bytes.NewReader(unhex("C58401010101"))) + s.List() + + want := unhex("8401010101") + raw, err := s.Raw() + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(want, raw) { + t.Errorf("raw mismatch: got %x, want %x", raw, want) + } +} + func TestDecodeErrors(t *testing.T) { r := bytes.NewReader(nil) @@ -331,7 +345,7 @@ var decodeTests = []decodeTest{ {input: "C109", ptr: new(*[]uint), value: &[]uint{9}}, {input: "C58403030303", ptr: new(*[][]byte), value: &[][]byte{{3, 3, 3, 3}}}, - // check that input position is advanced also empty values. + // check that input position is advanced also for empty values. {input: "C3808005", ptr: new([]*uint), value: []*uint{nil, nil, uintp(5)}}, // pointer should be reset to nil diff --git a/rlp/encode.go b/rlp/encode.go index 7ac74d8fb..289bc4eaa 100644 --- a/rlp/encode.go +++ b/rlp/encode.go @@ -70,7 +70,7 @@ func (e flatenc) EncodeRLP(out io.Writer) error { newhead := eb.lheads[prevnheads] copy(eb.lheads[prevnheads:], eb.lheads[prevnheads+1:]) eb.lheads = eb.lheads[:len(eb.lheads)-1] - eb.lhsize -= newhead.tagsize() + eb.lhsize -= headsize(uint64(newhead.size)) return nil } @@ -155,21 +155,29 @@ type listhead struct { // encode writes head to the given buffer, which must be at least // 9 bytes long. It returns the encoded bytes. func (head *listhead) encode(buf []byte) []byte { - if head.size < 56 { - buf[0] = 0xC0 + byte(head.size) - return buf[:1] - } else { - sizesize := putint(buf[1:], uint64(head.size)) - buf[0] = 0xF7 + byte(sizesize) - return buf[:sizesize+1] - } + return buf[:puthead(buf, 0xC0, 0xF7, uint64(head.size))] } -func (head *listhead) tagsize() int { - if head.size < 56 { +// headsize returns the size of a list or string header +// for a value of the given size. +func headsize(size uint64) int { + if size < 56 { return 1 } - return 1 + intsize(uint64(head.size)) + return 1 + intsize(size) +} + +// puthead writes a list or string header to buf. +// buf must be at least 9 bytes long. +func puthead(buf []byte, smalltag, largetag byte, size uint64) int { + if size < 56 { + buf[0] = smalltag + byte(size) + return 1 + } else { + sizesize := putint(buf[1:], size) + buf[0] = largetag + byte(sizesize) + return sizesize + 1 + } } func newencbuf() *encbuf { From 483d43a15af11a9c456e4d9df0520441b947cd35 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Sat, 21 Mar 2015 00:49:58 +0100 Subject: [PATCH 08/10] whisper: use package rlp --- whisper/envelope.go | 42 ++++++++++++++++-------------------------- whisper/peer.go | 21 +++++++++------------ 2 files changed, 25 insertions(+), 38 deletions(-) diff --git a/whisper/envelope.go b/whisper/envelope.go index 9ec9fc318..15a974a2c 100644 --- a/whisper/envelope.go +++ b/whisper/envelope.go @@ -28,9 +28,9 @@ type Envelope struct { func (self *Envelope) Hash() Hash { if self.hash == EmptyHash { - self.hash = H(crypto.Sha3(common.Encode(self))) + enc, _ := rlp.EncodeToBytes(self) + self.hash = H(crypto.Sha3(enc)) } - return self.hash } @@ -76,7 +76,8 @@ func (self *Envelope) Open(prv *ecdsa.PrivateKey) (msg *Message, err error) { func (self *Envelope) proveWork(dura time.Duration) { var bestBit int d := make([]byte, 64) - copy(d[:32], common.Encode(self.withoutNonce())) + enc, _ := rlp.EncodeToBytes(self.withoutNonce()) + copy(d[:32], enc) then := time.Now().Add(dura).UnixNano() for n := uint32(0); time.Now().UnixNano() < then; { @@ -96,39 +97,28 @@ func (self *Envelope) proveWork(dura time.Duration) { func (self *Envelope) valid() bool { d := make([]byte, 64) - copy(d[:32], common.Encode(self.withoutNonce())) + enc, _ := rlp.EncodeToBytes(self.withoutNonce()) + copy(d[:32], enc) binary.BigEndian.PutUint32(d[60:], self.Nonce) return common.FirstBitSet(common.BigD(crypto.Sha3(d))) > 0 } func (self *Envelope) withoutNonce() interface{} { - return []interface{}{self.Expiry, self.Ttl, common.ByteSliceToInterface(self.Topics), self.Data} + return []interface{}{self.Expiry, self.Ttl, self.Topics, self.Data} } -func (self *Envelope) RlpData() interface{} { - return []interface{}{self.Expiry, self.Ttl, common.ByteSliceToInterface(self.Topics), self.Data, self.Nonce} -} +// rlpenv is an Envelope but is not an rlp.Decoder. +// It is used for decoding because we need to +type rlpenv Envelope func (self *Envelope) DecodeRLP(s *rlp.Stream) error { - var extenv struct { - Expiry uint32 - Ttl uint32 - Topics [][]byte - Data []byte - Nonce uint32 - } - if err := s.Decode(&extenv); err != nil { + raw, err := s.Raw() + if err != nil { return err } - - self.Expiry = extenv.Expiry - self.Ttl = extenv.Ttl - self.Topics = extenv.Topics - self.Data = extenv.Data - self.Nonce = extenv.Nonce - - // TODO We should use the stream directly here. - self.hash = H(crypto.Sha3(common.Encode(self))) - + if err := rlp.DecodeBytes(raw, (*rlpenv)(self)); err != nil { + return err + } + self.hash = H(crypto.Sha3(raw)) return nil } diff --git a/whisper/peer.go b/whisper/peer.go index ee5bffd0b..338166c25 100644 --- a/whisper/peer.go +++ b/whisper/peer.go @@ -10,7 +10,7 @@ import ( ) const ( - protocolVersion = 0x02 + protocolVersion uint64 = 0x02 ) type peer struct { @@ -66,21 +66,18 @@ out: } func (self *peer) broadcast(envelopes []*Envelope) error { - envs := make([]interface{}, len(envelopes)) - i := 0 - for _, envelope := range envelopes { - if !self.known.Has(envelope.Hash()) { - envs[i] = envelope - self.known.Add(envelope.Hash()) - i++ + envs := make([]*Envelope, 0, len(envelopes)) + for _, env := range envelopes { + if !self.known.Has(env.Hash()) { + envs = append(envs, env) + self.known.Add(env.Hash()) } } - - if i > 0 { - if err := p2p.Send(self.ws, envelopesMsg, envs[:i]); err != nil { + if len(envs) > 0 { + if err := p2p.Send(self.ws, envelopesMsg, envs); err != nil { return err } - self.peer.DebugDetailln("broadcasted", i, "message(s)") + self.peer.DebugDetailln("broadcasted", len(envs), "message(s)") } return nil } From 7c4ff3abb4693bf93b2b348f572f14f2cfcf9142 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Sat, 21 Mar 2015 00:50:20 +0100 Subject: [PATCH 09/10] eth: enable whisper again --- eth/backend.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eth/backend.go b/eth/backend.go index b086d6a56..141c6c605 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -206,7 +206,7 @@ func New(config *Config) (*Ethereum, error) { ethProto := EthProtocol(config.ProtocolVersion, config.NetworkId, eth.txPool, eth.chainManager, eth.blockPool) protocols := []p2p.Protocol{ethProto} if config.Shh { - //protocols = append(protocols, eth.whisper.Protocol()) + protocols = append(protocols, eth.whisper.Protocol()) } eth.net = &p2p.Server{ From 069c87b960c48864dc4f1b9086adf582e1dc88a9 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Sat, 21 Mar 2015 00:57:18 +0100 Subject: [PATCH 10/10] whisper: use common.Hash --- whisper/envelope.go | 23 ++++++++++++++--------- whisper/sort.go | 8 ++++++-- whisper/sort_test.go | 16 ++++++++++------ whisper/whisper.go | 28 ++++------------------------ 4 files changed, 34 insertions(+), 41 deletions(-) diff --git a/whisper/envelope.go b/whisper/envelope.go index 15a974a2c..20e3e6d39 100644 --- a/whisper/envelope.go +++ b/whisper/envelope.go @@ -18,26 +18,31 @@ const ( type Envelope struct { Expiry uint32 // Whisper protocol specifies int32, really should be int64 - Ttl uint32 // ^^^^^^ + TTL uint32 // ^^^^^^ Topics [][]byte Data []byte Nonce uint32 - hash Hash + hash common.Hash } -func (self *Envelope) Hash() Hash { - if self.hash == EmptyHash { +func (self *Envelope) Hash() common.Hash { + if (self.hash == common.Hash{}) { enc, _ := rlp.EncodeToBytes(self) - self.hash = H(crypto.Sha3(enc)) + self.hash = crypto.Sha3Hash(enc) } return self.hash } func NewEnvelope(ttl time.Duration, topics [][]byte, data *Message) *Envelope { exp := time.Now().Add(ttl) - - return &Envelope{uint32(exp.Unix()), uint32(ttl.Seconds()), topics, data.Bytes(), 0, Hash{}} + return &Envelope{ + Expiry: uint32(exp.Unix()), + TTL: uint32(ttl.Seconds()), + Topics: topics, + Data: data.Bytes(), + Nonce: 0, + } } func (self *Envelope) Seal(pow time.Duration) { @@ -104,7 +109,7 @@ func (self *Envelope) valid() bool { } func (self *Envelope) withoutNonce() interface{} { - return []interface{}{self.Expiry, self.Ttl, self.Topics, self.Data} + return []interface{}{self.Expiry, self.TTL, self.Topics, self.Data} } // rlpenv is an Envelope but is not an rlp.Decoder. @@ -119,6 +124,6 @@ func (self *Envelope) DecodeRLP(s *rlp.Stream) error { if err := rlp.DecodeBytes(raw, (*rlpenv)(self)); err != nil { return err } - self.hash = H(crypto.Sha3(raw)) + self.hash = crypto.Sha3Hash(raw) return nil } diff --git a/whisper/sort.go b/whisper/sort.go index 8c5b46e9e..313ba5ac0 100644 --- a/whisper/sort.go +++ b/whisper/sort.go @@ -1,6 +1,10 @@ package whisper -import "sort" +import ( + "sort" + + "github.com/ethereum/go-ethereum/common" +) type sortedKeys struct { k []int32 @@ -10,7 +14,7 @@ func (self *sortedKeys) Len() int { return len(self.k) } func (self *sortedKeys) Less(i, j int) bool { return self.k[i] < self.k[j] } func (self *sortedKeys) Swap(i, j int) { self.k[i], self.k[j] = self.k[j], self.k[i] } -func sortKeys(m map[int32]Hash) []int32 { +func sortKeys(m map[int32]common.Hash) []int32 { sorted := new(sortedKeys) sorted.k = make([]int32, len(m)) i := 0 diff --git a/whisper/sort_test.go b/whisper/sort_test.go index 5d8177d41..a61fde4c2 100644 --- a/whisper/sort_test.go +++ b/whisper/sort_test.go @@ -1,13 +1,17 @@ package whisper -import "testing" +import ( + "testing" + + "github.com/ethereum/go-ethereum/common" +) func TestSorting(t *testing.T) { - m := map[int32]Hash{ - 1: HS("1"), - 3: HS("3"), - 2: HS("2"), - 5: HS("5"), + m := map[int32]common.Hash{ + 1: {1}, + 3: {3}, + 2: {2}, + 5: {5}, } exp := []int32{1, 2, 3, 5} res := sortKeys(m) diff --git a/whisper/whisper.go b/whisper/whisper.go index 908df973c..dbd4fc85f 100644 --- a/whisper/whisper.go +++ b/whisper/whisper.go @@ -1,12 +1,12 @@ package whisper import ( - "bytes" "crypto/ecdsa" "errors" "sync" "time" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/crypto/ecies" "github.com/ethereum/go-ethereum/event/filter" @@ -15,26 +15,6 @@ import ( "gopkg.in/fatih/set.v0" ) -// MOVE ME -type Hash struct { - hash string -} - -var EmptyHash Hash - -func H(hash []byte) Hash { - return Hash{string(hash)} -} -func HS(hash string) Hash { - return Hash{hash} -} - -func (self Hash) Compare(other Hash) int { - return bytes.Compare([]byte(self.hash), []byte(other.hash)) -} - -// MOVE ME END - const ( statusMsg = 0x0 envelopesMsg = 0x01 @@ -55,7 +35,7 @@ type Whisper struct { filters *filter.Filters mmu sync.RWMutex - messages map[Hash]*Envelope + messages map[common.Hash]*Envelope expiry map[uint32]*set.SetNonTS quit chan struct{} @@ -65,7 +45,7 @@ type Whisper struct { func New() *Whisper { whisper := &Whisper{ - messages: make(map[Hash]*Envelope), + messages: make(map[common.Hash]*Envelope), filters: filter.New(), expiry: make(map[uint32]*set.SetNonTS), quit: make(chan struct{}), @@ -239,7 +219,7 @@ func (self *Whisper) expire() { } hashSet.Each(func(v interface{}) bool { - delete(self.messages, v.(Hash)) + delete(self.messages, v.(common.Hash)) return true }) self.expiry[then].Clear()