diff --git a/.gitignore b/.gitignore index 346ef9afa..6b8d4ab21 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,5 @@ Cargo.lock *.pk *.sk *.raw_keypairs +flamegraph.svg +perf.data* diff --git a/.travis.yml b/.travis.yml index e725aa0ba..f89db54c9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,9 @@ language: rust +cache: + directories: + - /home/travis/.cargo +before_cache: + - rm -rf /home/travis/.cargo/registry before_install: - curl -OL https://github.com/google/protobuf/releases/download/v3.4.0/protoc-3.4.0-linux-x86_64.zip - unzip protoc-3.4.0-linux-x86_64.zip -d protoc3 @@ -6,11 +11,13 @@ before_install: - sudo mv protoc3/include/* /usr/local/include/ - sudo chown $USER /usr/local/bin/protoc - sudo chown -R $USER /usr/local/include/google +env: + - BUILD=--all + - BUILD=--release --all + - BUILD= --manifest-path eth2/state_processing/Cargo.toml --release --features fake_crypto script: - - cargo build --verbose --all - - cargo build --verbose --release --all - - cargo test --verbose --all - - cargo test --verbose --release --all + - cargo build --verbose $BUILD + - cargo test --verbose $BUILD - cargo fmt --all -- --check # No clippy until later... #- cargo clippy @@ -22,6 +29,15 @@ matrix: allow_failures: - rust: nightly fast_finish: true + exclude: + - rust: beta + env: BUILD=--release --all + - rust: beta + env: BUILD= --manifest-path eth2/state_processing/Cargo.toml --release --features fake_crypto + - rust: nightly + env: BUILD=--release --all + - rust: nightly + env: BUILD= --manifest-path eth2/state_processing/Cargo.toml --release --features fake_crypto install: - rustup component add rustfmt - rustup component add clippy diff --git a/Cargo.toml b/Cargo.toml index 5c9593f5a..c05e22286 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,6 +9,7 @@ members = [ "eth2/types", "eth2/utils/bls", "eth2/utils/boolean-bitfield", + "eth2/utils/cached_tree_hash", "eth2/utils/hashing", "eth2/utils/honey-badger-split", "eth2/utils/merkle_proof", @@ -18,6 +19,8 @@ members = [ "eth2/utils/ssz", "eth2/utils/ssz_derive", "eth2/utils/swap_or_not_shuffle", + "eth2/utils/tree_hash", + "eth2/utils/tree_hash_derive", "eth2/utils/fisher_yates_shuffle", "eth2/utils/test_random_derive", "beacon_node", diff --git a/Jenkinsfile b/Jenkinsfile index d12189941..11cbf0abe 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -23,6 +23,11 @@ pipeline { steps { sh 'cargo test --verbose --all' sh 'cargo test --verbose --all --release' + sh 'cargo test --manifest-path eth2/state_processing/Cargo.toml --verbose \ + --release --features fake_crypto' + sh 'cargo test --manifest-path eth2/state_processing/Cargo.toml --verbose \ + --release --features fake_crypto -- --ignored' + } } } diff --git a/LICENSE b/LICENSE index d159169d1..98016b543 100644 --- a/LICENSE +++ b/LICENSE @@ -1,339 +1,201 @@ - GNU GENERAL PUBLIC LICENSE - Version 2, June 1991 + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ - Copyright (C) 1989, 1991 Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - Everyone is permitted to copy and distribute verbatim copies - of this license document, but changing it is not allowed. + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - Preamble + 1. Definitions. - The licenses for most software are designed to take away your -freedom to share and change it. By contrast, the GNU General Public -License is intended to guarantee your freedom to share and change free -software--to make sure the software is free for all its users. This -General Public License applies to most of the Free Software -Foundation's software and to any other program whose authors commit to -using it. (Some other Free Software Foundation software is covered by -the GNU Lesser General Public License instead.) You can apply it to -your programs, too. + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. - When we speak of free software, we are referring to freedom, not -price. Our General Public Licenses are designed to make sure that you -have the freedom to distribute copies of free software (and charge for -this service if you wish), that you receive source code or can get it -if you want it, that you can change the software or use pieces of it -in new free programs; and that you know you can do these things. + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. - To protect your rights, we need to make restrictions that forbid -anyone to deny you these rights or to ask you to surrender the rights. -These restrictions translate to certain responsibilities for you if you -distribute copies of the software, or if you modify it. + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. - For example, if you distribute copies of such a program, whether -gratis or for a fee, you must give the recipients all the rights that -you have. You must make sure that they, too, receive or can get the -source code. And you must show them these terms so they know their -rights. + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. - We protect your rights with two steps: (1) copyright the software, and -(2) offer you this license which gives you legal permission to copy, -distribute and/or modify the software. + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. - Also, for each author's protection and ours, we want to make certain -that everyone understands that there is no warranty for this free -software. If the software is modified by someone else and passed on, we -want its recipients to know that what they have is not the original, so -that any problems introduced by others will not reflect on the original -authors' reputations. + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. - Finally, any free program is threatened constantly by software -patents. We wish to avoid the danger that redistributors of a free -program will individually obtain patent licenses, in effect making the -program proprietary. To prevent this, we have made it clear that any -patent must be licensed for everyone's free use or not licensed at all. + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). - The precise terms and conditions for copying, distribution and -modification follow. + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. - GNU GENERAL PUBLIC LICENSE - TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." - 0. This License applies to any program or other work which contains -a notice placed by the copyright holder saying it may be distributed -under the terms of this General Public License. The "Program", below, -refers to any such program or work, and a "work based on the Program" -means either the Program or any derivative work under copyright law: -that is to say, a work containing the Program or a portion of it, -either verbatim or with modifications and/or translated into another -language. (Hereinafter, translation is included without limitation in -the term "modification".) Each licensee is addressed as "you". + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. -Activities other than copying, distribution and modification are not -covered by this License; they are outside its scope. The act of -running the Program is not restricted, and the output from the Program -is covered only if its contents constitute a work based on the -Program (independent of having been made by running the Program). -Whether that is true depends on what the Program does. + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. - 1. You may copy and distribute verbatim copies of the Program's -source code as you receive it, in any medium, provided that you -conspicuously and appropriately publish on each copy an appropriate -copyright notice and disclaimer of warranty; keep intact all the -notices that refer to this License and to the absence of any warranty; -and give any other recipients of the Program a copy of this License -along with the Program. + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. -You may charge a fee for the physical act of transferring a copy, and -you may at your option offer warranty protection in exchange for a fee. + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: - 2. You may modify your copy or copies of the Program or any portion -of it, thus forming a work based on the Program, and copy and -distribute such modifications or work under the terms of Section 1 -above, provided that you also meet all of these conditions: + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and - a) You must cause the modified files to carry prominent notices - stating that you changed the files and the date of any change. + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and - b) You must cause any work that you distribute or publish, that in - whole or in part contains or is derived from the Program or any - part thereof, to be licensed as a whole at no charge to all third - parties under the terms of this License. + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and - c) If the modified program normally reads commands interactively - when run, you must cause it, when started running for such - interactive use in the most ordinary way, to print or display an - announcement including an appropriate copyright notice and a - notice that there is no warranty (or else, saying that you provide - a warranty) and that users may redistribute the program under - these conditions, and telling the user how to view a copy of this - License. (Exception: if the Program itself is interactive but - does not normally print such an announcement, your work based on - the Program is not required to print an announcement.) + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. -These requirements apply to the modified work as a whole. If -identifiable sections of that work are not derived from the Program, -and can be reasonably considered independent and separate works in -themselves, then this License, and its terms, do not apply to those -sections when you distribute them as separate works. But when you -distribute the same sections as part of a whole which is a work based -on the Program, the distribution of the whole must be on the terms of -this License, whose permissions for other licensees extend to the -entire whole, and thus to each and every part regardless of who wrote it. + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. -Thus, it is not the intent of this section to claim rights or contest -your rights to work written entirely by you; rather, the intent is to -exercise the right to control the distribution of derivative or -collective works based on the Program. + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. -In addition, mere aggregation of another work not based on the Program -with the Program (or with a work based on the Program) on a volume of -a storage or distribution medium does not bring the other work under -the scope of this License. + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. - 3. You may copy and distribute the Program (or a work based on it, -under Section 2) in object code or executable form under the terms of -Sections 1 and 2 above provided that you also do one of the following: + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. - a) Accompany it with the complete corresponding machine-readable - source code, which must be distributed under the terms of Sections - 1 and 2 above on a medium customarily used for software interchange; or, + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. - b) Accompany it with a written offer, valid for at least three - years, to give any third party, for a charge no more than your - cost of physically performing source distribution, a complete - machine-readable copy of the corresponding source code, to be - distributed under the terms of Sections 1 and 2 above on a medium - customarily used for software interchange; or, + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. - c) Accompany it with the information you received as to the offer - to distribute corresponding source code. (This alternative is - allowed only for noncommercial distribution and only if you - received the program in object code or executable form with such - an offer, in accord with Subsection b above.) + END OF TERMS AND CONDITIONS -The source code for a work means the preferred form of the work for -making modifications to it. For an executable work, complete source -code means all the source code for all modules it contains, plus any -associated interface definition files, plus the scripts used to -control compilation and installation of the executable. However, as a -special exception, the source code distributed need not include -anything that is normally distributed (in either source or binary -form) with the major components (compiler, kernel, and so on) of the -operating system on which the executable runs, unless that component -itself accompanies the executable. + APPENDIX: How to apply the Apache License to your work. -If distribution of executable or object code is made by offering -access to copy from a designated place, then offering equivalent -access to copy the source code from the same place counts as -distribution of the source code, even though third parties are not -compelled to copy the source along with the object code. + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. - 4. You may not copy, modify, sublicense, or distribute the Program -except as expressly provided under this License. Any attempt -otherwise to copy, modify, sublicense or distribute the Program is -void, and will automatically terminate your rights under this License. -However, parties who have received copies, or rights, from you under -this License will not have their licenses terminated so long as such -parties remain in full compliance. + Copyright 2018 Sigma Prime Pty Ltd - 5. You are not required to accept this License, since you have not -signed it. However, nothing else grants you permission to modify or -distribute the Program or its derivative works. These actions are -prohibited by law if you do not accept this License. Therefore, by -modifying or distributing the Program (or any work based on the -Program), you indicate your acceptance of this License to do so, and -all its terms and conditions for copying, distributing or modifying -the Program or works based on it. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at - 6. Each time you redistribute the Program (or any work based on the -Program), the recipient automatically receives a license from the -original licensor to copy, distribute or modify the Program subject to -these terms and conditions. You may not impose any further -restrictions on the recipients' exercise of the rights granted herein. -You are not responsible for enforcing compliance by third parties to -this License. + http://www.apache.org/licenses/LICENSE-2.0 - 7. If, as a consequence of a court judgment or allegation of patent -infringement or for any other reason (not limited to patent issues), -conditions are imposed on you (whether by court order, agreement or -otherwise) that contradict the conditions of this License, they do not -excuse you from the conditions of this License. If you cannot -distribute so as to satisfy simultaneously your obligations under this -License and any other pertinent obligations, then as a consequence you -may not distribute the Program at all. For example, if a patent -license would not permit royalty-free redistribution of the Program by -all those who receive copies directly or indirectly through you, then -the only way you could satisfy both it and this License would be to -refrain entirely from distribution of the Program. - -If any portion of this section is held invalid or unenforceable under -any particular circumstance, the balance of the section is intended to -apply and the section as a whole is intended to apply in other -circumstances. - -It is not the purpose of this section to induce you to infringe any -patents or other property right claims or to contest validity of any -such claims; this section has the sole purpose of protecting the -integrity of the free software distribution system, which is -implemented by public license practices. Many people have made -generous contributions to the wide range of software distributed -through that system in reliance on consistent application of that -system; it is up to the author/donor to decide if he or she is willing -to distribute software through any other system and a licensee cannot -impose that choice. - -This section is intended to make thoroughly clear what is believed to -be a consequence of the rest of this License. - - 8. If the distribution and/or use of the Program is restricted in -certain countries either by patents or by copyrighted interfaces, the -original copyright holder who places the Program under this License -may add an explicit geographical distribution limitation excluding -those countries, so that distribution is permitted only in or among -countries not thus excluded. In such case, this License incorporates -the limitation as if written in the body of this License. - - 9. The Free Software Foundation may publish revised and/or new versions -of the General Public License from time to time. Such new versions will -be similar in spirit to the present version, but may differ in detail to -address new problems or concerns. - -Each version is given a distinguishing version number. If the Program -specifies a version number of this License which applies to it and "any -later version", you have the option of following the terms and conditions -either of that version or of any later version published by the Free -Software Foundation. If the Program does not specify a version number of -this License, you may choose any version ever published by the Free Software -Foundation. - - 10. If you wish to incorporate parts of the Program into other free -programs whose distribution conditions are different, write to the author -to ask for permission. For software which is copyrighted by the Free -Software Foundation, write to the Free Software Foundation; we sometimes -make exceptions for this. Our decision will be guided by the two goals -of preserving the free status of all derivatives of our free software and -of promoting the sharing and reuse of software generally. - - NO WARRANTY - - 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY -FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN -OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES -PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED -OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS -TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE -PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, -REPAIR OR CORRECTION. - - 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING -WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR -REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, -INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING -OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED -TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY -YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER -PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE -POSSIBILITY OF SUCH DAMAGES. - - END OF TERMS AND CONDITIONS - - How to Apply These Terms to Your New Programs - - If you develop a new program, and you want it to be of the greatest -possible use to the public, the best way to achieve this is to make it -free software which everyone can redistribute and change under these terms. - - To do so, attach the following notices to the program. It is safest -to attach them to the start of each source file to most effectively -convey the exclusion of warranty; and each file should have at least -the "copyright" line and a pointer to where the full notice is found. - - - Copyright (C) - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -Also add information on how to contact you by electronic and paper mail. - -If the program is interactive, make it output a short notice like this -when it starts in an interactive mode: - - Gnomovision version 69, Copyright (C) year name of author - Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. - This is free software, and you are welcome to redistribute it - under certain conditions; type `show c' for details. - -The hypothetical commands `show w' and `show c' should show the appropriate -parts of the General Public License. Of course, the commands you use may -be called something other than `show w' and `show c'; they could even be -mouse-clicks or menu items--whatever suits your program. - -You should also get your employer (if you work as a programmer) or your -school, if any, to sign a "copyright disclaimer" for the program, if -necessary. Here is a sample; alter the names: - - Yoyodyne, Inc., hereby disclaims all copyright interest in the program - `Gnomovision' (which makes passes at compilers) written by James Hacker. - - , 1 April 1989 - Ty Coon, President of Vice - -This General Public License does not permit incorporating your program into -proprietary programs. If your program is a subroutine library, you may -consider it more useful to permit linking proprietary applications with the -library. If this is what you want to do, use the GNU Lesser General -Public License instead of this License. + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.md b/README.md index 7727154e7..abf9acb6a 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,7 @@ present-Ethereum functionality. - [About Lighthouse](docs/lighthouse.md): Goals, Ideology and Ethos surrounding this implementation. - [What is Ethereum Serenity](docs/serenity.md): an introduction to Ethereum Serenity. +- [Lighthouse Technical Documentation](http://lighthouse-docs.sigmaprime.io/): The Rust generated documentation, updated regularly. If you'd like some background on Sigma Prime, please see the [Lighthouse Update \#00](https://lighthouse.sigmaprime.io/update-00.html) blog post or the diff --git a/beacon_node/beacon_chain/Cargo.toml b/beacon_node/beacon_chain/Cargo.toml index 55d4bacfd..e2a4527a9 100644 --- a/beacon_node/beacon_chain/Cargo.toml +++ b/beacon_node/beacon_chain/Cargo.toml @@ -23,4 +23,5 @@ serde_json = "1.0" slot_clock = { path = "../../eth2/utils/slot_clock" } ssz = { path = "../../eth2/utils/ssz" } state_processing = { path = "../../eth2/state_processing" } +tree_hash = { path = "../../eth2/utils/tree_hash" } types = { path = "../../eth2/types" } diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index a22f4179e..41a718655 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -303,8 +303,6 @@ where /// then having it iteratively updated -- in such a case it's possible for another thread to /// find the state at an old slot. pub fn update_state(&self, mut state: BeaconState) -> Result<(), Error> { - let latest_block_header = self.head().beacon_block.block_header(); - let present_slot = match self.slot_clock.present_slot() { Ok(Some(slot)) => slot, _ => return Err(Error::UnableToReadSlot), @@ -312,7 +310,7 @@ where // If required, transition the new state to the present slot. for _ in state.slot.as_u64()..present_slot.as_u64() { - per_slot_processing(&mut state, &latest_block_header, &self.spec)?; + per_slot_processing(&mut state, &self.spec)?; } state.build_all_caches(&self.spec)?; @@ -324,8 +322,6 @@ where /// Ensures the current canonical `BeaconState` has been transitioned to match the `slot_clock`. pub fn catchup_state(&self) -> Result<(), Error> { - let latest_block_header = self.head().beacon_block.block_header(); - let present_slot = match self.slot_clock.present_slot() { Ok(Some(slot)) => slot, _ => return Err(Error::UnableToReadSlot), @@ -339,7 +335,7 @@ where state.build_epoch_cache(RelativeEpoch::NextWithoutRegistryChange, &self.spec)?; state.build_epoch_cache(RelativeEpoch::NextWithRegistryChange, &self.spec)?; - per_slot_processing(&mut *state, &latest_block_header, &self.spec)?; + per_slot_processing(&mut *state, &self.spec)?; } state.build_all_caches(&self.spec)?; @@ -617,9 +613,8 @@ where // Transition the parent state to the block slot. let mut state = parent_state; - let previous_block_header = parent_block.block_header(); for _ in state.slot.as_u64()..block.slot.as_u64() { - if let Err(e) = per_slot_processing(&mut state, &previous_block_header, &self.spec) { + if let Err(e) = per_slot_processing(&mut state, &self.spec) { return Ok(BlockProcessingOutcome::InvalidBlock( InvalidBlock::SlotProcessingError(e), )); diff --git a/beacon_node/beacon_chain/src/initialise.rs b/beacon_node/beacon_chain/src/initialise.rs index 0951e06fb..c66dd63b1 100644 --- a/beacon_node/beacon_chain/src/initialise.rs +++ b/beacon_node/beacon_chain/src/initialise.rs @@ -7,9 +7,9 @@ use db::stores::{BeaconBlockStore, BeaconStateStore}; use db::{DiskDB, MemoryDB}; use fork_choice::BitwiseLMDGhost; use slot_clock::SystemTimeSlotClock; -use ssz::TreeHash; use std::path::PathBuf; use std::sync::Arc; +use tree_hash::TreeHash; use types::test_utils::TestingBeaconStateBuilder; use types::{BeaconBlock, ChainSpec, Hash256}; @@ -32,7 +32,7 @@ pub fn initialise_beacon_chain( let (genesis_state, _keypairs) = state_builder.build(); let mut genesis_block = BeaconBlock::empty(&spec); - genesis_block.state_root = Hash256::from_slice(&genesis_state.hash_tree_root()); + genesis_block.state_root = Hash256::from_slice(&genesis_state.tree_hash_root()); // Slot clock let slot_clock = SystemTimeSlotClock::new( @@ -73,7 +73,7 @@ pub fn initialise_test_beacon_chain( let (genesis_state, _keypairs) = state_builder.build(); let mut genesis_block = BeaconBlock::empty(spec); - genesis_block.state_root = Hash256::from_slice(&genesis_state.hash_tree_root()); + genesis_block.state_root = Hash256::from_slice(&genesis_state.tree_hash_root()); // Slot clock let slot_clock = SystemTimeSlotClock::new( diff --git a/beacon_node/beacon_chain/src/test_utils/testing_beacon_chain_builder.rs b/beacon_node/beacon_chain/src/test_utils/testing_beacon_chain_builder.rs index 5c5477e55..d174670c0 100644 --- a/beacon_node/beacon_chain/src/test_utils/testing_beacon_chain_builder.rs +++ b/beacon_node/beacon_chain/src/test_utils/testing_beacon_chain_builder.rs @@ -5,8 +5,8 @@ use db::{ }; use fork_choice::BitwiseLMDGhost; use slot_clock::TestingSlotClock; -use ssz::TreeHash; use std::sync::Arc; +use tree_hash::TreeHash; use types::test_utils::TestingBeaconStateBuilder; use types::*; @@ -27,7 +27,7 @@ impl TestingBeaconChainBuilder { let (genesis_state, _keypairs) = self.state_builder.build(); let mut genesis_block = BeaconBlock::empty(&spec); - genesis_block.state_root = Hash256::from_slice(&genesis_state.hash_tree_root()); + genesis_block.state_root = Hash256::from_slice(&genesis_state.tree_hash_root()); // Create the Beacon Chain BeaconChain::from_genesis( diff --git a/beacon_node/beacon_chain/test_harness/Cargo.toml b/beacon_node/beacon_chain/test_harness/Cargo.toml index 50d154732..a2abf6c5a 100644 --- a/beacon_node/beacon_chain/test_harness/Cargo.toml +++ b/beacon_node/beacon_chain/test_harness/Cargo.toml @@ -38,5 +38,6 @@ serde_json = "1.0" serde_yaml = "0.8" slot_clock = { path = "../../../eth2/utils/slot_clock" } ssz = { path = "../../../eth2/utils/ssz" } +tree_hash = { path = "../../../eth2/utils/tree_hash" } types = { path = "../../../eth2/types" } yaml-rust = "0.4.2" diff --git a/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs b/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs index aeb734a4e..34b559478 100644 --- a/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs +++ b/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs @@ -9,8 +9,8 @@ use fork_choice::BitwiseLMDGhost; use log::debug; use rayon::prelude::*; use slot_clock::TestingSlotClock; -use ssz::TreeHash; use std::sync::Arc; +use tree_hash::TreeHash; use types::{test_utils::TestingBeaconStateBuilder, *}; type TestingBeaconChain = BeaconChain>; @@ -54,7 +54,7 @@ impl BeaconChainHarness { let (mut genesis_state, keypairs) = state_builder.build(); let mut genesis_block = BeaconBlock::empty(&spec); - genesis_block.state_root = Hash256::from_slice(&genesis_state.hash_tree_root()); + genesis_block.state_root = Hash256::from_slice(&genesis_state.tree_hash_root()); genesis_state .build_epoch_cache(RelativeEpoch::Previous, &spec) @@ -163,7 +163,7 @@ impl BeaconChainHarness { data: data.clone(), custody_bit: false, } - .hash_tree_root(); + .tree_hash_root(); let domain = self.spec.get_domain( state.slot.epoch(self.spec.slots_per_epoch), Domain::Attestation, diff --git a/beacon_node/beacon_chain/test_harness/src/lib.rs b/beacon_node/beacon_chain/test_harness/src/lib.rs index 0703fd4a5..e93fa7003 100644 --- a/beacon_node/beacon_chain/test_harness/src/lib.rs +++ b/beacon_node/beacon_chain/test_harness/src/lib.rs @@ -8,7 +8,7 @@ //! producing blocks and attestations. //! //! Example: -//! ``` +//! ```rust,no_run //! use test_harness::BeaconChainHarness; //! use types::ChainSpec; //! diff --git a/beacon_node/beacon_chain/test_harness/src/test_case.rs b/beacon_node/beacon_chain/test_harness/src/test_case.rs index f65b45505..28c7ae8a8 100644 --- a/beacon_node/beacon_chain/test_harness/src/test_case.rs +++ b/beacon_node/beacon_chain/test_harness/src/test_case.rs @@ -4,7 +4,7 @@ use crate::beacon_chain_harness::BeaconChainHarness; use beacon_chain::CheckPoint; use log::{info, warn}; -use ssz::SignedRoot; +use tree_hash::SignedRoot; use types::*; use types::test_utils::*; diff --git a/beacon_node/beacon_chain/test_harness/tests/chain.rs b/beacon_node/beacon_chain/test_harness/tests/chain.rs index e72c3a5aa..d47de6889 100644 --- a/beacon_node/beacon_chain/test_harness/tests/chain.rs +++ b/beacon_node/beacon_chain/test_harness/tests/chain.rs @@ -1,3 +1,5 @@ +#![cfg(not(debug_assertions))] + use env_logger::{Builder, Env}; use log::debug; use test_harness::BeaconChainHarness; diff --git a/beacon_node/network/Cargo.toml b/beacon_node/network/Cargo.toml index cd2c2269a..36bf1f141 100644 --- a/beacon_node/network/Cargo.toml +++ b/beacon_node/network/Cargo.toml @@ -15,6 +15,7 @@ version = { path = "../version" } types = { path = "../../eth2/types" } slog = { version = "^2.2.3" , features = ["max_level_trace", "release_max_level_debug"] } ssz = { path = "../../eth2/utils/ssz" } +tree_hash = { path = "../../eth2/utils/tree_hash" } futures = "0.1.25" error-chain = "0.12.0" crossbeam-channel = "0.3.8" diff --git a/beacon_node/network/src/sync/import_queue.rs b/beacon_node/network/src/sync/import_queue.rs index 0026347eb..106e3eb66 100644 --- a/beacon_node/network/src/sync/import_queue.rs +++ b/beacon_node/network/src/sync/import_queue.rs @@ -2,9 +2,9 @@ use crate::beacon_chain::BeaconChain; use eth2_libp2p::rpc::methods::*; use eth2_libp2p::PeerId; use slog::{debug, error}; -use ssz::TreeHash; use std::sync::Arc; use std::time::{Duration, Instant}; +use tree_hash::TreeHash; use types::{BeaconBlock, BeaconBlockBody, BeaconBlockHeader, Hash256, Slot}; /// Provides a queue for fully and partially built `BeaconBlock`s. @@ -15,7 +15,7 @@ use types::{BeaconBlock, BeaconBlockBody, BeaconBlockHeader, Hash256, Slot}; /// /// - When we receive a `BeaconBlockBody`, the only way we can find it's matching /// `BeaconBlockHeader` is to find a header such that `header.beacon_block_body == -/// hash_tree_root(body)`. Therefore, if we used a `HashMap` we would need to use the root of +/// tree_hash_root(body)`. Therefore, if we used a `HashMap` we would need to use the root of /// `BeaconBlockBody` as the key. /// - It is possible for multiple distinct blocks to have identical `BeaconBlockBodies`. Therefore /// we cannot use a `HashMap` keyed by the root of `BeaconBlockBody`. @@ -166,7 +166,7 @@ impl ImportQueue { let mut required_bodies: Vec = vec![]; for header in headers { - let block_root = Hash256::from_slice(&header.hash_tree_root()[..]); + let block_root = Hash256::from_slice(&header.tree_hash_root()[..]); if self.chain_has_not_seen_block(&block_root) { self.insert_header(block_root, header, sender.clone()); @@ -230,7 +230,7 @@ impl ImportQueue { /// /// If the body already existed, the `inserted` time is set to `now`. fn insert_body(&mut self, body: BeaconBlockBody, sender: PeerId) { - let body_root = Hash256::from_slice(&body.hash_tree_root()[..]); + let body_root = Hash256::from_slice(&body.tree_hash_root()[..]); self.partials.iter_mut().for_each(|mut p| { if let Some(header) = &mut p.header { @@ -250,7 +250,7 @@ impl ImportQueue { /// /// If the partial already existed, the `inserted` time is set to `now`. fn insert_full_block(&mut self, block: BeaconBlock, sender: PeerId) { - let block_root = Hash256::from_slice(&block.hash_tree_root()[..]); + let block_root = Hash256::from_slice(&block.tree_hash_root()[..]); let partial = PartialBeaconBlock { slot: block.slot, diff --git a/beacon_node/network/src/sync/simple_sync.rs b/beacon_node/network/src/sync/simple_sync.rs index 824458b89..1b57fbc00 100644 --- a/beacon_node/network/src/sync/simple_sync.rs +++ b/beacon_node/network/src/sync/simple_sync.rs @@ -5,10 +5,10 @@ use eth2_libp2p::rpc::methods::*; use eth2_libp2p::rpc::{RPCRequest, RPCResponse, RequestId}; use eth2_libp2p::PeerId; use slog::{debug, error, info, o, warn}; -use ssz::TreeHash; use std::collections::HashMap; use std::sync::Arc; use std::time::Duration; +use tree_hash::TreeHash; use types::{Attestation, BeaconBlock, Epoch, Hash256, Slot}; /// The number of slots that we can import blocks ahead of us, before going into full Sync mode. @@ -565,7 +565,7 @@ impl SimpleSync { return false; } - let block_root = Hash256::from_slice(&block.hash_tree_root()); + let block_root = Hash256::from_slice(&block.tree_hash_root()); // Ignore any block that the chain already knows about. if self.chain_has_seen_block(&block_root) { diff --git a/docs/documentation.md b/docs/documentation.md new file mode 100644 index 000000000..360055887 --- /dev/null +++ b/docs/documentation.md @@ -0,0 +1,14 @@ +# Lighthouse Technical Documentation + +The technical documentation, as generated by Rust, is available at [lighthouse-docs.sigmaprime.io](http://lighthouse-docs.sigmaprime.io/). + +This documentation is generated from Lighthouse and updated regularly. + + +### How to update: + +- `cargo doc`: builds the docs inside the `target/doc/` directory. +- `aws s3 sync target/doc/ s3://lighthouse-docs.sigmaprime.io/`: Uploads all of the docs, as generated with `cargo doc`, to the S3 bucket. + +**Note**: You will need appropriate credentials to make the upload. + diff --git a/eth2/attester/Cargo.toml b/eth2/attester/Cargo.toml index 956ecf565..41824274d 100644 --- a/eth2/attester/Cargo.toml +++ b/eth2/attester/Cargo.toml @@ -7,4 +7,5 @@ edition = "2018" [dependencies] slot_clock = { path = "../../eth2/utils/slot_clock" } ssz = { path = "../../eth2/utils/ssz" } +tree_hash = { path = "../../eth2/utils/tree_hash" } types = { path = "../../eth2/types" } diff --git a/eth2/attester/src/lib.rs b/eth2/attester/src/lib.rs index a4295f005..1bbbd6b43 100644 --- a/eth2/attester/src/lib.rs +++ b/eth2/attester/src/lib.rs @@ -2,8 +2,8 @@ pub mod test_utils; mod traits; use slot_clock::SlotClock; -use ssz::TreeHash; use std::sync::Arc; +use tree_hash::TreeHash; use types::{AttestationData, AttestationDataAndCustodyBit, FreeAttestation, Signature, Slot}; pub use self::traits::{ @@ -141,7 +141,7 @@ impl Attester BlockProducer bool { if bitfield.num_bytes() != ((committee_size + 7) / 8) { return false; @@ -18,3 +18,62 @@ pub fn verify_bitfield_length(bitfield: &Bitfield, committee_size: usize) -> boo true } + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn bitfield_length() { + assert_eq!( + verify_bitfield_length(&Bitfield::from_bytes(&[0b0000_0001]), 4), + true + ); + + assert_eq!( + verify_bitfield_length(&Bitfield::from_bytes(&[0b0001_0001]), 4), + false + ); + + assert_eq!( + verify_bitfield_length(&Bitfield::from_bytes(&[0b0000_0000]), 4), + true + ); + + assert_eq!( + verify_bitfield_length(&Bitfield::from_bytes(&[0b1000_0000]), 8), + true + ); + + assert_eq!( + verify_bitfield_length(&Bitfield::from_bytes(&[0b1000_0000, 0b0000_0000]), 16), + true + ); + + assert_eq!( + verify_bitfield_length(&Bitfield::from_bytes(&[0b1000_0000, 0b0000_0000]), 15), + false + ); + + assert_eq!( + verify_bitfield_length(&Bitfield::from_bytes(&[0b0000_0000, 0b0000_0000]), 8), + false + ); + + assert_eq!( + verify_bitfield_length( + &Bitfield::from_bytes(&[0b0000_0000, 0b0000_0000, 0b0000_0000]), + 8 + ), + false + ); + + assert_eq!( + verify_bitfield_length( + &Bitfield::from_bytes(&[0b0000_0000, 0b0000_0000, 0b0000_0000]), + 24 + ), + true + ); + } +} diff --git a/eth2/state_processing/src/get_genesis_state.rs b/eth2/state_processing/src/get_genesis_state.rs index 7c4d4cafd..4e9fb6caf 100644 --- a/eth2/state_processing/src/get_genesis_state.rs +++ b/eth2/state_processing/src/get_genesis_state.rs @@ -1,5 +1,5 @@ use super::per_block_processing::{errors::BlockProcessingError, process_deposits}; -use ssz::TreeHash; +use tree_hash::TreeHash; use types::*; pub enum GenesisError { @@ -9,13 +9,13 @@ pub enum GenesisError { /// Returns the genesis `BeaconState` /// -/// Spec v0.5.0 +/// Spec v0.5.1 pub fn get_genesis_state( genesis_validator_deposits: &[Deposit], genesis_time: u64, genesis_eth1_data: Eth1Data, spec: &ChainSpec, -) -> Result<(), BlockProcessingError> { +) -> Result { // Get the genesis `BeaconState` let mut state = BeaconState::genesis(genesis_time, genesis_eth1_data, spec); @@ -36,13 +36,13 @@ pub fn get_genesis_state( let active_validator_indices = state .get_cached_active_validator_indices(RelativeEpoch::Current, spec)? .to_vec(); - let genesis_active_index_root = Hash256::from_slice(&active_validator_indices.hash_tree_root()); + let genesis_active_index_root = Hash256::from_slice(&active_validator_indices.tree_hash_root()); state.fill_active_index_roots_with(genesis_active_index_root, spec); // Generate the current shuffling seed. state.current_shuffling_seed = state.generate_seed(spec.genesis_epoch, spec)?; - Ok(()) + Ok(state) } impl From for GenesisError { diff --git a/eth2/state_processing/src/per_block_processing.rs b/eth2/state_processing/src/per_block_processing.rs index 6c52a2676..58b948f62 100644 --- a/eth2/state_processing/src/per_block_processing.rs +++ b/eth2/state_processing/src/per_block_processing.rs @@ -1,7 +1,7 @@ use crate::common::slash_validator; use errors::{BlockInvalid as Invalid, BlockProcessingError as Error, IntoWithIndex}; use rayon::prelude::*; -use ssz::{SignedRoot, TreeHash}; +use tree_hash::{SignedRoot, TreeHash}; use types::*; pub use self::verify_attester_slashing::{ @@ -39,7 +39,7 @@ const VERIFY_DEPOSIT_MERKLE_PROOFS: bool = false; /// Returns `Ok(())` if the block is valid and the state was successfully updated. Otherwise /// returns an error describing why the block was invalid or how the function failed to execute. /// -/// Spec v0.5.0 +/// Spec v0.5.1 pub fn per_block_processing( state: &mut BeaconState, block: &BeaconBlock, @@ -54,7 +54,7 @@ pub fn per_block_processing( /// Returns `Ok(())` if the block is valid and the state was successfully updated. Otherwise /// returns an error describing why the block was invalid or how the function failed to execute. /// -/// Spec v0.5.0 +/// Spec v0.5.1 pub fn per_block_processing_without_verifying_block_signature( state: &mut BeaconState, block: &BeaconBlock, @@ -69,7 +69,7 @@ pub fn per_block_processing_without_verifying_block_signature( /// Returns `Ok(())` if the block is valid and the state was successfully updated. Otherwise /// returns an error describing why the block was invalid or how the function failed to execute. /// -/// Spec v0.5.0 +/// Spec v0.5.1 fn per_block_processing_signature_optional( mut state: &mut BeaconState, block: &BeaconBlock, @@ -99,7 +99,7 @@ fn per_block_processing_signature_optional( /// Processes the block header. /// -/// Spec v0.5.0 +/// Spec v0.5.1 pub fn process_block_header( state: &mut BeaconState, block: &BeaconBlock, @@ -107,12 +107,14 @@ pub fn process_block_header( ) -> Result<(), Error> { verify!(block.slot == state.slot, Invalid::StateSlotMismatch); - // NOTE: this is not to spec. I think spec is broken. See: - // - // https://github.com/ethereum/eth2.0-specs/issues/797 + let expected_previous_block_root = + Hash256::from_slice(&state.latest_block_header.signed_root()); verify!( - block.previous_block_root == *state.get_block_root(state.slot - 1, spec)?, - Invalid::ParentBlockRootMismatch + block.previous_block_root == expected_previous_block_root, + Invalid::ParentBlockRootMismatch { + state: expected_previous_block_root, + block: block.previous_block_root, + } ); state.latest_block_header = block.temporary_block_header(spec); @@ -122,7 +124,7 @@ pub fn process_block_header( /// Verifies the signature of a block. /// -/// Spec v0.5.0 +/// Spec v0.5.1 pub fn verify_block_signature( state: &BeaconState, block: &BeaconBlock, @@ -150,7 +152,7 @@ pub fn verify_block_signature( /// Verifies the `randao_reveal` against the block's proposer pubkey and updates /// `state.latest_randao_mixes`. /// -/// Spec v0.5.0 +/// Spec v0.5.1 pub fn process_randao( state: &mut BeaconState, block: &BeaconBlock, @@ -162,7 +164,7 @@ pub fn process_randao( // Verify the RANDAO is a valid signature of the proposer. verify!( block.body.randao_reveal.verify( - &state.current_epoch(spec).hash_tree_root()[..], + &state.current_epoch(spec).tree_hash_root()[..], spec.get_domain( block.slot.epoch(spec.slots_per_epoch), Domain::Randao, @@ -181,7 +183,7 @@ pub fn process_randao( /// Update the `state.eth1_data_votes` based upon the `eth1_data` provided. /// -/// Spec v0.5.0 +/// Spec v0.5.1 pub fn process_eth1_data(state: &mut BeaconState, eth1_data: &Eth1Data) -> Result<(), Error> { // Attempt to find a `Eth1DataVote` with matching `Eth1Data`. let matching_eth1_vote_index = state @@ -207,7 +209,7 @@ pub fn process_eth1_data(state: &mut BeaconState, eth1_data: &Eth1Data) -> Resul /// Returns `Ok(())` if the validation and state updates completed successfully, otherwise returns /// an `Err` describing the invalid object or cause of failure. /// -/// Spec v0.5.0 +/// Spec v0.5.1 pub fn process_proposer_slashings( state: &mut BeaconState, proposer_slashings: &[ProposerSlashing], @@ -240,7 +242,7 @@ pub fn process_proposer_slashings( /// Returns `Ok(())` if the validation and state updates completed successfully, otherwise returns /// an `Err` describing the invalid object or cause of failure. /// -/// Spec v0.5.0 +/// Spec v0.5.1 pub fn process_attester_slashings( state: &mut BeaconState, attester_slashings: &[AttesterSlashing], @@ -298,7 +300,7 @@ pub fn process_attester_slashings( /// Returns `Ok(())` if the validation and state updates completed successfully, otherwise returns /// an `Err` describing the invalid object or cause of failure. /// -/// Spec v0.5.0 +/// Spec v0.5.1 pub fn process_attestations( state: &mut BeaconState, attestations: &[Attestation], @@ -340,7 +342,7 @@ pub fn process_attestations( /// Returns `Ok(())` if the validation and state updates completed successfully, otherwise returns /// an `Err` describing the invalid object or cause of failure. /// -/// Spec v0.5.0 +/// Spec v0.5.1 pub fn process_deposits( state: &mut BeaconState, deposits: &[Deposit], @@ -410,7 +412,7 @@ pub fn process_deposits( /// Returns `Ok(())` if the validation and state updates completed successfully, otherwise returns /// an `Err` describing the invalid object or cause of failure. /// -/// Spec v0.5.0 +/// Spec v0.5.1 pub fn process_exits( state: &mut BeaconState, voluntary_exits: &[VoluntaryExit], @@ -442,7 +444,7 @@ pub fn process_exits( /// Returns `Ok(())` if the validation and state updates completed successfully, otherwise returns /// an `Err` describing the invalid object or cause of failure. /// -/// Spec v0.5.0 +/// Spec v0.5.1 pub fn process_transfers( state: &mut BeaconState, transfers: &[Transfer], diff --git a/eth2/state_processing/src/per_block_processing/errors.rs b/eth2/state_processing/src/per_block_processing/errors.rs index 6614f6f60..d8627d359 100644 --- a/eth2/state_processing/src/per_block_processing/errors.rs +++ b/eth2/state_processing/src/per_block_processing/errors.rs @@ -67,7 +67,10 @@ impl_from_beacon_state_error!(BlockProcessingError); #[derive(Debug, PartialEq)] pub enum BlockInvalid { StateSlotMismatch, - ParentBlockRootMismatch, + ParentBlockRootMismatch { + state: Hash256, + block: Hash256, + }, BadSignature, BadRandaoSignature, MaxAttestationsExceeded, @@ -271,10 +274,10 @@ pub enum ProposerSlashingValidationError { pub enum ProposerSlashingInvalid { /// The proposer index is not a known validator. ProposerUnknown(u64), - /// The two proposal have different slots. + /// The two proposal have different epochs. /// /// (proposal_1_slot, proposal_2_slot) - ProposalSlotMismatch(Slot, Slot), + ProposalEpochMismatch(Slot, Slot), /// The proposals are identical and therefore not slashable. ProposalsIdentical, /// The specified proposer has already been slashed. diff --git a/eth2/state_processing/src/per_block_processing/validate_attestation.rs b/eth2/state_processing/src/per_block_processing/validate_attestation.rs index 3b89bec99..438a75c94 100644 --- a/eth2/state_processing/src/per_block_processing/validate_attestation.rs +++ b/eth2/state_processing/src/per_block_processing/validate_attestation.rs @@ -1,6 +1,6 @@ use super::errors::{AttestationInvalid as Invalid, AttestationValidationError as Error}; use crate::common::verify_bitfield_length; -use ssz::TreeHash; +use tree_hash::TreeHash; use types::*; /// Indicates if an `Attestation` is valid to be included in a block in the current epoch of the @@ -8,7 +8,7 @@ use types::*; /// /// Returns `Ok(())` if the `Attestation` is valid, otherwise indicates the reason for invalidity. /// -/// Spec v0.5.0 +/// Spec v0.5.1 pub fn validate_attestation( state: &BeaconState, attestation: &Attestation, @@ -31,7 +31,7 @@ pub fn validate_attestation_time_independent_only( /// /// Returns `Ok(())` if the `Attestation` is valid, otherwise indicates the reason for invalidity. /// -/// Spec v0.5.0 +/// Spec v0.5.1 pub fn validate_attestation_without_signature( state: &BeaconState, attestation: &Attestation, @@ -44,7 +44,7 @@ pub fn validate_attestation_without_signature( /// given state, optionally validating the aggregate signature. /// /// -/// Spec v0.5.0 +/// Spec v0.5.1 fn validate_attestation_parametric( state: &BeaconState, attestation: &Attestation, @@ -167,7 +167,7 @@ fn validate_attestation_parametric( /// Verify that the `source_epoch` and `source_root` of an `Attestation` correctly /// match the current (or previous) justified epoch and root from the state. /// -/// Spec v0.5.0 +/// Spec v0.5.1 fn verify_justified_epoch_and_root( attestation: &Attestation, state: &BeaconState, @@ -222,7 +222,7 @@ fn verify_justified_epoch_and_root( /// - `custody_bitfield` does not have a bit for each index of `committee`. /// - A `validator_index` in `committee` is not in `state.validator_registry`. /// -/// Spec v0.5.0 +/// Spec v0.5.1 fn verify_attestation_signature( state: &BeaconState, committee: &[usize], @@ -270,14 +270,14 @@ fn verify_attestation_signature( data: a.data.clone(), custody_bit: false, } - .hash_tree_root(); + .tree_hash_root(); // Message when custody bitfield is `true` let message_1 = AttestationDataAndCustodyBit { data: a.data.clone(), custody_bit: true, } - .hash_tree_root(); + .tree_hash_root(); let mut messages = vec![]; let mut keys = vec![]; diff --git a/eth2/state_processing/src/per_block_processing/verify_attester_slashing.rs b/eth2/state_processing/src/per_block_processing/verify_attester_slashing.rs index abf99da64..3527b62e3 100644 --- a/eth2/state_processing/src/per_block_processing/verify_attester_slashing.rs +++ b/eth2/state_processing/src/per_block_processing/verify_attester_slashing.rs @@ -7,7 +7,7 @@ use types::*; /// /// Returns `Ok(())` if the `AttesterSlashing` is valid, otherwise indicates the reason for invalidity. /// -/// Spec v0.5.0 +/// Spec v0.5.1 pub fn verify_attester_slashing( state: &BeaconState, attester_slashing: &AttesterSlashing, @@ -41,7 +41,7 @@ pub fn verify_attester_slashing( /// /// Returns Ok(indices) if `indices.len() > 0`. /// -/// Spec v0.5.0 +/// Spec v0.5.1 pub fn gather_attester_slashing_indices( state: &BeaconState, attester_slashing: &AttesterSlashing, diff --git a/eth2/state_processing/src/per_block_processing/verify_deposit.rs b/eth2/state_processing/src/per_block_processing/verify_deposit.rs index a3a0f5734..22a62a321 100644 --- a/eth2/state_processing/src/per_block_processing/verify_deposit.rs +++ b/eth2/state_processing/src/per_block_processing/verify_deposit.rs @@ -15,7 +15,7 @@ use types::*; /// /// Note: this function is incomplete. /// -/// Spec v0.5.0 +/// Spec v0.5.1 pub fn verify_deposit( state: &BeaconState, deposit: &Deposit, @@ -46,7 +46,7 @@ pub fn verify_deposit( /// Verify that the `Deposit` index is correct. /// -/// Spec v0.5.0 +/// Spec v0.5.1 pub fn verify_deposit_index(state: &BeaconState, deposit: &Deposit) -> Result<(), Error> { verify!( deposit.index == state.deposit_index, @@ -88,7 +88,7 @@ pub fn get_existing_validator_index( /// Verify that a deposit is included in the state's eth1 deposit root. /// -/// Spec v0.5.0 +/// Spec v0.5.1 fn verify_deposit_merkle_proof(state: &BeaconState, deposit: &Deposit, spec: &ChainSpec) -> bool { let leaf = hash(&get_serialized_deposit_data(deposit)); verify_merkle_proof( @@ -102,7 +102,7 @@ fn verify_deposit_merkle_proof(state: &BeaconState, deposit: &Deposit, spec: &Ch /// Helper struct for easily getting the serialized data generated by the deposit contract. /// -/// Spec v0.5.0 +/// Spec v0.5.1 #[derive(Encode)] struct SerializedDepositData { amount: u64, @@ -113,7 +113,7 @@ struct SerializedDepositData { /// Return the serialized data generated by the deposit contract that is used to generate the /// merkle proof. /// -/// Spec v0.5.0 +/// Spec v0.5.1 fn get_serialized_deposit_data(deposit: &Deposit) -> Vec { let serialized_deposit_data = SerializedDepositData { amount: deposit.deposit_data.amount, diff --git a/eth2/state_processing/src/per_block_processing/verify_exit.rs b/eth2/state_processing/src/per_block_processing/verify_exit.rs index a3b694395..697188ee9 100644 --- a/eth2/state_processing/src/per_block_processing/verify_exit.rs +++ b/eth2/state_processing/src/per_block_processing/verify_exit.rs @@ -1,5 +1,5 @@ use super::errors::{ExitInvalid as Invalid, ExitValidationError as Error}; -use ssz::SignedRoot; +use tree_hash::SignedRoot; use types::*; /// Indicates if an `Exit` is valid to be included in a block in the current epoch of the given @@ -7,7 +7,7 @@ use types::*; /// /// Returns `Ok(())` if the `Exit` is valid, otherwise indicates the reason for invalidity. /// -/// Spec v0.5.0 +/// Spec v0.5.1 pub fn verify_exit( state: &BeaconState, exit: &VoluntaryExit, diff --git a/eth2/state_processing/src/per_block_processing/verify_proposer_slashing.rs b/eth2/state_processing/src/per_block_processing/verify_proposer_slashing.rs index dffb9d898..bbc03dd62 100644 --- a/eth2/state_processing/src/per_block_processing/verify_proposer_slashing.rs +++ b/eth2/state_processing/src/per_block_processing/verify_proposer_slashing.rs @@ -1,5 +1,5 @@ use super::errors::{ProposerSlashingInvalid as Invalid, ProposerSlashingValidationError as Error}; -use ssz::SignedRoot; +use tree_hash::SignedRoot; use types::*; /// Indicates if a `ProposerSlashing` is valid to be included in a block in the current epoch of the given @@ -7,7 +7,7 @@ use types::*; /// /// Returns `Ok(())` if the `ProposerSlashing` is valid, otherwise indicates the reason for invalidity. /// -/// Spec v0.5.0 +/// Spec v0.5.1 pub fn verify_proposer_slashing( proposer_slashing: &ProposerSlashing, state: &BeaconState, @@ -21,8 +21,9 @@ pub fn verify_proposer_slashing( })?; verify!( - proposer_slashing.header_1.slot == proposer_slashing.header_2.slot, - Invalid::ProposalSlotMismatch( + proposer_slashing.header_1.slot.epoch(spec.slots_per_epoch) + == proposer_slashing.header_2.slot.epoch(spec.slots_per_epoch), + Invalid::ProposalEpochMismatch( proposer_slashing.header_1.slot, proposer_slashing.header_2.slot ) @@ -66,7 +67,7 @@ pub fn verify_proposer_slashing( /// /// Returns `true` if the signature is valid. /// -/// Spec v0.5.0 +/// Spec v0.5.1 fn verify_header_signature( header: &BeaconBlockHeader, pubkey: &PublicKey, diff --git a/eth2/state_processing/src/per_block_processing/verify_slashable_attestation.rs b/eth2/state_processing/src/per_block_processing/verify_slashable_attestation.rs index d3ab5e398..89cb93ce5 100644 --- a/eth2/state_processing/src/per_block_processing/verify_slashable_attestation.rs +++ b/eth2/state_processing/src/per_block_processing/verify_slashable_attestation.rs @@ -2,7 +2,7 @@ use super::errors::{ SlashableAttestationInvalid as Invalid, SlashableAttestationValidationError as Error, }; use crate::common::verify_bitfield_length; -use ssz::TreeHash; +use tree_hash::TreeHash; use types::*; /// Indicates if a `SlashableAttestation` is valid to be included in a block in the current epoch of the given @@ -10,7 +10,7 @@ use types::*; /// /// Returns `Ok(())` if the `SlashableAttestation` is valid, otherwise indicates the reason for invalidity. /// -/// Spec v0.5.0 +/// Spec v0.5.1 pub fn verify_slashable_attestation( state: &BeaconState, slashable_attestation: &SlashableAttestation, @@ -77,12 +77,12 @@ pub fn verify_slashable_attestation( data: slashable_attestation.data.clone(), custody_bit: false, } - .hash_tree_root(); + .tree_hash_root(); let message_1 = AttestationDataAndCustodyBit { data: slashable_attestation.data.clone(), custody_bit: true, } - .hash_tree_root(); + .tree_hash_root(); let mut messages = vec![]; let mut keys = vec![]; diff --git a/eth2/state_processing/src/per_block_processing/verify_transfer.rs b/eth2/state_processing/src/per_block_processing/verify_transfer.rs index ac9e9aa09..8b0415508 100644 --- a/eth2/state_processing/src/per_block_processing/verify_transfer.rs +++ b/eth2/state_processing/src/per_block_processing/verify_transfer.rs @@ -1,6 +1,6 @@ use super::errors::{TransferInvalid as Invalid, TransferValidationError as Error}; use bls::get_withdrawal_credentials; -use ssz::SignedRoot; +use tree_hash::SignedRoot; use types::*; /// Indicates if a `Transfer` is valid to be included in a block in the current epoch of the given @@ -10,7 +10,7 @@ use types::*; /// /// Note: this function is incomplete. /// -/// Spec v0.5.0 +/// Spec v0.5.1 pub fn verify_transfer( state: &BeaconState, transfer: &Transfer, @@ -122,7 +122,7 @@ fn verify_transfer_parametric( /// /// Does not check that the transfer is valid, however checks for overflow in all actions. /// -/// Spec v0.5.0 +/// Spec v0.5.1 pub fn execute_transfer( state: &mut BeaconState, transfer: &Transfer, diff --git a/eth2/state_processing/src/per_epoch_processing.rs b/eth2/state_processing/src/per_epoch_processing.rs index fcdc668f4..87c9b9398 100644 --- a/eth2/state_processing/src/per_epoch_processing.rs +++ b/eth2/state_processing/src/per_epoch_processing.rs @@ -3,8 +3,8 @@ use errors::EpochProcessingError as Error; use process_ejections::process_ejections; use process_exit_queue::process_exit_queue; use process_slashings::process_slashings; -use ssz::TreeHash; use std::collections::HashMap; +use tree_hash::TreeHash; use types::*; use update_registry_and_shuffling_data::update_registry_and_shuffling_data; use validator_statuses::{TotalBalances, ValidatorStatuses}; @@ -32,7 +32,7 @@ pub type WinningRootHashSet = HashMap; /// Mutates the given `BeaconState`, returning early if an error is encountered. If an error is /// returned, a state might be "half-processed" and therefore in an invalid state. /// -/// Spec v0.5.0 +/// Spec v0.5.1 pub fn per_epoch_processing(state: &mut BeaconState, spec: &ChainSpec) -> Result<(), Error> { // Ensure the previous and next epoch caches are built. state.build_epoch_cache(RelativeEpoch::Previous, spec)?; @@ -86,7 +86,7 @@ pub fn per_epoch_processing(state: &mut BeaconState, spec: &ChainSpec) -> Result /// Maybe resets the eth1 period. /// -/// Spec v0.5.0 +/// Spec v0.5.1 pub fn maybe_reset_eth1_period(state: &mut BeaconState, spec: &ChainSpec) { let next_epoch = state.next_epoch(spec); let voting_period = spec.epochs_per_eth1_voting_period; @@ -108,7 +108,7 @@ pub fn maybe_reset_eth1_period(state: &mut BeaconState, spec: &ChainSpec) { /// - `justified_epoch` /// - `previous_justified_epoch` /// -/// Spec v0.5.0 +/// Spec v0.5.1 pub fn update_justification_and_finalization( state: &mut BeaconState, total_balances: &TotalBalances, @@ -178,7 +178,7 @@ pub fn update_justification_and_finalization( /// /// Also returns a `WinningRootHashSet` for later use during epoch processing. /// -/// Spec v0.5.0 +/// Spec v0.5.1 pub fn process_crosslinks( state: &mut BeaconState, spec: &ChainSpec, @@ -221,7 +221,7 @@ pub fn process_crosslinks( /// Finish up an epoch update. /// -/// Spec v0.5.0 +/// Spec v0.5.1 pub fn finish_epoch_update(state: &mut BeaconState, spec: &ChainSpec) -> Result<(), Error> { let current_epoch = state.current_epoch(spec); let next_epoch = state.next_epoch(spec); @@ -236,7 +236,7 @@ pub fn finish_epoch_update(state: &mut BeaconState, spec: &ChainSpec) -> Result< let active_index_root = Hash256::from_slice( &state .get_active_validator_indices(next_epoch + spec.activation_exit_delay) - .hash_tree_root()[..], + .tree_hash_root()[..], ); state.set_active_index_root(next_epoch, active_index_root, spec)?; @@ -261,7 +261,7 @@ pub fn finish_epoch_update(state: &mut BeaconState, spec: &ChainSpec) -> Result< let historical_batch: HistoricalBatch = state.historical_batch(); state .historical_roots - .push(Hash256::from_slice(&historical_batch.hash_tree_root()[..])); + .push(Hash256::from_slice(&historical_batch.tree_hash_root()[..])); } state.previous_epoch_attestations = state.current_epoch_attestations.clone(); diff --git a/eth2/state_processing/src/per_epoch_processing/apply_rewards.rs b/eth2/state_processing/src/per_epoch_processing/apply_rewards.rs index ce5fccb21..9af1ee8c3 100644 --- a/eth2/state_processing/src/per_epoch_processing/apply_rewards.rs +++ b/eth2/state_processing/src/per_epoch_processing/apply_rewards.rs @@ -32,7 +32,7 @@ impl std::ops::AddAssign for Delta { /// Apply attester and proposer rewards. /// -/// Spec v0.5.0 +/// Spec v0.5.1 pub fn apply_rewards( state: &mut BeaconState, validator_statuses: &mut ValidatorStatuses, @@ -79,7 +79,7 @@ pub fn apply_rewards( /// Applies the attestation inclusion reward to each proposer for every validator who included an /// attestation in the previous epoch. /// -/// Spec v0.5.0 +/// Spec v0.5.1 fn get_proposer_deltas( deltas: &mut Vec, state: &mut BeaconState, @@ -120,7 +120,7 @@ fn get_proposer_deltas( /// Apply rewards for participation in attestations during the previous epoch. /// -/// Spec v0.5.0 +/// Spec v0.5.1 fn get_justification_and_finalization_deltas( deltas: &mut Vec, state: &BeaconState, @@ -163,7 +163,7 @@ fn get_justification_and_finalization_deltas( /// Determine the delta for a single validator, if the chain is finalizing normally. /// -/// Spec v0.5.0 +/// Spec v0.5.1 fn compute_normal_justification_and_finalization_delta( validator: &ValidatorStatus, total_balances: &TotalBalances, @@ -215,7 +215,7 @@ fn compute_normal_justification_and_finalization_delta( /// Determine the delta for a single delta, assuming the chain is _not_ finalizing normally. /// -/// Spec v0.5.0 +/// Spec v0.5.1 fn compute_inactivity_leak_delta( validator: &ValidatorStatus, base_reward: u64, @@ -261,7 +261,7 @@ fn compute_inactivity_leak_delta( /// Calculate the deltas based upon the winning roots for attestations during the previous epoch. /// -/// Spec v0.5.0 +/// Spec v0.5.1 fn get_crosslink_deltas( deltas: &mut Vec, state: &BeaconState, @@ -295,7 +295,7 @@ fn get_crosslink_deltas( /// Returns the base reward for some validator. /// -/// Spec v0.5.0 +/// Spec v0.5.1 fn get_base_reward( state: &BeaconState, index: usize, @@ -312,7 +312,7 @@ fn get_base_reward( /// Returns the inactivity penalty for some validator. /// -/// Spec v0.5.0 +/// Spec v0.5.1 fn get_inactivity_penalty( state: &BeaconState, index: usize, @@ -328,7 +328,7 @@ fn get_inactivity_penalty( /// Returns the epochs since the last finalized epoch. /// -/// Spec v0.5.0 +/// Spec v0.5.1 fn epochs_since_finality(state: &BeaconState, spec: &ChainSpec) -> Epoch { state.current_epoch(spec) + 1 - state.finalized_epoch } diff --git a/eth2/state_processing/src/per_epoch_processing/get_attestation_participants.rs b/eth2/state_processing/src/per_epoch_processing/get_attestation_participants.rs index 52ba0274b..bea772204 100644 --- a/eth2/state_processing/src/per_epoch_processing/get_attestation_participants.rs +++ b/eth2/state_processing/src/per_epoch_processing/get_attestation_participants.rs @@ -3,7 +3,7 @@ use types::*; /// Returns validator indices which participated in the attestation. /// -/// Spec v0.5.0 +/// Spec v0.5.1 pub fn get_attestation_participants( state: &BeaconState, attestation_data: &AttestationData, diff --git a/eth2/state_processing/src/per_epoch_processing/inclusion_distance.rs b/eth2/state_processing/src/per_epoch_processing/inclusion_distance.rs index b52485947..6b221f513 100644 --- a/eth2/state_processing/src/per_epoch_processing/inclusion_distance.rs +++ b/eth2/state_processing/src/per_epoch_processing/inclusion_distance.rs @@ -5,7 +5,7 @@ use types::*; /// Returns the distance between the first included attestation for some validator and this /// slot. /// -/// Spec v0.5.0 +/// Spec v0.5.1 pub fn inclusion_distance( state: &BeaconState, attestations: &[&PendingAttestation], @@ -18,7 +18,7 @@ pub fn inclusion_distance( /// Returns the slot of the earliest included attestation for some validator. /// -/// Spec v0.5.0 +/// Spec v0.5.1 pub fn inclusion_slot( state: &BeaconState, attestations: &[&PendingAttestation], @@ -31,7 +31,7 @@ pub fn inclusion_slot( /// Finds the earliest included attestation for some validator. /// -/// Spec v0.5.0 +/// Spec v0.5.1 fn earliest_included_attestation( state: &BeaconState, attestations: &[&PendingAttestation], diff --git a/eth2/state_processing/src/per_epoch_processing/process_ejections.rs b/eth2/state_processing/src/per_epoch_processing/process_ejections.rs index a60d92187..6f64c46f7 100644 --- a/eth2/state_processing/src/per_epoch_processing/process_ejections.rs +++ b/eth2/state_processing/src/per_epoch_processing/process_ejections.rs @@ -4,7 +4,7 @@ use types::{BeaconStateError as Error, *}; /// Iterate through the validator registry and eject active validators with balance below /// ``EJECTION_BALANCE``. /// -/// Spec v0.5.0 +/// Spec v0.5.1 pub fn process_ejections(state: &mut BeaconState, spec: &ChainSpec) -> Result<(), Error> { // There is an awkward double (triple?) loop here because we can't loop across the borrowed // active validator indices and mutate state in the one loop. diff --git a/eth2/state_processing/src/per_epoch_processing/process_exit_queue.rs b/eth2/state_processing/src/per_epoch_processing/process_exit_queue.rs index 074db1d08..a6362188d 100644 --- a/eth2/state_processing/src/per_epoch_processing/process_exit_queue.rs +++ b/eth2/state_processing/src/per_epoch_processing/process_exit_queue.rs @@ -2,7 +2,7 @@ use types::*; /// Process the exit queue. /// -/// Spec v0.5.0 +/// Spec v0.5.1 pub fn process_exit_queue(state: &mut BeaconState, spec: &ChainSpec) { let current_epoch = state.current_epoch(spec); @@ -31,7 +31,7 @@ pub fn process_exit_queue(state: &mut BeaconState, spec: &ChainSpec) { /// Initiate an exit for the validator of the given `index`. /// -/// Spec v0.5.0 +/// Spec v0.5.1 fn prepare_validator_for_withdrawal( state: &mut BeaconState, validator_index: usize, diff --git a/eth2/state_processing/src/per_epoch_processing/process_slashings.rs b/eth2/state_processing/src/per_epoch_processing/process_slashings.rs index 88777472c..89a7dd484 100644 --- a/eth2/state_processing/src/per_epoch_processing/process_slashings.rs +++ b/eth2/state_processing/src/per_epoch_processing/process_slashings.rs @@ -2,7 +2,7 @@ use types::{BeaconStateError as Error, *}; /// Process slashings. /// -/// Spec v0.5.0 +/// Spec v0.5.1 pub fn process_slashings( state: &mut BeaconState, current_total_balance: u64, diff --git a/eth2/state_processing/src/per_epoch_processing/update_registry_and_shuffling_data.rs b/eth2/state_processing/src/per_epoch_processing/update_registry_and_shuffling_data.rs index 0b18c2571..d290d2987 100644 --- a/eth2/state_processing/src/per_epoch_processing/update_registry_and_shuffling_data.rs +++ b/eth2/state_processing/src/per_epoch_processing/update_registry_and_shuffling_data.rs @@ -4,7 +4,7 @@ use types::*; /// Peforms a validator registry update, if required. /// -/// Spec v0.5.0 +/// Spec v0.5.1 pub fn update_registry_and_shuffling_data( state: &mut BeaconState, current_total_balance: u64, @@ -49,7 +49,7 @@ pub fn update_registry_and_shuffling_data( /// Returns `true` if the validator registry should be updated during an epoch processing. /// -/// Spec v0.5.0 +/// Spec v0.5.1 pub fn should_update_validator_registry( state: &BeaconState, spec: &ChainSpec, @@ -78,7 +78,7 @@ pub fn should_update_validator_registry( /// /// Note: Utilizes the cache and will fail if the appropriate cache is not initialized. /// -/// Spec v0.5.0 +/// Spec v0.5.1 pub fn update_validator_registry( state: &mut BeaconState, current_total_balance: u64, @@ -133,7 +133,7 @@ pub fn update_validator_registry( /// Activate the validator of the given ``index``. /// -/// Spec v0.5.0 +/// Spec v0.5.1 pub fn activate_validator( state: &mut BeaconState, validator_index: usize, diff --git a/eth2/state_processing/src/per_epoch_processing/validator_statuses.rs b/eth2/state_processing/src/per_epoch_processing/validator_statuses.rs index 02149cc5a..afa78c9c0 100644 --- a/eth2/state_processing/src/per_epoch_processing/validator_statuses.rs +++ b/eth2/state_processing/src/per_epoch_processing/validator_statuses.rs @@ -160,7 +160,7 @@ impl ValidatorStatuses { /// - Active validators /// - Total balances for the current and previous epochs. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn new(state: &BeaconState, spec: &ChainSpec) -> Result { let mut statuses = Vec::with_capacity(state.validator_registry.len()); let mut total_balances = TotalBalances::default(); @@ -195,7 +195,7 @@ impl ValidatorStatuses { /// Process some attestations from the given `state` updating the `statuses` and /// `total_balances` fields. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn process_attestations( &mut self, state: &BeaconState, @@ -261,7 +261,7 @@ impl ValidatorStatuses { /// Update the `statuses` for each validator based upon whether or not they attested to the /// "winning" shard block root for the previous epoch. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn process_winning_roots( &mut self, state: &BeaconState, @@ -297,14 +297,14 @@ impl ValidatorStatuses { /// Returns the distance between when the attestation was created and when it was included in a /// block. /// -/// Spec v0.5.0 +/// Spec v0.5.1 fn inclusion_distance(a: &PendingAttestation) -> Slot { a.inclusion_slot - a.data.slot } /// Returns `true` if some `PendingAttestation` is from the supplied `epoch`. /// -/// Spec v0.5.0 +/// Spec v0.5.1 fn is_from_epoch(a: &PendingAttestation, epoch: Epoch, spec: &ChainSpec) -> bool { a.data.slot.epoch(spec.slots_per_epoch) == epoch } @@ -312,7 +312,7 @@ fn is_from_epoch(a: &PendingAttestation, epoch: Epoch, spec: &ChainSpec) -> bool /// Returns `true` if a `PendingAttestation` and `BeaconState` share the same beacon block hash for /// the first slot of the given epoch. /// -/// Spec v0.5.0 +/// Spec v0.5.1 fn has_common_epoch_boundary_root( a: &PendingAttestation, state: &BeaconState, @@ -328,7 +328,7 @@ fn has_common_epoch_boundary_root( /// Returns `true` if a `PendingAttestation` and `BeaconState` share the same beacon block hash for /// the current slot of the `PendingAttestation`. /// -/// Spec v0.5.0 +/// Spec v0.5.1 fn has_common_beacon_block_root( a: &PendingAttestation, state: &BeaconState, diff --git a/eth2/state_processing/src/per_epoch_processing/winning_root.rs b/eth2/state_processing/src/per_epoch_processing/winning_root.rs index 97cff3e13..5d31dff31 100644 --- a/eth2/state_processing/src/per_epoch_processing/winning_root.rs +++ b/eth2/state_processing/src/per_epoch_processing/winning_root.rs @@ -16,7 +16,7 @@ impl WinningRoot { /// A winning root is "better" than another if it has a higher `total_attesting_balance`. Ties /// are broken by favouring the higher `crosslink_data_root` value. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn is_better_than(&self, other: &Self) -> bool { if self.total_attesting_balance > other.total_attesting_balance { true @@ -34,7 +34,7 @@ impl WinningRoot { /// The `WinningRoot` object also contains additional fields that are useful in later stages of /// per-epoch processing. /// -/// Spec v0.5.0 +/// Spec v0.5.1 pub fn winning_root( state: &BeaconState, shard: u64, @@ -89,7 +89,7 @@ pub fn winning_root( /// Returns `true` if pending attestation `a` is eligible to become a winning root. /// -/// Spec v0.5.0 +/// Spec v0.5.1 fn is_eligible_for_winning_root(state: &BeaconState, a: &PendingAttestation, shard: Shard) -> bool { if shard >= state.latest_crosslinks.len() as u64 { return false; @@ -100,7 +100,7 @@ fn is_eligible_for_winning_root(state: &BeaconState, a: &PendingAttestation, sha /// Returns all indices which voted for a given crosslink. Does not contain duplicates. /// -/// Spec v0.5.0 +/// Spec v0.5.1 fn get_attesting_validator_indices( state: &BeaconState, shard: u64, diff --git a/eth2/state_processing/src/per_slot_processing.rs b/eth2/state_processing/src/per_slot_processing.rs index c6b5312c7..8f9606723 100644 --- a/eth2/state_processing/src/per_slot_processing.rs +++ b/eth2/state_processing/src/per_slot_processing.rs @@ -1,5 +1,5 @@ use crate::*; -use ssz::TreeHash; +use tree_hash::SignedRoot; use types::*; #[derive(Debug, PartialEq)] @@ -10,13 +10,9 @@ pub enum Error { /// Advances a state forward by one slot, performing per-epoch processing if required. /// -/// Spec v0.5.0 -pub fn per_slot_processing( - state: &mut BeaconState, - latest_block_header: &BeaconBlockHeader, - spec: &ChainSpec, -) -> Result<(), Error> { - cache_state(state, latest_block_header, spec)?; +/// Spec v0.5.1 +pub fn per_slot_processing(state: &mut BeaconState, spec: &ChainSpec) -> Result<(), Error> { + cache_state(state, spec)?; if (state.slot + 1) % spec.slots_per_epoch == 0 { per_epoch_processing(state, spec)?; @@ -27,12 +23,8 @@ pub fn per_slot_processing( Ok(()) } -fn cache_state( - state: &mut BeaconState, - latest_block_header: &BeaconBlockHeader, - spec: &ChainSpec, -) -> Result<(), Error> { - let previous_slot_state_root = Hash256::from_slice(&state.hash_tree_root()[..]); +fn cache_state(state: &mut BeaconState, spec: &ChainSpec) -> Result<(), Error> { + let previous_slot_state_root = state.update_tree_hash_cache()?; // Note: increment the state slot here to allow use of our `state_root` and `block_root` // getter/setter functions. @@ -46,7 +38,10 @@ fn cache_state( state.latest_block_header.state_root = previous_slot_state_root } - let latest_block_root = Hash256::from_slice(&latest_block_header.hash_tree_root()[..]); + // Store the previous slot's post state transition root. + state.set_state_root(previous_slot, previous_slot_state_root, spec)?; + + let latest_block_root = Hash256::from_slice(&state.latest_block_header.signed_root()[..]); state.set_block_root(previous_slot, latest_block_root, spec)?; // Set the state slot back to what it should be. diff --git a/eth2/state_processing/tests/tests.rs b/eth2/state_processing/tests/tests.rs index 1359508dc..e80340e9a 100644 --- a/eth2/state_processing/tests/tests.rs +++ b/eth2/state_processing/tests/tests.rs @@ -1,14 +1,59 @@ +#![cfg(not(debug_assertions))] + use serde_derive::Deserialize; use serde_yaml; -#[cfg(not(debug_assertions))] -use state_processing::{ - per_block_processing, per_block_processing_without_verifying_block_signature, - per_slot_processing, -}; +use state_processing::{per_block_processing, per_slot_processing}; use std::{fs::File, io::prelude::*, path::PathBuf}; use types::*; -#[allow(unused_imports)] -use yaml_utils; + +#[derive(Debug, Deserialize)] +pub struct ExpectedState { + pub slot: Option, + pub genesis_time: Option, + pub fork: Option, + pub validator_registry: Option>, + pub validator_balances: Option>, + pub previous_epoch_attestations: Option>, + pub current_epoch_attestations: Option>, + pub historical_roots: Option>, + pub finalized_epoch: Option, + pub latest_block_roots: Option>, +} + +impl ExpectedState { + // Return a list of fields that differ, and a string representation of the beacon state's field. + fn check(&self, state: &BeaconState) -> Vec<(&str, String)> { + // Check field equality + macro_rules! cfe { + ($field_name:ident) => { + if self.$field_name.as_ref().map_or(true, |$field_name| { + println!(" > Checking {}", stringify!($field_name)); + $field_name == &state.$field_name + }) { + vec![] + } else { + vec![(stringify!($field_name), format!("{:#?}", state.$field_name))] + } + }; + } + + vec![ + cfe!(slot), + cfe!(genesis_time), + cfe!(fork), + cfe!(validator_registry), + cfe!(validator_balances), + cfe!(previous_epoch_attestations), + cfe!(current_epoch_attestations), + cfe!(historical_roots), + cfe!(finalized_epoch), + cfe!(latest_block_roots), + ] + .into_iter() + .flat_map(|x| x) + .collect() + } +} #[derive(Debug, Deserialize)] pub struct TestCase { @@ -17,6 +62,7 @@ pub struct TestCase { pub verify_signatures: bool, pub initial_state: BeaconState, pub blocks: Vec, + pub expected_state: ExpectedState, } #[derive(Debug, Deserialize)] @@ -27,82 +73,81 @@ pub struct TestDoc { pub test_cases: Vec, } -#[test] -fn test_read_yaml() { - // Test sanity-check_small-config_32-vals.yaml +fn load_test_case(test_name: &str) -> TestDoc { let mut file = { let mut file_path_buf = PathBuf::from(env!("CARGO_MANIFEST_DIR")); - file_path_buf.push("yaml_utils/specs/sanity-check_small-config_32-vals.yaml"); + file_path_buf.push(format!("yaml_utils/specs/{}", test_name)); File::open(file_path_buf).unwrap() }; - let mut yaml_str = String::new(); - file.read_to_string(&mut yaml_str).unwrap(); - yaml_str = yaml_str.to_lowercase(); - let _doc: TestDoc = serde_yaml::from_str(&yaml_str.as_str()).unwrap(); + serde_yaml::from_str(&yaml_str.as_str()).unwrap() +} - // Test sanity-check_default-config_100-vals.yaml - file = { - let mut file_path_buf = PathBuf::from(env!("CARGO_MANIFEST_DIR")); - file_path_buf.push("yaml_utils/specs/sanity-check_default-config_100-vals.yaml"); +fn run_state_transition_test(test_name: &str) { + let doc = load_test_case(test_name); - File::open(file_path_buf).unwrap() - }; + // Run Tests + let mut ok = true; + for (i, test_case) in doc.test_cases.iter().enumerate() { + let fake_crypto = cfg!(feature = "fake_crypto"); + if !test_case.verify_signatures == fake_crypto { + println!("Running {}", test_case.name); + } else { + println!( + "Skipping {} (fake_crypto: {}, need fake: {})", + test_case.name, fake_crypto, !test_case.verify_signatures + ); + continue; + } + let mut state = test_case.initial_state.clone(); + for (j, block) in test_case.blocks.iter().enumerate() { + while block.slot > state.slot { + per_slot_processing(&mut state, &test_case.config).unwrap(); + } + let res = per_block_processing(&mut state, &block, &test_case.config); + if res.is_err() { + println!("Error in {} (#{}), on block {}", test_case.name, i, j); + println!("{:?}", res); + ok = false; + } + } - yaml_str = String::new(); + let mismatched_fields = test_case.expected_state.check(&state); + if !mismatched_fields.is_empty() { + println!( + "Error in expected state, these fields didn't match: {:?}", + mismatched_fields.iter().map(|(f, _)| f).collect::>() + ); + for (field_name, state_val) in mismatched_fields { + println!("state.{} was: {}", field_name, state_val); + } + ok = false; + } + } - file.read_to_string(&mut yaml_str).unwrap(); + assert!(ok, "one or more tests failed, see above"); +} - yaml_str = yaml_str.to_lowercase(); - - let _doc: TestDoc = serde_yaml::from_str(&yaml_str.as_str()).unwrap(); +#[test] +#[cfg(not(debug_assertions))] +fn test_read_yaml() { + load_test_case("sanity-check_small-config_32-vals.yaml"); + load_test_case("sanity-check_default-config_100-vals.yaml"); } #[test] #[cfg(not(debug_assertions))] fn run_state_transition_tests_small() { - // Test sanity-check_small-config_32-vals.yaml - let mut file = { - let mut file_path_buf = PathBuf::from(env!("CARGO_MANIFEST_DIR")); - file_path_buf.push("yaml_utils/specs/sanity-check_small-config_32-vals.yaml"); - - File::open(file_path_buf).unwrap() - }; - let mut yaml_str = String::new(); - file.read_to_string(&mut yaml_str).unwrap(); - yaml_str = yaml_str.to_lowercase(); - - let doc: TestDoc = serde_yaml::from_str(&yaml_str.as_str()).unwrap(); - - // Run Tests - for (i, test_case) in doc.test_cases.iter().enumerate() { - let mut state = test_case.initial_state.clone(); - for block in test_case.blocks.iter() { - while block.slot > state.slot { - let latest_block_header = state.latest_block_header.clone(); - per_slot_processing(&mut state, &latest_block_header, &test_case.config).unwrap(); - } - if test_case.verify_signatures { - let res = per_block_processing(&mut state, &block, &test_case.config); - if res.is_err() { - println!("{:?}", i); - println!("{:?}", res); - }; - } else { - let res = per_block_processing_without_verifying_block_signature( - &mut state, - &block, - &test_case.config, - ); - if res.is_err() { - println!("{:?}", i); - println!("{:?}", res); - } - } - } - } + run_state_transition_test("sanity-check_small-config_32-vals.yaml"); +} + +// Run with --ignored to run this test +#[test] +#[ignore] +fn run_state_transition_tests_large() { + run_state_transition_test("sanity-check_default-config_100-vals.yaml"); } diff --git a/eth2/state_processing/yaml_utils/Cargo.toml b/eth2/state_processing/yaml_utils/Cargo.toml index 4a7ae5b89..5f216fe1a 100644 --- a/eth2/state_processing/yaml_utils/Cargo.toml +++ b/eth2/state_processing/yaml_utils/Cargo.toml @@ -6,7 +6,6 @@ edition = "2018" [build-dependencies] reqwest = "0.9" -tempdir = "0.3" [dependencies] diff --git a/eth2/state_processing/yaml_utils/build.rs b/eth2/state_processing/yaml_utils/build.rs index 3b7f31471..7fb652cc1 100644 --- a/eth2/state_processing/yaml_utils/build.rs +++ b/eth2/state_processing/yaml_utils/build.rs @@ -1,5 +1,4 @@ extern crate reqwest; -extern crate tempdir; use std::fs::File; use std::io::copy; diff --git a/eth2/state_processing/yaml_utils/expected_state_fields.py b/eth2/state_processing/yaml_utils/expected_state_fields.py new file mode 100755 index 000000000..df4cb83f7 --- /dev/null +++ b/eth2/state_processing/yaml_utils/expected_state_fields.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python3 + +# Script to extract all the fields of the state mentioned in `expected_state` fields of tests +# in the `spec` directory. These fields can then be added to the `ExpectedState` struct. +# Might take a while to run. + +import os, yaml + +if __name__ == "__main__": + yaml_files = (filename for filename in os.listdir("specs") if filename.endswith(".yaml")) + parsed_yaml = (yaml.load(open("specs/" + filename, "r")) for filename in yaml_files) + all_fields = set() + for y in parsed_yaml: + all_fields.update(*({key for key in case["expected_state"]} for case in y["test_cases"])) + print(all_fields) diff --git a/eth2/types/Cargo.toml b/eth2/types/Cargo.toml index 613eb7936..36e251d7e 100644 --- a/eth2/types/Cargo.toml +++ b/eth2/types/Cargo.toml @@ -7,6 +7,7 @@ edition = "2018" [dependencies] bls = { path = "../utils/bls" } boolean-bitfield = { path = "../utils/boolean-bitfield" } +cached_tree_hash = { path = "../utils/cached_tree_hash" } dirs = "1.0" derivative = "1.0" ethereum-types = "0.5" @@ -26,6 +27,8 @@ ssz = { path = "../utils/ssz" } ssz_derive = { path = "../utils/ssz_derive" } swap_or_not_shuffle = { path = "../utils/swap_or_not_shuffle" } test_random_derive = { path = "../utils/test_random_derive" } +tree_hash = { path = "../utils/tree_hash" } +tree_hash_derive = { path = "../utils/tree_hash_derive" } libp2p = { git = "https://github.com/SigP/rust-libp2p", rev = "b3c32d9a821ae6cc89079499cc6e8a6bab0bffc3" } [dev-dependencies] diff --git a/eth2/types/src/attestation.rs b/eth2/types/src/attestation.rs index dabccfde7..d1511763d 100644 --- a/eth2/types/src/attestation.rs +++ b/eth2/types/src/attestation.rs @@ -2,13 +2,14 @@ use super::{AggregateSignature, AttestationData, Bitfield}; use crate::test_utils::TestRandom; use rand::RngCore; use serde_derive::{Deserialize, Serialize}; -use ssz::TreeHash; -use ssz_derive::{Decode, Encode, SignedRoot, TreeHash}; +use ssz_derive::{Decode, Encode}; use test_random_derive::TestRandom; +use tree_hash::TreeHash; +use tree_hash_derive::{CachedTreeHash, SignedRoot, TreeHash}; /// Details an attestation that can be slashable. /// -/// Spec v0.5.0 +/// Spec v0.5.1 #[derive( Debug, Clone, @@ -18,6 +19,7 @@ use test_random_derive::TestRandom; Encode, Decode, TreeHash, + CachedTreeHash, TestRandom, SignedRoot, )] @@ -25,6 +27,7 @@ pub struct Attestation { pub aggregation_bitfield: Bitfield, pub data: AttestationData, pub custody_bitfield: Bitfield, + #[signed_root(skip_hashing)] pub aggregate_signature: AggregateSignature, } @@ -56,4 +59,5 @@ mod tests { use super::*; ssz_tests!(Attestation); + cached_tree_hash_tests!(Attestation); } diff --git a/eth2/types/src/attestation_data.rs b/eth2/types/src/attestation_data.rs index 4a6b57823..c963d6001 100644 --- a/eth2/types/src/attestation_data.rs +++ b/eth2/types/src/attestation_data.rs @@ -2,13 +2,14 @@ use crate::test_utils::TestRandom; use crate::{Crosslink, Epoch, Hash256, Slot}; use rand::RngCore; use serde_derive::{Deserialize, Serialize}; -use ssz::TreeHash; -use ssz_derive::{Decode, Encode, SignedRoot, TreeHash}; +use ssz_derive::{Decode, Encode}; use test_random_derive::TestRandom; +use tree_hash::TreeHash; +use tree_hash_derive::{CachedTreeHash, SignedRoot, TreeHash}; /// The data upon which an attestation is based. /// -/// Spec v0.5.0 +/// Spec v0.5.1 #[derive( Debug, Clone, @@ -20,6 +21,7 @@ use test_random_derive::TestRandom; Encode, Decode, TreeHash, + CachedTreeHash, TestRandom, SignedRoot, )] @@ -46,4 +48,5 @@ mod tests { use super::*; ssz_tests!(AttestationData); + cached_tree_hash_tests!(AttestationData); } diff --git a/eth2/types/src/attestation_data_and_custody_bit.rs b/eth2/types/src/attestation_data_and_custody_bit.rs index 2cc6bc80c..85a5875ab 100644 --- a/eth2/types/src/attestation_data_and_custody_bit.rs +++ b/eth2/types/src/attestation_data_and_custody_bit.rs @@ -2,12 +2,13 @@ use super::AttestationData; use crate::test_utils::TestRandom; use rand::RngCore; use serde_derive::Serialize; -use ssz_derive::{Decode, Encode, TreeHash}; +use ssz_derive::{Decode, Encode}; +use tree_hash_derive::{CachedTreeHash, TreeHash}; /// Used for pairing an attestation with a proof-of-custody. /// -/// Spec v0.5.0 -#[derive(Debug, Clone, PartialEq, Default, Serialize, Encode, Decode, TreeHash)] +/// Spec v0.5.1 +#[derive(Debug, Clone, PartialEq, Default, Serialize, Encode, Decode, TreeHash, CachedTreeHash)] pub struct AttestationDataAndCustodyBit { pub data: AttestationData, pub custody_bit: bool, @@ -27,4 +28,5 @@ mod test { use super::*; ssz_tests!(AttestationDataAndCustodyBit); + cached_tree_hash_tests!(AttestationDataAndCustodyBit); } diff --git a/eth2/types/src/attester_slashing.rs b/eth2/types/src/attester_slashing.rs index 6fc404f42..d4848b01c 100644 --- a/eth2/types/src/attester_slashing.rs +++ b/eth2/types/src/attester_slashing.rs @@ -1,13 +1,25 @@ use crate::{test_utils::TestRandom, SlashableAttestation}; use rand::RngCore; use serde_derive::{Deserialize, Serialize}; -use ssz_derive::{Decode, Encode, TreeHash}; +use ssz_derive::{Decode, Encode}; use test_random_derive::TestRandom; +use tree_hash_derive::{CachedTreeHash, TreeHash}; /// Two conflicting attestations. /// -/// Spec v0.5.0 -#[derive(Debug, PartialEq, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom)] +/// Spec v0.5.1 +#[derive( + Debug, + PartialEq, + Clone, + Serialize, + Deserialize, + Encode, + Decode, + TreeHash, + CachedTreeHash, + TestRandom, +)] pub struct AttesterSlashing { pub slashable_attestation_1: SlashableAttestation, pub slashable_attestation_2: SlashableAttestation, @@ -18,4 +30,5 @@ mod tests { use super::*; ssz_tests!(AttesterSlashing); + cached_tree_hash_tests!(AttesterSlashing); } diff --git a/eth2/types/src/beacon_block.rs b/eth2/types/src/beacon_block.rs index 6a3f1a354..d198d16fc 100644 --- a/eth2/types/src/beacon_block.rs +++ b/eth2/types/src/beacon_block.rs @@ -3,13 +3,14 @@ use crate::*; use bls::Signature; use rand::RngCore; use serde_derive::{Deserialize, Serialize}; -use ssz::TreeHash; -use ssz_derive::{Decode, Encode, SignedRoot, TreeHash}; +use ssz_derive::{Decode, Encode}; use test_random_derive::TestRandom; +use tree_hash::TreeHash; +use tree_hash_derive::{CachedTreeHash, SignedRoot, TreeHash}; /// A block of the `BeaconChain`. /// -/// Spec v0.5.0 +/// Spec v0.5.1 #[derive( Debug, PartialEq, @@ -19,6 +20,7 @@ use test_random_derive::TestRandom; Encode, Decode, TreeHash, + CachedTreeHash, TestRandom, SignedRoot, )] @@ -27,13 +29,14 @@ pub struct BeaconBlock { pub previous_block_root: Hash256, pub state_root: Hash256, pub body: BeaconBlockBody, + #[signed_root(skip_hashing)] pub signature: Signature, } impl BeaconBlock { /// Returns an empty block to be used during genesis. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn empty(spec: &ChainSpec) -> BeaconBlock { BeaconBlock { slot: spec.genesis_slot, @@ -56,11 +59,11 @@ impl BeaconBlock { } } - /// Returns the `hash_tree_root` of the block. + /// Returns the `tree_hash_root | update` of the block. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn canonical_root(&self) -> Hash256 { - Hash256::from_slice(&self.hash_tree_root()[..]) + Hash256::from_slice(&self.tree_hash_root()[..]) } /// Returns a full `BeaconBlockHeader` of this block. @@ -70,20 +73,20 @@ impl BeaconBlock { /// /// Note: performs a full tree-hash of `self.body`. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn block_header(&self) -> BeaconBlockHeader { BeaconBlockHeader { slot: self.slot, previous_block_root: self.previous_block_root, state_root: self.state_root, - block_body_root: Hash256::from_slice(&self.body.hash_tree_root()[..]), + block_body_root: Hash256::from_slice(&self.body.tree_hash_root()[..]), signature: self.signature.clone(), } } /// Returns a "temporary" header, where the `state_root` is `spec.zero_hash`. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn temporary_block_header(&self, spec: &ChainSpec) -> BeaconBlockHeader { BeaconBlockHeader { state_root: spec.zero_hash, @@ -98,4 +101,5 @@ mod tests { use super::*; ssz_tests!(BeaconBlock); + cached_tree_hash_tests!(BeaconBlock); } diff --git a/eth2/types/src/beacon_block_body.rs b/eth2/types/src/beacon_block_body.rs index 677e24cec..15ba00d6b 100644 --- a/eth2/types/src/beacon_block_body.rs +++ b/eth2/types/src/beacon_block_body.rs @@ -2,13 +2,25 @@ use crate::test_utils::TestRandom; use crate::*; use rand::RngCore; use serde_derive::{Deserialize, Serialize}; -use ssz_derive::{Decode, Encode, TreeHash}; +use ssz_derive::{Decode, Encode}; use test_random_derive::TestRandom; +use tree_hash_derive::{CachedTreeHash, TreeHash}; /// The body of a `BeaconChain` block, containing operations. /// -/// Spec v0.5.0 -#[derive(Debug, PartialEq, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom)] +/// Spec v0.5.1 +#[derive( + Debug, + PartialEq, + Clone, + Serialize, + Deserialize, + Encode, + Decode, + TreeHash, + CachedTreeHash, + TestRandom, +)] pub struct BeaconBlockBody { pub randao_reveal: Signature, pub eth1_data: Eth1Data, @@ -25,4 +37,5 @@ mod tests { use super::*; ssz_tests!(BeaconBlockBody); + cached_tree_hash_tests!(BeaconBlockBody); } diff --git a/eth2/types/src/beacon_block_header.rs b/eth2/types/src/beacon_block_header.rs index f4bee27e1..5b35da1b6 100644 --- a/eth2/types/src/beacon_block_header.rs +++ b/eth2/types/src/beacon_block_header.rs @@ -3,13 +3,14 @@ use crate::*; use bls::Signature; use rand::RngCore; use serde_derive::{Deserialize, Serialize}; -use ssz::TreeHash; -use ssz_derive::{Decode, Encode, SignedRoot, TreeHash}; +use ssz_derive::{Decode, Encode}; use test_random_derive::TestRandom; +use tree_hash::{SignedRoot, TreeHash}; +use tree_hash_derive::{CachedTreeHash, SignedRoot, TreeHash}; /// A header of a `BeaconBlock`. /// -/// Spec v0.5.0 +/// Spec v0.5.1 #[derive( Debug, PartialEq, @@ -19,6 +20,7 @@ use test_random_derive::TestRandom; Encode, Decode, TreeHash, + CachedTreeHash, TestRandom, SignedRoot, )] @@ -27,20 +29,21 @@ pub struct BeaconBlockHeader { pub previous_block_root: Hash256, pub state_root: Hash256, pub block_body_root: Hash256, + #[signed_root(skip_hashing)] pub signature: Signature, } impl BeaconBlockHeader { - /// Returns the `hash_tree_root` of the header. + /// Returns the `tree_hash_root` of the header. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn canonical_root(&self) -> Hash256 { - Hash256::from_slice(&self.hash_tree_root()[..]) + Hash256::from_slice(&self.signed_root()[..]) } /// Given a `body`, consumes `self` and returns a complete `BeaconBlock`. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn into_block(self, body: BeaconBlockBody) -> BeaconBlock { BeaconBlock { slot: self.slot, @@ -57,4 +60,5 @@ mod tests { use super::*; ssz_tests!(BeaconBlockHeader); + cached_tree_hash_tests!(BeaconBlockHeader); } diff --git a/eth2/types/src/beacon_state.rs b/eth2/types/src/beacon_state.rs index 774e8eb76..e9b052f99 100644 --- a/eth2/types/src/beacon_state.rs +++ b/eth2/types/src/beacon_state.rs @@ -1,13 +1,16 @@ use self::epoch_cache::{get_active_validator_indices, EpochCache, Error as EpochCacheError}; use crate::test_utils::TestRandom; use crate::*; +use cached_tree_hash::{Error as TreeHashCacheError, TreeHashCache}; use int_to_bytes::int_to_bytes32; use pubkey_cache::PubkeyCache; use rand::RngCore; use serde_derive::{Deserialize, Serialize}; -use ssz::{hash, ssz_encode, TreeHash}; -use ssz_derive::{Decode, Encode, TreeHash}; +use ssz::{hash, ssz_encode}; +use ssz_derive::{Decode, Encode}; use test_random_derive::TestRandom; +use tree_hash::TreeHash; +use tree_hash_derive::{CachedTreeHash, TreeHash}; mod epoch_cache; mod pubkey_cache; @@ -40,12 +43,24 @@ pub enum Error { EpochCacheUninitialized(RelativeEpoch), RelativeEpochError(RelativeEpochError), EpochCacheError(EpochCacheError), + TreeHashCacheError(TreeHashCacheError), } /// The state of the `BeaconChain` at some slot. /// -/// Spec v0.5.0 -#[derive(Debug, PartialEq, Clone, Serialize, Deserialize, TestRandom, Encode, Decode, TreeHash)] +/// Spec v0.5.1 +#[derive( + Debug, + PartialEq, + Clone, + Serialize, + Deserialize, + TestRandom, + Encode, + Decode, + TreeHash, + CachedTreeHash, +)] pub struct BeaconState { // Misc pub slot: Slot, @@ -58,7 +73,7 @@ pub struct BeaconState { pub validator_registry_update_epoch: Epoch, // Randomness and committees - pub latest_randao_mixes: Vec, + pub latest_randao_mixes: TreeHashVector, pub previous_shuffling_start_shard: u64, pub current_shuffling_start_shard: u64, pub previous_shuffling_epoch: Epoch, @@ -78,11 +93,11 @@ pub struct BeaconState { pub finalized_root: Hash256, // Recent state - pub latest_crosslinks: Vec, - latest_block_roots: Vec, - latest_state_roots: Vec, - latest_active_index_roots: Vec, - latest_slashed_balances: Vec, + pub latest_crosslinks: TreeHashVector, + pub latest_block_roots: TreeHashVector, + latest_state_roots: TreeHashVector, + latest_active_index_roots: TreeHashVector, + latest_slashed_balances: TreeHashVector, pub latest_block_header: BeaconBlockHeader, pub historical_roots: Vec, @@ -110,6 +125,12 @@ pub struct BeaconState { #[tree_hash(skip_hashing)] #[test_random(default)] pub pubkey_cache: PubkeyCache, + #[serde(skip_serializing, skip_deserializing)] + #[ssz(skip_serializing)] + #[ssz(skip_deserializing)] + #[tree_hash(skip_hashing)] + #[test_random(default)] + pub tree_hash_cache: TreeHashCache, } impl BeaconState { @@ -118,7 +139,7 @@ impl BeaconState { /// This does not fully build a genesis beacon state, it omits processing of initial validator /// deposits. To obtain a full genesis beacon state, use the `BeaconStateBuilder`. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn genesis(genesis_time: u64, latest_eth1_data: Eth1Data, spec: &ChainSpec) -> BeaconState { let initial_crosslink = Crosslink { epoch: spec.genesis_epoch, @@ -137,7 +158,8 @@ impl BeaconState { validator_registry_update_epoch: spec.genesis_epoch, // Randomness and committees - latest_randao_mixes: vec![spec.zero_hash; spec.latest_randao_mixes_length as usize], + latest_randao_mixes: vec![spec.zero_hash; spec.latest_randao_mixes_length as usize] + .into(), previous_shuffling_start_shard: spec.genesis_start_shard, current_shuffling_start_shard: spec.genesis_start_shard, previous_shuffling_epoch: spec.genesis_epoch, @@ -157,11 +179,12 @@ impl BeaconState { finalized_root: spec.zero_hash, // Recent state - latest_crosslinks: vec![initial_crosslink; spec.shard_count as usize], - latest_block_roots: vec![spec.zero_hash; spec.slots_per_historical_root], - latest_state_roots: vec![spec.zero_hash; spec.slots_per_historical_root], - latest_active_index_roots: vec![spec.zero_hash; spec.latest_active_index_roots_length], - latest_slashed_balances: vec![0; spec.latest_slashed_exit_length], + latest_crosslinks: vec![initial_crosslink; spec.shard_count as usize].into(), + latest_block_roots: vec![spec.zero_hash; spec.slots_per_historical_root].into(), + latest_state_roots: vec![spec.zero_hash; spec.slots_per_historical_root].into(), + latest_active_index_roots: vec![spec.zero_hash; spec.latest_active_index_roots_length] + .into(), + latest_slashed_balances: vec![0; spec.latest_slashed_exit_length].into(), latest_block_header: BeaconBlock::empty(spec).temporary_block_header(spec), historical_roots: vec![], @@ -183,14 +206,15 @@ impl BeaconState { EpochCache::default(), ], pubkey_cache: PubkeyCache::default(), + tree_hash_cache: TreeHashCache::default(), } } - /// Returns the `hash_tree_root` of the state. + /// Returns the `tree_hash_root` of the state. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn canonical_root(&self) -> Hash256 { - Hash256::from_slice(&self.hash_tree_root()[..]) + Hash256::from_slice(&self.tree_hash_root()[..]) } pub fn historical_batch(&self) -> HistoricalBatch { @@ -217,7 +241,7 @@ impl BeaconState { /// The epoch corresponding to `self.slot`. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn current_epoch(&self, spec: &ChainSpec) -> Epoch { self.slot.epoch(spec.slots_per_epoch) } @@ -226,14 +250,14 @@ impl BeaconState { /// /// If the current epoch is the genesis epoch, the genesis_epoch is returned. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn previous_epoch(&self, spec: &ChainSpec) -> Epoch { self.current_epoch(&spec) - 1 } /// The epoch following `self.current_epoch()`. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn next_epoch(&self, spec: &ChainSpec) -> Epoch { self.current_epoch(spec) + 1 } @@ -246,7 +270,7 @@ impl BeaconState { /// /// Note: Utilizes the cache and will fail if the appropriate cache is not initialized. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn get_cached_active_validator_indices( &self, relative_epoch: RelativeEpoch, @@ -261,7 +285,7 @@ impl BeaconState { /// /// Does not utilize the cache, performs a full iteration over the validator registry. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn get_active_validator_indices(&self, epoch: Epoch) -> Vec { get_active_validator_indices(&self.validator_registry, epoch) } @@ -270,7 +294,7 @@ impl BeaconState { /// /// Note: Utilizes the cache and will fail if the appropriate cache is not initialized. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn get_crosslink_committees_at_slot( &self, slot: Slot, @@ -295,7 +319,7 @@ impl BeaconState { /// /// Note: Utilizes the cache and will fail if the appropriate cache is not initialized. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn get_crosslink_committee_for_shard( &self, epoch: Epoch, @@ -321,7 +345,7 @@ impl BeaconState { /// /// If the state does not contain an index for a beacon proposer at the requested `slot`, then `None` is returned. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn get_beacon_proposer_index( &self, slot: Slot, @@ -350,7 +374,7 @@ impl BeaconState { /// Safely obtains the index for latest block roots, given some `slot`. /// - /// Spec v0.5.0 + /// Spec v0.5.1 fn get_latest_block_roots_index(&self, slot: Slot, spec: &ChainSpec) -> Result { if (slot < self.slot) && (self.slot <= slot + spec.slots_per_historical_root as u64) { let i = slot.as_usize() % spec.slots_per_historical_root; @@ -366,7 +390,7 @@ impl BeaconState { /// Return the block root at a recent `slot`. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn get_block_root( &self, slot: Slot, @@ -378,7 +402,7 @@ impl BeaconState { /// Sets the block root for some given slot. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn set_block_root( &mut self, slot: Slot, @@ -392,7 +416,7 @@ impl BeaconState { /// Safely obtains the index for `latest_randao_mixes` /// - /// Spec v0.5.0 + /// Spec v0.5.1 fn get_randao_mix_index(&self, epoch: Epoch, spec: &ChainSpec) -> Result { let current_epoch = self.current_epoch(spec); @@ -416,7 +440,7 @@ impl BeaconState { /// /// See `Self::get_randao_mix`. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn update_randao_mix( &mut self, epoch: Epoch, @@ -434,7 +458,7 @@ impl BeaconState { /// Return the randao mix at a recent ``epoch``. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn get_randao_mix(&self, epoch: Epoch, spec: &ChainSpec) -> Result<&Hash256, Error> { let i = self.get_randao_mix_index(epoch, spec)?; Ok(&self.latest_randao_mixes[i]) @@ -442,7 +466,7 @@ impl BeaconState { /// Set the randao mix at a recent ``epoch``. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn set_randao_mix( &mut self, epoch: Epoch, @@ -456,7 +480,7 @@ impl BeaconState { /// Safely obtains the index for `latest_active_index_roots`, given some `epoch`. /// - /// Spec v0.5.0 + /// Spec v0.5.1 fn get_active_index_root_index(&self, epoch: Epoch, spec: &ChainSpec) -> Result { let current_epoch = self.current_epoch(spec); @@ -478,7 +502,7 @@ impl BeaconState { /// Return the `active_index_root` at a recent `epoch`. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn get_active_index_root(&self, epoch: Epoch, spec: &ChainSpec) -> Result { let i = self.get_active_index_root_index(epoch, spec)?; Ok(self.latest_active_index_roots[i]) @@ -486,7 +510,7 @@ impl BeaconState { /// Set the `active_index_root` at a recent `epoch`. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn set_active_index_root( &mut self, epoch: Epoch, @@ -500,15 +524,15 @@ impl BeaconState { /// Replace `active_index_roots` with clones of `index_root`. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn fill_active_index_roots_with(&mut self, index_root: Hash256, spec: &ChainSpec) { self.latest_active_index_roots = - vec![index_root; spec.latest_active_index_roots_length as usize] + vec![index_root; spec.latest_active_index_roots_length as usize].into() } /// Safely obtains the index for latest state roots, given some `slot`. /// - /// Spec v0.5.0 + /// Spec v0.5.1 fn get_latest_state_roots_index(&self, slot: Slot, spec: &ChainSpec) -> Result { if (slot < self.slot) && (self.slot <= slot + spec.slots_per_historical_root as u64) { let i = slot.as_usize() % spec.slots_per_historical_root; @@ -524,7 +548,7 @@ impl BeaconState { /// Gets the state root for some slot. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn get_state_root(&mut self, slot: Slot, spec: &ChainSpec) -> Result<&Hash256, Error> { let i = self.get_latest_state_roots_index(slot, spec)?; Ok(&self.latest_state_roots[i]) @@ -532,7 +556,7 @@ impl BeaconState { /// Sets the latest state root for slot. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn set_state_root( &mut self, slot: Slot, @@ -546,7 +570,7 @@ impl BeaconState { /// Safely obtains the index for `latest_slashed_balances`, given some `epoch`. /// - /// Spec v0.5.0 + /// Spec v0.5.1 fn get_slashed_balance_index(&self, epoch: Epoch, spec: &ChainSpec) -> Result { let i = epoch.as_usize() % spec.latest_slashed_exit_length; @@ -561,7 +585,7 @@ impl BeaconState { /// Gets the total slashed balances for some epoch. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn get_slashed_balance(&self, epoch: Epoch, spec: &ChainSpec) -> Result { let i = self.get_slashed_balance_index(epoch, spec)?; Ok(self.latest_slashed_balances[i]) @@ -569,7 +593,7 @@ impl BeaconState { /// Sets the total slashed balances for some epoch. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn set_slashed_balance( &mut self, epoch: Epoch, @@ -583,7 +607,7 @@ impl BeaconState { /// Generate a seed for the given `epoch`. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn generate_seed(&self, epoch: Epoch, spec: &ChainSpec) -> Result { let mut input = self .get_randao_mix(epoch - spec.min_seed_lookahead, spec)? @@ -599,7 +623,7 @@ impl BeaconState { /// Return the effective balance (also known as "balance at stake") for a validator with the given ``index``. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn get_effective_balance( &self, validator_index: usize, @@ -614,14 +638,14 @@ impl BeaconState { /// Return the epoch at which an activation or exit triggered in ``epoch`` takes effect. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn get_delayed_activation_exit_epoch(&self, epoch: Epoch, spec: &ChainSpec) -> Epoch { epoch + 1 + spec.activation_exit_delay } /// Initiate an exit for the validator of the given `index`. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn initiate_validator_exit(&mut self, validator_index: usize) { self.validator_registry[validator_index].initiated_exit = true; } @@ -633,7 +657,7 @@ impl BeaconState { /// /// Note: Utilizes the cache and will fail if the appropriate cache is not initialized. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn get_attestation_duties( &self, validator_index: usize, @@ -649,7 +673,7 @@ impl BeaconState { /// Return the combined effective balance of an array of validators. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn get_total_balance( &self, validator_indices: &[usize], @@ -668,6 +692,7 @@ impl BeaconState { self.build_epoch_cache(RelativeEpoch::NextWithoutRegistryChange, spec)?; self.build_epoch_cache(RelativeEpoch::NextWithRegistryChange, spec)?; self.update_pubkey_cache()?; + self.update_tree_hash_cache()?; Ok(()) } @@ -774,6 +799,39 @@ impl BeaconState { pub fn drop_pubkey_cache(&mut self) { self.pubkey_cache = PubkeyCache::default() } + + /// Update the tree hash cache, building it for the first time if it is empty. + /// + /// Returns the `tree_hash_root` resulting from the update. This root can be considered the + /// canonical root of `self`. + pub fn update_tree_hash_cache(&mut self) -> Result { + if self.tree_hash_cache.is_empty() { + self.tree_hash_cache = TreeHashCache::new(self)?; + } else { + // Move the cache outside of `self` to satisfy the borrow checker. + let mut cache = std::mem::replace(&mut self.tree_hash_cache, TreeHashCache::default()); + + cache.update(self)?; + + // Move the updated cache back into `self`. + self.tree_hash_cache = cache + } + + self.cached_tree_hash_root() + } + + /// Returns the tree hash root determined by the last execution of `self.update_tree_hash_cache(..)`. + /// + /// Note: does _not_ update the cache and may return an outdated root. + /// + /// Returns an error if the cache is not initialized or if an error is encountered during the + /// cache update. + pub fn cached_tree_hash_root(&self) -> Result { + self.tree_hash_cache + .tree_hash_root() + .and_then(|b| Ok(Hash256::from_slice(b))) + .map_err(|e| e.into()) + } } impl From for Error { @@ -787,3 +845,9 @@ impl From for Error { Error::EpochCacheError(e) } } + +impl From for Error { + fn from(e: TreeHashCacheError) -> Error { + Error::TreeHashCacheError(e) + } +} diff --git a/eth2/types/src/beacon_state/epoch_cache.rs b/eth2/types/src/beacon_state/epoch_cache.rs index 62df90271..1a63e9eb9 100644 --- a/eth2/types/src/beacon_state/epoch_cache.rs +++ b/eth2/types/src/beacon_state/epoch_cache.rs @@ -138,7 +138,7 @@ impl EpochCache { /// Returns a list of all `validator_registry` indices where the validator is active at the given /// `epoch`. /// -/// Spec v0.5.0 +/// Spec v0.5.1 pub fn get_active_validator_indices(validators: &[Validator], epoch: Epoch) -> Vec { let mut active = Vec::with_capacity(validators.len()); @@ -288,7 +288,7 @@ impl EpochCrosslinkCommitteesBuilder { self.active_validator_indices, spec.shuffle_round_count, &self.shuffling_seed[..], - true, + false, ) .ok_or_else(|| Error::UnableToShuffle)? }; diff --git a/eth2/types/src/beacon_state/epoch_cache/tests.rs b/eth2/types/src/beacon_state/epoch_cache/tests.rs index 5643776e2..5b1e53338 100644 --- a/eth2/types/src/beacon_state/epoch_cache/tests.rs +++ b/eth2/types/src/beacon_state/epoch_cache/tests.rs @@ -27,7 +27,7 @@ fn do_sane_cache_test( active_indices, spec.shuffle_round_count, &expected_seed[..], - true, + false, ) .unwrap(); diff --git a/eth2/types/src/beacon_state/tests.rs b/eth2/types/src/beacon_state/tests.rs index dc16a013b..d5862559a 100644 --- a/eth2/types/src/beacon_state/tests.rs +++ b/eth2/types/src/beacon_state/tests.rs @@ -3,6 +3,7 @@ use super::*; use crate::test_utils::*; ssz_tests!(BeaconState); +cached_tree_hash_tests!(BeaconState); /// Test that /// @@ -55,3 +56,22 @@ fn cache_initialization() { test_cache_initialization(&mut state, RelativeEpoch::NextWithRegistryChange, &spec); test_cache_initialization(&mut state, RelativeEpoch::NextWithoutRegistryChange, &spec); } + +#[test] +fn tree_hash_cache() { + use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; + use tree_hash::TreeHash; + + let mut rng = XorShiftRng::from_seed([42; 16]); + + let mut state = BeaconState::random_for_test(&mut rng); + + let root = state.update_tree_hash_cache().unwrap(); + + assert_eq!(root.as_bytes(), &state.tree_hash_root()[..]); + + state.slot = state.slot + 1; + + let root = state.update_tree_hash_cache().unwrap(); + assert_eq!(root.as_bytes(), &state.tree_hash_root()[..]); +} diff --git a/eth2/types/src/chain_spec.rs b/eth2/types/src/chain_spec.rs index 0042304f8..f3c92b42c 100644 --- a/eth2/types/src/chain_spec.rs +++ b/eth2/types/src/chain_spec.rs @@ -8,7 +8,7 @@ const GWEI: u64 = 1_000_000_000; /// Each of the BLS signature domains. /// -/// Spec v0.5.0 +/// Spec v0.5.1 pub enum Domain { BeaconBlock, Randao, @@ -20,7 +20,7 @@ pub enum Domain { /// Holds all the "constants" for a BeaconChain. /// -/// Spec v0.5.0 +/// Spec v0.5.1 #[derive(PartialEq, Debug, Clone, Deserialize)] #[serde(default)] pub struct ChainSpec { @@ -126,7 +126,7 @@ pub struct ChainSpec { impl ChainSpec { /// Return the number of committees in one epoch. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn get_epoch_committee_count(&self, active_validator_count: usize) -> u64 { std::cmp::max( 1, @@ -139,7 +139,7 @@ impl ChainSpec { /// Get the domain number that represents the fork meta and signature domain. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn get_domain(&self, epoch: Epoch, domain: Domain, fork: &Fork) -> u64 { let domain_constant = match domain { Domain::BeaconBlock => self.domain_beacon_block, @@ -161,7 +161,7 @@ impl ChainSpec { /// Returns a `ChainSpec` compatible with the Ethereum Foundation specification. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn foundation() -> Self { let genesis_slot = Slot::new(2_u64.pow(32)); let slots_per_epoch = 64; diff --git a/eth2/types/src/crosslink.rs b/eth2/types/src/crosslink.rs index f91680c75..448f5ea30 100644 --- a/eth2/types/src/crosslink.rs +++ b/eth2/types/src/crosslink.rs @@ -2,12 +2,13 @@ use crate::test_utils::TestRandom; use crate::{Epoch, Hash256}; use rand::RngCore; use serde_derive::{Deserialize, Serialize}; -use ssz_derive::{Decode, Encode, TreeHash}; +use ssz_derive::{Decode, Encode}; use test_random_derive::TestRandom; +use tree_hash_derive::{CachedTreeHash, TreeHash}; /// Specifies the block hash for a shard at an epoch. /// -/// Spec v0.5.0 +/// Spec v0.5.1 #[derive( Debug, Clone, @@ -19,6 +20,7 @@ use test_random_derive::TestRandom; Encode, Decode, TreeHash, + CachedTreeHash, TestRandom, )] pub struct Crosslink { @@ -31,4 +33,5 @@ mod tests { use super::*; ssz_tests!(Crosslink); + cached_tree_hash_tests!(Crosslink); } diff --git a/eth2/types/src/crosslink_committee.rs b/eth2/types/src/crosslink_committee.rs index af1778a1b..25c42c07b 100644 --- a/eth2/types/src/crosslink_committee.rs +++ b/eth2/types/src/crosslink_committee.rs @@ -1,8 +1,20 @@ use crate::*; use serde_derive::{Deserialize, Serialize}; -use ssz_derive::{Decode, Encode, TreeHash}; +use ssz_derive::{Decode, Encode}; +use tree_hash_derive::{CachedTreeHash, TreeHash}; -#[derive(Default, Clone, Debug, PartialEq, Serialize, Deserialize, Decode, Encode, TreeHash)] +#[derive( + Default, + Clone, + Debug, + PartialEq, + Serialize, + Deserialize, + Decode, + Encode, + TreeHash, + CachedTreeHash, +)] pub struct CrosslinkCommittee { pub slot: Slot, pub shard: Shard, diff --git a/eth2/types/src/deposit.rs b/eth2/types/src/deposit.rs index ff8d83d77..e8d2f5a4b 100644 --- a/eth2/types/src/deposit.rs +++ b/eth2/types/src/deposit.rs @@ -1,16 +1,28 @@ -use super::{DepositData, Hash256}; +use super::{DepositData, Hash256, TreeHashVector}; use crate::test_utils::TestRandom; use rand::RngCore; use serde_derive::{Deserialize, Serialize}; -use ssz_derive::{Decode, Encode, TreeHash}; +use ssz_derive::{Decode, Encode}; use test_random_derive::TestRandom; +use tree_hash_derive::{CachedTreeHash, TreeHash}; /// A deposit to potentially become a beacon chain validator. /// -/// Spec v0.5.0 -#[derive(Debug, PartialEq, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom)] +/// Spec v0.5.1 +#[derive( + Debug, + PartialEq, + Clone, + Serialize, + Deserialize, + Encode, + Decode, + TreeHash, + CachedTreeHash, + TestRandom, +)] pub struct Deposit { - pub proof: Vec, + pub proof: TreeHashVector, pub index: u64, pub deposit_data: DepositData, } @@ -20,4 +32,5 @@ mod tests { use super::*; ssz_tests!(Deposit); + cached_tree_hash_tests!(Deposit); } diff --git a/eth2/types/src/deposit_data.rs b/eth2/types/src/deposit_data.rs index a1e30032f..38c44d1a7 100644 --- a/eth2/types/src/deposit_data.rs +++ b/eth2/types/src/deposit_data.rs @@ -2,13 +2,25 @@ use super::DepositInput; use crate::test_utils::TestRandom; use rand::RngCore; use serde_derive::{Deserialize, Serialize}; -use ssz_derive::{Decode, Encode, TreeHash}; +use ssz_derive::{Decode, Encode}; use test_random_derive::TestRandom; +use tree_hash_derive::{CachedTreeHash, TreeHash}; /// Data generated by the deposit contract. /// -/// Spec v0.5.0 -#[derive(Debug, PartialEq, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom)] +/// Spec v0.5.1 +#[derive( + Debug, + PartialEq, + Clone, + Serialize, + Deserialize, + Encode, + Decode, + TreeHash, + CachedTreeHash, + TestRandom, +)] pub struct DepositData { pub amount: u64, pub timestamp: u64, @@ -20,4 +32,5 @@ mod tests { use super::*; ssz_tests!(DepositData); + cached_tree_hash_tests!(DepositData); } diff --git a/eth2/types/src/deposit_input.rs b/eth2/types/src/deposit_input.rs index 3f8a6177a..af1049a20 100644 --- a/eth2/types/src/deposit_input.rs +++ b/eth2/types/src/deposit_input.rs @@ -3,13 +3,14 @@ use crate::*; use bls::{PublicKey, Signature}; use rand::RngCore; use serde_derive::{Deserialize, Serialize}; -use ssz::{SignedRoot, TreeHash}; -use ssz_derive::{Decode, Encode, SignedRoot, TreeHash}; +use ssz_derive::{Decode, Encode}; use test_random_derive::TestRandom; +use tree_hash::{SignedRoot, TreeHash}; +use tree_hash_derive::{CachedTreeHash, SignedRoot, TreeHash}; /// The data supplied by the user to the deposit contract. /// -/// Spec v0.5.0 +/// Spec v0.5.1 #[derive( Debug, PartialEq, @@ -20,18 +21,20 @@ use test_random_derive::TestRandom; Decode, SignedRoot, TreeHash, + CachedTreeHash, TestRandom, )] pub struct DepositInput { pub pubkey: PublicKey, pub withdrawal_credentials: Hash256, + #[signed_root(skip_hashing)] pub proof_of_possession: Signature, } impl DepositInput { /// Generate the 'proof_of_posession' signature for a given DepositInput details. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn create_proof_of_possession( &self, secret_key: &SecretKey, @@ -47,7 +50,7 @@ impl DepositInput { /// Verify that proof-of-possession is valid. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn validate_proof_of_possession( &self, epoch: Epoch, @@ -66,6 +69,7 @@ mod tests { use super::*; ssz_tests!(DepositInput); + cached_tree_hash_tests!(DepositInput); #[test] fn can_create_and_validate() { diff --git a/eth2/types/src/epoch_cache.rs b/eth2/types/src/epoch_cache.rs deleted file mode 100644 index e69de29bb..000000000 diff --git a/eth2/types/src/eth1_data.rs b/eth2/types/src/eth1_data.rs index deced19fb..3c0c3af02 100644 --- a/eth2/types/src/eth1_data.rs +++ b/eth2/types/src/eth1_data.rs @@ -2,14 +2,25 @@ use super::Hash256; use crate::test_utils::TestRandom; use rand::RngCore; use serde_derive::{Deserialize, Serialize}; -use ssz_derive::{Decode, Encode, TreeHash}; +use ssz_derive::{Decode, Encode}; use test_random_derive::TestRandom; +use tree_hash_derive::{CachedTreeHash, TreeHash}; /// Contains data obtained from the Eth1 chain. /// -/// Spec v0.5.0 +/// Spec v0.5.1 #[derive( - Debug, PartialEq, Clone, Default, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom, + Debug, + PartialEq, + Clone, + Default, + Serialize, + Deserialize, + Encode, + Decode, + TreeHash, + CachedTreeHash, + TestRandom, )] pub struct Eth1Data { pub deposit_root: Hash256, @@ -21,4 +32,5 @@ mod tests { use super::*; ssz_tests!(Eth1Data); + cached_tree_hash_tests!(Eth1Data); } diff --git a/eth2/types/src/eth1_data_vote.rs b/eth2/types/src/eth1_data_vote.rs index 2f3a1ade1..00818ebf4 100644 --- a/eth2/types/src/eth1_data_vote.rs +++ b/eth2/types/src/eth1_data_vote.rs @@ -2,14 +2,25 @@ use super::Eth1Data; use crate::test_utils::TestRandom; use rand::RngCore; use serde_derive::{Deserialize, Serialize}; -use ssz_derive::{Decode, Encode, TreeHash}; +use ssz_derive::{Decode, Encode}; use test_random_derive::TestRandom; +use tree_hash_derive::{CachedTreeHash, TreeHash}; /// A summation of votes for some `Eth1Data`. /// -/// Spec v0.5.0 +/// Spec v0.5.1 #[derive( - Debug, PartialEq, Clone, Default, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom, + Debug, + PartialEq, + Clone, + Default, + Serialize, + Deserialize, + Encode, + Decode, + TreeHash, + CachedTreeHash, + TestRandom, )] pub struct Eth1DataVote { pub eth1_data: Eth1Data, @@ -21,4 +32,5 @@ mod tests { use super::*; ssz_tests!(Eth1DataVote); + cached_tree_hash_tests!(Eth1DataVote); } diff --git a/eth2/types/src/fork.rs b/eth2/types/src/fork.rs index b9d16c333..83d4f5dc6 100644 --- a/eth2/types/src/fork.rs +++ b/eth2/types/src/fork.rs @@ -5,14 +5,25 @@ use crate::{ use int_to_bytes::int_to_bytes4; use rand::RngCore; use serde_derive::{Deserialize, Serialize}; -use ssz_derive::{Decode, Encode, TreeHash}; +use ssz_derive::{Decode, Encode}; use test_random_derive::TestRandom; +use tree_hash_derive::{CachedTreeHash, TreeHash}; /// Specifies a fork of the `BeaconChain`, to prevent replay attacks. /// -/// Spec v0.5.0 +/// Spec v0.5.1 #[derive( - Debug, Clone, PartialEq, Default, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom, + Debug, + Clone, + PartialEq, + Default, + Serialize, + Deserialize, + Encode, + Decode, + TreeHash, + CachedTreeHash, + TestRandom, )] pub struct Fork { #[serde(deserialize_with = "fork_from_hex_str")] @@ -25,7 +36,7 @@ pub struct Fork { impl Fork { /// Initialize the `Fork` from the genesis parameters in the `spec`. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn genesis(spec: &ChainSpec) -> Self { let mut current_version: [u8; 4] = [0; 4]; current_version.copy_from_slice(&int_to_bytes4(spec.genesis_fork_version)); @@ -39,7 +50,7 @@ impl Fork { /// Return the fork version of the given ``epoch``. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn get_fork_version(&self, epoch: Epoch) -> [u8; 4] { if epoch < self.epoch { return self.previous_version; @@ -53,6 +64,7 @@ mod tests { use super::*; ssz_tests!(Fork); + cached_tree_hash_tests!(Fork); fn test_genesis(version: u32, epoch: Epoch) { let mut spec = ChainSpec::foundation(); diff --git a/eth2/types/src/historical_batch.rs b/eth2/types/src/historical_batch.rs index 77859ed1a..13e57131a 100644 --- a/eth2/types/src/historical_batch.rs +++ b/eth2/types/src/historical_batch.rs @@ -1,17 +1,29 @@ use crate::test_utils::TestRandom; -use crate::Hash256; +use crate::{Hash256, TreeHashVector}; use rand::RngCore; use serde_derive::{Deserialize, Serialize}; -use ssz_derive::{Decode, Encode, TreeHash}; +use ssz_derive::{Decode, Encode}; use test_random_derive::TestRandom; +use tree_hash_derive::{CachedTreeHash, TreeHash}; /// Historical block and state roots. /// -/// Spec v0.5.0 -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom)] +/// Spec v0.5.1 +#[derive( + Debug, + Clone, + PartialEq, + Serialize, + Deserialize, + Encode, + Decode, + TreeHash, + CachedTreeHash, + TestRandom, +)] pub struct HistoricalBatch { - pub block_roots: Vec, - pub state_roots: Vec, + pub block_roots: TreeHashVector, + pub state_roots: TreeHashVector, } #[cfg(test)] @@ -19,4 +31,5 @@ mod tests { use super::*; ssz_tests!(HistoricalBatch); + cached_tree_hash_tests!(HistoricalBatch); } diff --git a/eth2/types/src/lib.rs b/eth2/types/src/lib.rs index 118e862e8..070ed6745 100644 --- a/eth2/types/src/lib.rs +++ b/eth2/types/src/lib.rs @@ -27,6 +27,7 @@ pub mod pending_attestation; pub mod proposer_slashing; pub mod slashable_attestation; pub mod transfer; +pub mod tree_hash_vector; pub mod voluntary_exit; #[macro_use] pub mod slot_epoch_macros; @@ -65,6 +66,7 @@ pub use crate::slashable_attestation::SlashableAttestation; pub use crate::slot_epoch::{Epoch, Slot}; pub use crate::slot_height::SlotHeight; pub use crate::transfer::Transfer; +pub use crate::tree_hash_vector::TreeHashVector; pub use crate::validator::Validator; pub use crate::voluntary_exit::VoluntaryExit; diff --git a/eth2/types/src/pending_attestation.rs b/eth2/types/src/pending_attestation.rs index 938e59bef..b71351f9a 100644 --- a/eth2/types/src/pending_attestation.rs +++ b/eth2/types/src/pending_attestation.rs @@ -2,13 +2,25 @@ use crate::test_utils::TestRandom; use crate::{Attestation, AttestationData, Bitfield, Slot}; use rand::RngCore; use serde_derive::{Deserialize, Serialize}; -use ssz_derive::{Decode, Encode, TreeHash}; +use ssz_derive::{Decode, Encode}; use test_random_derive::TestRandom; +use tree_hash_derive::{CachedTreeHash, TreeHash}; /// An attestation that has been included in the state but not yet fully processed. /// -/// Spec v0.5.0 -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom)] +/// Spec v0.5.1 +#[derive( + Debug, + Clone, + PartialEq, + Serialize, + Deserialize, + Encode, + Decode, + TreeHash, + CachedTreeHash, + TestRandom, +)] pub struct PendingAttestation { pub aggregation_bitfield: Bitfield, pub data: AttestationData, @@ -33,4 +45,5 @@ mod tests { use super::*; ssz_tests!(PendingAttestation); + cached_tree_hash_tests!(PendingAttestation); } diff --git a/eth2/types/src/proposer_slashing.rs b/eth2/types/src/proposer_slashing.rs index 02216a2fc..bf26ae508 100644 --- a/eth2/types/src/proposer_slashing.rs +++ b/eth2/types/src/proposer_slashing.rs @@ -2,13 +2,25 @@ use super::BeaconBlockHeader; use crate::test_utils::TestRandom; use rand::RngCore; use serde_derive::{Deserialize, Serialize}; -use ssz_derive::{Decode, Encode, TreeHash}; +use ssz_derive::{Decode, Encode}; use test_random_derive::TestRandom; +use tree_hash_derive::{CachedTreeHash, TreeHash}; /// Two conflicting proposals from the same proposer (validator). /// -/// Spec v0.5.0 -#[derive(Debug, PartialEq, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom)] +/// Spec v0.5.1 +#[derive( + Debug, + PartialEq, + Clone, + Serialize, + Deserialize, + Encode, + Decode, + TreeHash, + CachedTreeHash, + TestRandom, +)] pub struct ProposerSlashing { pub proposer_index: u64, pub header_1: BeaconBlockHeader, @@ -20,4 +32,5 @@ mod tests { use super::*; ssz_tests!(ProposerSlashing); + cached_tree_hash_tests!(ProposerSlashing); } diff --git a/eth2/types/src/relative_epoch.rs b/eth2/types/src/relative_epoch.rs index 8f895e97a..6538ca4aa 100644 --- a/eth2/types/src/relative_epoch.rs +++ b/eth2/types/src/relative_epoch.rs @@ -10,7 +10,7 @@ pub enum Error { /// Defines the epochs relative to some epoch. Most useful when referring to the committees prior /// to and following some epoch. /// -/// Spec v0.5.0 +/// Spec v0.5.1 #[derive(Debug, PartialEq, Clone, Copy)] pub enum RelativeEpoch { /// The prior epoch. @@ -32,7 +32,7 @@ pub enum RelativeEpoch { impl RelativeEpoch { /// Returns the `epoch` that `self` refers to, with respect to the `base` epoch. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn into_epoch(self, base: Epoch) -> Epoch { match self { RelativeEpoch::Previous => base - 1, @@ -51,7 +51,7 @@ impl RelativeEpoch { /// - `AmbiguiousNextEpoch` whenever `other` is one after `base`, because it's unknowable if /// there will be a registry change. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn from_epoch(base: Epoch, other: Epoch) -> Result { if other == base - 1 { Ok(RelativeEpoch::Previous) diff --git a/eth2/types/src/slashable_attestation.rs b/eth2/types/src/slashable_attestation.rs index 05c41a72b..fb838e0c4 100644 --- a/eth2/types/src/slashable_attestation.rs +++ b/eth2/types/src/slashable_attestation.rs @@ -1,15 +1,16 @@ use crate::{test_utils::TestRandom, AggregateSignature, AttestationData, Bitfield, ChainSpec}; use rand::RngCore; use serde_derive::{Deserialize, Serialize}; -use ssz::TreeHash; -use ssz_derive::{Decode, Encode, SignedRoot, TreeHash}; +use ssz_derive::{Decode, Encode}; use test_random_derive::TestRandom; +use tree_hash::TreeHash; +use tree_hash_derive::{CachedTreeHash, SignedRoot, TreeHash}; /// Details an attestation that can be slashable. /// /// To be included in an `AttesterSlashing`. /// -/// Spec v0.5.0 +/// Spec v0.5.1 #[derive( Debug, PartialEq, @@ -19,6 +20,7 @@ use test_random_derive::TestRandom; Encode, Decode, TreeHash, + CachedTreeHash, TestRandom, SignedRoot, )] @@ -27,20 +29,21 @@ pub struct SlashableAttestation { pub validator_indices: Vec, pub data: AttestationData, pub custody_bitfield: Bitfield, + #[signed_root(skip_hashing)] pub aggregate_signature: AggregateSignature, } impl SlashableAttestation { /// Check if ``attestation_data_1`` and ``attestation_data_2`` have the same target. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn is_double_vote(&self, other: &SlashableAttestation, spec: &ChainSpec) -> bool { self.data.slot.epoch(spec.slots_per_epoch) == other.data.slot.epoch(spec.slots_per_epoch) } /// Check if ``attestation_data_1`` surrounds ``attestation_data_2``. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn is_surround_vote(&self, other: &SlashableAttestation, spec: &ChainSpec) -> bool { let source_epoch_1 = self.data.source_epoch; let source_epoch_2 = other.data.source_epoch; @@ -131,6 +134,7 @@ mod tests { } ssz_tests!(SlashableAttestation); + cached_tree_hash_tests!(SlashableAttestation); fn create_slashable_attestation( slot_factor: u64, diff --git a/eth2/types/src/slot_epoch.rs b/eth2/types/src/slot_epoch.rs index d334177e5..6c6a92ecb 100644 --- a/eth2/types/src/slot_epoch.rs +++ b/eth2/types/src/slot_epoch.rs @@ -14,7 +14,7 @@ use crate::test_utils::TestRandom; use rand::RngCore; use serde_derive::{Deserialize, Serialize}; use slog; -use ssz::{hash, ssz_encode, Decodable, DecodeError, Encodable, SszStream, TreeHash}; +use ssz::{ssz_encode, Decodable, DecodeError, Encodable, SszStream}; use std::cmp::{Ord, Ordering}; use std::fmt; use std::hash::{Hash, Hasher}; diff --git a/eth2/types/src/slot_epoch_macros.rs b/eth2/types/src/slot_epoch_macros.rs index 300ad3f6f..4a48bba9f 100644 --- a/eth2/types/src/slot_epoch_macros.rs +++ b/eth2/types/src/slot_epoch_macros.rs @@ -206,11 +206,41 @@ macro_rules! impl_ssz { } } - impl TreeHash for $type { - fn hash_tree_root(&self) -> Vec { - let mut result: Vec = vec![]; - result.append(&mut self.0.hash_tree_root()); - hash(&result) + impl tree_hash::TreeHash for $type { + fn tree_hash_type() -> tree_hash::TreeHashType { + tree_hash::TreeHashType::Basic + } + + fn tree_hash_packed_encoding(&self) -> Vec { + ssz_encode(self) + } + + fn tree_hash_packing_factor() -> usize { + 32 / 8 + } + + fn tree_hash_root(&self) -> Vec { + int_to_bytes::int_to_bytes32(self.0) + } + } + + impl cached_tree_hash::CachedTreeHash for $type { + fn new_tree_hash_cache( + &self, + depth: usize, + ) -> Result { + self.0.new_tree_hash_cache(depth) + } + + fn tree_hash_cache_schema(&self, depth: usize) -> cached_tree_hash::BTreeSchema { + self.0.tree_hash_cache_schema(depth) + } + + fn update_tree_hash_cache( + &self, + cache: &mut cached_tree_hash::TreeHashCache, + ) -> Result<(), cached_tree_hash::Error> { + self.0.update_tree_hash_cache(cache) } } @@ -535,6 +565,7 @@ macro_rules! all_tests { math_between_tests!($type, $type); math_tests!($type); ssz_tests!($type); + cached_tree_hash_tests!($type); mod u64_tests { use super::*; diff --git a/eth2/types/src/slot_height.rs b/eth2/types/src/slot_height.rs index 4a783d4a0..f7a34cbba 100644 --- a/eth2/types/src/slot_height.rs +++ b/eth2/types/src/slot_height.rs @@ -2,7 +2,7 @@ use crate::slot_epoch::{Epoch, Slot}; use crate::test_utils::TestRandom; use rand::RngCore; use serde_derive::Serialize; -use ssz::{hash, ssz_encode, Decodable, DecodeError, Encodable, SszStream, TreeHash}; +use ssz::{ssz_encode, Decodable, DecodeError, Encodable, SszStream}; use std::cmp::{Ord, Ordering}; use std::fmt; use std::hash::{Hash, Hasher}; diff --git a/eth2/types/src/test_utils/macros.rs b/eth2/types/src/test_utils/macros.rs index d580fd818..71f462c1a 100644 --- a/eth2/types/src/test_utils/macros.rs +++ b/eth2/types/src/test_utils/macros.rs @@ -17,14 +17,14 @@ macro_rules! ssz_tests { } #[test] - pub fn test_hash_tree_root() { + pub fn test_tree_hash_root() { use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::TreeHash; + use tree_hash::TreeHash; let mut rng = XorShiftRng::from_seed([42; 16]); let original = $type::random_for_test(&mut rng); - let result = original.hash_tree_root(); + let result = original.tree_hash_root(); assert_eq!(result.len(), 32); // TODO: Add further tests @@ -32,3 +32,51 @@ macro_rules! ssz_tests { } }; } + +#[cfg(test)] +#[macro_export] +macro_rules! cached_tree_hash_tests { + ($type: ident) => { + #[test] + pub fn test_cached_tree_hash() { + use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; + use tree_hash::TreeHash; + + let mut rng = XorShiftRng::from_seed([42; 16]); + + // Test the original hash + let original = $type::random_for_test(&mut rng); + let mut cache = cached_tree_hash::TreeHashCache::new(&original).unwrap(); + + assert_eq!( + cache.tree_hash_root().unwrap().to_vec(), + original.tree_hash_root(), + "Original hash failed." + ); + + // Test the updated hash + let modified = $type::random_for_test(&mut rng); + cache.update(&modified).unwrap(); + assert_eq!( + cache.tree_hash_root().unwrap().to_vec(), + modified.tree_hash_root(), + "Modification hash failed" + ); + + // Produce a new cache for the modified object and compare it to the updated cache. + let mut modified_cache = cached_tree_hash::TreeHashCache::new(&modified).unwrap(); + + // Reset the caches. + cache.reset_modifications(); + modified_cache.reset_modifications(); + + // Ensure the modified cache is the same as a newly created cache. This is a sanity + // check to make sure there are no artifacts of the original cache remaining after an + // update. + assert_eq!( + modified_cache, cache, + "The modified cache does not match a new cache." + ) + } + }; +} diff --git a/eth2/types/src/test_utils/mod.rs b/eth2/types/src/test_utils/mod.rs index 7d129dc53..20d53e72e 100644 --- a/eth2/types/src/test_utils/mod.rs +++ b/eth2/types/src/test_utils/mod.rs @@ -18,7 +18,10 @@ mod testing_voluntary_exit_builder; pub use generate_deterministic_keypairs::generate_deterministic_keypair; pub use generate_deterministic_keypairs::generate_deterministic_keypairs; pub use keypairs_file::KeypairsFile; -pub use rand::{prng::XorShiftRng, SeedableRng}; +pub use rand::{ + RngCore, + {prng::XorShiftRng, SeedableRng}, +}; pub use serde_utils::{fork_from_hex_str, u8_from_hex_str}; pub use test_random::TestRandom; pub use testing_attestation_builder::TestingAttestationBuilder; diff --git a/eth2/types/src/test_utils/test_random.rs b/eth2/types/src/test_utils/test_random.rs index cb7abe3a4..2d4269b08 100644 --- a/eth2/types/src/test_utils/test_random.rs +++ b/eth2/types/src/test_utils/test_random.rs @@ -44,11 +44,13 @@ where U: TestRandom, { fn random_for_test(rng: &mut T) -> Self { - vec![ - ::random_for_test(rng), - ::random_for_test(rng), - ::random_for_test(rng), - ] + let mut output = vec![]; + + for _ in 0..(usize::random_for_test(rng) % 4) { + output.push(::random_for_test(rng)); + } + + output } } diff --git a/eth2/types/src/test_utils/testing_attestation_builder.rs b/eth2/types/src/test_utils/testing_attestation_builder.rs index 60624b48d..162facc8e 100644 --- a/eth2/types/src/test_utils/testing_attestation_builder.rs +++ b/eth2/types/src/test_utils/testing_attestation_builder.rs @@ -1,6 +1,6 @@ use crate::test_utils::TestingAttestationDataBuilder; use crate::*; -use ssz::TreeHash; +use tree_hash::TreeHash; /// Builds an attestation to be used for testing purposes. /// @@ -74,7 +74,7 @@ impl TestingAttestationBuilder { data: self.attestation.data.clone(), custody_bit: false, } - .hash_tree_root(); + .tree_hash_root(); let domain = spec.get_domain( self.attestation.data.slot.epoch(spec.slots_per_epoch), diff --git a/eth2/types/src/test_utils/testing_attester_slashing_builder.rs b/eth2/types/src/test_utils/testing_attester_slashing_builder.rs index fcaa3285b..dc01f7fb0 100644 --- a/eth2/types/src/test_utils/testing_attester_slashing_builder.rs +++ b/eth2/types/src/test_utils/testing_attester_slashing_builder.rs @@ -1,5 +1,5 @@ use crate::*; -use ssz::TreeHash; +use tree_hash::TreeHash; /// Builds an `AttesterSlashing`. /// @@ -66,7 +66,7 @@ impl TestingAttesterSlashingBuilder { data: attestation.data.clone(), custody_bit: false, }; - let message = attestation_data_and_custody_bit.hash_tree_root(); + let message = attestation_data_and_custody_bit.tree_hash_root(); for (i, validator_index) in validator_indices.iter().enumerate() { attestation.custody_bitfield.set(i, false); diff --git a/eth2/types/src/test_utils/testing_beacon_block_builder.rs b/eth2/types/src/test_utils/testing_beacon_block_builder.rs index c5cd22ed4..549c00ac0 100644 --- a/eth2/types/src/test_utils/testing_beacon_block_builder.rs +++ b/eth2/types/src/test_utils/testing_beacon_block_builder.rs @@ -6,7 +6,7 @@ use crate::{ *, }; use rayon::prelude::*; -use ssz::{SignedRoot, TreeHash}; +use tree_hash::{SignedRoot, TreeHash}; /// Builds a beacon block to be used for testing purposes. /// @@ -43,7 +43,7 @@ impl TestingBeaconBlockBuilder { /// Modifying the block's slot after signing may invalidate the signature. pub fn set_randao_reveal(&mut self, sk: &SecretKey, fork: &Fork, spec: &ChainSpec) { let epoch = self.block.slot.epoch(spec.slots_per_epoch); - let message = epoch.hash_tree_root(); + let message = epoch.tree_hash_root(); let domain = spec.get_domain(epoch, Domain::Randao, fork); self.block.body.randao_reveal = Signature::new(&message, domain, sk); } diff --git a/eth2/types/src/test_utils/testing_deposit_builder.rs b/eth2/types/src/test_utils/testing_deposit_builder.rs index 326858c31..080ed5cfb 100644 --- a/eth2/types/src/test_utils/testing_deposit_builder.rs +++ b/eth2/types/src/test_utils/testing_deposit_builder.rs @@ -12,7 +12,7 @@ impl TestingDepositBuilder { /// Instantiates a new builder. pub fn new(pubkey: PublicKey, amount: u64) -> Self { let deposit = Deposit { - proof: vec![], + proof: vec![].into(), index: 0, deposit_data: DepositData { amount, diff --git a/eth2/types/src/test_utils/testing_proposer_slashing_builder.rs b/eth2/types/src/test_utils/testing_proposer_slashing_builder.rs index 2cfebd915..03c257b2d 100644 --- a/eth2/types/src/test_utils/testing_proposer_slashing_builder.rs +++ b/eth2/types/src/test_utils/testing_proposer_slashing_builder.rs @@ -1,5 +1,5 @@ use crate::*; -use ssz::SignedRoot; +use tree_hash::SignedRoot; /// Builds a `ProposerSlashing`. /// diff --git a/eth2/types/src/test_utils/testing_transfer_builder.rs b/eth2/types/src/test_utils/testing_transfer_builder.rs index 354e29aa5..2680f7b66 100644 --- a/eth2/types/src/test_utils/testing_transfer_builder.rs +++ b/eth2/types/src/test_utils/testing_transfer_builder.rs @@ -1,5 +1,5 @@ use crate::*; -use ssz::SignedRoot; +use tree_hash::SignedRoot; /// Builds a transfer to be used for testing purposes. /// diff --git a/eth2/types/src/test_utils/testing_voluntary_exit_builder.rs b/eth2/types/src/test_utils/testing_voluntary_exit_builder.rs index fe5c8325a..8583bc451 100644 --- a/eth2/types/src/test_utils/testing_voluntary_exit_builder.rs +++ b/eth2/types/src/test_utils/testing_voluntary_exit_builder.rs @@ -1,5 +1,5 @@ use crate::*; -use ssz::SignedRoot; +use tree_hash::SignedRoot; /// Builds an exit to be used for testing purposes. /// diff --git a/eth2/types/src/transfer.rs b/eth2/types/src/transfer.rs index 4b10ce1ca..aea13fdd7 100644 --- a/eth2/types/src/transfer.rs +++ b/eth2/types/src/transfer.rs @@ -4,13 +4,14 @@ use bls::{PublicKey, Signature}; use derivative::Derivative; use rand::RngCore; use serde_derive::{Deserialize, Serialize}; -use ssz::TreeHash; -use ssz_derive::{Decode, Encode, SignedRoot, TreeHash}; +use ssz_derive::{Decode, Encode}; use test_random_derive::TestRandom; +use tree_hash::TreeHash; +use tree_hash_derive::{CachedTreeHash, SignedRoot, TreeHash}; /// The data submitted to the deposit contract. /// -/// Spec v0.5.0 +/// Spec v0.5.1 #[derive( Debug, Clone, @@ -19,6 +20,7 @@ use test_random_derive::TestRandom; Encode, Decode, TreeHash, + CachedTreeHash, TestRandom, SignedRoot, Derivative, @@ -32,6 +34,7 @@ pub struct Transfer { pub slot: Slot, pub pubkey: PublicKey, #[derivative(Hash = "ignore")] + #[signed_root(skip_hashing)] pub signature: Signature, } @@ -40,4 +43,5 @@ mod tests { use super::*; ssz_tests!(Transfer); + cached_tree_hash_tests!(Transfer); } diff --git a/eth2/types/src/tree_hash_vector.rs b/eth2/types/src/tree_hash_vector.rs new file mode 100644 index 000000000..42a730f25 --- /dev/null +++ b/eth2/types/src/tree_hash_vector.rs @@ -0,0 +1,142 @@ +use crate::test_utils::{RngCore, TestRandom}; +use cached_tree_hash::CachedTreeHash; +use serde_derive::{Deserialize, Serialize}; +use ssz::{Decodable, DecodeError, Encodable, SszStream}; +use std::ops::{Deref, DerefMut}; +use tree_hash::TreeHash; + +#[derive(Debug, PartialEq, Clone, Serialize, Deserialize)] +pub struct TreeHashVector(Vec); + +impl From> for TreeHashVector { + fn from(vec: Vec) -> TreeHashVector { + TreeHashVector(vec) + } +} + +impl Into> for TreeHashVector { + fn into(self) -> Vec { + self.0 + } +} + +impl Deref for TreeHashVector { + type Target = Vec; + + fn deref(&self) -> &Vec { + &self.0 + } +} + +impl DerefMut for TreeHashVector { + fn deref_mut(&mut self) -> &mut Vec { + &mut self.0 + } +} + +impl tree_hash::TreeHash for TreeHashVector +where + T: TreeHash, +{ + fn tree_hash_type() -> tree_hash::TreeHashType { + tree_hash::TreeHashType::Vector + } + + fn tree_hash_packed_encoding(&self) -> Vec { + unreachable!("Vector should never be packed.") + } + + fn tree_hash_packing_factor() -> usize { + unreachable!("Vector should never be packed.") + } + + fn tree_hash_root(&self) -> Vec { + tree_hash::impls::vec_tree_hash_root(self) + } +} + +impl CachedTreeHash for TreeHashVector +where + T: CachedTreeHash + TreeHash, +{ + fn new_tree_hash_cache( + &self, + depth: usize, + ) -> Result { + let (cache, _overlay) = cached_tree_hash::vec::new_tree_hash_cache(self, depth)?; + + Ok(cache) + } + + fn tree_hash_cache_schema(&self, depth: usize) -> cached_tree_hash::BTreeSchema { + cached_tree_hash::vec::produce_schema(self, depth) + } + + fn update_tree_hash_cache( + &self, + cache: &mut cached_tree_hash::TreeHashCache, + ) -> Result<(), cached_tree_hash::Error> { + cached_tree_hash::vec::update_tree_hash_cache(self, cache)?; + + Ok(()) + } +} + +impl Encodable for TreeHashVector +where + T: Encodable, +{ + fn ssz_append(&self, s: &mut SszStream) { + s.append_vec(self) + } +} + +impl Decodable for TreeHashVector +where + T: Decodable, +{ + fn ssz_decode(bytes: &[u8], index: usize) -> Result<(Self, usize), DecodeError> { + ssz::decode_ssz_list(bytes, index).and_then(|(vec, i)| Ok((vec.into(), i))) + } +} + +impl TestRandom for TreeHashVector +where + U: TestRandom, +{ + fn random_for_test(rng: &mut T) -> Self { + TreeHashVector::from(vec![ + U::random_for_test(rng), + U::random_for_test(rng), + U::random_for_test(rng), + ]) + } +} + +#[cfg(test)] +mod test { + use super::*; + use tree_hash::TreeHash; + + #[test] + pub fn test_cached_tree_hash() { + let original = TreeHashVector::from(vec![1_u64, 2, 3, 4]); + + let mut cache = cached_tree_hash::TreeHashCache::new(&original).unwrap(); + + assert_eq!( + cache.tree_hash_root().unwrap().to_vec(), + original.tree_hash_root() + ); + + let modified = TreeHashVector::from(vec![1_u64, 1, 1, 1]); + + cache.update(&modified).unwrap(); + + assert_eq!( + cache.tree_hash_root().unwrap().to_vec(), + modified.tree_hash_root() + ); + } + +} diff --git a/eth2/types/src/validator.rs b/eth2/types/src/validator.rs index f57261175..a20eb6426 100644 --- a/eth2/types/src/validator.rs +++ b/eth2/types/src/validator.rs @@ -1,13 +1,25 @@ use crate::{test_utils::TestRandom, Epoch, Hash256, PublicKey}; use rand::RngCore; use serde_derive::{Deserialize, Serialize}; -use ssz_derive::{Decode, Encode, TreeHash}; +use ssz_derive::{Decode, Encode}; use test_random_derive::TestRandom; +use tree_hash_derive::{CachedTreeHash, TreeHash}; /// Information about a `BeaconChain` validator. /// -/// Spec v0.5.0 -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Encode, Decode, TestRandom, TreeHash)] +/// Spec v0.5.1 +#[derive( + Debug, + Clone, + PartialEq, + Serialize, + Deserialize, + Encode, + Decode, + TestRandom, + TreeHash, + CachedTreeHash, +)] pub struct Validator { pub pubkey: PublicKey, pub withdrawal_credentials: Hash256, @@ -110,4 +122,5 @@ mod tests { } ssz_tests!(Validator); + cached_tree_hash_tests!(Validator); } diff --git a/eth2/types/src/voluntary_exit.rs b/eth2/types/src/voluntary_exit.rs index f64f950cb..8a780db75 100644 --- a/eth2/types/src/voluntary_exit.rs +++ b/eth2/types/src/voluntary_exit.rs @@ -2,13 +2,14 @@ use crate::{test_utils::TestRandom, Epoch}; use bls::Signature; use rand::RngCore; use serde_derive::{Deserialize, Serialize}; -use ssz::TreeHash; -use ssz_derive::{Decode, Encode, SignedRoot, TreeHash}; +use ssz_derive::{Decode, Encode}; use test_random_derive::TestRandom; +use tree_hash::TreeHash; +use tree_hash_derive::{CachedTreeHash, SignedRoot, TreeHash}; /// An exit voluntarily submitted a validator who wishes to withdraw. /// -/// Spec v0.5.0 +/// Spec v0.5.1 #[derive( Debug, PartialEq, @@ -18,12 +19,14 @@ use test_random_derive::TestRandom; Encode, Decode, TreeHash, + CachedTreeHash, TestRandom, SignedRoot, )] pub struct VoluntaryExit { pub epoch: Epoch, pub validator_index: u64, + #[signed_root(skip_hashing)] pub signature: Signature, } @@ -32,4 +35,5 @@ mod tests { use super::*; ssz_tests!(VoluntaryExit); + cached_tree_hash_tests!(VoluntaryExit); } diff --git a/eth2/utils/bls/Cargo.toml b/eth2/utils/bls/Cargo.toml index 4230a06ea..dcace15c8 100644 --- a/eth2/utils/bls/Cargo.toml +++ b/eth2/utils/bls/Cargo.toml @@ -6,9 +6,14 @@ edition = "2018" [dependencies] bls-aggregates = { git = "https://github.com/sigp/signature-schemes", tag = "0.6.1" } +cached_tree_hash = { path = "../cached_tree_hash" } hashing = { path = "../hashing" } hex = "0.3" serde = "1.0" serde_derive = "1.0" serde_hex = { path = "../serde_hex" } ssz = { path = "../ssz" } +tree_hash = { path = "../tree_hash" } + +[features] +fake_crypto = [] diff --git a/eth2/utils/bls/build.rs b/eth2/utils/bls/build.rs new file mode 100644 index 000000000..7f08a1ed5 --- /dev/null +++ b/eth2/utils/bls/build.rs @@ -0,0 +1,19 @@ +// This build script is symlinked from each project that requires BLS's "fake crypto", +// so that the `fake_crypto` feature of every sub-crate can be turned on by running +// with FAKE_CRYPTO=1 from the top-level workspace. +// At some point in the future it might be possible to do: +// $ cargo test --all --release --features fake_crypto +// but at the present time this doesn't work. +// Related: https://github.com/rust-lang/cargo/issues/5364 +fn main() { + if let Ok(fake_crypto) = std::env::var("FAKE_CRYPTO") { + if fake_crypto == "1" { + println!("cargo:rustc-cfg=feature=\"fake_crypto\""); + println!("cargo:rerun-if-env-changed=FAKE_CRYPTO"); + println!( + "cargo:warning=[{}]: Compiled with fake BLS cryptography. DO NOT USE, TESTING ONLY", + std::env::var("CARGO_PKG_NAME").unwrap() + ); + } + } +} diff --git a/eth2/utils/bls/src/aggregate_signature.rs b/eth2/utils/bls/src/aggregate_signature.rs index 8c7ae5222..e6c6cff9a 100644 --- a/eth2/utils/bls/src/aggregate_signature.rs +++ b/eth2/utils/bls/src/aggregate_signature.rs @@ -2,10 +2,12 @@ use super::{AggregatePublicKey, Signature, BLS_AGG_SIG_BYTE_SIZE}; use bls_aggregates::{ AggregatePublicKey as RawAggregatePublicKey, AggregateSignature as RawAggregateSignature, }; +use cached_tree_hash::cached_tree_hash_ssz_encoding_as_vector; use serde::de::{Deserialize, Deserializer}; use serde::ser::{Serialize, Serializer}; use serde_hex::{encode as hex_encode, HexVisitor}; -use ssz::{decode, hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; +use ssz::{decode, Decodable, DecodeError, Encodable, SszStream}; +use tree_hash::tree_hash_ssz_encoding_as_vector; /// A BLS aggregate signature. /// @@ -165,11 +167,8 @@ impl<'de> Deserialize<'de> for AggregateSignature { } } -impl TreeHash for AggregateSignature { - fn hash_tree_root(&self) -> Vec { - hash(&self.as_bytes()) - } -} +tree_hash_ssz_encoding_as_vector!(AggregateSignature); +cached_tree_hash_ssz_encoding_as_vector!(AggregateSignature, 96); #[cfg(test)] mod tests { diff --git a/eth2/utils/bls/src/fake_aggregate_signature.rs b/eth2/utils/bls/src/fake_aggregate_signature.rs index 3f0ec0d6d..aeb89507d 100644 --- a/eth2/utils/bls/src/fake_aggregate_signature.rs +++ b/eth2/utils/bls/src/fake_aggregate_signature.rs @@ -1,8 +1,10 @@ use super::{fake_signature::FakeSignature, AggregatePublicKey, BLS_AGG_SIG_BYTE_SIZE}; +use cached_tree_hash::cached_tree_hash_ssz_encoding_as_vector; use serde::de::{Deserialize, Deserializer}; use serde::ser::{Serialize, Serializer}; use serde_hex::{encode as hex_encode, PrefixedHexVisitor}; -use ssz::{hash, ssz_encode, Decodable, DecodeError, Encodable, SszStream, TreeHash}; +use ssz::{ssz_encode, Decodable, DecodeError, Encodable, SszStream}; +use tree_hash::tree_hash_ssz_encoding_as_vector; /// A BLS aggregate signature. /// @@ -98,11 +100,8 @@ impl<'de> Deserialize<'de> for FakeAggregateSignature { } } -impl TreeHash for FakeAggregateSignature { - fn hash_tree_root(&self) -> Vec { - hash(&self.bytes) - } -} +tree_hash_ssz_encoding_as_vector!(FakeAggregateSignature); +cached_tree_hash_ssz_encoding_as_vector!(FakeAggregateSignature, 96); #[cfg(test)] mod tests { diff --git a/eth2/utils/bls/src/fake_signature.rs b/eth2/utils/bls/src/fake_signature.rs index 3c9f3a9f4..8a333b9c0 100644 --- a/eth2/utils/bls/src/fake_signature.rs +++ b/eth2/utils/bls/src/fake_signature.rs @@ -1,9 +1,11 @@ use super::{PublicKey, SecretKey, BLS_SIG_BYTE_SIZE}; +use cached_tree_hash::cached_tree_hash_ssz_encoding_as_vector; use hex::encode as hex_encode; use serde::de::{Deserialize, Deserializer}; use serde::ser::{Serialize, Serializer}; use serde_hex::HexVisitor; -use ssz::{hash, ssz_encode, Decodable, DecodeError, Encodable, SszStream, TreeHash}; +use ssz::{ssz_encode, Decodable, DecodeError, Encodable, SszStream}; +use tree_hash::tree_hash_ssz_encoding_as_vector; /// A single BLS signature. /// @@ -73,11 +75,8 @@ impl Decodable for FakeSignature { } } -impl TreeHash for FakeSignature { - fn hash_tree_root(&self) -> Vec { - hash(&self.bytes) - } -} +tree_hash_ssz_encoding_as_vector!(FakeSignature); +cached_tree_hash_ssz_encoding_as_vector!(FakeSignature, 96); impl Serialize for FakeSignature { fn serialize(&self, serializer: S) -> Result diff --git a/eth2/utils/bls/src/lib.rs b/eth2/utils/bls/src/lib.rs index b9a4d5c1d..fae41aeed 100644 --- a/eth2/utils/bls/src/lib.rs +++ b/eth2/utils/bls/src/lib.rs @@ -6,22 +6,22 @@ mod keypair; mod public_key; mod secret_key; -#[cfg(not(debug_assertions))] +#[cfg(not(feature = "fake_crypto"))] mod aggregate_signature; -#[cfg(not(debug_assertions))] +#[cfg(not(feature = "fake_crypto"))] mod signature; -#[cfg(not(debug_assertions))] +#[cfg(not(feature = "fake_crypto"))] pub use crate::aggregate_signature::AggregateSignature; -#[cfg(not(debug_assertions))] +#[cfg(not(feature = "fake_crypto"))] pub use crate::signature::Signature; -#[cfg(debug_assertions)] +#[cfg(feature = "fake_crypto")] mod fake_aggregate_signature; -#[cfg(debug_assertions)] +#[cfg(feature = "fake_crypto")] mod fake_signature; -#[cfg(debug_assertions)] +#[cfg(feature = "fake_crypto")] pub use crate::fake_aggregate_signature::FakeAggregateSignature as AggregateSignature; -#[cfg(debug_assertions)] +#[cfg(feature = "fake_crypto")] pub use crate::fake_signature::FakeSignature as Signature; pub use crate::aggregate_public_key::AggregatePublicKey; diff --git a/eth2/utils/bls/src/public_key.rs b/eth2/utils/bls/src/public_key.rs index 177a735c4..41b87d383 100644 --- a/eth2/utils/bls/src/public_key.rs +++ b/eth2/utils/bls/src/public_key.rs @@ -1,12 +1,14 @@ use super::{SecretKey, BLS_PUBLIC_KEY_BYTE_SIZE}; use bls_aggregates::PublicKey as RawPublicKey; +use cached_tree_hash::cached_tree_hash_ssz_encoding_as_vector; use serde::de::{Deserialize, Deserializer}; use serde::ser::{Serialize, Serializer}; use serde_hex::{encode as hex_encode, HexVisitor}; -use ssz::{decode, hash, ssz_encode, Decodable, DecodeError, Encodable, SszStream, TreeHash}; +use ssz::{decode, ssz_encode, Decodable, DecodeError, Encodable, SszStream}; use std::default; use std::fmt; use std::hash::{Hash, Hasher}; +use tree_hash::tree_hash_ssz_encoding_as_vector; /// A single BLS signature. /// @@ -104,11 +106,8 @@ impl<'de> Deserialize<'de> for PublicKey { } } -impl TreeHash for PublicKey { - fn hash_tree_root(&self) -> Vec { - hash(&self.0.as_bytes()) - } -} +tree_hash_ssz_encoding_as_vector!(PublicKey); +cached_tree_hash_ssz_encoding_as_vector!(PublicKey, 48); impl PartialEq for PublicKey { fn eq(&self, other: &PublicKey) -> bool { @@ -132,6 +131,7 @@ impl Hash for PublicKey { mod tests { use super::*; use ssz::ssz_encode; + use tree_hash::TreeHash; #[test] pub fn test_ssz_round_trip() { @@ -143,4 +143,27 @@ mod tests { assert_eq!(original, decoded); } + + #[test] + pub fn test_cached_tree_hash() { + let sk = SecretKey::random(); + let original = PublicKey::from_secret_key(&sk); + + let mut cache = cached_tree_hash::TreeHashCache::new(&original).unwrap(); + + assert_eq!( + cache.tree_hash_root().unwrap().to_vec(), + original.tree_hash_root() + ); + + let sk = SecretKey::random(); + let modified = PublicKey::from_secret_key(&sk); + + cache.update(&modified).unwrap(); + + assert_eq!( + cache.tree_hash_root().unwrap().to_vec(), + modified.tree_hash_root() + ); + } } diff --git a/eth2/utils/bls/src/secret_key.rs b/eth2/utils/bls/src/secret_key.rs index 40c469513..d1aaa96da 100644 --- a/eth2/utils/bls/src/secret_key.rs +++ b/eth2/utils/bls/src/secret_key.rs @@ -4,7 +4,8 @@ use hex::encode as hex_encode; use serde::de::{Deserialize, Deserializer}; use serde::ser::{Serialize, Serializer}; use serde_hex::HexVisitor; -use ssz::{decode, ssz_encode, Decodable, DecodeError, Encodable, SszStream, TreeHash}; +use ssz::{decode, ssz_encode, Decodable, DecodeError, Encodable, SszStream}; +use tree_hash::tree_hash_ssz_encoding_as_vector; /// A single BLS signature. /// @@ -69,11 +70,7 @@ impl<'de> Deserialize<'de> for SecretKey { } } -impl TreeHash for SecretKey { - fn hash_tree_root(&self) -> Vec { - self.0.as_bytes().clone() - } -} +tree_hash_ssz_encoding_as_vector!(SecretKey); #[cfg(test)] mod tests { diff --git a/eth2/utils/bls/src/signature.rs b/eth2/utils/bls/src/signature.rs index d19af545f..e2dbd9c27 100644 --- a/eth2/utils/bls/src/signature.rs +++ b/eth2/utils/bls/src/signature.rs @@ -1,10 +1,12 @@ use super::{PublicKey, SecretKey, BLS_SIG_BYTE_SIZE}; use bls_aggregates::Signature as RawSignature; +use cached_tree_hash::cached_tree_hash_ssz_encoding_as_vector; use hex::encode as hex_encode; use serde::de::{Deserialize, Deserializer}; use serde::ser::{Serialize, Serializer}; use serde_hex::HexVisitor; -use ssz::{decode, hash, ssz_encode, Decodable, DecodeError, Encodable, SszStream, TreeHash}; +use ssz::{decode, ssz_encode, Decodable, DecodeError, Encodable, SszStream}; +use tree_hash::tree_hash_ssz_encoding_as_vector; /// A single BLS signature. /// @@ -114,11 +116,8 @@ impl Decodable for Signature { } } -impl TreeHash for Signature { - fn hash_tree_root(&self) -> Vec { - hash(&self.as_bytes()) - } -} +tree_hash_ssz_encoding_as_vector!(Signature); +cached_tree_hash_ssz_encoding_as_vector!(Signature, 96); impl Serialize for Signature { /// Serde serialization is compliant the Ethereum YAML test format. @@ -148,6 +147,7 @@ mod tests { use super::super::Keypair; use super::*; use ssz::ssz_encode; + use tree_hash::TreeHash; #[test] pub fn test_ssz_round_trip() { @@ -161,6 +161,28 @@ mod tests { assert_eq!(original, decoded); } + #[test] + pub fn test_cached_tree_hash() { + let keypair = Keypair::random(); + let original = Signature::new(&[42, 42], 0, &keypair.sk); + + let mut cache = cached_tree_hash::TreeHashCache::new(&original).unwrap(); + + assert_eq!( + cache.tree_hash_root().unwrap().to_vec(), + original.tree_hash_root() + ); + + let modified = Signature::new(&[99, 99], 0, &keypair.sk); + + cache.update(&modified).unwrap(); + + assert_eq!( + cache.tree_hash_root().unwrap().to_vec(), + modified.tree_hash_root() + ); + } + #[test] pub fn test_empty_signature() { let sig = Signature::empty_signature(); diff --git a/eth2/utils/boolean-bitfield/Cargo.toml b/eth2/utils/boolean-bitfield/Cargo.toml index cf037c5d7..dfc97ce77 100644 --- a/eth2/utils/boolean-bitfield/Cargo.toml +++ b/eth2/utils/boolean-bitfield/Cargo.toml @@ -5,8 +5,14 @@ authors = ["Paul Hauner "] edition = "2018" [dependencies] +cached_tree_hash = { path = "../cached_tree_hash" } serde_hex = { path = "../serde_hex" } ssz = { path = "../ssz" } bit-vec = "0.5.0" +bit_reverse = "0.1" serde = "1.0" serde_derive = "1.0" +tree_hash = { path = "../tree_hash" } + +[dev-dependencies] +serde_yaml = "0.8" diff --git a/eth2/utils/boolean-bitfield/src/lib.rs b/eth2/utils/boolean-bitfield/src/lib.rs index d04516dba..d49da0d10 100644 --- a/eth2/utils/boolean-bitfield/src/lib.rs +++ b/eth2/utils/boolean-bitfield/src/lib.rs @@ -1,8 +1,9 @@ extern crate bit_vec; extern crate ssz; +use bit_reverse::LookupReverse; use bit_vec::BitVec; - +use cached_tree_hash::cached_tree_hash_bytes_as_list; use serde::de::{Deserialize, Deserializer}; use serde::ser::{Serialize, Serializer}; use serde_hex::{encode, PrefixedHexVisitor}; @@ -53,10 +54,15 @@ impl BooleanBitfield { /// Create a new bitfield using the supplied `bytes` as input pub fn from_bytes(bytes: &[u8]) -> Self { Self { - 0: BitVec::from_bytes(bytes), + 0: BitVec::from_bytes(&reverse_bit_order(bytes.to_vec())), } } + /// Returns a vector of bytes representing the bitfield + pub fn to_bytes(&self) -> Vec { + reverse_bit_order(self.0.to_bytes().to_vec()) + } + /// Read the value of a bit. /// /// If the index is in bounds, then result is Ok(value) where value is `true` if the bit is 1 and `false` if the bit is 0. @@ -85,11 +91,6 @@ impl BooleanBitfield { previous } - /// Returns the index of the highest set bit. Some(n) if some bit is set, None otherwise. - pub fn highest_set_bit(&self) -> Option { - self.0.iter().rposition(|bit| bit) - } - /// Returns the number of bits in this bitfield. pub fn len(&self) -> usize { self.0.len() @@ -115,12 +116,6 @@ impl BooleanBitfield { self.0.iter().filter(|&bit| bit).count() } - /// Returns a vector of bytes representing the bitfield - /// Note that this returns the bit layout of the underlying implementation in the `bit-vec` crate. - pub fn to_bytes(&self) -> Vec { - self.0.to_bytes() - } - /// Compute the intersection (binary-and) of this bitfield with another. Lengths must match. pub fn intersection(&self, other: &Self) -> Self { let mut res = self.clone(); @@ -217,17 +212,7 @@ impl Decodable for BooleanBitfield { Ok((BooleanBitfield::new(), index + ssz::LENGTH_BYTES)) } else { let bytes = &bytes[(index + 4)..(index + len + 4)]; - - let count = len * 8; - let mut field = BooleanBitfield::with_capacity(count); - for (byte_index, byte) in bytes.iter().enumerate() { - for i in 0..8 { - let bit = byte & (128 >> i); - if bit != 0 { - field.set(8 * byte_index + i, true); - } - } - } + let field = BooleanBitfield::from_bytes(bytes); let index = index + ssz::LENGTH_BYTES + len; Ok((field, index)) @@ -235,37 +220,86 @@ impl Decodable for BooleanBitfield { } } +// Reverse the bit order of a whole byte vec, so that the ith bit +// of the input vec is placed in the (N - i)th bit of the output vec. +// This function is necessary for converting bitfields to and from YAML, +// as the BitVec library and the hex-parser use opposing bit orders. +fn reverse_bit_order(mut bytes: Vec) -> Vec { + bytes.reverse(); + bytes.into_iter().map(|b| b.swap_bits()).collect() +} + impl Serialize for BooleanBitfield { - /// Serde serialization is compliant the Ethereum YAML test format. + /// Serde serialization is compliant with the Ethereum YAML test format. fn serialize(&self, serializer: S) -> Result where S: Serializer, { - serializer.serialize_str(&encode(&self.to_bytes())) + serializer.serialize_str(&encode(self.to_bytes())) } } impl<'de> Deserialize<'de> for BooleanBitfield { - /// Serde serialization is compliant the Ethereum YAML test format. + /// Serde serialization is compliant with the Ethereum YAML test format. fn deserialize(deserializer: D) -> Result where D: Deserializer<'de>, { + // We reverse the bit-order so that the BitVec library can read its 0th + // bit from the end of the hex string, e.g. + // "0xef01" => [0xef, 0x01] => [0b1000_0000, 0b1111_1110] let bytes = deserializer.deserialize_str(PrefixedHexVisitor)?; Ok(BooleanBitfield::from_bytes(&bytes)) } } -impl ssz::TreeHash for BooleanBitfield { - fn hash_tree_root(&self) -> Vec { - self.to_bytes().hash_tree_root() +impl tree_hash::TreeHash for BooleanBitfield { + fn tree_hash_type() -> tree_hash::TreeHashType { + tree_hash::TreeHashType::List + } + + fn tree_hash_packed_encoding(&self) -> Vec { + unreachable!("List should never be packed.") + } + + fn tree_hash_packing_factor() -> usize { + unreachable!("List should never be packed.") + } + + fn tree_hash_root(&self) -> Vec { + self.to_bytes().tree_hash_root() } } +cached_tree_hash_bytes_as_list!(BooleanBitfield); + #[cfg(test)] mod tests { use super::*; + use serde_yaml; use ssz::{decode, ssz_encode, SszStream}; + use tree_hash::TreeHash; + + #[test] + pub fn test_cached_tree_hash() { + let original = BooleanBitfield::from_bytes(&vec![18; 12][..]); + + let mut cache = cached_tree_hash::TreeHashCache::new(&original).unwrap(); + + assert_eq!( + cache.tree_hash_root().unwrap().to_vec(), + original.tree_hash_root() + ); + + let modified = BooleanBitfield::from_bytes(&vec![2; 1][..]); + + cache.update(&modified).unwrap(); + + assert_eq!( + cache.tree_hash_root().unwrap().to_vec(), + modified.tree_hash_root() + ); + } #[test] fn test_new_bitfield() { @@ -312,7 +346,7 @@ mod tests { assert_eq!(field.num_set_bits(), 100); } - const INPUT: &[u8] = &[0b0000_0010, 0b0000_0010]; + const INPUT: &[u8] = &[0b0100_0000, 0b0100_0000]; #[test] fn test_get_from_bitfield() { @@ -338,18 +372,6 @@ mod tests { assert!(!previous); } - #[test] - fn test_highest_set_bit() { - let field = BooleanBitfield::from_bytes(INPUT); - assert_eq!(field.highest_set_bit().unwrap(), 14); - - let field = BooleanBitfield::from_bytes(&[0b0000_0011]); - assert_eq!(field.highest_set_bit().unwrap(), 7); - - let field = BooleanBitfield::new(); - assert_eq!(field.highest_set_bit(), None); - } - #[test] fn test_len() { let field = BooleanBitfield::from_bytes(INPUT); @@ -430,15 +452,30 @@ mod tests { #[test] fn test_ssz_encode() { let field = create_test_bitfield(); - let mut stream = SszStream::new(); stream.append(&field); - assert_eq!(stream.drain(), vec![2, 0, 0, 0, 225, 192]); + assert_eq!(stream.drain(), vec![2, 0, 0, 0, 0b0000_0011, 0b1000_0111]); let field = BooleanBitfield::from_elem(18, true); let mut stream = SszStream::new(); stream.append(&field); - assert_eq!(stream.drain(), vec![3, 0, 0, 0, 255, 255, 192]); + assert_eq!( + stream.drain(), + vec![3, 0, 0, 0, 0b0000_0011, 0b1111_1111, 0b1111_1111] + ); + + let mut b = BooleanBitfield::new(); + b.set(1, true); + assert_eq!( + ssz_encode(&b), + vec![ + 0b0000_0001, + 0b0000_0000, + 0b0000_0000, + 0b0000_0000, + 0b0000_0010 + ] + ); } fn create_test_bitfield() -> BooleanBitfield { @@ -454,7 +491,7 @@ mod tests { #[test] fn test_ssz_decode() { - let encoded = vec![2, 0, 0, 0, 225, 192]; + let encoded = vec![2, 0, 0, 0, 0b0000_0011, 0b1000_0111]; let field = decode::(&encoded).unwrap(); let expected = create_test_bitfield(); assert_eq!(field, expected); @@ -465,6 +502,27 @@ mod tests { assert_eq!(field, expected); } + #[test] + fn test_serialize_deserialize() { + use serde_yaml::Value; + + let data: &[(_, &[_])] = &[ + ("0x01", &[0b00000001]), + ("0xf301", &[0b11110011, 0b00000001]), + ]; + for (hex_data, bytes) in data { + let bitfield = BooleanBitfield::from_bytes(bytes); + assert_eq!( + serde_yaml::from_str::(hex_data).unwrap(), + bitfield + ); + assert_eq!( + serde_yaml::to_value(&bitfield).unwrap(), + Value::String(hex_data.to_string()) + ); + } + } + #[test] fn test_ssz_round_trip() { let original = BooleanBitfield::from_bytes(&vec![18; 12][..]); diff --git a/eth2/utils/cached_tree_hash/Cargo.toml b/eth2/utils/cached_tree_hash/Cargo.toml new file mode 100644 index 000000000..7b331ad68 --- /dev/null +++ b/eth2/utils/cached_tree_hash/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "cached_tree_hash" +version = "0.1.0" +authors = ["Paul Hauner "] +edition = "2018" + +[[bench]] +name = "benches" +harness = false + +[dev-dependencies] +criterion = "0.2" +tree_hash_derive = { path = "../tree_hash_derive" } + +[dependencies] +tree_hash = { path = "../tree_hash" } +ethereum-types = "0.5" +hashing = { path = "../hashing" } +int_to_bytes = { path = "../int_to_bytes" } diff --git a/eth2/utils/cached_tree_hash/README.md b/eth2/utils/cached_tree_hash/README.md new file mode 100644 index 000000000..0498bfc3e --- /dev/null +++ b/eth2/utils/cached_tree_hash/README.md @@ -0,0 +1,76 @@ +# Tree hashing + +Provides both cached and non-cached tree hashing methods. + +## Standard Tree Hash + +```rust +use tree_hash_derive::TreeHash; + +#[derive(TreeHash)] +struct Foo { + a: u64, + b: Vec, +} + +fn main() { + let foo = Foo { + a: 42, + b: vec![1, 2, 3] + }; + + println!("root: {}", foo.tree_hash_root()); +} +``` + +## Cached Tree Hash + + +```rust +use tree_hash_derive::{TreeHash, CachedTreeHash}; + +#[derive(TreeHash, CachedTreeHash)] +struct Foo { + a: u64, + b: Vec, +} + +#[derive(TreeHash, CachedTreeHash)] +struct Bar { + a: Vec, + b: u64, +} + +fn main() { + let bar = Bar { + a: vec![ + Foo { + a: 42, + b: vec![1, 2, 3] + } + ], + b: 42 + }; + + let modified_bar = Bar { + a: vec![ + Foo { + a: 100, + b: vec![1, 2, 3, 4, 5, 6] + }, + Foo { + a: 42, + b: vec![] + } + ], + b: 99 + }; + + + let mut hasher = CachedTreeHasher::new(&bar).unwrap(); + hasher.update(&modified_bar).unwrap(); + + // Assert that the cached tree hash matches a standard tree hash. + assert_eq!(hasher.tree_hash_root(), modified_bar.tree_hash_root()); +} +``` diff --git a/eth2/utils/cached_tree_hash/benches/benches.rs b/eth2/utils/cached_tree_hash/benches/benches.rs new file mode 100644 index 000000000..be7e26bb5 --- /dev/null +++ b/eth2/utils/cached_tree_hash/benches/benches.rs @@ -0,0 +1,73 @@ +#[macro_use] +extern crate criterion; + +use cached_tree_hash::TreeHashCache; +use criterion::black_box; +use criterion::{Benchmark, Criterion}; +use ethereum_types::H256 as Hash256; +use hashing::hash; +use tree_hash::TreeHash; + +fn criterion_benchmark(c: &mut Criterion) { + let n = 1024; + + let source_vec: Vec = (0..n).map(|_| Hash256::random()).collect(); + + let mut source_modified_vec = source_vec.clone(); + source_modified_vec[n - 1] = Hash256::random(); + + let modified_vec = source_modified_vec.clone(); + c.bench( + &format!("vec_of_{}_hashes", n), + Benchmark::new("standard", move |b| { + b.iter_with_setup( + || modified_vec.clone(), + |modified_vec| black_box(modified_vec.tree_hash_root()), + ) + }) + .sample_size(100), + ); + + let modified_vec = source_modified_vec.clone(); + c.bench( + &format!("vec_of_{}_hashes", n), + Benchmark::new("build_cache", move |b| { + b.iter_with_setup( + || modified_vec.clone(), + |vec| black_box(TreeHashCache::new(&vec, 0)), + ) + }) + .sample_size(100), + ); + + let vec = source_vec.clone(); + let modified_vec = source_modified_vec.clone(); + c.bench( + &format!("vec_of_{}_hashes", n), + Benchmark::new("cache_update", move |b| { + b.iter_with_setup( + || { + let cache = TreeHashCache::new(&vec, 0).unwrap(); + (cache, modified_vec.clone()) + }, + |(mut cache, modified_vec)| black_box(cache.update(&modified_vec)), + ) + }) + .sample_size(100), + ); + + c.bench( + &format!("{}_hashes", n), + Benchmark::new("hash_64_bytes", move |b| { + b.iter(|| { + for _ in 0..n { + let _digest = hash(&[42; 64]); + } + }) + }) + .sample_size(100), + ); +} + +criterion_group!(benches, criterion_benchmark); +criterion_main!(benches); diff --git a/eth2/utils/cached_tree_hash/examples/8k_hashes_cached.rs b/eth2/utils/cached_tree_hash/examples/8k_hashes_cached.rs new file mode 100644 index 000000000..1e67571d5 --- /dev/null +++ b/eth2/utils/cached_tree_hash/examples/8k_hashes_cached.rs @@ -0,0 +1,21 @@ +use cached_tree_hash::TreeHashCache; +use ethereum_types::H256 as Hash256; + +fn run(vec: &Vec, modified_vec: &Vec) { + let mut cache = TreeHashCache::new(vec).unwrap(); + + cache.update(modified_vec).unwrap(); +} + +fn main() { + let n = 2048; + + let vec: Vec = (0..n).map(|_| Hash256::random()).collect(); + + let mut modified_vec = vec.clone(); + modified_vec[n - 1] = Hash256::random(); + + for _ in 0..10_000 { + run(&vec, &modified_vec); + } +} diff --git a/eth2/utils/cached_tree_hash/examples/8k_hashes_standard.rs b/eth2/utils/cached_tree_hash/examples/8k_hashes_standard.rs new file mode 100644 index 000000000..bcbb392e2 --- /dev/null +++ b/eth2/utils/cached_tree_hash/examples/8k_hashes_standard.rs @@ -0,0 +1,10 @@ +use ethereum_types::H256 as Hash256; +use tree_hash::TreeHash; + +fn main() { + let n = 2048; + + let vec: Vec = (0..n).map(|_| Hash256::random()).collect(); + + vec.tree_hash_root(); +} diff --git a/eth2/utils/cached_tree_hash/src/btree_overlay.rs b/eth2/utils/cached_tree_hash/src/btree_overlay.rs new file mode 100644 index 000000000..a96df769c --- /dev/null +++ b/eth2/utils/cached_tree_hash/src/btree_overlay.rs @@ -0,0 +1,340 @@ +use super::*; + +/// A schema defining a binary tree over a `TreeHashCache`. +/// +/// This structure is used for succinct storage, run-time functionality is gained by converting the +/// schema into a `BTreeOverlay`. +#[derive(Debug, PartialEq, Clone)] +pub struct BTreeSchema { + /// The depth of a schema defines how far it is nested within other fixed-length items. + /// + /// Each time a new variable-length object is created all items within it are assigned a depth + /// of `depth + 1`. + /// + /// When storing the schemas in a list, the depth parameter allows for removing all schemas + /// belonging to a specific variable-length item without removing schemas related to adjacent + /// variable-length items. + pub depth: usize, + lengths: Vec, +} + +impl BTreeSchema { + pub fn from_lengths(depth: usize, lengths: Vec) -> Self { + Self { depth, lengths } + } + + pub fn into_overlay(self, offset: usize) -> BTreeOverlay { + BTreeOverlay::from_schema(self, offset) + } +} + +impl Into for BTreeOverlay { + fn into(self) -> BTreeSchema { + BTreeSchema { + depth: self.depth, + lengths: self.lengths, + } + } +} + +/// Provides a status for some leaf-node in binary tree. +#[derive(Debug, PartialEq, Clone)] +pub enum LeafNode { + /// The leaf node does not exist in this tree. + DoesNotExist, + /// The leaf node exists in the tree and has a real value within the given `chunk` range. + Exists(Range), + /// The leaf node exists in the tree only as padding. + Padding, +} + +/// Instantiated from a `BTreeSchema`, allows for interpreting some chunks of a `TreeHashCache` as +/// a perfect binary tree. +/// +/// The primary purpose of this struct is to map from binary tree "nodes" to `TreeHashCache` +/// "chunks". Each tree has nodes `0..n` where `n` is the number of nodes and `0` is the root node. +/// Each of these nodes is mapped to a chunk, starting from `self.offset` and increasing in steps +/// of `1` for internal nodes and arbitrary steps for leaf-nodes. +#[derive(Debug, PartialEq, Clone)] +pub struct BTreeOverlay { + offset: usize, + /// See `BTreeSchema.depth` for a description. + pub depth: usize, + lengths: Vec, +} + +impl BTreeOverlay { + /// Instantiates a new instance for `item`, where it's first chunk is `inital_offset` and has + /// the specified `depth`. + pub fn new(item: &T, initial_offset: usize, depth: usize) -> Self + where + T: CachedTreeHash, + { + Self::from_schema(item.tree_hash_cache_schema(depth), initial_offset) + } + + /// Instantiates a new instance from a schema, where it's first chunk is `offset`. + pub fn from_schema(schema: BTreeSchema, offset: usize) -> Self { + Self { + offset, + depth: schema.depth, + lengths: schema.lengths, + } + } + + /// Returns a `LeafNode` for each of the `n` leaves of the tree. + /// + /// `LeafNode::DoesNotExist` is returned for each element `i` in `0..n` where `i >= + /// self.num_leaf_nodes()`. + pub fn get_leaf_nodes(&self, n: usize) -> Vec { + let mut running_offset = self.offset + self.num_internal_nodes(); + + let mut leaf_nodes: Vec = self + .lengths + .iter() + .map(|length| { + let range = running_offset..running_offset + length; + running_offset += length; + LeafNode::Exists(range) + }) + .collect(); + + leaf_nodes.resize(self.num_leaf_nodes(), LeafNode::Padding); + leaf_nodes.resize(n, LeafNode::DoesNotExist); + + leaf_nodes + } + + /// Returns the number of leaf nodes in the tree. + pub fn num_leaf_nodes(&self) -> usize { + self.lengths.len().next_power_of_two() + } + + /// Returns the number of leafs in the tree which are padding. + pub fn num_padding_leaves(&self) -> usize { + self.num_leaf_nodes() - self.lengths.len() + } + + /// Returns the number of nodes in the tree. + /// + /// Note: this is distinct from `num_chunks`, which returns the total number of chunks in + /// this tree. + pub fn num_nodes(&self) -> usize { + 2 * self.num_leaf_nodes() - 1 + } + + /// Returns the number of internal (non-leaf) nodes in the tree. + pub fn num_internal_nodes(&self) -> usize { + self.num_leaf_nodes() - 1 + } + + /// Returns the chunk of the first node of the tree. + fn first_node(&self) -> usize { + self.offset + } + + /// Returns the root chunk of the tree (the zero-th node) + pub fn root(&self) -> usize { + self.first_node() + } + + /// Returns the first chunk outside of the boundary of this tree. It is the root node chunk + /// plus the total number of chunks in the tree. + pub fn next_node(&self) -> usize { + self.first_node() + self.num_internal_nodes() + self.num_leaf_nodes() - self.lengths.len() + + self.lengths.iter().sum::() + } + + /// Returns the height of the tree where a tree with a single node has a height of 1. + pub fn height(&self) -> usize { + self.num_leaf_nodes().trailing_zeros() as usize + } + + /// Returns the range of chunks that belong to the internal nodes of the tree. + pub fn internal_chunk_range(&self) -> Range { + self.offset..self.offset + self.num_internal_nodes() + } + + /// Returns all of the chunks that are encompassed by the tree. + pub fn chunk_range(&self) -> Range { + self.first_node()..self.next_node() + } + + /// Returns the number of chunks inside this tree (including subtrees). + /// + /// Note: this is distinct from `num_nodes` which returns the number of nodes in the binary + /// tree. + pub fn num_chunks(&self) -> usize { + self.next_node() - self.first_node() + } + + /// Returns the first chunk of the first leaf node in the tree. + pub fn first_leaf_node(&self) -> usize { + self.offset + self.num_internal_nodes() + } + + /// Returns the chunks for some given parent node. + /// + /// Note: it is a parent _node_ not a parent _chunk_. + pub fn child_chunks(&self, parent: usize) -> (usize, usize) { + let children = children(parent); + + if children.1 < self.num_internal_nodes() { + (children.0 + self.offset, children.1 + self.offset) + } else { + let chunks = self.n_leaf_node_chunks(children.1); + (chunks[chunks.len() - 2], chunks[chunks.len() - 1]) + } + } + + /// Returns a vec of (parent_chunk, (left_child_chunk, right_child_chunk)). + pub fn internal_parents_and_children(&self) -> Vec<(usize, (usize, usize))> { + let mut chunks = Vec::with_capacity(self.num_nodes()); + chunks.append(&mut self.internal_node_chunks()); + chunks.append(&mut self.leaf_node_chunks()); + + (0..self.num_internal_nodes()) + .map(|parent| { + let children = children(parent); + (chunks[parent], (chunks[children.0], chunks[children.1])) + }) + .collect() + } + + /// Returns a vec of chunk indices for each internal node of the tree. + pub fn internal_node_chunks(&self) -> Vec { + (self.offset..self.offset + self.num_internal_nodes()).collect() + } + + /// Returns a vec of the first chunk for each leaf node of the tree. + pub fn leaf_node_chunks(&self) -> Vec { + self.n_leaf_node_chunks(self.num_leaf_nodes()) + } + + /// Returns a vec of the first chunk index for the first `n` leaf nodes of the tree. + fn n_leaf_node_chunks(&self, n: usize) -> Vec { + let mut chunks = Vec::with_capacity(n); + + let mut chunk = self.offset + self.num_internal_nodes(); + for i in 0..n { + chunks.push(chunk); + + match self.lengths.get(i) { + Some(len) => { + chunk += len; + } + None => chunk += 1, + } + } + + chunks + } +} + +fn children(parent: usize) -> (usize, usize) { + ((2 * parent + 1), (2 * parent + 2)) +} + +#[cfg(test)] +mod test { + use super::*; + + fn get_tree_a(n: usize) -> BTreeOverlay { + BTreeSchema::from_lengths(0, vec![1; n]).into_overlay(0) + } + + #[test] + fn leaf_node_chunks() { + let tree = get_tree_a(4); + + assert_eq!(tree.leaf_node_chunks(), vec![3, 4, 5, 6]) + } + + #[test] + fn internal_node_chunks() { + let tree = get_tree_a(4); + + assert_eq!(tree.internal_node_chunks(), vec![0, 1, 2]) + } + + #[test] + fn internal_parents_and_children() { + let tree = get_tree_a(4); + + assert_eq!( + tree.internal_parents_and_children(), + vec![(0, (1, 2)), (1, (3, 4)), (2, (5, 6))] + ) + } + + #[test] + fn chunk_range() { + let tree = get_tree_a(4); + assert_eq!(tree.chunk_range(), 0..7); + + let tree = get_tree_a(1); + assert_eq!(tree.chunk_range(), 0..1); + + let tree = get_tree_a(2); + assert_eq!(tree.chunk_range(), 0..3); + + let tree = BTreeSchema::from_lengths(0, vec![1, 1]).into_overlay(11); + assert_eq!(tree.chunk_range(), 11..14); + + let tree = BTreeSchema::from_lengths(0, vec![7, 7, 7]).into_overlay(0); + assert_eq!(tree.chunk_range(), 0..25); + } + + #[test] + fn get_leaf_node() { + let tree = get_tree_a(4); + let leaves = tree.get_leaf_nodes(5); + + assert_eq!(leaves[0], LeafNode::Exists(3..4)); + assert_eq!(leaves[1], LeafNode::Exists(4..5)); + assert_eq!(leaves[2], LeafNode::Exists(5..6)); + assert_eq!(leaves[3], LeafNode::Exists(6..7)); + assert_eq!(leaves[4], LeafNode::DoesNotExist); + + let tree = get_tree_a(3); + let leaves = tree.get_leaf_nodes(5); + + assert_eq!(leaves[0], LeafNode::Exists(3..4)); + assert_eq!(leaves[1], LeafNode::Exists(4..5)); + assert_eq!(leaves[2], LeafNode::Exists(5..6)); + assert_eq!(leaves[3], LeafNode::Padding); + assert_eq!(leaves[4], LeafNode::DoesNotExist); + + let tree = get_tree_a(0); + let leaves = tree.get_leaf_nodes(2); + + assert_eq!(leaves[0], LeafNode::Padding); + assert_eq!(leaves[1], LeafNode::DoesNotExist); + + let tree = BTreeSchema::from_lengths(0, vec![3]).into_overlay(0); + let leaves = tree.get_leaf_nodes(2); + assert_eq!(leaves[0], LeafNode::Exists(0..3)); + assert_eq!(leaves[1], LeafNode::DoesNotExist); + + let tree = BTreeSchema::from_lengths(0, vec![3]).into_overlay(10); + let leaves = tree.get_leaf_nodes(2); + assert_eq!(leaves[0], LeafNode::Exists(10..13)); + assert_eq!(leaves[1], LeafNode::DoesNotExist); + } + + #[test] + fn root_of_one_node() { + let tree = get_tree_a(1); + + assert_eq!(tree.root(), 0); + assert_eq!(tree.num_internal_nodes(), 0); + assert_eq!(tree.num_leaf_nodes(), 1); + } + + #[test] + fn child_chunks() { + let tree = get_tree_a(4); + + assert_eq!(tree.child_chunks(0), (1, 2)) + } +} diff --git a/eth2/utils/cached_tree_hash/src/errors.rs b/eth2/utils/cached_tree_hash/src/errors.rs new file mode 100644 index 000000000..d9ac02913 --- /dev/null +++ b/eth2/utils/cached_tree_hash/src/errors.rs @@ -0,0 +1,19 @@ +use tree_hash::TreeHashType; + +#[derive(Debug, PartialEq, Clone)] +pub enum Error { + ShouldNotProduceBTreeOverlay, + NoFirstNode, + NoBytesForRoot, + UnableToObtainSlices, + UnableToGrowMerkleTree, + UnableToShrinkMerkleTree, + TreeCannotHaveZeroNodes, + CacheNotInitialized, + ShouldNeverBePacked(TreeHashType), + BytesAreNotEvenChunks(usize), + NoModifiedFieldForChunk(usize), + NoBytesForChunk(usize), + NoSchemaForIndex(usize), + NotLeafNode(usize), +} diff --git a/eth2/utils/cached_tree_hash/src/impls.rs b/eth2/utils/cached_tree_hash/src/impls.rs new file mode 100644 index 000000000..5105ad6a7 --- /dev/null +++ b/eth2/utils/cached_tree_hash/src/impls.rs @@ -0,0 +1,110 @@ +use super::*; +use crate::merkleize::merkleize; +use ethereum_types::H256; + +pub mod vec; + +macro_rules! impl_for_single_leaf_int { + ($type: ident) => { + impl CachedTreeHash for $type { + fn new_tree_hash_cache(&self, _depth: usize) -> Result { + Ok(TreeHashCache::from_bytes( + merkleize(self.to_le_bytes().to_vec()), + false, + None, + )?) + } + + fn tree_hash_cache_schema(&self, depth: usize) -> BTreeSchema { + BTreeSchema::from_lengths(depth, vec![1]) + } + + fn update_tree_hash_cache(&self, cache: &mut TreeHashCache) -> Result<(), Error> { + let leaf = merkleize(self.to_le_bytes().to_vec()); + cache.maybe_update_chunk(cache.chunk_index, &leaf)?; + + cache.chunk_index += 1; + + Ok(()) + } + } + }; +} + +impl_for_single_leaf_int!(u8); +impl_for_single_leaf_int!(u16); +impl_for_single_leaf_int!(u32); +impl_for_single_leaf_int!(u64); +impl_for_single_leaf_int!(usize); + +impl CachedTreeHash for bool { + fn new_tree_hash_cache(&self, _depth: usize) -> Result { + Ok(TreeHashCache::from_bytes( + merkleize((*self as u8).to_le_bytes().to_vec()), + false, + None, + )?) + } + + fn tree_hash_cache_schema(&self, depth: usize) -> BTreeSchema { + BTreeSchema::from_lengths(depth, vec![1]) + } + + fn update_tree_hash_cache(&self, cache: &mut TreeHashCache) -> Result<(), Error> { + let leaf = merkleize((*self as u8).to_le_bytes().to_vec()); + cache.maybe_update_chunk(cache.chunk_index, &leaf)?; + + cache.chunk_index += 1; + + Ok(()) + } +} + +impl CachedTreeHash for [u8; 4] { + fn new_tree_hash_cache(&self, _depth: usize) -> Result { + Ok(TreeHashCache::from_bytes( + merkleize(self.to_vec()), + false, + None, + )?) + } + + fn tree_hash_cache_schema(&self, depth: usize) -> BTreeSchema { + BTreeSchema::from_lengths(depth, vec![1]) + } + + fn update_tree_hash_cache(&self, cache: &mut TreeHashCache) -> Result<(), Error> { + let leaf = merkleize(self.to_vec()); + cache.maybe_update_chunk(cache.chunk_index, &leaf)?; + + cache.chunk_index += 1; + + Ok(()) + } +} + +impl CachedTreeHash for H256 { + fn new_tree_hash_cache(&self, _depth: usize) -> Result { + Ok(TreeHashCache::from_bytes( + self.as_bytes().to_vec(), + false, + None, + )?) + } + + fn num_tree_hash_cache_chunks(&self) -> usize { + 1 + } + + fn tree_hash_cache_schema(&self, depth: usize) -> BTreeSchema { + BTreeSchema::from_lengths(depth, vec![1]) + } + + fn update_tree_hash_cache(&self, cache: &mut TreeHashCache) -> Result<(), Error> { + cache.maybe_update_chunk(cache.chunk_index, self.as_bytes())?; + + cache.chunk_index += 1; + + Ok(()) + } +} diff --git a/eth2/utils/cached_tree_hash/src/impls/vec.rs b/eth2/utils/cached_tree_hash/src/impls/vec.rs new file mode 100644 index 000000000..bdb7eb134 --- /dev/null +++ b/eth2/utils/cached_tree_hash/src/impls/vec.rs @@ -0,0 +1,338 @@ +use super::*; +use crate::btree_overlay::LeafNode; +use crate::merkleize::{merkleize, num_sanitized_leaves, sanitise_bytes}; + +macro_rules! impl_for_list { + ($type: ty) => { + impl CachedTreeHash for $type + where + T: CachedTreeHash + TreeHash, + { + fn new_tree_hash_cache(&self, depth: usize) -> Result { + let (mut cache, schema) = new_tree_hash_cache(self, depth)?; + + cache.add_length_nodes(schema.into_overlay(0).chunk_range(), self.len())?; + + Ok(cache) + } + + fn num_tree_hash_cache_chunks(&self) -> usize { + // Add two extra nodes to cater for the node before and after to allow mixing-in length. + BTreeOverlay::new(self, 0, 0).num_chunks() + 2 + } + + fn tree_hash_cache_schema(&self, depth: usize) -> BTreeSchema { + produce_schema(self, depth) + } + + fn update_tree_hash_cache(&self, cache: &mut TreeHashCache) -> Result<(), Error> { + // Skip the length-mixed-in root node. + cache.chunk_index += 1; + + // Update the cache, returning the new overlay. + let new_overlay = update_tree_hash_cache(&self, cache)?; + + // Mix in length + cache.mix_in_length(new_overlay.chunk_range(), self.len())?; + + // Skip an extra node to clear the length node. + cache.chunk_index += 1; + + Ok(()) + } + } + }; +} + +impl_for_list!(Vec); +impl_for_list!(&[T]); + +/// Build a new tree hash cache for some slice. +/// +/// Valid for both variable- and fixed-length slices. Does _not_ mix-in the length of the list, +/// the caller must do this. +pub fn new_tree_hash_cache( + vec: &[T], + depth: usize, +) -> Result<(TreeHashCache, BTreeSchema), Error> { + let schema = vec.tree_hash_cache_schema(depth); + + let cache = match T::tree_hash_type() { + TreeHashType::Basic => TreeHashCache::from_bytes( + merkleize(get_packed_leaves(vec)?), + false, + Some(schema.clone()), + ), + TreeHashType::Container | TreeHashType::List | TreeHashType::Vector => { + let subtrees = vec + .iter() + .map(|item| TreeHashCache::new_at_depth(item, depth + 1)) + .collect::, _>>()?; + + TreeHashCache::from_subtrees(&vec, subtrees, depth) + } + }?; + + Ok((cache, schema)) +} + +/// Produce a schema for some slice. +/// +/// Valid for both variable- and fixed-length slices. Does _not_ add the mix-in length nodes, the +/// caller must do this. +pub fn produce_schema(vec: &[T], depth: usize) -> BTreeSchema { + let lengths = match T::tree_hash_type() { + TreeHashType::Basic => { + // Ceil division. + let num_leaves = + (vec.len() + T::tree_hash_packing_factor() - 1) / T::tree_hash_packing_factor(); + + // Disallow zero-length as an empty list still has one all-padding node. + vec![1; std::cmp::max(1, num_leaves)] + } + TreeHashType::Container | TreeHashType::List | TreeHashType::Vector => { + let mut lengths = vec![]; + + for item in vec { + lengths.push(item.num_tree_hash_cache_chunks()) + } + + lengths + } + }; + + BTreeSchema::from_lengths(depth, lengths) +} + +/// Updates the cache for some slice. +/// +/// Valid for both variable- and fixed-length slices. Does _not_ cater for the mix-in length nodes, +/// the caller must do this. +#[allow(clippy::range_plus_one)] // Minor readability lint requiring structural changes; not worth it. +pub fn update_tree_hash_cache( + vec: &[T], + cache: &mut TreeHashCache, +) -> Result { + let old_overlay = cache.get_overlay(cache.schema_index, cache.chunk_index)?; + let new_overlay = BTreeOverlay::new(&vec, cache.chunk_index, old_overlay.depth); + + cache.replace_overlay(cache.schema_index, cache.chunk_index, new_overlay.clone())?; + + cache.schema_index += 1; + + match T::tree_hash_type() { + TreeHashType::Basic => { + let mut buf = vec![0; HASHSIZE]; + let item_bytes = HASHSIZE / T::tree_hash_packing_factor(); + + // If the number of leaf nodes has changed, resize the cache. + if new_overlay.num_leaf_nodes() < old_overlay.num_leaf_nodes() { + let start = new_overlay.next_node(); + let end = start + (old_overlay.num_leaf_nodes() - new_overlay.num_leaf_nodes()); + + cache.splice(start..end, vec![], vec![]); + } else if new_overlay.num_leaf_nodes() > old_overlay.num_leaf_nodes() { + let start = old_overlay.next_node(); + let new_nodes = new_overlay.num_leaf_nodes() - old_overlay.num_leaf_nodes(); + + cache.splice( + start..start, + vec![0; new_nodes * HASHSIZE], + vec![true; new_nodes], + ); + } + + // Iterate through each of the leaf nodes in the new list. + for i in 0..new_overlay.num_leaf_nodes() { + // Iterate through the number of items that may be packing into the leaf node. + for j in 0..T::tree_hash_packing_factor() { + // Create a mut slice that can be filled with either a serialized item or + // padding. + let buf_slice = &mut buf[j * item_bytes..(j + 1) * item_bytes]; + + // Attempt to get the item for this portion of the chunk. If it exists, + // update `buf` with it's serialized bytes. If it doesn't exist, update + // `buf` with padding. + match vec.get(i * T::tree_hash_packing_factor() + j) { + Some(item) => { + buf_slice.copy_from_slice(&item.tree_hash_packed_encoding()); + } + None => buf_slice.copy_from_slice(&vec![0; item_bytes]), + } + } + + // Update the chunk if the generated `buf` is not the same as the cache. + let chunk = new_overlay.first_leaf_node() + i; + cache.maybe_update_chunk(chunk, &buf)?; + } + } + TreeHashType::Container | TreeHashType::List | TreeHashType::Vector => { + let longest_len = + std::cmp::max(new_overlay.num_leaf_nodes(), old_overlay.num_leaf_nodes()); + + let old_leaf_nodes = old_overlay.get_leaf_nodes(longest_len); + let new_leaf_nodes = if old_overlay == new_overlay { + old_leaf_nodes.clone() + } else { + new_overlay.get_leaf_nodes(longest_len) + }; + + for i in 0..longest_len { + match (&old_leaf_nodes[i], &new_leaf_nodes[i]) { + // The item existed in the previous list and exists in the current list. + // + // Update the item. + (LeafNode::Exists(_old), LeafNode::Exists(new)) => { + cache.chunk_index = new.start; + + vec[i].update_tree_hash_cache(cache)?; + } + // The list has been lengthened and this is a new item that did not exist in + // the previous list. + // + // Splice the tree for the new item into the current chunk_index. + (LeafNode::DoesNotExist, LeafNode::Exists(new)) => { + splice_in_new_tree( + &vec[i], + new.start..new.start, + new_overlay.depth + 1, + cache, + )?; + + cache.chunk_index = new.end; + } + // The list has been lengthened and this is a new item that was prevously a + // padding item. + // + // Splice the tree for the new item over the padding chunk. + (LeafNode::Padding, LeafNode::Exists(new)) => { + splice_in_new_tree( + &vec[i], + new.start..new.start + 1, + new_overlay.depth + 1, + cache, + )?; + + cache.chunk_index = new.end; + } + // The list has been shortened and this item was removed from the list and made + // into padding. + // + // Splice a padding node over the number of nodes the previous item occupied, + // starting at the current chunk_index. + (LeafNode::Exists(old), LeafNode::Padding) => { + let num_chunks = old.end - old.start; + + cache.splice( + cache.chunk_index..cache.chunk_index + num_chunks, + vec![0; HASHSIZE], + vec![true], + ); + + cache.chunk_index += 1; + } + // The list has been shortened and the item for this leaf existed in the + // previous list, but does not exist in this list. + // + // Remove the number of nodes the previous item occupied, starting at the + // current chunk_index. + (LeafNode::Exists(old), LeafNode::DoesNotExist) => { + let num_chunks = old.end - old.start; + + cache.splice( + cache.chunk_index..cache.chunk_index + num_chunks, + vec![], + vec![], + ); + } + // The list has been shortened and this leaf was padding in the previous list, + // however it should not exist in this list. + // + // Remove one node, starting at the current `chunk_index`. + (LeafNode::Padding, LeafNode::DoesNotExist) => { + cache.splice(cache.chunk_index..cache.chunk_index + 1, vec![], vec![]); + } + // The list has been lengthened and this leaf did not exist in the previous + // list, but should be padding for this list. + // + // Splice in a new padding node at the current chunk_index. + (LeafNode::DoesNotExist, LeafNode::Padding) => { + cache.splice( + cache.chunk_index..cache.chunk_index, + vec![0; HASHSIZE], + vec![true], + ); + + cache.chunk_index += 1; + } + // This leaf was padding in both lists, there's nothing to do. + (LeafNode::Padding, LeafNode::Padding) => (), + // As we are looping through the larger of the lists of leaf nodes, it should + // be impossible for either leaf to be non-existant. + (LeafNode::DoesNotExist, LeafNode::DoesNotExist) => unreachable!(), + } + } + + // Clean out any excess schemas that may or may not be remaining if the list was + // shortened. + cache.remove_proceeding_child_schemas(cache.schema_index, new_overlay.depth); + } + } + + cache.update_internal_nodes(&new_overlay)?; + + cache.chunk_index = new_overlay.next_node(); + + Ok(new_overlay) +} + +/// Create a new `TreeHashCache` from `item` and splice it over the `chunks_to_replace` chunks of +/// the given `cache`. +/// +/// Useful for the case where a new element is added to a list. +/// +/// The schemas created for `item` will have the given `depth`. +fn splice_in_new_tree( + item: &T, + chunks_to_replace: Range, + depth: usize, + cache: &mut TreeHashCache, +) -> Result<(), Error> +where + T: CachedTreeHash, +{ + let (bytes, mut bools, schemas) = TreeHashCache::new_at_depth(item, depth)?.into_components(); + + // Record the number of schemas, this will be used later in the fn. + let num_schemas = schemas.len(); + + // Flag the root node of the new tree as dirty. + bools[0] = true; + + cache.splice(chunks_to_replace, bytes, bools); + cache + .schemas + .splice(cache.schema_index..cache.schema_index, schemas); + + cache.schema_index += num_schemas; + + Ok(()) +} + +/// Packs all of the leaves of `vec` into a single byte-array, appending `0` to ensure the number +/// of chunks in the byte-array is a power-of-two. +fn get_packed_leaves(vec: &[T]) -> Result, Error> +where + T: CachedTreeHash, +{ + let num_packed_bytes = (BYTES_PER_CHUNK / T::tree_hash_packing_factor()) * vec.len(); + let num_leaves = num_sanitized_leaves(num_packed_bytes); + + let mut packed = Vec::with_capacity(num_leaves * HASHSIZE); + + for item in vec { + packed.append(&mut item.tree_hash_packed_encoding()); + } + + Ok(sanitise_bytes(packed)) +} diff --git a/eth2/utils/cached_tree_hash/src/lib.rs b/eth2/utils/cached_tree_hash/src/lib.rs new file mode 100644 index 000000000..21fa786e4 --- /dev/null +++ b/eth2/utils/cached_tree_hash/src/lib.rs @@ -0,0 +1,150 @@ +//! Performs cached merkle-hashing adhering to the Ethereum 2.0 specification defined +//! [here](https://github.com/ethereum/eth2.0-specs/blob/v0.5.1/specs/simple-serialize.md#merkleization). +//! +//! Caching allows for reduced hashing when some object has only been partially modified. This +//! allows for significant CPU-time savings (at the cost of additional storage). For example, +//! determining the root of a list of 1024 items with a single modification has been observed to +//! run in 1/25th of the time of a full merkle hash. +//! +//! +//! # Example: +//! +//! ``` +//! use cached_tree_hash::TreeHashCache; +//! use tree_hash_derive::{TreeHash, CachedTreeHash}; +//! +//! #[derive(TreeHash, CachedTreeHash)] +//! struct Foo { +//! bar: u64, +//! baz: Vec +//! } +//! +//! let mut foo = Foo { +//! bar: 1, +//! baz: vec![0, 1, 2] +//! }; +//! +//! let mut cache = TreeHashCache::new(&foo).unwrap(); +//! +//! foo.baz[1] = 0; +//! +//! cache.update(&foo).unwrap(); +//! +//! println!("Root is: {:?}", cache.tree_hash_root().unwrap()); +//! ``` + +use hashing::hash; +use std::ops::Range; +use tree_hash::{TreeHash, TreeHashType, BYTES_PER_CHUNK, HASHSIZE}; + +mod btree_overlay; +mod errors; +mod impls; +pub mod merkleize; +mod resize; +mod tree_hash_cache; + +pub use btree_overlay::{BTreeOverlay, BTreeSchema}; +pub use errors::Error; +pub use impls::vec; +pub use tree_hash_cache::TreeHashCache; + +pub trait CachedTreeHash: TreeHash { + fn tree_hash_cache_schema(&self, depth: usize) -> BTreeSchema; + + fn num_tree_hash_cache_chunks(&self) -> usize { + self.tree_hash_cache_schema(0).into_overlay(0).num_chunks() + } + + fn new_tree_hash_cache(&self, depth: usize) -> Result; + + fn update_tree_hash_cache(&self, cache: &mut TreeHashCache) -> Result<(), Error>; +} + +/// Implements `CachedTreeHash` on `$type` as a fixed-length tree-hash vector of the ssz encoding +/// of `$type`. +#[macro_export] +macro_rules! cached_tree_hash_ssz_encoding_as_vector { + ($type: ident, $num_bytes: expr) => { + impl cached_tree_hash::CachedTreeHash for $type { + fn new_tree_hash_cache( + &self, + depth: usize, + ) -> Result { + let (cache, _schema) = + cached_tree_hash::vec::new_tree_hash_cache(&ssz::ssz_encode(self), depth)?; + + Ok(cache) + } + + fn tree_hash_cache_schema(&self, depth: usize) -> cached_tree_hash::BTreeSchema { + let lengths = + vec![1; cached_tree_hash::merkleize::num_unsanitized_leaves($num_bytes)]; + cached_tree_hash::BTreeSchema::from_lengths(depth, lengths) + } + + fn update_tree_hash_cache( + &self, + cache: &mut cached_tree_hash::TreeHashCache, + ) -> Result<(), cached_tree_hash::Error> { + cached_tree_hash::vec::update_tree_hash_cache(&ssz::ssz_encode(self), cache)?; + + Ok(()) + } + } + }; +} + +/// Implements `CachedTreeHash` on `$type` as a variable-length tree-hash list of the result of +/// calling `.as_bytes()` on `$type`. +#[macro_export] +macro_rules! cached_tree_hash_bytes_as_list { + ($type: ident) => { + impl cached_tree_hash::CachedTreeHash for $type { + fn new_tree_hash_cache( + &self, + depth: usize, + ) -> Result { + let bytes = self.to_bytes(); + + let (mut cache, schema) = + cached_tree_hash::vec::new_tree_hash_cache(&bytes, depth)?; + + cache.add_length_nodes(schema.into_overlay(0).chunk_range(), bytes.len())?; + + Ok(cache) + } + + fn num_tree_hash_cache_chunks(&self) -> usize { + // Add two extra nodes to cater for the node before and after to allow mixing-in length. + cached_tree_hash::BTreeOverlay::new(self, 0, 0).num_chunks() + 2 + } + + fn tree_hash_cache_schema(&self, depth: usize) -> cached_tree_hash::BTreeSchema { + let bytes = self.to_bytes(); + cached_tree_hash::vec::produce_schema(&bytes, depth) + } + + fn update_tree_hash_cache( + &self, + cache: &mut cached_tree_hash::TreeHashCache, + ) -> Result<(), cached_tree_hash::Error> { + let bytes = self.to_bytes(); + + // Skip the length-mixed-in root node. + cache.chunk_index += 1; + + // Update the cache, returning the new overlay. + let new_overlay = cached_tree_hash::vec::update_tree_hash_cache(&bytes, cache)?; + + // Mix in length + cache.mix_in_length(new_overlay.chunk_range(), bytes.len())?; + + // Skip an extra node to clear the length node. + cache.chunk_index += 1; + + Ok(()) + } + } + }; +} diff --git a/eth2/utils/cached_tree_hash/src/merkleize.rs b/eth2/utils/cached_tree_hash/src/merkleize.rs new file mode 100644 index 000000000..9d8c83200 --- /dev/null +++ b/eth2/utils/cached_tree_hash/src/merkleize.rs @@ -0,0 +1,83 @@ +use hashing::hash; +use tree_hash::{BYTES_PER_CHUNK, HASHSIZE, MERKLE_HASH_CHUNK}; + +/// Split `values` into a power-of-two, identical-length chunks (padding with `0`) and merkleize +/// them, returning the entire merkle tree. +/// +/// The root hash is `merkleize(values)[0..BYTES_PER_CHUNK]`. +pub fn merkleize(values: Vec) -> Vec { + let values = sanitise_bytes(values); + + let leaves = values.len() / HASHSIZE; + + if leaves == 0 { + panic!("No full leaves"); + } + + if !leaves.is_power_of_two() { + panic!("leaves is not power of two"); + } + + let mut o: Vec = vec![0; (num_nodes(leaves) - leaves) * HASHSIZE]; + o.append(&mut values.to_vec()); + + let mut i = o.len(); + let mut j = o.len() - values.len(); + + while i >= MERKLE_HASH_CHUNK { + i -= MERKLE_HASH_CHUNK; + let hash = hash(&o[i..i + MERKLE_HASH_CHUNK]); + + j -= HASHSIZE; + o[j..j + HASHSIZE].copy_from_slice(&hash); + } + + o +} + +/// Ensures that the given `bytes` are a power-of-two chunks, padding with zero if not. +pub fn sanitise_bytes(mut bytes: Vec) -> Vec { + let present_leaves = num_unsanitized_leaves(bytes.len()); + let required_leaves = present_leaves.next_power_of_two(); + + if (present_leaves != required_leaves) | last_leaf_needs_padding(bytes.len()) { + bytes.resize(num_bytes(required_leaves), 0); + } + + bytes +} + +/// Pads out `bytes` to ensure it is a clean `num_leaves` chunks. +pub fn pad_for_leaf_count(num_leaves: usize, bytes: &mut Vec) { + let required_leaves = num_leaves.next_power_of_two(); + + bytes.resize( + bytes.len() + (required_leaves - num_leaves) * BYTES_PER_CHUNK, + 0, + ); +} + +fn last_leaf_needs_padding(num_bytes: usize) -> bool { + num_bytes % HASHSIZE != 0 +} + +/// Returns the number of leaves for a given `bytes_len` number of bytes, rounding up if +/// `num_bytes` is not a client multiple of chunk size. +pub fn num_unsanitized_leaves(bytes_len: usize) -> usize { + (bytes_len + HASHSIZE - 1) / HASHSIZE +} + +fn num_bytes(num_leaves: usize) -> usize { + num_leaves * HASHSIZE +} + +fn num_nodes(num_leaves: usize) -> usize { + 2 * num_leaves - 1 +} + +/// Returns the power-of-two number of leaves that would result from the given `bytes_len` number +/// of bytes. +pub fn num_sanitized_leaves(bytes_len: usize) -> usize { + let leaves = (bytes_len + HASHSIZE - 1) / HASHSIZE; + leaves.next_power_of_two() +} diff --git a/eth2/utils/cached_tree_hash/src/resize.rs b/eth2/utils/cached_tree_hash/src/resize.rs new file mode 100644 index 000000000..5428e234b --- /dev/null +++ b/eth2/utils/cached_tree_hash/src/resize.rs @@ -0,0 +1,223 @@ +#![allow(clippy::range_plus_one)] // Minor readability lint requiring structural changes; not worth it. + +use super::*; + +/// New vec is bigger than old vec. +pub fn grow_merkle_tree( + old_bytes: &[u8], + old_flags: &[bool], + from_height: usize, + to_height: usize, +) -> Option<(Vec, Vec)> { + let to_nodes = nodes_in_tree_of_height(to_height); + + let mut bytes = vec![0; to_nodes * HASHSIZE]; + let mut flags = vec![true; to_nodes]; + + for i in 0..=from_height { + let old_byte_slice = old_bytes.get(byte_range_at_height(i))?; + let old_flag_slice = old_flags.get(node_range_at_height(i))?; + + let offset = i + to_height - from_height; + let new_byte_slice = bytes.get_mut(byte_range_at_height(offset))?; + let new_flag_slice = flags.get_mut(node_range_at_height(offset))?; + + new_byte_slice + .get_mut(0..old_byte_slice.len())? + .copy_from_slice(old_byte_slice); + new_flag_slice + .get_mut(0..old_flag_slice.len())? + .copy_from_slice(old_flag_slice); + } + + Some((bytes, flags)) +} + +/// New vec is smaller than old vec. +pub fn shrink_merkle_tree( + from_bytes: &[u8], + from_flags: &[bool], + from_height: usize, + to_height: usize, +) -> Option<(Vec, Vec)> { + let to_nodes = nodes_in_tree_of_height(to_height); + + let mut bytes = vec![0; to_nodes * HASHSIZE]; + let mut flags = vec![true; to_nodes]; + + for i in 0..=to_height as usize { + let offset = i + from_height - to_height; + let from_byte_slice = from_bytes.get(byte_range_at_height(offset))?; + let from_flag_slice = from_flags.get(node_range_at_height(offset))?; + + let to_byte_slice = bytes.get_mut(byte_range_at_height(i))?; + let to_flag_slice = flags.get_mut(node_range_at_height(i))?; + + to_byte_slice.copy_from_slice(from_byte_slice.get(0..to_byte_slice.len())?); + to_flag_slice.copy_from_slice(from_flag_slice.get(0..to_flag_slice.len())?); + } + + Some((bytes, flags)) +} + +pub fn nodes_in_tree_of_height(h: usize) -> usize { + 2 * (1 << h) - 1 +} + +fn byte_range_at_height(h: usize) -> Range { + let node_range = node_range_at_height(h); + node_range.start * HASHSIZE..node_range.end * HASHSIZE +} + +fn node_range_at_height(h: usize) -> Range { + first_node_at_height(h)..last_node_at_height(h) + 1 +} + +fn first_node_at_height(h: usize) -> usize { + (1 << h) - 1 +} + +fn last_node_at_height(h: usize) -> usize { + (1 << (h + 1)) - 2 +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn can_grow_and_shrink_three_levels() { + let small: usize = 1; + let big: usize = 15; + + let original_bytes = vec![42; small * HASHSIZE]; + let original_flags = vec![false; small]; + + let (grown_bytes, grown_flags) = grow_merkle_tree( + &original_bytes, + &original_flags, + (small + 1).trailing_zeros() as usize - 1, + (big + 1).trailing_zeros() as usize - 1, + ) + .unwrap(); + + let mut expected_bytes = vec![]; + let mut expected_flags = vec![]; + // First level + expected_bytes.append(&mut vec![0; 32]); + expected_flags.push(true); + // Second level + expected_bytes.append(&mut vec![0; 32]); + expected_bytes.append(&mut vec![0; 32]); + expected_flags.push(true); + expected_flags.push(true); + // Third level + expected_bytes.append(&mut vec![0; 32]); + expected_bytes.append(&mut vec![0; 32]); + expected_bytes.append(&mut vec![0; 32]); + expected_bytes.append(&mut vec![0; 32]); + expected_flags.push(true); + expected_flags.push(true); + expected_flags.push(true); + expected_flags.push(true); + // Fourth level + expected_bytes.append(&mut vec![42; 32]); + expected_bytes.append(&mut vec![0; 32]); + expected_bytes.append(&mut vec![0; 32]); + expected_bytes.append(&mut vec![0; 32]); + expected_bytes.append(&mut vec![0; 32]); + expected_bytes.append(&mut vec![0; 32]); + expected_bytes.append(&mut vec![0; 32]); + expected_bytes.append(&mut vec![0; 32]); + expected_flags.push(false); + expected_flags.push(true); + expected_flags.push(true); + expected_flags.push(true); + expected_flags.push(true); + expected_flags.push(true); + expected_flags.push(true); + expected_flags.push(true); + + assert_eq!(expected_bytes, grown_bytes); + assert_eq!(expected_flags, grown_flags); + + let (shrunk_bytes, shrunk_flags) = shrink_merkle_tree( + &grown_bytes, + &grown_flags, + (big + 1).trailing_zeros() as usize - 1, + (small + 1).trailing_zeros() as usize - 1, + ) + .unwrap(); + + assert_eq!(original_bytes, shrunk_bytes); + assert_eq!(original_flags, shrunk_flags); + } + + #[test] + fn can_grow_and_shrink_one_level() { + let small: usize = 7; + let big: usize = 15; + + let original_bytes = vec![42; small * HASHSIZE]; + let original_flags = vec![false; small]; + + let (grown_bytes, grown_flags) = grow_merkle_tree( + &original_bytes, + &original_flags, + (small + 1).trailing_zeros() as usize - 1, + (big + 1).trailing_zeros() as usize - 1, + ) + .unwrap(); + + let mut expected_bytes = vec![]; + let mut expected_flags = vec![]; + // First level + expected_bytes.append(&mut vec![0; 32]); + expected_flags.push(true); + // Second level + expected_bytes.append(&mut vec![42; 32]); + expected_bytes.append(&mut vec![0; 32]); + expected_flags.push(false); + expected_flags.push(true); + // Third level + expected_bytes.append(&mut vec![42; 32]); + expected_bytes.append(&mut vec![42; 32]); + expected_bytes.append(&mut vec![0; 32]); + expected_bytes.append(&mut vec![0; 32]); + expected_flags.push(false); + expected_flags.push(false); + expected_flags.push(true); + expected_flags.push(true); + // Fourth level + expected_bytes.append(&mut vec![42; 32]); + expected_bytes.append(&mut vec![42; 32]); + expected_bytes.append(&mut vec![42; 32]); + expected_bytes.append(&mut vec![42; 32]); + expected_bytes.append(&mut vec![0; 32]); + expected_bytes.append(&mut vec![0; 32]); + expected_bytes.append(&mut vec![0; 32]); + expected_bytes.append(&mut vec![0; 32]); + expected_flags.push(false); + expected_flags.push(false); + expected_flags.push(false); + expected_flags.push(false); + expected_flags.push(true); + expected_flags.push(true); + expected_flags.push(true); + expected_flags.push(true); + + assert_eq!(expected_bytes, grown_bytes); + assert_eq!(expected_flags, grown_flags); + + let (shrunk_bytes, shrunk_flags) = shrink_merkle_tree( + &grown_bytes, + &grown_flags, + (big + 1).trailing_zeros() as usize - 1, + (small + 1).trailing_zeros() as usize - 1, + ) + .unwrap(); + + assert_eq!(original_bytes, shrunk_bytes); + assert_eq!(original_flags, shrunk_flags); + } +} diff --git a/eth2/utils/cached_tree_hash/src/tree_hash_cache.rs b/eth2/utils/cached_tree_hash/src/tree_hash_cache.rs new file mode 100644 index 000000000..8f7b9de86 --- /dev/null +++ b/eth2/utils/cached_tree_hash/src/tree_hash_cache.rs @@ -0,0 +1,446 @@ +#![allow(clippy::range_plus_one)] // Minor readability lint requiring structural changes; not worth it. + +use super::*; +use crate::merkleize::{merkleize, pad_for_leaf_count}; +use int_to_bytes::int_to_bytes32; + +/// Provides cached tree hashing for some object implementing `CachedTreeHash`. +/// +/// Caching allows for doing minimal internal-node hashing when an object has only been partially +/// changed. +/// +/// See the crate root for an example. +#[derive(Debug, PartialEq, Clone)] +pub struct TreeHashCache { + /// Stores the binary-tree in 32-byte chunks. + pub bytes: Vec, + /// Maps to each chunk of `self.bytes`, indicating if the chunk is dirty. + pub chunk_modified: Vec, + /// Contains a schema for each variable-length item stored in the cache. + pub schemas: Vec, + + /// A counter used during updates. + pub chunk_index: usize, + /// A counter used during updates. + pub schema_index: usize, +} + +impl Default for TreeHashCache { + /// Create an empty cache. + /// + /// Note: an empty cache is effectively useless, an error will be raised if `self.update` is + /// called. + fn default() -> TreeHashCache { + TreeHashCache { + bytes: vec![], + chunk_modified: vec![], + schemas: vec![], + chunk_index: 0, + schema_index: 0, + } + } +} + +impl TreeHashCache { + /// Instantiates a new cache from `item` at a depth of `0`. + /// + /// The returned cache is fully-built and will return an accurate tree-hash root. + pub fn new(item: &T) -> Result + where + T: CachedTreeHash, + { + Self::new_at_depth(item, 0) + } + + /// Instantiates a new cache from `item` at the specified `depth`. + /// + /// The returned cache is fully-built and will return an accurate tree-hash root. + pub fn new_at_depth(item: &T, depth: usize) -> Result + where + T: CachedTreeHash, + { + item.new_tree_hash_cache(depth) + } + + /// Updates the cache with `item`. + /// + /// `item` _must_ be of the same type as the `item` used to build the cache, otherwise an error + /// may be returned. + /// + /// After calling `update`, the cache will return an accurate tree-hash root using + /// `self.tree_hash_root()`. + pub fn update(&mut self, item: &T) -> Result<(), Error> + where + T: CachedTreeHash, + { + if self.is_empty() { + Err(Error::CacheNotInitialized) + } else { + self.reset_modifications(); + + item.update_tree_hash_cache(self) + } + } + + /// Builds a new cache for `item`, given `subtrees` contains a `Self` for field/item of `item`. + /// + /// Each `subtree` in `subtree` will become a leaf-node of the merkle-tree of `item`. + pub fn from_subtrees(item: &T, subtrees: Vec, depth: usize) -> Result + where + T: CachedTreeHash, + { + let overlay = BTreeOverlay::new(item, 0, depth); + + // Note how many leaves were provided. If is not a power-of-two, we'll need to pad it out + // later. + let num_provided_leaf_nodes = subtrees.len(); + + // Allocate enough bytes to store the internal nodes and the leaves and subtrees, then fill + // all the to-be-built internal nodes with zeros and append the leaves and subtrees. + let internal_node_bytes = overlay.num_internal_nodes() * BYTES_PER_CHUNK; + let subtrees_bytes = subtrees.iter().fold(0, |acc, t| acc + t.bytes.len()); + let mut bytes = Vec::with_capacity(subtrees_bytes + internal_node_bytes); + bytes.resize(internal_node_bytes, 0); + + // Allocate enough bytes to store all the leaves. + let mut leaves = Vec::with_capacity(overlay.num_leaf_nodes() * HASHSIZE); + let mut schemas = Vec::with_capacity(subtrees.len()); + + if T::tree_hash_type() == TreeHashType::List { + schemas.push(overlay.into()); + } + + // Iterate through all of the leaves/subtrees, adding their root as a leaf node and then + // concatenating their merkle trees. + for t in subtrees { + leaves.append(&mut t.tree_hash_root()?.to_vec()); + + let (mut t_bytes, _bools, mut t_schemas) = t.into_components(); + bytes.append(&mut t_bytes); + schemas.append(&mut t_schemas); + } + + // Pad the leaves to an even power-of-two, using zeros. + pad_for_leaf_count(num_provided_leaf_nodes, &mut bytes); + + // Merkleize the leaves, then split the leaf nodes off them. Then, replace all-zeros + // internal nodes created earlier with the internal nodes generated by `merkleize`. + let mut merkleized = merkleize(leaves); + merkleized.split_off(internal_node_bytes); + bytes.splice(0..internal_node_bytes, merkleized); + + Ok(Self { + chunk_modified: vec![true; bytes.len() / BYTES_PER_CHUNK], + bytes, + schemas, + chunk_index: 0, + schema_index: 0, + }) + } + + /// Instantiate a new cache from the pre-built `bytes` where each `self.chunk_modified` will be + /// set to `intitial_modified_state`. + /// + /// Note: `bytes.len()` must be a multiple of 32 + pub fn from_bytes( + bytes: Vec, + initial_modified_state: bool, + schema: Option, + ) -> Result { + if bytes.len() % BYTES_PER_CHUNK > 0 { + return Err(Error::BytesAreNotEvenChunks(bytes.len())); + } + + let schemas = match schema { + Some(schema) => vec![schema], + None => vec![], + }; + + Ok(Self { + chunk_modified: vec![initial_modified_state; bytes.len() / BYTES_PER_CHUNK], + bytes, + schemas, + chunk_index: 0, + schema_index: 0, + }) + } + + /// Returns `true` if this cache is empty (i.e., it has never been built for some item). + /// + /// Note: an empty cache is effectively useless, an error will be raised if `self.update` is + /// called. + pub fn is_empty(&self) -> bool { + self.chunk_modified.is_empty() + } + + /// Return an overlay, built from the schema at `schema_index` with an offset of `chunk_index`. + pub fn get_overlay( + &self, + schema_index: usize, + chunk_index: usize, + ) -> Result { + Ok(self + .schemas + .get(schema_index) + .ok_or_else(|| Error::NoSchemaForIndex(schema_index))? + .clone() + .into_overlay(chunk_index)) + } + + /// Resets the per-update counters, allowing a new update to start. + /// + /// Note: this does _not_ delete the contents of the cache. + pub fn reset_modifications(&mut self) { + // Reset the per-hash counters. + self.chunk_index = 0; + self.schema_index = 0; + + for chunk_modified in &mut self.chunk_modified { + *chunk_modified = false; + } + } + + /// Replace the schema at `schema_index` with the schema derived from `new_overlay`. + /// + /// If the `new_overlay` schema has a different number of internal nodes to the schema at + /// `schema_index`, the cache will be updated to add/remove these new internal nodes. + pub fn replace_overlay( + &mut self, + schema_index: usize, + // TODO: remove chunk index (if possible) + chunk_index: usize, + new_overlay: BTreeOverlay, + ) -> Result { + let old_overlay = self.get_overlay(schema_index, chunk_index)?; + // If the merkle tree required to represent the new list is of a different size to the one + // required for the previous list, then update the internal nodes. + // + // Leaf nodes are not touched, they should be updated externally to this function. + // + // This grows/shrinks the bytes to accommodate the new tree, preserving as much of the tree + // as possible. + if new_overlay.num_internal_nodes() != old_overlay.num_internal_nodes() { + // Get slices of the existing tree from the cache. + let (old_bytes, old_flags) = self + .slices(old_overlay.internal_chunk_range()) + .ok_or_else(|| Error::UnableToObtainSlices)?; + + let (new_bytes, new_flags) = if new_overlay.num_internal_nodes() == 0 { + // The new tree has zero internal nodes, simply return empty lists. + (vec![], vec![]) + } else if old_overlay.num_internal_nodes() == 0 { + // The old tree has zero nodes and the new tree has some nodes. Create new nodes to + // suit. + let nodes = resize::nodes_in_tree_of_height(new_overlay.height() - 1); + + (vec![0; nodes * HASHSIZE], vec![true; nodes]) + } else if new_overlay.num_internal_nodes() > old_overlay.num_internal_nodes() { + // The new tree is bigger than the old tree. + // + // Grow the internal nodes, preserving any existing nodes. + resize::grow_merkle_tree( + old_bytes, + old_flags, + old_overlay.height() - 1, + new_overlay.height() - 1, + ) + .ok_or_else(|| Error::UnableToGrowMerkleTree)? + } else { + // The new tree is smaller than the old tree. + // + // Shrink the internal nodes, preserving any existing nodes. + resize::shrink_merkle_tree( + old_bytes, + old_flags, + old_overlay.height() - 1, + new_overlay.height() - 1, + ) + .ok_or_else(|| Error::UnableToShrinkMerkleTree)? + }; + + // Splice the resized created elements over the existing elements, effectively updating + // the number of stored internal nodes for this tree. + self.splice(old_overlay.internal_chunk_range(), new_bytes, new_flags); + } + + let old_schema = std::mem::replace(&mut self.schemas[schema_index], new_overlay.into()); + + Ok(old_schema.into_overlay(chunk_index)) + } + + /// Remove all of the child schemas following `schema_index`. + /// + /// Schema `a` is a child of schema `b` if `a.depth < b.depth`. + pub fn remove_proceeding_child_schemas(&mut self, schema_index: usize, depth: usize) { + let end = self + .schemas + .iter() + .skip(schema_index) + .position(|o| o.depth <= depth) + .and_then(|i| Some(i + schema_index)) + .unwrap_or_else(|| self.schemas.len()); + + self.schemas.splice(schema_index..end, vec![]); + } + + /// Iterate through the internal nodes chunks of `overlay`, updating the chunk with the + /// merkle-root of it's children if either of those children are dirty. + pub fn update_internal_nodes(&mut self, overlay: &BTreeOverlay) -> Result<(), Error> { + for (parent, children) in overlay.internal_parents_and_children().into_iter().rev() { + if self.either_modified(children)? { + self.modify_chunk(parent, &self.hash_children(children)?)?; + } + } + + Ok(()) + } + + /// Returns to the tree-hash root of the cache. + pub fn tree_hash_root(&self) -> Result<&[u8], Error> { + if self.is_empty() { + Err(Error::CacheNotInitialized) + } else { + self.bytes + .get(0..HASHSIZE) + .ok_or_else(|| Error::NoBytesForRoot) + } + } + + /// Splices the given `bytes` over `self.bytes` and `bools` over `self.chunk_modified` at the + /// specified `chunk_range`. + pub fn splice(&mut self, chunk_range: Range, bytes: Vec, bools: Vec) { + // Update the `chunk_modified` vec, marking all spliced-in nodes as changed. + self.chunk_modified.splice(chunk_range.clone(), bools); + self.bytes + .splice(node_range_to_byte_range(&chunk_range), bytes); + } + + /// If the bytes at `chunk` are not the same as `to`, `self.bytes` is updated and + /// `self.chunk_modified` is set to `true`. + pub fn maybe_update_chunk(&mut self, chunk: usize, to: &[u8]) -> Result<(), Error> { + let start = chunk * BYTES_PER_CHUNK; + let end = start + BYTES_PER_CHUNK; + + if !self.chunk_equals(chunk, to)? { + self.bytes + .get_mut(start..end) + .ok_or_else(|| Error::NoModifiedFieldForChunk(chunk))? + .copy_from_slice(to); + self.chunk_modified[chunk] = true; + } + + Ok(()) + } + + /// Returns the slices of `self.bytes` and `self.chunk_modified` at the given `chunk_range`. + fn slices(&self, chunk_range: Range) -> Option<(&[u8], &[bool])> { + Some(( + self.bytes.get(node_range_to_byte_range(&chunk_range))?, + self.chunk_modified.get(chunk_range)?, + )) + } + + /// Updates `self.bytes` at `chunk` and sets `self.chunk_modified` for the `chunk` to `true`. + pub fn modify_chunk(&mut self, chunk: usize, to: &[u8]) -> Result<(), Error> { + let start = chunk * BYTES_PER_CHUNK; + let end = start + BYTES_PER_CHUNK; + + self.bytes + .get_mut(start..end) + .ok_or_else(|| Error::NoBytesForChunk(chunk))? + .copy_from_slice(to); + + self.chunk_modified[chunk] = true; + + Ok(()) + } + + /// Returns the bytes at `chunk`. + fn get_chunk(&self, chunk: usize) -> Result<&[u8], Error> { + let start = chunk * BYTES_PER_CHUNK; + let end = start + BYTES_PER_CHUNK; + + Ok(self + .bytes + .get(start..end) + .ok_or_else(|| Error::NoModifiedFieldForChunk(chunk))?) + } + + /// Returns `true` if the bytes at `chunk` are equal to `other`. + fn chunk_equals(&mut self, chunk: usize, other: &[u8]) -> Result { + Ok(self.get_chunk(chunk)? == other) + } + + /// Returns `true` if `chunk` is dirty. + pub fn changed(&self, chunk: usize) -> Result { + self.chunk_modified + .get(chunk) + .cloned() + .ok_or_else(|| Error::NoModifiedFieldForChunk(chunk)) + } + + /// Returns `true` if either of the `children` chunks is dirty. + fn either_modified(&self, children: (usize, usize)) -> Result { + Ok(self.changed(children.0)? | self.changed(children.1)?) + } + + /// Returns the hash of the concatenation of the given `children`. + pub fn hash_children(&self, children: (usize, usize)) -> Result, Error> { + let mut child_bytes = Vec::with_capacity(BYTES_PER_CHUNK * 2); + child_bytes.append(&mut self.get_chunk(children.0)?.to_vec()); + child_bytes.append(&mut self.get_chunk(children.1)?.to_vec()); + + Ok(hash(&child_bytes)) + } + + /// Adds a chunk before and after the given `chunk` range and calls `self.mix_in_length()`. + pub fn add_length_nodes( + &mut self, + chunk_range: Range, + length: usize, + ) -> Result<(), Error> { + self.chunk_modified[chunk_range.start] = true; + + let byte_range = node_range_to_byte_range(&chunk_range); + + // Add the last node. + self.bytes + .splice(byte_range.end..byte_range.end, vec![0; HASHSIZE]); + self.chunk_modified + .splice(chunk_range.end..chunk_range.end, vec![false]); + + // Add the first node. + self.bytes + .splice(byte_range.start..byte_range.start, vec![0; HASHSIZE]); + self.chunk_modified + .splice(chunk_range.start..chunk_range.start, vec![false]); + + self.mix_in_length(chunk_range.start + 1..chunk_range.end + 1, length)?; + + Ok(()) + } + + /// Sets `chunk_range.end + 1` equal to the little-endian serialization of `length`. Sets + /// `chunk_range.start - 1` equal to `self.hash_children(chunk_range.start, chunk_range.end + 1)`. + pub fn mix_in_length(&mut self, chunk_range: Range, length: usize) -> Result<(), Error> { + // Update the length chunk. + self.maybe_update_chunk(chunk_range.end, &int_to_bytes32(length as u64))?; + + // Update the mixed-in root if the main root or the length have changed. + let children = (chunk_range.start, chunk_range.end); + if self.either_modified(children)? { + self.modify_chunk(chunk_range.start - 1, &self.hash_children(children)?)?; + } + + Ok(()) + } + + /// Returns `(self.bytes, self.chunk_modified, self.schemas)`. + pub fn into_components(self) -> (Vec, Vec, Vec) { + (self.bytes, self.chunk_modified, self.schemas) + } +} + +fn node_range_to_byte_range(node_range: &Range) -> Range { + node_range.start * HASHSIZE..node_range.end * HASHSIZE +} diff --git a/eth2/utils/cached_tree_hash/tests/tests.rs b/eth2/utils/cached_tree_hash/tests/tests.rs new file mode 100644 index 000000000..3e2598e2b --- /dev/null +++ b/eth2/utils/cached_tree_hash/tests/tests.rs @@ -0,0 +1,677 @@ +use cached_tree_hash::{merkleize::merkleize, *}; +use ethereum_types::H256 as Hash256; +use int_to_bytes::int_to_bytes32; +use tree_hash_derive::{CachedTreeHash, TreeHash}; + +#[test] +fn modifications() { + let n = 2048; + + let vec: Vec = (0..n).map(|_| Hash256::random()).collect(); + + let mut cache = TreeHashCache::new(&vec).unwrap(); + cache.update(&vec).unwrap(); + + let modifications = cache.chunk_modified.iter().filter(|b| **b).count(); + + assert_eq!(modifications, 0); + + let mut modified_vec = vec.clone(); + modified_vec[n - 1] = Hash256::random(); + + cache.update(&modified_vec).unwrap(); + + let modifications = cache.chunk_modified.iter().filter(|b| **b).count(); + + assert_eq!(modifications, n.trailing_zeros() as usize + 2); +} + +#[derive(Clone, Debug, TreeHash, CachedTreeHash)] +pub struct NestedStruct { + pub a: u64, + pub b: Inner, +} + +fn test_routine(original: T, modified: Vec) +where + T: CachedTreeHash + std::fmt::Debug, +{ + let mut cache = TreeHashCache::new(&original).unwrap(); + + let standard_root = original.tree_hash_root(); + let cached_root = cache.tree_hash_root().unwrap(); + assert_eq!(standard_root, cached_root, "Initial cache build failed."); + + for (i, modified) in modified.iter().enumerate() { + println!("-- Start of modification {} --", i); + + // Update the existing hasher. + cache + .update(modified) + .expect(&format!("Modification {}", i)); + + // Create a new hasher from the "modified" struct. + let modified_cache = TreeHashCache::new(modified).unwrap(); + + assert_eq!( + cache.chunk_modified.len(), + modified_cache.chunk_modified.len(), + "Number of chunks is different" + ); + + assert_eq!( + cache.bytes.len(), + modified_cache.bytes.len(), + "Number of bytes is different" + ); + + assert_eq!(cache.bytes, modified_cache.bytes, "Bytes are different"); + + assert_eq!( + cache.schemas.len(), + modified_cache.schemas.len(), + "Number of schemas is different" + ); + + assert_eq!( + cache.schemas, modified_cache.schemas, + "Schemas are different" + ); + + // Test the root generated by the updated hasher matches a non-cached tree hash root. + let standard_root = modified.tree_hash_root(); + let cached_root = cache + .tree_hash_root() + .expect(&format!("Modification {}", i)); + assert_eq!( + standard_root, cached_root, + "Modification {} failed. \n Cache: {:?}", + i, cache + ); + } +} + +#[test] +fn test_nested_struct() { + let original = NestedStruct { + a: 42, + b: Inner { + a: 12, + b: 13, + c: 14, + d: 15, + }, + }; + let modified = vec![NestedStruct { + a: 99, + ..original.clone() + }]; + + test_routine(original, modified); +} + +#[test] +fn test_inner() { + let original = Inner { + a: 12, + b: 13, + c: 14, + d: 15, + }; + + let modified = vec![Inner { + a: 99, + ..original.clone() + }]; + + test_routine(original, modified); +} + +#[test] +fn test_vec_of_hash256() { + let n = 16; + + let original: Vec = (0..n).map(|_| Hash256::random()).collect(); + + let modified: Vec> = vec![ + original[..].to_vec(), + original[0..n / 2].to_vec(), + vec![], + original[0..1].to_vec(), + original[0..3].to_vec(), + original[0..n - 12].to_vec(), + ]; + + test_routine(original, modified); +} + +#[test] +fn test_vec_of_u64() { + let original: Vec = vec![1, 2, 3, 4, 5]; + + let modified: Vec> = vec![ + vec![1, 2, 3, 4, 42], + vec![1, 2, 3, 4], + vec![], + vec![42; 2_usize.pow(4)], + vec![], + vec![], + vec![1, 2, 3, 4, 42], + vec![1, 2, 3], + vec![1], + ]; + + test_routine(original, modified); +} + +#[test] +fn test_nested_list_of_u64() { + let original: Vec> = vec![vec![42]]; + + let modified = vec![ + vec![vec![1]], + vec![vec![1], vec![2]], + vec![vec![1], vec![3], vec![4]], + vec![], + vec![vec![1], vec![3], vec![4]], + vec![], + vec![vec![1, 2], vec![3], vec![4, 5, 6, 7, 8]], + vec![], + vec![vec![1], vec![2], vec![3]], + vec![vec![1, 2, 3, 4, 5, 6], vec![1, 2, 3, 4, 5, 6, 7]], + vec![vec![], vec![], vec![]], + vec![vec![0, 0, 0], vec![0], vec![0]], + ]; + + test_routine(original, modified); +} + +#[test] +fn test_shrinking_vec_of_vec() { + let original: Vec> = vec![vec![1], vec![2], vec![3], vec![4], vec![5]]; + let modified: Vec> = original[0..3].to_vec(); + + let new_cache = TreeHashCache::new(&modified).unwrap(); + + let mut modified_cache = TreeHashCache::new(&original).unwrap(); + modified_cache.update(&modified).unwrap(); + + assert_eq!( + new_cache.schemas.len(), + modified_cache.schemas.len(), + "Schema count is different" + ); + + assert_eq!( + new_cache.chunk_modified.len(), + modified_cache.chunk_modified.len(), + "Chunk count is different" + ); +} + +#[derive(Clone, Debug, TreeHash, CachedTreeHash)] +pub struct StructWithVec { + pub a: u64, + pub b: Inner, + pub c: Vec, +} + +#[test] +fn test_struct_with_vec() { + let original = StructWithVec { + a: 42, + b: Inner { + a: 12, + b: 13, + c: 14, + d: 15, + }, + c: vec![1, 2, 3, 4, 5], + }; + + let modified = vec![ + StructWithVec { + a: 99, + ..original.clone() + }, + StructWithVec { + a: 100, + ..original.clone() + }, + StructWithVec { + c: vec![1, 2, 3, 4, 5], + ..original.clone() + }, + StructWithVec { + c: vec![1, 3, 4, 5, 6], + ..original.clone() + }, + StructWithVec { + c: vec![1, 3, 4, 5, 6, 7, 8, 9], + ..original.clone() + }, + StructWithVec { + c: vec![1, 3, 4, 5], + ..original.clone() + }, + StructWithVec { + b: Inner { + a: u64::max_value(), + b: u64::max_value(), + c: u64::max_value(), + d: u64::max_value(), + }, + c: vec![], + ..original.clone() + }, + StructWithVec { + b: Inner { + a: 0, + b: 1, + c: 2, + d: 3, + }, + ..original.clone() + }, + ]; + + test_routine(original, modified); +} + +#[test] +fn test_vec_of_struct_with_vec() { + let a = StructWithVec { + a: 42, + b: Inner { + a: 12, + b: 13, + c: 14, + d: 15, + }, + c: vec![1, 2, 3, 4, 5], + }; + let b = StructWithVec { + c: vec![], + ..a.clone() + }; + let c = StructWithVec { + b: Inner { + a: 99, + b: 100, + c: 101, + d: 102, + }, + ..a.clone() + }; + let d = StructWithVec { a: 0, ..a.clone() }; + + let original: Vec = vec![a.clone(), c.clone()]; + + let modified = vec![ + vec![a.clone(), c.clone()], + vec![], + vec![a.clone(), b.clone(), c.clone(), d.clone()], + vec![b.clone(), a.clone(), c.clone(), d.clone()], + vec![], + vec![a.clone()], + vec![], + vec![a.clone(), b.clone(), c.clone(), d.clone()], + ]; + + test_routine(original, modified); +} + +#[derive(Clone, Debug, TreeHash, CachedTreeHash)] +pub struct StructWithVecOfStructs { + pub a: u64, + pub b: Inner, + pub c: Vec, +} + +fn get_inners() -> Vec { + vec![ + Inner { + a: 12, + b: 13, + c: 14, + d: 15, + }, + Inner { + a: 99, + b: 100, + c: 101, + d: 102, + }, + Inner { + a: 255, + b: 256, + c: 257, + d: 0, + }, + Inner { + a: 1000, + b: 2000, + c: 3000, + d: 0, + }, + Inner { + a: 0, + b: 0, + c: 0, + d: 0, + }, + ] +} + +fn get_struct_with_vec_of_structs() -> Vec { + let inner_a = Inner { + a: 12, + b: 13, + c: 14, + d: 15, + }; + + let inner_b = Inner { + a: 99, + b: 100, + c: 101, + d: 102, + }; + + let inner_c = Inner { + a: 255, + b: 256, + c: 257, + d: 0, + }; + + let a = StructWithVecOfStructs { + a: 42, + b: inner_a.clone(), + c: vec![inner_a.clone(), inner_b.clone(), inner_c.clone()], + }; + + let b = StructWithVecOfStructs { + c: vec![], + ..a.clone() + }; + + let c = StructWithVecOfStructs { + a: 800, + ..a.clone() + }; + + let d = StructWithVecOfStructs { + b: inner_c.clone(), + ..a.clone() + }; + + let e = StructWithVecOfStructs { + c: vec![inner_a.clone(), inner_b.clone()], + ..a.clone() + }; + + let f = StructWithVecOfStructs { + c: vec![inner_a.clone()], + ..a.clone() + }; + + vec![a, b, c, d, e, f] +} + +#[test] +fn test_struct_with_vec_of_structs() { + let variants = get_struct_with_vec_of_structs(); + + test_routine(variants[0].clone(), variants.clone()); + test_routine(variants[1].clone(), variants.clone()); + test_routine(variants[2].clone(), variants.clone()); + test_routine(variants[3].clone(), variants.clone()); + test_routine(variants[4].clone(), variants.clone()); + test_routine(variants[5].clone(), variants.clone()); +} + +#[derive(Clone, Debug, TreeHash, CachedTreeHash)] +pub struct StructWithVecOfStructWithVecOfStructs { + pub a: Vec, + pub b: u64, +} + +#[test] +fn test_struct_with_vec_of_struct_with_vec_of_structs() { + let structs = get_struct_with_vec_of_structs(); + + let variants = vec![ + StructWithVecOfStructWithVecOfStructs { + a: structs[..].to_vec(), + b: 99, + }, + StructWithVecOfStructWithVecOfStructs { a: vec![], b: 99 }, + StructWithVecOfStructWithVecOfStructs { + a: structs[0..2].to_vec(), + b: 99, + }, + StructWithVecOfStructWithVecOfStructs { + a: structs[0..2].to_vec(), + b: 100, + }, + StructWithVecOfStructWithVecOfStructs { + a: structs[0..1].to_vec(), + b: 100, + }, + StructWithVecOfStructWithVecOfStructs { + a: structs[0..4].to_vec(), + b: 100, + }, + StructWithVecOfStructWithVecOfStructs { + a: structs[0..5].to_vec(), + b: 8, + }, + ]; + + for v in &variants { + test_routine(v.clone(), variants.clone()); + } +} + +#[derive(Clone, Debug, TreeHash, CachedTreeHash)] +pub struct StructWithTwoVecs { + pub a: Vec, + pub b: Vec, +} + +fn get_struct_with_two_vecs() -> Vec { + let inners = get_inners(); + + vec![ + StructWithTwoVecs { + a: inners[..].to_vec(), + b: inners[..].to_vec(), + }, + StructWithTwoVecs { + a: inners[0..1].to_vec(), + b: inners[..].to_vec(), + }, + StructWithTwoVecs { + a: inners[0..1].to_vec(), + b: inners[0..2].to_vec(), + }, + StructWithTwoVecs { + a: inners[0..4].to_vec(), + b: inners[0..2].to_vec(), + }, + StructWithTwoVecs { + a: vec![], + b: inners[..].to_vec(), + }, + StructWithTwoVecs { + a: inners[..].to_vec(), + b: vec![], + }, + StructWithTwoVecs { + a: inners[0..3].to_vec(), + b: inners[0..1].to_vec(), + }, + ] +} + +#[test] +fn test_struct_with_two_vecs() { + let variants = get_struct_with_two_vecs(); + + for v in &variants { + test_routine(v.clone(), variants.clone()); + } +} + +#[test] +fn test_vec_of_struct_with_two_vecs() { + let structs = get_struct_with_two_vecs(); + + let variants = vec![ + structs[0..].to_vec(), + structs[0..2].to_vec(), + structs[2..3].to_vec(), + vec![], + structs[2..4].to_vec(), + ]; + + test_routine(variants[0].clone(), vec![variants[2].clone()]); + + for v in &variants { + test_routine(v.clone(), variants.clone()); + } +} + +#[derive(Clone, Debug, TreeHash, CachedTreeHash)] +pub struct U64AndTwoStructs { + pub a: u64, + pub b: Inner, + pub c: Inner, +} + +#[test] +fn test_u64_and_two_structs() { + let inners = get_inners(); + + let variants = vec![ + U64AndTwoStructs { + a: 99, + b: inners[0].clone(), + c: inners[1].clone(), + }, + U64AndTwoStructs { + a: 10, + b: inners[2].clone(), + c: inners[3].clone(), + }, + U64AndTwoStructs { + a: 0, + b: inners[1].clone(), + c: inners[1].clone(), + }, + U64AndTwoStructs { + a: 0, + b: inners[1].clone(), + c: inners[1].clone(), + }, + ]; + + for v in &variants { + test_routine(v.clone(), variants.clone()); + } +} + +#[derive(Clone, Debug, TreeHash, CachedTreeHash)] +pub struct Inner { + pub a: u64, + pub b: u64, + pub c: u64, + pub d: u64, +} + +fn generic_test(index: usize) { + let inner = Inner { + a: 1, + b: 2, + c: 3, + d: 4, + }; + + let mut cache = TreeHashCache::new(&inner).unwrap(); + + let changed_inner = match index { + 0 => Inner { + a: 42, + ..inner.clone() + }, + 1 => Inner { + b: 42, + ..inner.clone() + }, + 2 => Inner { + c: 42, + ..inner.clone() + }, + 3 => Inner { + d: 42, + ..inner.clone() + }, + _ => panic!("bad index"), + }; + + changed_inner.update_tree_hash_cache(&mut cache).unwrap(); + + let data1 = int_to_bytes32(1); + let data2 = int_to_bytes32(2); + let data3 = int_to_bytes32(3); + let data4 = int_to_bytes32(4); + + let mut data = vec![data1, data2, data3, data4]; + + data[index] = int_to_bytes32(42); + + let expected = merkleize(join(data)); + + let (cache_bytes, _, _) = cache.into_components(); + + assert_eq!(expected, cache_bytes); +} + +#[test] +fn cached_hash_on_inner() { + generic_test(0); + generic_test(1); + generic_test(2); + generic_test(3); +} + +#[test] +fn inner_builds() { + let data1 = int_to_bytes32(1); + let data2 = int_to_bytes32(2); + let data3 = int_to_bytes32(3); + let data4 = int_to_bytes32(4); + + let data = join(vec![data1, data2, data3, data4]); + let expected = merkleize(data); + + let inner = Inner { + a: 1, + b: 2, + c: 3, + d: 4, + }; + + let (cache_bytes, _, _) = TreeHashCache::new(&inner).unwrap().into_components(); + + assert_eq!(expected, cache_bytes); +} + +fn join(many: Vec>) -> Vec { + let mut all = vec![]; + for one in many { + all.extend_from_slice(&mut one.clone()) + } + all +} diff --git a/eth2/utils/ssz/Cargo.toml b/eth2/utils/ssz/Cargo.toml index fa042a8ac..ecefd69fe 100644 --- a/eth2/utils/ssz/Cargo.toml +++ b/eth2/utils/ssz/Cargo.toml @@ -8,5 +8,6 @@ edition = "2018" bytes = "0.4.9" ethereum-types = "0.5" hashing = { path = "../hashing" } +int_to_bytes = { path = "../int_to_bytes" } hex = "0.3" yaml-rust = "0.4" diff --git a/eth2/utils/ssz/src/impl_tree_hash.rs b/eth2/utils/ssz/src/impl_tree_hash.rs deleted file mode 100644 index 03976f637..000000000 --- a/eth2/utils/ssz/src/impl_tree_hash.rs +++ /dev/null @@ -1,85 +0,0 @@ -use super::ethereum_types::{Address, H256}; -use super::{merkle_hash, ssz_encode, TreeHash}; -use hashing::hash; - -impl TreeHash for u8 { - fn hash_tree_root(&self) -> Vec { - ssz_encode(self) - } -} - -impl TreeHash for u16 { - fn hash_tree_root(&self) -> Vec { - ssz_encode(self) - } -} - -impl TreeHash for u32 { - fn hash_tree_root(&self) -> Vec { - ssz_encode(self) - } -} - -impl TreeHash for u64 { - fn hash_tree_root(&self) -> Vec { - ssz_encode(self) - } -} - -impl TreeHash for usize { - fn hash_tree_root(&self) -> Vec { - ssz_encode(self) - } -} - -impl TreeHash for bool { - fn hash_tree_root(&self) -> Vec { - ssz_encode(self) - } -} - -impl TreeHash for Address { - fn hash_tree_root(&self) -> Vec { - ssz_encode(self) - } -} - -impl TreeHash for H256 { - fn hash_tree_root(&self) -> Vec { - ssz_encode(self) - } -} - -impl TreeHash for [u8] { - fn hash_tree_root(&self) -> Vec { - if self.len() > 32 { - return hash(&self); - } - self.to_vec() - } -} - -impl TreeHash for Vec -where - T: TreeHash, -{ - /// Returns the merkle_hash of a list of hash_tree_root values created - /// from the given list. - /// Note: A byte vector, Vec, must be converted to a slice (as_slice()) - /// to be handled properly (i.e. hashed) as byte array. - fn hash_tree_root(&self) -> Vec { - let mut tree_hashes = self.iter().map(|x| x.hash_tree_root()).collect(); - merkle_hash(&mut tree_hashes) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_impl_tree_hash_vec() { - let result = vec![1u32, 2, 3, 4, 5, 6, 7].hash_tree_root(); - assert_eq!(result.len(), 32); - } -} diff --git a/eth2/utils/ssz/src/lib.rs b/eth2/utils/ssz/src/lib.rs index cb3f63c48..0a00efa5d 100644 --- a/eth2/utils/ssz/src/lib.rs +++ b/eth2/utils/ssz/src/lib.rs @@ -12,17 +12,12 @@ extern crate ethereum_types; pub mod decode; pub mod encode; -mod signed_root; -pub mod tree_hash; mod impl_decode; mod impl_encode; -mod impl_tree_hash; pub use crate::decode::{decode, decode_ssz_list, Decodable, DecodeError}; pub use crate::encode::{Encodable, SszStream}; -pub use crate::signed_root::SignedRoot; -pub use crate::tree_hash::{merkle_hash, TreeHash}; pub use hashing::hash; diff --git a/eth2/utils/ssz/src/signed_root.rs b/eth2/utils/ssz/src/signed_root.rs deleted file mode 100644 index f7aeca4af..000000000 --- a/eth2/utils/ssz/src/signed_root.rs +++ /dev/null @@ -1,5 +0,0 @@ -use crate::TreeHash; - -pub trait SignedRoot: TreeHash { - fn signed_root(&self) -> Vec; -} diff --git a/eth2/utils/ssz/src/tree_hash.rs b/eth2/utils/ssz/src/tree_hash.rs deleted file mode 100644 index 85e56924c..000000000 --- a/eth2/utils/ssz/src/tree_hash.rs +++ /dev/null @@ -1,107 +0,0 @@ -use hashing::hash; - -const BYTES_PER_CHUNK: usize = 32; -const HASHSIZE: usize = 32; - -pub trait TreeHash { - fn hash_tree_root(&self) -> Vec; -} - -/// Returns a 32 byte hash of 'list' - a vector of byte vectors. -/// Note that this will consume 'list'. -pub fn merkle_hash(list: &mut Vec>) -> Vec { - // flatten list - let mut chunkz = list_to_blob(list); - - // get data_len as bytes. It will hashed will the merkle root - let mut datalen = list.len().to_le_bytes().to_vec(); - zpad(&mut datalen, 32); - - // merklelize - while chunkz.len() > HASHSIZE { - let mut new_chunkz: Vec = Vec::new(); - - for two_chunks in chunkz.chunks(BYTES_PER_CHUNK * 2) { - // Hash two chuncks together - new_chunkz.append(&mut hash(two_chunks)); - } - - chunkz = new_chunkz; - } - - chunkz.append(&mut datalen); - hash(&chunkz) -} - -fn list_to_blob(list: &mut Vec>) -> Vec { - // pack - fit as many many items per chunk as we can and then - // right pad to BYTES_PER_CHUNCK - let (items_per_chunk, chunk_count) = if list.is_empty() { - (1, 1) - } else { - let items_per_chunk = BYTES_PER_CHUNK / list[0].len(); - let chunk_count = list.len() / items_per_chunk; - (items_per_chunk, chunk_count) - }; - - let mut chunkz = Vec::new(); - if list.is_empty() { - // handle and empty list - chunkz.append(&mut vec![0; BYTES_PER_CHUNK * 2]); - } else if list[0].len() <= BYTES_PER_CHUNK { - // just create a blob here; we'll divide into - // chunked slices when we merklize - let mut chunk = Vec::with_capacity(BYTES_PER_CHUNK); - let mut item_count_in_chunk = 0; - chunkz.reserve(chunk_count * BYTES_PER_CHUNK); - for item in list.iter_mut() { - item_count_in_chunk += 1; - chunk.append(item); - - // completed chunk? - if item_count_in_chunk == items_per_chunk { - zpad(&mut chunk, BYTES_PER_CHUNK); - chunkz.append(&mut chunk); - item_count_in_chunk = 0; - } - } - - // left-over uncompleted chunk? - if item_count_in_chunk != 0 { - zpad(&mut chunk, BYTES_PER_CHUNK); - chunkz.append(&mut chunk); - } - } - - // extend the number of chunks to a power of two if necessary - if !chunk_count.is_power_of_two() { - let zero_chunks_count = chunk_count.next_power_of_two() - chunk_count; - chunkz.append(&mut vec![0; zero_chunks_count * BYTES_PER_CHUNK]); - } - - chunkz -} - -/// right pads with zeros making 'bytes' 'size' in length -fn zpad(bytes: &mut Vec, size: usize) { - if bytes.len() < size { - bytes.resize(size, 0); - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_merkle_hash() { - let data1 = vec![1; 32]; - let data2 = vec![2; 32]; - let data3 = vec![3; 32]; - let mut list = vec![data1, data2, data3]; - let result = merkle_hash(&mut list); - - //note: should test againt a known test hash value - assert_eq!(HASHSIZE, result.len()); - } -} diff --git a/eth2/utils/ssz_derive/src/lib.rs b/eth2/utils/ssz_derive/src/lib.rs index 9ba1de416..c5b53e934 100644 --- a/eth2/utils/ssz_derive/src/lib.rs +++ b/eth2/utils/ssz_derive/src/lib.rs @@ -188,145 +188,3 @@ pub fn ssz_decode_derive(input: TokenStream) -> TokenStream { }; output.into() } - -/// Returns a Vec of `syn::Ident` for each named field in the struct, whilst filtering out fields -/// that should not be tree hashed. -/// -/// # Panics -/// Any unnamed struct field (like in a tuple struct) will raise a panic at compile time. -fn get_tree_hashable_named_field_idents<'a>( - struct_data: &'a syn::DataStruct, -) -> Vec<&'a syn::Ident> { - struct_data - .fields - .iter() - .filter_map(|f| { - if should_skip_tree_hash(&f) { - None - } else { - Some(match &f.ident { - Some(ref ident) => ident, - _ => panic!("ssz_derive only supports named struct fields."), - }) - } - }) - .collect() -} - -/// Returns true if some field has an attribute declaring it should not be tree-hashed. -/// -/// The field attribute is: `#[tree_hash(skip_hashing)]` -fn should_skip_tree_hash(field: &syn::Field) -> bool { - for attr in &field.attrs { - if attr.tts.to_string() == "( skip_hashing )" { - return true; - } - } - false -} - -/// Implements `ssz::TreeHash` for some `struct`. -/// -/// Fields are processed in the order they are defined. -#[proc_macro_derive(TreeHash, attributes(tree_hash))] -pub fn ssz_tree_hash_derive(input: TokenStream) -> TokenStream { - let item = parse_macro_input!(input as DeriveInput); - - let name = &item.ident; - - let struct_data = match &item.data { - syn::Data::Struct(s) => s, - _ => panic!("ssz_derive only supports structs."), - }; - - let field_idents = get_tree_hashable_named_field_idents(&struct_data); - - let output = quote! { - impl ssz::TreeHash for #name { - fn hash_tree_root(&self) -> Vec { - let mut list: Vec> = Vec::new(); - #( - list.push(self.#field_idents.hash_tree_root()); - )* - - ssz::merkle_hash(&mut list) - } - } - }; - output.into() -} - -/// Returns `true` if some `Ident` should be considered to be a signature type. -fn type_ident_is_signature(ident: &syn::Ident) -> bool { - match ident.to_string().as_ref() { - "Signature" => true, - "AggregateSignature" => true, - _ => false, - } -} - -/// Takes a `Field` where the type (`ty`) portion is a path (e.g., `types::Signature`) and returns -/// the final `Ident` in that path. -/// -/// E.g., for `types::Signature` returns `Signature`. -fn final_type_ident(field: &syn::Field) -> &syn::Ident { - match &field.ty { - syn::Type::Path(path) => &path.path.segments.last().unwrap().value().ident, - _ => panic!("ssz_derive only supports Path types."), - } -} - -/// Implements `ssz::TreeHash` for some `struct`, whilst excluding any fields following and -/// including a field that is of type "Signature" or "AggregateSignature". -/// -/// See: -/// https://github.com/ethereum/eth2.0-specs/blob/master/specs/simple-serialize.md#signed-roots -/// -/// This is a rather horrendous macro, it will read the type of the object as a string and decide -/// if it's a signature by matching that string against "Signature" or "AggregateSignature". So, -/// it's important that you use those exact words as your type -- don't alias it to something else. -/// -/// If you can think of a better way to do this, please make an issue! -/// -/// Fields are processed in the order they are defined. -#[proc_macro_derive(SignedRoot)] -pub fn ssz_signed_root_derive(input: TokenStream) -> TokenStream { - let item = parse_macro_input!(input as DeriveInput); - - let name = &item.ident; - - let struct_data = match &item.data { - syn::Data::Struct(s) => s, - _ => panic!("ssz_derive only supports structs."), - }; - - let mut field_idents: Vec<&syn::Ident> = vec![]; - - for field in struct_data.fields.iter() { - let final_type_ident = final_type_ident(&field); - - if type_ident_is_signature(final_type_ident) { - break; - } else { - let ident = field - .ident - .as_ref() - .expect("ssz_derive only supports named_struct fields."); - field_idents.push(ident); - } - } - - let output = quote! { - impl ssz::SignedRoot for #name { - fn signed_root(&self) -> Vec { - let mut list: Vec> = Vec::new(); - #( - list.push(self.#field_idents.hash_tree_root()); - )* - - ssz::merkle_hash(&mut list) - } - } - }; - output.into() -} diff --git a/eth2/utils/swap_or_not_shuffle/src/shuffle_list.rs b/eth2/utils/swap_or_not_shuffle/src/shuffle_list.rs index e7e1e18e6..f60d793f2 100644 --- a/eth2/utils/swap_or_not_shuffle/src/shuffle_list.rs +++ b/eth2/utils/swap_or_not_shuffle/src/shuffle_list.rs @@ -18,6 +18,8 @@ const TOTAL_SIZE: usize = SEED_SIZE + ROUND_SIZE + POSITION_WINDOW_SIZE; /// Credits to [@protolambda](https://github.com/protolambda) for defining this algorithm. /// /// Shuffles if `forwards == true`, otherwise un-shuffles. +/// It holds that: shuffle_list(shuffle_list(l, r, s, true), r, s, false) == l +/// and: shuffle_list(shuffle_list(l, r, s, false), r, s, true) == l /// /// Returns `None` under any of the following conditions: /// - `list_size == 0` diff --git a/eth2/utils/tree_hash/Cargo.toml b/eth2/utils/tree_hash/Cargo.toml new file mode 100644 index 000000000..7e23d2165 --- /dev/null +++ b/eth2/utils/tree_hash/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "tree_hash" +version = "0.1.0" +authors = ["Paul Hauner "] +edition = "2018" + +[dev-dependencies] +tree_hash_derive = { path = "../tree_hash_derive" } + +[dependencies] +ethereum-types = "0.5" +hashing = { path = "../hashing" } +int_to_bytes = { path = "../int_to_bytes" } diff --git a/eth2/utils/tree_hash/README.md b/eth2/utils/tree_hash/README.md new file mode 100644 index 000000000..0498bfc3e --- /dev/null +++ b/eth2/utils/tree_hash/README.md @@ -0,0 +1,76 @@ +# Tree hashing + +Provides both cached and non-cached tree hashing methods. + +## Standard Tree Hash + +```rust +use tree_hash_derive::TreeHash; + +#[derive(TreeHash)] +struct Foo { + a: u64, + b: Vec, +} + +fn main() { + let foo = Foo { + a: 42, + b: vec![1, 2, 3] + }; + + println!("root: {}", foo.tree_hash_root()); +} +``` + +## Cached Tree Hash + + +```rust +use tree_hash_derive::{TreeHash, CachedTreeHash}; + +#[derive(TreeHash, CachedTreeHash)] +struct Foo { + a: u64, + b: Vec, +} + +#[derive(TreeHash, CachedTreeHash)] +struct Bar { + a: Vec, + b: u64, +} + +fn main() { + let bar = Bar { + a: vec![ + Foo { + a: 42, + b: vec![1, 2, 3] + } + ], + b: 42 + }; + + let modified_bar = Bar { + a: vec![ + Foo { + a: 100, + b: vec![1, 2, 3, 4, 5, 6] + }, + Foo { + a: 42, + b: vec![] + } + ], + b: 99 + }; + + + let mut hasher = CachedTreeHasher::new(&bar).unwrap(); + hasher.update(&modified_bar).unwrap(); + + // Assert that the cached tree hash matches a standard tree hash. + assert_eq!(hasher.tree_hash_root(), modified_bar.tree_hash_root()); +} +``` diff --git a/eth2/utils/tree_hash/src/impls.rs b/eth2/utils/tree_hash/src/impls.rs new file mode 100644 index 000000000..42ea9add0 --- /dev/null +++ b/eth2/utils/tree_hash/src/impls.rs @@ -0,0 +1,166 @@ +use super::*; +use crate::merkleize::merkle_root; +use ethereum_types::H256; +use hashing::hash; +use int_to_bytes::int_to_bytes32; + +macro_rules! impl_for_bitsize { + ($type: ident, $bit_size: expr) => { + impl TreeHash for $type { + fn tree_hash_type() -> TreeHashType { + TreeHashType::Basic + } + + fn tree_hash_packed_encoding(&self) -> Vec { + self.to_le_bytes().to_vec() + } + + fn tree_hash_packing_factor() -> usize { + HASHSIZE / ($bit_size / 8) + } + + #[allow(clippy::cast_lossless)] + fn tree_hash_root(&self) -> Vec { + int_to_bytes32(*self as u64) + } + } + }; +} + +impl_for_bitsize!(u8, 8); +impl_for_bitsize!(u16, 16); +impl_for_bitsize!(u32, 32); +impl_for_bitsize!(u64, 64); +impl_for_bitsize!(usize, 64); + +impl TreeHash for bool { + fn tree_hash_type() -> TreeHashType { + TreeHashType::Basic + } + + fn tree_hash_packed_encoding(&self) -> Vec { + (*self as u8).tree_hash_packed_encoding() + } + + fn tree_hash_packing_factor() -> usize { + u8::tree_hash_packing_factor() + } + + fn tree_hash_root(&self) -> Vec { + int_to_bytes32(*self as u64) + } +} + +impl TreeHash for [u8; 4] { + fn tree_hash_type() -> TreeHashType { + TreeHashType::Vector + } + + fn tree_hash_packed_encoding(&self) -> Vec { + unreachable!("bytesN should never be packed.") + } + + fn tree_hash_packing_factor() -> usize { + unreachable!("bytesN should never be packed.") + } + + fn tree_hash_root(&self) -> Vec { + merkle_root(&self[..]) + } +} + +impl TreeHash for H256 { + fn tree_hash_type() -> TreeHashType { + TreeHashType::Vector + } + + fn tree_hash_packed_encoding(&self) -> Vec { + self.as_bytes().to_vec() + } + + fn tree_hash_packing_factor() -> usize { + 1 + } + + fn tree_hash_root(&self) -> Vec { + merkle_root(&self.as_bytes().to_vec()) + } +} + +macro_rules! impl_for_list { + ($type: ty) => { + impl TreeHash for $type + where + T: TreeHash, + { + fn tree_hash_type() -> TreeHashType { + TreeHashType::List + } + + fn tree_hash_packed_encoding(&self) -> Vec { + unreachable!("List should never be packed.") + } + + fn tree_hash_packing_factor() -> usize { + unreachable!("List should never be packed.") + } + + fn tree_hash_root(&self) -> Vec { + let mut root_and_len = Vec::with_capacity(HASHSIZE * 2); + root_and_len.append(&mut vec_tree_hash_root(self)); + root_and_len.append(&mut int_to_bytes32(self.len() as u64)); + + hash(&root_and_len) + } + } + }; +} + +impl_for_list!(Vec); +impl_for_list!(&[T]); + +pub fn vec_tree_hash_root(vec: &[T]) -> Vec +where + T: TreeHash, +{ + let leaves = match T::tree_hash_type() { + TreeHashType::Basic => { + let mut leaves = + Vec::with_capacity((HASHSIZE / T::tree_hash_packing_factor()) * vec.len()); + + for item in vec { + leaves.append(&mut item.tree_hash_packed_encoding()); + } + + leaves + } + TreeHashType::Container | TreeHashType::List | TreeHashType::Vector => { + let mut leaves = Vec::with_capacity(vec.len() * HASHSIZE); + + for item in vec { + leaves.append(&mut item.tree_hash_root()) + } + + leaves + } + }; + + merkle_root(&leaves) +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn bool() { + let mut true_bytes: Vec = vec![1]; + true_bytes.append(&mut vec![0; 31]); + + let false_bytes: Vec = vec![0; 32]; + + assert_eq!(true.tree_hash_root(), true_bytes); + assert_eq!(false.tree_hash_root(), false_bytes); + } + +} diff --git a/eth2/utils/tree_hash/src/lib.rs b/eth2/utils/tree_hash/src/lib.rs new file mode 100644 index 000000000..2554e70c3 --- /dev/null +++ b/eth2/utils/tree_hash/src/lib.rs @@ -0,0 +1,74 @@ +pub mod impls; +pub mod merkleize; + +pub const BYTES_PER_CHUNK: usize = 32; +pub const HASHSIZE: usize = 32; +pub const MERKLE_HASH_CHUNK: usize = 2 * BYTES_PER_CHUNK; + +#[derive(Debug, PartialEq, Clone)] +pub enum TreeHashType { + Basic, + Vector, + List, + Container, +} + +pub trait TreeHash { + fn tree_hash_type() -> TreeHashType; + + fn tree_hash_packed_encoding(&self) -> Vec; + + fn tree_hash_packing_factor() -> usize; + + fn tree_hash_root(&self) -> Vec; +} + +pub trait SignedRoot: TreeHash { + fn signed_root(&self) -> Vec; +} + +#[macro_export] +macro_rules! tree_hash_ssz_encoding_as_vector { + ($type: ident) => { + impl tree_hash::TreeHash for $type { + fn tree_hash_type() -> tree_hash::TreeHashType { + tree_hash::TreeHashType::Vector + } + + fn tree_hash_packed_encoding(&self) -> Vec { + unreachable!("Vector should never be packed.") + } + + fn tree_hash_packing_factor() -> usize { + unreachable!("Vector should never be packed.") + } + + fn tree_hash_root(&self) -> Vec { + tree_hash::merkleize::merkle_root(&ssz::ssz_encode(self)) + } + } + }; +} + +#[macro_export] +macro_rules! tree_hash_ssz_encoding_as_list { + ($type: ident) => { + impl tree_hash::TreeHash for $type { + fn tree_hash_type() -> tree_hash::TreeHashType { + tree_hash::TreeHashType::List + } + + fn tree_hash_packed_encoding(&self) -> Vec { + unreachable!("List should never be packed.") + } + + fn tree_hash_packing_factor() -> usize { + unreachable!("List should never be packed.") + } + + fn tree_hash_root(&self) -> Vec { + ssz::ssz_encode(self).tree_hash_root() + } + } + }; +} diff --git a/eth2/utils/tree_hash/src/merkleize.rs b/eth2/utils/tree_hash/src/merkleize.rs new file mode 100644 index 000000000..9482895ec --- /dev/null +++ b/eth2/utils/tree_hash/src/merkleize.rs @@ -0,0 +1,69 @@ +use super::*; +use hashing::hash; + +pub fn merkle_root(bytes: &[u8]) -> Vec { + // TODO: replace this with a more memory efficient method. + efficient_merkleize(&bytes)[0..32].to_vec() +} + +pub fn efficient_merkleize(bytes: &[u8]) -> Vec { + // If the bytes are just one chunk (or less than one chunk) just return them. + if bytes.len() <= HASHSIZE { + let mut o = bytes.to_vec(); + o.resize(HASHSIZE, 0); + return o; + } + + let leaves = num_sanitized_leaves(bytes.len()); + let nodes = num_nodes(leaves); + let internal_nodes = nodes - leaves; + + let num_bytes = std::cmp::max(internal_nodes, 1) * HASHSIZE + bytes.len(); + + let mut o: Vec = vec![0; internal_nodes * HASHSIZE]; + + o.append(&mut bytes.to_vec()); + + assert_eq!(o.len(), num_bytes); + + let empty_chunk_hash = hash(&[0; MERKLE_HASH_CHUNK]); + + let mut i = nodes * HASHSIZE; + let mut j = internal_nodes * HASHSIZE; + + while i >= MERKLE_HASH_CHUNK { + i -= MERKLE_HASH_CHUNK; + + j -= HASHSIZE; + let hash = match o.get(i..i + MERKLE_HASH_CHUNK) { + // All bytes are available, hash as ususal. + Some(slice) => hash(slice), + // Unable to get all the bytes. + None => { + match o.get(i..) { + // Able to get some of the bytes, pad them out. + Some(slice) => { + let mut bytes = slice.to_vec(); + bytes.resize(MERKLE_HASH_CHUNK, 0); + hash(&bytes) + } + // Unable to get any bytes, use the empty-chunk hash. + None => empty_chunk_hash.clone(), + } + } + }; + + o[j..j + HASHSIZE].copy_from_slice(&hash); + } + + o +} + +fn num_sanitized_leaves(num_bytes: usize) -> usize { + let leaves = (num_bytes + HASHSIZE - 1) / HASHSIZE; + leaves.next_power_of_two() +} + +fn num_nodes(num_leaves: usize) -> usize { + 2 * num_leaves - 1 +} diff --git a/eth2/utils/tree_hash_derive/Cargo.toml b/eth2/utils/tree_hash_derive/Cargo.toml new file mode 100644 index 000000000..8544108a7 --- /dev/null +++ b/eth2/utils/tree_hash_derive/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "tree_hash_derive" +version = "0.1.0" +authors = ["Paul Hauner "] +edition = "2018" +description = "Procedural derive macros for SSZ tree hashing." + +[lib] +proc-macro = true + +[dev-dependencies] +tree_hash = { path = "../tree_hash" } +cached_tree_hash = { path = "../cached_tree_hash" } + +[dependencies] +syn = "0.15" +quote = "0.6" diff --git a/eth2/utils/tree_hash_derive/src/lib.rs b/eth2/utils/tree_hash_derive/src/lib.rs new file mode 100644 index 000000000..50727a89f --- /dev/null +++ b/eth2/utils/tree_hash_derive/src/lib.rs @@ -0,0 +1,210 @@ +#![recursion_limit = "256"] +extern crate proc_macro; + +use proc_macro::TokenStream; +use quote::{quote, ToTokens}; +use syn::{parse_macro_input, DeriveInput}; + +/// Returns a Vec of `syn::Ident` for each named field in the struct, whilst filtering out fields +/// that should not be hashed. +/// +/// # Panics +/// Any unnamed struct field (like in a tuple struct) will raise a panic at compile time. +fn get_hashable_named_field_idents<'a>(struct_data: &'a syn::DataStruct) -> Vec<&'a syn::Ident> { + struct_data + .fields + .iter() + .filter_map(|f| { + if should_skip_hashing(&f) { + None + } else { + Some(match &f.ident { + Some(ref ident) => ident, + _ => panic!("tree_hash_derive only supports named struct fields."), + }) + } + }) + .collect() +} + +/// Returns true if some field has an attribute declaring it should not be hashedd. +/// +/// The field attribute is: `#[tree_hash(skip_hashing)]` +fn should_skip_hashing(field: &syn::Field) -> bool { + field + .attrs + .iter() + .any(|attr| attr.into_token_stream().to_string() == "# [ tree_hash ( skip_hashing ) ]") +} + +/// Implements `tree_hash::CachedTreeHash` for some `struct`. +/// +/// Fields are hashed in the order they are defined. +#[proc_macro_derive(CachedTreeHash, attributes(tree_hash))] +pub fn subtree_derive(input: TokenStream) -> TokenStream { + let item = parse_macro_input!(input as DeriveInput); + + let name = &item.ident; + + let struct_data = match &item.data { + syn::Data::Struct(s) => s, + _ => panic!("tree_hash_derive only supports structs."), + }; + + let idents_a = get_hashable_named_field_idents(&struct_data); + let idents_b = idents_a.clone(); + let idents_c = idents_a.clone(); + + let output = quote! { + impl cached_tree_hash::CachedTreeHash for #name { + fn new_tree_hash_cache(&self, depth: usize) -> Result { + let tree = cached_tree_hash::TreeHashCache::from_subtrees( + self, + vec![ + #( + self.#idents_a.new_tree_hash_cache(depth)?, + )* + ], + depth + )?; + + Ok(tree) + } + + fn num_tree_hash_cache_chunks(&self) -> usize { + cached_tree_hash::BTreeOverlay::new(self, 0, 0).num_chunks() + } + + fn tree_hash_cache_schema(&self, depth: usize) -> cached_tree_hash::BTreeSchema { + let mut lengths = vec![]; + + #( + lengths.push(self.#idents_b.num_tree_hash_cache_chunks()); + )* + + cached_tree_hash::BTreeSchema::from_lengths(depth, lengths) + } + + fn update_tree_hash_cache(&self, cache: &mut cached_tree_hash::TreeHashCache) -> Result<(), cached_tree_hash::Error> { + let overlay = cached_tree_hash::BTreeOverlay::new(self, cache.chunk_index, 0); + + + // Skip the chunk index to the first leaf node of this struct. + cache.chunk_index = overlay.first_leaf_node(); + // Skip the overlay index to the first leaf node of this struct. + // cache.overlay_index += 1; + + // Recurse into the struct items, updating their caches. + #( + self.#idents_c.update_tree_hash_cache(cache)?; + )* + + // Iterate through the internal nodes, updating them if their children have changed. + cache.update_internal_nodes(&overlay)?; + + cache.chunk_index = overlay.next_node(); + + Ok(()) + } + } + }; + output.into() +} + +/// Implements `tree_hash::TreeHash` for some `struct`. +/// +/// Fields are hashed in the order they are defined. +#[proc_macro_derive(TreeHash, attributes(tree_hash))] +pub fn tree_hash_derive(input: TokenStream) -> TokenStream { + let item = parse_macro_input!(input as DeriveInput); + + let name = &item.ident; + + let struct_data = match &item.data { + syn::Data::Struct(s) => s, + _ => panic!("tree_hash_derive only supports structs."), + }; + + let idents = get_hashable_named_field_idents(&struct_data); + + let output = quote! { + impl tree_hash::TreeHash for #name { + fn tree_hash_type() -> tree_hash::TreeHashType { + tree_hash::TreeHashType::Container + } + + fn tree_hash_packed_encoding(&self) -> Vec { + unreachable!("Struct should never be packed.") + } + + fn tree_hash_packing_factor() -> usize { + unreachable!("Struct should never be packed.") + } + + fn tree_hash_root(&self) -> Vec { + let mut leaves = Vec::with_capacity(4 * tree_hash::HASHSIZE); + + #( + leaves.append(&mut self.#idents.tree_hash_root()); + )* + + tree_hash::merkleize::merkle_root(&leaves) + } + } + }; + output.into() +} + +#[proc_macro_derive(SignedRoot, attributes(signed_root))] +pub fn tree_hash_signed_root_derive(input: TokenStream) -> TokenStream { + let item = parse_macro_input!(input as DeriveInput); + + let name = &item.ident; + + let struct_data = match &item.data { + syn::Data::Struct(s) => s, + _ => panic!("tree_hash_derive only supports structs."), + }; + + let idents = get_signed_root_named_field_idents(&struct_data); + let num_elems = idents.len(); + + let output = quote! { + impl tree_hash::SignedRoot for #name { + fn signed_root(&self) -> Vec { + let mut leaves = Vec::with_capacity(#num_elems * tree_hash::HASHSIZE); + + #( + leaves.append(&mut self.#idents.tree_hash_root()); + )* + + tree_hash::merkleize::merkle_root(&leaves) + } + } + }; + output.into() +} + +fn get_signed_root_named_field_idents(struct_data: &syn::DataStruct) -> Vec<&syn::Ident> { + struct_data + .fields + .iter() + .filter_map(|f| { + if should_skip_signed_root(&f) { + None + } else { + Some(match &f.ident { + Some(ref ident) => ident, + _ => panic!("tree_hash_derive only supports named struct fields"), + }) + } + }) + .collect() +} + +fn should_skip_signed_root(field: &syn::Field) -> bool { + field + .attrs + .iter() + .any(|attr| attr.into_token_stream().to_string() == "# [ signed_root ( skip_hashing ) ]") +} diff --git a/eth2/utils/tree_hash_derive/tests/tests.rs b/eth2/utils/tree_hash_derive/tests/tests.rs new file mode 100644 index 000000000..d4fd55165 --- /dev/null +++ b/eth2/utils/tree_hash_derive/tests/tests.rs @@ -0,0 +1,179 @@ +use cached_tree_hash::{CachedTreeHash, TreeHashCache}; +use tree_hash::{merkleize::merkle_root, SignedRoot, TreeHash}; +use tree_hash_derive::{CachedTreeHash, SignedRoot, TreeHash}; + +#[derive(Clone, Debug, TreeHash, CachedTreeHash)] +pub struct Inner { + pub a: u64, + pub b: u64, + pub c: u64, + pub d: u64, +} + +fn test_standard_and_cached(original: &T, modified: &T) { + // let mut cache = original.new_tree_hash_cache().unwrap(); + let mut cache = TreeHashCache::new(original).unwrap(); + + let standard_root = original.tree_hash_root(); + let cached_root = cache.tree_hash_root().unwrap(); + assert_eq!(standard_root, cached_root); + + // Test after a modification + cache.update(modified).unwrap(); + let standard_root = modified.tree_hash_root(); + let cached_root = cache.tree_hash_root().unwrap(); + assert_eq!(standard_root, cached_root); +} + +#[test] +fn inner_standard_vs_cached() { + let original = Inner { + a: 1, + b: 2, + c: 3, + d: 4, + }; + let modified = Inner { + b: 42, + ..original.clone() + }; + + test_standard_and_cached(&original, &modified); +} + +#[derive(Clone, Debug, TreeHash, CachedTreeHash)] +pub struct Uneven { + pub a: u64, + pub b: u64, + pub c: u64, + pub d: u64, + pub e: u64, +} + +#[test] +fn uneven_standard_vs_cached() { + let original = Uneven { + a: 1, + b: 2, + c: 3, + d: 4, + e: 5, + }; + let modified = Uneven { + e: 42, + ..original.clone() + }; + + test_standard_and_cached(&original, &modified); +} + +#[derive(Clone, Debug, TreeHash, SignedRoot)] +pub struct SignedInner { + pub a: u64, + pub b: u64, + pub c: u64, + pub d: u64, + #[signed_root(skip_hashing)] + pub e: u64, +} + +#[test] +fn signed_root() { + let unsigned = Inner { + a: 1, + b: 2, + c: 3, + d: 4, + }; + let signed = SignedInner { + a: 1, + b: 2, + c: 3, + d: 4, + e: 5, + }; + + assert_eq!(unsigned.tree_hash_root(), signed.signed_root()); +} + +#[derive(TreeHash, SignedRoot)] +struct CryptoKitties { + best_kitty: u64, + worst_kitty: u8, + kitties: Vec, +} + +impl CryptoKitties { + fn new() -> Self { + CryptoKitties { + best_kitty: 9999, + worst_kitty: 1, + kitties: vec![2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43], + } + } + + fn hash(&self) -> Vec { + let mut leaves = vec![]; + leaves.append(&mut self.best_kitty.tree_hash_root()); + leaves.append(&mut self.worst_kitty.tree_hash_root()); + leaves.append(&mut self.kitties.tree_hash_root()); + merkle_root(&leaves) + } +} + +#[test] +fn test_simple_tree_hash_derive() { + let kitties = CryptoKitties::new(); + assert_eq!(kitties.tree_hash_root(), kitties.hash()); +} + +#[test] +fn test_simple_signed_root_derive() { + let kitties = CryptoKitties::new(); + assert_eq!(kitties.signed_root(), kitties.hash()); +} + +#[derive(TreeHash, SignedRoot)] +struct Casper { + friendly: bool, + #[tree_hash(skip_hashing)] + friends: Vec, + #[signed_root(skip_hashing)] + dead: bool, +} + +impl Casper { + fn new() -> Self { + Casper { + friendly: true, + friends: vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10], + dead: true, + } + } + + fn expected_signed_hash(&self) -> Vec { + let mut list = Vec::new(); + list.append(&mut self.friendly.tree_hash_root()); + list.append(&mut self.friends.tree_hash_root()); + merkle_root(&list) + } + + fn expected_tree_hash(&self) -> Vec { + let mut list = Vec::new(); + list.append(&mut self.friendly.tree_hash_root()); + list.append(&mut self.dead.tree_hash_root()); + merkle_root(&list) + } +} + +#[test] +fn test_annotated_tree_hash_derive() { + let casper = Casper::new(); + assert_eq!(casper.tree_hash_root(), casper.expected_tree_hash()); +} + +#[test] +fn test_annotated_signed_root_derive() { + let casper = Casper::new(); + assert_eq!(casper.signed_root(), casper.expected_signed_hash()); +} diff --git a/validator_client/Cargo.toml b/validator_client/Cargo.toml index 80477c8ea..7f6b0cee9 100644 --- a/validator_client/Cargo.toml +++ b/validator_client/Cargo.toml @@ -17,6 +17,7 @@ block_proposer = { path = "../eth2/block_proposer" } attester = { path = "../eth2/attester" } bls = { path = "../eth2/utils/bls" } ssz = { path = "../eth2/utils/ssz" } +tree_hash = { path = "../eth2/utils/tree_hash" } clap = "2.32.0" dirs = "1.0.3" grpcio = { version = "0.4", default-features = false, features = ["protobuf-codec"] } diff --git a/validator_client/src/attestation_producer/mod.rs b/validator_client/src/attestation_producer/mod.rs index 0fbc7bcba..d2dbdf2e2 100644 --- a/validator_client/src/attestation_producer/mod.rs +++ b/validator_client/src/attestation_producer/mod.rs @@ -8,7 +8,7 @@ use super::block_producer::{BeaconNodeError, PublishOutcome, ValidatorEvent}; use crate::signer::Signer; use beacon_node_attestation::BeaconNodeAttestation; use slog::{error, info, warn}; -use ssz::TreeHash; +use tree_hash::TreeHash; use types::{ AggregateSignature, Attestation, AttestationData, AttestationDataAndCustodyBit, AttestationDuty, Bitfield, @@ -123,7 +123,7 @@ impl<'a, B: BeaconNodeAttestation, S: Signer> AttestationProducer<'a, B, S> { data: attestation.clone(), custody_bit: false, } - .hash_tree_root(); + .tree_hash_root(); let sig = self.signer.sign_message(&message, domain)?; diff --git a/validator_client/src/block_producer/mod.rs b/validator_client/src/block_producer/mod.rs index 8b4f5abda..2689b302d 100644 --- a/validator_client/src/block_producer/mod.rs +++ b/validator_client/src/block_producer/mod.rs @@ -6,8 +6,8 @@ pub use self::beacon_node_block::{BeaconNodeError, PublishOutcome}; pub use self::grpc::BeaconBlockGrpcClient; use crate::signer::Signer; use slog::{error, info, warn}; -use ssz::{SignedRoot, TreeHash}; use std::sync::Arc; +use tree_hash::{SignedRoot, TreeHash}; use types::{BeaconBlock, ChainSpec, Domain, Fork, Slot}; #[derive(Debug, PartialEq)] @@ -86,7 +86,7 @@ impl<'a, B: BeaconNodeBlock, S: Signer> BlockProducer<'a, B, S> { pub fn produce_block(&mut self) -> Result { let epoch = self.slot.epoch(self.spec.slots_per_epoch); - let message = epoch.hash_tree_root(); + let message = epoch.tree_hash_root(); let randao_reveal = match self.signer.sign_message( &message, self.spec.get_domain(epoch, Domain::Randao, &self.fork),