From 444395960f05439df0648c63afceb65128252db3 Mon Sep 17 00:00:00 2001 From: Mathias Baumann Date: Tue, 13 Aug 2019 12:43:04 +0200 Subject: [PATCH 01/12] Fix link to Contract ABI in source code --- libsolidity/interface/ABI.cpp | 2 +- libsolidity/interface/ABI.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/libsolidity/interface/ABI.cpp b/libsolidity/interface/ABI.cpp index 6d450141d..5001620f9 100644 --- a/libsolidity/interface/ABI.cpp +++ b/libsolidity/interface/ABI.cpp @@ -15,7 +15,7 @@ along with solidity. If not, see . */ /** - * Utilities to handle the Contract ABI (https://github.com/ethereum/wiki/wiki/Ethereum-Contract-ABI) + * Utilities to handle the Contract ABI (https://solidity.readthedocs.io/en/develop/abi-spec.html) */ #include diff --git a/libsolidity/interface/ABI.h b/libsolidity/interface/ABI.h index f52081503..9e1ba294e 100644 --- a/libsolidity/interface/ABI.h +++ b/libsolidity/interface/ABI.h @@ -15,7 +15,7 @@ along with solidity. If not, see . */ /** - * Utilities to handle the Contract ABI (https://github.com/ethereum/wiki/wiki/Ethereum-Contract-ABI) + * Utilities to handle the Contract ABI (https://solidity.readthedocs.io/en/develop/abi-spec.html) */ #pragma once From 30373d5719014f1269cbd9492ae63c459285fe03 Mon Sep 17 00:00:00 2001 From: Mathias Baumann Date: Tue, 13 Aug 2019 12:44:29 +0200 Subject: [PATCH 02/12] Test scripts: Make build dir a variable --- scripts/tests.sh | 3 ++- test/cmdlineTests.sh | 7 ++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/scripts/tests.sh b/scripts/tests.sh index 80c19752f..32fc4134c 100755 --- a/scripts/tests.sh +++ b/scripts/tests.sh @@ -29,6 +29,7 @@ set -e REPO_ROOT="$(dirname "$0")/.." +SOLIDITY_BUILD_DIR="${SOLIDITY_BUILD_DIR:-build}" source "${REPO_ROOT}/scripts/common.sh" @@ -120,7 +121,7 @@ do fi set +e - "$REPO_ROOT"/build/test/soltest --show-progress $log -- --testpath "$REPO_ROOT"/test "$optimize" --evm-version "$vm" $SMT_FLAGS $force_abiv2_flag + "$REPO_ROOT"/${SOLIDITY_BUILD_DIR}/test/soltest --show-progress $log -- --testpath "$REPO_ROOT"/test "$optimize" --evm-version "$vm" $SMT_FLAGS $force_abiv2_flag if test "0" -ne "$?"; then exit 1 diff --git a/test/cmdlineTests.sh b/test/cmdlineTests.sh index ba0b1cf8b..a9ab91bc7 100755 --- a/test/cmdlineTests.sh +++ b/test/cmdlineTests.sh @@ -31,8 +31,9 @@ set -e ## GLOBAL VARIABLES REPO_ROOT=$(cd $(dirname "$0")/.. && pwd) +SOLIDITY_BUILD_DIR=${SOLIDITY_BUILD_DIR:-build} source "${REPO_ROOT}/scripts/common.sh" -SOLC="$REPO_ROOT/build/solc/solc" +SOLC="$REPO_ROOT/${SOLIDITY_BUILD_DIR}/solc/solc" INTERACTIVE=true if ! tty -s || [ "$CI" ] then @@ -439,8 +440,8 @@ SOLTMPDIR=$(mktemp -d) "$REPO_ROOT"/scripts/isolate_tests.py "$REPO_ROOT"/test/ "$REPO_ROOT"/scripts/isolate_tests.py "$REPO_ROOT"/docs/ docs - echo *.sol | xargs -P 4 -n 50 "$REPO_ROOT"/build/test/tools/solfuzzer --quiet --input-files - echo *.sol | xargs -P 4 -n 50 "$REPO_ROOT"/build/test/tools/solfuzzer --without-optimizer --quiet --input-files + echo *.sol | xargs -P 4 -n 50 "$REPO_ROOT"/${SOLIDITY_BUILD_DIR}/test/tools/solfuzzer --quiet --input-files + echo *.sol | xargs -P 4 -n 50 "$REPO_ROOT"/${SOLIDITY_BUILD_DIR}/test/tools/solfuzzer --without-optimizer --quiet --input-files ) rm -rf "$SOLTMPDIR" From 53af4e082ebb316bcfd682c3f2dfe9a23b3aff89 Mon Sep 17 00:00:00 2001 From: mingchuan Date: Tue, 6 Aug 2019 18:46:16 +0800 Subject: [PATCH 03/12] Callgraph generator. --- libyul/CMakeLists.txt | 2 + libyul/optimiser/CallGraphGenerator.cpp | 54 +++++++++++++++++++++++ libyul/optimiser/CallGraphGenerator.h | 57 +++++++++++++++++++++++++ 3 files changed, 113 insertions(+) create mode 100644 libyul/optimiser/CallGraphGenerator.cpp create mode 100644 libyul/optimiser/CallGraphGenerator.h diff --git a/libyul/CMakeLists.txt b/libyul/CMakeLists.txt index bc686ddf3..dee8d1224 100644 --- a/libyul/CMakeLists.txt +++ b/libyul/CMakeLists.txt @@ -62,6 +62,8 @@ add_library(yul optimiser/BlockFlattener.h optimiser/BlockHasher.cpp optimiser/BlockHasher.h + optimiser/CallGraphGenerator.cpp + optimiser/CallGraphGenerator.h optimiser/CommonSubexpressionEliminator.cpp optimiser/CommonSubexpressionEliminator.h optimiser/ControlFlowSimplifier.cpp diff --git a/libyul/optimiser/CallGraphGenerator.cpp b/libyul/optimiser/CallGraphGenerator.cpp new file mode 100644 index 000000000..0e2a63578 --- /dev/null +++ b/libyul/optimiser/CallGraphGenerator.cpp @@ -0,0 +1,54 @@ +/* + This file is part of solidity. + + solidity is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + solidity is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with solidity. If not, see . +*/ +/** + * Specific AST walker that generates the call graph. + */ + +#include +#include + +#include + +#include + +using namespace std; +using namespace dev; +using namespace yul; + + +void CallGraphGenerator::operator()(FunctionalInstruction const& _functionalInstruction) +{ + m_callGraph.insert(m_currentFunction, YulString{ + dev::eth::instructionInfo(_functionalInstruction.instruction).name + }); + ASTWalker::operator()(_functionalInstruction); +} + +void CallGraphGenerator::operator()(FunctionCall const& _functionCall) +{ + m_callGraph.insert(m_currentFunction, _functionCall.functionName.name); + ASTWalker::operator()(_functionCall); +} + +void CallGraphGenerator::operator()(FunctionDefinition const& _functionDefinition) +{ + YulString previousFunction = m_currentFunction; + m_currentFunction = _functionDefinition.name; + ASTWalker::operator()(_functionDefinition); + m_currentFunction = previousFunction; +} + diff --git a/libyul/optimiser/CallGraphGenerator.h b/libyul/optimiser/CallGraphGenerator.h new file mode 100644 index 000000000..212d29d04 --- /dev/null +++ b/libyul/optimiser/CallGraphGenerator.h @@ -0,0 +1,57 @@ +/* + This file is part of solidity. + + solidity is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + solidity is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with solidity. If not, see . +*/ +/** + * Specific AST walker that generates the call graph. + */ + +#pragma once + +#include + +#include + +#include + +#include +#include + +namespace yul +{ + +/** + * Specific AST walker that generates the call graph. + * + * The outermost (non-function) context is denoted by the empty string. + */ +class CallGraphGenerator: public ASTWalker +{ +public: + /// @returns the call graph of the visited AST. + InvertibleRelation const& callGraph() const { return m_callGraph; } + + using ASTWalker::operator(); + void operator()(FunctionalInstruction const& _functionalInstruction) override; + void operator()(FunctionCall const& _functionCall) override; + void operator()(FunctionDefinition const& _functionDefinition) override; + +private: + InvertibleRelation m_callGraph; + /// The name of the function we are currently visiting during traversal. + YulString m_currentFunction; +}; + +} From 57125de9ef504c7d4ff1c226d2d1d4cb5d16a17e Mon Sep 17 00:00:00 2001 From: chriseth Date: Tue, 13 Aug 2019 13:34:33 +0200 Subject: [PATCH 04/12] Remove ContainsMSize from side-effect-collector. --- libsolidity/analysis/SyntaxChecker.cpp | 5 +---- libyul/optimiser/LoadResolver.cpp | 2 +- libyul/optimiser/Semantics.cpp | 28 ++++++++++++++++++++++---- libyul/optimiser/Semantics.h | 28 +++++++++++++++++++++----- libyul/optimiser/StackCompressor.cpp | 2 +- libyul/optimiser/UnusedPruner.cpp | 2 +- 6 files changed, 51 insertions(+), 16 deletions(-) diff --git a/libsolidity/analysis/SyntaxChecker.cpp b/libsolidity/analysis/SyntaxChecker.cpp index 5402d7ff4..7bd917d0d 100644 --- a/libsolidity/analysis/SyntaxChecker.cpp +++ b/libsolidity/analysis/SyntaxChecker.cpp @@ -262,10 +262,7 @@ bool SyntaxChecker::visit(InlineAssembly const& _inlineAssembly) if (!m_useYulOptimizer) return false; - if (yul::SideEffectsCollector( - _inlineAssembly.dialect(), - _inlineAssembly.operations() - ).containsMSize()) + if (yul::MSizeFinder::containsMSize(_inlineAssembly.dialect(), _inlineAssembly.operations())) m_errorReporter.syntaxError( _inlineAssembly.location(), "The msize instruction cannot be used when the Yul optimizer is activated because " diff --git a/libyul/optimiser/LoadResolver.cpp b/libyul/optimiser/LoadResolver.cpp index b33963e94..2a68149fe 100644 --- a/libyul/optimiser/LoadResolver.cpp +++ b/libyul/optimiser/LoadResolver.cpp @@ -31,7 +31,7 @@ using namespace yul; void LoadResolver::run(Dialect const& _dialect, Block& _ast) { - bool containsMSize = SideEffectsCollector(_dialect, _ast).containsMSize(); + bool containsMSize = MSizeFinder::containsMSize(_dialect, _ast); LoadResolver{_dialect, !containsMSize}(_ast); } diff --git a/libyul/optimiser/Semantics.cpp b/libyul/optimiser/Semantics.cpp index 154a63bb5..5f1846ee3 100644 --- a/libyul/optimiser/Semantics.cpp +++ b/libyul/optimiser/Semantics.cpp @@ -61,8 +61,6 @@ void SideEffectsCollector::operator()(FunctionalInstruction const& _instr) m_sideEffectFree = false; if (!eth::SemanticInformation::sideEffectFreeIfNoMSize(_instr.instruction)) m_sideEffectFreeIfNoMSize = false; - if (_instr.instruction == eth::Instruction::MSIZE) - m_containsMSize = true; if (eth::SemanticInformation::invalidatesStorage(_instr.instruction)) m_invalidatesStorage = true; if (eth::SemanticInformation::invalidatesMemory(_instr.instruction)) @@ -81,8 +79,6 @@ void SideEffectsCollector::operator()(FunctionCall const& _functionCall) m_sideEffectFree = false; if (!f->sideEffectFreeIfNoMSize) m_sideEffectFreeIfNoMSize = false; - if (f->isMSize) - m_containsMSize = true; if (f->invalidatesStorage) m_invalidatesStorage = true; if (f->invalidatesMemory) @@ -98,6 +94,30 @@ void SideEffectsCollector::operator()(FunctionCall const& _functionCall) } } +bool MSizeFinder::containsMSize(Dialect const& _dialect, Block const& _ast) +{ + MSizeFinder finder(_dialect); + finder(_ast); + return finder.m_msizeFound; +} + +void MSizeFinder::operator()(FunctionalInstruction const& _instr) +{ + ASTWalker::operator()(_instr); + + if (_instr.instruction == eth::Instruction::MSIZE) + m_msizeFound = true; +} + +void MSizeFinder::operator()(FunctionCall const& _functionCall) +{ + ASTWalker::operator()(_functionCall); + + if (BuiltinFunction const* f = m_dialect.builtin(_functionCall.functionName.name)) + if (f->isMSize) + m_msizeFound = true; +} + MovableChecker::MovableChecker(Dialect const& _dialect, Expression const& _expression): MovableChecker(_dialect) { diff --git a/libyul/optimiser/Semantics.h b/libyul/optimiser/Semantics.h index 14fa128c9..393e2d114 100644 --- a/libyul/optimiser/Semantics.h +++ b/libyul/optimiser/Semantics.h @@ -53,7 +53,6 @@ public: return m_sideEffectFree; } bool sideEffectFreeIfNoMSize() const { return m_sideEffectFreeIfNoMSize; } - bool containsMSize() const { return m_containsMSize; } bool invalidatesStorage() const { return m_invalidatesStorage; } bool invalidatesMemory() const { return m_invalidatesMemory; } @@ -67,16 +66,35 @@ private: /// Is the current expression side-effect free up to msize, i.e. can be removed /// without changing the semantics except for the value returned by the msize instruction. bool m_sideEffectFreeIfNoMSize = true; - /// Does the current code contain the MSize operation? - /// Note that this is a purely syntactic property meaning that even if this is false, - /// the code can still contain calls to functions that contain the msize instruction. - bool m_containsMSize = false; /// If false, storage is guaranteed to be unchanged by the code under all /// circumstances. bool m_invalidatesStorage = false; bool m_invalidatesMemory = false; }; +/** + * Class that can be used to find out if certain code contains the MSize instruction. + * + * Note that this is a purely syntactic property meaning that even if this is false, + * the code can still contain calls to functions that contain the msize instruction. + * + * The only safe way to determine this is by passing the full AST. + */ +class MSizeFinder: public ASTWalker +{ +public: + static bool containsMSize(Dialect const& _dialect, Block const& _ast); + + using ASTWalker::operator(); + void operator()(FunctionalInstruction const& _instr); + void operator()(FunctionCall const& _funCall); + +private: + MSizeFinder(Dialect const& _dialect): m_dialect(_dialect) {} + Dialect const& m_dialect; + bool m_msizeFound = false; +}; + /** * Specific AST walker that determines whether an expression is movable * and collects the referenced variables. diff --git a/libyul/optimiser/StackCompressor.cpp b/libyul/optimiser/StackCompressor.cpp index 3c417c2dd..72f63b76d 100644 --- a/libyul/optimiser/StackCompressor.cpp +++ b/libyul/optimiser/StackCompressor.cpp @@ -165,7 +165,7 @@ bool StackCompressor::run( _object.code->statements.size() > 0 && _object.code->statements.at(0).type() == typeid(Block), "Need to run the function grouper before the stack compressor." ); - bool allowMSizeOptimzation = !SideEffectsCollector(_dialect, *_object.code).containsMSize(); + bool allowMSizeOptimzation = !MSizeFinder::containsMSize(_dialect, *_object.code); for (size_t iterations = 0; iterations < _maxIterations; iterations++) { map stackSurplus = CompilabilityChecker::run(_dialect, _object, _optimizeStackAllocation); diff --git a/libyul/optimiser/UnusedPruner.cpp b/libyul/optimiser/UnusedPruner.cpp index 1ce3af0a0..32d8a75b6 100644 --- a/libyul/optimiser/UnusedPruner.cpp +++ b/libyul/optimiser/UnusedPruner.cpp @@ -138,7 +138,7 @@ void UnusedPruner::runUntilStabilised( set const& _externallyUsedFunctions ) { - bool allowMSizeOptimization = !SideEffectsCollector(_dialect, _ast).containsMSize(); + bool allowMSizeOptimization = !MSizeFinder::containsMSize(_dialect, _ast); runUntilStabilised(_dialect, _ast, allowMSizeOptimization, _externallyUsedFunctions); } From bd105ad4b1d6f2026a33375f853139c0dcf6454b Mon Sep 17 00:00:00 2001 From: Daniel Kirchner Date: Mon, 3 Jun 2019 18:27:29 +0200 Subject: [PATCH 05/12] Experimental static Z3 PPA script. --- scripts/deps-ppa/static_z3.sh | 229 ++++++++++++++++++++++++++++++++++ scripts/install_static_z3.sh | 9 ++ scripts/release_ppa.sh | 9 +- 3 files changed, 246 insertions(+), 1 deletion(-) create mode 100755 scripts/deps-ppa/static_z3.sh create mode 100644 scripts/install_static_z3.sh diff --git a/scripts/deps-ppa/static_z3.sh b/scripts/deps-ppa/static_z3.sh new file mode 100755 index 000000000..b971fdd24 --- /dev/null +++ b/scripts/deps-ppa/static_z3.sh @@ -0,0 +1,229 @@ +#!/usr/bin/env bash +############################################################################## +## This is used to package .deb packages and upload them to the launchpad +## ppa servers for building. +## +## The gnupg key for "builds@ethereum.org" has to be present in order to sign +## the package. +## +## It will clone the Z3 git from github on the specified version tag, +## create a source archive and push it to the ubuntu ppa servers. +## +## This requires the following entries in /etc/dput.cf: +## +## [cpp-build-deps] +## fqdn = ppa.launchpad.net +## method = ftp +## incoming = ~ethereum/cpp-build-deps +## login = anonymous + +## +############################################################################## + +set -ev + +keyid=70D110489D66E2F6 +email=builds@ethereum.org +packagename=libz3-static-dev +version=4.8.5 + +DISTRIBUTIONS="bionic disco" + +for distribution in $DISTRIBUTIONS +do +cd /tmp/ +rm -rf $distribution +mkdir $distribution +cd $distribution + +pparepo=cpp-build-deps +ppafilesurl=https://launchpad.net/~ethereum/+archive/ubuntu/${pparepo}/+files + +# Fetch source +git clone --depth 1 --branch Z3-${version} https://github.com/Z3Prover/z3.git +cd z3 +debversion="$version" + +CMAKE_OPTIONS="-DBUILD_LIBZ3_SHARED=OFF -DCMAKE_BUILD_TYPE=Release" + +# gzip will create different tars all the time and we are not allowed +# to upload the same file twice with different contents, so we only +# create it once. +if [ ! -e /tmp/${packagename}_${debversion}.orig.tar.gz ] +then + tar --exclude .git -czf /tmp/${packagename}_${debversion}.orig.tar.gz . +fi +cp /tmp/${packagename}_${debversion}.orig.tar.gz ../ + +# Create debian package information + +mkdir debian +echo 9 > debian/compat +# TODO: the Z3 packages have different build dependencies +cat < debian/control +Source: libz3-static-dev +Section: science +Priority: extra +Maintainer: Daniel Kirchner +Build-Depends: debhelper (>= 9.0.0), + cmake, + g++, + git, + libgmp-dev, + dh-python, + python +Standards-Version: 3.9.6 +Homepage: https://github.com/Z3Prover/z3 +Vcs-Git: git://github.com/Z3Prover/z3.git +Vcs-Browser: https://github.com/Z3Prover/z3 + +Package: libz3-static-dev +Section: libdevel +Architecture: any-i386 any-amd64 +Multi-Arch: same +Depends: \${shlibs:Depends}, \${misc:Depends} +Description: theorem prover from Microsoft Research - development files (static library) + Z3 is a state-of-the art theorem prover from Microsoft Research. It can be + used to check the satisfiability of logical formulas over one or more + theories. Z3 offers a compelling match for software analysis and verification + tools, since several common software constructs map directly into supported + theories. + . + This package can be used to invoke Z3 via its C++ API. +EOF +cat < debian/rules +#!/usr/bin/make -f +# -*- makefile -*- +# Sample debian/rules that uses debhelper. +# +# This file was originally written by Joey Hess and Craig Small. +# As a special exception, when this file is copied by dh-make into a +# dh-make output file, you may use that output file without restriction. +# This special exception was added by Craig Small in version 0.37 of dh-make. +# +# Modified to make a template file for a multi-binary package with separated +# build-arch and build-indep targets by Bill Allombert 2001 + +# Uncomment this to turn on verbose mode. +export DH_VERBOSE=1 + +# This has to be exported to make some magic below work. +export DH_OPTIONS + + +%: + dh \$@ --buildsystem=cmake + +override_dh_auto_test: + +override_dh_shlibdeps: + dh_shlibdeps --dpkg-shlibdeps-params=--ignore-missing-info + +override_dh_auto_configure: + dh_auto_configure -- ${CMAKE_OPTIONS} + +override_dh_auto_install: + dh_auto_install --destdir debian/tmp +EOF +cat < debian/libz3-static-dev.install +usr/include/* +usr/lib/*/libz3.a +usr/lib/*/cmake/z3/* +EOF +cat < debian/copyright +Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ +Upstream-Name: z3 +Source: https://github.com/Z3Prover/z3 + +Files: * +Copyright: Microsoft Corporation +License: Expat + Permission is hereby granted, free of charge, to any person obtaining a copy of + this software and associated documentation files (the "Software"), to deal in + the Software without restriction, including without limitation the rights to + use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + of the Software, and to permit persons to whom the Software is furnished to do + so, subject to the following conditions: + . + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + . + THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. + +Files: debian/* +Copyright: 2019 Ethereum +License: GPL-3.0+ +This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + . + This package is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + . + You should have received a copy of the GNU General Public License + along with this program. If not, see . + . + On Debian systems, the complete text of the GNU General + Public License version 3 can be found in "/usr/share/common-licenses/GPL-3". +EOF +cat < debian/changelog +libz3-static-dev (0.0.1-0ubuntu1) saucy; urgency=low + + * Initial release. + + -- Daniel Mon, 03 Jun 2019 14:50:20 +0000 +EOF +mkdir debian/source +echo "3.0 (quilt)" > debian/source/format +chmod +x debian/rules + +versionsuffix=0ubuntu1~${distribution} +EMAIL="$email" dch -v 1:${debversion}-${versionsuffix} "build of ${version}" + +# build source package +# If packages is rejected because original source is already present, add +# -sd to remove it from the .changes file +# -d disables the build dependencies check +debuild -S -d -sa -us -uc + +# prepare .changes file for Launchpad +sed -i -e s/UNRELEASED/${distribution}/ -e s/urgency=medium/urgency=low/ ../*.changes + +# check if ubuntu already has the source tarball +( +cd .. +orig=${packagename}_${debversion}.orig.tar.gz +orig_size=$(ls -l $orig | cut -d ' ' -f 5) +orig_sha1=$(sha1sum $orig | cut -d ' ' -f 1) +orig_sha256=$(sha256sum $orig | cut -d ' ' -f 1) +orig_md5=$(md5sum $orig | cut -d ' ' -f 1) + +if wget --quiet -O $orig-tmp "$ppafilesurl/$orig" +then + echo "[WARN] Original tarball found in Ubuntu archive, using it instead" + mv $orig-tmp $orig + new_size=$(ls -l *.orig.tar.gz | cut -d ' ' -f 5) + new_sha1=$(sha1sum $orig | cut -d ' ' -f 1) + new_sha256=$(sha256sum $orig | cut -d ' ' -f 1) + new_md5=$(md5sum $orig | cut -d ' ' -f 1) + sed -i -e s,$orig_sha1,$new_sha1,g -e s,$orig_sha256,$new_sha256,g -e s,$orig_size,$new_size,g -e s,$orig_md5,$new_md5,g *.dsc + sed -i -e s,$orig_sha1,$new_sha1,g -e s,$orig_sha256,$new_sha256,g -e s,$orig_size,$new_size,g -e s,$orig_md5,$new_md5,g *.changes +fi +) + +# sign the package +debsign --re-sign -k ${keyid} ../${packagename}_${debversion}-${versionsuffix}_source.changes + +# upload +dput ${pparepo} ../${packagename}_${debversion}-${versionsuffix}_source.changes + +done diff --git a/scripts/install_static_z3.sh b/scripts/install_static_z3.sh new file mode 100644 index 000000000..7748fb1f3 --- /dev/null +++ b/scripts/install_static_z3.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +git clone --depth 1 --branch z3-4.8.1 https://github.com/Z3Prover/z3.git +cd z3 +mkdir build +cd build +LDFLAGS="-static" cmake -DBUILD_LIBZ3_SHARED=OFF .. +make -j 4 +make install \ No newline at end of file diff --git a/scripts/release_ppa.sh b/scripts/release_ppa.sh index 74daaa0b5..78ea3118b 100755 --- a/scripts/release_ppa.sh +++ b/scripts/release_ppa.sh @@ -83,8 +83,15 @@ else else pparepo=ethereum-dev fi - SMTDEPENDENCY="libcvc4-dev, + if [ $distribution = disco ] + then + SMTDEPENDENCY="libz3-static-dev, + libcvc4-dev, " + else + SMTDEPENDENCY="libz3-static-dev, + " + fi CMAKE_OPTIONS="" fi ppafilesurl=https://launchpad.net/~ethereum/+archive/ubuntu/${pparepo}/+files From e91c6acbc30efdfaee3303c1787a1d97e0af6a5c Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Fri, 9 Aug 2019 11:28:22 +0200 Subject: [PATCH 06/12] Defaulting to C++17 for building. --- .circleci/config.yml | 8 ++++---- .travis.yml | 2 +- CMakeLists.txt | 7 +------ cmake/jsoncpp.cmake | 4 ++-- cmake/toolchains/cxx17.cmake | 4 ---- cmake/toolchains/cxx20.cmake | 4 ++++ cmake/toolchains/default.cmake | 4 ++-- docs/installing-solidity.rst | 2 +- libdevcore/CMakeLists.txt | 2 +- scripts/install_cmake.sh | 14 ++++++++------ 10 files changed, 24 insertions(+), 27 deletions(-) delete mode 100644 cmake/toolchains/cxx17.cmake create mode 100644 cmake/toolchains/cxx20.cmake diff --git a/.circleci/config.yml b/.circleci/config.yml index e4b4d0654..f6346c2d1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -292,13 +292,13 @@ jobs: command: codecov --flags all --gcov-root build - store_artifacts: *artifacts_test_results - # Builds in C++17 mode and uses debug build in order to speed up. + # Builds in C++20 mode and uses debug build in order to speed up. # Do *NOT* store any artifacts or workspace as we don't run tests on this build. - b_ubu_cxx17: + b_ubu_cxx20: <<: *build_ubuntu1904 environment: CMAKE_BUILD_TYPE: Debug - CMAKE_OPTIONS: -DCMAKE_TOOLCHAIN_FILE=cmake/toolchains/cxx17.cmake -DUSE_CVC4=OFF + CMAKE_OPTIONS: -DCMAKE_TOOLCHAIN_FILE=cmake/toolchains/cxx20.cmake -DUSE_CVC4=OFF steps: - checkout - run: *run_build @@ -612,7 +612,7 @@ workflows: # build-only - b_docs: *workflow_trigger_on_tags - b_archlinux: *workflow_trigger_on_tags - - b_ubu_cxx17: *workflow_trigger_on_tags + - b_ubu_cxx20: *workflow_trigger_on_tags - b_ubu_ossfuzz: *workflow_trigger_on_tags # OS/X build and tests diff --git a/.travis.yml b/.travis.yml index 7e1114974..5754fa0a9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -188,7 +188,7 @@ cache: install: - test $SOLC_INSTALL_DEPS_TRAVIS != On || (scripts/install_deps.sh) - - test "$TRAVIS_OS_NAME" != "linux" || (scripts/install_cmake.sh) + - test "$TRAVIS_OS_NAME" != "linux" || (sudo scripts/install_cmake.sh) before_script: # Disable tests unless run on the release branch, on tags or with daily cron diff --git a/CMakeLists.txt b/CMakeLists.txt index f3d724d26..a1927bf33 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.5.0) +cmake_minimum_required(VERSION 3.9.0) set(ETH_CMAKE_DIR "${CMAKE_CURRENT_LIST_DIR}/cmake" CACHE PATH "The the path to the cmake directory") list(APPEND CMAKE_MODULE_PATH ${ETH_CMAKE_DIR}) @@ -13,11 +13,6 @@ eth_policy() set(PROJECT_VERSION "0.5.12") project(solidity VERSION ${PROJECT_VERSION} LANGUAGES C CXX) -if (${CMAKE_VERSION} VERSION_LESS "3.9.0") - # needed for the big endian test for older cmake versions - enable_language(C) -endif() - include(TestBigEndian) TEST_BIG_ENDIAN(IS_BIG_ENDIAN) if (IS_BIG_ENDIAN) diff --git a/cmake/jsoncpp.cmake b/cmake/jsoncpp.cmake index 4ca8581dc..48cec7318 100644 --- a/cmake/jsoncpp.cmake +++ b/cmake/jsoncpp.cmake @@ -15,9 +15,9 @@ set(JSONCPP_INCLUDE_DIR "${prefix}/include") # versions used in the CI runs. if(EMSCRIPTEN) # Do not include all flags in CMAKE_CXX_FLAGS for emscripten, - # but only use -std=c++14. Using all flags causes build failures + # but only use -std=c++17. Using all flags causes build failures # at the moment. - set(JSONCPP_CXX_FLAGS -std=c++14) + set(JSONCPP_CXX_FLAGS -std=c++17) else() set(JSONCPP_CXX_FLAGS ${CMAKE_CXX_FLAGS}) endif() diff --git a/cmake/toolchains/cxx17.cmake b/cmake/toolchains/cxx17.cmake deleted file mode 100644 index 04a865ebd..000000000 --- a/cmake/toolchains/cxx17.cmake +++ /dev/null @@ -1,4 +0,0 @@ -# Require C++17. -set(CMAKE_CXX_STANDARD 17) -set(CMAKE_CXX_STANDARD_REQUIRED TRUE) -set(CMAKE_CXX_EXTENSIONS OFF) diff --git a/cmake/toolchains/cxx20.cmake b/cmake/toolchains/cxx20.cmake new file mode 100644 index 000000000..ad34e5749 --- /dev/null +++ b/cmake/toolchains/cxx20.cmake @@ -0,0 +1,4 @@ +# Require C++20. +set(CMAKE_CXX_STANDARD 20) # This requires at least CMake 3.12 to understand this C++20 flag +set(CMAKE_CXX_STANDARD_REQUIRED TRUE) +set(CMAKE_CXX_EXTENSIONS OFF) diff --git a/cmake/toolchains/default.cmake b/cmake/toolchains/default.cmake index baf859b70..07fc80e8e 100644 --- a/cmake/toolchains/default.cmake +++ b/cmake/toolchains/default.cmake @@ -1,4 +1,4 @@ -# Require C++14. -set(CMAKE_CXX_STANDARD 14) +# Require C++17. +set(CMAKE_CXX_STANDARD 17) # This requires at least CMake 3.8 to accept this C++17 flag. set(CMAKE_CXX_STANDARD_REQUIRED TRUE) set(CMAKE_CXX_EXTENSIONS OFF) diff --git a/docs/installing-solidity.rst b/docs/installing-solidity.rst index 2d8f257d4..ade678087 100644 --- a/docs/installing-solidity.rst +++ b/docs/installing-solidity.rst @@ -184,7 +184,7 @@ The following are dependencies for all builds of Solidity: +-----------------------------------+-------------------------------------------------------+ | Software | Notes | +===================================+=======================================================+ -| `CMake`_ (version 3.5+) | Cross-platform build file generator. | +| `CMake`_ (version 3.9+) | Cross-platform build file generator. | +-----------------------------------+-------------------------------------------------------+ | `Boost`_ (version 1.65+) | C++ libraries. | +-----------------------------------+-------------------------------------------------------+ diff --git a/libdevcore/CMakeLists.txt b/libdevcore/CMakeLists.txt index a092e3140..8bc3bb525 100644 --- a/libdevcore/CMakeLists.txt +++ b/libdevcore/CMakeLists.txt @@ -34,6 +34,6 @@ set(sources ) add_library(devcore ${sources}) -target_link_libraries(devcore PUBLIC jsoncpp Boost::boost Boost::filesystem Boost::regex Boost::system Threads::Threads) +target_link_libraries(devcore PUBLIC jsoncpp Boost::boost Boost::filesystem Boost::regex Boost::system) target_include_directories(devcore PUBLIC "${CMAKE_SOURCE_DIR}") add_dependencies(devcore solidity_BuildInfo.h) diff --git a/scripts/install_cmake.sh b/scripts/install_cmake.sh index e334b2c90..134b86f09 100755 --- a/scripts/install_cmake.sh +++ b/scripts/install_cmake.sh @@ -6,16 +6,18 @@ set -e -VERSION=3.7.1 -PREFIX=~/.local +VERSION_MAJOR=3 +VERSION_MINOR=15 +VERSION_MICRO=2 +VERSION=$VERSION_MAJOR.$VERSION_MINOR.$VERSION_MICRO +PREFIX="/usr/local" OS=$(uname -s) case $OS in -Linux) SHA256=7b4b7a1d9f314f45722899c0521c261e4bfab4a6b532609e37fef391da6bade2;; -Darwin) SHA256=1851d1448964893fdc5a8c05863326119f397a3790e0c84c40b83499c7960267;; +Linux) SHA256=f8cbec2abc433938bd9378b129d1d288bb33b8b5a277afe19644683af6e32a59;; +Darwin) SHA256=7ec056d641b8cbea98b220efdcc99da1991758a370063dcac3a0cd388d6b30b6;; esac - BIN=$PREFIX/bin PATH=$PREFIX/bin:$PATH @@ -24,7 +26,7 @@ if test -f $BIN/cmake && ($BIN/cmake --version | grep -q "$VERSION"); then echo "CMake $VERSION already installed in $BIN" else FILE=cmake-$VERSION-$OS-x86_64.tar.gz - URL=https://cmake.org/files/v3.7/$FILE + URL=https://cmake.org/files/v$VERSION_MAJOR.$VERSION_MINOR/$FILE ERROR=0 TMPFILE=$(mktemp --tmpdir cmake-$VERSION-$OS-x86_64.XXXXXXXX.tar.gz) echo "Downloading CMake ($URL)..." From e396dc7246250245e789fb9038c112a2c4f7a073 Mon Sep 17 00:00:00 2001 From: chriseth Date: Tue, 13 Aug 2019 18:42:51 +0200 Subject: [PATCH 07/12] Properly set storage and memory after erasing potentially destroyed keys --- libyul/optimiser/DataFlowAnalyzer.cpp | 6 ++---- .../loadResolver/re_store_memory.yul | 21 +++++++++++++++++++ .../loadResolver/re_store_storage.yul | 21 +++++++++++++++++++ 3 files changed, 44 insertions(+), 4 deletions(-) create mode 100644 test/libyul/yulOptimizerTests/loadResolver/re_store_memory.yul create mode 100644 test/libyul/yulOptimizerTests/loadResolver/re_store_storage.yul diff --git a/libyul/optimiser/DataFlowAnalyzer.cpp b/libyul/optimiser/DataFlowAnalyzer.cpp index a927a7eca..7c852fb47 100644 --- a/libyul/optimiser/DataFlowAnalyzer.cpp +++ b/libyul/optimiser/DataFlowAnalyzer.cpp @@ -43,7 +43,6 @@ void DataFlowAnalyzer::operator()(ExpressionStatement& _statement) if (auto vars = isSimpleStore(dev::eth::Instruction::SSTORE, _statement)) { ASTModifier::operator()(_statement); - m_storage.set(vars->first, vars->second); set keysToErase; for (auto const& item: m_storage.values) if (!( @@ -53,6 +52,7 @@ void DataFlowAnalyzer::operator()(ExpressionStatement& _statement) keysToErase.insert(item.first); for (YulString const& key: keysToErase) m_storage.eraseKey(key); + m_storage.set(vars->first, vars->second); } else if (auto vars = isSimpleStore(dev::eth::Instruction::MSTORE, _statement)) { @@ -61,11 +61,9 @@ void DataFlowAnalyzer::operator()(ExpressionStatement& _statement) for (auto const& item: m_memory.values) if (!m_knowledgeBase.knownToBeDifferentByAtLeast32(vars->first, item.first)) keysToErase.insert(item.first); - // TODO is it fine to do that here? - // can we also move the storage above? - m_memory.set(vars->first, vars->second); for (YulString const& key: keysToErase) m_memory.eraseKey(key); + m_memory.set(vars->first, vars->second); } else { diff --git a/test/libyul/yulOptimizerTests/loadResolver/re_store_memory.yul b/test/libyul/yulOptimizerTests/loadResolver/re_store_memory.yul new file mode 100644 index 000000000..7fb27c996 --- /dev/null +++ b/test/libyul/yulOptimizerTests/loadResolver/re_store_memory.yul @@ -0,0 +1,21 @@ +{ + let a := 0 + let b := 1 + let c := 2 + mstore(a, b) + sstore(0, mload(a)) + mstore(a, c) + sstore(10, mload(a)) +} +// ==== +// step: loadResolver +// ---- +// { +// let a := 0 +// let b := 1 +// let c := 2 +// mstore(a, b) +// sstore(a, b) +// mstore(a, c) +// sstore(10, c) +// } diff --git a/test/libyul/yulOptimizerTests/loadResolver/re_store_storage.yul b/test/libyul/yulOptimizerTests/loadResolver/re_store_storage.yul new file mode 100644 index 000000000..aed91e002 --- /dev/null +++ b/test/libyul/yulOptimizerTests/loadResolver/re_store_storage.yul @@ -0,0 +1,21 @@ +{ + let a := 0 + let b := 1 + let c := 2 + sstore(a, b) + mstore(0, sload(a)) + sstore(a, c) + mstore(32, sload(a)) +} +// ==== +// step: loadResolver +// ---- +// { +// let a := 0 +// let b := 1 +// let c := 2 +// sstore(a, b) +// mstore(a, b) +// sstore(a, c) +// mstore(32, c) +// } From 3c927eeeda35d03e60c7855d46d9e5b9e909be93 Mon Sep 17 00:00:00 2001 From: Mathias Baumann Date: Fri, 9 Aug 2019 12:28:17 +0200 Subject: [PATCH 08/12] Add missing docker file for Ubuntu 18.04 --- .circleci/docker/Dockerfile.ubuntu1804 | 117 +++++++++++++++++++++++++ 1 file changed, 117 insertions(+) create mode 100644 .circleci/docker/Dockerfile.ubuntu1804 diff --git a/.circleci/docker/Dockerfile.ubuntu1804 b/.circleci/docker/Dockerfile.ubuntu1804 new file mode 100644 index 000000000..8d6e25289 --- /dev/null +++ b/.circleci/docker/Dockerfile.ubuntu1804 @@ -0,0 +1,117 @@ +# vim:syntax=dockerfile +#------------------------------------------------------------------------------ +# Dockerfile for building and testing Solidity Compiler on CI +# Target: Ubuntu 18.04 (Bionic Beaver) +# URL: https://hub.docker.com/r/ethereum/solidity-buildpack-deps +# +# This file is part of solidity. +# +# solidity is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# solidity is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with solidity. If not, see +# +# (c) 2016-2019 solidity contributors. +#------------------------------------------------------------------------------ +FROM buildpack-deps:bionic AS base + +ARG DEBIAN_FRONTEND=noninteractive + +RUN set -ex; \ + dist=$(grep DISTRIB_CODENAME /etc/lsb-release | cut -d= -f2); \ + echo "deb http://ppa.launchpad.net/ethereum/cpp-build-deps/ubuntu $dist main" >> /etc/apt/sources.list ; \ + apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 1c52189c923f6ca9 ; \ + apt-get update; \ + apt-get install -qqy --no-install-recommends \ + build-essential \ + software-properties-common \ + cmake ninja-build clang++-8 \ + libboost-regex-dev libboost-filesystem-dev libboost-test-dev libboost-system-dev \ + libboost-program-options-dev \ + libjsoncpp-dev \ + llvm-8-dev libz3-static-dev \ + ; \ + apt-get install -qy python-pip python-sphinx; \ + update-alternatives --install /usr/bin/llvm-symbolizer llvm-symbolizer /usr/bin/llvm-symbolizer-8 1; \ + pip install codecov; \ + rm -rf /var/lib/apt/lists/* + +FROM base AS libraries + +# OSSFUZZ: libprotobuf-mutator +RUN set -ex; \ + git clone https://github.com/google/libprotobuf-mutator.git \ + /usr/src/libprotobuf-mutator; \ + cd /usr/src/libprotobuf-mutator; \ + git checkout d1fe8a7d8ae18f3d454f055eba5213c291986f21; \ + mkdir build; \ + cd build; \ + cmake .. -GNinja -DLIB_PROTO_MUTATOR_DOWNLOAD_PROTOBUF=ON \ + -DLIB_PROTO_MUTATOR_TESTING=OFF -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX="/usr"; \ + ninja; \ + cp -vpr external.protobuf/bin/* /usr/bin/; \ + cp -vpr external.protobuf/include/* /usr/include/; \ + cp -vpr external.protobuf/lib/* /usr/lib/; \ + ninja install/strip; \ + rm -rf /usr/src/libprotobuf-mutator + +# OSSFUZZ: libfuzzer +RUN set -ex; \ + cd /var/tmp; \ + svn co https://llvm.org/svn/llvm-project/compiler-rt/trunk/lib/fuzzer libfuzzer; \ + mkdir -p build-libfuzzer; \ + cd build-libfuzzer; \ + clang++-8 -O1 -stdlib=libstdc++ -std=c++11 -O2 -fPIC -c ../libfuzzer/*.cpp -I../libfuzzer; \ + ar r /usr/lib/libFuzzingEngine.a *.o; \ + rm -rf /var/lib/libfuzzer + +# ETHASH +RUN set -ex; \ + cd /usr/src; \ + git clone --branch="v0.4.4" https://github.com/chfast/ethash.git; \ + cd ethash; \ + mkdir build; \ + cd build; \ + cmake .. -G Ninja -DBUILD_SHARED_LIBS=OFF -DETHASH_BUILD_TESTS=OFF -DCMAKE_INSTALL_PREFIX="/usr"; \ + ninja; \ + ninja install/strip; \ + rm -rf /usr/src/ethash + +# INTX +RUN set -ex; \ + cd /usr/src; \ + git clone --branch="v0.2.0" https://github.com/chfast/intx.git; \ + cd intx; \ + mkdir build; \ + cd build; \ + cmake .. -G Ninja -DBUILD_SHARED_LIBS=OFF -DINTX_TESTING=OFF -DINTX_BENCHMARKING=OFF -DCMAKE_INSTALL_PREFIX="/usr"; \ + ninja; \ + ninja install/strip; \ + rm -rf /usr/src/intx; + +# EVMONE +ARG EVMONE_HASH="f10d12c190f55a9d373e78b2dc0074d35d752c02cb536bb6fe754fb3719dd69e" +ARG EVMONE_MAJOR="0" +ARG EVMONE_MINOR="1" +ARG EVMONE_MICRO="0" +RUN set -ex; \ + EVMONE_VERSION="$EVMONE_MAJOR.$EVMONE_MINOR.$EVMONE_MICRO"; \ + TGZFILE="evmone-$EVMONE_VERSION-linux-x86_64.tar.gz"; \ + wget https://github.com/ethereum/evmone/releases/download/v$EVMONE_VERSION/$TGZFILE; \ + sha256sum $TGZFILE; \ + tar xzpf $TGZFILE -C /usr; \ + rm -f $TGZFILE; + +FROM base +COPY --from=libraries /usr/lib /usr/lib +COPY --from=libraries /usr/bin /usr/bin +COPY --from=libraries /usr/include /usr/include From ddc3ce0eae256f5a7357584d61d79953cc8b2fec Mon Sep 17 00:00:00 2001 From: Bhargava Shastry Date: Wed, 14 Aug 2019 12:07:37 +0200 Subject: [PATCH 09/12] Avoid aggregate initialization of EWasmToText class --- libyul/backends/wasm/EWasmCodeTransform.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libyul/backends/wasm/EWasmCodeTransform.cpp b/libyul/backends/wasm/EWasmCodeTransform.cpp index 3bcc18fe8..f9e28e94a 100644 --- a/libyul/backends/wasm/EWasmCodeTransform.cpp +++ b/libyul/backends/wasm/EWasmCodeTransform.cpp @@ -55,7 +55,7 @@ string EWasmCodeTransform::run(Dialect const& _dialect, yul::Block const& _ast) std::vector imports; for (auto& imp: transform.m_functionsToImport) imports.emplace_back(std::move(imp.second)); - return EWasmToText{}.run( + return EWasmToText().run( transform.m_globalVariables, imports, functions From 7d30fbdef0df39d0fdc0ad275faf087d303add0d Mon Sep 17 00:00:00 2001 From: chriseth Date: Tue, 13 Aug 2019 14:40:26 +0200 Subject: [PATCH 10/12] Extract side effects into their own struct. --- libyul/Dialect.h | 18 +------ libyul/SideEffects.h | 78 ++++++++++++++++++++++++++++ libyul/backends/evm/EVMDialect.cpp | 58 +++++++++++---------- libyul/backends/evm/EVMDialect.h | 2 + libyul/backends/wasm/WasmDialect.cpp | 28 ++++------ libyul/optimiser/Semantics.cpp | 33 ++---------- libyul/optimiser/Semantics.h | 24 +++------ test/libyul/Parser.cpp | 2 +- 8 files changed, 136 insertions(+), 107 deletions(-) create mode 100644 libyul/SideEffects.h diff --git a/libyul/Dialect.h b/libyul/Dialect.h index bf1a352e1..64377cc37 100644 --- a/libyul/Dialect.h +++ b/libyul/Dialect.h @@ -21,6 +21,7 @@ #pragma once #include +#include #include @@ -45,24 +46,9 @@ struct BuiltinFunction YulString name; std::vector parameters; std::vector returns; - /// If true, calls to this function can be freely moved and copied (as long as their - /// arguments are either variables or also movable) without altering the semantics. - /// This means the function cannot depend on storage or memory, cannot have any side-effects, - /// but it can depend on state that is constant across an EVM-call. - bool movable = false; - /// If true, a call to this function can be omitted without changing semantics. - bool sideEffectFree = false; - /// If true, a call to this function can be omitted without changing semantics if the - /// program does not contain the msize instruction. - bool sideEffectFreeIfNoMSize = false; + SideEffects sideEffects; /// If true, this is the msize instruction. bool isMSize = false; - /// If false, storage of the current contract before and after the function is the same - /// under every circumstance. If the function does not return, this can be false. - bool invalidatesStorage = true; - /// If false, memory before and after the function is the same under every circumstance. - /// If the function does not return, this can be false. - bool invalidatesMemory = true; /// If true, can only accept literals as arguments and they cannot be moved to variables. bool literalArguments = false; }; diff --git a/libyul/SideEffects.h b/libyul/SideEffects.h new file mode 100644 index 000000000..fff907bec --- /dev/null +++ b/libyul/SideEffects.h @@ -0,0 +1,78 @@ +/* + This file is part of solidity. + + solidity is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + solidity is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with solidity. If not, see . +*/ + +#pragma once + +#include + +namespace yul +{ + +/** + * Side effects of code. + * + * The default-constructed value applies to the "empty code". + */ +struct SideEffects +{ + /// If true, expressions in this code can be freely moved and copied without altering the + /// semantics. + /// At statement level, it means that functions containing this code can be + /// called multiple times, their calls can be rearranged and calls can also be + /// deleted without changing the semantics. + /// This means it cannot depend on storage or memory, cannot have any side-effects, + /// but it can depend on state that is constant across an EVM-call. + bool movable = true; + /// If true, the code can be removed without changing the semantics. + bool sideEffectFree = true; + /// If true, the code can be removed without changing the semantics as long as + /// the whole program does not contain the msize instruction. + bool sideEffectFreeIfNoMSize = true; + /// If false, storage is guaranteed to be unchanged by the code under all + /// circumstances. + bool invalidatesStorage = false; + /// If false, memory is guaranteed to be unchanged by the code under all + /// circumstances. + bool invalidatesMemory = false; + + /// @returns the worst-case side effects. + static SideEffects worst() + { + return SideEffects{false, false, false, true, true}; + } + + /// @returns the combined side effects of two pieces of code. + SideEffects operator+(SideEffects const& _other) + { + return SideEffects{ + movable && _other.movable, + sideEffectFree && _other.sideEffectFree, + sideEffectFreeIfNoMSize && _other.sideEffectFreeIfNoMSize, + invalidatesStorage || _other.invalidatesStorage, + invalidatesMemory || _other.invalidatesMemory + }; + } + + /// Adds the side effects of another piece of code to this side effect. + SideEffects& operator+=(SideEffects const& _other) + { + *this = *this + _other; + return *this; + } +}; + +} diff --git a/libyul/backends/evm/EVMDialect.cpp b/libyul/backends/evm/EVMDialect.cpp index b4a7cbd49..a9cbd2cbc 100644 --- a/libyul/backends/evm/EVMDialect.cpp +++ b/libyul/backends/evm/EVMDialect.cpp @@ -50,12 +50,8 @@ pair createEVMFunction( f.name = YulString{_name}; f.parameters.resize(info.args); f.returns.resize(info.ret); - f.movable = eth::SemanticInformation::movable(_instruction); - f.sideEffectFree = eth::SemanticInformation::sideEffectFree(_instruction); - f.sideEffectFreeIfNoMSize = eth::SemanticInformation::sideEffectFreeIfNoMSize(_instruction); + f.sideEffects = EVMDialect::sideEffectsOfInstruction(_instruction); f.isMSize = _instruction == dev::eth::Instruction::MSIZE; - f.invalidatesStorage = eth::SemanticInformation::invalidatesStorage(_instruction); - f.invalidatesMemory = eth::SemanticInformation::invalidatesMemory(_instruction); f.literalArguments = false; f.instruction = _instruction; f.generateCode = [_instruction]( @@ -75,11 +71,7 @@ pair createFunction( string _name, size_t _params, size_t _returns, - bool _movable, - bool _sideEffectFree, - bool _sideEffectFreeIfNoMSize, - bool _invalidatesStorage, - bool _invalidatesMemory, + SideEffects _sideEffects, bool _literalArguments, std::function)> _generateCode ) @@ -89,13 +81,9 @@ pair createFunction( f.name = name; f.parameters.resize(_params); f.returns.resize(_returns); - f.movable = _movable; + f.sideEffects = std::move(_sideEffects); f.literalArguments = _literalArguments; - f.sideEffectFree = _sideEffectFree; - f.sideEffectFreeIfNoMSize = _sideEffectFreeIfNoMSize; f.isMSize = false; - f.invalidatesStorage = _invalidatesStorage; - f.invalidatesMemory = _invalidatesMemory; f.instruction = {}; f.generateCode = std::move(_generateCode); return {name, f}; @@ -116,7 +104,7 @@ map createBuiltins(langutil::EVMVersion _evmVe if (_objectAccess) { - builtins.emplace(createFunction("datasize", 1, 1, true, true, true, false, false, true, []( + builtins.emplace(createFunction("datasize", 1, 1, SideEffects{}, true, []( FunctionCall const& _call, AbstractAssembly& _assembly, BuiltinContext& _context, @@ -137,7 +125,7 @@ map createBuiltins(langutil::EVMVersion _evmVe _assembly.appendDataSize(_context.subIDs.at(dataName)); } })); - builtins.emplace(createFunction("dataoffset", 1, 1, true, true, true, false, false, true, []( + builtins.emplace(createFunction("dataoffset", 1, 1, SideEffects{}, true, []( FunctionCall const& _call, AbstractAssembly& _assembly, BuiltinContext& _context, @@ -158,15 +146,22 @@ map createBuiltins(langutil::EVMVersion _evmVe _assembly.appendDataOffset(_context.subIDs.at(dataName)); } })); - builtins.emplace(createFunction("datacopy", 3, 0, false, false, false, false, true, false, []( - FunctionCall const&, - AbstractAssembly& _assembly, - BuiltinContext&, - std::function _visitArguments - ) { - _visitArguments(); - _assembly.appendInstruction(dev::eth::Instruction::CODECOPY); - })); + builtins.emplace(createFunction( + "datacopy", + 3, + 0, + SideEffects{false, false, false, false, true}, + false, + []( + FunctionCall const&, + AbstractAssembly& _assembly, + BuiltinContext&, + std::function _visitArguments + ) { + _visitArguments(); + _assembly.appendInstruction(dev::eth::Instruction::CODECOPY); + } + )); } return builtins; } @@ -225,3 +220,14 @@ EVMDialect const& EVMDialect::yulForEVM(langutil::EVMVersion _version) dialects[_version] = make_unique(AsmFlavour::Yul, false, _version); return *dialects[_version]; } + +SideEffects EVMDialect::sideEffectsOfInstruction(eth::Instruction _instruction) +{ + return SideEffects{ + eth::SemanticInformation::movable(_instruction), + eth::SemanticInformation::sideEffectFree(_instruction), + eth::SemanticInformation::sideEffectFreeIfNoMSize(_instruction), + eth::SemanticInformation::invalidatesStorage(_instruction), + eth::SemanticInformation::invalidatesMemory(_instruction) + }; +} diff --git a/libyul/backends/evm/EVMDialect.h b/libyul/backends/evm/EVMDialect.h index bcea185f4..f4c0e4eed 100644 --- a/libyul/backends/evm/EVMDialect.h +++ b/libyul/backends/evm/EVMDialect.h @@ -80,6 +80,8 @@ struct EVMDialect: public Dialect bool providesObjectAccess() const { return m_objectAccess; } + static SideEffects sideEffectsOfInstruction(dev::eth::Instruction _instruction); + protected: bool const m_objectAccess; langutil::EVMVersion const m_evmVersion; diff --git a/libyul/backends/wasm/WasmDialect.cpp b/libyul/backends/wasm/WasmDialect.cpp index d06fe2edd..f781292ae 100644 --- a/libyul/backends/wasm/WasmDialect.cpp +++ b/libyul/backends/wasm/WasmDialect.cpp @@ -49,19 +49,19 @@ WasmDialect::WasmDialect(): addFunction("i64.eqz", 1, 1); addFunction("i64.store", 2, 0, false); - m_functions["i64.store"_yulstring].invalidatesStorage = false; + m_functions["i64.store"_yulstring].sideEffects.invalidatesStorage = false; addFunction("i64.load", 1, 1, false); - m_functions["i64.load"_yulstring].invalidatesStorage = false; - m_functions["i64.load"_yulstring].invalidatesMemory = false; - m_functions["i64.load"_yulstring].sideEffectFree = true; - m_functions["i64.load"_yulstring].sideEffectFreeIfNoMSize = true; + m_functions["i64.load"_yulstring].sideEffects.invalidatesStorage = false; + m_functions["i64.load"_yulstring].sideEffects.invalidatesMemory = false; + m_functions["i64.load"_yulstring].sideEffects.sideEffectFree = true; + m_functions["i64.load"_yulstring].sideEffects.sideEffectFreeIfNoMSize = true; addFunction("drop", 1, 0); addFunction("unreachable", 0, 0, false); - m_functions["unreachable"_yulstring].invalidatesStorage = false; - m_functions["unreachable"_yulstring].invalidatesMemory = false; + m_functions["unreachable"_yulstring].sideEffects.invalidatesStorage = false; + m_functions["unreachable"_yulstring].sideEffects.invalidatesMemory = false; addFunction("datasize", 1, 4, true, true); addFunction("dataoffset", 1, 4, true, true); @@ -138,14 +138,10 @@ void WasmDialect::addEthereumExternals() f.parameters.emplace_back(YulString(p)); for (string const& p: ext.returns) f.returns.emplace_back(YulString(p)); - f.movable = false; // TODO some of them are side effect free. - f.sideEffectFree = false; - f.sideEffectFreeIfNoMSize = false; + f.sideEffects = SideEffects::worst(); f.isMSize = false; - f.invalidatesStorage = (ext.name == "storageStore"); - // TODO some of them do not invalidate memory - f.invalidatesMemory = true; + f.sideEffects.invalidatesStorage = (ext.name == "storageStore"); f.literalArguments = false; } } @@ -163,11 +159,7 @@ void WasmDialect::addFunction( f.name = name; f.parameters.resize(_params); f.returns.resize(_returns); - f.movable = _movable; - f.sideEffectFree = _movable; - f.sideEffectFreeIfNoMSize = _movable; + f.sideEffects = _movable ? SideEffects{} : SideEffects::worst(); f.isMSize = false; - f.invalidatesStorage = !_movable; - f.invalidatesMemory = !_movable; f.literalArguments = _literalArguments; } diff --git a/libyul/optimiser/Semantics.cpp b/libyul/optimiser/Semantics.cpp index 5f1846ee3..8fe3cc474 100644 --- a/libyul/optimiser/Semantics.cpp +++ b/libyul/optimiser/Semantics.cpp @@ -33,6 +33,7 @@ using namespace std; using namespace dev; using namespace yul; + SideEffectsCollector::SideEffectsCollector(Dialect const& _dialect, Expression const& _expression): SideEffectsCollector(_dialect) { @@ -55,16 +56,7 @@ void SideEffectsCollector::operator()(FunctionalInstruction const& _instr) { ASTWalker::operator()(_instr); - if (!eth::SemanticInformation::movable(_instr.instruction)) - m_movable = false; - if (!eth::SemanticInformation::sideEffectFree(_instr.instruction)) - m_sideEffectFree = false; - if (!eth::SemanticInformation::sideEffectFreeIfNoMSize(_instr.instruction)) - m_sideEffectFreeIfNoMSize = false; - if (eth::SemanticInformation::invalidatesStorage(_instr.instruction)) - m_invalidatesStorage = true; - if (eth::SemanticInformation::invalidatesMemory(_instr.instruction)) - m_invalidatesMemory = true; + m_sideEffects += EVMDialect::sideEffectsOfInstruction(_instr.instruction); } void SideEffectsCollector::operator()(FunctionCall const& _functionCall) @@ -72,26 +64,9 @@ void SideEffectsCollector::operator()(FunctionCall const& _functionCall) ASTWalker::operator()(_functionCall); if (BuiltinFunction const* f = m_dialect.builtin(_functionCall.functionName.name)) - { - if (!f->movable) - m_movable = false; - if (!f->sideEffectFree) - m_sideEffectFree = false; - if (!f->sideEffectFreeIfNoMSize) - m_sideEffectFreeIfNoMSize = false; - if (f->invalidatesStorage) - m_invalidatesStorage = true; - if (f->invalidatesMemory) - m_invalidatesMemory = true; - } + m_sideEffects += f->sideEffects; else - { - m_movable = false; - m_sideEffectFree = false; - m_sideEffectFreeIfNoMSize = false; - m_invalidatesStorage = true; - m_invalidatesMemory = true; - } + m_sideEffects += SideEffects::worst(); } bool MSizeFinder::containsMSize(Dialect const& _dialect, Block const& _ast) diff --git a/libyul/optimiser/Semantics.h b/libyul/optimiser/Semantics.h index 393e2d114..a924fe5ff 100644 --- a/libyul/optimiser/Semantics.h +++ b/libyul/optimiser/Semantics.h @@ -21,6 +21,7 @@ #pragma once #include +#include #include @@ -44,32 +45,21 @@ public: void operator()(FunctionalInstruction const& _functionalInstruction) override; void operator()(FunctionCall const& _functionCall) override; - bool movable() const { return m_movable; } + bool movable() const { return m_sideEffects.movable; } bool sideEffectFree(bool _allowMSizeModification = false) const { if (_allowMSizeModification) return sideEffectFreeIfNoMSize(); else - return m_sideEffectFree; + return m_sideEffects.sideEffectFree; } - bool sideEffectFreeIfNoMSize() const { return m_sideEffectFreeIfNoMSize; } - bool invalidatesStorage() const { return m_invalidatesStorage; } - bool invalidatesMemory() const { return m_invalidatesMemory; } + bool sideEffectFreeIfNoMSize() const { return m_sideEffects.sideEffectFreeIfNoMSize; } + bool invalidatesStorage() const { return m_sideEffects.invalidatesStorage; } + bool invalidatesMemory() const { return m_sideEffects.invalidatesMemory; } private: Dialect const& m_dialect; - /// Is the current expression movable or not. - bool m_movable = true; - /// Is the current expression side-effect free, i.e. can be removed - /// without changing the semantics. - bool m_sideEffectFree = true; - /// Is the current expression side-effect free up to msize, i.e. can be removed - /// without changing the semantics except for the value returned by the msize instruction. - bool m_sideEffectFreeIfNoMSize = true; - /// If false, storage is guaranteed to be unchanged by the code under all - /// circumstances. - bool m_invalidatesStorage = false; - bool m_invalidatesMemory = false; + SideEffects m_sideEffects; }; /** diff --git a/test/libyul/Parser.cpp b/test/libyul/Parser.cpp index e747f66bd..899f40f72 100644 --- a/test/libyul/Parser.cpp +++ b/test/libyul/Parser.cpp @@ -565,7 +565,7 @@ BOOST_AUTO_TEST_CASE(builtins_analysis) { return _name == "builtin"_yulstring ? &f : nullptr; } - BuiltinFunction f{"builtin"_yulstring, vector(2), vector(3), false, false}; + BuiltinFunction f{"builtin"_yulstring, vector(2), vector(3), {}}; }; SimpleDialect dialect; From a075588618e353f6b01b09d41164ddd4cce798a9 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Wed, 14 Aug 2019 16:20:18 +0200 Subject: [PATCH 11/12] circleci: Fixes logfile naming, not accidentally overriding others --- .circleci/soltest.sh | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.circleci/soltest.sh b/.circleci/soltest.sh index dda335102..41f08db77 100755 --- a/.circleci/soltest.sh +++ b/.circleci/soltest.sh @@ -45,7 +45,15 @@ mkdir -p test_results # in case we run with ASAN enabled, we must increase stck size. ulimit -s 16384 -BOOST_TEST_ARGS="--color_output=no --show_progress=yes --logger=JUNIT,error,test_results/$EVM.xml" +get_logfile_basename() { + local filename="${EVM}" + test "${OPTIMIZE}" = "1" && filename="${filename}_opt" + test "${ABI_ENCODER_V2}" = "1" && filename="${filename}_abiv2" + + echo -ne "${filename}" +} + +BOOST_TEST_ARGS="--color_output=no --show_progress=yes --logger=JUNIT,error,test_results/`get_logfile_basename`.xml" SOLTEST_ARGS="--evm-version=$EVM --evmonepath /usr/lib/libevmone.so $flags" test "${OPTIMIZE}" = "1" && SOLTEST_ARGS="${SOLTEST_ARGS} --optimize" test "${ABI_ENCODER_V2}" = "1" && SOLTEST_ARGS="${SOLTEST_ARGS} --abiencoderv2 --optimize-yul" From b94bdfcec040f8c526ba2630b621e3783d3a1ca7 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Wed, 14 Aug 2019 15:43:24 +0200 Subject: [PATCH 12/12] Merges soltest tests per EVM/flags into a single one. Because of EVMONE, speeds up test runs we can do that now. --- .circleci/config.yml | 101 +++++++++++---------------------------- .circleci/soltest_all.sh | 37 ++++++++++++++ 2 files changed, 64 insertions(+), 74 deletions(-) create mode 100755 .circleci/soltest_all.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index f6346c2d1..952cc87d1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -96,22 +96,35 @@ defaults: name: soltest command: ./.circleci/soltest.sh + - run_soltest_all: &run_soltest_all + name: soltest_all + command: ./.circleci/soltest_all.sh + - run_cmdline_tests: &run_cmdline_tests name: command line tests command: ./test/cmdlineTests.sh - - test_steps: &test_steps - - checkout - - attach_workspace: - at: build - - run: *run_soltest - - store_test_results: *store_test_results - - store_artifacts: *artifacts_test_results - - test_ubuntu1904: &test_ubuntu1904 docker: - image: ethereum/solidity-buildpack-deps:ubuntu1904 - steps: *test_steps + steps: + - checkout + - attach_workspace: + at: build + - run: *run_soltest + - store_test_results: *store_test_results + - store_artifacts: *artifacts_test_results + + - test_ubuntu1904_all: &test_ubuntu1904 + docker: + - image: ethereum/solidity-buildpack-deps:ubuntu1904 + steps: + - checkout + - attach_workspace: + at: build + - run: *run_soltest_all + - store_test_results: *store_test_results + - store_artifacts: *artifacts_test_results - test_asan: &test_asan <<: *test_ubuntu1904 @@ -445,6 +458,9 @@ jobs: path: docs/_build/html/ destination: docs-html + t_ubu_soltest: &t_ubu_soltest + <<: *test_ubuntu1904 + t_ubu_cli: &t_ubu_cli docker: - image: ethereum/solidity-buildpack-deps:ubuntu1904 @@ -480,61 +496,6 @@ jobs: OPTIMIZE: 0 ASAN_OPTIONS: check_initialization_order=true:detect_stack_use_after_return=true:strict_init_order=true:strict_string_checks=true:detect_invalid_pointer_pairs=2 - t_ubu_homestead: - <<: *test_ubuntu1904 - environment: - EVM: homestead - OPTIMIZE: 0 - - t_ubu_homestead_opt: - <<: *test_ubuntu1904 - environment: - EVM: homestead - OPTIMIZE: 1 - - t_ubu_byzantium: - <<: *test_ubuntu1904 - environment: - EVM: byzantium - OPTIMIZE: 0 - - t_ubu_byzantium_opt: - <<: *test_ubuntu1904 - environment: - EVM: byzantium - OPTIMIZE: 1 - - t_ubu_constantinople: - <<: *test_ubuntu1904 - environment: - EVM: constantinople - OPTIMIZE: 0 - - t_ubu_constantinople_opt: - <<: *test_ubuntu1904 - environment: - EVM: constantinople - OPTIMIZE: 1 - - t_ubu_constantinople_opt_abiv2: - <<: *test_ubuntu1904 - environment: - EVM: constantinople - OPTIMIZE: 1 - ABI_ENCODER_V2: 1 - - t_ubu_petersburg: - <<: *test_ubuntu1904 - environment: - EVM: petersburg - OPTIMIZE: 0 - - t_ubu_petersburg_opt: - <<: *test_ubuntu1904 - environment: - EVM: petersburg - OPTIMIZE: 1 - t_ems_solcjs: docker: - image: circleci/node:10 @@ -619,19 +580,11 @@ workflows: - b_osx: *workflow_trigger_on_tags - t_osx_cli: *workflow_osx - # Ubuntu 18.10 build and tests + # Ubuntu build and tests - b_ubu: *workflow_trigger_on_tags - b_ubu18: *workflow_trigger_on_tags - t_ubu_cli: *workflow_ubuntu1904 - - t_ubu_homestead: *workflow_ubuntu1904 - - t_ubu_homestead_opt: *workflow_ubuntu1904 - - t_ubu_byzantium: *workflow_ubuntu1904 - - t_ubu_byzantium_opt: *workflow_ubuntu1904 - - t_ubu_constantinople: *workflow_ubuntu1904 - - t_ubu_constantinople_opt: *workflow_ubuntu1904 - - t_ubu_constantinople_opt_abiv2: *workflow_ubuntu1904 - - t_ubu_petersburg: *workflow_ubuntu1904 - - t_ubu_petersburg_opt: *workflow_ubuntu1904 + - t_ubu_soltest: *workflow_ubuntu1904 # ASan build and tests - b_ubu_asan: *workflow_trigger_on_tags diff --git a/.circleci/soltest_all.sh b/.circleci/soltest_all.sh new file mode 100755 index 000000000..7b4564ee7 --- /dev/null +++ b/.circleci/soltest_all.sh @@ -0,0 +1,37 @@ +#! /bin/bash +#------------------------------------------------------------------------------ +# Bash script to execute the Solidity tests by CircleCI. +# +# The documentation for solidity is hosted at: +# +# https://solidity.readthedocs.org +# +# ------------------------------------------------------------------------------ +# This file is part of solidity. +# +# solidity is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# solidity is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with solidity. If not, see +# +# (c) 2016-2019 solidity contributors. +# ------------------------------------------------------------------------------ +set -e + +REPODIR="$(realpath $(dirname $0)/..)" + +for OPTIMIZE in 0 1; do + for EVM in homestead byzantium constantinople petersburg; do + EVM=$EVM OPTIMIZE=$OPTIMIZE ${REPODIR}/.circleci/soltest.sh + done +done + +EVM=constantinople OPTIMIZE=1 ABI_ENCODER_V2=1 ${REPODIR}/.circleci/soltest.sh \ No newline at end of file