Merge pull request #5975 from ethereum/develop

Release 0.5.4

.circleci/config.yml

@@ -66,6 +66,7 @@ jobs:
           paths:
             - soljson.js
             - version.txt
+
   test_emscripten_solcjs:
     docker:
       - image: circleci/node:10
@@ -84,6 +85,7 @@ jobs:
           name: Test solcjs
           command: |
             test/solcjsTests.sh /tmp/workspace/soljson.js $(cat /tmp/workspace/version.txt)
+
   test_emscripten_external:
     docker:
       - image: circleci/node:10
@@ -102,6 +104,7 @@ jobs:
           name: External tests
           command: |
             test/externalTests.sh /tmp/workspace/soljson.js || test/externalTests.sh /tmp/workspace/soljson.js
+
   build_x86_linux:
     docker:
       - image: buildpack-deps:bionic
@@ -114,7 +117,7 @@ jobs:
           name: Install build dependencies
           command: |
             apt-get -qq update
-            apt-get -qy install cmake libboost-regex-dev libboost-filesystem-dev libboost-test-dev libboost-system-dev libboost-program-options-dev libz3-dev
+            apt-get -qy install cmake libboost-regex-dev libboost-filesystem-dev libboost-test-dev libboost-system-dev libboost-program-options-dev libcvc4-dev
             ./scripts/install_obsolete_jsoncpp_1_7_4.sh
       - run: *setup_prerelease_commit_hash
       - run: *run_build
@@ -124,6 +127,23 @@ jobs:
           paths:
             - "*"
 
+  build_x86_linux_cxx17:
+    docker:
+      - image: buildpack-deps:disco
+    environment:
+      TERM: xterm
+      CMAKE_OPTIONS: -DCMAKE_TOOLCHAIN_FILE=cmake/toolchains/cxx17.cmake
+    steps:
+      - checkout
+      - run:
+          name: Install build dependencies
+          command: |
+            apt-get -qq update
+            apt-get -qy install cmake libboost-regex-dev libboost-filesystem-dev libboost-test-dev libboost-system-dev libboost-program-options-dev libcvc4-dev
+            ./scripts/install_obsolete_jsoncpp_1_7_4.sh
+      - run: *setup_prerelease_commit_hash
+      - run: *run_build
+
   build_x86_archlinux:
     docker:
       - image: archlinux/base
@@ -145,21 +165,21 @@ jobs:
             - test/soltest
             - test/tools/solfuzzer
 
-  build_x86_clang7:
+  build_x86_clang7_asan:
     docker:
       - image: buildpack-deps:cosmic
     environment:
       TERM: xterm
       CC: /usr/bin/clang-7
       CXX: /usr/bin/clang++-7
-      CMAKE_OPTIONS: -DLLL=ON
+      CMAKE_OPTIONS: -DSANITIZE=address -DCMAKE_BUILD_TYPE=Debug
     steps:
       - checkout
       - run:
           name: Install build dependencies
           command: |
             apt-get -qq update
-            apt-get -qy install clang-7 cmake libboost-regex-dev libboost-filesystem-dev libboost-test-dev libboost-system-dev libboost-program-options-dev libz3-dev
+            apt-get -qy install clang-7 cmake libboost-regex-dev libboost-filesystem-dev libboost-test-dev libboost-system-dev libboost-program-options-dev libcvc4-dev
             ./scripts/install_obsolete_jsoncpp_1_7_4.sh
       - run: *setup_prerelease_commit_hash
       - run: *run_build
@@ -167,7 +187,9 @@ jobs:
       - persist_to_workspace:
           root: build
           paths:
-            - "*"
+            - solc/solc
+            - test/soltest
+            - test/tools/solfuzzer
 
   build_x86_mac:
     macos:
@@ -249,7 +271,7 @@ jobs:
           name: Install dependencies
           command: |
             apt-get -qq update
-            apt-get -qy install libz3-dev libleveldb1v5 python-pip
+            apt-get -qy install libcvc4-dev libleveldb1v5 python-pip
             pip install codecov
       - run: mkdir -p test_results
       - run:
@@ -268,6 +290,36 @@ jobs:
           path: test_results/
           destination: test_results/
 
+  test_x86_clang7_asan:
+    docker:
+      - image: buildpack-deps:cosmic
+    environment:
+      TERM: xterm
+    steps:
+      - checkout
+      - attach_workspace:
+          at: build
+      - run:
+          name: Install dependencies
+          command: |
+            apt-get -qq update
+            apt-get -qy install llvm-7-dev libcvc4-dev libleveldb1v5 python-pip
+            # This is needed to resolve the symbols. Since we're using clang7 in the build, we must use the appropriate symbolizer.
+            update-alternatives --install /usr/bin/llvm-symbolizer llvm-symbolizer /usr/bin/llvm-symbolizer-7 1
+      - run: mkdir -p test_results
+      - run:
+          name: Run tests with ASAN
+          command: |
+            ulimit -a
+            # Increase stack size because ASan makes stack frames bigger and that breaks our assumptions (in tests).
+            ulimit -s 16384
+            build/test/soltest --logger=JUNIT,test_suite,test_results/result.xml -- --no-ipc --testpath test
+      - store_test_results:
+          path: test_results/
+      - store_artifacts:
+          path: test_results/
+          destination: test_results/
+
   test_x86_archlinux:
     docker:
       - image: archlinux/base
@@ -350,12 +402,19 @@ workflows:
           requires:
             - build_emscripten
       - build_x86_linux: *build_on_tags
-      - build_x86_clang7: *build_on_tags
+      - build_x86_linux_cxx17: *build_on_tags
+      - build_x86_clang7_asan: *build_on_tags
       - build_x86_mac: *build_on_tags
       - test_x86_linux:
           <<: *build_on_tags
           requires:
             - build_x86_linux
+      - test_x86_clang7_asan:
+          filters:
+            branches:
+              only: develop
+          requires:
+            - build_x86_clang7_asan
       - test_x86_mac:
           <<: *build_on_tags
           requires:

.travis.yml

@@ -58,13 +58,11 @@ matrix:
           before_install:
               - sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test
               - sudo add-apt-repository -y ppa:mhier/libboost-latest
-              - sudo add-apt-repository -y ppa:hvr/z3
               - sudo apt-get update -qq
           install:
               - sudo apt-get install -qq g++-8 gcc-8
               - sudo apt-get install -qq libboost1.67-dev
               - sudo apt-get install -qq libleveldb1
-              - sudo apt-get install -qq libz3-dev
               - sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-8 90
               - sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-8 90
 
@@ -78,13 +76,11 @@ matrix:
           before_install:
               - sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test
               - sudo add-apt-repository -y ppa:mhier/libboost-latest
-              - sudo add-apt-repository -y ppa:hvr/z3
               - sudo apt-get update -qq
           install:
               - sudo apt-get install -qq g++-8 gcc-8
               - sudo apt-get install -qq libboost1.67-dev
               - sudo apt-get install -qq libleveldb1
-              - sudo apt-get install -qq libz3-dev
               - sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-8 90
               - sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-8 90
 

CMakeLists.txt

@@ -10,7 +10,7 @@ include(EthPolicy)
 eth_policy()
 
 # project name and version should be set after cmake_policy CMP0048
-set(PROJECT_VERSION "0.5.3")
+set(PROJECT_VERSION "0.5.4")
 project(solidity VERSION ${PROJECT_VERSION} LANGUAGES CXX)
 
 option(LLL "Build LLL" OFF)

Changelog.md

@@ -1,3 +1,32 @@
+### 0.5.4 (2019-02-12)
+
+Language Features:
+ * Allow calldata structs without dynamically encoded members with ABIEncoderV2.
+
+
+Compiler Features:
+ * ABIEncoderV2: Implement packed encoding.
+ * C API (``libsolc`` / raw ``soljson.js``): Introduce ``solidity_free`` method which releases all internal buffers to save memory.
+ * Commandline Interface: Adds new option ``--new-reporter`` for improved diagnostics formatting
+   along with ``--color`` and ``--no-color`` for colorized output to be forced (or explicitly disabled).
+
+
+Bugfixes:
+ * Code Generator: Defensively pad allocation of creationCode and runtimeCode to multiples of 32 bytes.
+ * Commandline Interface: Allow yul optimizer only for strict assembly.
+ * Parser: Disallow empty import statements.
+ * Type Checker: Disallow mappings with data locations other than ``storage``.
+ * Type Checker: Fix internal error when a struct array index does not fit into a uint256.
+ * Type System: Properly report packed encoded size for arrays and structs (mostly unused until now).
+
+
+Build System:
+ * Add support for continuous fuzzing via Google oss-fuzz
+ * SMT: If using Z3, require version 4.6.0 or newer.
+ * Soltest: Add parser that is used in the file-based unit test environment.
+ * Ubuntu PPA Packages: Use CVC4 as SMT solver instead of Z3
+
+
 ### 0.5.3 (2019-01-22)
 
 Language Features:

cmake/EthCompilerSettings.cmake

@@ -74,7 +74,6 @@ if (("${CMAKE_CXX_COMPILER_ID}" MATCHES "GNU") OR ("${CMAKE_CXX_COMPILER_ID}" MA
 			# into errors, which makes sense.
 			# http://stackoverflow.com/questions/21617158/how-to-silence-unused-command-line-argument-error-with-clang-without-disabling-i
 			add_compile_options(-Qunused-arguments)
-
 		elseif(EMSCRIPTEN)
 			set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} --memory-init-file 0")
 			# Leave only exported symbols as public and aggressively remove others
@@ -124,6 +123,7 @@ elseif (DEFINED MSVC)
 	add_compile_options(/wd4800)					# disable forcing value to bool 'true' or 'false' (performance warning) (4800)
 	add_compile_options(-D_WIN32_WINNT=0x0600)		# declare Windows Vista API requirement
 	add_compile_options(-DNOMINMAX)					# undefine windows.h MAX && MIN macros cause it cause conflicts with std::min && std::max functions
+	add_compile_options(/utf-8)					# enable utf-8 encoding (solves warning 4819)
 
 	# disable empty object file warning
 	set(CMAKE_STATIC_LINKER_FLAGS "${CMAKE_STATIC_LINKER_FLAGS} /ignore:4221")

cmake/EthOptions.cmake

@@ -3,6 +3,7 @@ macro(configure_project)
 
 	# features
 	eth_default_option(COVERAGE OFF)
+	eth_default_option(OSSFUZZ OFF)
 
 	# components
 	eth_default_option(TESTS ON)

cmake/FindZ3.cmake

@@ -1,8 +1,24 @@
 if (USE_Z3)
     find_path(Z3_INCLUDE_DIR NAMES z3++.h PATH_SUFFIXES z3)
     find_library(Z3_LIBRARY NAMES z3)
+    find_program(Z3_EXECUTABLE z3 PATH_SUFFIXES bin)
+
+    if(Z3_INCLUDE_DIR AND Z3_LIBRARY AND Z3_EXECUTABLE)
+        execute_process (COMMAND ${Z3_EXECUTABLE} -version
+            OUTPUT_VARIABLE libz3_version_str
+            ERROR_QUIET
+            OUTPUT_STRIP_TRAILING_WHITESPACE)
+
+        string(REGEX REPLACE "^Z3 version ([0-9.]+).*" "\\1"
+               Z3_VERSION_STRING "${libz3_version_str}")
+        unset(libz3_version_str)
+    endif()
+    mark_as_advanced(Z3_VERSION_STRING z3_DIR)
+
     include(FindPackageHandleStandardArgs)
-    find_package_handle_standard_args(Z3 DEFAULT_MSG Z3_LIBRARY Z3_INCLUDE_DIR)
+    find_package_handle_standard_args(Z3
+        REQUIRED_VARS Z3_LIBRARY Z3_INCLUDE_DIR
+        VERSION_VAR Z3_VERSION_STRING)
 
     if (NOT TARGET Z3::Z3)
         add_library(Z3::Z3 UNKNOWN IMPORTED)

cmake/toolchains/cxx17.cmake

@@ -0,0 +1,4 @@
+# Require C++17.
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD_REQUIRED TRUE)
+set(CMAKE_CXX_EXTENSIONS OFF)

docs/abi-spec.rst

@@ -418,10 +418,22 @@ In effect, a log entry using this ABI is described as:
 
 - ``address``: the address of the contract (intrinsically provided by Ethereum);
 - ``topics[0]``: ``keccak(EVENT_NAME+"("+EVENT_ARGS.map(canonical_type_of).join(",")+")")`` (``canonical_type_of`` is a function that simply returns the canonical type of a given argument, e.g. for ``uint indexed foo``, it would return ``uint256``). If the event is declared as ``anonymous`` the ``topics[0]`` is not generated;
-- ``topics[n]``: ``EVENT_INDEXED_ARGS[n - 1]`` (``EVENT_INDEXED_ARGS`` is the series of ``EVENT_ARGS`` that are indexed);
+- ``topics[n]``: ``abi_encode(EVENT_INDEXED_ARGS[n - 1])`` (``EVENT_INDEXED_ARGS`` is the series of ``EVENT_ARGS`` that are indexed);
-- ``data``: ``abi_serialise(EVENT_NON_INDEXED_ARGS)`` (``EVENT_NON_INDEXED_ARGS`` is the series of ``EVENT_ARGS`` that are not indexed, ``abi_serialise`` is the ABI serialisation function used for returning a series of typed values from a function, as described above).
+- ``data``: ABI encoding of ``EVENT_NON_INDEXED_ARGS`` (``EVENT_NON_INDEXED_ARGS`` is the series of ``EVENT_ARGS`` that are not indexed, ``abi_encode`` is the ABI encoding function used for returning a series of typed values from a function, as described above).
 
-For all fixed-length Solidity types, the ``EVENT_INDEXED_ARGS`` array contains the 32-byte encoded value directly. However, for *types of dynamic length*, which include ``string``, ``bytes``, and arrays, ``EVENT_INDEXED_ARGS`` will contain the *Keccak hash* of the packed encoded value (see :ref:`abi_packed_mode`), rather than the encoded value directly. This allows applications to efficiently query for values of dynamic-length types (by setting the hash of the encoded value as the topic), but leaves applications unable to decode indexed values they have not queried for. For dynamic-length types, application developers face a trade-off between fast search for predetermined values (if the argument is indexed) and legibility of arbitrary values (which requires that the arguments not be indexed). Developers may overcome this tradeoff and achieve both efficient search and arbitrary legibility by defining events with two arguments — one indexed, one not — intended to hold the same value.
+For all types of length at most 32 bytes, the ``EVENT_INDEXED_ARGS`` array contains
+the value directly, padded or sign-extended (for signed integers) to 32 bytes, just as for regular ABI encoding.
+However, for all "complex" types or types of dynamic length, including all arrays, ``string``, ``bytes`` and structs,
+``EVENT_INDEXED_ARGS`` will contain the *Keccak hash* of a special in-place encoded value
+(see :ref:`indexed_event_encoding`), rather than the encoded value directly.
+This allows applications to efficiently query for values of dynamic-length types
+(by setting the hash of the encoded value as the topic), but leaves applications unable
+to decode indexed values they have not queried for. For dynamic-length types,
+application developers face a trade-off between fast search for predetermined values
+(if the argument is indexed) and legibility of arbitrary values (which requires that
+the arguments not be indexed). Developers may overcome this tradeoff and achieve both
+efficient search and arbitrary legibility by defining events with two arguments — one
+indexed, one not — intended to hold the same value.
 
 .. _abi_json:
 
@@ -608,8 +620,9 @@ Through ``abi.encodePacked()``, Solidity supports a non-standard packed mode whe
 
 - types shorter than 32 bytes are neither zero padded nor sign extended and
 - dynamic types are encoded in-place and without the length.
+- array elements are padded, but still encoded in-place
 
-This packed mode is mainly used for indexed event parameters.
+Furthermore, structs as well as nested arrays are not supported.
 
 As an example, the encoding of ``int16(-1), bytes1(0x42), uint16(0x03), string("Hello, world!")`` results in:
 
@@ -622,12 +635,18 @@ As an example, the encoding of ``int16(-1), bytes1(0x42), uint16(0x03), string("
                 ^^^^^^^^^^^^^^^^^^^^^^^^^^ string("Hello, world!") without a length field
 
 More specifically:
- - Each value type takes as many bytes as its range has.
+ - During the encoding, everything is encoded in-place. This means that there is
- - The encoding of a struct or fixed-size array is the concatenation of the
+   no distinction between head and tail, as in the ABI encoding, and the length
-   encoding of its members/elements without any separator or padding.
+   of an array is not encoded.
- - Mapping members of structs are ignored as usual.
+ - The direct arguments of ``abi.encodePacked`` are encoded without padding,
- - Dynamically-sized types like ``string``, ``bytes`` or ``uint[]`` are encoded without
+   as long as they are not arrays (or ``string`` or ``bytes``).
-   their length field.
+ - The encoding of an array is the concatenation of the
+   encoding of its elements **with** padding.
+ - Dynamically-sized types like ``string``, ``bytes`` or ``uint[]`` are encoded
+   without their length field.
+ - The encoding of ``string`` or ``bytes`` does not apply padding at the end
+   unless it is part of an array or struct (then it is padded to a multiple of
+   32 bytes).
 
 In general, the encoding is ambiguous as soon as there are two dynamically-sized elements,
 because of the missing length field.
@@ -636,3 +655,39 @@ If padding is needed, explicit type conversions can be used: ``abi.encodePacked(
 
 Since packed encoding is not used when calling functions, there is no special support
 for prepending a function selector. Since the encoding is ambiguous, there is no decoding function.
+
+.. warning::
+
+  If you use ``keccak256(abi.encodePacked(a, b))`` and both ``a`` and ``b`` are dynamic types,
+  it is easy to craft collisions in the hash value by moving parts of ``a`` into ``b`` and
+  vice-versa. More specifically, ``abi.encodePacked("a", "bc") == abi.encodePacked("ab", "c")``.
+  If you use ``abi.encodePacked`` for signatures, authentication or data integrity, make
+  sure to always use the same types and check that at most one of them is dynamic.
+  Unless there is a compelling reason, ``abi.encode`` should be preferred.
+
+
+.. _indexed_event_encoding:
+
+Encoding of Indexed Event Parameters
+====================================
+
+Indexed event parameters that are not value types, i.e. arrays and structs are not
+stored directly but instead a keccak256-hash of an encoding is stored. This encoding
+is defined as follows:
+
+ - the encoding of a ``bytes`` and ``string`` value is just the string contents
+   without any padding or length prefix.
+ - the encoding of a struct is the concatenation of the encoding of its members,
+   always padded to a multiple of 32 bytes (even ``bytes`` and ``string``).
+ - the encoding of an array (both dynamically- and statically-sized) is
+   the concatenation of the encoding of its elements, always padded to a multiple
+   of 32 bytes (even ``bytes`` and ``string``) and without any length prefix
+
+In the above, as usual, a negative number is padded by sign extension and not zero padded.
+``bytesNN`` types are padded on the right while ``uintNN`` / ``intNN`` are padded on the left.
+
+.. warning::
+
+    The encoding of a struct is ambiguous if it contains more than one dynamically-sized
+    array. Because of that, always re-check the event data and do not rely on the search result
+    based on the indexed parameters alone.

docs/bugs_by_version.json

@@ -624,5 +624,9 @@
     "0.5.3": {
         "bugs": [],
         "released": "2019-01-22"
+    },
+    "0.5.4": {
+        "bugs": [],
+        "released": "2019-02-12"
     }
 }

docs/control-structures.rst

@@ -264,13 +264,13 @@ Complications for Arrays and Structs
 The semantics of assignments are a bit more complicated for non-value types like arrays and structs.
 Assigning *to* a state variable always creates an independent copy. On the other hand, assigning to a local variable creates an independent copy only for elementary types, i.e. static types that fit into 32 bytes. If structs or arrays (including ``bytes`` and ``string``) are assigned from a state variable to a local variable, the local variable holds a reference to the original state variable. A second assignment to the local variable does not modify the state but only changes the reference. Assignments to members (or elements) of the local variable *do* change the state.
 
-In the example below the call to ``g(x)`` has no effect on ``x`` because it needs
+In the example below the call to ``g(x)`` has no effect on ``x`` because it creates
-to create an independent copy of the storage value in memory. However ``h(x)`` modifies ``x`` because a reference and
+an independent copy of the storage value in memory. However, ``h(x)`` successfully modifies ``x``
-not a copy is passed.
+because only a reference and not a copy is passed.
 
 ::
 
-     pragma solidity >=0.4.16 <0.6.0;
+    pragma solidity >=0.4.16 <0.6.0;
 
      contract C {
         uint[20] x;

docs/frequently-asked-questions.rst

@@ -1,96 +0,0 @@
-###########################
-Frequently Asked Questions
-###########################
-
-This list was originally compiled by `fivedogit <mailto:fivedogit@gmail.com>`_.
-
-
-***************
-Basic Questions
-***************
-
-If I return an ``enum``, I only get integer values in web3.js. How to get the named values?
-===========================================================================================
-
-Enums are not supported by the ABI, they are just supported by Solidity.
-You have to do the mapping yourself for now, we might provide some help
-later.
-
-What are some examples of basic string manipulation (``substring``, ``indexOf``, ``charAt``, etc)?
-==================================================================================================
-
-There are some string utility functions at `stringUtils.sol <https://github.com/ethereum/dapp-bin/blob/master/library/stringUtils.sol>`_
-which will be extended in the future. In addition, Arachnid has written `solidity-stringutils <https://github.com/Arachnid/solidity-stringutils>`_.
-
-For now, if you want to modify a string (even when you only want to know its length),
-you should always convert it to a ``bytes`` first::
-
-    pragma solidity >=0.4.0 <0.6.0;
-
-    contract C {
-        string s;
-
-        function append(byte c) public {
-            bytes(s).push(c);
-        }
-
-        function set(uint i, byte c) public {
-            bytes(s)[i] = c;
-        }
-    }
-
-
-Can I concatenate two strings?
-==============================
-
-Yes, you can use ``abi.encodePacked``::
-
-    pragma solidity >=0.4.0 <0.6.0;
-
-    library ConcatHelper {
-        function concat(bytes memory a, bytes memory b)
-                internal pure returns (bytes memory) {
-            return abi.encodePacked(a, b);
-        }
-    }
-
-******************
-Advanced Questions
-******************
-
-Get return value from non-constant function from another contract
-=================================================================
-
-The key point is that the calling contract needs to know about the function it intends to call.
-
-See `ping.sol <https://github.com/fivedogit/solidity-baby-steps/blob/master/contracts/45_ping.sol>`_
-and `pong.sol <https://github.com/fivedogit/solidity-baby-steps/blob/master/contracts/45_pong.sol>`_.
-
-How do I initialize a contract with only a specific amount of wei?
-==================================================================
-
-Currently the approach is a little ugly, but there is little that can be done to improve it.
-In the case of a ``contract A`` calling a new instance of ``contract B``, parentheses have to be used around
-``new B`` because ``B.value`` would refer to a member of ``B`` called ``value``.
-You will need to make sure that you have both contracts aware of each other's presence and that ``contract B`` has a ``payable`` constructor.
-In this example::
-
-    pragma solidity ^0.5.0;
-
-    contract B {
-        constructor() public payable {}
-    }
-
-    contract A {
-        B child;
-
-        function test() public {
-            child = (new B).value(10)(); //construct a new B with 10 wei
-        }
-    }
-
-More Questions?
-===============
-
-If you have more questions or your question is not answered here, please talk to us on
-`gitter <https://gitter.im/ethereum/solidity>`_ or file an `issue <https://github.com/ethereum/solidity/issues>`_.

docs/index.rst

@@ -87,5 +87,4 @@ Contents
    common-patterns.rst
    bugs.rst
    contributing.rst
-   frequently-asked-questions.rst
    lll.rst

docs/miscellaneous.rst

@@ -351,7 +351,7 @@ Global Variables
 
 - ``abi.decode(bytes memory encodedData, (...)) returns (...)``: :ref:`ABI <ABI>`-decodes the provided data. The types are given in parentheses as second argument. Example: ``(uint a, uint[2] memory b, bytes memory c) = abi.decode(data, (uint, uint[2], bytes))``
 - ``abi.encode(...) returns (bytes memory)``: :ref:`ABI <ABI>`-encodes the given arguments
-- ``abi.encodePacked(...) returns (bytes memory)``: Performs :ref:`packed encoding <abi_packed_mode>` of the given arguments
+- ``abi.encodePacked(...) returns (bytes memory)``: Performs :ref:`packed encoding <abi_packed_mode>` of the given arguments. Note that this encoding can be ambiguous!
 - ``abi.encodeWithSelector(bytes4 selector, ...) returns (bytes memory)``: :ref:`ABI <ABI>`-encodes the given arguments
    starting from the second and prepends the given four-byte selector
 - ``abi.encodeWithSignature(string memory signature, ...) returns (bytes memory)``: Equivalent to ``abi.encodeWithSelector(bytes4(keccak256(bytes(signature)), ...)```

docs/types/reference-types.rst

@@ -106,13 +106,23 @@ Array elements can be of any type, including mapping or struct. The general
 restrictions for types apply, in that mappings can only be stored in the
 ``storage`` data location and publicly-visible functions need parameters that are :ref:`ABI types <ABI>`.
 
+It is possible to mark state variable arrays ``public`` and have Solidity create a :ref:`getter <visibility-and-getters>`.
+The numeric index becomes a required parameter for the getter.
+
 Accessing an array past its end causes a failing assertion. You can use the ``.push()`` method to append a new element at the end or assign to the ``.length`` :ref:`member <array-members>` to change the size (see below for caveats).
 method or increase the ``.length`` :ref:`member <array-members>` to add elements.
 
+``bytes`` and ``strings`` as Arrays
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
 Variables of type ``bytes`` and ``string`` are special arrays. A ``bytes`` is similar to ``byte[]``,
 but it is packed tightly in calldata and memory. ``string`` is equal to ``bytes`` but does not allow
 length or index access.
 
+Solidity does not have string manipulation functions, but there are
+third-party string libraries. You can also compare two strings by their keccak256-hash using
+``keccak256(abi.encodePacked(s1)) == keccak256(abi.encodePacked(s2))`` and concatenate two strings using ``abi.encodePacked(s1, s2)``.
+
 You should use ``bytes`` over ``byte[]`` because it is cheaper, since ``byte[]`` adds 31 padding bytes between the elements. As a general rule,
 use ``bytes`` for arbitrary-length raw byte data and ``string`` for arbitrary-length
 string (UTF-8) data. If you can limit the length to a certain number of bytes,
@@ -124,9 +134,6 @@ always use one of the value types ``bytes1`` to ``bytes32`` because they are muc
     that you are accessing the low-level bytes of the UTF-8 representation,
     and not the individual characters.
 
-It is possible to mark arrays ``public`` and have Solidity create a :ref:`getter <visibility-and-getters>`.
-The numeric index becomes a required parameter for the getter.
-
 .. index:: ! array;allocating, new
 
 Allocating Memory Arrays

docs/types/value-types.rst

@@ -713,4 +713,4 @@ Another example that uses external function types::
     }
 
 .. note::
-    Lambda or inline functions are planned but not yet supported.
+    Lambda or inline functions are planned but not yet supported.

docs/units-and-global-variables.rst

@@ -117,7 +117,7 @@ ABI Encoding and Decoding Functions
 
 - ``abi.decode(bytes memory encodedData, (...)) returns (...)``: ABI-decodes the given data, while the types are given in parentheses as second argument. Example: ``(uint a, uint[2] memory b, bytes memory c) = abi.decode(data, (uint, uint[2], bytes))``
 - ``abi.encode(...) returns (bytes memory)``: ABI-encodes the given arguments
-- ``abi.encodePacked(...) returns (bytes memory)``: Performs :ref:`packed encoding <abi_packed_mode>` of the given arguments
+- ``abi.encodePacked(...) returns (bytes memory)``: Performs :ref:`packed encoding <abi_packed_mode>` of the given arguments. Note that packed encoding can be ambiguous!
 - ``abi.encodeWithSelector(bytes4 selector, ...) returns (bytes memory)``: ABI-encodes the given arguments starting from the second and prepends the given four-byte selector
 - ``abi.encodeWithSignature(string memory signature, ...) returns (bytes memory)``: Equivalent to ``abi.encodeWithSelector(bytes4(keccak256(bytes(signature))), ...)```
 
@@ -255,7 +255,7 @@ Type Information
 The expression ``type(X)`` can be used to retrieve information about the
 type ``X``. Currently, there is limited support for this feature, but
 it might be expanded in the future. The following properties are
-available for a conract type ``C``:
+available for a contract type ``C``:
 
 ``type(C).creationCode``:
     Memory byte array that contains the creation bytecode of the contract.

docs/using-the-compiler.rst

@@ -19,8 +19,8 @@ If you only want to compile a single file, you run it as ``solc --bin sourceFile
 Before you deploy your contract, activate the optimizer when compiling using ``solc --optimize --bin sourceFile.sol``.
 By default, the optimizer will optimize the contract assuming it is called 200 times across its lifetime.
 If you want the initial contract deployment to be cheaper and the later function executions to be more expensive,
-set it to ``--runs=1``. If you expect many transactions and do not care for higher deployment cost and
+set it to ``--optimize-runs=1``. If you expect many transactions and do not care for higher deployment cost and
-output size, set ``--runs`` to a high number.
+output size, set ``--optimize-runs`` to a high number.
 
 The commandline compiler will automatically read imported files from the filesystem, but
 it is also possible to provide path redirects using ``prefix=path`` in the following way:
@@ -140,9 +140,9 @@ Input Description
 
     {
       // Required: Source code language, such as "Solidity", "Vyper", "lll", "assembly", etc.
-      language: "Solidity",
+      "language": "Solidity",
       // Required
-      sources:
+      "sources":
       {
         // The keys here are the "global" names of the source files,
         // imports can use other files via remappings (see below).
@@ -155,13 +155,16 @@ Input Description
           // URL(s) should be imported in this order and the result checked against the
           // keccak256 hash (if available). If the hash doesn't match or none of the
           // URL(s) result in success, an error should be raised.
+          // Using the commandline interface only filesystem paths are supported.
+          // With the JavaScript interface the URL will be passed to the user-supplied
+          // read callback, so any URL supported by the callback can be used.
           "urls":
           [
             "bzzr://56ab...",
             "ipfs://Qma...",
+            "/tmp/path/to/file.sol"
             // If files are used, their directories should be added to the command line via
             // `--allow-paths <path>`.
-            "file:///tmp/path/to/file.sol"
           ]
         },
         "mortal":
@@ -173,26 +176,26 @@ Input Description
         }
       },
       // Optional
-      settings:
+      "settings":
       {
         // Optional: Sorted list of remappings
-        remappings: [ ":g/dir" ],
+        "remappings": [ ":g/dir" ],
         // Optional: Optimizer settings
-        optimizer: {
+        "optimizer": {
           // disabled by default
-          enabled: true,
+          "enabled": true,
           // Optimize for how many times you intend to run the code.
           // Lower values will optimize more for initial deployment cost, higher values will optimize more for high-frequency usage.
-          runs: 200
+          "runs": 200
         },
-        evmVersion: "byzantium", // Version of the EVM to compile for. Affects type checking and code generation. Can be homestead, tangerineWhistle, spuriousDragon, byzantium or constantinople
+        "evmVersion": "byzantium", // Version of the EVM to compile for. Affects type checking and code generation. Can be homestead, tangerineWhistle, spuriousDragon, byzantium or constantinople
         // Metadata settings (optional)
-        metadata: {
+        "metadata": {
           // Use only literal content and not URLs (false by default)
-          useLiteralContent: true
+          "useLiteralContent": true
         },
         // Addresses of the libraries. If not all libraries are given here, it can result in unlinked objects whose output data is different.
-        libraries: {
+        "libraries": {
           // The top level key is the the name of the source file where the library is used.
           // If remappings are used, this source file should match the global path after remappings were applied.
           // If this key is an empty string, that refers to a global level.
@@ -240,22 +243,19 @@ Input Description
         // Note that using a using `evm`, `evm.bytecode`, `ewasm`, etc. will select every
         // target part of that output. Additionally, `*` can be used as a wildcard to request everything.
         //
-        outputSelection: {
+        "outputSelection": {
-          // Enable the metadata and bytecode outputs of every single contract.
           "*": {
-            "*": [ "metadata", "evm.bytecode" ]
+            "*": [
+              "metadata", "evm.bytecode" // Enable the metadata and bytecode outputs of every single contract.
+              , "evm.bytecode.sourceMap" // Enable the source map output of every single contract.
+            ],
+            "": [
+              "ast" // Enable the AST output of every single file.
+            ]
           },
           // Enable the abi and opcodes output of MyContract defined in file def.
           "def": {
             "MyContract": [ "abi", "evm.bytecode.opcodes" ]
-          },
-          // Enable the source map output of every single contract.
-          "*": {
-            "*": [ "evm.bytecode.sourceMap" ]
-          },
-          // Enable the legacy AST output of every single file.
-          "*": {
-            "": [ "legacyAST" ]
           }
         }
       }
@@ -269,106 +269,106 @@ Output Description
 
     {
       // Optional: not present if no errors/warnings were encountered
-      errors: [
+      "errors": [
         {
           // Optional: Location within the source file.
-          sourceLocation: {
+          "sourceLocation": {
-            file: "sourceFile.sol",
+            "file": "sourceFile.sol",
-            start: 0,
+            "start": 0,
-            end: 100
+            "end": 100
           ],
           // Mandatory: Error type, such as "TypeError", "InternalCompilerError", "Exception", etc.
           // See below for complete list of types.
-          type: "TypeError",
+          "type": "TypeError",
           // Mandatory: Component where the error originated, such as "general", "ewasm", etc.
-          component: "general",
+          "component": "general",
           // Mandatory ("error" or "warning")
-          severity: "error",
+          "severity": "error",
           // Mandatory
-          message: "Invalid keyword"
+          "message": "Invalid keyword"
           // Optional: the message formatted with source location
-          formattedMessage: "sourceFile.sol:100: Invalid keyword"
+          "formattedMessage": "sourceFile.sol:100: Invalid keyword"
         }
       ],
       // This contains the file-level outputs. In can be limited/filtered by the outputSelection settings.
-      sources: {
+      "sources": {
         "sourceFile.sol": {
           // Identifier of the source (used in source maps)
-          id: 1,
+          "id": 1,
           // The AST object
-          ast: {},
+          "ast": {},
           // The legacy AST object
-          legacyAST: {}
+          "legacyAST": {}
         }
       },
       // This contains the contract-level outputs. It can be limited/filtered by the outputSelection settings.
-      contracts: {
+      "contracts": {
         "sourceFile.sol": {
           // If the language used has no contract names, this field should equal to an empty string.
           "ContractName": {
             // The Ethereum Contract ABI. If empty, it is represented as an empty array.
             // See https://github.com/ethereum/wiki/wiki/Ethereum-Contract-ABI
-            abi: [],
+            "abi": [],
             // See the Metadata Output documentation (serialised JSON string)
-            metadata: "{...}",
+            "metadata": "{...}",
             // User documentation (natspec)
-            userdoc: {},
+            "userdoc": {},
             // Developer documentation (natspec)
-            devdoc: {},
+            "devdoc": {},
             // Intermediate representation (string)
-            ir: "",
+            "ir": "",
             // EVM-related outputs
-            evm: {
+            "evm": {
               // Assembly (string)
-              assembly: "",
+              "assembly": "",
               // Old-style assembly (object)
-              legacyAssembly: {},
+              "legacyAssembly": {},
               // Bytecode and related details.
-              bytecode: {
+              "bytecode": {
                 // The bytecode as a hex string.
-                object: "00fe",
+                "object": "00fe",
                 // Opcodes list (string)
-                opcodes: "",
+                "opcodes": "",
                 // The source mapping as a string. See the source mapping definition.
-                sourceMap: "",
+                "sourceMap": "",
                 // If given, this is an unlinked object.
-                linkReferences: {
+                "linkReferences": {
                   "libraryFile.sol": {
                     // Byte offsets into the bytecode. Linking replaces the 20 bytes located there.
                     "Library1": [
-                      { start: 0, length: 20 },
+                      { "start": 0, "length": 20 },
-                      { start: 200, length: 20 }
+                      { "start": 200, "length": 20 }
                     ]
                   }
                 }
               },
               // The same layout as above.
-              deployedBytecode: { },
+              "deployedBytecode": { },
               // The list of function hashes
-              methodIdentifiers: {
+              "methodIdentifiers": {
                 "delegate(address)": "5c19a95c"
               },
               // Function gas estimates
-              gasEstimates: {
+              "gasEstimates": {
-                creation: {
+                "creation": {
-                  codeDepositCost: "420000",
+                  "codeDepositCost": "420000",
-                  executionCost: "infinite",
+                  "executionCost": "infinite",
-                  totalCost: "infinite"
+                  "totalCost": "infinite"
                 },
-                external: {
+                "external": {
                   "delegate(address)": "25000"
                 },
-                internal: {
+                "internal": {
                   "heavyLifting()": "infinite"
                 }
               }
             },
             // eWASM related outputs
-            ewasm: {
+            "ewasm": {
               // S-expressions format
-              wast: "",
+              "wast": "",
               // Binary format (hex string)
-              wasm: ""
+              "wasm": ""
             }
           }
         }

libdevcore/AnsiColorized.h

@@ -0,0 +1,91 @@
+/*
+	This file is part of solidity.
+
+	solidity is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	solidity is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with solidity.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#pragma once
+
+#include <ostream>
+#include <vector>
+
+namespace dev
+{
+
+namespace formatting
+{
+
+// control codes
+static constexpr char const* RESET = "\033[0m";
+static constexpr char const* INVERSE = "\033[7m";
+static constexpr char const* BOLD = "\033[1m";
+static constexpr char const* BRIGHT = BOLD;
+
+// standard foreground colors
+static constexpr char const* BLACK = "\033[30m";
+static constexpr char const* RED = "\033[31m";
+static constexpr char const* GREEN = "\033[32m";
+static constexpr char const* YELLOW = "\033[33m";
+static constexpr char const* BLUE = "\033[34m";
+static constexpr char const* MAGENTA = "\033[35m";
+static constexpr char const* CYAN = "\033[36m";
+static constexpr char const* WHITE = "\033[37m";
+
+// standard background colors
+static constexpr char const* BLACK_BACKGROUND = "\033[40m";
+static constexpr char const* RED_BACKGROUND = "\033[41m";
+static constexpr char const* GREEN_BACKGROUND = "\033[42m";
+static constexpr char const* YELLOW_BACKGROUND = "\033[43m";
+static constexpr char const* BLUE_BACKGROUND = "\033[44m";
+static constexpr char const* MAGENTA_BACKGROUND = "\033[45m";
+static constexpr char const* CYAN_BACKGROUND = "\033[46m";
+static constexpr char const* WHITE_BACKGROUND = "\033[47m";
+
+// 256-bit-colors (incomplete set)
+static constexpr char const* RED_BACKGROUND_256 = "\033[48;5;160m";
+static constexpr char const* ORANGE_BACKGROUND_256 = "\033[48;5;166m";
+
+}
+
+/// AnsiColorized provides a convenience helper to colorize ostream with formatting-reset assured.
+class AnsiColorized
+{
+public:
+	AnsiColorized(std::ostream& _os, bool const _enabled, std::vector<char const*>&& _formatting):
+		m_stream{_os}, m_enabled{_enabled}, m_codes{std::move(_formatting)}
+	{
+		if (m_enabled)
+			for (auto const& code: m_codes)
+				m_stream << code;
+	}
+
+	~AnsiColorized()
+	{
+		if (m_enabled)
+			m_stream << formatting::RESET;
+	}
+
+	template <typename T>
+	std::ostream& operator<<(T&& _t)
+	{
+		return m_stream << std::forward<T>(_t);
+	}
+
+private:
+	std::ostream& m_stream;
+	bool m_enabled;
+	std::vector<char const*> m_codes;
+};
+
+}

libdevcore/CMakeLists.txt

@@ -1,5 +1,6 @@
 set(sources
 	Algorithms.h
+	AnsiColorized.h
 	Assertions.h
 	Common.h
 	CommonData.cpp

libdevcore/CommonData.h

@@ -334,4 +334,11 @@ bool containerEqual(Container const& _lhs, Container const& _rhs, Compare&& _com
 	return std::equal(std::begin(_lhs), std::end(_lhs), std::begin(_rhs), std::end(_rhs), std::forward<Compare>(_compare));
 }
 
+inline std::string findAnyOf(std::string const& _haystack, std::vector<std::string> const& _needles)
+{
+	for (std::string const& needle: _needles)
+		if (_haystack.find(needle) != std::string::npos)
+			return needle;
+	return "";
+}
 }

libevmasm/Assembly.cpp

@@ -496,8 +496,7 @@ map<u256, u256> Assembly::optimiseInternal(
 			_settings.isCreation,
 			_settings.isCreation ? 1 : _settings.expectedExecutionsPerDeployment,
 			_settings.evmVersion,
-			*this,
+			*this
-			m_items
 		);
 
 	return tagReplacements;

libevmasm/Assembly.h

@@ -86,6 +86,9 @@ public:
 	/// Returns the assembly items.
 	AssemblyItems const& items() const { return m_items; }
 
+	/// Returns the mutable assembly items. Use with care!
+	AssemblyItems& items() { return m_items; }
+
 	int deposit() const { return m_deposit; }
 	void adjustDeposit(int _adjustment) { m_deposit += _adjustment; assertThrow(m_deposit >= 0, InvalidDeposit, ""); }
 	void setDeposit(int _deposit) { m_deposit = _deposit; assertThrow(m_deposit >= 0, InvalidDeposit, ""); }

libevmasm/ConstantOptimiser.cpp

@@ -30,10 +30,12 @@ unsigned ConstantOptimisationMethod::optimiseConstants(
 	bool _isCreation,
 	size_t _runs,
 	solidity::EVMVersion _evmVersion,
-	Assembly& _assembly,
+	Assembly& _assembly
-	AssemblyItems& _items
 )
 {
+	// TODO: design the optimiser in a way this is not needed
+	AssemblyItems& _items = _assembly.items();
+
 	unsigned optimisations = 0;
 	map<AssemblyItem, size_t> pushes;
 	for (AssemblyItem const& item: _items)
@@ -129,16 +131,11 @@ bigint LiteralMethod::gasNeeded() const
 	return combineGas(
 		simpleRunGas({Instruction::PUSH1}),
 		// PUSHX plus data
-		(m_params.isCreation ? GasCosts::txDataNonZeroGas : GasCosts::createDataGas) + dataGas(),
+		(m_params.isCreation ? GasCosts::txDataNonZeroGas : GasCosts::createDataGas) + dataGas(toCompactBigEndian(m_value, 1)),
 		0
 	);
 }
 
-CodeCopyMethod::CodeCopyMethod(Params const& _params, u256 const& _value):
-	ConstantOptimisationMethod(_params, _value)
-{
-}
-
 bigint CodeCopyMethod::gasNeeded() const
 {
 	return combineGas(

libevmasm/ConstantOptimiser.h

@@ -47,16 +47,18 @@ class ConstantOptimisationMethod
 {
 public:
 	/// Tries to optimised how constants are represented in the source code and modifies
-	/// @a _assembly and its @a _items.
+	/// @a _assembly.
 	/// @returns zero if no optimisations could be performed.
 	static unsigned optimiseConstants(
 		bool _isCreation,
 		size_t _runs,
 		solidity::EVMVersion _evmVersion,
-		Assembly& _assembly,
+		Assembly& _assembly
-		AssemblyItems& _items
 	);
 
+protected:
+	/// This is the public API for the optimiser methods, but it doesn't need to be exposed to the caller.
+
 	struct Params
 	{
 		bool isCreation; ///< Whether this is called during contract creation or runtime.
@@ -79,8 +81,6 @@ protected:
 	static bigint simpleRunGas(AssemblyItems const& _items);
 	/// @returns the gas needed to store the given data literally
 	bigint dataGas(bytes const& _data) const;
-	/// @returns the gas needed to store the value literally
-	bigint dataGas() const { return dataGas(toCompactBigEndian(m_value, 1)); }
 	static size_t bytesRequired(AssemblyItems const& _items);
 	/// @returns the combined estimated gas usage taking @a m_params into account.
 	bigint combineGas(
@@ -119,7 +119,8 @@ public:
 class CodeCopyMethod: public ConstantOptimisationMethod
 {
 public:
-	explicit CodeCopyMethod(Params const& _params, u256 const& _value);
+	explicit CodeCopyMethod(Params const& _params, u256 const& _value):
+		ConstantOptimisationMethod(_params, _value) {}
 	bigint gasNeeded() const override;
 	AssemblyItems execute(Assembly& _assembly) const override;
 

liblangutil/CMakeLists.txt

@@ -16,6 +16,8 @@ set(sources
 	SourceReferenceExtractor.h
 	SourceReferenceFormatter.cpp
 	SourceReferenceFormatter.h
+	SourceReferenceFormatterHuman.cpp
+	SourceReferenceFormatterHuman.h
 	Token.cpp
 	Token.h
 	UndefMacros.h

liblangutil/ErrorReporter.h

@@ -22,6 +22,8 @@
 
 #pragma once
 
+#include <libdevcore/CommonData.h>
+
 #include <liblangutil/Exceptions.h>
 #include <liblangutil/SourceLocation.h>
 
@@ -40,6 +42,11 @@ public:
 
 	ErrorReporter& operator=(ErrorReporter const& _errorReporter);
 
+	void append(ErrorList const& _errorList)
+	{
+		m_errorList += _errorList;
+	}
+
 	void warning(std::string const& _description);
 
 	void warning(SourceLocation const& _location, std::string const& _description);

liblangutil/SourceReferenceFormatter.h

@@ -44,11 +44,14 @@ public:
 		m_stream(_stream)
 	{}
 
+	virtual ~SourceReferenceFormatter() = default;
+
 	/// Prints source location if it is given.
-	void printSourceLocation(SourceLocation const* _location);
+	virtual void printSourceLocation(SourceReference const& _ref);
-	void printSourceLocation(SourceReference const& _ref);
+	virtual void printExceptionInformation(SourceReferenceExtractor::Message const& _msg);
-	void printExceptionInformation(dev::Exception const& _error, std::string const& _category);
+
-	void printExceptionInformation(SourceReferenceExtractor::Message const& _msg);
+	virtual void printSourceLocation(SourceLocation const* _location);
+	virtual void printExceptionInformation(dev::Exception const& _error, std::string const& _category);
 
 	static std::string formatExceptionInformation(
 		dev::Exception const& _exception,
@@ -62,7 +65,7 @@ public:
 		return errorOutput.str();
 	}
 
-private:
+protected:
 	/// Prints source name if location is given.
 	void printSourceName(SourceReference const& _ref);
 

liblangutil/SourceReferenceFormatterHuman.cpp

@@ -0,0 +1,136 @@
+/*
+	This file is part of solidity.
+
+	solidity is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	solidity is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with solidity.  If not, see <http://www.gnu.org/licenses/>.
+*/
+/**
+ * Formatting functions for errors referencing positions and locations in the source.
+ */
+
+#include <liblangutil/SourceReferenceFormatterHuman.h>
+#include <liblangutil/Scanner.h>
+#include <liblangutil/Exceptions.h>
+#include <cmath>
+#include <iomanip>
+
+using namespace std;
+using namespace dev;
+using namespace dev::formatting;
+using namespace langutil;
+
+AnsiColorized SourceReferenceFormatterHuman::normalColored() const
+{
+	return AnsiColorized(m_stream, m_colored, {WHITE});
+}
+
+AnsiColorized SourceReferenceFormatterHuman::frameColored() const
+{
+	return AnsiColorized(m_stream, m_colored, {BOLD, BLUE});
+}
+
+AnsiColorized SourceReferenceFormatterHuman::errorColored() const
+{
+	return AnsiColorized(m_stream, m_colored, {BOLD, RED});
+}
+
+AnsiColorized SourceReferenceFormatterHuman::messageColored() const
+{
+	return AnsiColorized(m_stream, m_colored, {BOLD, WHITE});
+}
+
+AnsiColorized SourceReferenceFormatterHuman::secondaryColored() const
+{
+	return AnsiColorized(m_stream, m_colored, {BOLD, CYAN});
+}
+
+AnsiColorized SourceReferenceFormatterHuman::highlightColored() const
+{
+	return AnsiColorized(m_stream, m_colored, {YELLOW});
+}
+
+AnsiColorized SourceReferenceFormatterHuman::diagColored() const
+{
+	return AnsiColorized(m_stream, m_colored, {BOLD, YELLOW});
+}
+
+void SourceReferenceFormatterHuman::printSourceLocation(SourceReference const& _ref)
+{
+	if (_ref.position.line < 0)
+		return; // Nothing we can print here
+
+	int const leftpad = static_cast<int>(log10(max(_ref.position.line, 1))) + 1;
+
+	// line 0: source name
+	frameColored() << string(leftpad, ' ') << "--> ";
+	m_stream << _ref.sourceName << ":" << (_ref.position.line + 1) << ":" << (_ref.position.column + 1) << ": " << '\n';
+
+	if (!_ref.multiline)
+	{
+		int const locationLength = _ref.endColumn - _ref.startColumn;
+
+		// line 1:
+		m_stream << string(leftpad, ' ');
+		frameColored() << " |" << '\n';
+
+		// line 2:
+		frameColored() << (_ref.position.line + 1) << " | ";
+		m_stream << _ref.text.substr(0, _ref.startColumn);
+		highlightColored() << _ref.text.substr(_ref.startColumn, locationLength);
+		m_stream << _ref.text.substr(_ref.endColumn) << '\n';
+
+		// line 3:
+		m_stream << string(leftpad, ' ');
+		frameColored() << " | ";
+		for_each(
+			_ref.text.cbegin(),
+			_ref.text.cbegin() + _ref.startColumn,
+			[this](char ch) { m_stream << (ch == '\t' ? '\t' : ' '); }
+		);
+		diagColored() << string(locationLength, '^') << '\n';
+	}
+	else
+	{
+		// line 1:
+		m_stream << string(leftpad, ' ');
+		frameColored() << " |" << '\n';
+
+		// line 2:
+		frameColored() << (_ref.position.line + 1) << " | ";
+		m_stream << _ref.text.substr(0, _ref.startColumn);
+		highlightColored() << _ref.text.substr(_ref.startColumn) << '\n';
+
+		// line 3:
+		frameColored() << string(leftpad, ' ') << " | ";
+		m_stream << string(_ref.startColumn, ' ');
+		diagColored() << "^ (Relevant source part starts here and spans across multiple lines).\n";
+	}
+}
+
+void SourceReferenceFormatterHuman::printExceptionInformation(SourceReferenceExtractor::Message const& _msg)
+{
+	// exception header line
+	errorColored() << _msg.category;
+	messageColored() << ": " << _msg.primary.message << '\n';
+
+	printSourceLocation(_msg.primary);
+
+	for (auto const& secondary: _msg.secondary)
+	{
+		secondaryColored() << "Note";
+		messageColored() << ": " << secondary.message << '\n';
+		printSourceLocation(secondary);
+	}
+
+	m_stream << '\n';
+}

liblangutil/SourceReferenceFormatterHuman.h

@@ -0,0 +1,80 @@
+/*
+	This file is part of solidity.
+
+	solidity is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	solidity is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with solidity.  If not, see <http://www.gnu.org/licenses/>.
+*/
+/**
+ * Formatting functions for errors referencing positions and locations in the source.
+ */
+
+#pragma once
+
+#include <liblangutil/SourceReferenceExtractor.h>
+#include <liblangutil/SourceReferenceFormatter.h> // SourceReferenceFormatterBase
+
+#include <libdevcore/AnsiColorized.h>
+
+#include <ostream>
+#include <sstream>
+#include <functional>
+
+namespace dev
+{
+struct Exception; // forward
+}
+
+namespace langutil
+{
+
+struct SourceLocation;
+struct SourceReference;
+
+class SourceReferenceFormatterHuman: public SourceReferenceFormatter
+{
+public:
+	SourceReferenceFormatterHuman(std::ostream& _stream, bool colored):
+		SourceReferenceFormatter{_stream}, m_colored{colored}
+	{}
+
+	void printSourceLocation(SourceReference const& _ref) override;
+	void printExceptionInformation(SourceReferenceExtractor::Message const& _msg) override;
+	using SourceReferenceFormatter::printExceptionInformation;
+
+	static std::string formatExceptionInformation(
+		dev::Exception const& _exception,
+		std::string const& _name,
+		bool colored = false
+	)
+	{
+		std::ostringstream errorOutput;
+
+		SourceReferenceFormatterHuman formatter(errorOutput, colored);
+		formatter.printExceptionInformation(_exception, _name);
+		return errorOutput.str();
+	}
+
+private:
+	dev::AnsiColorized normalColored() const;
+	dev::AnsiColorized frameColored() const;
+	dev::AnsiColorized errorColored() const;
+	dev::AnsiColorized messageColored() const;
+	dev::AnsiColorized secondaryColored() const;
+	dev::AnsiColorized highlightColored() const;
+	dev::AnsiColorized diagColored() const;
+
+private:
+	bool m_colored;
+};
+
+}

libsolc/libsolc.cpp

@@ -98,4 +98,8 @@ extern char const* solidity_compile(char const* _input, CStyleReadFileCallback _
 	s_outputBuffer = compile(_input, _readCallback);
 	return s_outputBuffer.c_str();
 }
+extern void solidity_free() noexcept
+{
+	s_outputBuffer.clear();
+}
 }

libsolc/libsolc.h

@@ -34,14 +34,32 @@
 extern "C" {
 #endif
 
-/// Callback used to retrieve additional source files. "Returns" two pointers that should be
+/// Callback used to retrieve additional source files.
-/// heap-allocated and are free'd by the caller.
+///
+/// "Returns" two pointers that should be heap-allocated and are free'd by the caller.
 typedef void (*CStyleReadFileCallback)(char const* _path, char** o_contents, char** o_error);
 
+/// Returns the complete license document.
+///
+/// The pointer returned must not be freed by the caller.
 char const* solidity_license() SOLC_NOEXCEPT;
+
+/// Returns the compiler version.
+///
+/// The pointer returned must not be freed by the caller.
 char const* solidity_version() SOLC_NOEXCEPT;
+
+/// Takes a "Standard Input JSON" and an optional callback (can be set to null). Returns
+/// a "Standard Output JSON". Both are to be UTF-8 encoded.
+///
+/// The pointer returned must not be freed by the caller.
 char const* solidity_compile(char const* _input, CStyleReadFileCallback _readCallback) SOLC_NOEXCEPT;
 
+/// Frees up any allocated memory.
+///
+/// NOTE: the pointer returned by solidity_compile is invalid after calling this!
+void solidity_free() SOLC_NOEXCEPT;
+
 #ifdef __cplusplus
 }
 #endif

libsolidity/CMakeLists.txt

@@ -102,7 +102,7 @@ set(sources
 	parsing/Token.h
 )
 
-find_package(Z3 QUIET)
+find_package(Z3 4.6.0)
 if (${Z3_FOUND})
   add_definitions(-DHAVE_Z3)
   message("Z3 SMT solver found. This enables optional SMT checking with Z3.")

libsolidity/analysis/TypeChecker.cpp

@@ -48,9 +48,9 @@ using namespace dev::solidity;
 namespace
 {
 
-bool typeSupportedByOldABIEncoder(Type const& _type)
+bool typeSupportedByOldABIEncoder(Type const& _type, bool _isLibraryCall)
 {
-	if (_type.dataStoredIn(DataLocation::Storage))
+	if (_isLibraryCall && _type.dataStoredIn(DataLocation::Storage))
 		return true;
 	if (_type.category() == Type::Category::Struct)
 		return false;
@@ -58,7 +58,7 @@ bool typeSupportedByOldABIEncoder(Type const& _type)
 	{
 		auto const& arrayType = dynamic_cast<ArrayType const&>(_type);
 		auto base = arrayType.baseType();
-		if (!typeSupportedByOldABIEncoder(*base) || (base->category() == Type::Category::Array && base->isDynamicallySized()))
+		if (!typeSupportedByOldABIEncoder(*base, _isLibraryCall) || (base->category() == Type::Category::Array && base->isDynamicallySized()))
 			return false;
 	}
 	return true;
@@ -287,8 +287,7 @@ void TypeChecker::endVisit(UsingForDirective const& _usingFor)
 bool TypeChecker::visit(StructDefinition const& _struct)
 {
 	for (ASTPointer<VariableDeclaration> const& member: _struct.members())
-		if (!type(*member)->canBeStored())
+		solAssert(type(*member)->canBeStored(), "Type cannot be used in struct.");
-			m_errorReporter.typeError(member->location(), "Type cannot be used in struct.");
 
 	// Check recursion, fatal error if detected.
 	auto visitor = [&](StructDefinition const& _struct, CycleDetector<StructDefinition>& _cycleDetector, size_t _depth)
@@ -334,10 +333,17 @@ bool TypeChecker::visit(FunctionDefinition const& _function)
 	auto checkArgumentAndReturnParameter = [&](VariableDeclaration const& var) {
 		if (type(var)->category() == Type::Category::Mapping)
 		{
-			if (!type(var)->dataStoredIn(DataLocation::Storage))
+			if (var.referenceLocation() != VariableDeclaration::Location::Storage)
-				m_errorReporter.typeError(var.location(), "Mapping types can only have a data location of \"storage\"." );
+			{
-			else if (!isLibraryFunction && _function.isPublic())
+				if (!isLibraryFunction && _function.isPublic())
-				m_errorReporter.typeError(var.location(), "Mapping types for parameters or return variables can only be used in internal or library functions.");
+					m_errorReporter.typeError(var.location(), "Mapping types can only have a data location of \"storage\" and thus only be parameters or return variables for internal or library functions.");
+				else
+					m_errorReporter.typeError(var.location(), "Mapping types can only have a data location of \"storage\"." );
+			}
+			else
+			{
+				solAssert(isLibraryFunction || !_function.isPublic(), "Mapping types for parameters or return variables can only be used in internal or library functions.");
+			}
 		}
 		else
 		{
@@ -349,7 +355,7 @@ bool TypeChecker::visit(FunctionDefinition const& _function)
 		if (
 			_function.isPublic() &&
 			!_function.sourceUnit().annotation().experimentalFeatures.count(ExperimentalFeature::ABIEncoderV2) &&
-			!typeSupportedByOldABIEncoder(*type(var))
+			!typeSupportedByOldABIEncoder(*type(var), isLibraryFunction)
 		)
 			m_errorReporter.typeError(
 				var.location(),
@@ -360,15 +366,27 @@ bool TypeChecker::visit(FunctionDefinition const& _function)
 	for (ASTPointer<VariableDeclaration> const& var: _function.parameters())
 	{
 		TypePointer baseType = type(*var);
+		if (auto const* arrayType = dynamic_cast<ArrayType const*>(baseType.get()))
+		{
+			baseType = arrayType->baseType();
+			if (
+				!m_scope->isInterface() &&
+				baseType->dataStoredIn(DataLocation::CallData) &&
+				baseType->isDynamicallyEncoded()
+			)
+				m_errorReporter.typeError(var->location(), "Calldata arrays with dynamically encoded base types are not yet supported.");
+		}
 		while (auto const* arrayType = dynamic_cast<ArrayType const*>(baseType.get()))
 			baseType = arrayType->baseType();
 
 		if (
 			!m_scope->isInterface() &&
-			baseType->category() == Type::Category::Struct &&
 			baseType->dataStoredIn(DataLocation::CallData)
 		)
-			m_errorReporter.typeError(var->location(), "Calldata structs are not yet supported.");
+			if (auto const* structType = dynamic_cast<StructType const*>(baseType.get()))
+				if (structType->isDynamicallyEncoded())
+					m_errorReporter.typeError(var->location(), "Dynamically encoded calldata structs are not yet supported.");
+
 		checkArgumentAndReturnParameter(*var);
 		var->accept(*this);
 	}
@@ -469,7 +487,7 @@ bool TypeChecker::visit(VariableDeclaration const& _variable)
 		{
 			vector<string> unsupportedTypes;
 			for (auto const& param: getter.parameterTypes() + getter.returnParameterTypes())
-				if (!typeSupportedByOldABIEncoder(*param))
+				if (!typeSupportedByOldABIEncoder(*param, false /* isLibrary */))
 					unsupportedTypes.emplace_back(param->toString());
 			if (!unsupportedTypes.empty())
 				m_errorReporter.typeError(_variable.location(),
@@ -572,24 +590,14 @@ bool TypeChecker::visit(EventDefinition const& _eventDef)
 	for (ASTPointer<VariableDeclaration> const& var: _eventDef.parameters())
 	{
 		if (var->isIndexed())
-		{
 			numIndexed++;
-			if (
-				_eventDef.sourceUnit().annotation().experimentalFeatures.count(ExperimentalFeature::ABIEncoderV2) &&
-				dynamic_cast<ReferenceType const*>(type(*var).get())
-			)
-				m_errorReporter.typeError(
-					var->location(),
-					"Indexed reference types cannot yet be used with ABIEncoderV2."
-				);
-		}
 		if (!type(*var)->canLiveOutsideStorage())
 			m_errorReporter.typeError(var->location(), "Type is required to live outside storage.");
 		if (!type(*var)->interfaceType(false))
 			m_errorReporter.typeError(var->location(), "Internal or recursive type is not allowed as event parameter type.");
 		if (
 			!_eventDef.sourceUnit().annotation().experimentalFeatures.count(ExperimentalFeature::ABIEncoderV2) &&
-			!typeSupportedByOldABIEncoder(*type(*var))
+			!typeSupportedByOldABIEncoder(*type(*var), false /* isLibrary */)
 		)
 			m_errorReporter.typeError(
 				var->location(),
@@ -608,8 +616,7 @@ void TypeChecker::endVisit(FunctionTypeName const& _funType)
 {
 	FunctionType const& fun = dynamic_cast<FunctionType const&>(*_funType.annotation().type);
 	if (fun.kind() == FunctionType::Kind::External)
-		if (!fun.canBeUsedExternally(false))
+		solAssert(fun.canBeUsedExternally(false), "External function type uses internal types.");
-			m_errorReporter.typeError(_funType.location(), "External function type uses internal types.");
 }
 
 bool TypeChecker::visit(InlineAssembly const& _inlineAssembly)
@@ -880,8 +887,7 @@ bool TypeChecker::visit(VariableDeclarationStatement const& _statement)
 			if (ref->dataStoredIn(DataLocation::Storage))
 			{
 				string errorText{"Uninitialized storage pointer."};
-				if (varDecl.referenceLocation() == VariableDeclaration::Location::Unspecified)
+				solAssert(varDecl.referenceLocation() != VariableDeclaration::Location::Unspecified, "Expected a specified location at this point");
-					errorText += " Did you mean '<type> memory " + varDecl.name() + "'?";
 				solAssert(m_scope, "");
 				m_errorReporter.declarationError(varDecl.location(), errorText);
 			}
@@ -949,10 +955,7 @@ bool TypeChecker::visit(VariableDeclarationStatement const& _statement)
 					solAssert(false, "");
 			}
 			else if (*var.annotation().type == TupleType())
-				m_errorReporter.typeError(
+				solAssert(false, "Cannot declare variable with void (empty tuple) type.");
-					var.location(),
-					"Cannot declare variable with void (empty tuple) type."
-				);
 			else if (valueComponentType->category() == Type::Category::RationalNumber)
 			{
 				string typeName = var.annotation().type->toString(true);
@@ -1067,25 +1070,39 @@ bool TypeChecker::visit(Conditional const& _conditional)
 
 	TypePointer trueType = type(_conditional.trueExpression())->mobileType();
 	TypePointer falseType = type(_conditional.falseExpression())->mobileType();
-	if (!trueType)
-		m_errorReporter.fatalTypeError(_conditional.trueExpression().location(), "Invalid mobile type.");
-	if (!falseType)
-		m_errorReporter.fatalTypeError(_conditional.falseExpression().location(), "Invalid mobile type.");
 
-	TypePointer commonType = Type::commonType(trueType, falseType);
+	TypePointer commonType;
-	if (!commonType)
+
-	{
+	if (!trueType)
-		m_errorReporter.typeError(
+		m_errorReporter.typeError(_conditional.trueExpression().location(), "Invalid mobile type in true expression.");
-				_conditional.location(),
+	else
-				"True expression's type " +
-				trueType->toString() +
-				" doesn't match false expression's type " +
-				falseType->toString() +
-				"."
-		);
-		// even we can't find a common type, we have to set a type here,
-		// otherwise the upper statement will not be able to check the type.
 		commonType = trueType;
+
+	if (!falseType)
+		m_errorReporter.typeError(_conditional.falseExpression().location(), "Invalid mobile type in false expression.");
+	else
+		commonType = falseType;
+
+	if (!trueType && !falseType)
+		BOOST_THROW_EXCEPTION(FatalError());
+	else if (trueType && falseType)
+	{
+		commonType = Type::commonType(trueType, falseType);
+
+		if (!commonType)
+		{
+			m_errorReporter.typeError(
+					_conditional.location(),
+					"True expression's type " +
+					trueType->toString() +
+					" doesn't match false expression's type " +
+					falseType->toString() +
+					"."
+					);
+			// even we can't find a common type, we have to set a type here,
+			// otherwise the upper statement will not be able to check the type.
+			commonType = trueType;
+		}
 	}
 
 	_conditional.annotation().type = commonType;
@@ -1233,10 +1250,10 @@ bool TypeChecker::visit(TupleExpression const& _tuple)
 					if (!dynamic_cast<RationalNumberType const&>(*types[i]).mobileType())
 						m_errorReporter.fatalTypeError(components[i]->location(), "Invalid rational number.");
 
-				if (_tuple.isInlineArray())
-					solAssert(!!types[i], "Inline array cannot have empty components");
 				if (_tuple.isInlineArray())
 				{
+					solAssert(!!types[i], "Inline array cannot have empty components");
+
 					if ((i == 0 || inlineArrayType) && !types[i]->mobileType())
 						m_errorReporter.fatalTypeError(components[i]->location(), "Invalid mobile type.");
 
@@ -1559,6 +1576,15 @@ void TypeChecker::typeCheckABIEncodeFunctions(
 			}
 		}
 
+		if (isPacked && !typeSupportedByOldABIEncoder(*argType, false /* isLibrary */))
+		{
+			m_errorReporter.typeError(
+				arguments[i]->location(),
+				"Type not supported in packed mode."
+			);
+			continue;
+		}
+
 		if (!argType->fullEncodingType(false, abiEncoderV2, !_functionType->padArguments()))
 			m_errorReporter.typeError(
 				arguments[i]->location(),
@@ -1677,17 +1703,10 @@ void TypeChecker::typeCheckFunctionGeneralChecks(
 	{
 		auto const& parameterNames = _functionType->parameterNames();
 
-		// Check for expected number of named arguments
+		solAssert(
-		if (parameterNames.size() != argumentNames.size())
+			parameterNames.size() == argumentNames.size(),
-		{
+			"Unexpected parameter length mismatch!"
-			m_errorReporter.typeError(
+		);
-				_functionCall.location(),
-				parameterNames.size() > argumentNames.size() ?
-				"Some argument names are missing." :
-				"Too many arguments."
-			);
-			return;
-		}
 
 		// Check for duplicate argument names
 		{
@@ -1971,8 +1990,8 @@ void TypeChecker::endVisit(NewExpression const& _newExpression)
 		_newExpression.annotation().type = make_shared<FunctionType>(
 			TypePointers{make_shared<IntegerType>(256)},
 			TypePointers{type},
-			strings(),
+			strings(1, ""),
-			strings(),
+			strings(1, ""),
 			FunctionType::Kind::ObjectCreation,
 			false,
 			StateMutability::Pure
@@ -2078,8 +2097,8 @@ bool TypeChecker::visit(MemberAccess const& _memberAccess)
 				exprType->toString() + " (expected " + funType->selfType()->toString() + ")."
 			);
 
-	if (exprType->category() == Type::Category::Struct)
+	if (auto const* structType = dynamic_cast<StructType const*>(exprType.get()))
-		annotation.isLValue = true;
+		annotation.isLValue = !structType->dataStoredIn(DataLocation::CallData);
 	else if (exprType->category() == Type::Category::Array)
 	{
 		auto const& arrayType(dynamic_cast<ArrayType const&>(*exprType));
@@ -2198,15 +2217,22 @@ bool TypeChecker::visit(IndexAccess const& _access)
 			resultType = make_shared<TypeType>(make_shared<ArrayType>(DataLocation::Memory, typeType.actualType()));
 		else
 		{
-			expectType(*index, IntegerType::uint256());
+			u256 length = 1;
-			if (auto length = dynamic_cast<RationalNumberType const*>(type(*index).get()))
+			if (expectType(*index, IntegerType::uint256()))
-				resultType = make_shared<TypeType>(make_shared<ArrayType>(
+			{
-					DataLocation::Memory,
+				if (auto indexValue = dynamic_cast<RationalNumberType const*>(type(*index).get()))
-					typeType.actualType(),
+					length = indexValue->literalValue(nullptr);
-					length->literalValue(nullptr)
+				else
-				));
+					m_errorReporter.fatalTypeError(index->location(), "Integer constant expected.");
+			}
 			else
-				m_errorReporter.fatalTypeError(index->location(), "Integer constant expected.");
+				solAssert(m_errorReporter.hasErrors(), "Expected errors as expectType returned false");
+
+			resultType = make_shared<TypeType>(make_shared<ArrayType>(
+				DataLocation::Memory,
+				typeType.actualType(),
+				length
+			));
 		}
 		break;
 	}

libsolidity/ast/Types.cpp

@@ -415,7 +415,7 @@ MemberList const& Type::members(ContractDefinition const* _currentScope) const
 	return *m_members[_currentScope];
 }
 
-TypePointer Type::fullEncodingType(bool _inLibraryCall, bool _encoderV2, bool _packed) const
+TypePointer Type::fullEncodingType(bool _inLibraryCall, bool _encoderV2, bool) const
 {
 	TypePointer encodingType = mobileType();
 	if (encodingType)
@@ -423,7 +423,7 @@ TypePointer Type::fullEncodingType(bool _inLibraryCall, bool _encoderV2, bool _p
 	if (encodingType)
 		encodingType = encodingType->encodingType();
 	// Structs are fine in the following circumstances:
-	// - ABIv2 without packed encoding or,
+	// - ABIv2 or,
 	// - storage struct for a library
 	if (_inLibraryCall && encodingType->dataStoredIn(DataLocation::Storage))
 		return encodingType;
@@ -431,7 +431,7 @@ TypePointer Type::fullEncodingType(bool _inLibraryCall, bool _encoderV2, bool _p
 	while (auto const* arrayType = dynamic_cast<ArrayType const*>(baseType.get()))
 		baseType = arrayType->baseType();
 	if (dynamic_cast<StructType const*>(baseType.get()))
-		if (!_encoderV2 || _packed)
+		if (!_encoderV2)
 			return TypePointer();
 	return encodingType;
 }
@@ -1716,8 +1716,10 @@ bigint ArrayType::unlimitedCalldataEncodedSize(bool _padded) const
 {
 	if (isDynamicallySized())
 		return 32;
-	bigint size = bigint(length()) * (isByteArray() ? 1 : baseType()->calldataEncodedSize(_padded));
+	// Array elements are always padded.
-	size = ((size + 31) / 32) * 32;
+	bigint size = bigint(length()) * (isByteArray() ? 1 : baseType()->calldataEncodedSize(true));
+	if (_padded)
+		size = ((size + 31) / 32) * 32;
 	return size;
 }
 
@@ -1834,8 +1836,8 @@ MemberList::MemberMap ArrayType::nativeMembers(ContractDefinition const*) const
 			members.emplace_back("pop", make_shared<FunctionType>(
 				TypePointers{},
 				TypePointers{},
-				strings{string()},
+				strings{},
-				strings{string()},
+				strings{},
 				FunctionType::Kind::ArrayPop
 			));
 		}
@@ -2034,7 +2036,7 @@ bool StructType::operator==(Type const& _other) const
 	return ReferenceType::operator==(other) && other.m_struct == m_struct;
 }
 
-unsigned StructType::calldataEncodedSize(bool _padded) const
+unsigned StructType::calldataEncodedSize(bool) const
 {
 	unsigned size = 0;
 	for (auto const& member: members(nullptr))
@@ -2042,7 +2044,8 @@ unsigned StructType::calldataEncodedSize(bool _padded) const
 			return 0;
 		else
 		{
-			unsigned memberSize = member.type->calldataEncodedSize(_padded);
+			// Struct members are always padded.
+			unsigned memberSize = member.type->calldataEncodedSize(true);
 			if (memberSize == 0)
 				return 0;
 			size += memberSize;
@@ -2050,6 +2053,24 @@ unsigned StructType::calldataEncodedSize(bool _padded) const
 	return size;
 }
 
+unsigned StructType::calldataOffsetOfMember(std::string const& _member) const
+{
+	unsigned offset = 0;
+	for (auto const& member: members(nullptr))
+	{
+		solAssert(member.type->canLiveOutsideStorage(), "");
+		if (member.name == _member)
+			return offset;
+		{
+			// Struct members are always padded.
+			unsigned memberSize = member.type->calldataEncodedSize(true);
+			solAssert(memberSize != 0, "");
+			offset += memberSize;
+		}
+	}
+	solAssert(false, "Struct member not found.");
+}
+
 bool StructType::isDynamicallyEncoded() const
 {
 	solAssert(!recursive(), "");
@@ -2188,7 +2209,7 @@ FunctionTypePointer StructType::constructorType() const
 		paramTypes,
 		TypePointers{copyForLocation(DataLocation::Memory, false)},
 		paramNames,
-		strings(),
+		strings(1, ""),
 		FunctionType::Kind::Internal
 	);
 }
@@ -2420,6 +2441,16 @@ FunctionType::FunctionType(FunctionDefinition const& _function, bool _isInternal
 		m_returnParameterNames.push_back(var->name());
 		m_returnParameterTypes.push_back(var->annotation().type);
 	}
+
+	solAssert(
+		m_parameterNames.size() == m_parameterTypes.size(),
+		"Parameter names list must match parameter types list!"
+	);
+
+	solAssert(
+		m_returnParameterNames.size() == m_returnParameterTypes.size(),
+		"Return parameter names list must match return parameter types list!"
+	);
 }
 
 FunctionType::FunctionType(VariableDeclaration const& _varDecl):
@@ -2476,6 +2507,15 @@ FunctionType::FunctionType(VariableDeclaration const& _varDecl):
 		));
 		m_returnParameterNames.emplace_back("");
 	}
+
+	solAssert(
+			m_parameterNames.size() == m_parameterTypes.size(),
+			"Parameter names list must match parameter types list!"
+			);
+	solAssert(
+			m_returnParameterNames.size() == m_returnParameterTypes.size(),
+			"Return parameter names list must match return parameter types list!"
+			);
 }
 
 FunctionType::FunctionType(EventDefinition const& _event):
@@ -2488,9 +2528,20 @@ FunctionType::FunctionType(EventDefinition const& _event):
 		m_parameterNames.push_back(var->name());
 		m_parameterTypes.push_back(var->annotation().type);
 	}
+
+	solAssert(
+			m_parameterNames.size() == m_parameterTypes.size(),
+			"Parameter names list must match parameter types list!"
+			);
+	solAssert(
+			m_returnParameterNames.size() == m_returnParameterTypes.size(),
+			"Return parameter names list must match return parameter types list!"
+			);
 }
 
 FunctionType::FunctionType(FunctionTypeName const& _typeName):
+	m_parameterNames(_typeName.parameterTypes().size(), ""),
+	m_returnParameterNames(_typeName.returnParameterTypes().size(), ""),
 	m_kind(_typeName.visibility() == VariableDeclaration::Visibility::External ? Kind::External : Kind::Internal),
 	m_stateMutability(_typeName.stateMutability())
 {
@@ -2516,6 +2567,15 @@ FunctionType::FunctionType(FunctionTypeName const& _typeName):
 			);
 		m_returnParameterTypes.push_back(t->annotation().type);
 	}
+
+	solAssert(
+			m_parameterNames.size() == m_parameterTypes.size(),
+			"Parameter names list must match parameter types list!"
+			);
+	solAssert(
+			m_returnParameterNames.size() == m_returnParameterTypes.size(),
+			"Return parameter names list must match return parameter types list!"
+			);
 }
 
 FunctionTypePointer FunctionType::newExpressionType(ContractDefinition const& _contract)
@@ -2853,8 +2913,8 @@ MemberList::MemberMap FunctionType::nativeMembers(ContractDefinition const*) con
 					make_shared<FunctionType>(
 						parseElementaryTypeVector({"uint"}),
 						TypePointers{copyAndSetGasOrValue(false, true)},
-						strings(),
+						strings(1, ""),
-						strings(),
+						strings(1, ""),
 						Kind::SetValue,
 						false,
 						StateMutability::NonPayable,
@@ -2870,8 +2930,8 @@ MemberList::MemberMap FunctionType::nativeMembers(ContractDefinition const*) con
 				make_shared<FunctionType>(
 					parseElementaryTypeVector({"uint"}),
 					TypePointers{copyAndSetGasOrValue(true, false)},
-					strings(),
+					strings(1, ""),
-					strings(),
+					strings(1, ""),
 					Kind::SetGas,
 					false,
 					StateMutability::NonPayable,
@@ -3024,8 +3084,8 @@ u256 FunctionType::externalIdentifier() const
 
 bool FunctionType::isPure() const
 {
-	// FIXME: replace this with m_stateMutability == StateMutability::Pure once
+	// TODO: replace this with m_stateMutability == StateMutability::Pure once
-	//        the callgraph analyzer is in place
+	//       the callgraph analyzer is in place
 	return
 		m_kind == Kind::KECCAK256 ||
 		m_kind == Kind::ECRecover ||
@@ -3374,7 +3434,7 @@ MemberList::MemberMap MagicType::nativeMembers(ContractDefinition const*) const
 				TypePointers(),
 				TypePointers{make_shared<ArrayType>(DataLocation::Memory)},
 				strings{},
-				strings{},
+				strings{1, ""},
 				FunctionType::Kind::ABIEncode,
 				true,
 				StateMutability::Pure
@@ -3383,7 +3443,7 @@ MemberList::MemberMap MagicType::nativeMembers(ContractDefinition const*) const
 				TypePointers(),
 				TypePointers{make_shared<ArrayType>(DataLocation::Memory)},
 				strings{},
-				strings{},
+				strings{1, ""},
 				FunctionType::Kind::ABIEncodePacked,
 				true,
 				StateMutability::Pure
@@ -3391,8 +3451,8 @@ MemberList::MemberMap MagicType::nativeMembers(ContractDefinition const*) const
 			{"encodeWithSelector", make_shared<FunctionType>(
 				TypePointers{make_shared<FixedBytesType>(4)},
 				TypePointers{make_shared<ArrayType>(DataLocation::Memory)},
-				strings{},
+				strings{1, ""},
-				strings{},
+				strings{1, ""},
 				FunctionType::Kind::ABIEncodeWithSelector,
 				true,
 				StateMutability::Pure
@@ -3400,8 +3460,8 @@ MemberList::MemberMap MagicType::nativeMembers(ContractDefinition const*) const
 			{"encodeWithSignature", make_shared<FunctionType>(
 				TypePointers{make_shared<ArrayType>(DataLocation::Memory, true)},
 				TypePointers{make_shared<ArrayType>(DataLocation::Memory)},
-				strings{},
+				strings{1, ""},
-				strings{},
+				strings{1, ""},
 				FunctionType::Kind::ABIEncodeWithSignature,
 				true,
 				StateMutability::Pure

libsolidity/ast/Types.h

@@ -852,6 +852,7 @@ public:
 
 	std::pair<u256, unsigned> const& storageOffsetsOfMember(std::string const& _name) const;
 	u256 memoryOffsetOfMember(std::string const& _name) const;
+	unsigned calldataOffsetOfMember(std::string const& _name) const;
 
 	StructDefinition const& structDefinition() const { return m_struct; }
 
@@ -1012,8 +1013,8 @@ public:
 	): FunctionType(
 		parseElementaryTypeVector(_parameterTypes),
 		parseElementaryTypeVector(_returnParameterTypes),
-		strings(),
+		strings(_parameterTypes.size(), ""),
-		strings(),
+		strings(_returnParameterTypes.size(), ""),
 		_kind,
 		_arbitraryParameters,
 		_stateMutability
@@ -1050,6 +1051,14 @@ public:
 		m_bound(_bound),
 		m_declaration(_declaration)
 	{
+		solAssert(
+			m_parameterNames.size() == m_parameterTypes.size(),
+			"Parameter names list must match parameter types list!"
+		);
+		solAssert(
+			m_returnParameterNames.size() == m_returnParameterTypes.size(),
+			"Return parameter names list must match return parameter types list!"
+		);
 		solAssert(
 			!m_bound || !m_parameterTypes.empty(),
 			"Attempted construction of bound function without self type"

libsolidity/codegen/ABIFunctions.cpp

@@ -107,6 +107,74 @@ string ABIFunctions::tupleEncoder(
 	});
 }
 
+string ABIFunctions::tupleEncoderPacked(
+	TypePointers const& _givenTypes,
+	TypePointers const& _targetTypes
+)
+{
+	EncodingOptions options;
+	options.encodeAsLibraryTypes = false;
+	options.encodeFunctionFromStack = true;
+	options.padded = false;
+	options.dynamicInplace = true;
+
+	string functionName = string("abi_encode_tuple_packed_");
+	for (auto const& t: _givenTypes)
+		functionName += t->identifier() + "_";
+	functionName += "_to_";
+	for (auto const& t: _targetTypes)
+		functionName += t->identifier() + "_";
+	functionName += options.toFunctionNameSuffix();
+
+	return createExternallyUsedFunction(functionName, [&]() {
+		solAssert(!_givenTypes.empty(), "");
+
+		// Note that the values are in reverse due to the difference in calling semantics.
+		Whiskers templ(R"(
+			function <functionName>(pos <valueParams>) -> end {
+				<encodeElements>
+				end := pos
+			}
+		)");
+		templ("functionName", functionName);
+		string valueParams;
+		string encodeElements;
+		size_t stackPos = 0;
+		for (size_t i = 0; i < _givenTypes.size(); ++i)
+		{
+			solAssert(_givenTypes[i], "");
+			solAssert(_targetTypes[i], "");
+			size_t sizeOnStack = _givenTypes[i]->sizeOnStack();
+			string valueNames = "";
+			for (size_t j = 0; j < sizeOnStack; j++)
+			{
+				valueNames += "value" + to_string(stackPos) + ", ";
+				valueParams = ", value" + to_string(stackPos) + valueParams;
+				stackPos++;
+			}
+			bool dynamic = _targetTypes[i]->isDynamicallyEncoded();
+			Whiskers elementTempl(
+				dynamic ?
+				string(R"(
+					pos := <abiEncode>(<values> pos)
+				)") :
+				string(R"(
+					<abiEncode>(<values> pos)
+					pos := add(pos, <calldataEncodedSize>)
+				)")
+			);
+			elementTempl("values", valueNames);
+			if (!dynamic)
+				elementTempl("calldataEncodedSize", to_string(_targetTypes[i]->calldataEncodedSize(false)));
+			elementTempl("abiEncode", abiEncodingFunction(*_givenTypes[i], *_targetTypes[i], options));
+			encodeElements += elementTempl.render();
+		}
+		templ("valueParams", valueParams);
+		templ("encodeElements", encodeElements);
+
+		return templ.render();
+	});
+}
 string ABIFunctions::tupleDecoder(TypePointers const& _types, bool _fromMemory)
 {
 	string functionName = string("abi_decode_tuple_");
@@ -216,7 +284,7 @@ string ABIFunctions::cleanupFunction(Type const& _type, bool _revertOnFailure)
 		switch (_type.category())
 		{
 		case Type::Category::Address:
-			templ("body", "cleaned := " + cleanupFunction(IntegerType(160)) + "(value)");
+			templ("body", "cleaned := " + cleanupFunction(IntegerType(160), _revertOnFailure) + "(value)");
 			break;
 		case Type::Category::Integer:
 		{
@@ -265,7 +333,7 @@ string ABIFunctions::cleanupFunction(Type const& _type, bool _revertOnFailure)
 				StateMutability::Payable :
 				StateMutability::NonPayable
 			);
-			templ("body", "cleaned := " + cleanupFunction(addressType) + "(value)");
+			templ("body", "cleaned := " + cleanupFunction(addressType, _revertOnFailure) + "(value)");
 			break;
 		}
 		case Type::Category::Enum:
@@ -341,7 +409,7 @@ string ABIFunctions::conversionFunction(Type const& _from, Type const& _to)
 				solAssert(_from.mobileType(), "");
 				body =
 					Whiskers("converted := <cleanEnum>(<cleanInt>(value))")
-					("cleanEnum", cleanupFunction(_to, false))
+					("cleanEnum", cleanupFunction(_to))
 					// "mobileType()" returns integer type for rational
 					("cleanInt", cleanupFunction(*_from.mobileType()))
 					.render();
@@ -569,25 +637,71 @@ string ABIFunctions::abiEncodingFunction(
 		)");
 		templ("functionName", functionName);
 
-		if (_from.dataStoredIn(DataLocation::Storage) && to.isValueType())
+		if (_from.dataStoredIn(DataLocation::Storage))
 		{
 			// special case: convert storage reference type to value type - this is only
 			// possible for library calls where we just forward the storage reference
 			solAssert(_options.encodeAsLibraryTypes, "");
+			solAssert(_options.padded && !_options.dynamicInplace, "Non-padded / inplace encoding for library call requested.");
 			solAssert(to == IntegerType::uint256(), "");
 			templ("cleanupConvert", "value");
 		}
 		else
 		{
+			string cleanupConvert;
 			if (_from == to)
-				templ("cleanupConvert", cleanupFunction(_from) + "(value)");
+				cleanupConvert = cleanupFunction(_from) + "(value)";
 			else
-				templ("cleanupConvert", conversionFunction(_from, to) + "(value)");
+				cleanupConvert = conversionFunction(_from, to) + "(value)";
+			if (!_options.padded)
+				cleanupConvert = leftAlignFunction(to) + "(" + cleanupConvert + ")";
+			templ("cleanupConvert", cleanupConvert);
 		}
 		return templ.render();
 	});
 }
 
+string ABIFunctions::abiEncodeAndReturnUpdatedPosFunction(
+	Type const& _givenType,
+	Type const& _targetType,
+	ABIFunctions::EncodingOptions const& _options
+)
+{
+	string functionName =
+		"abi_encodeUpdatedPos_" +
+		_givenType.identifier() +
+		"_to_" +
+		_targetType.identifier() +
+		_options.toFunctionNameSuffix();
+	return createFunction(functionName, [&]() {
+		string encoder = abiEncodingFunction(_givenType, _targetType, _options);
+		if (_targetType.isDynamicallyEncoded())
+			return Whiskers(R"(
+				function <functionName>(value, pos) -> updatedPos {
+					updatedPos := <encode>(value, pos)
+				}
+			)")
+			("functionName", functionName)
+			("encode", encoder)
+			.render();
+		else
+		{
+			unsigned encodedSize = _targetType.calldataEncodedSize(_options.padded);
+			solAssert(encodedSize != 0, "Invalid encoded size.");
+			return Whiskers(R"(
+				function <functionName>(value, pos) -> updatedPos {
+					<encode>(value, pos)
+					updatedPos := add(pos, <encodedSize>)
+				}
+			)")
+			("functionName", functionName)
+			("encode", encoder)
+			("encodedSize", toCompactHexWithPrefix(encodedSize))
+			.render();
+		}
+	});
+}
+
 string ABIFunctions::abiEncodingFunctionCalldataArray(
 	Type const& _from,
 	Type const& _to,
@@ -621,17 +735,17 @@ string ABIFunctions::abiEncodingFunctionCalldataArray(
 		Whiskers templ(R"(
 			// <readableTypeNameFrom> -> <readableTypeNameTo>
 			function <functionName>(start, length, pos) -> end {
-				<storeLength> // might update pos
+				pos := <storeLength>(pos, length)
 				<copyFun>(start, pos, length)
-				end := add(pos, <roundUpFun>(length))
+				end := add(pos, <lengthPadded>)
 			}
 		)");
-		templ("storeLength", _to.isDynamicallySized() ? "mstore(pos, length) pos := add(pos, 0x20)" : "");
+		templ("storeLength", arrayStoreLengthForEncodingFunction(toArrayType, _options));
 		templ("functionName", functionName);
 		templ("readableTypeNameFrom", _from.toString(true));
 		templ("readableTypeNameTo", _to.toString(true));
 		templ("copyFun", copyToMemoryFunction(true));
-		templ("roundUpFun", roundUpFunction());
+		templ("lengthPadded", _options.padded ? roundUpFunction() + "(length)" : "length");
 		return templ.render();
 	});
 }
@@ -659,13 +773,14 @@ string ABIFunctions::abiEncodingFunctionSimpleArray(
 		bool dynamic = _to.isDynamicallyEncoded();
 		bool dynamicBase = _to.baseType()->isDynamicallyEncoded();
 		bool inMemory = _from.dataStoredIn(DataLocation::Memory);
+		bool const usesTail = dynamicBase && !_options.dynamicInplace;
 		Whiskers templ(
-			dynamicBase ?
+			usesTail ?
 			R"(
 				// <readableTypeNameFrom> -> <readableTypeNameTo>
 				function <functionName>(value, pos) <return> {
 					let length := <lengthFun>(value)
-					<storeLength> // might update pos
+					pos := <storeLength>(pos, length)
 					let headStart := pos
 					let tail := add(pos, mul(length, 0x20))
 					let srcPtr := <dataAreaFun>(value)
@@ -684,13 +799,12 @@ string ABIFunctions::abiEncodingFunctionSimpleArray(
 				// <readableTypeNameFrom> -> <readableTypeNameTo>
 				function <functionName>(value, pos) <return> {
 					let length := <lengthFun>(value)
-					<storeLength> // might update pos
+					pos := <storeLength>(pos, length)
 					let srcPtr := <dataAreaFun>(value)
 					for { let i := 0 } lt(i, length) { i := add(i, 1) }
 					{
-						<encodeToMemoryFun>(<arrayElementAccess>, pos)
+						pos := <encodeToMemoryFun>(<arrayElementAccess>, pos)
 						srcPtr := <nextArrayElement>(srcPtr)
-						pos := add(pos, <elementEncodedSize>)
 					}
 					<assignEnd>
 				}
@@ -702,20 +816,13 @@ string ABIFunctions::abiEncodingFunctionSimpleArray(
 		templ("return", dynamic ? " -> end " : "");
 		templ("assignEnd", dynamic ? "end := pos" : "");
 		templ("lengthFun", arrayLengthFunction(_from));
-		if (_to.isDynamicallySized())
+		templ("storeLength", arrayStoreLengthForEncodingFunction(_to, _options));
-			templ("storeLength", "mstore(pos, length) pos := add(pos, 0x20)");
-		else
-			templ("storeLength", "");
 		templ("dataAreaFun", arrayDataAreaFunction(_from));
-		templ("elementEncodedSize", toCompactHexWithPrefix(_to.baseType()->calldataEncodedSize()));
 
 		EncodingOptions subOptions(_options);
 		subOptions.encodeFunctionFromStack = false;
-		templ("encodeToMemoryFun", abiEncodingFunction(
+		subOptions.padded = true;
-			*_from.baseType(),
+		templ("encodeToMemoryFun", abiEncodeAndReturnUpdatedPosFunction(*_from.baseType(), *_to.baseType(), subOptions));
-			*_to.baseType(),
-			subOptions
-		));
 		templ("arrayElementAccess", inMemory ? "mload(srcPtr)" : _from.baseType()->isValueType() ? "sload(srcPtr)" : "srcPtr" );
 		templ("nextArrayElement", nextArrayElementFunction(_from));
 		return templ.render();
@@ -745,15 +852,16 @@ string ABIFunctions::abiEncodingFunctionMemoryByteArray(
 		Whiskers templ(R"(
 			function <functionName>(value, pos) -> end {
 				let length := <lengthFun>(value)
-				mstore(pos, length)
+				pos := <storeLength>(pos, length)
-				<copyFun>(add(value, 0x20), add(pos, 0x20), length)
+				<copyFun>(add(value, 0x20), pos, length)
-				end := add(add(pos, 0x20), <roundUpFun>(length))
+				end := add(pos, <lengthPadded>)
 			}
 		)");
 		templ("functionName", functionName);
 		templ("lengthFun", arrayLengthFunction(_from));
+		templ("storeLength", arrayStoreLengthForEncodingFunction(_to, _options));
 		templ("copyFun", copyToMemoryFunction(false));
-		templ("roundUpFun", roundUpFunction());
+		templ("lengthPadded", _options.padded ? roundUpFunction() + "(length)" : "length");
 		return templ.render();
 	});
 }
@@ -787,28 +895,30 @@ string ABIFunctions::abiEncodingFunctionCompactStorageArray(
 					case 0 {
 						// short byte array
 						let length := and(div(slotValue, 2), 0x7f)
-						mstore(pos, length)
+						pos := <storeLength>(pos, length)
-						mstore(add(pos, 0x20), and(slotValue, not(0xff)))
+						mstore(pos, and(slotValue, not(0xff)))
-						ret := add(pos, 0x40)
+						ret := add(pos, <lengthPaddedShort>)
 					}
 					case 1 {
 						// long byte array
 						let length := div(slotValue, 2)
-						mstore(pos, length)
+						pos := <storeLength>(pos, length)
-						pos := add(pos, 0x20)
 						let dataPos := <arrayDataSlot>(value)
 						let i := 0
 						for { } lt(i, length) { i := add(i, 0x20) } {
 							mstore(add(pos, i), sload(dataPos))
 							dataPos := add(dataPos, 1)
 						}
-						ret := add(pos, i)
+						ret := add(pos, <lengthPaddedLong>)
 					}
 				}
 			)");
 			templ("functionName", functionName);
 			templ("readableTypeNameFrom", _from.toString(true));
 			templ("readableTypeNameTo", _to.toString(true));
+			templ("storeLength", arrayStoreLengthForEncodingFunction(_to, _options));
+			templ("lengthPaddedShort", _options.padded ? "0x20" : "length");
+			templ("lengthPaddedLong", _options.padded ? "i" : "length");
 			templ("arrayDataSlot", arrayDataAreaFunction(_from));
 			return templ.render();
 		}
@@ -822,13 +932,13 @@ string ABIFunctions::abiEncodingFunctionCompactStorageArray(
 			size_t storageBytes = _from.baseType()->storageBytes();
 			size_t itemsPerSlot = 32 / storageBytes;
 			// This always writes full slot contents to memory, which might be
-			// more than desired, i.e. it writes beyond the end of memory.
+			// more than desired, i.e. it always writes beyond the end of memory.
 			Whiskers templ(
 				R"(
 					// <readableTypeNameFrom> -> <readableTypeNameTo>
 					function <functionName>(value, pos) <return> {
 						let length := <lengthFun>(value)
-						<storeLength> // might update pos
+						pos := <storeLength>(pos, length)
 						let originalPos := pos
 						let srcPtr := <dataArea>(value)
 						for { let i := 0 } lt(i, length) { i := add(i, <itemsPerSlot>) }
@@ -851,17 +961,16 @@ string ABIFunctions::abiEncodingFunctionCompactStorageArray(
 			templ("return", dynamic ? " -> end " : "");
 			templ("assignEnd", dynamic ? "end := pos" : "");
 			templ("lengthFun", arrayLengthFunction(_from));
-			if (_to.isDynamicallySized())
+			templ("storeLength", arrayStoreLengthForEncodingFunction(_to, _options));
-				templ("storeLength", "mstore(pos, length) pos := add(pos, 0x20)");
-			else
-				templ("storeLength", "");
 			templ("dataArea", arrayDataAreaFunction(_from));
 			templ("itemsPerSlot", to_string(itemsPerSlot));
+			// We use padded size because array elements are always padded.
 			string elementEncodedSize = toCompactHexWithPrefix(_to.baseType()->calldataEncodedSize());
 			templ("elementEncodedSize", elementEncodedSize);
 
 			EncodingOptions subOptions(_options);
 			subOptions.encodeFunctionFromStack = false;
+			subOptions.padded = true;
 			string encodeToMemoryFun = abiEncodingFunction(
 				*_from.baseType(),
 				*_to.baseType(),
@@ -904,6 +1013,8 @@ string ABIFunctions::abiEncodingFunctionStruct(
 				<#members>
 				{
 					// <memberName>
+					<preprocess>
+					let memberValue := <retrieveValue>
 					<encode>
 				}
 				</members>
@@ -914,7 +1025,12 @@ string ABIFunctions::abiEncodingFunctionStruct(
 		templ("readableTypeNameFrom", _from.toString(true));
 		templ("readableTypeNameTo", _to.toString(true));
 		templ("return", dynamic ? " -> end " : "");
-		templ("assignEnd", dynamic ? "end := tail" : "");
+		if (dynamic && _options.dynamicInplace)
+			templ("assignEnd", "end := pos");
+		else if (dynamic && !_options.dynamicInplace)
+			templ("assignEnd", "end := tail");
+		else
+			templ("assignEnd", "");
 		// to avoid multiple loads from the same slot for subsequent members
 		templ("init", fromStorage ? "let slotValue := 0" : "");
 		u256 previousSlotOffset(-1);
@@ -932,20 +1048,10 @@ string ABIFunctions::abiEncodingFunctionStruct(
 			bool dynamicMember = memberTypeTo->isDynamicallyEncoded();
 			if (dynamicMember)
 				solAssert(dynamic, "");
-			Whiskers memberTempl(R"(
+
-				<preprocess>
+			members.push_back({});
-				let memberValue := <retrieveValue>
+			members.back()["preprocess"] = "";
-				)" + (
+
-					dynamicMember ?
-					string(R"(
-						mstore(add(pos, <encodingOffset>), sub(tail, pos))
-						tail := <abiEncode>(memberValue, tail)
-					)") :
-					string(R"(
-						<abiEncode>(memberValue, add(pos, <encodingOffset>))
-					)")
-				)
-			);
 			if (fromStorage)
 			{
 				solAssert(memberTypeFrom->isValueType() == memberTypeTo->isValueType(), "");
@@ -956,39 +1062,56 @@ string ABIFunctions::abiEncodingFunctionStruct(
 				{
 					if (storageSlotOffset != previousSlotOffset)
 					{
-						memberTempl("preprocess", "slotValue := sload(add(value, " + toCompactHexWithPrefix(storageSlotOffset) + "))");
+						members.back()["preprocess"] = "slotValue := sload(add(value, " + toCompactHexWithPrefix(storageSlotOffset) + "))";
 						previousSlotOffset = storageSlotOffset;
 					}
-					else
+					members.back()["retrieveValue"] = shiftRightFunction(intraSlotOffset * 8) + "(slotValue)";
-						memberTempl("preprocess", "");
-					memberTempl("retrieveValue", shiftRightFunction(intraSlotOffset * 8) + "(slotValue)");
 				}
 				else
 				{
 					solAssert(memberTypeFrom->dataStoredIn(DataLocation::Storage), "");
 					solAssert(intraSlotOffset == 0, "");
-					memberTempl("preprocess", "");
+					members.back()["retrieveValue"] = "add(value, " + toCompactHexWithPrefix(storageSlotOffset) + ")";
-					memberTempl("retrieveValue", "add(value, " + toCompactHexWithPrefix(storageSlotOffset) + ")");
 				}
 			}
 			else
 			{
-				memberTempl("preprocess", "");
 				string sourceOffset = toCompactHexWithPrefix(_from.memoryOffsetOfMember(member.name));
-				memberTempl("retrieveValue", "mload(add(value, " + sourceOffset + "))");
+				members.back()["retrieveValue"] = "mload(add(value, " + sourceOffset + "))";
 			}
-			memberTempl("encodingOffset", toCompactHexWithPrefix(encodingOffset));
-			encodingOffset += dynamicMember ? 0x20 : memberTypeTo->calldataEncodedSize();
 
 			EncodingOptions subOptions(_options);
 			subOptions.encodeFunctionFromStack = false;
-			memberTempl("abiEncode", abiEncodingFunction(*memberTypeFrom, *memberTypeTo, subOptions));
+			// Like with arrays, struct members are always padded.
+			subOptions.padded = true;
+
+			string encode;
+			if (_options.dynamicInplace)
+				encode = Whiskers{"pos := <encode>(memberValue, pos)"}
+					("encode", abiEncodeAndReturnUpdatedPosFunction(*memberTypeFrom, *memberTypeTo, subOptions))
+					 .render();
+			else
+			{
+				Whiskers encodeTempl(
+					dynamicMember ?
+					string(R"(
+						mstore(add(pos, <encodingOffset>), sub(tail, pos))
+						tail := <abiEncode>(memberValue, tail)
+					)") :
+					"<abiEncode>(memberValue, add(pos, <encodingOffset>))"
+				);
+				encodeTempl("encodingOffset", toCompactHexWithPrefix(encodingOffset));
+				encodingOffset += dynamicMember ? 0x20 : memberTypeTo->calldataEncodedSize();
+				encodeTempl("abiEncode", abiEncodingFunction(*memberTypeFrom, *memberTypeTo, subOptions));
+				encode = encodeTempl.render();
+			}
+			members.back()["encode"] = encode;
 
-			members.push_back({});
-			members.back()["encode"] = memberTempl.render();
 			members.back()["memberName"] = member.name;
 		}
 		templ("members", members);
+		if (_options.dynamicInplace)
+			solAssert(encodingOffset == 0, "In-place encoding should enforce zero head size.");
 		templ("headSize", toCompactHexWithPrefix(encodingOffset));
 		return templ.render();
 	});
@@ -1015,9 +1138,10 @@ string ABIFunctions::abiEncodingFunctionStringLiteral(
 
 		if (_to.isDynamicallySized())
 		{
+			solAssert(_to.category() == Type::Category::Array, "");
 			Whiskers templ(R"(
 				function <functionName>(pos) -> end {
-					mstore(pos, <length>)
+					pos := <storeLength>(pos, <length>)
 					<#word>
 						mstore(add(pos, <offset>), <wordValue>)
 					</word>
@@ -1028,12 +1152,17 @@ string ABIFunctions::abiEncodingFunctionStringLiteral(
 
 			// TODO this can make use of CODECOPY for large strings once we have that in Yul
 			size_t words = (value.size() + 31) / 32;
-			templ("overallSize", to_string(32 + words * 32));
 			templ("length", to_string(value.size()));
+			templ("storeLength", arrayStoreLengthForEncodingFunction(dynamic_cast<ArrayType const&>(_to), _options));
+			if (_options.padded)
+				templ("overallSize", to_string(words * 32));
+			else
+				templ("overallSize", to_string(value.size()));
+
 			vector<map<string, string>> wordParams(words);
 			for (size_t i = 0; i < words; ++i)
 			{
-				wordParams[i]["offset"] = to_string(32 + i * 32);
+				wordParams[i]["offset"] = to_string(i * 32);
 				wordParams[i]["wordValue"] = "0x" + h256(value.substr(32 * i, 32), h256::AlignLeft).hex();
 			}
 			templ("word", wordParams);
@@ -1117,7 +1246,16 @@ string ABIFunctions::abiDecodingFunction(Type const& _type, bool _fromMemory, bo
 			return abiDecodingFunctionArray(*arrayType, _fromMemory);
 	}
 	else if (auto const* structType = dynamic_cast<StructType const*>(decodingType.get()))
-		return abiDecodingFunctionStruct(*structType, _fromMemory);
+	{
+		if (structType->dataStoredIn(DataLocation::CallData))
+		{
+			solAssert(!_fromMemory, "");
+			solUnimplementedAssert(!structType->isDynamicallyEncoded(), "Dynamically encoded calldata structs are not yet implemented.");
+			return abiDecodingFunctionCalldataStruct(*structType);
+		}
+		else
+			return abiDecodingFunctionStruct(*structType, _fromMemory);
+	}
 	else if (auto const* functionType = dynamic_cast<FunctionType const*>(decodingType.get()))
 		return abiDecodingFunctionFunctionType(*functionType, _fromMemory, _forUseOnStack);
 	else
@@ -1294,15 +1432,37 @@ string ABIFunctions::abiDecodingFunctionByteArray(ArrayType const& _type, bool _
 	});
 }
 
+string ABIFunctions::abiDecodingFunctionCalldataStruct(StructType const& _type)
+{
+	solAssert(_type.dataStoredIn(DataLocation::CallData), "");
+	solAssert(_type.calldataEncodedSize(true) != 0, "");
+	string functionName =
+		"abi_decode_" +
+		_type.identifier();
+
+	return createFunction(functionName, [&]() {
+		Whiskers w{R"(
+				// <readableTypeName>
+				function <functionName>(offset, end) -> value {
+					if slt(sub(end, offset), <minimumSize>) { revert(0, 0) }
+					value := offset
+				}
+		)"};
+		w("functionName", functionName);
+		w("readableTypeName", _type.toString(true));
+		w("minimumSize", to_string(_type.calldataEncodedSize(true)));
+		return w.render();
+	});
+}
+
 string ABIFunctions::abiDecodingFunctionStruct(StructType const& _type, bool _fromMemory)
 {
+	solAssert(!_type.dataStoredIn(DataLocation::CallData), "");
 	string functionName =
 		"abi_decode_" +
 		_type.identifier() +
 		(_fromMemory ? "_fromMemory" : "");
 
-	solUnimplementedAssert(!_type.dataStoredIn(DataLocation::CallData), "");
-
 	return createFunction(functionName, [&]() {
 		Whiskers templ(R"(
 			// <readableTypeName>
@@ -1435,6 +1595,66 @@ string ABIFunctions::copyToMemoryFunction(bool _fromCalldata)
 	});
 }
 
+string ABIFunctions::leftAlignFunction(Type const& _type)
+{
+	string functionName = string("leftAlign_") + _type.identifier();
+	return createFunction(functionName, [&]() {
+		Whiskers templ(R"(
+			function <functionName>(value) -> aligned {
+				<body>
+			}
+		)");
+		templ("functionName", functionName);
+		switch (_type.category())
+		{
+		case Type::Category::Address:
+			templ("body", "aligned := " + leftAlignFunction(IntegerType(160)) + "(value)");
+			break;
+		case Type::Category::Integer:
+		{
+			IntegerType const& type = dynamic_cast<IntegerType const&>(_type);
+			if (type.numBits() == 256)
+				templ("body", "aligned := value");
+			else
+				templ("body", "aligned := " + shiftLeftFunction(256 - type.numBits()) + "(value)");
+			break;
+		}
+		case Type::Category::RationalNumber:
+			solAssert(false, "Left align requested for rational number.");
+			break;
+		case Type::Category::Bool:
+			templ("body", "aligned := " + leftAlignFunction(IntegerType(8)) + "(value)");
+			break;
+		case Type::Category::FixedPoint:
+			solUnimplemented("Fixed point types not implemented.");
+			break;
+		case Type::Category::Array:
+		case Type::Category::Struct:
+			solAssert(false, "Left align requested for non-value type.");
+			break;
+		case Type::Category::FixedBytes:
+			templ("body", "aligned := value");
+			break;
+		case Type::Category::Contract:
+			templ("body", "aligned := " + leftAlignFunction(AddressType::address()) + "(value)");
+			break;
+		case Type::Category::Enum:
+		{
+			unsigned storageBytes = dynamic_cast<EnumType const&>(_type).storageBytes();
+			templ("body", "aligned := " + leftAlignFunction(IntegerType(8 * storageBytes)) + "(value)");
+			break;
+		}
+		case Type::Category::InaccessibleDynamic:
+			solAssert(false, "Left align requested for inaccessible dynamic type.");
+			break;
+		default:
+			solAssert(false, "Left align of type " + _type.identifier() + " requested.");
+		}
+
+		return templ.render();
+	});
+}
+
 string ABIFunctions::shiftLeftFunction(size_t _numBits)
 {
 	solAssert(_numBits < 256, "");
@@ -1688,6 +1908,30 @@ string ABIFunctions::nextArrayElementFunction(ArrayType const& _type)
 	});
 }
 
+string ABIFunctions::arrayStoreLengthForEncodingFunction(ArrayType const& _type, EncodingOptions const& _options)
+{
+	string functionName = "array_storeLengthForEncoding_" + _type.identifier() + _options.toFunctionNameSuffix();
+	return createFunction(functionName, [&]() {
+		if (_type.isDynamicallySized() && !_options.dynamicInplace)
+			return Whiskers(R"(
+				function <functionName>(pos, length) -> updated_pos {
+					mstore(pos, length)
+					updated_pos := add(pos, 0x20)
+				}
+			)")
+			("functionName", functionName)
+			.render();
+		else
+			return Whiskers(R"(
+				function <functionName>(pos, length) -> updated_pos {
+					updated_pos := pos
+				}
+			)")
+			("functionName", functionName)
+			.render();
+	});
+}
+
 string ABIFunctions::allocationFunction()
 {
 	string functionName = "allocateMemory";

libsolidity/codegen/ABIFunctions.h

@@ -60,16 +60,26 @@ public:
 	/// The values represent stack slots. If a type occupies more or less than one
 	/// stack slot, it takes exactly that number of values.
 	/// Returns a pointer to the end of the area written in memory.
-	/// Does not allocate memory (does not change the memory head pointer), but writes
+	/// Does not allocate memory (does not change the free memory pointer), but writes
 	/// to memory starting at $headStart and an unrestricted amount after that.
-	/// Assigns the end of encoded memory either to $value0 or (if that is not present)
-	/// to $headStart.
 	std::string tupleEncoder(
 		TypePointers const& _givenTypes,
 		TypePointers const& _targetTypes,
 		bool _encodeAsLibraryTypes = false
 	);
 
+	/// @returns name of an assembly function to encode values of @a _givenTypes
+	/// with packed encoding into memory, converting the types to @a _targetTypes on the fly.
+	/// Parameters are: <memPos> <value_n> ... <value_1>, i.e.
+	/// the layout on the stack is <value_1> ... <value_n> <memPos> with
+	/// the top of the stack on the right.
+	/// The values represent stack slots. If a type occupies more or less than one
+	/// stack slot, it takes exactly that number of values.
+	/// Returns a pointer to the end of the area written in memory.
+	/// Does not allocate memory (does not change the free memory pointer), but writes
+	/// to memory starting at memPos and an unrestricted amount after that.
+	std::string tupleEncoderPacked(TypePointers const& _givenTypes, TypePointers const& _targetTypes);
+
 	/// @returns name of an assembly function to ABI-decode values of @a _types
 	/// into memory. If @a _fromMemory is true, decodes from memory instead of
 	/// from calldata.
@@ -89,7 +99,9 @@ public:
 private:
 	struct EncodingOptions
 	{
-		/// Pad/signextend value types and bytes/string to multiples of  32 bytes.
+		/// Pad/signextend value types and bytes/string to multiples of 32 bytes.
+		/// If false, data is always left-aligned.
+		/// Note that this is always re-set to true for the elements of arrays and structs.
 		bool padded = true;
 		/// Store arrays and structs in place without "data pointer" and do not store the length.
 		bool dynamicInplace = false;
@@ -134,6 +146,14 @@ private:
 		Type const& _targetType,
 		EncodingOptions const& _options
 	);
+	/// @returns the name of a function that internally calls `abiEncodingFunction`
+	/// but always returns the updated encoding position, even if the type is
+	/// statically encoded.
+	std::string abiEncodeAndReturnUpdatedPosFunction(
+		Type const& _givenType,
+		Type const& _targetType,
+		EncodingOptions const& _options
+	);
 	/// Part of @a abiEncodingFunction for array target type and given calldata array.
 	std::string abiEncodingFunctionCalldataArray(
 		Type const& _givenType,
@@ -202,6 +222,8 @@ private:
 	std::string abiDecodingFunctionCalldataArray(ArrayType const& _type);
 	/// Part of @a abiDecodingFunction for byte array types.
 	std::string abiDecodingFunctionByteArray(ArrayType const& _type, bool _fromMemory);
+	/// Part of @a abiDecodingFunction for calldata struct types.
+	std::string abiDecodingFunctionCalldataStruct(StructType const& _type);
 	/// Part of @a abiDecodingFunction for struct types.
 	std::string abiDecodingFunctionStruct(StructType const& _type, bool _fromMemory);
 	/// Part of @a abiDecodingFunction for array types.
@@ -212,6 +234,10 @@ private:
 	/// Pads with zeros and might write more than exactly length.
 	std::string copyToMemoryFunction(bool _fromCalldata);
 
+	/// @returns the name of a function that takes a (cleaned) value of the given value type and
+	/// left-aligns it, usually for use in non-padded encoding.
+	std::string leftAlignFunction(Type const& _type);
+
 	std::string shiftLeftFunction(size_t _numBits);
 	std::string shiftRightFunction(size_t _numBits);
 	/// @returns the name of a function that rounds its input to the next multiple
@@ -231,6 +257,13 @@ private:
 	/// Only works for memory arrays and storage arrays that store one item per slot.
 	std::string nextArrayElementFunction(ArrayType const& _type);
 
+	/// @returns the name of a function used during encoding that stores the length
+	/// if the array is dynamically sized (and the options do not request in-place encoding).
+	/// It returns the new encoding position.
+	/// If the array is not dynamically sized (or in-place encoding was requested),
+	/// does nothing and just returns the position again.
+	std::string arrayStoreLengthForEncodingFunction(ArrayType const& _type, EncodingOptions const& _options);
+
 	/// @returns the name of a function that allocates memory.
 	/// Modifies the "free memory pointer"
 	/// Arguments: size

libsolidity/codegen/AsmCodeGen.cpp

@@ -184,14 +184,28 @@ void CodeGenerator::assemble(
 )
 {
 	EthAssemblyAdapter assemblyAdapter(_assembly);
-	CodeTransform(
+	shared_ptr<EVMDialect> dialect = EVMDialect::strictAssemblyForEVM();
+	CodeTransform transform(
 		assemblyAdapter,
 		_analysisInfo,
 		_parsedData,
-		*EVMDialect::strictAssemblyForEVM(),
+		*dialect,
 		_optimize,
 		false,
 		_identifierAccess,
 		_useNamedLabelsForFunctions
-	)(_parsedData);
+	);
+	try
+	{
+		transform(_parsedData);
+	}
+	catch (StackTooDeepError const& _e)
+	{
+		BOOST_THROW_EXCEPTION(
+			InternalCompilerError() << errinfo_comment(
+				"Stack too deep when compiling inline assembly" +
+				(_e.comment() ? ": " + *_e.comment() : ".")
+			));
+	}
+	solAssert(transform.stackErrors().empty(), "Stack errors present but not thrown.");
 }

libsolidity/codegen/CompilerContext.cpp

@@ -336,9 +336,11 @@ void CompilerContext::appendInlineAssembly(
 	identifierAccess.resolve = [&](
 		yul::Identifier const& _identifier,
 		yul::IdentifierContext,
-		bool
+		bool _insideFunction
-	)
+	) -> size_t
 	{
+		if (_insideFunction)
+			return size_t(-1);
 		auto it = std::find(_localVariables.begin(), _localVariables.end(), _identifier.name.str());
 		return it == _localVariables.end() ? size_t(-1) : 1;
 	};

libsolidity/codegen/CompilerUtils.cpp

@@ -348,11 +348,15 @@ void CompilerUtils::encodeToMemory(
 
 	if (_givenTypes.empty())
 		return;
-	else if (_padToWordBoundaries && !_copyDynamicDataInPlace && encoderV2)
+	if (encoderV2)
 	{
 		// Use the new Yul-based encoding function
+		solAssert(
+			_padToWordBoundaries != _copyDynamicDataInPlace,
+			"Non-padded and in-place encoding can only be combined."
+		);
 		auto stackHeightBefore = m_context.stackHeight();
-		abiEncodeV2(_givenTypes, targetTypes, _encodeAsLibraryTypes);
+		abiEncodeV2(_givenTypes, targetTypes, _encodeAsLibraryTypes, _padToWordBoundaries);
 		solAssert(stackHeightBefore - m_context.stackHeight() == sizeOnStack(_givenTypes), "");
 		return;
 	}
@@ -466,15 +470,22 @@ void CompilerUtils::encodeToMemory(
 void CompilerUtils::abiEncodeV2(
 	TypePointers const& _givenTypes,
 	TypePointers const& _targetTypes,
-	bool _encodeAsLibraryTypes
+	bool _encodeAsLibraryTypes,
+	bool _padToWordBoundaries
 )
 {
+	if (!_padToWordBoundaries)
+		solAssert(!_encodeAsLibraryTypes, "Library calls cannot be packed.");
+
 	// stack: <$value0> <$value1> ... <$value(n-1)> <$headStart>
 
 	auto ret = m_context.pushNewTag();
 	moveIntoStack(sizeOnStack(_givenTypes) + 1);
 
-	string encoderName = m_context.abiFunctions().tupleEncoder(_givenTypes, _targetTypes, _encodeAsLibraryTypes);
+	string encoderName =
+		_padToWordBoundaries ?
+		m_context.abiFunctions().tupleEncoder(_givenTypes, _targetTypes, _encodeAsLibraryTypes) :
+		m_context.abiFunctions().tupleEncoderPacked(_givenTypes, _targetTypes);
 	m_context.appendJumpTo(m_context.namedTag(encoderName));
 	m_context.adjustStackOffset(-int(sizeOnStack(_givenTypes)) - 1);
 	m_context << ret.tag();
@@ -907,8 +918,7 @@ void CompilerUtils::convertType(
 		auto& targetType = dynamic_cast<StructType const&>(_targetType);
 		auto& typeOnStack = dynamic_cast<StructType const&>(_typeOnStack);
 		solAssert(
-			targetType.location() != DataLocation::CallData &&
+			targetType.location() != DataLocation::CallData
-			typeOnStack.location() != DataLocation::CallData
 		, "");
 		switch (targetType.location())
 		{
@@ -922,9 +932,9 @@ void CompilerUtils::convertType(
 			break;
 		case DataLocation::Memory:
 			// Copy the array to a free position in memory, unless it is already in memory.
-			if (typeOnStack.location() != DataLocation::Memory)
+			switch (typeOnStack.location())
 			{
-				solAssert(typeOnStack.location() == DataLocation::Storage, "");
+			case DataLocation::Storage:
 				// stack: <source ref>
 				m_context << typeOnStack.memorySize();
 				allocateMemory();
@@ -944,6 +954,19 @@ void CompilerUtils::convertType(
 					storeInMemoryDynamic(*targetMemberType, true);
 				}
 				m_context << Instruction::POP << Instruction::POP;
+				break;
+			case DataLocation::CallData:
+			{
+				solUnimplementedAssert(!typeOnStack.isDynamicallyEncoded(), "");
+				m_context << Instruction::DUP1;
+				m_context << Instruction::CALLDATASIZE;
+				m_context << Instruction::SUB;
+				abiDecode({targetType.shared_from_this()}, false);
+				break;
+			}
+			case DataLocation::Memory:
+				// nothing to do
+				break;
 			}
 			break;
 		case DataLocation::CallData:

libsolidity/codegen/CompilerUtils.h

@@ -159,7 +159,8 @@ public:
 	void abiEncodeV2(
 		TypePointers const& _givenTypes,
 		TypePointers const& _targetTypes,
-		bool _encodeAsLibraryTypes = false
+		bool _encodeAsLibraryTypes = false,
+		bool _padToWordBoundaries = true
 	);
 
 	/// Decodes data from ABI encoding into internal encoding. If @a _fromMemory is set to true,

libsolidity/codegen/ExpressionCompiler.cpp

@@ -731,6 +731,7 @@ bool ExpressionCompiler::visit(FunctionCall const& _functionCall)
 			}
 			arguments.front()->accept(*this);
 			utils().fetchFreeMemoryPointer();
+			solAssert(function.parameterTypes().front()->isValueType(), "");
 			utils().packedEncode(
 				{arguments.front()->annotation().type},
 				{function.parameterTypes().front()}
@@ -744,28 +745,32 @@ bool ExpressionCompiler::visit(FunctionCall const& _functionCall)
 			_functionCall.expression().accept(*this);
 			auto const& event = dynamic_cast<EventDefinition const&>(function.declaration());
 			unsigned numIndexed = 0;
+			TypePointers paramTypes = function.parameterTypes();
 			// All indexed arguments go to the stack
 			for (unsigned arg = arguments.size(); arg > 0; --arg)
 				if (event.parameters()[arg - 1]->isIndexed())
 				{
 					++numIndexed;
 					arguments[arg - 1]->accept(*this);
-					if (auto const& arrayType = dynamic_pointer_cast<ArrayType const>(function.parameterTypes()[arg - 1]))
+					if (auto const& referenceType = dynamic_pointer_cast<ReferenceType const>(paramTypes[arg - 1]))
 					{
 						utils().fetchFreeMemoryPointer();
 						utils().packedEncode(
 							{arguments[arg - 1]->annotation().type},
-							{arrayType}
+							{referenceType}
 						);
 						utils().toSizeAfterFreeMemoryPointer();
 						m_context << Instruction::KECCAK256;
 					}
 					else
+					{
+						solAssert(paramTypes[arg - 1]->isValueType(), "");
 						utils().convertType(
 							*arguments[arg - 1]->annotation().type,
-							*function.parameterTypes()[arg - 1],
+							*paramTypes[arg - 1],
 							true
 						);
+					}
 				}
 			if (!event.isAnonymous())
 			{
@@ -782,7 +787,7 @@ bool ExpressionCompiler::visit(FunctionCall const& _functionCall)
 				{
 					arguments[arg]->accept(*this);
 					nonIndexedArgTypes.push_back(arguments[arg]->annotation().type);
-					nonIndexedParamTypes.push_back(function.parameterTypes()[arg]);
+					nonIndexedParamTypes.push_back(paramTypes[arg]);
 				}
 			utils().fetchFreeMemoryPointer();
 			utils().abiEncode(nonIndexedArgTypes, nonIndexedParamTypes);
@@ -1348,7 +1353,7 @@ bool ExpressionCompiler::visit(MemberAccess const& _memberAccess)
 			m_context.appendInlineAssembly(
 				Whiskers(R"({
 					mstore(start, sub(end, add(start, 0x20)))
-					mstore(<free>, end)
+					mstore(<free>, and(add(end, 31), not(31)))
 				})")("free", to_string(CompilerUtils::freeMemoryPointer)).render(),
 				{"start", "end"}
 			);
@@ -1375,6 +1380,24 @@ bool ExpressionCompiler::visit(MemberAccess const& _memberAccess)
 			setLValue<MemoryItem>(_memberAccess, *_memberAccess.annotation().type);
 			break;
 		}
+		case DataLocation::CallData:
+		{
+			solUnimplementedAssert(!type.isDynamicallyEncoded(), "");
+			m_context << type.calldataOffsetOfMember(member) << Instruction::ADD;
+			// For non-value types the calldata offset is returned directly.
+			if (_memberAccess.annotation().type->isValueType())
+			{
+				solAssert(_memberAccess.annotation().type->calldataEncodedSize(false) > 0, "");
+				CompilerUtils(m_context).loadFromMemoryDynamic(*_memberAccess.annotation().type, true, true, false);
+			}
+			else
+				solAssert(
+					_memberAccess.annotation().type->category() == Type::Category::Array ||
+					_memberAccess.annotation().type->category() == Type::Category::Struct,
+					""
+				);
+			break;
+		}
 		default:
 			solAssert(false, "Illegal data location for struct.");
 		}

libsolidity/formal/CVC4Interface.cpp

@@ -163,6 +163,8 @@ CVC4::Expr CVC4Interface::toCVC4Expr(Expression const& _expr)
 		return m_context.mkExpr(CVC4::kind::MULT, arguments[0], arguments[1]);
 	else if (n == "/")
 		return m_context.mkExpr(CVC4::kind::INTS_DIVISION_TOTAL, arguments[0], arguments[1]);
+	else if (n == "mod")
+		return m_context.mkExpr(CVC4::kind::INTS_MODULUS, arguments[0], arguments[1]);
 	else if (n == "select")
 		return m_context.mkExpr(CVC4::kind::SELECT, arguments[0], arguments[1]);
 	else if (n == "store")

libsolidity/formal/SMTChecker.cpp

@@ -34,7 +34,8 @@ using namespace dev::solidity;
 
 SMTChecker::SMTChecker(ErrorReporter& _errorReporter, map<h256, string> const& _smtlib2Responses):
 	m_interface(make_shared<smt::SMTPortfolio>(_smtlib2Responses)),
-	m_errorReporter(_errorReporter)
+	m_errorReporterReference(_errorReporter),
+	m_errorReporter(m_smtErrors)
 {
 #if defined (HAVE_Z3) || defined (HAVE_CVC4)
 	if (!_smtlib2Responses.empty())
@@ -53,6 +54,25 @@ void SMTChecker::analyze(SourceUnit const& _source, shared_ptr<Scanner> const& _
 	m_scanner = _scanner;
 	if (_source.annotation().experimentalFeatures.count(ExperimentalFeature::SMTChecker))
 		_source.accept(*this);
+
+	solAssert(m_interface->solvers() > 0, "");
+	// If this check is true, Z3 and CVC4 are not available
+	// and the query answers were not provided, since SMTPortfolio
+	// guarantees that SmtLib2Interface is the first solver.
+	if (!m_interface->unhandledQueries().empty() && m_interface->solvers() == 1)
+	{
+		if (!m_noSolverWarning)
+		{
+			m_noSolverWarning = true;
+			m_errorReporterReference.warning(
+				SourceLocation(),
+				"SMTChecker analysis was not possible since no integrated SMT solver (Z3 or CVC4) was found."
+			);
+		}
+	}
+	else
+		m_errorReporterReference.append(m_errorReporter.errors());
+	m_errorReporter.clear();
 }
 
 bool SMTChecker::visit(ContractDefinition const& _contract)

libsolidity/formal/SMTChecker.h

@@ -23,6 +23,7 @@
 
 #include <libsolidity/ast/ASTVisitor.h>
 #include <libsolidity/interface/ReadFile.h>
+#include <liblangutil/ErrorReporter.h>
 #include <liblangutil/Scanner.h>
 
 #include <string>
@@ -215,6 +216,8 @@ private:
 	std::shared_ptr<VariableUsage> m_variableUsage;
 	bool m_loopExecutionHappened = false;
 	bool m_arrayAssignmentHappened = false;
+	// True if the "No SMT solver available" warning was already created.
+	bool m_noSolverWarning = false;
 	/// An Expression may have multiple smt::Expression due to
 	/// repeated calls to the same function.
 	std::unordered_map<Expression const*, std::shared_ptr<SymbolicVariable>> m_expressions;
@@ -225,7 +228,13 @@ private:
 	/// Used to retrieve models.
 	std::set<Expression const*> m_uninterpretedTerms;
 	std::vector<smt::Expression> m_pathConditions;
-	langutil::ErrorReporter& m_errorReporter;
+	/// ErrorReporter that comes from CompilerStack.
+	langutil::ErrorReporter& m_errorReporterReference;
+	/// Local SMTChecker ErrorReporter.
+	/// This is necessary to show the "No SMT solver available"
+	/// warning before the others in case it's needed.
+	langutil::ErrorReporter m_errorReporter;
+	langutil::ErrorList m_smtErrors;
 	std::shared_ptr<langutil::Scanner> m_scanner;
 
 	/// Stores the current path of function calls.

libsolidity/formal/SMTPortfolio.cpp

@@ -129,6 +129,15 @@ pair<CheckResult, vector<string>> SMTPortfolio::check(vector<Expression> const&
 	return make_pair(lastResult, finalValues);
 }
 
+vector<string> SMTPortfolio::unhandledQueries()
+{
+	// This code assumes that the constructor guarantees that
+	// SmtLib2Interface is in position 0.
+	solAssert(!m_solvers.empty(), "");
+	solAssert(dynamic_cast<smt::SMTLib2Interface*>(m_solvers.at(0).get()), "");
+	return m_solvers.at(0)->unhandledQueries();
+}
+
 bool SMTPortfolio::solverAnswered(CheckResult result)
 {
 	return result == CheckResult::SATISFIABLE || result == CheckResult::UNSATISFIABLE;

libsolidity/formal/SMTPortfolio.h

@@ -54,7 +54,8 @@ public:
 	void addAssertion(Expression const& _expr) override;
 	std::pair<CheckResult, std::vector<std::string>> check(std::vector<Expression> const& _expressionsToEvaluate) override;
 
-	std::vector<std::string> unhandledQueries() override { return m_solvers.at(0)->unhandledQueries(); }
+	std::vector<std::string> unhandledQueries() override;
+	unsigned solvers() override { return m_solvers.size(); }
 private:
 	static bool solverAnswered(CheckResult result);
 

libsolidity/formal/SolverInterface.h

@@ -141,6 +141,7 @@ public:
 			{"-", 2},
 			{"*", 2},
 			{"/", 2},
+			{"mod", 2},
 			{"select", 2},
 			{"store", 3}
 		};
@@ -246,6 +247,10 @@ public:
 	{
 		return Expression("/", std::move(_a), std::move(_b), Kind::Int);
 	}
+	friend Expression operator%(Expression _a, Expression _b)
+	{
+		return Expression("mod", std::move(_a), std::move(_b), Kind::Int);
+	}
 	Expression operator()(std::vector<Expression> _arguments) const
 	{
 		solAssert(
@@ -305,6 +310,9 @@ public:
 	/// @returns a list of queries that the system was not able to respond to.
 	virtual std::vector<std::string> unhandledQueries() { return {}; }
 
+	/// @returns how many SMT solvers this interface has.
+	virtual unsigned solvers() { return 1; }
+
 protected:
 	// SMT query timeout in milliseconds.
 	static int const queryTimeout = 10000;

libsolidity/formal/Z3Interface.cpp

@@ -162,6 +162,8 @@ z3::expr Z3Interface::toZ3Expr(Expression const& _expr)
 		return arguments[0] * arguments[1];
 	else if (n == "/")
 		return arguments[0] / arguments[1];
+	else if (n == "mod")
+		return z3::mod(arguments[0], arguments[1]);
 	else if (n == "select")
 		return z3::select(arguments[0], arguments[1]);
 	else if (n == "store")

libsolidity/interface/AssemblyStack.cpp

@@ -84,7 +84,8 @@ bool AssemblyStack::parseAndAnalyze(std::string const& _sourceName, std::string
 
 void AssemblyStack::optimize()
 {
-	solAssert(m_language != Language::Assembly, "Optimization requested for loose assembly.");
+	if (m_language != Language::StrictAssembly)
+		solUnimplemented("Optimizer for both loose assembly and Yul is not yet implemented");
 	solAssert(m_analysisSuccessful, "Analysis was not successful.");
 	m_analysisSuccessful = false;
 	optimize(*m_parserResult);
@@ -134,7 +135,7 @@ void AssemblyStack::optimize(yul::Object& _object)
 	for (auto& subNode: _object.subObjects)
 		if (auto subObject = dynamic_cast<yul::Object*>(subNode.get()))
 			optimize(*subObject);
-	yul::OptimiserSuite::run(*languageToDialect(m_language), *_object.code, *_object.analysisInfo);
+	yul::OptimiserSuite::run(languageToDialect(m_language), *_object.code, *_object.analysisInfo);
 }
 
 MachineAssemblyObject AssemblyStack::assemble(Machine _machine, bool _optimize) const

libsolidity/interface/CompilerStack.cpp

@@ -473,7 +473,7 @@ eth::LinkerObject const& CompilerStack::runtimeObject(string const& _contractNam
 	return contract(_contractName).runtimeObject;
 }
 
-/// FIXME: cache this string
+/// TODO: cache this string
 string CompilerStack::assemblyString(string const& _contractName, StringMap _sourceCodes) const
 {
 	if (m_stackState != CompilationSuccessful)
@@ -486,7 +486,7 @@ string CompilerStack::assemblyString(string const& _contractName, StringMap _sou
 		return string();
 }
 
-/// FIXME: cache the JSON
+/// TODO: cache the JSON
 Json::Value CompilerStack::assemblyJSON(string const& _contractName, StringMap _sourceCodes) const
 {
 	if (m_stackState != CompilationSuccessful)
@@ -681,6 +681,8 @@ StringMap CompilerStack::loadMissingSources(SourceUnit const& _ast, std::string
 	for (auto const& node: _ast.nodes())
 		if (ImportDirective const* import = dynamic_cast<ImportDirective*>(node.get()))
 		{
+			solAssert(!import->path().empty(), "Import path cannot be empty.");
+
 			string importPath = dev::absolutePath(import->path(), _sourcePath);
 			// The current value of `path` is the absolute path as seen from this source file.
 			// We first have to apply remappings before we can store the actual absolute path

libsolidity/interface/StandardCompiler.cpp

@@ -298,6 +298,8 @@ boost::optional<Json::Value> checkOptimizerKeys(Json::Value const& _input)
 
 boost::optional<Json::Value> checkMetadataKeys(Json::Value const& _input)
 {
+	if (_input.isObject() && _input.isMember("useLiteralContent") && !_input["useLiteralContent"].isBool())
+		return formatFatalError("JSONError", "\"settings.metadata.useLiteralContent\" must be Boolean");
 	static set<string> keys{"useLiteralContent"};
 	return checkKeys(_input, keys, "settings.metadata");
 }

libsolidity/parsing/DocStringParser.cpp

@@ -20,7 +20,7 @@
 #include <liblangutil/ErrorReporter.h>
 #include <liblangutil/Exceptions.h>
 
-#include <boost/range/algorithm.hpp>
+#include <boost/range/algorithm/find_first_of.hpp>
 #include <boost/range/irange.hpp>
 
 using namespace std;

libsolidity/parsing/Parser.cpp

@@ -221,6 +221,8 @@ ASTPointer<ImportDirective> Parser::parseImportDirective()
 			fatalParserError("Expected import path.");
 		path = getLiteralAndAdvance();
 	}
+	if (path->empty())
+		fatalParserError("Import path cannot be empty.");
 	nodeFactory.markEndPosition();
 	expectToken(Token::Semicolon);
 	return nodeFactory.createNode<ImportDirective>(path, unitAlias, move(symbolAliases));

libyul/AsmAnalysis.cpp

@@ -55,6 +55,26 @@ bool AsmAnalyzer::analyze(Block const& _block)
 	return (*this)(_block);
 }
 
+AsmAnalysisInfo AsmAnalyzer::analyzeStrictAssertCorrect(
+	shared_ptr<Dialect> _dialect,
+	EVMVersion _evmVersion,
+	Block const& _ast
+)
+{
+	ErrorList errorList;
+	langutil::ErrorReporter errors(errorList);
+	yul::AsmAnalysisInfo analysisInfo;
+	bool success = yul::AsmAnalyzer(
+		analysisInfo,
+		errors,
+		_evmVersion,
+		Error::Type::SyntaxError,
+		_dialect
+	).analyze(_ast);
+	solAssert(success && errorList.empty(), "Invalid assembly/yul code.");
+	return analysisInfo;
+}
+
 bool AsmAnalyzer::operator()(Label const& _label)
 {
 	solAssert(!_label.name.empty(), "");

libyul/AsmAnalysis.h

@@ -72,6 +72,12 @@ public:
 
 	bool analyze(Block const& _block);
 
+	static AsmAnalysisInfo analyzeStrictAssertCorrect(
+		std::shared_ptr<Dialect> _dialect,
+		dev::solidity::EVMVersion _evmVersion,
+		Block const& _ast
+	);
+
 	bool operator()(Instruction const&);
 	bool operator()(Literal const& _literal);
 	bool operator()(Identifier const&);

libyul/CMakeLists.txt

@@ -13,6 +13,8 @@ add_library(yul
 	AsmScope.h
 	AsmScopeFiller.cpp
 	AsmScopeFiller.h
+	CompilabilityChecker.cpp
+	CompilabilityChecker.h
 	Dialect.cpp
 	Dialect.h
 	Exceptions.h
@@ -32,6 +34,8 @@ add_library(yul
 	backends/evm/EVMDialect.h
 	backends/evm/EVMObjectCompiler.cpp
 	backends/evm/EVMObjectCompiler.h
+	backends/evm/NoOutputAssembly.h
+	backends/evm/NoOutputAssembly.cpp
 	optimiser/ASTCopier.cpp
 	optimiser/ASTCopier.h
 	optimiser/ASTWalker.cpp
@@ -90,6 +94,8 @@ add_library(yul
 	optimiser/Semantics.h
 	optimiser/SimplificationRules.cpp
 	optimiser/SimplificationRules.h
+	optimiser/StackCompressor.cpp
+	optimiser/StackCompressor.h
 	optimiser/StructuralSimplifier.cpp
 	optimiser/StructuralSimplifier.h
 	optimiser/Substitution.cpp

libyul/CompilabilityChecker.cpp

@@ -0,0 +1,64 @@
+/*(
+	This file is part of solidity.
+
+	solidity is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	solidity is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with solidity.  If not, see <http://www.gnu.org/licenses/>.
+*/
+/**
+ * Component that checks whether all variables are reachable on the stack.
+ */
+
+#include <libyul/CompilabilityChecker.h>
+
+#include <libyul/AsmAnalysis.h>
+#include <libyul/AsmAnalysisInfo.h>
+
+#include <libyul/backends/evm/EVMCodeTransform.h>
+#include <libyul/backends/evm/NoOutputAssembly.h>
+
+#include <liblangutil/EVMVersion.h>
+
+using namespace std;
+using namespace yul;
+using namespace dev;
+using namespace dev::solidity;
+
+std::map<YulString, int> CompilabilityChecker::run(std::shared_ptr<Dialect> _dialect, Block const& _ast)
+{
+	if (_dialect->flavour == AsmFlavour::Yul)
+		return {};
+
+	solAssert(_dialect->flavour == AsmFlavour::Strict, "");
+
+	EVMDialect const& evmDialect = dynamic_cast<EVMDialect const&>(*_dialect);
+
+	bool optimize = true;
+	yul::AsmAnalysisInfo analysisInfo =
+		yul::AsmAnalyzer::analyzeStrictAssertCorrect(_dialect, EVMVersion(), _ast);
+	NoOutputAssembly assembly;
+	CodeTransform transform(assembly, analysisInfo, _ast, evmDialect, optimize);
+	try
+	{
+		transform(_ast);
+	}
+	catch (StackTooDeepError const&)
+	{
+		solAssert(!transform.stackErrors().empty(), "Got stack too deep exception that was not stored.");
+	}
+
+	std::map<YulString, int> functions;
+	for (StackTooDeepError const& error: transform.stackErrors())
+		functions[error.functionName] = max(error.depth, functions[error.functionName]);
+
+	return functions;
+}

libyul/CompilabilityChecker.h

@@ -0,0 +1,45 @@
+/*
+	This file is part of solidity.
+
+	solidity is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	solidity is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with solidity.  If not, see <http://www.gnu.org/licenses/>.
+*/
+/**
+ * Component that checks whether all variables are reachable on the stack.
+ */
+
+#pragma once
+
+#include <libyul/Dialect.h>
+#include <libyul/AsmDataForward.h>
+
+#include <map>
+#include <memory>
+
+namespace yul
+{
+
+/**
+ * Component that checks whether all variables are reachable on the stack and
+ * returns a mapping from function name to the largest stack difference found
+ * in that function (no entry present if that function is compilable).
+ * This only works properly if the outermost block is compilable and
+ * functions are not nested. Otherwise, it might miss reporting some functions.
+ */
+class CompilabilityChecker
+{
+public:
+	static std::map<YulString, int> run(std::shared_ptr<Dialect> _dialect, Block const& _ast);
+};
+
+}

libyul/backends/evm/EVMCodeTransform.cpp

@@ -192,7 +192,8 @@ void CodeTransform::operator()(VariableDeclaration const& _varDecl)
 	bool atTopOfStack = true;
 	for (int varIndex = numVariables - 1; varIndex >= 0; --varIndex)
 	{
-		auto& var = boost::get<Scope::Variable>(m_scope->identifiers.at(_varDecl.variables[varIndex].name));
+		YulString varName = _varDecl.variables[varIndex].name;
+		auto& var = boost::get<Scope::Variable>(m_scope->identifiers.at(varName));
 		m_context->variableStackHeights[&var] = height + varIndex;
 		if (!m_allowStackOpt)
 			continue;
@@ -217,7 +218,7 @@ void CodeTransform::operator()(VariableDeclaration const& _varDecl)
 			m_unusedStackSlots.erase(m_unusedStackSlots.begin());
 			m_context->variableStackHeights[&var] = slot;
 			m_assembly.setSourceLocation(_varDecl.location);
-			if (int heightDiff = variableHeightDiff(var, true))
+			if (int heightDiff = variableHeightDiff(var, varName, true))
 				m_assembly.appendInstruction(solidity::swapInstruction(heightDiff - 1));
 			m_assembly.appendInstruction(solidity::Instruction::POP);
 			--m_stackAdjustment;
@@ -226,6 +227,18 @@ void CodeTransform::operator()(VariableDeclaration const& _varDecl)
 	checkStackHeight(&_varDecl);
 }
 
+void CodeTransform::stackError(StackTooDeepError _error, int _targetStackHeight)
+{
+	m_assembly.appendInstruction(solidity::Instruction::INVALID);
+	// Correct the stack.
+	while (m_assembly.stackHeight() > _targetStackHeight)
+		m_assembly.appendInstruction(solidity::Instruction::POP);
+	while (m_assembly.stackHeight() < _targetStackHeight)
+		m_assembly.appendConstant(u256(0));
+	// Store error.
+	m_stackErrors.emplace_back(std::move(_error));
+}
+
 void CodeTransform::operator()(Assignment const& _assignment)
 {
 	int height = m_assembly.stackHeight();
@@ -353,7 +366,7 @@ void CodeTransform::operator()(Identifier const& _identifier)
 		{
 			// TODO: opportunity for optimization: Do not DUP if this is the last reference
 			// to the top most element of the stack
-			if (int heightDiff = variableHeightDiff(_var, false))
+			if (int heightDiff = variableHeightDiff(_var, _identifier.name, false))
 				m_assembly.appendInstruction(solidity::dupInstruction(heightDiff));
 			else
 				// Store something to balance the stack
@@ -512,18 +525,32 @@ void CodeTransform::operator()(FunctionDefinition const& _function)
 		m_assembly.appendConstant(u256(0));
 	}
 
-	CodeTransform(
+	try
-		m_assembly,
+	{
-		m_info,
+		CodeTransform(
-		_function.body,
+			m_assembly,
-		m_allowStackOpt,
+			m_info,
-		m_dialect,
+			_function.body,
-		m_evm15,
+			m_allowStackOpt,
-		m_identifierAccess,
+			m_dialect,
-		m_useNamedLabelsForFunctions,
+			m_evm15,
-		localStackAdjustment,
+			m_identifierAccess,
-		m_context
+			m_useNamedLabelsForFunctions,
-	)(_function.body);
+			localStackAdjustment,
+			m_context
+		)(_function.body);
+	}
+	catch (StackTooDeepError const& _error)
+	{
+		// This exception will be re-thrown after the end of the surrounding block.
+		// It enables us to see which functions compiled successfully and which did not.
+		// Even if we emit actual code, add an illegal instruction to make sure that tests
+		// will catch it.
+		StackTooDeepError error(_error);
+		if (error.functionName.empty())
+			error.functionName = _function.name;
+		stackError(error, height);
+	}
 
 	{
 		// The stack layout here is:
@@ -542,22 +569,35 @@ void CodeTransform::operator()(FunctionDefinition const& _function)
 		for (size_t i = 0; i < _function.returnVariables.size(); ++i)
 			stackLayout.push_back(i); // Move return values down, but keep order.
 
-		solAssert(stackLayout.size() <= 17, "Stack too deep");
+		if (stackLayout.size() > 17)
-		while (!stackLayout.empty() && stackLayout.back() != int(stackLayout.size() - 1))
+		{
-			if (stackLayout.back() < 0)
+			StackTooDeepError error(_function.name, YulString{}, stackLayout.size() - 17);
-			{
+			error << errinfo_comment(
-				m_assembly.appendInstruction(solidity::Instruction::POP);
+				"The function " +
-				stackLayout.pop_back();
+				_function.name.str() +
-			}
+				" has " +
-			else
+				to_string(stackLayout.size() - 17) +
-			{
+				" parameters or return variables too many to fit the stack size."
-				m_assembly.appendInstruction(swapInstruction(stackLayout.size() - stackLayout.back() - 1));
+			);
-				swap(stackLayout[stackLayout.back()], stackLayout.back());
+			stackError(error, m_assembly.stackHeight() - _function.parameters.size());
-			}
+		}
-		for (int i = 0; size_t(i) < stackLayout.size(); ++i)
+		else
-			solAssert(i == stackLayout[i], "Error reshuffling stack.");
+		{
+			while (!stackLayout.empty() && stackLayout.back() != int(stackLayout.size() - 1))
+				if (stackLayout.back() < 0)
+				{
+					m_assembly.appendInstruction(solidity::Instruction::POP);
+					stackLayout.pop_back();
+				}
+				else
+				{
+					m_assembly.appendInstruction(swapInstruction(stackLayout.size() - stackLayout.back() - 1));
+					swap(stackLayout[stackLayout.back()], stackLayout.back());
+				}
+			for (int i = 0; size_t(i) < stackLayout.size(); ++i)
+				solAssert(i == stackLayout[i], "Error reshuffling stack.");
+		}
 	}
-
 	if (m_evm15)
 		m_assembly.appendReturnsub(_function.returnVariables.size(), stackHeightBefore);
 	else
@@ -615,6 +655,9 @@ void CodeTransform::operator()(Block const& _block)
 
 	finalizeBlock(_block, blockStartStackHeight);
 	m_scope = originalScope;
+
+	if (!m_stackErrors.empty())
+		BOOST_THROW_EXCEPTION(m_stackErrors.front());
 }
 
 AbstractAssembly::LabelID CodeTransform::labelFromIdentifier(Identifier const& _identifier)
@@ -711,7 +754,7 @@ void CodeTransform::generateAssignment(Identifier const& _variableName)
 	if (auto var = m_scope->lookup(_variableName.name))
 	{
 		Scope::Variable const& _var = boost::get<Scope::Variable>(*var);
-		if (int heightDiff = variableHeightDiff(_var, true))
+		if (int heightDiff = variableHeightDiff(_var, _variableName.name, true))
 			m_assembly.appendInstruction(solidity::swapInstruction(heightDiff - 1));
 		m_assembly.appendInstruction(solidity::Instruction::POP);
 		decreaseReference(_variableName.name, _var);
@@ -726,19 +769,25 @@ void CodeTransform::generateAssignment(Identifier const& _variableName)
 	}
 }
 
-int CodeTransform::variableHeightDiff(Scope::Variable const& _var, bool _forSwap) const
+int CodeTransform::variableHeightDiff(Scope::Variable const& _var, YulString _varName, bool _forSwap)
 {
 	solAssert(m_context->variableStackHeights.count(&_var), "");
 	int heightDiff = m_assembly.stackHeight() - m_context->variableStackHeights[&_var];
-	if (heightDiff <= (_forSwap ? 1 : 0) || heightDiff > (_forSwap ? 17 : 16))
+	solAssert(heightDiff > (_forSwap ? 1 : 0), "Negative stack difference for variable.");
+	int limit = _forSwap ? 17 : 16;
+	if (heightDiff > limit)
 	{
-		solUnimplemented(
+		m_stackErrors.emplace_back(_varName, heightDiff - limit);
-			"Variable inaccessible, too deep inside stack (" + to_string(heightDiff) + ")"
+		m_stackErrors.back() << errinfo_comment(
+			"Variable " +
+			_varName.str() +
+			" is " +
+			to_string(heightDiff - limit) +
+			" slot(s) too deep inside the stack."
 		);
-		return 0;
+		BOOST_THROW_EXCEPTION(m_stackErrors.back());
 	}
-	else
+	return heightDiff;
-		return heightDiff;
 }
 
 void CodeTransform::expectDeposit(int _deposit, int _oldHeight) const

libyul/backends/evm/EVMCodeTransform.h

@@ -40,6 +40,17 @@ namespace yul
 struct AsmAnalysisInfo;
 class EVMAssembly;
 
+struct StackTooDeepError: virtual YulException
+{
+	StackTooDeepError(YulString _variable, int _depth): variable(_variable), depth(_depth) {}
+	StackTooDeepError(YulString _functionName, YulString _variable, int _depth):
+		functionName(_functionName), variable(_variable), depth(_depth)
+	{}
+	YulString functionName;
+	YulString variable;
+	int depth;
+};
+
 struct CodeTransformContext
 {
 	std::map<Scope::Label const*, AbstractAssembly::LabelID> labelIDs;
@@ -85,6 +96,10 @@ class CodeTransform: public boost::static_visitor<>
 public:
 	/// Create the code transformer.
 	/// @param _identifierAccess used to resolve identifiers external to the inline assembly
+	/// As a side-effect of its construction, translates the Yul code and appends it to the
+	/// given assembly.
+	/// Throws StackTooDeepError if a variable is not accessible or if a function has too
+	/// many parameters.
 	CodeTransform(
 		AbstractAssembly& _assembly,
 		AsmAnalysisInfo& _analysisInfo,
@@ -109,6 +124,8 @@ public:
 	{
 	}
 
+	std::vector<StackTooDeepError> const& stackErrors() const { return m_stackErrors; }
+
 protected:
 	using Context = CodeTransformContext;
 
@@ -172,12 +189,16 @@ private:
 	/// Determines the stack height difference to the given variables. Throws
 	/// if it is not yet in scope or the height difference is too large. Returns
 	/// the (positive) stack height difference otherwise.
-	int variableHeightDiff(Scope::Variable const& _var, bool _forSwap) const;
+	int variableHeightDiff(Scope::Variable const& _var, YulString _name, bool _forSwap);
 
 	void expectDeposit(int _deposit, int _oldHeight) const;
 
 	void checkStackHeight(void const* _astElement) const;
 
+	/// Stores the stack error in the list of errors, appends an invalid opcode
+	/// and corrects the stack height to the target stack height.
+	void stackError(StackTooDeepError _error, int _targetStackSize);
+
 	AbstractAssembly& m_assembly;
 	AsmAnalysisInfo& m_info;
 	Scope* m_scope = nullptr;
@@ -198,6 +219,8 @@ private:
 	/// statement level in the scope where the variable was defined.
 	std::set<Scope::Variable const*> m_variablesScheduledForDeletion;
 	std::set<int> m_unusedStackSlots;
+
+	std::vector<StackTooDeepError> m_stackErrors;
 };
 
 }

libyul/backends/evm/EVMObjectCompiler.cpp

@@ -60,5 +60,9 @@ void EVMObjectCompiler::run(Object& _object, bool _optimize)
 
 	yulAssert(_object.analysisInfo, "No analysis info.");
 	yulAssert(_object.code, "No code.");
-	CodeTransform{m_assembly, *_object.analysisInfo, *_object.code, m_dialect, _optimize, m_evm15}(*_object.code);
+	// We do not catch and re-throw the stack too deep exception here because it is a YulException,
+	// which should be native to this part of the code.
+	CodeTransform transform{m_assembly, *_object.analysisInfo, *_object.code, m_dialect, _optimize, m_evm15};
+	transform(*_object.code);
+	yulAssert(transform.stackErrors().empty(), "Stack errors present but not thrown.");
 }

libyul/backends/evm/NoOutputAssembly.cpp

@@ -0,0 +1,143 @@
+/*
+	This file is part of solidity.
+
+	solidity is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	solidity is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with solidity.  If not, see <http://www.gnu.org/licenses/>.
+*/
+/**
+ * Assembly interface that ignores everything. Can be used as a backend for a compilation dry-run.
+ */
+
+#include <libyul/backends/evm/NoOutputAssembly.h>
+
+#include <libevmasm/Instruction.h>
+
+#include <liblangutil/Exceptions.h>
+
+using namespace std;
+using namespace dev;
+using namespace langutil;
+using namespace yul;
+
+
+void NoOutputAssembly::appendInstruction(solidity::Instruction _instr)
+{
+	m_stackHeight += solidity::instructionInfo(_instr).ret - solidity::instructionInfo(_instr).args;
+}
+
+void NoOutputAssembly::appendConstant(u256 const&)
+{
+	appendInstruction(solidity::pushInstruction(1));
+}
+
+void NoOutputAssembly::appendLabel(LabelID)
+{
+	appendInstruction(solidity::Instruction::JUMPDEST);
+}
+
+void NoOutputAssembly::appendLabelReference(LabelID)
+{
+	solAssert(!m_evm15, "Cannot use plain label references in EMV1.5 mode.");
+	appendInstruction(solidity::pushInstruction(1));
+}
+
+NoOutputAssembly::LabelID NoOutputAssembly::newLabelId()
+{
+	return 1;
+}
+
+AbstractAssembly::LabelID NoOutputAssembly::namedLabel(string const&)
+{
+	return 1;
+}
+
+void NoOutputAssembly::appendLinkerSymbol(string const&)
+{
+	solAssert(false, "Linker symbols not yet implemented.");
+}
+
+void NoOutputAssembly::appendJump(int _stackDiffAfter)
+{
+	solAssert(!m_evm15, "Plain JUMP used for EVM 1.5");
+	appendInstruction(solidity::Instruction::JUMP);
+	m_stackHeight += _stackDiffAfter;
+}
+
+void NoOutputAssembly::appendJumpTo(LabelID _labelId, int _stackDiffAfter)
+{
+	if (m_evm15)
+		m_stackHeight += _stackDiffAfter;
+	else
+	{
+		appendLabelReference(_labelId);
+		appendJump(_stackDiffAfter);
+	}
+}
+
+void NoOutputAssembly::appendJumpToIf(LabelID _labelId)
+{
+	if (m_evm15)
+		m_stackHeight--;
+	else
+	{
+		appendLabelReference(_labelId);
+		appendInstruction(solidity::Instruction::JUMPI);
+	}
+}
+
+void NoOutputAssembly::appendBeginsub(LabelID, int _arguments)
+{
+	solAssert(m_evm15, "BEGINSUB used for EVM 1.0");
+	solAssert(_arguments >= 0, "");
+	m_stackHeight += _arguments;
+}
+
+void NoOutputAssembly::appendJumpsub(LabelID, int _arguments, int _returns)
+{
+	solAssert(m_evm15, "JUMPSUB used for EVM 1.0");
+	solAssert(_arguments >= 0 && _returns >= 0, "");
+	m_stackHeight += _returns - _arguments;
+}
+
+void NoOutputAssembly::appendReturnsub(int _returns, int _stackDiffAfter)
+{
+	solAssert(m_evm15, "RETURNSUB used for EVM 1.0");
+	solAssert(_returns >= 0, "");
+	m_stackHeight += _stackDiffAfter - _returns;
+}
+
+void NoOutputAssembly::appendAssemblySize()
+{
+	appendInstruction(solidity::Instruction::PUSH1);
+}
+
+pair<shared_ptr<AbstractAssembly>, AbstractAssembly::SubID> NoOutputAssembly::createSubAssembly()
+{
+	solAssert(false, "Sub assemblies not implemented.");
+	return {};
+}
+
+void NoOutputAssembly::appendDataOffset(AbstractAssembly::SubID)
+{
+	appendInstruction(solidity::Instruction::PUSH1);
+}
+
+void NoOutputAssembly::appendDataSize(AbstractAssembly::SubID)
+{
+	appendInstruction(solidity::Instruction::PUSH1);
+}
+
+AbstractAssembly::SubID NoOutputAssembly::appendData(bytes const&)
+{
+	return 1;
+}

libyul/backends/evm/NoOutputAssembly.h

@@ -0,0 +1,75 @@
+/*
+	This file is part of solidity.
+
+	solidity is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	solidity is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with solidity.  If not, see <http://www.gnu.org/licenses/>.
+*/
+/**
+ * Assembly interface that ignores everything. Can be used as a backend for a compilation dry-run.
+ */
+
+#pragma once
+
+#include <libyul/backends/evm/AbstractAssembly.h>
+
+#include <libevmasm/LinkerObject.h>
+
+#include <map>
+
+namespace langutil
+{
+struct SourceLocation;
+}
+
+namespace yul
+{
+
+/**
+ * Assembly class that just ignores everything and only performs stack counting.
+ * The purpose is to use this assembly for compilation dry-runs.
+ */
+class NoOutputAssembly: public AbstractAssembly
+{
+public:
+	explicit NoOutputAssembly(bool _evm15 = false): m_evm15(_evm15) { }
+	virtual ~NoOutputAssembly() = default;
+
+	void setSourceLocation(langutil::SourceLocation const&) override {}
+	int stackHeight() const override { return m_stackHeight; }
+	void appendInstruction(dev::solidity::Instruction _instruction) override;
+	void appendConstant(dev::u256 const& _constant) override;
+	void appendLabel(LabelID _labelId) override;
+	void appendLabelReference(LabelID _labelId) override;
+	LabelID newLabelId() override;
+	LabelID namedLabel(std::string const& _name) override;
+	void appendLinkerSymbol(std::string const& _name) override;
+
+	void appendJump(int _stackDiffAfter) override;
+	void appendJumpTo(LabelID _labelId, int _stackDiffAfter) override;
+	void appendJumpToIf(LabelID _labelId) override;
+	void appendBeginsub(LabelID _labelId, int _arguments) override;
+	void appendJumpsub(LabelID _labelId, int _arguments, int _returns) override;
+	void appendReturnsub(int _returns, int _stackDiffAfter) override;
+
+	void appendAssemblySize() override;
+	std::pair<std::shared_ptr<AbstractAssembly>, SubID> createSubAssembly() override;
+	void appendDataOffset(SubID _sub) override;
+	void appendDataSize(SubID _sub) override;
+	SubID appendData(dev::bytes const& _data) override;
+
+private:
+	bool m_evm15 = false; ///< if true, switch to evm1.5 mode
+	int m_stackHeight = 0;
+};
+
+}

libyul/optimiser/ForLoopInitRewriter.cpp

@@ -38,7 +38,7 @@ void ForLoopInitRewriter::operator()(Block& _block)
 				vector<Statement> rewrite;
 				swap(rewrite, forLoop.pre.statements);
 				rewrite.emplace_back(move(forLoop));
-				return rewrite;
+				return std::move(rewrite);
 			}
 			else
 			{

libyul/optimiser/InlinableExpressionFunctionFinder.cpp

@@ -50,7 +50,7 @@ void InlinableExpressionFunctionFinder::operator()(FunctionDefinition const& _fu
 			Assignment const& assignment = boost::get<Assignment>(bodyStatement);
 			if (assignment.variableNames.size() == 1 && assignment.variableNames.front().name == retVariable)
 			{
-				// FIXME: use code size metric here
+				// TODO: use code size metric here
 
 				// We cannot overwrite previous settings, because this function definition
 				// would not be valid here if we were searching inside a functionally inlinable

libyul/optimiser/NameCollector.cpp

@@ -60,6 +60,13 @@ map<YulString, size_t> ReferencesCounter::countReferences(Block const& _block)
 	return counter.references();
 }
 
+map<YulString, size_t> ReferencesCounter::countReferences(FunctionDefinition const& _function)
+{
+	ReferencesCounter counter;
+	counter(_function);
+	return counter.references();
+}
+
 map<YulString, size_t> ReferencesCounter::countReferences(Expression const& _expression)
 {
 	ReferencesCounter counter;

libyul/optimiser/NameCollector.h

@@ -59,6 +59,7 @@ public:
 	virtual void operator()(FunctionCall const& _funCall);
 
 	static std::map<YulString, size_t> countReferences(Block const& _block);
+	static std::map<YulString, size_t> countReferences(FunctionDefinition const& _function);
 	static std::map<YulString, size_t> countReferences(Expression const& _expression);
 
 	std::map<YulString, size_t> const& references() const { return m_references; }

libyul/optimiser/Rematerialiser.cpp

@@ -30,14 +30,39 @@ using namespace std;
 using namespace dev;
 using namespace yul;
 
-void Rematerialiser::run(Dialect const& _dialect, Block& _ast)
+void Rematerialiser::run(Dialect const& _dialect, Block& _ast, set<YulString> _varsToAlwaysRematerialize)
 {
-	Rematerialiser{_dialect, _ast}(_ast);
+	Rematerialiser{_dialect, _ast, std::move(_varsToAlwaysRematerialize)}(_ast);
 }
 
-Rematerialiser::Rematerialiser(Dialect const& _dialect, Block& _ast):
+void Rematerialiser::run(
+	Dialect const& _dialect,
+	FunctionDefinition& _function,
+	set<YulString> _varsToAlwaysRematerialize
+)
+{
+	Rematerialiser{_dialect, _function, std::move(_varsToAlwaysRematerialize)}(_function);
+}
+
+Rematerialiser::Rematerialiser(
+	Dialect const& _dialect,
+	Block& _ast,
+	set<YulString> _varsToAlwaysRematerialize
+):
 	DataFlowAnalyzer(_dialect),
-	m_referenceCounts(ReferencesCounter::countReferences(_ast))
+	m_referenceCounts(ReferencesCounter::countReferences(_ast)),
+	m_varsToAlwaysRematerialize(std::move(_varsToAlwaysRematerialize))
+{
+}
+
+Rematerialiser::Rematerialiser(
+	Dialect const& _dialect,
+	FunctionDefinition& _function,
+	set<YulString> _varsToAlwaysRematerialize
+):
+	DataFlowAnalyzer(_dialect),
+	m_referenceCounts(ReferencesCounter::countReferences(_function)),
+	m_varsToAlwaysRematerialize(std::move(_varsToAlwaysRematerialize))
 {
 }
 
@@ -53,7 +78,7 @@ void Rematerialiser::visit(Expression& _e)
 			auto const& value = *m_value.at(name);
 			size_t refs = m_referenceCounts[name];
 			size_t cost = CodeCost::codeCost(value);
-			if (refs <= 1 || cost == 0 || (refs <= 5 && cost <= 1))
+			if (refs <= 1 || cost == 0 || (refs <= 5 && cost <= 1) || m_varsToAlwaysRematerialize.count(name))
 			{
 				assertThrow(m_referenceCounts[name] > 0, OptimizerException, "");
 				for (auto const& ref: m_references[name])

libyul/optimiser/Rematerialiser.h

@@ -38,15 +38,34 @@ namespace yul
 class Rematerialiser: public DataFlowAnalyzer
 {
 public:
-	static void run(Dialect const& _dialect, Block& _ast);
+	static void run(
+		Dialect const& _dialect,
+		Block& _ast,
+		std::set<YulString> _varsToAlwaysRematerialize = {}
+	);
+	static void run(
+		Dialect const& _dialect,
+		FunctionDefinition& _function,
+		std::set<YulString> _varsToAlwaysRematerialize = {}
+	);
 
 protected:
-	Rematerialiser(Dialect const& _dialect, Block& _ast);
+	Rematerialiser(
+		Dialect const& _dialect,
+		Block& _ast,
+		std::set<YulString> _varsToAlwaysRematerialize = {}
+	);
+	Rematerialiser(
+		Dialect const& _dialect,
+		FunctionDefinition& _function,
+		std::set<YulString> _varsToAlwaysRematerialize = {}
+	);
 
 	using ASTModifier::visit;
 	void visit(Expression& _e) override;
 
 	std::map<YulString, size_t> m_referenceCounts;
+	std::set<YulString> m_varsToAlwaysRematerialize;
 };
 
 }

libyul/optimiser/SSATransform.cpp

@@ -97,7 +97,7 @@ void SSATransform::operator()(Block& _block)
 					varDecl.value
 				));
 				v.emplace_back(move(varDecl));
-				return v;
+				return std::move(v);
 			}
 			else if (_s.type() == typeid(Assignment))
 			{
@@ -115,7 +115,7 @@ void SSATransform::operator()(Block& _block)
 					assignment.value
 				));
 				v.emplace_back(move(assignment));
-				return v;
+				return std::move(v);
 			}
 			else
 				visit(_s);

libyul/optimiser/StackCompressor.cpp

@@ -0,0 +1,107 @@
+/*(
+	This file is part of solidity.
+
+	solidity is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	solidity is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with solidity.  If not, see <http://www.gnu.org/licenses/>.
+*/
+/**
+ * Optimisation stage that aggressively rematerializes certain variables ina a function to free
+ * space on the stack until it is compilable.
+ */
+
+#include <libyul/optimiser/StackCompressor.h>
+
+#include <libyul/optimiser/SSAValueTracker.h>
+#include <libyul/optimiser/NameCollector.h>
+#include <libyul/optimiser/Rematerialiser.h>
+#include <libyul/optimiser/UnusedPruner.h>
+#include <libyul/optimiser/Metrics.h>
+#include <libyul/optimiser/Semantics.h>
+
+#include <libyul/CompilabilityChecker.h>
+
+#include <libyul/AsmData.h>
+
+using namespace std;
+using namespace dev;
+using namespace yul;
+
+namespace
+{
+
+template <typename ASTNode>
+void eliminateVariables(shared_ptr<Dialect> const& _dialect, ASTNode& _node, size_t _numVariables)
+{
+	SSAValueTracker ssaValues;
+	ssaValues(_node);
+
+	map<YulString, size_t> references = ReferencesCounter::countReferences(_node);
+
+	set<pair<size_t, YulString>> rematCosts;
+	for (auto const& ssa: ssaValues.values())
+	{
+		if (!MovableChecker{*_dialect, *ssa.second}.movable())
+			continue;
+		size_t numRef = references[ssa.first];
+		size_t cost = 0;
+		if (numRef > 1)
+			cost = CodeCost::codeCost(*ssa.second) * (numRef - 1);
+		rematCosts.insert(make_pair(cost, ssa.first));
+	}
+
+	// Select at most _numVariables
+	set<YulString> varsToEliminate;
+	for (auto const& costs: rematCosts)
+	{
+		if (varsToEliminate.size() >= _numVariables)
+			break;
+		varsToEliminate.insert(costs.second);
+	}
+
+	Rematerialiser::run(*_dialect, _node, std::move(varsToEliminate));
+	UnusedPruner::runUntilStabilised(*_dialect, _node);
+}
+
+}
+
+bool StackCompressor::run(shared_ptr<Dialect> const& _dialect, Block& _ast)
+{
+	yulAssert(
+		_ast.statements.size() > 0 && _ast.statements.at(0).type() == typeid(Block),
+		"Need to run the function grouper before the stack compressor."
+	);
+	for (size_t iterations = 0; iterations < 4; iterations++)
+	{
+		map<YulString, int> stackSurplus = CompilabilityChecker::run(_dialect, _ast);
+		if (stackSurplus.empty())
+			return true;
+
+		if (stackSurplus.count(YulString{}))
+		{
+			yulAssert(stackSurplus.at({}) > 0, "Invalid surplus value.");
+			eliminateVariables(_dialect, boost::get<Block>(_ast.statements.at(0)), stackSurplus.at({}));
+		}
+
+		for (size_t i = 1; i < _ast.statements.size(); ++i)
+		{
+			FunctionDefinition& fun = boost::get<FunctionDefinition>(_ast.statements[i]);
+			if (!stackSurplus.count(fun.name))
+				continue;
+
+			yulAssert(stackSurplus.at(fun.name) > 0, "Invalid surplus value.");
+			eliminateVariables(_dialect, fun, stackSurplus.at(fun.name));
+		}
+	}
+	return false;
+}
+

libyul/optimiser/StackCompressor.h

@@ -0,0 +1,47 @@
+/*
+	This file is part of solidity.
+
+	solidity is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	solidity is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with solidity.  If not, see <http://www.gnu.org/licenses/>.
+*/
+/**
+ * Optimisation stage that aggressively rematerializes certain variables ina a function to free
+ * space on the stack until it is compilable.
+ */
+
+#pragma once
+
+#include <memory>
+
+namespace yul
+{
+
+struct Dialect;
+struct Block;
+struct FunctionDefinition;
+
+/**
+ * Optimisation stage that aggressively rematerializes certain variables in a function to free
+ * space on the stack until it is compilable.
+ *
+ * Prerequisite: Disambiguator, Function Grouper
+ */
+class StackCompressor
+{
+public:
+	/// Try to remove local variables until the AST is compilable.
+	/// @returns true if it was successful.
+	static bool run(std::shared_ptr<Dialect> const& _dialect, Block& _ast);
+};
+
+}

libyul/optimiser/Suite.cpp

@@ -37,6 +37,7 @@
 #include <libyul/optimiser/CommonSubexpressionEliminator.h>
 #include <libyul/optimiser/SSAReverser.h>
 #include <libyul/optimiser/SSATransform.h>
+#include <libyul/optimiser/StackCompressor.h>
 #include <libyul/optimiser/StructuralSimplifier.h>
 #include <libyul/optimiser/RedundantAssignEliminator.h>
 #include <libyul/AsmAnalysisInfo.h>
@@ -50,7 +51,7 @@ using namespace dev;
 using namespace yul;
 
 void OptimiserSuite::run(
-	Dialect const& _dialect,
+	shared_ptr<Dialect> const& _dialect,
 	Block& _ast,
 	AsmAnalysisInfo const& _analysisInfo,
 	set<YulString> const& _externallyUsedIdentifiers
@@ -58,85 +59,121 @@ void OptimiserSuite::run(
 {
 	set<YulString> reservedIdentifiers = _externallyUsedIdentifiers;
 
-	Block ast = boost::get<Block>(Disambiguator(_dialect, _analysisInfo, reservedIdentifiers)(_ast));
+	Block ast = boost::get<Block>(Disambiguator(*_dialect, _analysisInfo, reservedIdentifiers)(_ast));
 
 	(VarDeclInitializer{})(ast);
 	(FunctionHoister{})(ast);
 	(BlockFlattener{})(ast);
 	(FunctionGrouper{})(ast);
 	EquivalentFunctionCombiner::run(ast);
-	UnusedPruner::runUntilStabilised(_dialect, ast, reservedIdentifiers);
+	UnusedPruner::runUntilStabilised(*_dialect, ast, reservedIdentifiers);
 	(ForLoopInitRewriter{})(ast);
 	(BlockFlattener{})(ast);
-	StructuralSimplifier{_dialect}(ast);
+	StructuralSimplifier{*_dialect}(ast);
+	(BlockFlattener{})(ast);
 
-	NameDispenser dispenser{_dialect, ast};
+	// None of the above can make stack problems worse.
+
+	NameDispenser dispenser{*_dialect, ast};
 
 	for (size_t i = 0; i < 4; i++)
 	{
-		ExpressionSplitter{_dialect, dispenser}(ast);
+		{
-		SSATransform::run(ast, dispenser);
+			// Turn into SSA and simplify
-		RedundantAssignEliminator::run(_dialect, ast);
+			ExpressionSplitter{*_dialect, dispenser}(ast);
-		RedundantAssignEliminator::run(_dialect, ast);
+			SSATransform::run(ast, dispenser);
+			RedundantAssignEliminator::run(*_dialect, ast);
+			RedundantAssignEliminator::run(*_dialect, ast);
 
-		CommonSubexpressionEliminator{_dialect}(ast);
+			ExpressionSimplifier::run(*_dialect, ast);
-		ExpressionSimplifier::run(_dialect, ast);
+			CommonSubexpressionEliminator{*_dialect}(ast);
-		StructuralSimplifier{_dialect}(ast);
+		}
-		(BlockFlattener{})(ast);
-		SSATransform::run(ast, dispenser);
-		RedundantAssignEliminator::run(_dialect, ast);
-		RedundantAssignEliminator::run(_dialect, ast);
-		UnusedPruner::runUntilStabilised(_dialect, ast, reservedIdentifiers);
-		CommonSubexpressionEliminator{_dialect}(ast);
-		UnusedPruner::runUntilStabilised(_dialect, ast, reservedIdentifiers);
 
-		SSAReverser::run(ast);
+		{
-		CommonSubexpressionEliminator{_dialect}(ast);
+			// still in SSA, perform structural simplification
-		UnusedPruner::runUntilStabilised(_dialect, ast, reservedIdentifiers);
+			StructuralSimplifier{*_dialect}(ast);
+			(BlockFlattener{})(ast);
+			UnusedPruner::runUntilStabilised(*_dialect, ast, reservedIdentifiers);
+		}
+		{
+			// simplify again
+			CommonSubexpressionEliminator{*_dialect}(ast);
+			UnusedPruner::runUntilStabilised(*_dialect, ast, reservedIdentifiers);
+		}
 
-		ExpressionJoiner::run(ast);
+		{
-		ExpressionJoiner::run(ast);
+			// reverse SSA
-		ExpressionInliner(_dialect, ast).run();
+			SSAReverser::run(ast);
-		UnusedPruner::runUntilStabilised(_dialect, ast);
+			CommonSubexpressionEliminator{*_dialect}(ast);
+			UnusedPruner::runUntilStabilised(*_dialect, ast, reservedIdentifiers);
 
-		ExpressionSplitter{_dialect, dispenser}(ast);
+			ExpressionJoiner::run(ast);
-		SSATransform::run(ast, dispenser);
+			ExpressionJoiner::run(ast);
-		RedundantAssignEliminator::run(_dialect, ast);
+		}
-		RedundantAssignEliminator::run(_dialect, ast);
-		CommonSubexpressionEliminator{_dialect}(ast);
 
-		(FunctionGrouper{})(ast);
+		// should have good "compilability" property here.
-		EquivalentFunctionCombiner::run(ast);
-		FullInliner{ast, dispenser}.run();
 
-		SSATransform::run(ast, dispenser);
+		{
-		RedundantAssignEliminator::run(_dialect, ast);
+			// run functional expression inliner
-		RedundantAssignEliminator::run(_dialect, ast);
+			ExpressionInliner(*_dialect, ast).run();
-		ExpressionSimplifier::run(_dialect, ast);
+			UnusedPruner::runUntilStabilised(*_dialect, ast, reservedIdentifiers);
-		StructuralSimplifier{_dialect}(ast);
+		}
-		(BlockFlattener{})(ast);
+
-		CommonSubexpressionEliminator{_dialect}(ast);
+		{
-		SSATransform::run(ast, dispenser);
+			// Turn into SSA again and simplify
-		RedundantAssignEliminator::run(_dialect, ast);
+			ExpressionSplitter{*_dialect, dispenser}(ast);
-		RedundantAssignEliminator::run(_dialect, ast);
+			SSATransform::run(ast, dispenser);
-		UnusedPruner::runUntilStabilised(_dialect, ast, reservedIdentifiers);
+			RedundantAssignEliminator::run(*_dialect, ast);
-		CommonSubexpressionEliminator{_dialect}(ast);
+			RedundantAssignEliminator::run(*_dialect, ast);
+			CommonSubexpressionEliminator{*_dialect}(ast);
+		}
+
+		{
+			// run full inliner
+			(FunctionGrouper{})(ast);
+			EquivalentFunctionCombiner::run(ast);
+			FullInliner{ast, dispenser}.run();
+			(BlockFlattener{})(ast);
+		}
+
+		{
+			// SSA plus simplify
+			SSATransform::run(ast, dispenser);
+			RedundantAssignEliminator::run(*_dialect, ast);
+			RedundantAssignEliminator::run(*_dialect, ast);
+			ExpressionSimplifier::run(*_dialect, ast);
+			StructuralSimplifier{*_dialect}(ast);
+			(BlockFlattener{})(ast);
+			CommonSubexpressionEliminator{*_dialect}(ast);
+			SSATransform::run(ast, dispenser);
+			RedundantAssignEliminator::run(*_dialect, ast);
+			RedundantAssignEliminator::run(*_dialect, ast);
+			UnusedPruner::runUntilStabilised(*_dialect, ast, reservedIdentifiers);
+			CommonSubexpressionEliminator{*_dialect}(ast);
+		}
 	}
+
+	// Make source short and pretty.
+
 	ExpressionJoiner::run(ast);
-	Rematerialiser::run(_dialect, ast);
+	Rematerialiser::run(*_dialect, ast);
-	UnusedPruner::runUntilStabilised(_dialect, ast);
+	UnusedPruner::runUntilStabilised(*_dialect, ast, reservedIdentifiers);
 	ExpressionJoiner::run(ast);
-	UnusedPruner::runUntilStabilised(_dialect, ast);
+	UnusedPruner::runUntilStabilised(*_dialect, ast, reservedIdentifiers);
 	ExpressionJoiner::run(ast);
-	UnusedPruner::runUntilStabilised(_dialect, ast);
+	UnusedPruner::runUntilStabilised(*_dialect, ast, reservedIdentifiers);
 
 	SSAReverser::run(ast);
-	CommonSubexpressionEliminator{_dialect}(ast);
+	CommonSubexpressionEliminator{*_dialect}(ast);
-	UnusedPruner::runUntilStabilised(_dialect, ast, reservedIdentifiers);
+	UnusedPruner::runUntilStabilised(*_dialect, ast, reservedIdentifiers);
 
 	ExpressionJoiner::run(ast);
-	Rematerialiser::run(_dialect, ast);
+	Rematerialiser::run(*_dialect, ast);
-	UnusedPruner::runUntilStabilised(_dialect, ast);
+	UnusedPruner::runUntilStabilised(*_dialect, ast, reservedIdentifiers);
+
+	(FunctionGrouper{})(ast);
+	StackCompressor::run(_dialect, ast);
+	(BlockFlattener{})(ast);
 
 	_ast = std::move(ast);
 }

libyul/optimiser/Suite.h

@@ -38,7 +38,7 @@ class OptimiserSuite
 {
 public:
 	static void run(
-		Dialect const& _dialect,
+		std::shared_ptr<Dialect> const& _dialect,
 		Block& _ast,
 		AsmAnalysisInfo const& _analysisInfo,
 		std::set<YulString> const& _externallyUsedIdentifiers = {}

libyul/optimiser/UnusedPruner.cpp

@@ -35,10 +35,15 @@ using namespace yul;
 UnusedPruner::UnusedPruner(Dialect const& _dialect, Block& _ast, set<YulString> const& _externallyUsedFunctions):
 	m_dialect(_dialect)
 {
-	ReferencesCounter counter;
+	m_references = ReferencesCounter::countReferences(_ast);
-	counter(_ast);
+	for (auto const& f: _externallyUsedFunctions)
+		++m_references[f];
+}
 
-	m_references = counter.references();
+UnusedPruner::UnusedPruner(Dialect const& _dialect, FunctionDefinition& _function, set<YulString> const& _externallyUsedFunctions):
+	m_dialect(_dialect)
+{
+	m_references = ReferencesCounter::countReferences(_function);
 	for (auto const& f: _externallyUsedFunctions)
 		++m_references[f];
 }
@@ -116,6 +121,21 @@ void UnusedPruner::runUntilStabilised(
 	}
 }
 
+void UnusedPruner::runUntilStabilised(
+	Dialect const& _dialect,
+	FunctionDefinition& _function,
+	set<YulString> const& _externallyUsedFunctions
+)
+{
+	while (true)
+	{
+		UnusedPruner pruner(_dialect, _function, _externallyUsedFunctions);
+		pruner(_function);
+		if (!pruner.shouldRunAgain())
+			return;
+	}
+}
+
 bool UnusedPruner::used(YulString _name) const
 {
 	return m_references.count(_name) && m_references.at(_name) > 0;

libyul/optimiser/UnusedPruner.h

@@ -46,6 +46,11 @@ public:
 		Block& _ast,
 		std::set<YulString> const& _externallyUsedFunctions = {}
 	);
+	UnusedPruner(
+		Dialect const& _dialect,
+		FunctionDefinition& _function,
+		std::set<YulString> const& _externallyUsedFunctions = {}
+	);
 
 	using ASTModifier::operator();
 	void operator()(Block& _block) override;
@@ -60,6 +65,14 @@ public:
 		std::set<YulString> const& _externallyUsedFunctions = {}
 	);
 
+	// Run the pruner until the code does not change anymore.
+	// Only run on the given function.
+	static void runUntilStabilised(
+		Dialect const& _dialect,
+		FunctionDefinition& _functionDefinition,
+		std::set<YulString> const& _externallyUsedFunctions = {}
+	);
+
 private:
 	bool used(YulString _name) const;
 	void subtractReferences(std::map<YulString, size_t> const& _subtrahend);

scripts/install_deps.sh

@@ -162,7 +162,7 @@ case $(uname -s) in
                 # See https://pkgs.alpinelinux.org/
 
                 apk update
-                apk add boost-dev build-base cmake git
+                apk add boost-dev boost-static build-base cmake git
 
                 ;;
 

scripts/release_ppa.sh

@@ -53,7 +53,14 @@ packagename=solc
 
 static_build_distribution=cosmic
 
-for distribution in bionic cosmic STATIC
+DISTRIBUTIONS="bionic cosmic disco"
+
+if [ branch != develop ]
+then
+    DISTRIBUTIONS="$DISTRIBUTIONS STATIC"
+fi
+
+for distribution in $DISTRIBUTIONS
 do
 cd /tmp/
 rm -rf $distribution
@@ -63,7 +70,7 @@ cd $distribution
 if [ $distribution = STATIC ]
 then
     pparepo=ethereum-static
-    Z3DEPENDENCY=""
+    SMTDEPENDENCY=""
     CMAKE_OPTIONS="-DSOLC_LINK_STATIC=On"
 else
     if [ "$branch" = develop ]
@@ -72,7 +79,7 @@ else
     else
         pparepo=ethereum
     fi
-    Z3DEPENDENCY="libz3-dev,
+    SMTDEPENDENCY="libcvc4-dev,
                "
     CMAKE_OPTIONS=""
 fi
@@ -96,7 +103,7 @@ commitdate=$(git show --format=%ci HEAD | head -n 1 | cut - -b1-10 | sed -e 's/-
 echo "$commithash" > commit_hash.txt
 if [ $branch = develop ]
 then
-    debversion="$version-develop-$commitdate-$commithash"
+    debversion="$version~develop-$commitdate-$commithash"
 else
     debversion="$version"
     echo -n > prerelease.txt # proper release
@@ -120,7 +127,7 @@ Source: solc
 Section: science
 Priority: extra
 Maintainer: Christian (Buildserver key) <builds@ethereum.org>
-Build-Depends: ${Z3DEPENDENCY}debhelper (>= 9.0.0),
+Build-Depends: ${SMTDEPENDENCY}debhelper (>= 9.0.0),
                cmake,
                g++,
                git,

snap/snapcraft.yaml

@@ -27,7 +27,7 @@ parts:
     source-type: git
     plugin: cmake
     build-packages: [build-essential, libboost-all-dev, libz3-dev]
-    stage-packages: [libicu55]
+    stage-packages: [libicu60]
     prepare: |
       if git describe --exact-match --tags 2> /dev/null
       then

solc/CommandLineInterface.cpp

@@ -35,6 +35,7 @@
 #include <libsolidity/interface/CompilerStack.h>
 #include <libsolidity/interface/StandardCompiler.h>
 #include <liblangutil/SourceReferenceFormatter.h>
+#include <liblangutil/SourceReferenceFormatterHuman.h>
 #include <libsolidity/interface/GasEstimator.h>
 #include <libsolidity/interface/AssemblyStack.h>
 
@@ -46,6 +47,8 @@
 #include <libdevcore/CommonIO.h>
 #include <libdevcore/JSON.h>
 
+#include <memory>
+
 #include <boost/filesystem.hpp>
 #include <boost/filesystem/operations.hpp>
 #include <boost/algorithm/string.hpp>
@@ -57,10 +60,15 @@
 #else // unix
 	#include <unistd.h>
 #endif
+
 #include <string>
 #include <iostream>
 #include <fstream>
 
+#if !defined(STDERR_FILENO)
+	#define STDERR_FILENO 2
+#endif
+
 using namespace std;
 using namespace langutil;
 namespace po = boost::program_options;
@@ -134,6 +142,9 @@ static string const g_strStrictAssembly = "strict-assembly";
 static string const g_strPrettyJson = "pretty-json";
 static string const g_strVersion = "version";
 static string const g_strIgnoreMissingFiles = "ignore-missing";
+static string const g_strColor = "color";
+static string const g_strNoColor = "no-color";
+static string const g_strNewReporter = "new-reporter";
 
 static string const g_argAbi = g_strAbi;
 static string const g_argPrettyJson = g_strPrettyJson;
@@ -169,6 +180,9 @@ static string const g_argStrictAssembly = g_strStrictAssembly;
 static string const g_argVersion = g_strVersion;
 static string const g_stdinFileName = g_stdinFileNameStr;
 static string const g_argIgnoreMissingFiles = g_strIgnoreMissingFiles;
+static string const g_argColor = g_strColor;
+static string const g_argNoColor = g_strNoColor;
+static string const g_argNewReporter = g_strNewReporter;
 
 /// Possible arguments to for --combined-json
 static set<string> const g_combinedJsonArgs
@@ -652,6 +666,9 @@ Allowed options)",
 			po::value<string>()->value_name("path(s)"),
 			"Allow a given path for imports. A list of paths can be supplied by separating them with a comma."
 		)
+		(g_argColor.c_str(), "Force colored output.")
+		(g_argNoColor.c_str(), "Explicitly disable colored output, disabling terminal auto-detection.")
+		(g_argNewReporter.c_str(), "Enables new diagnostics reporter.")
 		(g_argIgnoreMissingFiles.c_str(), "Ignore missing files.");
 	po::options_description outputComponents("Output Components");
 	outputComponents.add_options()
@@ -691,6 +708,14 @@ Allowed options)",
 		return false;
 	}
 
+	if (m_args.count(g_argColor) && m_args.count(g_argNoColor))
+	{
+		serr() << "Option " << g_argColor << " and " << g_argNoColor << " are mutualy exclusive." << endl;
+		return false;
+	}
+
+	m_coloredOutput = !m_args.count(g_argNoColor) && (isatty(STDERR_FILENO) || m_args.count(g_argColor));
+
 	if (m_args.count(g_argHelp) || (isatty(fileno(stdin)) && _argc == 1))
 	{
 		sout() << desc;
@@ -836,13 +861,11 @@ bool CommandLineInterface::processInput()
 				return false;
 			}
 		}
-		if (optimize && inputLanguage == Input::Assembly)
+		if (optimize && inputLanguage != Input::StrictAssembly)
 		{
 			serr() <<
-				"Optimizer cannot be used for loose assembly. Use --" <<
+				"Optimizer can only be used for strict assembly. Use --" <<
 				g_strStrictAssembly <<
-				" or --" <<
-				g_strYul <<
 				"." <<
 				endl;
 			return false;
@@ -858,7 +881,11 @@ bool CommandLineInterface::processInput()
 
 	m_compiler.reset(new CompilerStack(fileReader));
 
-	SourceReferenceFormatter formatter(serr(false));
+	unique_ptr<SourceReferenceFormatter> formatter;
+	if (m_args.count(g_argNewReporter))
+		formatter = make_unique<SourceReferenceFormatterHuman>(serr(false), m_coloredOutput);
+	else
+		formatter = make_unique<SourceReferenceFormatter>(serr(false));
 
 	try
 	{
@@ -881,7 +908,7 @@ bool CommandLineInterface::processInput()
 		for (auto const& error: m_compiler->errors())
 		{
 			g_hasOutput = true;
-			formatter.printExceptionInformation(
+			formatter->printExceptionInformation(
 				*error,
 				(error->type() == Error::Type::Warning) ? "Warning" : "Error"
 			);
@@ -893,7 +920,7 @@ bool CommandLineInterface::processInput()
 	catch (CompilerError const& _exception)
 	{
 		g_hasOutput = true;
-		formatter.printExceptionInformation(_exception, "Compiler error");
+		formatter->printExceptionInformation(_exception, "Compiler error");
 		return false;
 	}
 	catch (InternalCompilerError const& _exception)
@@ -915,7 +942,7 @@ bool CommandLineInterface::processInput()
 		else
 		{
 			g_hasOutput = true;
-			formatter.printExceptionInformation(_error, _error.typeName());
+			formatter->printExceptionInformation(_error, _error.typeName());
 		}
 
 		return false;
@@ -1221,12 +1248,16 @@ bool CommandLineInterface::assemble(
 	for (auto const& sourceAndStack: assemblyStacks)
 	{
 		auto const& stack = sourceAndStack.second;
-		SourceReferenceFormatter formatter(serr(false));
+		unique_ptr<SourceReferenceFormatter> formatter;
+		if (m_args.count(g_argNewReporter))
+			formatter = make_unique<SourceReferenceFormatterHuman>(serr(false), m_coloredOutput);
+		else
+			formatter = make_unique<SourceReferenceFormatter>(serr(false));
 
 		for (auto const& error: stack.errors())
 		{
 			g_hasOutput = true;
-			formatter.printExceptionInformation(
+			formatter->printExceptionInformation(
 				*error,
 				(error->type() == Error::Type::Warning) ? "Warning" : "Error"
 			);

solc/CommandLineInterface.h

@@ -109,6 +109,8 @@ private:
 	std::unique_ptr<dev::solidity::CompilerStack> m_compiler;
 	/// EVM version to use
 	EVMVersion m_evmVersion;
+	/// Whether or not to colorize diagnostics output.
+	bool m_coloredOutput = true;
 };
 
 }

test/CMakeLists.txt

@@ -17,6 +17,8 @@ if (LLL)
 endif()
 file(GLOB libsolidity_sources "libsolidity/*.cpp")
 file(GLOB libsolidity_headers "libsolidity/*.h")
+file(GLOB libsolidity_util_sources "libsolidity/util/*.cpp")
+file(GLOB libsolidity_util_headers "libsolidity/util/*.h")
 
 add_executable(soltest ${sources} ${headers}
     ${contracts_sources} ${contracts_headers}
@@ -26,6 +28,7 @@ add_executable(soltest ${sources} ${headers}
     ${libyul_sources} ${libyul_headers}
     ${liblll_sources} ${liblll_headers}
     ${libsolidity_sources} ${libsolidity_headers}
+    ${libsolidity_util_sources} ${libsolidity_util_headers}
 )
 target_link_libraries(soltest PRIVATE libsolc yul solidity evmasm devcore ${Boost_UNIT_TEST_FRAMEWORK_LIBRARIES})
 

test/Metadata.cpp

@@ -22,6 +22,7 @@
 #include <string>
 #include <iostream>
 #include <libdevcore/JSON.h>
+#include <test/Metadata.h>
 
 using namespace std;
 
@@ -30,21 +31,24 @@ namespace dev
 namespace test
 {
 
+bytes bytecodeSansMetadata(bytes const& _bytecode)
+{
+	unsigned size = _bytecode.size();
+	if (size < 5)
+		return bytes{};
+	size_t metadataSize = (_bytecode[size - 2] << 8) + _bytecode[size - 1];
+	if (size < (metadataSize + 2))
+		return bytes{};
+	// Sanity check: assume the first byte is a fixed-size CBOR array with either 1 or 2 entries
+	unsigned char firstByte = _bytecode[size - metadataSize - 2];
+	if (firstByte != 0xa1 && firstByte != 0xa2)
+		return bytes{};
+	return bytes(_bytecode.begin(), _bytecode.end() - metadataSize - 2);
+}
+
 string bytecodeSansMetadata(string const& _bytecode)
 {
-	/// The metadata hash takes up 43 bytes (or 86 characters in hex)
+	return toHex(bytecodeSansMetadata(fromHex(_bytecode, WhenError::Throw)));
-	/// /a165627a7a72305820([0-9a-f]{64})0029$/
-
-	if (_bytecode.size() < 88)
-		return _bytecode;
-
-	if (_bytecode.substr(_bytecode.size() - 4, 4) != "0029")
-		return _bytecode;
-
-	if (_bytecode.substr(_bytecode.size() - 86, 18) != "a165627a7a72305820")
-		return _bytecode;
-
-	return _bytecode.substr(0, _bytecode.size() - 86);
 }
 
 bool isValidMetadata(string const& _metadata)

test/Metadata.h

@@ -19,6 +19,7 @@
  * Metadata processing helpers.
  */
 
+#include <libdevcore/CommonData.h>
 #include <string>
 
 namespace dev
@@ -27,6 +28,10 @@ namespace test
 {
 
 /// Returns the bytecode with the metadata hash stripped out.
+bytes bytecodeSansMetadata(bytes const& _bytecode);
+
+/// Returns the bytecode with the metadata hash stripped out.
+/// Throws exception on invalid hex string.
 std::string bytecodeSansMetadata(std::string const& _bytecode);
 
 /// Expects a serialised metadata JSON and returns true if the

test/RPCSession.cpp

@@ -324,7 +324,15 @@ Json::Value RPCSession::rpcCall(string const& _methodName, vector<string> const&
 	Json::Value result;
 	string errorMsg;
 	if (!jsonParseStrict(reply, result, &errorMsg))
-		BOOST_REQUIRE_MESSAGE(false, errorMsg);
+		BOOST_FAIL("Failed to parse JSON-RPC response: " + errorMsg);
+
+	if (!result.isMember("id") || !result["id"].isUInt())
+		BOOST_FAIL("Badly formatted JSON-RPC response (missing or non-integer \"id\")");
+	if (result["id"].asUInt() != (m_rpcSequence - 1))
+		BOOST_FAIL(
+			"Response identifier mismatch. "
+			"Expected " + to_string(m_rpcSequence - 1) + " but got " + to_string(result["id"].asUInt()) + "."
+		);
 
 	if (result.isMember("error"))
 	{

test/RPCSession.h

@@ -57,7 +57,10 @@ class IPCSocket: public boost::noncopyable
 public:
 	explicit IPCSocket(std::string const& _path);
 	std::string sendRequest(std::string const& _req);
-	~IPCSocket() { close(m_socket); }
+	~IPCSocket() {
+		shutdown(m_socket, SHUT_RDWR);
+		close(m_socket);
+	}
 
 	std::string const& path() const { return m_path; }
 

test/cmdlineTests.sh

@@ -332,13 +332,12 @@ printTask "Testing assemble, yul, strict-assembly and optimize..."
     # Test options above in conjunction with --optimize.
     # Using both, --assemble and --optimize should fail.
     ! echo '{}' | "$SOLC" - --assemble --optimize &>/dev/null
+    ! echo '{}' | "$SOLC" - --yul --optimize &>/dev/null
 
     # Test yul and strict assembly output
     # Non-empty code results in non-empty binary representation with optimizations turned off,
     # while it results in empty binary representation with optimizations turned on.
     test_solc_assembly_output "{ let x:u256 := 0:u256 }" "{ let x:u256 := 0:u256 }" "--yul"
-    test_solc_assembly_output "{ let x:u256 := 0:u256 }" "{ }" "--yul --optimize"
-
     test_solc_assembly_output "{ let x := 0 }" "{ let x := 0 }" "--strict-assembly"
     test_solc_assembly_output "{ let x := 0 }" "{ }" "--strict-assembly --optimize"
 )

test/cmdlineTests/standard_missing_key_useLiteralContent/exit

@@ -0,0 +1 @@
+0

test/cmdlineTests/standard_missing_key_useLiteralContent/input.json

@@ -0,0 +1,19 @@
+{
+	"language": "Solidity",
+	"sources":
+	{
+		"A":
+		{
+			"content": "pragma solidity >=0.0; contract C { function f() public pure {} }"
+		}
+	},
+	"settings":
+	{
+		"optimizer": {
+			"enabled": true,
+			"runs": 200
+		},
+		"evmVersion": "byzantium",
+		"metadata": {}
+	}
+}

test/cmdlineTests/standard_missing_key_useLiteralContent/output.json

@@ -0,0 +1 @@
+{"sources":{"A":{"id":0}}}

test/cmdlineTests/standard_wrong_type_useLiteralContent/exit

@@ -0,0 +1 @@
+0

test/cmdlineTests/standard_wrong_type_useLiteralContent/input.json

@@ -0,0 +1,21 @@
+{
+	"language": "Solidity",
+	"sources":
+	{
+		"A":
+		{
+			"content": "pragma solidity >=0.0; contract C { function f() public pure {} }"
+		}
+	},
+	"settings":
+	{
+		"optimizer": {
+			"enabled": true,
+			"runs": 200
+		},
+		"evmVersion": "byzantium",
+		"metadata": {
+			"useLiteralContent": "literalContent"
+		}
+	}
+}

test/cmdlineTests/standard_wrong_type_useLiteralContent/output.json

@@ -0,0 +1 @@
+{"errors":[{"component":"general","formattedMessage":"\"settings.metadata.useLiteralContent\" must be Boolean","message":"\"settings.metadata.useLiteralContent\" must be Boolean","severity":"error","type":"JSONError"}]}

test/externalTests.sh

@@ -59,10 +59,13 @@ function test_truffle
       for d in node_modules node_modules/truffle/node_modules
       do
       (
-        cd $d
+        if [ -d "$d" ]
-        rm -rf solc
+        then
-        git clone --depth 1 -b v0.5.0 https://github.com/ethereum/solc-js.git solc
+          cd $d
-        cp "$SOLJSON" solc/
+          rm -rf solc
+          git clone --depth 1 -b v0.5.0 https://github.com/ethereum/solc-js.git solc
+          cp "$SOLJSON" solc/
+        fi
       )
       done
       if [ "$name" == "Zeppelin" -o "$name" == "Gnosis" ]; then
@@ -85,8 +88,8 @@ function test_truffle
     rm -rf "$DIR"
 }
 
-# Using our temporary fork here. Hopefully to be merged into upstream after the 0.5.0 release.
+# Since Zeppelin 2.1.1 it supports Solidity 0.5.0.
-test_truffle Zeppelin https://github.com/axic/openzeppelin-solidity.git solidity-050
+test_truffle Zeppelin https://github.com/OpenZeppelin/openzeppelin-solidity.git master
 
 # Disabled temporarily as it needs to be updated to latest Truffle first.
 #test_truffle Gnosis https://github.com/axic/pm-contracts.git solidity-050

test/libsolidity/ASTJSONTest.cpp

@@ -17,6 +17,7 @@
 
 #include <test/libsolidity/ASTJSONTest.h>
 #include <test/Options.h>
+#include <libdevcore/AnsiColorized.h>
 #include <libsolidity/ast/ASTJsonConverter.h>
 #include <libsolidity/interface/CompilerStack.h>
 #include <boost/algorithm/string.hpp>
@@ -26,10 +27,10 @@
 #include <memory>
 #include <stdexcept>
 
-using namespace dev;
+using namespace dev::solidity;
-using namespace solidity;
 using namespace dev::solidity::test;
-using namespace dev::solidity::test::formatting;
+using namespace dev::formatting;
+using namespace dev;
 using namespace std;
 namespace fs = boost::filesystem;
 using namespace boost::unit_test;
@@ -116,7 +117,7 @@ bool ASTJSONTest::run(ostream& _stream, string const& _linePrefix, bool const _f
 	if (m_expectation != m_result)
 	{
 		string nextIndentLevel = _linePrefix + "  ";
-		FormattedScope(_stream, _formatted, {BOLD, CYAN}) << _linePrefix << "Expected result:" << endl;
+		AnsiColorized(_stream, _formatted, {BOLD, CYAN}) << _linePrefix << "Expected result:" << endl;
 		{
 			istringstream stream(m_expectation);
 			string line;
@@ -124,7 +125,7 @@ bool ASTJSONTest::run(ostream& _stream, string const& _linePrefix, bool const _f
 				_stream << nextIndentLevel << line << endl;
 		}
 		_stream << endl;
-		FormattedScope(_stream, _formatted, {BOLD, CYAN}) << _linePrefix << "Obtained result:" << endl;
+		AnsiColorized(_stream, _formatted, {BOLD, CYAN}) << _linePrefix << "Obtained result:" << endl;
 		{
 			istringstream stream(m_result);
 			string line;
@@ -148,7 +149,7 @@ bool ASTJSONTest::run(ostream& _stream, string const& _linePrefix, bool const _f
 	if (m_expectationLegacy != m_resultLegacy)
 	{
 		string nextIndentLevel = _linePrefix + "  ";
-		FormattedScope(_stream, _formatted, {BOLD, CYAN}) << _linePrefix << "Expected result (legacy):" << endl;
+		AnsiColorized(_stream, _formatted, {BOLD, CYAN}) << _linePrefix << "Expected result (legacy):" << endl;
 		{
 			istringstream stream(m_expectationLegacy);
 			string line;
@@ -156,7 +157,7 @@ bool ASTJSONTest::run(ostream& _stream, string const& _linePrefix, bool const _f
 				_stream << nextIndentLevel << line << endl;
 		}
 		_stream << endl;
-		FormattedScope(_stream, _formatted, {BOLD, CYAN}) << _linePrefix << "Obtained result (legacy):" << endl;
+		AnsiColorized(_stream, _formatted, {BOLD, CYAN}) << _linePrefix << "Obtained result (legacy):" << endl;
 		{
 			istringstream stream(m_resultLegacy);
 			string line;

test/libsolidity/ASTJSONTest.h

@@ -17,7 +17,7 @@
 
 #pragma once
 
-#include <test/libsolidity/FormattedScope.h>
+#include <libdevcore/AnsiColorized.h>
 #include <test/TestCase.h>
 
 #include <iosfwd>

test/libsolidity/FormattedScope.h

@@ -1,68 +0,0 @@
-/*
-	This file is part of solidity.
-
-	solidity is free software: you can redistribute it and/or modify
-	it under the terms of the GNU General Public License as published by
-	the Free Software Foundation, either version 3 of the License, or
-	(at your option) any later version.
-
-	solidity is distributed in the hope that it will be useful,
-	but WITHOUT ANY WARRANTY; without even the implied warranty of
-	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-	GNU General Public License for more details.
-
-	You should have received a copy of the GNU General Public License
-	along with solidity.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-#pragma once
-
-#include <boost/noncopyable.hpp>
-
-#include <ostream>
-#include <vector>
-
-namespace dev
-{
-namespace solidity
-{
-namespace test
-{
-
-namespace formatting
-{
-
-static constexpr char const* RESET = "\033[0m";
-static constexpr char const* RED  = "\033[1;31m";
-static constexpr char const* GREEN = "\033[1;32m";
-static constexpr char const* YELLOW = "\033[1;33m";
-static constexpr char const* CYAN = "\033[1;36m";
-static constexpr char const* BOLD = "\033[1m";
-static constexpr char const* RED_BACKGROUND  = "\033[48;5;160m";
-static constexpr char const* ORANGE_BACKGROUND  = "\033[48;5;166m";
-static constexpr char const* INVERSE = "\033[7m";
-
-}
-
-class FormattedScope: boost::noncopyable
-{
-public:
-	/// @arg _formatting List of formatting strings (e.g. colors) defined in the formatting namespace.
-	FormattedScope(std::ostream& _stream, bool const _enabled, std::vector<char const*> const& _formatting):
-		m_stream(_stream), m_enabled(_enabled)
-	{
-		if (m_enabled)
-			for (auto const& format: _formatting)
-				m_stream << format;
-	}
-	~FormattedScope() { if (m_enabled) m_stream << formatting::RESET; }
-	template<typename T>
-	std::ostream& operator<<(T&& _t) { return m_stream << std::forward<T>(_t); }
-private:
-	std::ostream& m_stream;
-	bool m_enabled;
-};
-
-}
-}
-}

test/libsolidity/LibSolc.cpp

@@ -25,9 +25,6 @@
 #include <libsolidity/interface/Version.h>
 #include <libsolc/libsolc.h>
 
-#include <test/Metadata.h>
-#include <test/Options.h>
-
 using namespace std;
 
 namespace dev
@@ -40,11 +37,31 @@ namespace test
 namespace
 {
 
-Json::Value compile(string const& _input)
+/// TODO: share this between StandardCompiler.cpp
+/// Helper to match a specific error type and message
+bool containsError(Json::Value const& _compilerResult, string const& _type, string const& _message)
 {
-	string output(solidity_compile(_input.c_str(), nullptr));
+	if (!_compilerResult.isMember("errors"))
+		return false;
+
+	for (auto const& error: _compilerResult["errors"])
+	{
+		BOOST_REQUIRE(error.isObject());
+		BOOST_REQUIRE(error["type"].isString());
+		BOOST_REQUIRE(error["message"].isString());
+		if ((error["type"].asString() == _type) && (error["message"].asString() == _message))
+			return true;
+	}
+
+	return false;
+}
+
+Json::Value compile(string const& _input, CStyleReadFileCallback _callback = nullptr)
+{
+	string output(solidity_compile(_input.c_str(), _callback));
 	Json::Value ret;
 	BOOST_REQUIRE(jsonParseStrict(output, ret));
+	solidity_free();
 	return ret;
 }
 
@@ -56,12 +73,14 @@ BOOST_AUTO_TEST_CASE(read_version)
 {
 	string output(solidity_version());
 	BOOST_CHECK(output.find(VersionString) == 0);
+	solidity_free();
 }
 
 BOOST_AUTO_TEST_CASE(read_license)
 {
 	string output(solidity_license());
 	BOOST_CHECK(output.find("GNU GENERAL PUBLIC LICENSE") != string::npos);
+	solidity_free();
 }
 
 BOOST_AUTO_TEST_CASE(standard_compilation)
@@ -86,6 +105,71 @@ BOOST_AUTO_TEST_CASE(standard_compilation)
 	BOOST_CHECK(!result.isMember("contracts"));
 }
 
+BOOST_AUTO_TEST_CASE(missing_callback)
+{
+	char const* input = R"(
+	{
+		"language": "Solidity",
+		"sources": {
+			"fileA": {
+				"content": "import \"missing.sol\"; contract A { }"
+			}
+		}
+	}
+	)";
+	Json::Value result = compile(input);
+	BOOST_CHECK(result.isObject());
+
+	BOOST_CHECK(containsError(result, "ParserError", "Source \"missing.sol\" not found: File not supplied initially."));
+}
+
+BOOST_AUTO_TEST_CASE(with_callback)
+{
+	char const* input = R"(
+	{
+		"language": "Solidity",
+		"sources": {
+			"fileA": {
+				"content": "import \"found.sol\"; import \"notfound.sol\"; contract A { }"
+			}
+		}
+	}
+	)";
+
+	CStyleReadFileCallback callback{
+		[](char const* _path, char** o_contents, char** o_error)
+		{
+			// Caller frees the pointers.
+			if (string(_path) == "found.sol")
+			{
+				static string content{"import \"missing.sol\"; contract B {}"};
+				*o_contents = strdup(content.c_str());
+				*o_error = nullptr;
+			}
+			else if (string(_path) == "missing.sol")
+			{
+				static string errorMsg{"Missing file."};
+				*o_error = strdup(errorMsg.c_str());
+				*o_contents = nullptr;
+			}
+			else
+			{
+				*o_error = nullptr;
+				*o_contents = nullptr;
+			}
+		}
+	};
+
+	Json::Value result = compile(input, callback);
+	BOOST_CHECK(result.isObject());
+
+	// This ensures that "found.sol" was properly loaded which triggered the second import statement.
+	BOOST_CHECK(containsError(result, "ParserError", "Source \"missing.sol\" not found: Missing file."));
+
+	// This should be placed due to the missing "notfound.sol" which sets both pointers to null.
+	BOOST_CHECK(containsError(result, "ParserError", "Source \"notfound.sol\" not found: File not found."));
+}
+
 BOOST_AUTO_TEST_SUITE_END()
 
 }