From 22466acf39c41c168a9a24b05b2f00f48cbfa30e Mon Sep 17 00:00:00 2001 From: Daniel Kirchner Date: Thu, 9 Jan 2020 00:20:27 +0100 Subject: [PATCH] Experimental wasm rebuild scripts --- .../docker-scripts/genbytecode.sh | 98 ++++++++ .../docker-scripts/isolate_tests.py | 55 ++++ scripts/wasm-rebuild/docker-scripts/patch.sh | 7 + .../docker-scripts/rebuild_current.sh | 67 +++++ .../docker-scripts/rebuild_tags.sh | 238 ++++++++++++++++++ scripts/wasm-rebuild/rebuild.sh | 28 +++ 6 files changed, 493 insertions(+) create mode 100755 scripts/wasm-rebuild/docker-scripts/genbytecode.sh create mode 100755 scripts/wasm-rebuild/docker-scripts/isolate_tests.py create mode 100755 scripts/wasm-rebuild/docker-scripts/patch.sh create mode 100755 scripts/wasm-rebuild/docker-scripts/rebuild_current.sh create mode 100755 scripts/wasm-rebuild/docker-scripts/rebuild_tags.sh create mode 100755 scripts/wasm-rebuild/rebuild.sh diff --git a/scripts/wasm-rebuild/docker-scripts/genbytecode.sh b/scripts/wasm-rebuild/docker-scripts/genbytecode.sh new file mode 100755 index 000000000..ef28d59a4 --- /dev/null +++ b/scripts/wasm-rebuild/docker-scripts/genbytecode.sh @@ -0,0 +1,98 @@ +#!/usr/bin/env bash + +#------------------------------------------------------------------------------ +# Script used for cross-platform comparison as part of the travis automation. +# Splits all test source code into multiple files, generates bytecode and +# uploads the bytecode into github.com/ethereum/solidity-test-bytecode where +# another travis job is triggered to do the actual comparison. +# +# ------------------------------------------------------------------------------ +# This file is part of solidity. +# +# solidity is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# solidity is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with solidity. If not, see +# +# (c) 2017 solidity contributors. +#------------------------------------------------------------------------------ + +set -e + +SCRIPTDIR=$(dirname "$0") +SCRIPTDIR=$(realpath "${SCRIPTDIR}") + + +echo "Compiling all test contracts into bytecode..." +TMPDIR=$(mktemp -d) +( + cd "${TMPDIR}" + "${SCRIPTDIR}/isolate_tests.py" /src/test/ + + cat > solc < /tmp/report.txt +) +rm -rf "$TMPDIR" diff --git a/scripts/wasm-rebuild/docker-scripts/isolate_tests.py b/scripts/wasm-rebuild/docker-scripts/isolate_tests.py new file mode 100755 index 000000000..eab461bff --- /dev/null +++ b/scripts/wasm-rebuild/docker-scripts/isolate_tests.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python2 + +import sys +import re +import os +import hashlib +from os.path import join, isfile + + +def extract_test_cases(path): + lines = open(path, 'rb').read().splitlines() + + inside = False + delimiter = '' + tests = [] + + for l in lines: + if inside: + if l.strip().endswith(')' + delimiter + '";'): + tests[-1] += l.strip()[:-(3 + len(delimiter))] + inside = False + else: + tests[-1] += l + '\n' + else: + m = re.search(r'R"([^(]*)\((.*)$', l.strip()) + if m: + inside = True + delimiter = m.group(1) + tests += [m.group(2)] + + return tests + +def extract_and_write(f, path): + if f.endswith('.sol'): + cases = [open(path, 'r').read()] + else: + cases = extract_test_cases(path) + write_cases(f, cases) + +def write_cases(f, tests): + cleaned_filename = f.replace(".","_").replace("-","_").replace(" ","_").lower() + for test in tests: + remainder = re.sub(r'^ {4}', '', test, 0, re.MULTILINE) + open('test_%s_%s.sol' % (hashlib.sha256(test).hexdigest(), cleaned_filename), 'w').write(remainder) + + +if __name__ == '__main__': + path = sys.argv[1] + + for root, subdirs, files in os.walk(path): + if '_build' in subdirs: + subdirs.remove('_build') + for f in files: + path = join(root, f) + extract_and_write(f, path) diff --git a/scripts/wasm-rebuild/docker-scripts/patch.sh b/scripts/wasm-rebuild/docker-scripts/patch.sh new file mode 100755 index 000000000..a03fc73d0 --- /dev/null +++ b/scripts/wasm-rebuild/docker-scripts/patch.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +TAG="$1" +SOLJSON_JS="$2" + +# If we ever want to patch the binaries e.g. for compatibility with older solc-js versions, +# we can do that here. diff --git a/scripts/wasm-rebuild/docker-scripts/rebuild_current.sh b/scripts/wasm-rebuild/docker-scripts/rebuild_current.sh new file mode 100755 index 000000000..b48b37fc5 --- /dev/null +++ b/scripts/wasm-rebuild/docker-scripts/rebuild_current.sh @@ -0,0 +1,67 @@ +#!/bin/bash -e + +# Do not call this script directly. + +# This script is expected to be run inside the docker image trzeci/emscripten:sdk-tag-1.39.3-64bit and +# be called by ./rebuild_tags.sh. + +echo "========== STAGE 1: PREPARE ========== ($(date))" +COMMIT_DATE="$(git show -s --format=%cI HEAD)" +git rev-parse --short=8 HEAD >commit_hash.txt +echo -e "" >prerelease.txt +sed -i -e 's/-Wl,--gc-sections//' cmake/EthCompilerSettings.cmake +echo "set(CMAKE_CXX_FLAGS \"\${CMAKE_CXX_FLAGS} -s EXTRA_EXPORTED_RUNTIME_METHODS=['cwrap','addFunction','removeFunction','UTF8ToString','lengthBytesUTF8','_malloc','stringToUTF8','setValue'] -s WASM=1 -s WASM_ASYNC_COMPILATION=0 -s SINGLE_FILE=1 -Wno-almost-asm\")" >>cmake/EthCompilerSettings.cmake +# Needed for < 0.5.0. +sed -i -e 's/-Werror/-Wno-error/' cmake/EthCompilerSettings.cmake + +echo "========== STAGE 2: BUILD ========== ($(date))" +scripts/travis-emscripten/install_deps.sh +if [ -d cryptopp ]; then + # Needed for < 0.4.4. Will not affect >= 0.4.5. + # Unfortunately we need to update to the latest + # release in the 5.6 series for it to build. + # Hopefully we don't miss any bugs. + rm -rf cryptopp + git clone https://github.com/weidai11/cryptopp/ + ( + set -e + cd cryptopp + git checkout CRYPTOPP_5_6_5 + ln -s . src + ) +fi +if [ -d jsoncpp ]; then + # Needed for < 0.4.4. Will not affect >= 0.4.5. + ( + set -e + cd jsoncpp + # Checkout the latest commit at the time of our release. + git checkout $(git rev-list -1 --before=$COMMIT_DATE master) + ) +fi + +set +e +scripts/travis-emscripten/build_emscripten.sh +set -e + +mkdir -p upload + +if [ ! -f upload/soljson.js ]; then + if [ -f build/solc/soljson.js ]; then + cp build/solc/soljson.js upload + elif [ -f build/libsolc/soljson.js ]; then + cp build/libsolc/soljson.js upload + elif [ -f emscripten_build/solc/soljson.js ]; then + cp emscripten_build/solc/soljson.js upload + elif [ -f emscripten_build/libsolc/soljson.js ]; then + cp emscripten_build/libsolc/soljson.js upload + fi +fi + +if [ -f upload/soljson.js ]; then + echo "========== SUCCESS ========== ($(date))" + exit 0 +else + echo "========== FAILURE ========== ($(date))" + exit 1 +fi diff --git a/scripts/wasm-rebuild/docker-scripts/rebuild_tags.sh b/scripts/wasm-rebuild/docker-scripts/rebuild_tags.sh new file mode 100755 index 000000000..6a3065bca --- /dev/null +++ b/scripts/wasm-rebuild/docker-scripts/rebuild_tags.sh @@ -0,0 +1,238 @@ +#!/bin/bash -e + +# This script is expected to be run inside the docker image trzeci/emscripten:sdk-tag-1.39.3-64bit. +# Its main purpose is to be called by ../rebuild.sh. + +# Usage: $0 [tagFilter] [outputDirectory] + +# The output directory must be outside the repository, +# since the script will prune the repository directory after +# each build. + +TAG_FILTER="$1" +OUTPUTDIR="$2" +RETEST=0 +shift +shift +while (( "$#" )); do + if [[ "$1" == "--retest" ]]; then + RETEST=1 + else + echo "Unrecognized option: $1" + exit 1 + fi + shift +done + +SOLIDITY_REPO_URL="https://github.com/ethereum/solidity" +SOLC_JS_REPO_URL="https://github.com/ethereum/solc-js" +SOLC_JS_BRANCH=wasmRebuildTests +RELEASE_URL="https://raw.githubusercontent.com/ethereum/solc-bin/gh-pages/bin" +RELEASE_COMMIT_LIST_URL="$RELEASE_URL/list.txt" + +SCRIPTDIR=$(dirname "$0") +SCRIPTDIR=$(realpath "${SCRIPTDIR}") +RED='\033[0;31m' +GREEN='\033[0;32m' +ORANGE='\033[0;33m' +CYAN='\033[0;36m' +RESET='\033[0m' + +function generate_bytecode_report() { + rm -rf /tmp/report.txt + + local EXIT_STATUS + + if semver -r "<0.4.12" $3 > /dev/null; then + set +e + "${SCRIPTDIR}/genbytecode.sh" "$1" >/dev/null 2>&1 + EXIT_STATUS=$? + set -e + else + set +e + ( + set -e + + git reset --hard HEAD --quiet + git clean -f -d -x --quiet + + for dir in build/solc build/libsolc emscripten_build/libsolc; do + mkdir -p $dir + rm -rf $dir/soljson.js + ln -sf "$1" $dir/soljson.js + done + + /tmp/storebytecode.sh >/dev/null 2>&1 + ) + EXIT_STATUS=$? + fi + + if [ $EXIT_STATUS -eq 0 ] && [ -f /tmp/report.txt ] && grep -q -v -c -e "ERROR" -e "NO BYTECODE" /tmp/report.txt; then + mv /tmp/report.txt "$2" + echo -e "${GREEN}SUCCESS${RESET}" + else + echo -e "${RED}FAILURE${RESET}" + fi +} +function clean_git_checkout() { + git submodule deinit --all -q + git reset --hard HEAD --quiet + git clean -f -d -x --quiet + git checkout "$1" --quiet + git submodule init -q + git submodule update -q +} +function process_tag() { + local TAG=$1 + cd /src + # Checkout the historic commit instead of the tag directly. + local HISTORIC_COMMIT_HASH="$(grep "${TAG}+" /tmp/release_commit_list.txt | cut -d '+' -f 2 | cut -d '.' -f 2)" + if [ "$(git cat-file -t ${HISTORIC_COMMIT_HASH} 2>/dev/null)" == "commit" ]; then + clean_git_checkout "$HISTORIC_COMMIT_HASH" + else + clean_git_checkout "${TAG}" + fi + + # compatibility symlink + ln -s . solidity + + local VERSION + if [ -f ./scripts/get_version.sh ]; then + VERSION=$(./scripts/get_version.sh) + else + VERSION=$(echo "$TAG" | cut -d v -f 2) + fi + + local COMMIT_HASH=$(git rev-parse --short=8 HEAD) + local FULL_VERSION_SUFFIX="${TAG}+commit.${COMMIT_HASH}" + local HISTORIC_VERSION_SUFFIX="${TAG}+commit.${HISTORIC_COMMIT_HASH}" + + if [ ! -f "${OUTPUTDIR}/bin/soljson-${FULL_VERSION_SUFFIX}.js" ]; then + echo -ne "BUILDING ${CYAN}${TAG}${RESET}... " + set +e + ( + set -e + "${SCRIPTDIR}/rebuild_current.sh" "${VERSION}" >"${OUTPUTDIR}/log/running/build-$TAG.txt" 2>&1 + "${SCRIPTDIR}/patch.sh" "$TAG" upload/soljson.js + cp upload/soljson.js "${OUTPUTDIR}/bin/soljson-${FULL_VERSION_SUFFIX}.js" + rm upload/soljson.js + ) + local EXIT_STATUS=$? + set -e + rm -f "${OUTPUTDIR}/log/success/build-$TAG.txt" + rm -f "${OUTPUTDIR}/log/fail/build-$TAG.txt" + if [ $EXIT_STATUS -eq 0 ]; then + mv "${OUTPUTDIR}/log/running/build-$TAG.txt" "${OUTPUTDIR}/log/success" + echo -e "${GREEN}SUCCESS${RESET}" + else + mv "${OUTPUTDIR}/log/running/build-$TAG.txt" "${OUTPUTDIR}/log/fail" + echo -e "${RED}FAIL${RESET}" + fi + else + echo -e "${CYAN}${TAG}${RESET} ALREADY EXISTS." + if [ $RETEST -eq 0 ]; then + return 0 + fi + fi + + if [ -f "${OUTPUTDIR}/bin/soljson-${FULL_VERSION_SUFFIX}.js" ]; then + + echo -ne "GENERATE BYTECODE REPORT FOR ${CYAN}${TAG}${RESET}... " + generate_bytecode_report "${OUTPUTDIR}/bin/soljson-${FULL_VERSION_SUFFIX}.js" "${OUTPUTDIR}"/log/reports/report-${TAG}.txt "${TAG}" + echo -ne "GENERATE BYTECODE REPORT FOR HISTORIC ${CYAN}${TAG}${RESET}... " + rm -rf /tmp/soljson.js + if wget -q "$RELEASE_URL/soljson-${HISTORIC_VERSION_SUFFIX}.js" -O /tmp/soljson.js; then + generate_bytecode_report /tmp/soljson.js "${OUTPUTDIR}"/log/reports/report-historic-${TAG}.txt "${TAG}" + else + echo -e "${ORANGE}CANNOT FETCH RELEASE${RESET}" + fi + rm -rf /tmp/soljson.js + + if [ -f "${OUTPUTDIR}/log/reports/report-${TAG}.txt" ] && [ -f "${OUTPUTDIR}/log/reports/report-historic-${TAG}.txt" ]; then + rm -rf "${OUTPUTDIR}/log/success/bytecode-${TAG}.txt" + rm -rf "${OUTPUTDIR}/log/fail/bytecode-${TAG}.txt" + if diff -q "${OUTPUTDIR}/log/reports/report-${TAG}.txt" "${OUTPUTDIR}/log/reports/report-historic-${TAG}.txt" >/dev/null 2>&1; then + echo -e "${GREEN}BYTECODE MATCHES FOR ${CYAN}${TAG}${RESET}" + grep -v -c -e "ERROR" -e "NO BYTECODE" "${OUTPUTDIR}/log/reports/report-${TAG}.txt" >"${OUTPUTDIR}/log/success/bytecode-${TAG}.txt" + else + echo -e "${RED}BYTECODE DOES NOT MATCH FOR ${CYAN}${TAG}${RESET}" + echo "MISMATCH" >"${OUTPUTDIR}/log/fail/bytecode-${TAG}.txt" + fi + fi + + echo -ne "TESTING ${CYAN}${TAG}${RESET}... " + cd /root/solc-js + npm version --allow-same-version --no-git-tag-version "${VERSION}" >/dev/null + sed -i -e "s/runTests(solc, .*)/runTests(solc, '${FULL_VERSION_SUFFIX}')/" test/compiler.js + ln -sf "${OUTPUTDIR}/bin/soljson-${FULL_VERSION_SUFFIX}.js" soljson.js + rm -f "${OUTPUTDIR}/log/success/test-$TAG.txt" + rm -f "${OUTPUTDIR}/log/fail/test-$TAG.txt" + if npm test >"${OUTPUTDIR}/log/running/test-$TAG.txt" 2>&1; then + mv "${OUTPUTDIR}/log/running/test-$TAG.txt" "${OUTPUTDIR}/log/success" + echo -e "${GREEN}SUCCESS${RESET}" + else + mv "${OUTPUTDIR}/log/running/test-$TAG.txt" "${OUTPUTDIR}/log/fail" + echo -e "${RED}FAIL${RESET}" + fi + fi +} + +cd /tmp + +echo "Check out solidity repository..." +if [ -d /root/project ]; then + echo "Solidity repo checkout already exists." +else + git clone "${SOLIDITY_REPO_URL}" /root/project --quiet +fi + +echo "Extract bytecode comparison scripts from v0.6.1..." +cd /root/project +git checkout v0.6.1 --quiet +cp scripts/bytecodecompare/storebytecode.sh /tmp +sed -i -e 's/rm -rf "\$TMPDIR"/cp "\$TMPDIR"\/report.txt \/tmp\/report.txt ; rm -rf "\$TMPDIR"/' /tmp/storebytecode.sh +sed -i -e 's/REPO_ROOT=.*/REPO_ROOT=\/src/' /tmp/storebytecode.sh +export SOLC_EMSCRIPTEN="On" + +echo "Check out solc-js repository..." +if [ -d /root/solc-js ]; then + echo "solc-js repo checkout already exists." +else + git clone --branch "${SOLC_JS_BRANCH}" "${SOLC_JS_REPO_URL}" /root/solc-js --quiet +fi + +echo "Create symbolic links for backwards compatibility with older emscripten docker images." +ln -sf /emsdk_portable/node/current/* /emsdk_portable/node/ +ln -sf /emsdk_portable/emscripten/sdk/ /emsdk_portable/ +ln -sf sdk /emsdk_portable/emscripten/bin +ln -sf /emsdk_portable/emscripten/bin/* /usr/local/bin +rm -rf /src +ln -sf /root/project /src + +apt-get -qq update >/dev/null 2>&1 +apt-get -qq install cmake >/dev/null 2>&1 + +echo "Create output directories." +mkdir -p "${OUTPUTDIR}" +mkdir -p "${OUTPUTDIR}"/log +mkdir -p "${OUTPUTDIR}"/log/success +mkdir -p "${OUTPUTDIR}"/log/fail +mkdir -p "${OUTPUTDIR}"/log/running +mkdir -p "${OUTPUTDIR}"/log/reports +mkdir -p "${OUTPUTDIR}"/bin + +echo "Prepare solc-js." +cd /root/solc-js +npm install >/dev/null 2>&1 + +echo "Install semver helper." +npm install -g semver >/dev/null 2>&1 + +echo "Fetching release commit list." +wget -q "${RELEASE_COMMIT_LIST_URL}" -O /tmp/release_commit_list.txt + +cd /src +TAGS=$(git tag --list "${TAG_FILTER}" | tac) +for TAG in ${TAGS}; do + process_tag "${TAG}" +done diff --git a/scripts/wasm-rebuild/rebuild.sh b/scripts/wasm-rebuild/rebuild.sh new file mode 100755 index 000000000..6ac275764 --- /dev/null +++ b/scripts/wasm-rebuild/rebuild.sh @@ -0,0 +1,28 @@ +#!/bin/sh + +# This script is expected to produce working builds for all compiler versions >= 0.3.6 and +# succeeding solc-js test runs for all compiler versions >= 0.5.0. + +if [ $# -lt 2 ]; then + echo "Usage: $0 [tagFilter] [outputDirectory] [options...]" + echo + echo " [tagFilter] will be passed to "git tag --list" to filter the tags to be built." + echo " [outputDirectory] will contain log files and the resulting soljson.js builds." + echo " --retest will re-run tests and bytecode comparisons, even if soljson.js is already built." + exit 1 +fi + +TAGS="$1" +OUTPUTDIR="$2" +shift +shift +SCRIPTDIR=$(dirname "$0") +SCRIPTDIR=$(realpath "${SCRIPTDIR}") + +if [ ! -d "${OUTPUTDIR}" ]; then + echo "Output directory ${OUTPUTDIR} does not exist!." + exit 1 +fi +OUTPUTDIR=$(realpath "${OUTPUTDIR}") + +docker run --rm -v "${OUTPUTDIR}":/tmp/output -v "${SCRIPTDIR}":/tmp/scripts:ro -it trzeci/emscripten:sdk-tag-1.39.3-64bit /tmp/scripts/docker-scripts/rebuild_tags.sh "${TAGS}" /tmp/output $@