Experimental wasm rebuild scripts

This commit is contained in:
Daniel Kirchner 2020-01-09 00:20:27 +01:00
parent 3d4a2219a6
commit 22466acf39
6 changed files with 493 additions and 0 deletions

View File

@ -0,0 +1,98 @@
#!/usr/bin/env bash
#------------------------------------------------------------------------------
# Script used for cross-platform comparison as part of the travis automation.
# Splits all test source code into multiple files, generates bytecode and
# uploads the bytecode into github.com/ethereum/solidity-test-bytecode where
# another travis job is triggered to do the actual comparison.
#
# ------------------------------------------------------------------------------
# This file is part of solidity.
#
# solidity is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# solidity is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with solidity. If not, see <http://www.gnu.org/licenses/>
#
# (c) 2017 solidity contributors.
#------------------------------------------------------------------------------
set -e
SCRIPTDIR=$(dirname "$0")
SCRIPTDIR=$(realpath "${SCRIPTDIR}")
echo "Compiling all test contracts into bytecode..."
TMPDIR=$(mktemp -d)
(
cd "${TMPDIR}"
"${SCRIPTDIR}/isolate_tests.py" /src/test/
cat > solc <<EOF
#!/usr/bin/env node
var process = require('process')
var fs = require('fs')
var compiler = require('/root/solc-js/wrapper.js')(require("${1}"))
for (var optimize of [false, true])
{
for (var filename of process.argv.slice(2))
{
if (filename !== undefined)
{
var inputs = {}
inputs[filename] = { content: fs.readFileSync(filename).toString() }
var input = {
language: 'Solidity',
sources: inputs,
settings: {
optimizer: { enabled: optimize },
outputSelection: { '*': { '*': ['*'] } }
}
}
try {
var result = JSON.parse(compiler.compile(JSON.stringify(input)))
if (
!('contracts' in result) ||
Object.keys(result['contracts']).length === 0
)
{
// NOTE: do not exit here because this may be run on source which cannot be compiled
console.log(filename + ': ERROR')
}
else
{
for (var outputName in result['contracts'])
for (var contractName in result['contracts'][outputName])
{
var contractData = result['contracts'][outputName][contractName];
if (contractData.evm !== undefined && contractData.evm.bytecode !== undefined)
console.log(filename + ':' + contractName + ' ' + contractData.evm.bytecode.object)
else
console.log(filename + ':' + contractName + ' NO BYTECODE')
console.log(filename + ':' + contractName + ' ' + contractData.metadata)
}
}
} catch (e) {
console.log(filename + ': FATAL ERROR')
console.error(filename)
console.error(inputs)
}
}
}
}
EOF
chmod +x solc
./solc *.sol > /tmp/report.txt
)
rm -rf "$TMPDIR"

View File

@ -0,0 +1,55 @@
#!/usr/bin/env python2
import sys
import re
import os
import hashlib
from os.path import join, isfile
def extract_test_cases(path):
lines = open(path, 'rb').read().splitlines()
inside = False
delimiter = ''
tests = []
for l in lines:
if inside:
if l.strip().endswith(')' + delimiter + '";'):
tests[-1] += l.strip()[:-(3 + len(delimiter))]
inside = False
else:
tests[-1] += l + '\n'
else:
m = re.search(r'R"([^(]*)\((.*)$', l.strip())
if m:
inside = True
delimiter = m.group(1)
tests += [m.group(2)]
return tests
def extract_and_write(f, path):
if f.endswith('.sol'):
cases = [open(path, 'r').read()]
else:
cases = extract_test_cases(path)
write_cases(f, cases)
def write_cases(f, tests):
cleaned_filename = f.replace(".","_").replace("-","_").replace(" ","_").lower()
for test in tests:
remainder = re.sub(r'^ {4}', '', test, 0, re.MULTILINE)
open('test_%s_%s.sol' % (hashlib.sha256(test).hexdigest(), cleaned_filename), 'w').write(remainder)
if __name__ == '__main__':
path = sys.argv[1]
for root, subdirs, files in os.walk(path):
if '_build' in subdirs:
subdirs.remove('_build')
for f in files:
path = join(root, f)
extract_and_write(f, path)

View File

@ -0,0 +1,7 @@
#!/bin/bash
TAG="$1"
SOLJSON_JS="$2"
# If we ever want to patch the binaries e.g. for compatibility with older solc-js versions,
# we can do that here.

View File

@ -0,0 +1,67 @@
#!/bin/bash -e
# Do not call this script directly.
# This script is expected to be run inside the docker image trzeci/emscripten:sdk-tag-1.39.3-64bit and
# be called by ./rebuild_tags.sh.
echo "========== STAGE 1: PREPARE ========== ($(date))"
COMMIT_DATE="$(git show -s --format=%cI HEAD)"
git rev-parse --short=8 HEAD >commit_hash.txt
echo -e "" >prerelease.txt
sed -i -e 's/-Wl,--gc-sections//' cmake/EthCompilerSettings.cmake
echo "set(CMAKE_CXX_FLAGS \"\${CMAKE_CXX_FLAGS} -s EXTRA_EXPORTED_RUNTIME_METHODS=['cwrap','addFunction','removeFunction','UTF8ToString','lengthBytesUTF8','_malloc','stringToUTF8','setValue'] -s WASM=1 -s WASM_ASYNC_COMPILATION=0 -s SINGLE_FILE=1 -Wno-almost-asm\")" >>cmake/EthCompilerSettings.cmake
# Needed for < 0.5.0.
sed -i -e 's/-Werror/-Wno-error/' cmake/EthCompilerSettings.cmake
echo "========== STAGE 2: BUILD ========== ($(date))"
scripts/travis-emscripten/install_deps.sh
if [ -d cryptopp ]; then
# Needed for < 0.4.4. Will not affect >= 0.4.5.
# Unfortunately we need to update to the latest
# release in the 5.6 series for it to build.
# Hopefully we don't miss any bugs.
rm -rf cryptopp
git clone https://github.com/weidai11/cryptopp/
(
set -e
cd cryptopp
git checkout CRYPTOPP_5_6_5
ln -s . src
)
fi
if [ -d jsoncpp ]; then
# Needed for < 0.4.4. Will not affect >= 0.4.5.
(
set -e
cd jsoncpp
# Checkout the latest commit at the time of our release.
git checkout $(git rev-list -1 --before=$COMMIT_DATE master)
)
fi
set +e
scripts/travis-emscripten/build_emscripten.sh
set -e
mkdir -p upload
if [ ! -f upload/soljson.js ]; then
if [ -f build/solc/soljson.js ]; then
cp build/solc/soljson.js upload
elif [ -f build/libsolc/soljson.js ]; then
cp build/libsolc/soljson.js upload
elif [ -f emscripten_build/solc/soljson.js ]; then
cp emscripten_build/solc/soljson.js upload
elif [ -f emscripten_build/libsolc/soljson.js ]; then
cp emscripten_build/libsolc/soljson.js upload
fi
fi
if [ -f upload/soljson.js ]; then
echo "========== SUCCESS ========== ($(date))"
exit 0
else
echo "========== FAILURE ========== ($(date))"
exit 1
fi

View File

@ -0,0 +1,238 @@
#!/bin/bash -e
# This script is expected to be run inside the docker image trzeci/emscripten:sdk-tag-1.39.3-64bit.
# Its main purpose is to be called by ../rebuild.sh.
# Usage: $0 [tagFilter] [outputDirectory]
# The output directory must be outside the repository,
# since the script will prune the repository directory after
# each build.
TAG_FILTER="$1"
OUTPUTDIR="$2"
RETEST=0
shift
shift
while (( "$#" )); do
if [[ "$1" == "--retest" ]]; then
RETEST=1
else
echo "Unrecognized option: $1"
exit 1
fi
shift
done
SOLIDITY_REPO_URL="https://github.com/ethereum/solidity"
SOLC_JS_REPO_URL="https://github.com/ethereum/solc-js"
SOLC_JS_BRANCH=wasmRebuildTests
RELEASE_URL="https://raw.githubusercontent.com/ethereum/solc-bin/gh-pages/bin"
RELEASE_COMMIT_LIST_URL="$RELEASE_URL/list.txt"
SCRIPTDIR=$(dirname "$0")
SCRIPTDIR=$(realpath "${SCRIPTDIR}")
RED='\033[0;31m'
GREEN='\033[0;32m'
ORANGE='\033[0;33m'
CYAN='\033[0;36m'
RESET='\033[0m'
function generate_bytecode_report() {
rm -rf /tmp/report.txt
local EXIT_STATUS
if semver -r "<0.4.12" $3 > /dev/null; then
set +e
"${SCRIPTDIR}/genbytecode.sh" "$1" >/dev/null 2>&1
EXIT_STATUS=$?
set -e
else
set +e
(
set -e
git reset --hard HEAD --quiet
git clean -f -d -x --quiet
for dir in build/solc build/libsolc emscripten_build/libsolc; do
mkdir -p $dir
rm -rf $dir/soljson.js
ln -sf "$1" $dir/soljson.js
done
/tmp/storebytecode.sh >/dev/null 2>&1
)
EXIT_STATUS=$?
fi
if [ $EXIT_STATUS -eq 0 ] && [ -f /tmp/report.txt ] && grep -q -v -c -e "ERROR" -e "NO BYTECODE" /tmp/report.txt; then
mv /tmp/report.txt "$2"
echo -e "${GREEN}SUCCESS${RESET}"
else
echo -e "${RED}FAILURE${RESET}"
fi
}
function clean_git_checkout() {
git submodule deinit --all -q
git reset --hard HEAD --quiet
git clean -f -d -x --quiet
git checkout "$1" --quiet
git submodule init -q
git submodule update -q
}
function process_tag() {
local TAG=$1
cd /src
# Checkout the historic commit instead of the tag directly.
local HISTORIC_COMMIT_HASH="$(grep "${TAG}+" /tmp/release_commit_list.txt | cut -d '+' -f 2 | cut -d '.' -f 2)"
if [ "$(git cat-file -t ${HISTORIC_COMMIT_HASH} 2>/dev/null)" == "commit" ]; then
clean_git_checkout "$HISTORIC_COMMIT_HASH"
else
clean_git_checkout "${TAG}"
fi
# compatibility symlink
ln -s . solidity
local VERSION
if [ -f ./scripts/get_version.sh ]; then
VERSION=$(./scripts/get_version.sh)
else
VERSION=$(echo "$TAG" | cut -d v -f 2)
fi
local COMMIT_HASH=$(git rev-parse --short=8 HEAD)
local FULL_VERSION_SUFFIX="${TAG}+commit.${COMMIT_HASH}"
local HISTORIC_VERSION_SUFFIX="${TAG}+commit.${HISTORIC_COMMIT_HASH}"
if [ ! -f "${OUTPUTDIR}/bin/soljson-${FULL_VERSION_SUFFIX}.js" ]; then
echo -ne "BUILDING ${CYAN}${TAG}${RESET}... "
set +e
(
set -e
"${SCRIPTDIR}/rebuild_current.sh" "${VERSION}" >"${OUTPUTDIR}/log/running/build-$TAG.txt" 2>&1
"${SCRIPTDIR}/patch.sh" "$TAG" upload/soljson.js
cp upload/soljson.js "${OUTPUTDIR}/bin/soljson-${FULL_VERSION_SUFFIX}.js"
rm upload/soljson.js
)
local EXIT_STATUS=$?
set -e
rm -f "${OUTPUTDIR}/log/success/build-$TAG.txt"
rm -f "${OUTPUTDIR}/log/fail/build-$TAG.txt"
if [ $EXIT_STATUS -eq 0 ]; then
mv "${OUTPUTDIR}/log/running/build-$TAG.txt" "${OUTPUTDIR}/log/success"
echo -e "${GREEN}SUCCESS${RESET}"
else
mv "${OUTPUTDIR}/log/running/build-$TAG.txt" "${OUTPUTDIR}/log/fail"
echo -e "${RED}FAIL${RESET}"
fi
else
echo -e "${CYAN}${TAG}${RESET} ALREADY EXISTS."
if [ $RETEST -eq 0 ]; then
return 0
fi
fi
if [ -f "${OUTPUTDIR}/bin/soljson-${FULL_VERSION_SUFFIX}.js" ]; then
echo -ne "GENERATE BYTECODE REPORT FOR ${CYAN}${TAG}${RESET}... "
generate_bytecode_report "${OUTPUTDIR}/bin/soljson-${FULL_VERSION_SUFFIX}.js" "${OUTPUTDIR}"/log/reports/report-${TAG}.txt "${TAG}"
echo -ne "GENERATE BYTECODE REPORT FOR HISTORIC ${CYAN}${TAG}${RESET}... "
rm -rf /tmp/soljson.js
if wget -q "$RELEASE_URL/soljson-${HISTORIC_VERSION_SUFFIX}.js" -O /tmp/soljson.js; then
generate_bytecode_report /tmp/soljson.js "${OUTPUTDIR}"/log/reports/report-historic-${TAG}.txt "${TAG}"
else
echo -e "${ORANGE}CANNOT FETCH RELEASE${RESET}"
fi
rm -rf /tmp/soljson.js
if [ -f "${OUTPUTDIR}/log/reports/report-${TAG}.txt" ] && [ -f "${OUTPUTDIR}/log/reports/report-historic-${TAG}.txt" ]; then
rm -rf "${OUTPUTDIR}/log/success/bytecode-${TAG}.txt"
rm -rf "${OUTPUTDIR}/log/fail/bytecode-${TAG}.txt"
if diff -q "${OUTPUTDIR}/log/reports/report-${TAG}.txt" "${OUTPUTDIR}/log/reports/report-historic-${TAG}.txt" >/dev/null 2>&1; then
echo -e "${GREEN}BYTECODE MATCHES FOR ${CYAN}${TAG}${RESET}"
grep -v -c -e "ERROR" -e "NO BYTECODE" "${OUTPUTDIR}/log/reports/report-${TAG}.txt" >"${OUTPUTDIR}/log/success/bytecode-${TAG}.txt"
else
echo -e "${RED}BYTECODE DOES NOT MATCH FOR ${CYAN}${TAG}${RESET}"
echo "MISMATCH" >"${OUTPUTDIR}/log/fail/bytecode-${TAG}.txt"
fi
fi
echo -ne "TESTING ${CYAN}${TAG}${RESET}... "
cd /root/solc-js
npm version --allow-same-version --no-git-tag-version "${VERSION}" >/dev/null
sed -i -e "s/runTests(solc, .*)/runTests(solc, '${FULL_VERSION_SUFFIX}')/" test/compiler.js
ln -sf "${OUTPUTDIR}/bin/soljson-${FULL_VERSION_SUFFIX}.js" soljson.js
rm -f "${OUTPUTDIR}/log/success/test-$TAG.txt"
rm -f "${OUTPUTDIR}/log/fail/test-$TAG.txt"
if npm test >"${OUTPUTDIR}/log/running/test-$TAG.txt" 2>&1; then
mv "${OUTPUTDIR}/log/running/test-$TAG.txt" "${OUTPUTDIR}/log/success"
echo -e "${GREEN}SUCCESS${RESET}"
else
mv "${OUTPUTDIR}/log/running/test-$TAG.txt" "${OUTPUTDIR}/log/fail"
echo -e "${RED}FAIL${RESET}"
fi
fi
}
cd /tmp
echo "Check out solidity repository..."
if [ -d /root/project ]; then
echo "Solidity repo checkout already exists."
else
git clone "${SOLIDITY_REPO_URL}" /root/project --quiet
fi
echo "Extract bytecode comparison scripts from v0.6.1..."
cd /root/project
git checkout v0.6.1 --quiet
cp scripts/bytecodecompare/storebytecode.sh /tmp
sed -i -e 's/rm -rf "\$TMPDIR"/cp "\$TMPDIR"\/report.txt \/tmp\/report.txt ; rm -rf "\$TMPDIR"/' /tmp/storebytecode.sh
sed -i -e 's/REPO_ROOT=.*/REPO_ROOT=\/src/' /tmp/storebytecode.sh
export SOLC_EMSCRIPTEN="On"
echo "Check out solc-js repository..."
if [ -d /root/solc-js ]; then
echo "solc-js repo checkout already exists."
else
git clone --branch "${SOLC_JS_BRANCH}" "${SOLC_JS_REPO_URL}" /root/solc-js --quiet
fi
echo "Create symbolic links for backwards compatibility with older emscripten docker images."
ln -sf /emsdk_portable/node/current/* /emsdk_portable/node/
ln -sf /emsdk_portable/emscripten/sdk/ /emsdk_portable/
ln -sf sdk /emsdk_portable/emscripten/bin
ln -sf /emsdk_portable/emscripten/bin/* /usr/local/bin
rm -rf /src
ln -sf /root/project /src
apt-get -qq update >/dev/null 2>&1
apt-get -qq install cmake >/dev/null 2>&1
echo "Create output directories."
mkdir -p "${OUTPUTDIR}"
mkdir -p "${OUTPUTDIR}"/log
mkdir -p "${OUTPUTDIR}"/log/success
mkdir -p "${OUTPUTDIR}"/log/fail
mkdir -p "${OUTPUTDIR}"/log/running
mkdir -p "${OUTPUTDIR}"/log/reports
mkdir -p "${OUTPUTDIR}"/bin
echo "Prepare solc-js."
cd /root/solc-js
npm install >/dev/null 2>&1
echo "Install semver helper."
npm install -g semver >/dev/null 2>&1
echo "Fetching release commit list."
wget -q "${RELEASE_COMMIT_LIST_URL}" -O /tmp/release_commit_list.txt
cd /src
TAGS=$(git tag --list "${TAG_FILTER}" | tac)
for TAG in ${TAGS}; do
process_tag "${TAG}"
done

28
scripts/wasm-rebuild/rebuild.sh Executable file
View File

@ -0,0 +1,28 @@
#!/bin/sh
# This script is expected to produce working builds for all compiler versions >= 0.3.6 and
# succeeding solc-js test runs for all compiler versions >= 0.5.0.
if [ $# -lt 2 ]; then
echo "Usage: $0 [tagFilter] [outputDirectory] [options...]"
echo
echo " [tagFilter] will be passed to "git tag --list" to filter the tags to be built."
echo " [outputDirectory] will contain log files and the resulting soljson.js builds."
echo " --retest will re-run tests and bytecode comparisons, even if soljson.js is already built."
exit 1
fi
TAGS="$1"
OUTPUTDIR="$2"
shift
shift
SCRIPTDIR=$(dirname "$0")
SCRIPTDIR=$(realpath "${SCRIPTDIR}")
if [ ! -d "${OUTPUTDIR}" ]; then
echo "Output directory ${OUTPUTDIR} does not exist!."
exit 1
fi
OUTPUTDIR=$(realpath "${OUTPUTDIR}")
docker run --rm -v "${OUTPUTDIR}":/tmp/output -v "${SCRIPTDIR}":/tmp/scripts:ro -it trzeci/emscripten:sdk-tag-1.39.3-64bit /tmp/scripts/docker-scripts/rebuild_tags.sh "${TAGS}" /tmp/output $@