Merge pull request #8118 from ethereum/wasmRebuilds

Experimental wasm soljson.js rebuild scripts
This commit is contained in:
chriseth 2020-02-14 13:00:33 +01:00 committed by GitHub
commit 2917cf4bbc
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 497 additions and 0 deletions

View File

@ -0,0 +1,4 @@
This directory contains scripts that we used to rebuild historic releases to webassembly.
The rebuild took place at the time of Solidity version 0.6.1 and was used to rebuild all versions from 0.3.6 to 0.6.1.
The scripts are not actively tested or maintained, but we keep them for future reference in case we want to rebuild
historic releases again at a later point.

View File

@ -0,0 +1,98 @@
#!/usr/bin/env bash
#------------------------------------------------------------------------------
# Script used for cross-platform comparison as part of the travis automation.
# Splits all test source code into multiple files, generates bytecode and
# uploads the bytecode into github.com/ethereum/solidity-test-bytecode where
# another travis job is triggered to do the actual comparison.
#
# ------------------------------------------------------------------------------
# This file is part of solidity.
#
# solidity is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# solidity is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with solidity. If not, see <http://www.gnu.org/licenses/>
#
# (c) 2017 solidity contributors.
#------------------------------------------------------------------------------
set -e
SCRIPTDIR=$(dirname "$0")
SCRIPTDIR=$(realpath "${SCRIPTDIR}")
echo "Compiling all test contracts into bytecode..."
TMPDIR=$(mktemp -d)
(
cd "${TMPDIR}"
"${SCRIPTDIR}/isolate_tests.py" /src/test/
cat > solc <<EOF
#!/usr/bin/env node
var process = require('process')
var fs = require('fs')
var compiler = require('/root/solc-js/wrapper.js')(require("${1}"))
for (var optimize of [false, true])
{
for (var filename of process.argv.slice(2))
{
if (filename !== undefined)
{
var inputs = {}
inputs[filename] = { content: fs.readFileSync(filename).toString() }
var input = {
language: 'Solidity',
sources: inputs,
settings: {
optimizer: { enabled: optimize },
outputSelection: { '*': { '*': ['*'] } }
}
}
try {
var result = JSON.parse(compiler.compile(JSON.stringify(input)))
if (
!('contracts' in result) ||
Object.keys(result['contracts']).length === 0
)
{
// NOTE: do not exit here because this may be run on source which cannot be compiled
console.log(filename + ': ERROR')
}
else
{
for (var outputName in result['contracts'])
for (var contractName in result['contracts'][outputName])
{
var contractData = result['contracts'][outputName][contractName];
if (contractData.evm !== undefined && contractData.evm.bytecode !== undefined)
console.log(filename + ':' + contractName + ' ' + contractData.evm.bytecode.object)
else
console.log(filename + ':' + contractName + ' NO BYTECODE')
console.log(filename + ':' + contractName + ' ' + contractData.metadata)
}
}
} catch (e) {
console.log(filename + ': FATAL ERROR')
console.error(filename)
console.error(inputs)
}
}
}
}
EOF
chmod +x solc
./solc *.sol > /tmp/report.txt
)
rm -rf "$TMPDIR"

View File

@ -0,0 +1,55 @@
#!/usr/bin/env python2
import sys
import re
import os
import hashlib
from os.path import join, isfile
def extract_test_cases(path):
lines = open(path, 'rb').read().splitlines()
inside = False
delimiter = ''
tests = []
for l in lines:
if inside:
if l.strip().endswith(')' + delimiter + '";'):
tests[-1] += l.strip()[:-(3 + len(delimiter))]
inside = False
else:
tests[-1] += l + '\n'
else:
m = re.search(r'R"([^(]*)\((.*)$', l.strip())
if m:
inside = True
delimiter = m.group(1)
tests += [m.group(2)]
return tests
def extract_and_write(f, path):
if f.endswith('.sol'):
cases = [open(path, 'r').read()]
else:
cases = extract_test_cases(path)
write_cases(f, cases)
def write_cases(f, tests):
cleaned_filename = f.replace(".","_").replace("-","_").replace(" ","_").lower()
for test in tests:
remainder = re.sub(r'^ {4}', '', test, 0, re.MULTILINE)
open('test_%s_%s.sol' % (hashlib.sha256(test).hexdigest(), cleaned_filename), 'w').write(remainder)
if __name__ == '__main__':
path = sys.argv[1]
for root, subdirs, files in os.walk(path):
if '_build' in subdirs:
subdirs.remove('_build')
for f in files:
path = join(root, f)
extract_and_write(f, path)

View File

@ -0,0 +1,7 @@
#!/bin/bash
TAG="$1"
SOLJSON_JS="$2"
# If we ever want to patch the binaries e.g. for compatibility with older solc-js versions,
# we can do that here.

View File

@ -0,0 +1,67 @@
#!/bin/bash -e
# Do not call this script directly.
# This script is expected to be run inside the docker image trzeci/emscripten:sdk-tag-1.39.3-64bit and
# be called by ./rebuild_tags.sh.
echo "========== STAGE 1: PREPARE ========== ($(date))"
COMMIT_DATE="$(git show -s --format=%cI HEAD)"
git rev-parse --short=8 HEAD >commit_hash.txt
echo -e "" >prerelease.txt
sed -i -e 's/-Wl,--gc-sections//' cmake/EthCompilerSettings.cmake
echo "set(CMAKE_CXX_FLAGS \"\${CMAKE_CXX_FLAGS} -s EXTRA_EXPORTED_RUNTIME_METHODS=['cwrap','addFunction','removeFunction','UTF8ToString','lengthBytesUTF8','_malloc','stringToUTF8','setValue'] -s WASM=1 -s WASM_ASYNC_COMPILATION=0 -s SINGLE_FILE=1 -Wno-almost-asm\")" >>cmake/EthCompilerSettings.cmake
# Needed for < 0.5.0.
sed -i -e 's/-Werror/-Wno-error/' cmake/EthCompilerSettings.cmake
echo "========== STAGE 2: BUILD ========== ($(date))"
scripts/travis-emscripten/install_deps.sh
if [ -d cryptopp ]; then
# Needed for < 0.4.4. Will not affect >= 0.4.5.
# Unfortunately we need to update to the latest
# release in the 5.6 series for it to build.
# Hopefully we don't miss any bugs.
rm -rf cryptopp
git clone https://github.com/weidai11/cryptopp/
(
set -e
cd cryptopp
git checkout CRYPTOPP_5_6_5
ln -s . src
)
fi
if [ -d jsoncpp ]; then
# Needed for < 0.4.4. Will not affect >= 0.4.5.
(
set -e
cd jsoncpp
# Checkout the latest commit at the time of our release.
git checkout $(git rev-list -1 --before=$COMMIT_DATE master)
)
fi
set +e
scripts/travis-emscripten/build_emscripten.sh
set -e
mkdir -p upload
if [ ! -f upload/soljson.js ]; then
if [ -f build/solc/soljson.js ]; then
cp build/solc/soljson.js upload
elif [ -f build/libsolc/soljson.js ]; then
cp build/libsolc/soljson.js upload
elif [ -f emscripten_build/solc/soljson.js ]; then
cp emscripten_build/solc/soljson.js upload
elif [ -f emscripten_build/libsolc/soljson.js ]; then
cp emscripten_build/libsolc/soljson.js upload
fi
fi
if [ -f upload/soljson.js ]; then
echo "========== SUCCESS ========== ($(date))"
exit 0
else
echo "========== FAILURE ========== ($(date))"
exit 1
fi

View File

@ -0,0 +1,238 @@
#!/bin/bash -e
# This script is expected to be run inside the docker image trzeci/emscripten:sdk-tag-1.39.3-64bit.
# Its main purpose is to be called by ../rebuild.sh.
# Usage: $0 [tagFilter] [outputDirectory]
# The output directory must be outside the repository,
# since the script will prune the repository directory after
# each build.
TAG_FILTER="$1"
OUTPUTDIR="$2"
RETEST=0
shift
shift
while (( "$#" )); do
if [[ "$1" == "--retest" ]]; then
RETEST=1
else
echo "Unrecognized option: $1"
exit 1
fi
shift
done
SOLIDITY_REPO_URL="https://github.com/ethereum/solidity"
SOLC_JS_REPO_URL="https://github.com/ethereum/solc-js"
SOLC_JS_BRANCH=wasmRebuildTests
RELEASE_URL="https://raw.githubusercontent.com/ethereum/solc-bin/gh-pages/bin"
RELEASE_COMMIT_LIST_URL="$RELEASE_URL/list.txt"
SCRIPTDIR=$(dirname "$0")
SCRIPTDIR=$(realpath "${SCRIPTDIR}")
RED='\033[0;31m'
GREEN='\033[0;32m'
ORANGE='\033[0;33m'
CYAN='\033[0;36m'
RESET='\033[0m'
function generate_bytecode_report() {
rm -rf /tmp/report.txt
local EXIT_STATUS
if semver -r "<0.4.12" $3 > /dev/null; then
set +e
"${SCRIPTDIR}/genbytecode.sh" "$1" >/dev/null 2>&1
EXIT_STATUS=$?
set -e
else
set +e
(
set -e
git reset --hard HEAD --quiet
git clean -f -d -x --quiet
for dir in build/solc build/libsolc emscripten_build/libsolc; do
mkdir -p $dir
rm -rf $dir/soljson.js
ln -sf "$1" $dir/soljson.js
done
/tmp/storebytecode.sh >/dev/null 2>&1
)
EXIT_STATUS=$?
fi
if [ $EXIT_STATUS -eq 0 ] && [ -f /tmp/report.txt ] && grep -q -v -c -e "ERROR" -e "NO BYTECODE" /tmp/report.txt; then
mv /tmp/report.txt "$2"
echo -e "${GREEN}SUCCESS${RESET}"
else
echo -e "${RED}FAILURE${RESET}"
fi
}
function clean_git_checkout() {
git submodule deinit --all -q
git reset --hard HEAD --quiet
git clean -f -d -x --quiet
git checkout "$1" --quiet
git submodule init -q
git submodule update -q
}
function process_tag() {
local TAG=$1
cd /src
# Checkout the historic commit instead of the tag directly.
local HISTORIC_COMMIT_HASH="$(grep "${TAG}+" /tmp/release_commit_list.txt | cut -d '+' -f 2 | cut -d '.' -f 2)"
if [ "$(git cat-file -t ${HISTORIC_COMMIT_HASH} 2>/dev/null)" == "commit" ]; then
clean_git_checkout "$HISTORIC_COMMIT_HASH"
else
clean_git_checkout "${TAG}"
fi
# compatibility symlink
ln -s . solidity
local VERSION
if [ -f ./scripts/get_version.sh ]; then
VERSION=$(./scripts/get_version.sh)
else
VERSION=$(echo "$TAG" | cut -d v -f 2)
fi
local COMMIT_HASH=$(git rev-parse --short=8 HEAD)
local FULL_VERSION_SUFFIX="${TAG}+commit.${COMMIT_HASH}"
local HISTORIC_VERSION_SUFFIX="${TAG}+commit.${HISTORIC_COMMIT_HASH}"
if [ ! -f "${OUTPUTDIR}/bin/soljson-${FULL_VERSION_SUFFIX}.js" ]; then
echo -ne "BUILDING ${CYAN}${TAG}${RESET}... "
set +e
(
set -e
"${SCRIPTDIR}/rebuild_current.sh" "${VERSION}" >"${OUTPUTDIR}/log/running/build-$TAG.txt" 2>&1
"${SCRIPTDIR}/patch.sh" "$TAG" upload/soljson.js
cp upload/soljson.js "${OUTPUTDIR}/bin/soljson-${FULL_VERSION_SUFFIX}.js"
rm upload/soljson.js
)
local EXIT_STATUS=$?
set -e
rm -f "${OUTPUTDIR}/log/success/build-$TAG.txt"
rm -f "${OUTPUTDIR}/log/fail/build-$TAG.txt"
if [ $EXIT_STATUS -eq 0 ]; then
mv "${OUTPUTDIR}/log/running/build-$TAG.txt" "${OUTPUTDIR}/log/success"
echo -e "${GREEN}SUCCESS${RESET}"
else
mv "${OUTPUTDIR}/log/running/build-$TAG.txt" "${OUTPUTDIR}/log/fail"
echo -e "${RED}FAIL${RESET}"
fi
else
echo -e "${CYAN}${TAG}${RESET} ALREADY EXISTS."
if [ $RETEST -eq 0 ]; then
return 0
fi
fi
if [ -f "${OUTPUTDIR}/bin/soljson-${FULL_VERSION_SUFFIX}.js" ]; then
echo -ne "GENERATE BYTECODE REPORT FOR ${CYAN}${TAG}${RESET}... "
generate_bytecode_report "${OUTPUTDIR}/bin/soljson-${FULL_VERSION_SUFFIX}.js" "${OUTPUTDIR}"/log/reports/report-${TAG}.txt "${TAG}"
echo -ne "GENERATE BYTECODE REPORT FOR HISTORIC ${CYAN}${TAG}${RESET}... "
rm -rf /tmp/soljson.js
if wget -q "$RELEASE_URL/soljson-${HISTORIC_VERSION_SUFFIX}.js" -O /tmp/soljson.js; then
generate_bytecode_report /tmp/soljson.js "${OUTPUTDIR}"/log/reports/report-historic-${TAG}.txt "${TAG}"
else
echo -e "${ORANGE}CANNOT FETCH RELEASE${RESET}"
fi
rm -rf /tmp/soljson.js
if [ -f "${OUTPUTDIR}/log/reports/report-${TAG}.txt" ] && [ -f "${OUTPUTDIR}/log/reports/report-historic-${TAG}.txt" ]; then
rm -rf "${OUTPUTDIR}/log/success/bytecode-${TAG}.txt"
rm -rf "${OUTPUTDIR}/log/fail/bytecode-${TAG}.txt"
if diff -q "${OUTPUTDIR}/log/reports/report-${TAG}.txt" "${OUTPUTDIR}/log/reports/report-historic-${TAG}.txt" >/dev/null 2>&1; then
echo -e "${GREEN}BYTECODE MATCHES FOR ${CYAN}${TAG}${RESET}"
grep -v -c -e "ERROR" -e "NO BYTECODE" "${OUTPUTDIR}/log/reports/report-${TAG}.txt" >"${OUTPUTDIR}/log/success/bytecode-${TAG}.txt"
else
echo -e "${RED}BYTECODE DOES NOT MATCH FOR ${CYAN}${TAG}${RESET}"
echo "MISMATCH" >"${OUTPUTDIR}/log/fail/bytecode-${TAG}.txt"
fi
fi
echo -ne "TESTING ${CYAN}${TAG}${RESET}... "
cd /root/solc-js
npm version --allow-same-version --no-git-tag-version "${VERSION}" >/dev/null
sed -i -e "s/runTests(solc, .*)/runTests(solc, '${FULL_VERSION_SUFFIX}')/" test/compiler.js
ln -sf "${OUTPUTDIR}/bin/soljson-${FULL_VERSION_SUFFIX}.js" soljson.js
rm -f "${OUTPUTDIR}/log/success/test-$TAG.txt"
rm -f "${OUTPUTDIR}/log/fail/test-$TAG.txt"
if npm test >"${OUTPUTDIR}/log/running/test-$TAG.txt" 2>&1; then
mv "${OUTPUTDIR}/log/running/test-$TAG.txt" "${OUTPUTDIR}/log/success"
echo -e "${GREEN}SUCCESS${RESET}"
else
mv "${OUTPUTDIR}/log/running/test-$TAG.txt" "${OUTPUTDIR}/log/fail"
echo -e "${RED}FAIL${RESET}"
fi
fi
}
cd /tmp
echo "Check out solidity repository..."
if [ -d /root/project ]; then
echo "Solidity repo checkout already exists."
else
git clone "${SOLIDITY_REPO_URL}" /root/project --quiet
fi
echo "Extract bytecode comparison scripts from v0.6.1..."
cd /root/project
git checkout v0.6.1 --quiet
cp scripts/bytecodecompare/storebytecode.sh /tmp
sed -i -e 's/rm -rf "\$TMPDIR"/cp "\$TMPDIR"\/report.txt \/tmp\/report.txt ; rm -rf "\$TMPDIR"/' /tmp/storebytecode.sh
sed -i -e 's/REPO_ROOT=.*/REPO_ROOT=\/src/' /tmp/storebytecode.sh
export SOLC_EMSCRIPTEN="On"
echo "Check out solc-js repository..."
if [ -d /root/solc-js ]; then
echo "solc-js repo checkout already exists."
else
git clone --branch "${SOLC_JS_BRANCH}" "${SOLC_JS_REPO_URL}" /root/solc-js --quiet
fi
echo "Create symbolic links for backwards compatibility with older emscripten docker images."
ln -sf /emsdk_portable/node/current/* /emsdk_portable/node/
ln -sf /emsdk_portable/emscripten/sdk/ /emsdk_portable/
ln -sf sdk /emsdk_portable/emscripten/bin
ln -sf /emsdk_portable/emscripten/bin/* /usr/local/bin
rm -rf /src
ln -sf /root/project /src
apt-get -qq update >/dev/null 2>&1
apt-get -qq install cmake >/dev/null 2>&1
echo "Create output directories."
mkdir -p "${OUTPUTDIR}"
mkdir -p "${OUTPUTDIR}"/log
mkdir -p "${OUTPUTDIR}"/log/success
mkdir -p "${OUTPUTDIR}"/log/fail
mkdir -p "${OUTPUTDIR}"/log/running
mkdir -p "${OUTPUTDIR}"/log/reports
mkdir -p "${OUTPUTDIR}"/bin
echo "Prepare solc-js."
cd /root/solc-js
npm install >/dev/null 2>&1
echo "Install semver helper."
npm install -g semver >/dev/null 2>&1
echo "Fetching release commit list."
wget -q "${RELEASE_COMMIT_LIST_URL}" -O /tmp/release_commit_list.txt
cd /src
TAGS=$(git tag --list "${TAG_FILTER}" | tac)
for TAG in ${TAGS}; do
process_tag "${TAG}"
done

28
scripts/wasm-rebuild/rebuild.sh Executable file
View File

@ -0,0 +1,28 @@
#!/bin/sh
# This script is expected to produce working builds for all compiler versions >= 0.3.6 and
# succeeding solc-js test runs for all compiler versions >= 0.5.0.
if [ $# -lt 2 ]; then
echo "Usage: $0 [tagFilter] [outputDirectory] [options...]"
echo
echo " [tagFilter] will be passed to "git tag --list" to filter the tags to be built."
echo " [outputDirectory] will contain log files and the resulting soljson.js builds."
echo " --retest will re-run tests and bytecode comparisons, even if soljson.js is already built."
exit 1
fi
TAGS="$1"
OUTPUTDIR="$2"
shift
shift
SCRIPTDIR=$(dirname "$0")
SCRIPTDIR=$(realpath "${SCRIPTDIR}")
if [ ! -d "${OUTPUTDIR}" ]; then
echo "Output directory ${OUTPUTDIR} does not exist!."
exit 1
fi
OUTPUTDIR=$(realpath "${OUTPUTDIR}")
docker run --rm -v "${OUTPUTDIR}":/tmp/output -v "${SCRIPTDIR}":/tmp/scripts:ro -it trzeci/emscripten:sdk-tag-1.39.3-64bit /tmp/scripts/docker-scripts/rebuild_tags.sh "${TAGS}" /tmp/output $@