Merge pull request #12265 from ethereum/packedSoljson

Packed soljson.js

.circleci/config.yml

@@ -982,10 +982,10 @@ jobs:
           command: |
             scripts/ci/build_emscripten.sh
       - store_artifacts:
-          path: emscripten_build/libsolc/soljson.js
+          path: upload/soljson.js
           destination: soljson.js
       - run: mkdir -p workspace
-      - run: cp emscripten_build/libsolc/soljson.js workspace/soljson.js
+      - run: cp upload/soljson.js workspace/soljson.js
       - run: scripts/get_version.sh > workspace/version.txt
       - persist_to_workspace:
           root: workspace

cmake/EthCompilerSettings.cmake

@@ -142,8 +142,6 @@ if (("${CMAKE_CXX_COMPILER_ID}" MATCHES "GNU") OR ("${CMAKE_CXX_COMPILER_ID}" MA
 			set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -s WASM=1")
 			# Set webassembly build to synchronous loading.
 			set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -s WASM_ASYNC_COMPILATION=0")
-			# Output a single js file with the wasm binary embedded as base64 string.
-			set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -s SINGLE_FILE=1")
 			# Allow new functions to be added to the wasm module via addFunction.
 			set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -s ALLOW_TABLE_GROWTH=1")
 			# Disable warnings about not being pure asm.js due to memory growth.

cmake/templates/license.h.in

@@ -137,6 +137,36 @@ evmc:
    See the License for the specific language governing permissions and
    limitations under the License.
 
+mini-lz4:
+  The file scripts/ci/mini-lz4.js is derived from the emscripten adaptation of
+  node-lz4 and licensed under the following terms:
+
+  Copyright (c) 2012 Pierre Curto
+
+  Permission is hereby granted, free of charge, to any person obtaining a copy
+  of this software and associated documentation files (the "Software"), to deal
+  in the Software without restriction, including without limitation the rights
+  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+  copies of the Software, and to permit persons to whom the Software is
+  furnished to do so, subject to the following conditions:
+
+  The above copyright notice and this permission notice shall be included in
+  all copies or substantial portions of the Software.
+
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+  THE SOFTWARE.
+
+base64:
+  The file scripts/ci/base64DecToArr.js is derived from a code example
+  in the MDN Web Docs, which permits use under CC0 terms:
+
+  Any copyright is dedicated to the Public Domain. http://creativecommons.org/publicdomain/zero/1.0/
+
 
 All other code is licensed under GPL version 3:
 

scripts/ci/base64DecToArr.js

@@ -0,0 +1,46 @@
+function base64DecToArr (sBase64) {
+/*\
+|*|
+|*|  Base64 / binary data / UTF-8 strings utilities
+|*|
+|*|  https://developer.mozilla.org/en-US/docs/Web/JavaScript/Base64_encoding_and_decoding
+|*|
+\*/
+
+/* Array of bytes to Base64 string decoding */
+
+function b64ToUint6 (nChr) {
+
+  return nChr > 64 && nChr < 91 ?
+      nChr - 65
+    : nChr > 96 && nChr < 123 ?
+      nChr - 71
+    : nChr > 47 && nChr < 58 ?
+      nChr + 4
+    : nChr === 43 ?
+      62
+    : nChr === 47 ?
+      63
+    :
+      0;
+
+}
+
+  var
+    nInLen = sBase64.length,
+    nOutLen = nInLen * 3 + 1 >> 2, taBytes = new Uint8Array(nOutLen);
+
+  for (var nMod3, nMod4, nUint24 = 0, nOutIdx = 0, nInIdx = 0; nInIdx < nInLen; nInIdx++) {
+    nMod4 = nInIdx & 3;
+    nUint24 |= b64ToUint6(sBase64.charCodeAt(nInIdx)) << 6 * (3 - nMod4);
+    if (nMod4 === 3 || nInLen - nInIdx === 1) {
+      for (nMod3 = 0; nMod3 < 3 && nOutIdx < nOutLen; nMod3++, nOutIdx++) {
+        taBytes[nOutIdx] = nUint24 >>> (16 >>> nMod3 & 24) & 255;
+      }
+      nUint24 = 0;
+
+    }
+  }
+
+  return taBytes;
+}

scripts/ci/build_emscripten.sh

@@ -40,6 +40,8 @@ else
 	BUILD_DIR="$1"
 fi
 
+apt-get update && apt-get install lz4
+
 WORKSPACE=/root/project
 
 cd $WORKSPACE
@@ -71,8 +73,8 @@ make soljson
 
 cd ..
 mkdir -p upload
-cp "$BUILD_DIR/libsolc/soljson.js" upload/
-cp "$BUILD_DIR/libsolc/soljson.js" ./
+scripts/ci/pack_soljson.sh "$BUILD_DIR/libsolc/soljson.js" "$BUILD_DIR/libsolc/soljson.wasm" upload/soljson.js
+cp upload/soljson.js ./
 
 OUTPUT_SIZE=$(ls -la soljson.js)
 

scripts/ci/mini-lz4.js

@@ -0,0 +1,116 @@
+function uncompress(source, uncompressedSize) {
+/*
+based off https://github.com/emscripten-core/emscripten/blob/main/third_party/mini-lz4.js
+The license only applies to the body of this function (``uncompress``).
+====
+MiniLZ4: Minimal LZ4 block decoding and encoding.
+
+based off of node-lz4, https://github.com/pierrec/node-lz4
+
+====
+Copyright (c) 2012 Pierre Curto
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+====
+
+changes have the same license
+*/
+/**
+ * Decode a block. Assumptions: input contains all sequences of a
+ * chunk, output is large enough to receive the decoded data.
+ * If the output buffer is too small, an error will be thrown.
+ * If the returned value is negative, an error occurred at the returned offset.
+ *
+ * @param {ArrayBufferView} input input data
+ * @param {ArrayBufferView} output output data
+ * @param {number=} sIdx
+ * @param {number=} eIdx
+ * @return {number} number of decoded bytes
+ * @private
+ */
+function uncompressBlock (input, output, sIdx, eIdx) {
+	sIdx = sIdx || 0
+	eIdx = eIdx || (input.length - sIdx)
+	// Process each sequence in the incoming data
+	for (var i = sIdx, n = eIdx, j = 0; i < n;) {
+		var token = input[i++]
+
+		// Literals
+		var literals_length = (token >> 4)
+		if (literals_length > 0) {
+			// length of literals
+			var l = literals_length + 240
+			while (l === 255) {
+				l = input[i++]
+				literals_length += l
+			}
+
+			// Copy the literals
+			var end = i + literals_length
+			while (i < end) output[j++] = input[i++]
+
+			// End of buffer?
+			if (i === n) return j
+		}
+
+		// Match copy
+		// 2 bytes offset (little endian)
+		var offset = input[i++] | (input[i++] << 8)
+
+		// XXX 0 is an invalid offset value
+		if (offset === 0) return j
+		if (offset > j) return -(i-2)
+
+		// length of match copy
+		var match_length = (token & 0xf)
+		var l = match_length + 240
+		while (l === 255) {
+			l = input[i++]
+			match_length += l
+		}
+		// Copy the match
+		var pos = j - offset // position of the match copy in the current output
+		var end = j + match_length + 4 // minmatch = 4
+		while (j < end) output[j++] = output[pos++]
+	}
+
+	return j
+}
+var result = new ArrayBuffer(uncompressedSize);
+var sourceIndex = 0;
+var destIndex = 0;
+var blockSize;
+while((blockSize = (source[sourceIndex] | (source[sourceIndex + 1] << 8) | (source[sourceIndex + 2] << 16) | (source[sourceIndex + 3] << 24))) > 0)
+{
+	sourceIndex += 4;
+	if (blockSize & 0x80000000)
+	{
+		blockSize &= 0x7FFFFFFFF;
+		for (var i = 0; i < blockSize; i++) {
+			result[destIndex++] = source[sourceIndex++];
+		}
+	}
+	else
+	{
+		destIndex += uncompressBlock(source, new Uint8Array(result, destIndex, uncompressedSize - destIndex), sourceIndex, sourceIndex + blockSize);
+		sourceIndex += blockSize;
+	}
+}
+return new Uint8Array(result, 0, uncompressedSize);
+}

scripts/ci/pack_soljson.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+script_dir="$(realpath "$(dirname "$0")")"
+soljson_js="$1"
+soljson_wasm="$2"
+soljson_wasm_size=$(wc -c "${soljson_wasm}" | cut -d ' ' -f 1)
+output="$3"
+
+(( $# == 3 )) || { >&2 echo "Usage: $0 soljson.js soljson.wasm packed_soljson.js"; exit 1; }
+
+# If this changes in an emscripten update, it's probably nothing to worry about,
+# but we should double-check when it happens and adjust the tail command below.
+[[ $(head -c 5 "${soljson_js}") == "null;" ]] || { >&2 echo 'Expected soljson.js to start with "null;"'; exit 1; }
+
+echo "Packing $soljson_js and $soljson_wasm to $output."
+(
+    echo -n 'var Module = Module || {}; Module["wasmBinary"] = '
+    echo -n '(function(source, uncompressedSize) {'
+    # Note that base64DecToArr assumes no trailing equals signs.
+    cpp "${script_dir}/base64DecToArr.js" | grep -v "^#.*"
+    # Note that mini-lz4.js assumes no file header and no frame crc checksums.
+    cpp "${script_dir}/mini-lz4.js" | grep -v "^#.*"
+    echo 'return uncompress(base64DecToArr(source), uncompressedSize);})('
+    echo -n '"'
+    # We fix lz4 format settings, remove the 8 bytes file header and remove the trailing equals signs of the base64 encoding.
+    lz4c --no-frame-crc --best --favor-decSpeed "${soljson_wasm}" - | tail -c +8 | base64 -w 0 | sed 's/[^A-Za-z0-9\+\/]//g'
+    echo '",'
+    echo -n "${soljson_wasm_size});"
+    # Remove "null;" from the js wrapper.
+    tail -c +6 "${soljson_js}"
+) > "$output"
+
+echo "Testing $output."
+echo "process.stdout.write(require('$(realpath "${output}")').wasmBinary)" | node | cmp "${soljson_wasm}" && echo "Binaries match."
+# Allow the wasm binary to be garbage collected after compilation.
+echo 'Module["wasmBinary"] = undefined;' >> "${output}"