Fix bug where the defensive copy loses the computed leaf paths. (#401)

* Statediff missing parent blocks automatically.

* Use local ipld-eth-db.

.github/CODEOWNERS

@@ -5,16 +5,14 @@ accounts/usbwallet              @karalabe
 accounts/scwallet               @gballet
 accounts/abi                    @gballet @MariusVanDerWijden
 cmd/clef                        @holiman
-cmd/puppeth                     @karalabe
 consensus                       @karalabe
 core/                           @karalabe @holiman @rjl493456442
 eth/                            @karalabe @holiman @rjl493456442
 eth/catalyst/                   @gballet
 eth/tracers/                    @s1na
-graphql/                        @gballet @s1na
+graphql/                        @s1na
 les/                            @zsfelfoldi @rjl493456442
 light/                          @zsfelfoldi @rjl493456442
-mobile/                         @karalabe @ligi
 node/                           @fjl
 p2p/                            @fjl @zsfelfoldi
 rpc/                            @fjl @holiman

.github/CONTRIBUTING.md

@@ -35,6 +35,6 @@ and help.
 
 ## Configuration, dependencies, and tests
 
-Please see the [Developers' Guide](https://geth.ethereum.org/docs/developers/devguide)
+Please see the [Developers' Guide](https://geth.ethereum.org/docs/developers/geth-developer/dev-guide)
 for more details on configuring your environment, managing project dependencies
 and testing procedures.

.github/ISSUE_TEMPLATE/bug.md

@@ -9,6 +9,7 @@ assignees: ''
 #### System information
 
 Geth version: `geth version`
+CL client & version: e.g. lighthouse/nimbus/prysm@v1.0.0
 OS & Version: Windows/Linux/OSX
 Commit hash : (if `develop`)
 
@@ -27,4 +28,4 @@ Commit hash : (if `develop`)
 [backtrace]
 ````
 
-When submitting logs: please submit them as text and not screenshots.
+When submitting logs: please submit them as text and not screenshots.

.github/workflows/checks.yml

@@ -0,0 +1,15 @@
+name: checks
+
+on: [pull_request]
+
+jobs:
+  linter-check:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/setup-go@v3
+        with:
+          go-version: "1.19"
+          check-latest: true
+      - uses: actions/checkout@v2
+      - name: Run linter
+        run: go run build/ci.go lint

.github/workflows/manual_binary_publish.yaml

@@ -0,0 +1,21 @@
+name: MANUAL Override Publish geth binary to release
+on:
+  workflow_dispatch:
+    inputs:
+      giteaPublishTag:
+        description: 'Package to publish TO on gitea; e.g. v1.10.25-statediff-4.2.1-alpha'
+        required: true
+      cercContainerTag:
+        description: 'Tagged Container to extract geth binary FROM'
+        required: true
+jobs:
+  build:
+    name: Manual override publish of geth binary FROM tagged release TO TAGGED package on git.vdb.to
+    runs-on: ubuntu-latest
+    steps:
+      - name: Copy ethereum binary file
+        run: docker run --rm --entrypoint cat git.vdb.to/cerc-io/go-ethereum/go-ethereum:${{ github.event.inputs.cercContainerTag }} /usr/local/bin/geth > geth-linux-amd64
+      - name: curl
+        uses: enflo/curl-action@master
+        with:
+          curl: --user cerccicd:${{ secrets.GITEA_TOKEN }} --upload-file geth-linux-amd64 https://git.vdb.to/api/packages/cerc-io/generic/go-ethereum/${{ github.event.inputs.giteaPublishTag }}/geth-linux-amd64

.github/workflows/manual_publish.yaml

@@ -0,0 +1,21 @@
+name: MANUAL Override Publish geth binary to release
+on:
+  workflow_dispatch:
+    inputs:
+      giteaPublishTag:
+        description: 'Package to publish TO on gitea; e.g. v1.10.25-statediff-4.2.1-alpha'
+        required: true
+      cercContainerTag:
+        description: 'Tagged Container to extract geth binary FROM'
+        required: true
+jobs:
+  build:
+    name: Manual override publish of geth binary FROM tagged release TO TAGGED package on git.vdb.to
+    runs-on: ubuntu-latest
+    steps:
+      - name: Copy ethereum binary file
+        run: docker run --rm --entrypoint cat git.vdb.to/cerc-io/go-ethereum/go-ethereum:${{ github.event.inputs.cercContainerTag }} /usr/local/bin/geth > geth-linux-amd64
+      - name: curl
+        uses: enflo/curl-action@master
+        with:
+          curl: --user circcicd:${{ secrets.GITEA_TOKEN }} --upload-file geth-linux-amd64 https://git.vdb.to/api/packages/cerc-io/generic/go-ethereum/${{ github.event.inputs.giteaPublishTag }}/geth-linux-amd64

.github/workflows/on-pr.yml

@@ -0,0 +1,7 @@
+name: Build and test
+
+on: [pull_request]
+
+jobs:
+  run-tests:
+    uses: ./.github/workflows/tests.yml

.github/workflows/publish.yaml

@@ -0,0 +1,36 @@
+name: Publish geth to release
+on:
+  release:
+    types: [published]
+jobs:
+  run-tests:
+    uses: ./.github/workflows/tests.yml
+  build:
+    name: Run docker build and publish
+    needs: run-tests
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Run docker build
+        run: docker build -t cerc-io/go-ethereum -f Dockerfile .
+      - name: Get the version
+        id: vars
+        run: |
+          echo ::set-output name=sha::$(echo ${GITHUB_SHA:0:7})
+          echo ::set-output name=tag::$(echo ${GITHUB_REF#refs/tags/})
+      - name: Tag docker image
+        run: docker tag cerc-io/go-ethereum git.vdb.to/cerc-io/go-ethereum/go-ethereum:${{steps.vars.outputs.sha}}
+      - name: Tag docker image
+        run: docker tag git.vdb.to/cerc-io/go-ethereum/go-ethereum:${{steps.vars.outputs.sha}} git.vdb.to/cerc-io/go-ethereum/go-ethereum:${{steps.vars.outputs.tag}}
+      - name: Docker Login
+        run: echo ${{ secrets.GITEA_PUBLISH_TOKEN }} | docker login https://git.vdb.to -u cerccicd --password-stdin
+      - name: Docker Push
+        run: docker push git.vdb.to/cerc-io/go-ethereum/go-ethereum:${{steps.vars.outputs.sha}}
+      - name: Docker Push TAGGED
+        run: docker push git.vdb.to/cerc-io/go-ethereum/go-ethereum:${{steps.vars.outputs.tag}}
+      - name: Copy ethereum binary file
+        run: docker run --rm --entrypoint cat git.vdb.to/cerc-io/go-ethereum/go-ethereum:${{steps.vars.outputs.sha}} /usr/local/bin/geth > geth-linux-amd64
+      - name: curl
+        uses: enflo/curl-action@master
+        with:
+          curl: --user cerccicd:${{ secrets.GITEA_PUBLISH_TOKEN }} --upload-file geth-linux-amd64 https://git.vdb.to/api/packages/cerc-io/generic/go-ethereum/${{steps.vars.outputs.tag}}/geth-linux-amd64

.github/workflows/tests.yml

@@ -0,0 +1,150 @@
+name: Tests for Geth that are used in multiple jobs.
+
+on:
+  workflow_call:
+
+env:
+  stack-orchestrator-ref: ${{ github.event.inputs.stack-orchestrator-ref  || 'e62830c982d4dfc5f3c1c2b12c1754a7e9b538f1'}}
+  ipld-eth-db-ref: ${{ github.event.inputs.ipld-eth-db-ref  || '1b922dbff350bfe2a9aec5fe82079e9d855ea7ed' }}
+  GOPATH: /tmp/go
+
+jobs:
+  build:
+    name: Run docker build
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Run docker build
+        run: docker build -t cerc-io/go-ethereum .
+
+  geth-unit-test:
+    name: Run geth unit test
+    runs-on: ubuntu-latest
+    env:
+      GO111MODULE: on
+    steps:
+      - name: Create GOPATH
+        run: mkdir -p /tmp/go
+
+      - uses: actions/setup-go@v3
+        with:
+          go-version: "1.19"
+          check-latest: true
+
+      - name: Checkout code
+        uses: actions/checkout@v2
+
+      - name: Run unit tests
+        run: |
+          make test
+
+  statediff-unit-test:
+    name: Run state diff unit test
+    runs-on: ubuntu-latest
+    steps:
+      - name: Create GOPATH
+        run: mkdir -p /tmp/go
+
+      - uses: actions/setup-go@v3
+        with:
+          go-version: "1.19"
+          check-latest: true
+
+      - name: Checkout code
+        uses: actions/checkout@v2
+
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ env.ipld-eth-db-ref }}
+          repository: cerc-io/ipld-eth-db
+          path: "./ipld-eth-db/"
+          fetch-depth: 0
+
+      - name: Build ipld-eth-db
+        run: |
+          docker build -f ./ipld-eth-db/Dockerfile ./ipld-eth-db/ -t cerc/ipld-eth-db:local
+
+      - name: Run docker compose
+        run: |
+          docker-compose up -d
+      - name: Give the migration a few seconds
+        run: sleep 30;
+
+      - name: Run unit tests
+        run: make statedifftest
+
+  private-network-test:
+    name: Start Geth in a private network.
+    runs-on: ubuntu-latest
+    steps:
+      - name: Create GOPATH
+        run: mkdir -p /tmp/go
+
+      - uses: actions/setup-go@v3
+        with:
+          go-version: "1.19"
+          check-latest: true
+
+      - name: Checkout code
+        uses: actions/checkout@v3
+        with:
+          path: "./go-ethereum"
+
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ env.stack-orchestrator-ref }}
+          path: "./stack-orchestrator/"
+          repository: cerc-io/mshaw_stack_hack
+          fetch-depth: 0
+
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ env.ipld-eth-db-ref }}
+          repository: cerc-io/ipld-eth-db
+          path: "./ipld-eth-db/"
+          fetch-depth: 0
+
+      - name: Create config file
+        run: |
+          echo vulcanize_ipld_eth_db=$GITHUB_WORKSPACE/ipld-eth-db/ > $GITHUB_WORKSPACE/config.sh
+          echo vulcanize_go_ethereum=$GITHUB_WORKSPACE/go-ethereum/ >> $GITHUB_WORKSPACE/config.sh
+          echo db_write=true >> $GITHUB_WORKSPACE/config.sh
+          echo genesis_file_path=start-up-files/go-ethereum/genesis.json >> $GITHUB_WORKSPACE/config.sh
+          cat $GITHUB_WORKSPACE/config.sh
+      - name: Compile Geth
+        run: |
+          cd $GITHUB_WORKSPACE/stack-orchestrator/helper-scripts
+          ./compile-geth.sh -e docker -p $GITHUB_WORKSPACE/config.sh
+          cd -
+      - name: Run docker compose
+        run: |
+          docker-compose  \
+          -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-go-ethereum.yml" \
+          -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-db-sharding.yml" \
+          --env-file $GITHUB_WORKSPACE/config.sh \
+          up -d --build
+      - name: Make sure the /root/transaction_info/STATEFUL_TEST_DEPLOYED_ADDRESS exists within a certain time frame.
+        shell: bash
+        run: |
+          COUNT=0
+          ATTEMPTS=15
+          docker logs local_go-ethereum_1
+          docker compose -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-db-sharding.yml" -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-go-ethereum.yml" exec go-ethereum ps aux
+          until $(docker compose -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-db-sharding.yml" -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-go-ethereum.yml" cp go-ethereum:/root/transaction_info/STATEFUL_TEST_DEPLOYED_ADDRESS ./STATEFUL_TEST_DEPLOYED_ADDRESS) || [[ $COUNT -eq $ATTEMPTS ]]; do echo -e "$(( COUNT++ ))... \c"; sleep 10; done
+          [[ $COUNT -eq $ATTEMPTS ]] && echo "Could not find the successful contract deployment" && (exit 1)
+          cat ./STATEFUL_TEST_DEPLOYED_ADDRESS
+          echo "Address length: `wc ./STATEFUL_TEST_DEPLOYED_ADDRESS`"
+          sleep 15;
+      - name: Create a new transaction.
+        shell: bash
+        run: |
+          docker logs local_go-ethereum_1
+          docker compose -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-db-sharding.yml" -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-go-ethereum.yml" exec go-ethereum /bin/bash /root/transaction_info/NEW_TRANSACTION
+          echo $?
+      - name: Make sure we see entries in the header table
+        shell: bash
+        run: |
+          rows=$(docker compose -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-db-sharding.yml" -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-go-ethereum.yml" exec ipld-eth-db psql -U vdbm -d vulcanize_testing  -AXqtc "SELECT COUNT(*) FROM eth.header_cids")
+          [[ "$rows" -lt "1" ]] && echo "We could not find any rows in postgres table." && (exit 1)
+          echo $rows
+          docker compose -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-db-sharding.yml" -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-go-ethereum.yml" exec ipld-eth-db psql -U vdbm -d vulcanize_testing  -AXqtc "SELECT * FROM eth.header_cids"

.gitignore

@@ -47,3 +47,19 @@ profile.cov
 /dashboard/assets/package-lock.json
 
 **/yarn-error.log
+logs/
+foundry/deployments/local-private-network/geth-linux-amd64
+foundry/projects/local-private-network/geth-linux-amd64
+
+# Helpful repos
+related-repositories/foundry-test/**
+related-repositories/hive/**
+related-repositories/ipld-eth-db/**
+related-repositories/foundry-test/
+related-repositories/ipld-eth-db/
+
+# files generated by statediffing tests
+statediff/indexer/database/sql/statediffing_test_file.sql
+statediff/statediffing_test_file.sql
+statediff/known_gaps.sql
+statediff/indexer/database/file/statediffing_test

.gitmodules

@@ -5,4 +5,4 @@
 [submodule "evm-benchmarks"]
 	path = tests/evm-benchmarks
 	url = https://github.com/ipsilon/evm-benchmarks
-	shallow = true
+	shallow = true

.golangci.yml

@@ -12,7 +12,6 @@ run:
 linters:
   disable-all: true
   enable:
-    - deadcode
     - goconst
     - goimports
     - gosimple
@@ -20,23 +19,22 @@ linters:
     - ineffassign
     - misspell
     - unconvert
-    - varcheck
     - typecheck
     - unused
     - staticcheck
     - bidichk
     - durationcheck
     - exportloopref
-    - gosec
+    - whitespace
 
-    #- structcheck # lots of false positives
-    #- errcheck #lot of false positives
-   # - contextcheck
-   # - errchkjson # lots of false positives
-   # - errorlint # this check crashes
-   # - exhaustive # silly check
-   # - makezero # false positives
-   # - nilerr # several intentional
+    # - structcheck # lots of false positives
+    # - errcheck #lot of false positives
+    # - contextcheck
+    # - errchkjson # lots of false positives
+    # - errorlint # this check crashes
+    # - exhaustive # silly check
+    # - makezero # false positives
+    # - nilerr # several intentional
 
 linters-settings:
   gofmt:
@@ -44,11 +42,6 @@ linters-settings:
   goconst:
     min-len: 3 # minimum length of string constant
     min-occurrences: 6 # minimum number of occurrences
-  gosec:
-    excludes:
-     - G404 # Use of weak random number generator - lots of FP
-     - G107 # Potential http request -- those are intentional
-     - G306 # G306: Expect WriteFile permissions to be 0600 or less
 
 issues:
   exclude-rules:
@@ -57,15 +50,15 @@ issues:
         - deadcode
         - staticcheck
     - path: internal/build/pgp.go
-      text: 'SA1019: package golang.org/x/crypto/openpgp is deprecated'
+      text: 'SA1019: "golang.org/x/crypto/openpgp" is deprecated: this package is unmaintained except for security fixes.'
     - path: core/vm/contracts.go
-      text: 'SA1019: package golang.org/x/crypto/ripemd160 is deprecated'
+      text: 'SA1019: "golang.org/x/crypto/ripemd160" is deprecated: RIPEMD-160 is a legacy hash and should not be used for new applications.'
     - path: accounts/usbwallet/trezor.go
-      text: 'SA1019: package github.com/golang/protobuf/proto is deprecated'
+      text: 'SA1019: "github.com/golang/protobuf/proto" is deprecated: Use the "google.golang.org/protobuf/proto" package instead.'
     - path: accounts/usbwallet/trezor/
-      text: 'SA1019: package github.com/golang/protobuf/proto is deprecated'
+      text: 'SA1019: "github.com/golang/protobuf/proto" is deprecated: Use the "google.golang.org/protobuf/proto" package instead.'
   exclude:
     - 'SA1019: event.TypeMux is deprecated: use Feed'
     - 'SA1019: strings.Title is deprecated'
+    - 'SA1019: strings.Title has been deprecated since Go 1.18 and an alternative has been available since Go 1.0: The rule Title uses for word boundaries does not handle Unicode punctuation properly. Use golang.org/x/text/cases instead.'
     - 'SA1029: should not use built-in type string as key for value'
-    - 'G306: Expect WriteFile permissions to be 0600 or less'

.travis.yml

@@ -5,18 +5,15 @@ jobs:
   allow_failures:
     - stage: build
       os: osx
-      go: 1.17.x
       env:
         - azure-osx
-        - azure-ios
-        - cocoapods-ios
 
   include:
     # This builder only tests code linters on latest version of Go
     - stage: lint
       os: linux
       dist: bionic
-      go: 1.18.x
+      go: 1.20.x
       env:
         - lint
       git:
@@ -31,7 +28,7 @@ jobs:
       os: linux
       arch: amd64
       dist: bionic
-      go: 1.18.x
+      go: 1.20.x
       env:
         - docker
       services:
@@ -48,7 +45,7 @@ jobs:
       os: linux
       arch: arm64
       dist: bionic
-      go: 1.18.x
+      go: 1.20.x
       env:
         - docker
       services:
@@ -60,37 +57,13 @@ jobs:
       script:
         - go run build/ci.go docker -image -manifest amd64,arm64 -upload ethereum/client-go
 
-    # This builder does the Ubuntu PPA upload
-    - stage: build
-      if: type = push
-      os: linux
-      dist: bionic
-      go: 1.18.x
-      env:
-        - ubuntu-ppa
-        - GO111MODULE=on
-      git:
-        submodules: false # avoid cloning ethereum/tests
-      addons:
-        apt:
-          packages:
-            - devscripts
-            - debhelper
-            - dput
-            - fakeroot
-            - python-bzrlib
-            - python-paramiko
-      script:
-        - echo '|1|7SiYPr9xl3uctzovOTj4gMwAC1M=|t6ReES75Bo/PxlOPJ6/GsGbTrM0= ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA0aKz5UTUndYgIGG7dQBV+HaeuEZJ2xPHo2DS2iSKvUL4xNMSAY4UguNW+pX56nAQmZKIZZ8MaEvSj6zMEDiq6HFfn5JcTlM80UwlnyKe8B8p7Nk06PPQLrnmQt5fh0HmEcZx+JU9TZsfCHPnX7MNz4ELfZE6cFsclClrKim3BHUIGq//t93DllB+h4O9LHjEUsQ1Sr63irDLSutkLJD6RXchjROXkNirlcNVHH/jwLWR5RcYilNX7S5bIkK8NlWPjsn/8Ua5O7I9/YoE97PpO6i73DTGLh5H9JN/SITwCKBkgSDWUt61uPK3Y11Gty7o2lWsBjhBUm2Y38CBsoGmBw==' >> ~/.ssh/known_hosts
-        - go run build/ci.go debsrc -upload ethereum/ethereum -sftp-user geth-ci -signer "Go Ethereum Linux Builder <geth-ci@ethereum.org>"
-
     # This builder does the Linux Azure uploads
     - stage: build
       if: type = push
       os: linux
       dist: bionic
       sudo: required
-      go: 1.18.x
+      go: 1.20.x
       env:
         - azure-linux
         - GO111MODULE=on
@@ -120,53 +93,13 @@ jobs:
         - go run build/ci.go install -dlgo -arch arm64 -cc aarch64-linux-gnu-gcc
         - go run build/ci.go archive -arch arm64 -type tar -signer LINUX_SIGNING_KEY -signify SIGNIFY_KEY -upload gethstore/builds
 
-    # This builder does the Android Maven and Azure uploads
-    - stage: build
-      if: type = push
-      os: linux
-      dist: bionic
-      addons:
-        apt:
-          packages:
-            - openjdk-8-jdk
-      env:
-        - azure-android
-        - maven-android
-        - GO111MODULE=on
-      git:
-        submodules: false # avoid cloning ethereum/tests
-      before_install:
-        # Install Android and it's dependencies manually, Travis is stale
-        - export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
-        - curl https://dl.google.com/android/repository/commandlinetools-linux-6858069_latest.zip -o android.zip
-        - unzip -q android.zip -d $HOME/sdk && rm android.zip
-        - mv $HOME/sdk/cmdline-tools $HOME/sdk/latest && mkdir $HOME/sdk/cmdline-tools && mv $HOME/sdk/latest $HOME/sdk/cmdline-tools
-        - export PATH=$PATH:$HOME/sdk/cmdline-tools/latest/bin
-        - export ANDROID_HOME=$HOME/sdk
-
-        - yes | sdkmanager --licenses >/dev/null
-        - sdkmanager "platform-tools" "platforms;android-15" "platforms;android-19" "platforms;android-24" "ndk-bundle"
-
-        # Install Go to allow building with
-        - curl https://dl.google.com/go/go1.18.linux-amd64.tar.gz | tar -xz
-        - export PATH=`pwd`/go/bin:$PATH
-        - export GOROOT=`pwd`/go
-        - export GOPATH=$HOME/go
-      script:
-        # Build the Android archive and upload it to Maven Central and Azure
-        - mkdir -p $GOPATH/src/github.com/ethereum
-        - ln -s `pwd` $GOPATH/src/github.com/ethereum/go-ethereum
-        - go run build/ci.go aar -signer ANDROID_SIGNING_KEY -signify SIGNIFY_KEY  -deploy https://oss.sonatype.org -upload gethstore/builds
-
-    # This builder does the OSX Azure, iOS CocoaPods and iOS Azure uploads
+    # This builder does the OSX Azure uploads
     - stage: build
       if: type = push
       os: osx
-      go: 1.18.x
+      go: 1.20.x
       env:
         - azure-osx
-        - azure-ios
-        - cocoapods-ios
         - GO111MODULE=on
       git:
         submodules: false # avoid cloning ethereum/tests
@@ -174,58 +107,67 @@ jobs:
         - go run build/ci.go install -dlgo
         - go run build/ci.go archive -type tar -signer OSX_SIGNING_KEY -signify SIGNIFY_KEY -upload gethstore/builds
 
-        # Build the iOS framework and upload it to CocoaPods and Azure
-        - gem uninstall cocoapods -a -x
-        - gem install cocoapods
-
-        - mv ~/.cocoapods/repos/master ~/.cocoapods/repos/master.bak
-        - sed -i '.bak' 's/repo.join/!repo.join/g' $(dirname `gem which cocoapods`)/cocoapods/sources_manager.rb
-        - if [ "$TRAVIS_PULL_REQUEST" = "false" ]; then git clone --depth=1 https://github.com/CocoaPods/Specs.git ~/.cocoapods/repos/master && pod setup --verbose; fi
-
-        - xctool -version
-        - xcrun simctl list
-
-        # Workaround for https://github.com/golang/go/issues/23749
-        - export CGO_CFLAGS_ALLOW='-fmodules|-fblocks|-fobjc-arc'
-        - go run build/ci.go xcode -signer IOS_SIGNING_KEY -signify SIGNIFY_KEY -deploy trunk -upload gethstore/builds
-
     # These builders run the tests
     - stage: build
       os: linux
       arch: amd64
       dist: bionic
-      go: 1.18.x
+      go: 1.20.x
       env:
         - GO111MODULE=on
       script:
-        - go run build/ci.go test -coverage $TEST_PACKAGES
+        - go run build/ci.go test $TEST_PACKAGES
 
     - stage: build
       if: type = pull_request
       os: linux
       arch: arm64
       dist: bionic
-      go: 1.18.x
+      go: 1.19.x
       env:
         - GO111MODULE=on
       script:
-        - go run build/ci.go test -coverage $TEST_PACKAGES
+        - go run build/ci.go test $TEST_PACKAGES
 
     - stage: build
       os: linux
       dist: bionic
-      go: 1.17.x
+      go: 1.19.x
       env:
         - GO111MODULE=on
       script:
-        - go run build/ci.go test -coverage $TEST_PACKAGES
+        - go run build/ci.go test $TEST_PACKAGES
+
+    # This builder does the Ubuntu PPA nightly uploads
+    - stage: build
+      if: type = cron || (type = push && tag ~= /^v[0-9]/)
+      os: linux
+      dist: bionic
+      go: 1.20.x
+      env:
+        - ubuntu-ppa
+        - GO111MODULE=on
+      git:
+        submodules: false # avoid cloning ethereum/tests
+      addons:
+        apt:
+          packages:
+            - devscripts
+            - debhelper
+            - dput
+            - fakeroot
+            - python-bzrlib
+            - python-paramiko
+      script:
+        - echo '|1|7SiYPr9xl3uctzovOTj4gMwAC1M=|t6ReES75Bo/PxlOPJ6/GsGbTrM0= ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA0aKz5UTUndYgIGG7dQBV+HaeuEZJ2xPHo2DS2iSKvUL4xNMSAY4UguNW+pX56nAQmZKIZZ8MaEvSj6zMEDiq6HFfn5JcTlM80UwlnyKe8B8p7Nk06PPQLrnmQt5fh0HmEcZx+JU9TZsfCHPnX7MNz4ELfZE6cFsclClrKim3BHUIGq//t93DllB+h4O9LHjEUsQ1Sr63irDLSutkLJD6RXchjROXkNirlcNVHH/jwLWR5RcYilNX7S5bIkK8NlWPjsn/8Ua5O7I9/YoE97PpO6i73DTGLh5H9JN/SITwCKBkgSDWUt61uPK3Y11Gty7o2lWsBjhBUm2Y38CBsoGmBw==' >> ~/.ssh/known_hosts
+        - go run build/ci.go debsrc -upload ethereum/ethereum -sftp-user geth-ci -signer "Go Ethereum Linux Builder <geth-ci@ethereum.org>"
 
     # This builder does the Azure archive purges to avoid accumulating junk
     - stage: build
       if: type = cron
       os: linux
       dist: bionic
-      go: 1.18.x
+      go: 1.20.x
       env:
         - azure-purge
         - GO111MODULE=on
@@ -239,9 +181,9 @@ jobs:
       if: type = cron
       os: linux
       dist: bionic
-      go: 1.18.x
+      go: 1.20.x
       env:
         - GO111MODULE=on
       script:
-        - go run build/ci.go test  -race -coverage $TEST_PACKAGES
+        - go run build/ci.go test  -race $TEST_PACKAGES
 

Dockerfile

@@ -4,7 +4,7 @@ ARG VERSION=""
 ARG BUILDNUM=""
 
 # Build Geth in a stock Go builder container
-FROM golang:1.18-alpine as builder
+FROM golang:1.20-alpine as builder
 
 RUN apk add --no-cache gcc musl-dev linux-headers git
 
@@ -14,7 +14,7 @@ COPY go.sum /go-ethereum/
 RUN cd /go-ethereum && go mod download
 
 ADD . /go-ethereum
-RUN cd /go-ethereum && go run build/ci.go install ./cmd/geth
+RUN cd /go-ethereum && go run build/ci.go install -static ./cmd/geth
 
 # Pull Geth into a second stage deploy alpine container
 FROM alpine:latest

Dockerfile.alltools

@@ -4,7 +4,7 @@ ARG VERSION=""
 ARG BUILDNUM=""
 
 # Build Geth in a stock Go builder container
-FROM golang:1.18-alpine as builder
+FROM golang:1.20-alpine as builder
 
 RUN apk add --no-cache gcc musl-dev linux-headers git
 
@@ -14,7 +14,7 @@ COPY go.sum /go-ethereum/
 RUN cd /go-ethereum && go mod download
 
 ADD . /go-ethereum
-RUN cd /go-ethereum && go run build/ci.go install
+RUN cd /go-ethereum && go run build/ci.go install -static
 
 # Pull all binaries into a second stage deploy alpine container
 FROM alpine:latest

Dockerfile.amd64

@@ -0,0 +1,7 @@
+# Build Geth in a stock Go builder container
+FROM golang:1.15.5 as builder
+
+#RUN apk add --no-cache make gcc musl-dev linux-headers git
+
+ADD . /go-ethereum
+RUN cd /go-ethereum && make geth

Jenkinsfile

@@ -0,0 +1,50 @@
+pipeline {
+    agent any
+
+    stages {
+        stage('Build') {
+            steps {
+                script{
+                    docker.withRegistry('https://git.vdb.to'){
+                        echo 'Building geth image...'
+                        //def geth_image = docker.build("cerc-io/go-ethereum:jenkinscicd")
+                        echo 'built geth image'
+                    }
+                }
+            }
+        }
+        stage('Test') {
+            agent {
+                docker {
+                    image 'cerc-io/foundation:jenkinscicd'
+                    //image 'cerc-io/foundation_alpine:jenkinscicd'
+                }
+            }
+
+            environment {
+                GO111MODULE = "on"
+                CGO_ENABLED = 1
+                //GOPATH = "${JENKINS_HOME}/jobs/${JOB_NAME}/builds/${BUILD_ID}"
+                //GOPATH = "/go"
+                GOPATH = "/tmp/go"
+                //GOMODCACHE = "/go/pkg/mod"
+                GOCACHE = "${WORKSPACE}/.cache/go-build"
+                GOENV = "${WORKSPACE}/.config/go/env"
+                GOMODCACHE = "/tmp/go/pkg/mod"
+                GOWORK=""
+                //GOFLAGS=""
+
+            }
+            steps {
+                echo 'Testing ...'
+                //sh '/usr/local/go/bin/go test -p 1 -v ./...'
+                sh 'make test'
+            }
+        }
+        stage('Packaging') {
+            steps {
+                echo 'Packaging ...'
+            }
+        }
+    }
+}

Makefile

@@ -4,10 +4,31 @@
 
 .PHONY: geth android ios evm all test clean
 
+BIN = $(GOPATH)/bin
+
+## Migration tool
+GOOSE = $(BIN)/goose
+$(BIN)/goose:
+	go get -u github.com/pressly/goose/cmd/goose
+
 GOBIN = ./build/bin
 GO ?= latest
 GORUN = env GO111MODULE=on go run
 
+#Database
+HOST_NAME = localhost
+PORT = 5432
+USER = vdbm
+PASSWORD = password
+
+# Set env variable
+# `PGPASSWORD` is used by `createdb` and `dropdb`
+export PGPASSWORD=$(PASSWORD)
+
+#Test
+TEST_DB = cerc_testing
+TEST_CONNECT_STRING = postgresql://$(USER):$(PASSWORD)@$(HOST_NAME):$(PORT)/$(TEST_DB)?sslmode=disable
+
 geth:
 	$(GORUN) build/ci.go install ./cmd/geth
 	@echo "Done building."
@@ -16,18 +37,6 @@ geth:
 all:
 	$(GORUN) build/ci.go install
 
-android:
-	$(GORUN) build/ci.go aar --local
-	@echo "Done building."
-	@echo "Import \"$(GOBIN)/geth.aar\" to use the library."
-	@echo "Import \"$(GOBIN)/geth-sources.jar\" to add javadocs"
-	@echo "For more info see https://stackoverflow.com/questions/20994336/android-studio-how-to-attach-javadoc"
-
-ios:
-	$(GORUN) build/ci.go xcode --local
-	@echo "Done building."
-	@echo "Import \"$(GOBIN)/Geth.framework\" to use the library."
-
 test: all
 	$(GORUN) build/ci.go test
 
@@ -48,3 +57,13 @@ devtools:
 	env GOBIN= go install ./cmd/abigen
 	@type "solc" 2> /dev/null || echo 'Please install solc'
 	@type "protoc" 2> /dev/null || echo 'Please install protoc'
+
+.PHONY: statedifftest
+statedifftest: | $(GOOSE)
+	GO111MODULE=on go get github.com/stretchr/testify/assert@v1.7.0
+	GO111MODULE=on MODE=statediff go test -p 1 ./statediff/... -v
+
+.PHONY: statediff_filewriting_test
+statediff_filetest: | $(GOOSE)
+	GO111MODULE=on go get github.com/stretchr/testify/assert@v1.7.0
+	GO111MODULE=on MODE=statediff STATEDIFF_DB=file go test -p 1 ./statediff/... -v

README.md

@@ -1,10 +1,8 @@
 ## Go Ethereum
 
-Official Golang implementation of the Ethereum protocol.
+Official Golang execution layer implementation of the Ethereum protocol.
 
-[![API Reference](
-https://camo.githubusercontent.com/915b7be44ada53c290eb157634330494ebe3e30a/68747470733a2f2f676f646f632e6f72672f6769746875622e636f6d2f676f6c616e672f6764646f3f7374617475732e737667
-)](https://pkg.go.dev/github.com/ethereum/go-ethereum?tab=doc)
+[![API Reference](https://camo.githubusercontent.com/915b7be44ada53c290eb157634330494ebe3e30a/68747470733a2f2f676f646f632e6f72672f6769746875622e636f6d2f676f6c616e672f6764646f3f7374617475732e737667)](https://pkg.go.dev/github.com/ethereum/go-ethereum?tab=doc)
 [![Go Report Card](https://goreportcard.com/badge/github.com/ethereum/go-ethereum)](https://goreportcard.com/report/github.com/ethereum/go-ethereum)
 [![Travis](https://travis-ci.com/ethereum/go-ethereum.svg?branch=master)](https://travis-ci.com/ethereum/go-ethereum)
 [![Discord](https://img.shields.io/badge/discord-join%20chat-blue.svg)](https://discord.gg/nthXNEv)
@@ -12,11 +10,26 @@ https://camo.githubusercontent.com/915b7be44ada53c290eb157634330494ebe3e30a/6874
 Automated builds are available for stable releases and the unstable master branch. Binary
 archives are published at https://geth.ethereum.org/downloads/.
 
+## Vulcanize Specific
+
+This section captures components specific to vulcanize.
+
+### Branching Structure
+
+We currently follow the following branching structure.
+
+1. Create a branch: `v1.10.18-statediff-vX` --> feature/some-feature`
+2. Create a PR upstream: `feature/some-feature` --> `v1.10.18-statediff-vX`
+3. When a release is ready, create a release branch: `v1.10.18-statediff-vX` --> `v1.10.18-statediff-X.Y.Z`
+4. When `v1.10.18-statediff-vX` is stable, merge it to `statediff`.
+
+This process is subject to change.
+
 ## Building the source
 
-For prerequisites and detailed build instructions please read the [Installation Instructions](https://geth.ethereum.org/docs/install-and-build/installing-geth).
+For prerequisites and detailed build instructions please read the [Installation Instructions](https://geth.ethereum.org/docs/getting-started/installing-geth).
 
-Building `geth` requires both a Go (version 1.16 or later) and a C compiler. You can install
+Building `geth` requires both a Go (version 1.19 or later) and a C compiler. You can install
 them using your favourite package manager. Once the dependencies are installed, run
 
 ```shell
@@ -34,21 +47,20 @@ make all
 The go-ethereum project comes with several wrappers/executables found in the `cmd`
 directory.
 
-|    Command    | Description                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                          |
-| :-----------: | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-|  **`geth`**   | Our main Ethereum CLI client. It is the entry point into the Ethereum network (main-, test- or private net), capable of running as a full node (default), archive node (retaining all historical state) or a light node (retrieving data live). It can be used by other processes as a gateway into the Ethereum network via JSON RPC endpoints exposed on top of HTTP, WebSocket and/or IPC transports. `geth --help` and the [CLI page](https://geth.ethereum.org/docs/interface/command-line-options) for command line options.          |
-|   `clef`    | Stand-alone signing tool, which can be used as a backend signer for `geth`.  |
-|   `devp2p`    | Utilities to interact with nodes on the networking layer, without running a full blockchain. |
-|   `abigen`    | Source code generator to convert Ethereum contract definitions into easy to use, compile-time type-safe Go packages. It operates on plain [Ethereum contract ABIs](https://docs.soliditylang.org/en/develop/abi-spec.html) with expanded functionality if the contract bytecode is also available. However, it also accepts Solidity source files, making development much more streamlined. Please see our [Native DApps](https://geth.ethereum.org/docs/dapp/native-bindings) page for details. |
-|  `bootnode`   | Stripped down version of our Ethereum client implementation that only takes part in the network node discovery protocol, but does not run any of the higher level application protocols. It can be used as a lightweight bootstrap node to aid in finding peers in private networks.                                                                                                                                                                                                                                                                 |
-|     `evm`     | Developer utility version of the EVM (Ethereum Virtual Machine) that is capable of running bytecode snippets within a configurable environment and execution mode. Its purpose is to allow isolated, fine-grained debugging of EVM opcodes (e.g. `evm --code 60ff60ff --debug run`).                                                                                                                                                                                                                                                                     |
-|   `rlpdump`   | Developer utility tool to convert binary RLP ([Recursive Length Prefix](https://ethereum.org/en/developers/docs/data-structures-and-encoding/rlp)) dumps (data encoding used by the Ethereum protocol both network as well as consensus wise) to user-friendlier hierarchical representation (e.g. `rlpdump --hex CE0183FFFFFFC4C304050583616263`).                                                                                                                                                                                                                                 |
-|   `puppeth`   | a CLI wizard that aids in creating a new Ethereum network.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                           |
+|  Command   | Description                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        |
+| :--------: | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| **`geth`** | Our main Ethereum CLI client. It is the entry point into the Ethereum network (main-, test- or private net), capable of running as a full node (default), archive node (retaining all historical state) or a light node (retrieving data live). It can be used by other processes as a gateway into the Ethereum network via JSON RPC endpoints exposed on top of HTTP, WebSocket and/or IPC transports. `geth --help` and the [CLI page](https://geth.ethereum.org/docs/fundamentals/command-line-options) for command line options. |
+|   `clef`   | Stand-alone signing tool, which can be used as a backend signer for `geth`.                                                                                                                                                                                                                                                                                                                                                                                                                                                        |
+|  `devp2p`  | Utilities to interact with nodes on the networking layer, without running a full blockchain.                                                                                                                                                                                                                                                                                                                                                                                                                                       |
+|  `abigen`  | Source code generator to convert Ethereum contract definitions into easy-to-use, compile-time type-safe Go packages. It operates on plain [Ethereum contract ABIs](https://docs.soliditylang.org/en/develop/abi-spec.html) with expanded functionality if the contract bytecode is also available. However, it also accepts Solidity source files, making development much more streamlined. Please see our [Native DApps](https://geth.ethereum.org/docs/developers/dapp-developer/native-bindings) page for details.                                  |
+| `bootnode` | Stripped down version of our Ethereum client implementation that only takes part in the network node discovery protocol, but does not run any of the higher level application protocols. It can be used as a lightweight bootstrap node to aid in finding peers in private networks.                                                                                                                                                                                                                                               |
+|   `evm`    | Developer utility version of the EVM (Ethereum Virtual Machine) that is capable of running bytecode snippets within a configurable environment and execution mode. Its purpose is to allow isolated, fine-grained debugging of EVM opcodes (e.g. `evm --code 60ff60ff --debug run`).                                                                                                                                                                                                                                               |
+| `rlpdump`  | Developer utility tool to convert binary RLP ([Recursive Length Prefix](https://ethereum.org/en/developers/docs/data-structures-and-encoding/rlp)) dumps (data encoding used by the Ethereum protocol both network as well as consensus wise) to user-friendlier hierarchical representation (e.g. `rlpdump --hex CE0183FFFFFFC4C304050583616263`).                                                                                                                                                                                |
 
 ## Running `geth`
 
 Going through all the possible command line flags is out of scope here (please consult our
-[CLI Wiki page](https://geth.ethereum.org/docs/interface/command-line-options)),
+[CLI Wiki page](https://geth.ethereum.org/docs/fundamentals/command-line-options)),
 but we've enumerated a few common parameter combos to get you up to speed quickly
 on how you can run your own `geth` instance.
 
@@ -56,23 +68,23 @@ on how you can run your own `geth` instance.
 
 Minimum:
 
-* CPU with 2+ cores
-* 4GB RAM
-* 1TB free storage space to sync the Mainnet
-* 8 MBit/sec download Internet service
+- CPU with 2+ cores
+- 4GB RAM
+- 1TB free storage space to sync the Mainnet
+- 8 MBit/sec download Internet service
 
 Recommended:
 
 * Fast CPU with 4+ cores
 * 16GB+ RAM
-* High Performance SSD with at least 1TB free space
+* High-performance SSD with at least 1TB of free space
 * 25+ MBit/sec download Internet service
 
 ### Full node on the main Ethereum network
 
 By far the most common scenario is people wanting to simply interact with the Ethereum
 network: create accounts; transfer funds; deploy and interact with contracts. For this
-particular use-case the user doesn't care about years-old historical data, so we can
+particular use case, the user doesn't care about years-old historical data, so we can
 sync quickly to the current state of the network. To do so:
 
 ```shell
@@ -83,11 +95,11 @@ This command will:
  * Start `geth` in snap sync mode (default, can be changed with the `--syncmode` flag),
    causing it to download more data in exchange for avoiding processing the entire history
    of the Ethereum network, which is very CPU intensive.
- * Start up `geth`'s built-in interactive [JavaScript console](https://geth.ethereum.org/docs/interface/javascript-console),
+ * Start the built-in interactive [JavaScript console](https://geth.ethereum.org/docs/interacting-with-geth/javascript-console),
    (via the trailing `console` subcommand) through which you can interact using [`web3` methods](https://github.com/ChainSafe/web3.js/blob/0.20.7/DOCUMENTATION.md) 
    (note: the `web3` version bundled within `geth` is very old, and not up to date with official docs),
-   as well as `geth`'s own [management APIs](https://geth.ethereum.org/docs/rpc/server).
-   This tool is optional and if you leave it out you can always attach to an already running
+   as well as `geth`'s own [management APIs](https://geth.ethereum.org/docs/interacting-with-geth/rpc).
+   This tool is optional and if you leave it out you can always attach it to an already running
    `geth` instance with `geth attach`.
 
 ### A Full node on the Görli test network
@@ -102,12 +114,12 @@ the main network, but with play-Ether only.
 $ geth --goerli console
 ```
 
-The `console` subcommand has the exact same meaning as above and they are equally
-useful on the testnet too. Please, see above for their explanations if you've skipped here.
+The `console` subcommand has the same meaning as above and is equally
+useful on the testnet too.
 
 Specifying the `--goerli` flag, however, will reconfigure your `geth` instance a bit:
 
- * Instead of connecting the main Ethereum network, the client will connect to the Görli
+ * Instead of connecting to the main Ethereum network, the client will connect to the Görli
    test network, which uses different P2P bootnodes, different network IDs and genesis
    states.
  * Instead of using the default data directory (`~/.ethereum` on Linux for example), `geth`
@@ -118,34 +130,21 @@ Specifying the `--goerli` flag, however, will reconfigure your `geth` instance a
    `geth attach <datadir>/goerli/geth.ipc`. Windows users are not affected by
    this.
 
-*Note: Although there are some internal protective measures to prevent transactions from
-crossing over between the main network and test network, you should make sure to always
-use separate accounts for play-money and real-money. Unless you manually move
+*Note: Although some internal protective measures prevent transactions from
+crossing over between the main network and test network, you should always
+use separate accounts for play and real money. Unless you manually move
 accounts, `geth` will by default correctly separate the two networks and will not make any
-accounts available between them.*
+accounts available between them._
 
 ### Full node on the Rinkeby test network
 
 Go Ethereum also supports connecting to the older proof-of-authority based test network
-called [*Rinkeby*](https://www.rinkeby.io) which is operated by members of the community.
+called [_Rinkeby_](https://www.rinkeby.io) which is operated by members of the community.
 
 ```shell
 $ geth --rinkeby console
 ```
 
-### Full node on the Ropsten test network
-
-In addition to Görli and Rinkeby, Geth also supports the ancient Ropsten testnet. The
-Ropsten test network is based on the Ethash proof-of-work consensus algorithm. As such,
-it has certain extra overhead and is more susceptible to reorganization attacks due to the
-network's low difficulty/security.
-
-```shell
-$ geth --ropsten console
-```
-
-*Note: Older Geth configurations store the Ropsten database in the `testnet` subdirectory.*
-
 ### Configuration
 
 As an alternative to passing the numerous flags to the `geth` binary, you can also pass a
@@ -155,14 +154,14 @@ configuration file via:
 $ geth --config /path/to/your_config.toml
 ```
 
-To get an idea how the file should look like you can use the `dumpconfig` subcommand to
+To get an idea of how the file should look like you can use the `dumpconfig` subcommand to
 export your existing configuration:
 
 ```shell
 $ geth --your-favourite-flags dumpconfig
 ```
 
-*Note: This works only with `geth` v1.6.0 and above.*
+_Note: This works only with `geth` v1.6.0 and above._
 
 #### Docker quick start
 
@@ -175,7 +174,7 @@ docker run -d --name ethereum-node -v /Users/alice/ethereum:/root \
            ethereum/client-go
 ```
 
-This will start `geth` in snap-sync mode with a DB memory allowance of 1GB just as the
+This will start `geth` in snap-sync mode with a DB memory allowance of 1GB, as the
 above command does.  It will also create a persistent volume in your home directory for
 saving your blockchain as well as map the default ports. There is also an `alpine` tag
 available for a slim version of the image.
@@ -189,7 +188,7 @@ accessible from the outside.
 As a developer, sooner rather than later you'll want to start interacting with `geth` and the
 Ethereum network via your own programs and not manually through the console. To aid
 this, `geth` has built-in support for a JSON-RPC based APIs ([standard APIs](https://ethereum.github.io/execution-apis/api-documentation/)
-and [`geth` specific APIs](https://geth.ethereum.org/docs/rpc/server)).
+and [`geth` specific APIs](https://geth.ethereum.org/docs/interacting-with-geth/rpc)).
 These can be exposed via HTTP, WebSockets and IPC (UNIX sockets on UNIX based
 platforms, and named pipes on Windows).
 
@@ -209,9 +208,9 @@ HTTP based JSON-RPC API options:
   * `--ws.addr` WS-RPC server listening interface (default: `localhost`)
   * `--ws.port` WS-RPC server listening port (default: `8546`)
   * `--ws.api` API's offered over the WS-RPC interface (default: `eth,net,web3`)
-  * `--ws.origins` Origins from which to accept websockets requests
+  * `--ws.origins` Origins from which to accept WebSocket requests
   * `--ipcdisable` Disable the IPC-RPC server
-  * `--ipcapi` API's offered over the IPC-RPC interface (default: `admin,debug,eth,miner,net,personal,shh,txpool,web3`)
+  * `--ipcapi` API's offered over the IPC-RPC interface (default: `admin,debug,eth,miner,net,personal,txpool,web3`)
   * `--ipcpath` Filename for IPC socket/pipe within the datadir (explicit paths escape it)
 
 You'll need to use your own programming environments' capabilities (libraries, tools, etc) to
@@ -302,8 +301,8 @@ that other nodes can use to connect to it and exchange peer information. Make su
 replace the displayed IP address information (most probably `[::]`) with your externally
 accessible IP to get the actual `enode` URL.
 
-*Note: You could also use a full-fledged `geth` node as a bootnode, but it's the less
-recommended way.*
+_Note: You could also use a full-fledged `geth` node as a bootnode, but it's the less
+recommended way._
 
 #### Starting up your member nodes
 
@@ -317,17 +316,13 @@ do also specify a custom `--datadir` flag.
 $ geth --datadir=path/to/custom/data/folder --bootnodes=<bootnode-enode-url-from-above>
 ```
 
-*Note: Since your network will be completely cut off from the main and test networks, you'll
-also need to configure a miner to process transactions and create new blocks for you.*
+_Note: Since your network will be completely cut off from the main and test networks, you'll
+also need to configure a miner to process transactions and create new blocks for you._
 
 #### Running a private miner
 
-Mining on the public Ethereum network is a complex task as it's only feasible using GPUs,
-requiring an OpenCL or CUDA enabled `ethminer` instance. For information on such a
-setup, please consult the [EtherMining subreddit](https://www.reddit.com/r/EtherMining/)
-and the [ethminer](https://github.com/ethereum-mining/ethminer) repository.
 
-In a private network setting, however a single CPU miner instance is more than enough for
+In a private network setting a single CPU miner instance is more than enough for
 practical purposes as it can produce a stable stream of blocks at the correct intervals
 without needing heavy resources (consider running on a single thread, no need for multiple
 ones either). To start a `geth` instance for mining, run it with all your usual flags, extended
@@ -344,7 +339,7 @@ transactions are accepted at (`--miner.gasprice`).
 
 ## Contribution
 
-Thank you for considering to help out with the source code! We welcome contributions
+Thank you for considering helping out with the source code! We welcome contributions
 from anyone on the internet, and are grateful for even the smallest of fixes!
 
 If you'd like to contribute to go-ethereum, please fork, fix, commit and send a pull request
@@ -356,24 +351,30 @@ and merge procedures quick and simple.
 
 Please make sure your contributions adhere to our coding guidelines:
 
- * Code must adhere to the official Go [formatting](https://golang.org/doc/effective_go.html#formatting)
-   guidelines (i.e. uses [gofmt](https://golang.org/cmd/gofmt/)).
- * Code must be documented adhering to the official Go [commentary](https://golang.org/doc/effective_go.html#commentary)
-   guidelines.
- * Pull requests need to be based on and opened against the `master` branch.
- * Commit messages should be prefixed with the package(s) they modify.
-   * E.g. "eth, rpc: make trace configs optional"
+- Code must adhere to the official Go [formatting](https://golang.org/doc/effective_go.html#formatting)
+  guidelines (i.e. uses [gofmt](https://golang.org/cmd/gofmt/)).
+- Code must be documented adhering to the official Go [commentary](https://golang.org/doc/effective_go.html#commentary)
+  guidelines.
+- Pull requests need to be based on and opened against the `master` branch.
+- Commit messages should be prefixed with the package(s) they modify.
+  - E.g. "eth, rpc: make trace configs optional"
 
-Please see the [Developers' Guide](https://geth.ethereum.org/docs/developers/devguide)
+Please see the [Developers' Guide](https://geth.ethereum.org/docs/developers/geth-developer/dev-guide)
 for more details on configuring your environment, managing project dependencies, and
 testing procedures.
 
+### Contributing to geth.ethereum.org
+
+For contributions to the [go-ethereum website](https://geth.ethereum.org), please checkout and raise pull requests against the `website` branch.
+For more detailed instructions please see the `website` branch [README](https://github.com/ethereum/go-ethereum/tree/website#readme) or the 
+[contributing](https://geth.ethereum.org/docs/developers/geth-developer/contributing) page of the website.
+
 ## License
 
 The go-ethereum library (i.e. all code outside of the `cmd` directory) is licensed under the
 [GNU Lesser General Public License v3.0](https://www.gnu.org/licenses/lgpl-3.0.en.html),
 also included in our repository in the `COPYING.LESSER` file.
 
-The go-ethereum binaries (i.e. all code inside of the `cmd` directory) is licensed under the
+The go-ethereum binaries (i.e. all code inside of the `cmd` directory) are licensed under the
 [GNU General Public License v3.0](https://www.gnu.org/licenses/gpl-3.0.en.html), also
 included in our repository in the `COPYING` file.

accounts/abi/abi.go

@@ -87,7 +87,7 @@ func (abi ABI) getArguments(name string, data []byte) (Arguments, error) {
 	var args Arguments
 	if method, ok := abi.Methods[name]; ok {
 		if len(data)%32 != 0 {
-			return nil, fmt.Errorf("abi: improperly formatted output: %s - Bytes: [%+v]", string(data), data)
+			return nil, fmt.Errorf("abi: improperly formatted output: %q - Bytes: %+v", data, data)
 		}
 		args = method.Outputs
 	}
@@ -95,7 +95,7 @@ func (abi ABI) getArguments(name string, data []byte) (Arguments, error) {
 		args = event.Inputs
 	}
 	if args == nil {
-		return nil, errors.New("abi: could not locate named method or event")
+		return nil, fmt.Errorf("abi: could not locate named method or event: %s", name)
 	}
 	return args, nil
 }
@@ -246,7 +246,10 @@ func UnpackRevert(data []byte) (string, error) {
 	if !bytes.Equal(data[:4], revertSelector) {
 		return "", errors.New("invalid data for unpacking")
 	}
-	typ, _ := NewType("string", "", nil)
+	typ, err := NewType("string", "", nil)
+	if err != nil {
+		return "", err
+	}
 	unpacked, err := (Arguments{{Type: typ}}).Unpack(data[4:])
 	if err != nil {
 		return "", err

accounts/abi/abi_test.go

@@ -165,8 +165,9 @@ func TestInvalidABI(t *testing.T) {
 
 // TestConstructor tests a constructor function.
 // The test is based on the following contract:
-// 	contract TestConstructor {
-// 		constructor(uint256 a, uint256 b) public{}
+//
+//	contract TestConstructor {
+//		constructor(uint256 a, uint256 b) public{}
 //	}
 func TestConstructor(t *testing.T) {
 	json := `[{	"inputs": [{"internalType": "uint256","name": "a","type": "uint256"	},{	"internalType": "uint256","name": "b","type": "uint256"}],"stateMutability": "nonpayable","type": "constructor"}]`
@@ -724,16 +725,19 @@ func TestBareEvents(t *testing.T) {
 }
 
 // TestUnpackEvent is based on this contract:
-//    contract T {
-//      event received(address sender, uint amount, bytes memo);
-//      event receivedAddr(address sender);
-//      function receive(bytes memo) external payable {
-//        received(msg.sender, msg.value, memo);
-//        receivedAddr(msg.sender);
-//      }
-//    }
+//
+//	contract T {
+//		event received(address sender, uint amount, bytes memo);
+//		event receivedAddr(address sender);
+//		function receive(bytes memo) external payable {
+//			received(msg.sender, msg.value, memo);
+//			receivedAddr(msg.sender);
+//		}
+//	}
+//
 // When receive("X") is called with sender 0x00... and value 1, it produces this tx receipt:
-//   receipt{status=1 cgas=23949 bloom=00000000004000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000800000000000000000000000000000000000040200000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000080000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 logs=[log: b6818c8064f645cd82d99b59a1a267d6d61117ef [75fd880d39c1daf53b6547ab6cb59451fc6452d27caa90e5b6649dd8293b9eed] 000000000000000000000000376c47978271565f56deb45495afa69e59c16ab200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000060000000000000000000000000000000000000000000000000000000000000000158 9ae378b6d4409eada347a5dc0c180f186cb62dc68fcc0f043425eb917335aa28 0 95d429d309bb9d753954195fe2d69bd140b4ae731b9b5b605c34323de162cf00 0]}
+//
+//	receipt{status=1 cgas=23949 bloom=00000000004000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000800000000000000000000000000000000000040200000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000080000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 logs=[log: b6818c8064f645cd82d99b59a1a267d6d61117ef [75fd880d39c1daf53b6547ab6cb59451fc6452d27caa90e5b6649dd8293b9eed] 000000000000000000000000376c47978271565f56deb45495afa69e59c16ab200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000060000000000000000000000000000000000000000000000000000000000000000158 9ae378b6d4409eada347a5dc0c180f186cb62dc68fcc0f043425eb917335aa28 0 95d429d309bb9d753954195fe2d69bd140b4ae731b9b5b605c34323de162cf00 0]}
 func TestUnpackEvent(t *testing.T) {
 	const abiJSON = `[{"constant":false,"inputs":[{"name":"memo","type":"bytes"}],"name":"receive","outputs":[],"payable":true,"stateMutability":"payable","type":"function"},{"anonymous":false,"inputs":[{"indexed":false,"name":"sender","type":"address"},{"indexed":false,"name":"amount","type":"uint256"},{"indexed":false,"name":"memo","type":"bytes"}],"name":"received","type":"event"},{"anonymous":false,"inputs":[{"indexed":false,"name":"sender","type":"address"}],"name":"receivedAddr","type":"event"}]`
 	abi, err := JSON(strings.NewReader(abiJSON))
@@ -1078,8 +1082,9 @@ func TestDoubleDuplicateMethodNames(t *testing.T) {
 // TestDoubleDuplicateEventNames checks that if send0 already exists, there won't be a name
 // conflict and that the second send event will be renamed send1.
 // The test runs the abi of the following contract.
-// 	contract DuplicateEvent {
-// 		event send(uint256 a);
+//
+//	contract DuplicateEvent {
+//		event send(uint256 a);
 //		event send0();
 //		event send();
 //	}
@@ -1106,7 +1111,8 @@ func TestDoubleDuplicateEventNames(t *testing.T) {
 // TestUnnamedEventParam checks that an event with unnamed parameters is
 // correctly handled.
 // The test runs the abi of the following contract.
-// 	contract TestEvent {
+//
+//	contract TestEvent {
 //		event send(uint256, uint256);
 //	}
 func TestUnnamedEventParam(t *testing.T) {

accounts/abi/argument.go

@@ -18,6 +18,7 @@ package abi
 
 import (
 	"encoding/json"
+	"errors"
 	"fmt"
 	"reflect"
 	"strings"
@@ -79,7 +80,7 @@ func (arguments Arguments) isTuple() bool {
 func (arguments Arguments) Unpack(data []byte) ([]interface{}, error) {
 	if len(data) == 0 {
 		if len(arguments.NonIndexed()) != 0 {
-			return nil, fmt.Errorf("abi: attempting to unmarshall an empty string while arguments are expected")
+			return nil, errors.New("abi: attempting to unmarshall an empty string while arguments are expected")
 		}
 		return make([]interface{}, 0), nil
 	}
@@ -90,11 +91,11 @@ func (arguments Arguments) Unpack(data []byte) ([]interface{}, error) {
 func (arguments Arguments) UnpackIntoMap(v map[string]interface{}, data []byte) error {
 	// Make sure map is not nil
 	if v == nil {
-		return fmt.Errorf("abi: cannot unpack into a nil map")
+		return errors.New("abi: cannot unpack into a nil map")
 	}
 	if len(data) == 0 {
 		if len(arguments.NonIndexed()) != 0 {
-			return fmt.Errorf("abi: attempting to unmarshall an empty string while arguments are expected")
+			return errors.New("abi: attempting to unmarshall an empty string while arguments are expected")
 		}
 		return nil // Nothing to unmarshal, return
 	}
@@ -116,7 +117,7 @@ func (arguments Arguments) Copy(v interface{}, values []interface{}) error {
 	}
 	if len(values) == 0 {
 		if len(arguments.NonIndexed()) != 0 {
-			return fmt.Errorf("abi: attempting to copy no values while arguments are expected")
+			return errors.New("abi: attempting to copy no values while arguments are expected")
 		}
 		return nil // Nothing to copy, return
 	}
@@ -186,6 +187,9 @@ func (arguments Arguments) UnpackValues(data []byte) ([]interface{}, error) {
 	virtualArgs := 0
 	for index, arg := range nonIndexedArgs {
 		marshalledValue, err := toGoType((index+virtualArgs)*32, arg.Type, data)
+		if err != nil {
+			return nil, err
+		}
 		if arg.Type.T == ArrayTy && !isDynamicType(arg.Type) {
 			// If we have a static array, like [3]uint256, these are coded as
 			// just like uint256,uint256,uint256.
@@ -203,9 +207,6 @@ func (arguments Arguments) UnpackValues(data []byte) ([]interface{}, error) {
 			// coded as just like uint256,bool,uint256
 			virtualArgs += getTypeSize(arg.Type)/32 - 1
 		}
-		if err != nil {
-			return nil, err
-		}
 		retval = append(retval, marshalledValue)
 	}
 	return retval, nil

accounts/abi/bind/backends/simulated.go

@@ -68,7 +68,8 @@ type SimulatedBackend struct {
 	pendingState    *state.StateDB // Currently pending state that will be the active on request
 	pendingReceipts types.Receipts // Currently receipts for the pending block
 
-	events *filters.EventSystem // Event system for filtering log events live
+	events       *filters.EventSystem  // for filtering log events live
+	filterSystem *filters.FilterSystem // for filtering database logs
 
 	config *params.ChainConfig
 }
@@ -77,17 +78,27 @@ type SimulatedBackend struct {
 // and uses a simulated blockchain for testing purposes.
 // A simulated backend always uses chainID 1337.
 func NewSimulatedBackendWithDatabase(database ethdb.Database, alloc core.GenesisAlloc, gasLimit uint64) *SimulatedBackend {
-	genesis := core.Genesis{Config: params.AllEthashProtocolChanges, GasLimit: gasLimit, Alloc: alloc}
-	genesis.MustCommit(database)
-	blockchain, _ := core.NewBlockChain(database, nil, genesis.Config, ethash.NewFaker(), vm.Config{}, nil, nil)
+	genesis := core.Genesis{
+		Config:   params.AllEthashProtocolChanges,
+		GasLimit: gasLimit,
+		Alloc:    alloc,
+	}
+	blockchain, _ := core.NewBlockChain(database, nil, &genesis, nil, ethash.NewFaker(), vm.Config{}, nil, nil)
 
 	backend := &SimulatedBackend{
 		database:   database,
 		blockchain: blockchain,
 		config:     genesis.Config,
 	}
-	backend.events = filters.NewEventSystem(&filterBackend{database, blockchain, backend}, false)
-	backend.rollback(blockchain.CurrentBlock())
+
+	filterBackend := &filterBackend{database, blockchain, backend}
+	backend.filterSystem = filters.NewFilterSystem(filterBackend, filters.Config{})
+	backend.events = filters.NewEventSystem(backend.filterSystem, false)
+
+	header := backend.blockchain.CurrentBlock()
+	block := backend.blockchain.GetBlock(header.Hash(), header.Number.Uint64())
+
+	backend.rollback(block)
 	return backend
 }
 
@@ -106,16 +117,20 @@ func (b *SimulatedBackend) Close() error {
 
 // Commit imports all the pending transactions as a single block and starts a
 // fresh new state.
-func (b *SimulatedBackend) Commit() {
+func (b *SimulatedBackend) Commit() common.Hash {
 	b.mu.Lock()
 	defer b.mu.Unlock()
 
 	if _, err := b.blockchain.InsertChain([]*types.Block{b.pendingBlock}); err != nil {
 		panic(err) // This cannot happen unless the simulator is wrong, fail in that case
 	}
+	blockHash := b.pendingBlock.Hash()
+
 	// Using the last inserted block here makes it possible to build on a side
 	// chain after a fork.
 	b.rollback(b.pendingBlock)
+
+	return blockHash
 }
 
 // Rollback aborts all pending transactions, reverting to the last committed state.
@@ -123,7 +138,10 @@ func (b *SimulatedBackend) Rollback() {
 	b.mu.Lock()
 	defer b.mu.Unlock()
 
-	b.rollback(b.blockchain.CurrentBlock())
+	header := b.blockchain.CurrentBlock()
+	block := b.blockchain.GetBlock(header.Hash(), header.Number.Uint64())
+
+	b.rollback(block)
 }
 
 func (b *SimulatedBackend) rollback(parent *types.Block) {
@@ -162,7 +180,7 @@ func (b *SimulatedBackend) Fork(ctx context.Context, parent common.Hash) error {
 
 // stateByBlockNumber retrieves a state by a given blocknumber.
 func (b *SimulatedBackend) stateByBlockNumber(ctx context.Context, blockNumber *big.Int) (*state.StateDB, error) {
-	if blockNumber == nil || blockNumber.Cmp(b.blockchain.CurrentBlock().Number()) == 0 {
+	if blockNumber == nil || blockNumber.Cmp(b.blockchain.CurrentBlock().Number) == 0 {
 		return b.blockchain.State()
 	}
 	block, err := b.blockByNumber(ctx, blockNumber)
@@ -291,7 +309,7 @@ func (b *SimulatedBackend) BlockByNumber(ctx context.Context, number *big.Int) (
 // (associated with its hash) if found without Lock.
 func (b *SimulatedBackend) blockByNumber(ctx context.Context, number *big.Int) (*types.Block, error) {
 	if number == nil || number.Cmp(b.pendingBlock.Number()) == 0 {
-		return b.blockchain.CurrentBlock(), nil
+		return b.blockByHash(ctx, b.blockchain.CurrentBlock().Hash())
 	}
 
 	block := b.blockchain.GetBlockByNumber(uint64(number.Int64()))
@@ -419,7 +437,7 @@ func (b *SimulatedBackend) CallContract(ctx context.Context, call ethereum.CallM
 	b.mu.Lock()
 	defer b.mu.Unlock()
 
-	if blockNumber != nil && blockNumber.Cmp(b.blockchain.CurrentBlock().Number()) != 0 {
+	if blockNumber != nil && blockNumber.Cmp(b.blockchain.CurrentBlock().Number) != 0 {
 		return nil, errBlockNumberUnsupported
 	}
 	stateDB, err := b.blockchain.State()
@@ -443,7 +461,7 @@ func (b *SimulatedBackend) PendingCallContract(ctx context.Context, call ethereu
 	defer b.mu.Unlock()
 	defer b.pendingState.RevertToSnapshot(b.pendingState.Snapshot())
 
-	res, err := b.callContract(ctx, call, b.pendingBlock, b.pendingState)
+	res, err := b.callContract(ctx, call, b.pendingBlock.Header(), b.pendingState)
 	if err != nil {
 		return nil, err
 	}
@@ -515,7 +533,7 @@ func (b *SimulatedBackend) EstimateGas(ctx context.Context, call ethereum.CallMs
 		available := new(big.Int).Set(balance)
 		if call.Value != nil {
 			if call.Value.Cmp(available) >= 0 {
-				return 0, errors.New("insufficient funds for transfer")
+				return 0, core.ErrInsufficientFundsForTransfer
 			}
 			available.Sub(available, call.Value)
 		}
@@ -537,7 +555,7 @@ func (b *SimulatedBackend) EstimateGas(ctx context.Context, call ethereum.CallMs
 		call.Gas = gas
 
 		snapshot := b.pendingState.Snapshot()
-		res, err := b.callContract(ctx, call, b.pendingBlock, b.pendingState)
+		res, err := b.callContract(ctx, call, b.pendingBlock.Header(), b.pendingState)
 		b.pendingState.RevertToSnapshot(snapshot)
 
 		if err != nil {
@@ -587,7 +605,7 @@ func (b *SimulatedBackend) EstimateGas(ctx context.Context, call ethereum.CallMs
 
 // callContract implements common code between normal and pending contract calls.
 // state is modified during execution, make sure to copy it if necessary.
-func (b *SimulatedBackend) callContract(ctx context.Context, call ethereum.CallMsg, block *types.Block, stateDB *state.StateDB) (*core.ExecutionResult, error) {
+func (b *SimulatedBackend) callContract(ctx context.Context, call ethereum.CallMsg, header *types.Header, stateDB *state.StateDB) (*core.ExecutionResult, error) {
 	// Gas prices post 1559 need to be initialized
 	if call.GasPrice != nil && (call.GasFeeCap != nil || call.GasTipCap != nil) {
 		return nil, errors.New("both gasPrice and (maxFeePerGas or maxPriorityFeePerGas) specified")
@@ -605,7 +623,7 @@ func (b *SimulatedBackend) callContract(ctx context.Context, call ethereum.CallM
 			// User specified the legacy gas field, convert to 1559 gas typing
 			call.GasFeeCap, call.GasTipCap = call.GasPrice, call.GasPrice
 		} else {
-			// User specified 1559 gas feilds (or none), use those
+			// User specified 1559 gas fields (or none), use those
 			if call.GasFeeCap == nil {
 				call.GasFeeCap = new(big.Int)
 			}
@@ -626,20 +644,33 @@ func (b *SimulatedBackend) callContract(ctx context.Context, call ethereum.CallM
 	if call.Value == nil {
 		call.Value = new(big.Int)
 	}
+
 	// Set infinite balance to the fake caller account.
 	from := stateDB.GetOrNewStateObject(call.From)
 	from.SetBalance(math.MaxBig256)
-	// Execute the call.
-	msg := callMsg{call}
 
-	txContext := core.NewEVMTxContext(msg)
-	evmContext := core.NewEVMBlockContext(block.Header(), b.blockchain, nil)
+	// Execute the call.
+	msg := &core.Message{
+		From:              call.From,
+		To:                call.To,
+		Value:             call.Value,
+		GasLimit:          call.Gas,
+		GasPrice:          call.GasPrice,
+		GasFeeCap:         call.GasFeeCap,
+		GasTipCap:         call.GasTipCap,
+		Data:              call.Data,
+		AccessList:        call.AccessList,
+		SkipAccountChecks: true,
+	}
+
 	// Create a new environment which holds all relevant information
 	// about the transaction and calling mechanisms.
+	txContext := core.NewEVMTxContext(msg)
+	evmContext := core.NewEVMBlockContext(header, b.blockchain, nil)
 	vmEnv := vm.NewEVM(evmContext, txContext, stateDB, b.config, vm.Config{NoBaseFee: true})
 	gasPool := new(core.GasPool).AddGas(math.MaxUint64)
 
-	return core.NewStateTransition(vmEnv, msg, gasPool).TransitionDb()
+	return core.ApplyMessage(vmEnv, msg, gasPool)
 }
 
 // SendTransaction updates the pending block to include the given transaction.
@@ -685,7 +716,7 @@ func (b *SimulatedBackend) FilterLogs(ctx context.Context, query ethereum.Filter
 	var filter *filters.Filter
 	if query.BlockHash != nil {
 		// Block filter requested, construct a single-shot filter
-		filter = filters.NewBlockFilter(&filterBackend{b.database, b.blockchain, b}, *query.BlockHash, query.Addresses, query.Topics)
+		filter = b.filterSystem.NewBlockFilter(*query.BlockHash, query.Addresses, query.Topics)
 	} else {
 		// Initialize unset filter boundaries to run from genesis to chain head
 		from := int64(0)
@@ -697,7 +728,7 @@ func (b *SimulatedBackend) FilterLogs(ctx context.Context, query ethereum.Filter
 			to = query.ToBlock.Int64()
 		}
 		// Construct the range filter
-		filter = filters.NewRangeFilter(&filterBackend{b.database, b.blockchain, b}, from, to, query.Addresses, query.Topics)
+		filter = b.filterSystem.NewRangeFilter(from, to, query.Addresses, query.Topics)
 	}
 	// Run the filter and return all the logs
 	logs, err := filter.Logs(ctx)
@@ -781,8 +812,13 @@ func (b *SimulatedBackend) AdjustTime(adjustment time.Duration) error {
 	if len(b.pendingBlock.Transactions()) != 0 {
 		return errors.New("Could not adjust time on non-empty block")
 	}
+	// Get the last block
+	block := b.blockchain.GetBlockByHash(b.pendingBlock.ParentHash())
+	if block == nil {
+		return fmt.Errorf("could not find parent")
+	}
 
-	blocks, _ := core.GenerateChain(b.config, b.blockchain.CurrentBlock(), ethash.NewFaker(), b.database, 1, func(number int, block *core.BlockGen) {
+	blocks, _ := core.GenerateChain(b.config, block, ethash.NewFaker(), b.database, 1, func(number int, block *core.BlockGen) {
 		block.OffsetTime(int64(adjustment.Seconds()))
 	})
 	stateDB, _ := b.blockchain.State()
@@ -798,23 +834,6 @@ func (b *SimulatedBackend) Blockchain() *core.BlockChain {
 	return b.blockchain
 }
 
-// callMsg implements core.Message to allow passing it as a transaction simulator.
-type callMsg struct {
-	ethereum.CallMsg
-}
-
-func (m callMsg) From() common.Address         { return m.CallMsg.From }
-func (m callMsg) Nonce() uint64                { return 0 }
-func (m callMsg) IsFake() bool                 { return true }
-func (m callMsg) To() *common.Address          { return m.CallMsg.To }
-func (m callMsg) GasPrice() *big.Int           { return m.CallMsg.GasPrice }
-func (m callMsg) GasFeeCap() *big.Int          { return m.CallMsg.GasFeeCap }
-func (m callMsg) GasTipCap() *big.Int          { return m.CallMsg.GasTipCap }
-func (m callMsg) Gas() uint64                  { return m.CallMsg.Gas }
-func (m callMsg) Value() *big.Int              { return m.CallMsg.Value }
-func (m callMsg) Data() []byte                 { return m.CallMsg.Data }
-func (m callMsg) AccessList() types.AccessList { return m.CallMsg.AccessList }
-
 // filterBackend implements filters.Backend to support filtering for logs without
 // taking bloom-bits acceleration structures into account.
 type filterBackend struct {
@@ -823,20 +842,39 @@ type filterBackend struct {
 	backend *SimulatedBackend
 }
 
-func (fb *filterBackend) ChainDb() ethdb.Database  { return fb.db }
+func (fb *filterBackend) ChainDb() ethdb.Database { return fb.db }
+
 func (fb *filterBackend) EventMux() *event.TypeMux { panic("not supported") }
 
-func (fb *filterBackend) HeaderByNumber(ctx context.Context, block rpc.BlockNumber) (*types.Header, error) {
-	if block == rpc.LatestBlockNumber {
+func (fb *filterBackend) HeaderByNumber(ctx context.Context, number rpc.BlockNumber) (*types.Header, error) {
+	switch number {
+	case rpc.PendingBlockNumber:
+		if block := fb.backend.pendingBlock; block != nil {
+			return block.Header(), nil
+		}
+		return nil, nil
+	case rpc.LatestBlockNumber:
 		return fb.bc.CurrentHeader(), nil
+	case rpc.FinalizedBlockNumber:
+		return fb.bc.CurrentFinalBlock(), nil
+	case rpc.SafeBlockNumber:
+		return fb.bc.CurrentSafeBlock(), nil
+	default:
+		return fb.bc.GetHeaderByNumber(uint64(number.Int64())), nil
 	}
-	return fb.bc.GetHeaderByNumber(uint64(block.Int64())), nil
 }
 
 func (fb *filterBackend) HeaderByHash(ctx context.Context, hash common.Hash) (*types.Header, error) {
 	return fb.bc.GetHeaderByHash(hash), nil
 }
 
+func (fb *filterBackend) GetBody(ctx context.Context, hash common.Hash, number rpc.BlockNumber) (*types.Body, error) {
+	if body := fb.bc.GetBody(hash); body != nil {
+		return body, nil
+	}
+	return nil, errors.New("block body not found")
+}
+
 func (fb *filterBackend) PendingBlockAndReceipts() (*types.Block, types.Receipts) {
 	return fb.backend.pendingBlock, fb.backend.pendingReceipts
 }
@@ -849,19 +887,8 @@ func (fb *filterBackend) GetReceipts(ctx context.Context, hash common.Hash) (typ
 	return rawdb.ReadReceipts(fb.db, hash, *number, fb.bc.Config()), nil
 }
 
-func (fb *filterBackend) GetLogs(ctx context.Context, hash common.Hash) ([][]*types.Log, error) {
-	number := rawdb.ReadHeaderNumber(fb.db, hash)
-	if number == nil {
-		return nil, nil
-	}
-	receipts := rawdb.ReadReceipts(fb.db, hash, *number, fb.bc.Config())
-	if receipts == nil {
-		return nil, nil
-	}
-	logs := make([][]*types.Log, len(receipts))
-	for i, receipt := range receipts {
-		logs[i] = receipt.Logs
-	}
+func (fb *filterBackend) GetLogs(ctx context.Context, hash common.Hash, number uint64) ([][]*types.Log, error) {
+	logs := rawdb.ReadLogs(fb.db, hash, number, fb.bc.Config())
 	return logs, nil
 }
 
@@ -891,6 +918,14 @@ func (fb *filterBackend) ServiceFilter(ctx context.Context, ms *bloombits.Matche
 	panic("not supported")
 }
 
+func (fb *filterBackend) ChainConfig() *params.ChainConfig {
+	panic("not supported")
+}
+
+func (fb *filterBackend) CurrentHeader() *types.Header {
+	panic("not supported")
+}
+
 func nullSubscription() event.Subscription {
 	return event.NewSubscription(func(quit <-chan struct{}) error {
 		<-quit

accounts/abi/bind/backends/simulated_test.go

@@ -93,17 +93,18 @@ func TestSimulatedBackend(t *testing.T) {
 
 var testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
 
-//  the following is based on this contract:
-//  contract T {
-//  	event received(address sender, uint amount, bytes memo);
-//  	event receivedAddr(address sender);
+// the following is based on this contract:
 //
-//  	function receive(bytes calldata memo) external payable returns (string memory res) {
-//  		emit received(msg.sender, msg.value, memo);
-//  		emit receivedAddr(msg.sender);
-//		    return "hello world";
-//  	}
-//  }
+//	 contract T {
+//	 	event received(address sender, uint amount, bytes memo);
+//	 	event receivedAddr(address sender);
+//
+//	 	function receive(bytes calldata memo) external payable returns (string memory res) {
+//	 		emit received(msg.sender, msg.value, memo);
+//	 		emit receivedAddr(msg.sender);
+//			return "hello world";
+//	 	}
+//	 }
 const abiJSON = `[ { "constant": false, "inputs": [ { "name": "memo", "type": "bytes" } ], "name": "receive", "outputs": [ { "name": "res", "type": "string" } ], "payable": true, "stateMutability": "payable", "type": "function" }, { "anonymous": false, "inputs": [ { "indexed": false, "name": "sender", "type": "address" }, { "indexed": false, "name": "amount", "type": "uint256" }, { "indexed": false, "name": "memo", "type": "bytes" } ], "name": "received", "type": "event" }, { "anonymous": false, "inputs": [ { "indexed": false, "name": "sender", "type": "address" } ], "name": "receivedAddr", "type": "event" } ]`
 const abiBin = `0x608060405234801561001057600080fd5b506102a0806100206000396000f3fe60806040526004361061003b576000357c010000000000000000000000000000000000000000000000000000000090048063a69b6ed014610040575b600080fd5b6100b76004803603602081101561005657600080fd5b810190808035906020019064010000000081111561007357600080fd5b82018360208201111561008557600080fd5b803590602001918460018302840111640100000000831117156100a757600080fd5b9091929391929390505050610132565b6040518080602001828103825283818151815260200191508051906020019080838360005b838110156100f75780820151818401526020810190506100dc565b50505050905090810190601f1680156101245780820380516001836020036101000a031916815260200191505b509250505060405180910390f35b60607f75fd880d39c1daf53b6547ab6cb59451fc6452d27caa90e5b6649dd8293b9eed33348585604051808573ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff168152602001848152602001806020018281038252848482818152602001925080828437600081840152601f19601f8201169050808301925050509550505050505060405180910390a17f46923992397eac56cf13058aced2a1871933622717e27b24eabc13bf9dd329c833604051808273ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff16815260200191505060405180910390a16040805190810160405280600b81526020017f68656c6c6f20776f726c6400000000000000000000000000000000000000000081525090509291505056fea165627a7a72305820ff0c57dad254cfeda48c9cfb47f1353a558bccb4d1bc31da1dae69315772d29e0029`
 const deployedCode = `60806040526004361061003b576000357c010000000000000000000000000000000000000000000000000000000090048063a69b6ed014610040575b600080fd5b6100b76004803603602081101561005657600080fd5b810190808035906020019064010000000081111561007357600080fd5b82018360208201111561008557600080fd5b803590602001918460018302840111640100000000831117156100a757600080fd5b9091929391929390505050610132565b6040518080602001828103825283818151815260200191508051906020019080838360005b838110156100f75780820151818401526020810190506100dc565b50505050905090810190601f1680156101245780820380516001836020036101000a031916815260200191505b509250505060405180910390f35b60607f75fd880d39c1daf53b6547ab6cb59451fc6452d27caa90e5b6649dd8293b9eed33348585604051808573ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff168152602001848152602001806020018281038252848482818152602001925080828437600081840152601f19601f8201169050808301925050509550505050505060405180910390a17f46923992397eac56cf13058aced2a1871933622717e27b24eabc13bf9dd329c833604051808273ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff16815260200191505060405180910390a16040805190810160405280600b81526020017f68656c6c6f20776f726c6400000000000000000000000000000000000000000081525090509291505056fea165627a7a72305820ff0c57dad254cfeda48c9cfb47f1353a558bccb4d1bc31da1dae69315772d29e0029`
@@ -417,12 +418,13 @@ func TestEstimateGas(t *testing.T) {
 	/*
 		pragma solidity ^0.6.4;
 		contract GasEstimation {
-		    function PureRevert() public { revert(); }
-		    function Revert() public { revert("revert reason");}
-		    function OOG() public { for (uint i = 0; ; i++) {}}
-		    function Assert() public { assert(false);}
-		    function Valid() public {}
-		}*/
+			function PureRevert() public { revert(); }
+			function Revert() public { revert("revert reason");}
+			function OOG() public { for (uint i = 0; ; i++) {}}
+			function Assert() public { assert(false);}
+			function Valid() public {}
+		}
+	*/
 	const contractAbi = "[{\"inputs\":[],\"name\":\"Assert\",\"outputs\":[],\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"OOG\",\"outputs\":[],\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"PureRevert\",\"outputs\":[],\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"Revert\",\"outputs\":[],\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"Valid\",\"outputs\":[],\"stateMutability\":\"nonpayable\",\"type\":\"function\"}]"
 	const contractBin = "0x60806040523480156100115760006000fd5b50610017565b61016e806100266000396000f3fe60806040523480156100115760006000fd5b506004361061005c5760003560e01c806350f6fe3414610062578063aa8b1d301461006c578063b9b046f914610076578063d8b9839114610080578063e09fface1461008a5761005c565b60006000fd5b61006a610094565b005b6100746100ad565b005b61007e6100b5565b005b6100886100c2565b005b610092610135565b005b6000600090505b5b808060010191505061009b565b505b565b60006000fd5b565b600015156100bf57fe5b5b565b6040517f08c379a000000000000000000000000000000000000000000000000000000000815260040180806020018281038252600d8152602001807f72657665727420726561736f6e0000000000000000000000000000000000000081526020015060200191505060405180910390fd5b565b5b56fea2646970667358221220345bbcbb1a5ecf22b53a78eaebf95f8ee0eceff6d10d4b9643495084d2ec934a64736f6c63430006040033"
 
@@ -994,7 +996,8 @@ func TestCodeAt(t *testing.T) {
 }
 
 // When receive("X") is called with sender 0x00... and value 1, it produces this tx receipt:
-//   receipt{status=1 cgas=23949 bloom=00000000004000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000800000000000000000000000000000000000040200000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000080000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 logs=[log: b6818c8064f645cd82d99b59a1a267d6d61117ef [75fd880d39c1daf53b6547ab6cb59451fc6452d27caa90e5b6649dd8293b9eed] 000000000000000000000000376c47978271565f56deb45495afa69e59c16ab200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000060000000000000000000000000000000000000000000000000000000000000000158 9ae378b6d4409eada347a5dc0c180f186cb62dc68fcc0f043425eb917335aa28 0 95d429d309bb9d753954195fe2d69bd140b4ae731b9b5b605c34323de162cf00 0]}
+//
+//	receipt{status=1 cgas=23949 bloom=00000000004000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000800000000000000000000000000000000000040200000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000080000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 logs=[log: b6818c8064f645cd82d99b59a1a267d6d61117ef [75fd880d39c1daf53b6547ab6cb59451fc6452d27caa90e5b6649dd8293b9eed] 000000000000000000000000376c47978271565f56deb45495afa69e59c16ab200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000060000000000000000000000000000000000000000000000000000000000000000158 9ae378b6d4409eada347a5dc0c180f186cb62dc68fcc0f043425eb917335aa28 0 95d429d309bb9d753954195fe2d69bd140b4ae731b9b5b605c34323de162cf00 0]}
 func TestPendingAndCallContract(t *testing.T) {
 	testAddr := crypto.PubkeyToAddress(testKey.PublicKey)
 	sim := simTestBackend(testAddr)
@@ -1057,27 +1060,27 @@ func TestPendingAndCallContract(t *testing.T) {
 // This test is based on the following contract:
 /*
 contract Reverter {
-    function revertString() public pure{
-        require(false, "some error");
-    }
-    function revertNoString() public pure {
-        require(false, "");
-    }
-    function revertASM() public pure {
-        assembly {
-            revert(0x0, 0x0)
-        }
-    }
-    function noRevert() public pure {
-        assembly {
-            // Assembles something that looks like require(false, "some error") but is not reverted
-            mstore(0x0, 0x08c379a000000000000000000000000000000000000000000000000000000000)
-            mstore(0x4, 0x0000000000000000000000000000000000000000000000000000000000000020)
-            mstore(0x24, 0x000000000000000000000000000000000000000000000000000000000000000a)
-            mstore(0x44, 0x736f6d65206572726f7200000000000000000000000000000000000000000000)
-            return(0x0, 0x64)
-        }
-    }
+	function revertString() public pure{
+		require(false, "some error");
+	}
+	function revertNoString() public pure {
+		require(false, "");
+	}
+	function revertASM() public pure {
+		assembly {
+			revert(0x0, 0x0)
+		}
+	}
+	function noRevert() public pure {
+		assembly {
+			// Assembles something that looks like require(false, "some error") but is not reverted
+			mstore(0x0, 0x08c379a000000000000000000000000000000000000000000000000000000000)
+			mstore(0x4, 0x0000000000000000000000000000000000000000000000000000000000000020)
+			mstore(0x24, 0x000000000000000000000000000000000000000000000000000000000000000a)
+			mstore(0x44, 0x736f6d65206572726f7200000000000000000000000000000000000000000000)
+			return(0x0, 0x64)
+		}
+	}
 }*/
 func TestCallContractRevert(t *testing.T) {
 	testAddr := crypto.PubkeyToAddress(testKey.PublicKey)
@@ -1186,7 +1189,7 @@ func TestFork(t *testing.T) {
 		sim.Commit()
 	}
 	// 3.
-	if sim.blockchain.CurrentBlock().NumberU64() != uint64(n) {
+	if sim.blockchain.CurrentBlock().Number.Uint64() != uint64(n) {
 		t.Error("wrong chain length")
 	}
 	// 4.
@@ -1196,7 +1199,7 @@ func TestFork(t *testing.T) {
 		sim.Commit()
 	}
 	// 6.
-	if sim.blockchain.CurrentBlock().NumberU64() != uint64(n+1) {
+	if sim.blockchain.CurrentBlock().Number.Uint64() != uint64(n+1) {
 		t.Error("wrong chain length")
 	}
 }
@@ -1204,11 +1207,11 @@ func TestFork(t *testing.T) {
 /*
 Example contract to test event emission:
 
-pragma solidity >=0.7.0 <0.9.0;
-contract Callable {
-    event Called();
-    function Call() public { emit Called(); }
-}
+	pragma solidity >=0.7.0 <0.9.0;
+	contract Callable {
+		event Called();
+		function Call() public { emit Called(); }
+	}
 */
 const callableAbi = "[{\"anonymous\":false,\"inputs\":[],\"name\":\"Called\",\"type\":\"event\"},{\"inputs\":[],\"name\":\"Call\",\"outputs\":[],\"stateMutability\":\"nonpayable\",\"type\":\"function\"}]"
 
@@ -1226,7 +1229,7 @@ const callableBin = "6080604052348015600f57600080fd5b5060998061001e6000396000f3f
 //  7. Mine two blocks to trigger a reorg.
 //  8. Check that the event was removed.
 //  9. Re-send the transaction and mine a block.
-// 10. Check that the event was reborn.
+//  10. Check that the event was reborn.
 func TestForkLogsReborn(t *testing.T) {
 	testAddr := crypto.PubkeyToAddress(testKey.PublicKey)
 	sim := simTestBackend(testAddr)
@@ -1335,3 +1338,62 @@ func TestForkResendTx(t *testing.T) {
 		t.Errorf("TX included in wrong block: %d", h)
 	}
 }
+
+func TestCommitReturnValue(t *testing.T) {
+	testAddr := crypto.PubkeyToAddress(testKey.PublicKey)
+	sim := simTestBackend(testAddr)
+	defer sim.Close()
+
+	startBlockHeight := sim.blockchain.CurrentBlock().Number.Uint64()
+
+	// Test if Commit returns the correct block hash
+	h1 := sim.Commit()
+	if h1 != sim.blockchain.CurrentBlock().Hash() {
+		t.Error("Commit did not return the hash of the last block.")
+	}
+
+	// Create a block in the original chain (containing a transaction to force different block hashes)
+	head, _ := sim.HeaderByNumber(context.Background(), nil) // Should be child's, good enough
+	gasPrice := new(big.Int).Add(head.BaseFee, big.NewInt(1))
+	_tx := types.NewTransaction(0, testAddr, big.NewInt(1000), params.TxGas, gasPrice, nil)
+	tx, _ := types.SignTx(_tx, types.HomesteadSigner{}, testKey)
+	sim.SendTransaction(context.Background(), tx)
+	h2 := sim.Commit()
+
+	// Create another block in the original chain
+	sim.Commit()
+
+	// Fork at the first bock
+	if err := sim.Fork(context.Background(), h1); err != nil {
+		t.Errorf("forking: %v", err)
+	}
+
+	// Test if Commit returns the correct block hash after the reorg
+	h2fork := sim.Commit()
+	if h2 == h2fork {
+		t.Error("The block in the fork and the original block are the same block!")
+	}
+	if sim.blockchain.GetHeader(h2fork, startBlockHeight+2) == nil {
+		t.Error("Could not retrieve the just created block (side-chain)")
+	}
+}
+
+// TestAdjustTimeAfterFork ensures that after a fork, AdjustTime uses the pending fork
+// block's parent rather than the canonical head's parent.
+func TestAdjustTimeAfterFork(t *testing.T) {
+	testAddr := crypto.PubkeyToAddress(testKey.PublicKey)
+	sim := simTestBackend(testAddr)
+	defer sim.Close()
+
+	sim.Commit() // h1
+	h1 := sim.blockchain.CurrentHeader().Hash()
+	sim.Commit() // h2
+	sim.Fork(context.Background(), h1)
+	sim.AdjustTime(1 * time.Second)
+	sim.Commit()
+
+	head := sim.blockchain.CurrentHeader()
+	if head.Number == common.Big2 && head.ParentHash != h1 {
+		t.Errorf("failed to build block on fork")
+	}
+}

accounts/abi/bind/base.go

@@ -32,6 +32,13 @@ import (
 	"github.com/ethereum/go-ethereum/event"
 )
 
+const basefeeWiggleMultiplier = 2
+
+var (
+	errNoEventSignature       = errors.New("no event signature")
+	errEventSignatureMismatch = errors.New("event signature mismatch")
+)
+
 // SignerFn is a signer function callback when a contract requires a method to
 // sign the transaction before submission.
 type SignerFn func(common.Address, *types.Transaction) (*types.Transaction, error)
@@ -254,7 +261,7 @@ func (c *BoundContract) createDynamicTx(opts *TransactOpts, contract *common.Add
 	if gasFeeCap == nil {
 		gasFeeCap = new(big.Int).Add(
 			gasTipCap,
-			new(big.Int).Mul(head.BaseFee, big.NewInt(2)),
+			new(big.Int).Mul(head.BaseFee, big.NewInt(basefeeWiggleMultiplier)),
 		)
 	}
 	if gasFeeCap.Cmp(gasTipCap) < 0 {
@@ -371,6 +378,8 @@ func (c *BoundContract) transact(opts *TransactOpts, contract *common.Address, i
 	)
 	if opts.GasPrice != nil {
 		rawTx, err = c.createLegacyTx(opts, contract, input)
+	} else if opts.GasFeeCap != nil && opts.GasTipCap != nil {
+		rawTx, err = c.createDynamicTx(opts, contract, input, nil)
 	} else {
 		// Only query for basefee if gasPrice not specified
 		if head, errHead := c.transactor.HeaderByNumber(ensureContext(opts.Context), nil); errHead != nil {
@@ -484,8 +493,12 @@ func (c *BoundContract) WatchLogs(opts *WatchOpts, name string, query ...[]inter
 
 // UnpackLog unpacks a retrieved log into the provided output structure.
 func (c *BoundContract) UnpackLog(out interface{}, event string, log types.Log) error {
+	// Anonymous events are not supported.
+	if len(log.Topics) == 0 {
+		return errNoEventSignature
+	}
 	if log.Topics[0] != c.abi.Events[event].ID {
-		return fmt.Errorf("event signature mismatch")
+		return errEventSignatureMismatch
 	}
 	if len(log.Data) > 0 {
 		if err := c.abi.UnpackIntoInterface(out, event, log.Data); err != nil {
@@ -503,8 +516,12 @@ func (c *BoundContract) UnpackLog(out interface{}, event string, log types.Log)
 
 // UnpackLogIntoMap unpacks a retrieved log into the provided map.
 func (c *BoundContract) UnpackLogIntoMap(out map[string]interface{}, event string, log types.Log) error {
+	// Anonymous events are not supported.
+	if len(log.Topics) == 0 {
+		return errNoEventSignature
+	}
 	if log.Topics[0] != c.abi.Events[event].ID {
-		return fmt.Errorf("event signature mismatch")
+		return errEventSignatureMismatch
 	}
 	if len(log.Data) > 0 {
 		if err := c.abi.UnpackIntoMap(out, event, log.Data); err != nil {

accounts/abi/bind/base_test.go

@@ -115,7 +115,6 @@ func (mc *mockPendingCaller) PendingCallContract(ctx context.Context, call ether
 }
 
 func TestPassingBlockNumber(t *testing.T) {
-
 	mc := &mockPendingCaller{
 		mockCaller: &mockCaller{
 			codeAtBytes: []byte{1, 2, 3},
@@ -187,6 +186,23 @@ func TestUnpackIndexedStringTyLogIntoMap(t *testing.T) {
 	unpackAndCheck(t, bc, expectedReceivedMap, mockLog)
 }
 
+func TestUnpackAnonymousLogIntoMap(t *testing.T) {
+	mockLog := newMockLog(nil, common.HexToHash("0x0"))
+
+	abiString := `[{"anonymous":false,"inputs":[{"indexed":false,"name":"amount","type":"uint256"}],"name":"received","type":"event"}]`
+	parsedAbi, _ := abi.JSON(strings.NewReader(abiString))
+	bc := bind.NewBoundContract(common.HexToAddress("0x0"), parsedAbi, nil, nil, nil)
+
+	var received map[string]interface{}
+	err := bc.UnpackLogIntoMap(received, "received", mockLog)
+	if err == nil {
+		t.Error("unpacking anonymous event is not supported")
+	}
+	if err.Error() != "no event signature" {
+		t.Errorf("expected error 'no event signature', got '%s'", err)
+	}
+}
+
 func TestUnpackIndexedSliceTyLogIntoMap(t *testing.T) {
 	sliceBytes, err := rlp.EncodeToBytes([]string{"name1", "name2", "name3", "name4"})
 	if err != nil {

accounts/abi/bind/bind.go

@@ -22,7 +22,6 @@ package bind
 
 import (
 	"bytes"
-	"errors"
 	"fmt"
 	"go/format"
 	"regexp"
@@ -39,10 +38,45 @@ type Lang int
 
 const (
 	LangGo Lang = iota
-	LangJava
-	LangObjC
 )
 
+func isKeyWord(arg string) bool {
+	switch arg {
+	case "break":
+	case "case":
+	case "chan":
+	case "const":
+	case "continue":
+	case "default":
+	case "defer":
+	case "else":
+	case "fallthrough":
+	case "for":
+	case "func":
+	case "go":
+	case "goto":
+	case "if":
+	case "import":
+	case "interface":
+	case "iota":
+	case "map":
+	case "make":
+	case "new":
+	case "package":
+	case "range":
+	case "return":
+	case "select":
+	case "struct":
+	case "switch":
+	case "type":
+	case "var":
+	default:
+		return false
+	}
+
+	return true
+}
+
 // Bind generates a Go wrapper around a contract ABI. This wrapper isn't meant
 // to be used as is in client code, but rather as an intermediate struct which
 // enforces compile time type safety and naming convention opposed to having to
@@ -114,7 +148,7 @@ func Bind(types []string, abis []string, bytecodes []string, fsigs []map[string]
 			normalized.Inputs = make([]abi.Argument, len(original.Inputs))
 			copy(normalized.Inputs, original.Inputs)
 			for j, input := range normalized.Inputs {
-				if input.Name == "" {
+				if input.Name == "" || isKeyWord(input.Name) {
 					normalized.Inputs[j].Name = fmt.Sprintf("arg%d", j)
 				}
 				if hasStruct(input.Type) {
@@ -158,7 +192,7 @@ func Bind(types []string, abis []string, bytecodes []string, fsigs []map[string]
 			normalized.Inputs = make([]abi.Argument, len(original.Inputs))
 			copy(normalized.Inputs, original.Inputs)
 			for j, input := range normalized.Inputs {
-				if input.Name == "" {
+				if input.Name == "" || isKeyWord(input.Name) {
 					normalized.Inputs[j].Name = fmt.Sprintf("arg%d", j)
 				}
 				// Event is a bit special, we need to define event struct in binding,
@@ -184,11 +218,6 @@ func Bind(types []string, abis []string, bytecodes []string, fsigs []map[string]
 		if evmABI.HasReceive() {
 			receive = &tmplMethod{Original: evmABI.Receive}
 		}
-		// There is no easy way to pass arbitrary java objects to the Go side.
-		if len(structs) > 0 && lang == LangJava {
-			return "", errors.New("java binding for tuple arguments is not supported yet")
-		}
-
 		contracts[types[i]] = &tmplContract{
 			Type:        capitalise(types[i]),
 			InputABI:    strings.ReplaceAll(strippedABI, "\"", "\\\""),
@@ -261,8 +290,7 @@ func Bind(types []string, abis []string, bytecodes []string, fsigs []map[string]
 // bindType is a set of type binders that convert Solidity types to some supported
 // programming language types.
 var bindType = map[Lang]func(kind abi.Type, structs map[string]*tmplStruct) string{
-	LangGo:   bindTypeGo,
-	LangJava: bindTypeJava,
+	LangGo: bindTypeGo,
 }
 
 // bindBasicTypeGo converts basic solidity types(except array, slice and tuple) to Go ones.
@@ -305,86 +333,10 @@ func bindTypeGo(kind abi.Type, structs map[string]*tmplStruct) string {
 	}
 }
 
-// bindBasicTypeJava converts basic solidity types(except array, slice and tuple) to Java ones.
-func bindBasicTypeJava(kind abi.Type) string {
-	switch kind.T {
-	case abi.AddressTy:
-		return "Address"
-	case abi.IntTy, abi.UintTy:
-		// Note that uint and int (without digits) are also matched,
-		// these are size 256, and will translate to BigInt (the default).
-		parts := regexp.MustCompile(`(u)?int([0-9]*)`).FindStringSubmatch(kind.String())
-		if len(parts) != 3 {
-			return kind.String()
-		}
-		// All unsigned integers should be translated to BigInt since gomobile doesn't
-		// support them.
-		if parts[1] == "u" {
-			return "BigInt"
-		}
-
-		namedSize := map[string]string{
-			"8":  "byte",
-			"16": "short",
-			"32": "int",
-			"64": "long",
-		}[parts[2]]
-
-		// default to BigInt
-		if namedSize == "" {
-			namedSize = "BigInt"
-		}
-		return namedSize
-	case abi.FixedBytesTy, abi.BytesTy:
-		return "byte[]"
-	case abi.BoolTy:
-		return "boolean"
-	case abi.StringTy:
-		return "String"
-	case abi.FunctionTy:
-		return "byte[24]"
-	default:
-		return kind.String()
-	}
-}
-
-// pluralizeJavaType explicitly converts multidimensional types to predefined
-// types in go side.
-func pluralizeJavaType(typ string) string {
-	switch typ {
-	case "boolean":
-		return "Bools"
-	case "String":
-		return "Strings"
-	case "Address":
-		return "Addresses"
-	case "byte[]":
-		return "Binaries"
-	case "BigInt":
-		return "BigInts"
-	}
-	return typ + "[]"
-}
-
-// bindTypeJava converts a Solidity type to a Java one. Since there is no clear mapping
-// from all Solidity types to Java ones (e.g. uint17), those that cannot be exactly
-// mapped will use an upscaled type (e.g. BigDecimal).
-func bindTypeJava(kind abi.Type, structs map[string]*tmplStruct) string {
-	switch kind.T {
-	case abi.TupleTy:
-		return structs[kind.TupleRawName+kind.String()].Name
-	case abi.ArrayTy, abi.SliceTy:
-		return pluralizeJavaType(bindTypeJava(*kind.Elem, structs))
-	default:
-		return bindBasicTypeJava(kind)
-	}
-}
-
 // bindTopicType is a set of type binders that convert Solidity types to some
 // supported programming language topic types.
 var bindTopicType = map[Lang]func(kind abi.Type, structs map[string]*tmplStruct) string{
-	LangGo:   bindTopicTypeGo,
-	LangJava: bindTopicTypeJava,
+	LangGo: bindTopicTypeGo,
 }
 
 // bindTopicTypeGo converts a Solidity topic type to a Go one. It is almost the same
@@ -404,28 +356,10 @@ func bindTopicTypeGo(kind abi.Type, structs map[string]*tmplStruct) string {
 	return bound
 }
 
-// bindTopicTypeJava converts a Solidity topic type to a Java one. It is almost the same
-// functionality as for simple types, but dynamic types get converted to hashes.
-func bindTopicTypeJava(kind abi.Type, structs map[string]*tmplStruct) string {
-	bound := bindTypeJava(kind, structs)
-
-	// todo(rjl493456442) according solidity documentation, indexed event
-	// parameters that are not value types i.e. arrays and structs are not
-	// stored directly but instead a keccak256-hash of an encoding is stored.
-	//
-	// We only convert strings and bytes to hash, still need to deal with
-	// array(both fixed-size and dynamic-size) and struct.
-	if bound == "String" || bound == "byte[]" {
-		bound = "Hash"
-	}
-	return bound
-}
-
 // bindStructType is a set of type binders that convert Solidity tuple types to some supported
 // programming language struct definition.
 var bindStructType = map[Lang]func(kind abi.Type, structs map[string]*tmplStruct) string{
-	LangGo:   bindStructTypeGo,
-	LangJava: bindStructTypeJava,
+	LangGo: bindStructTypeGo,
 }
 
 // bindStructTypeGo converts a Solidity tuple type to a Go one and records the mapping
@@ -474,74 +408,10 @@ func bindStructTypeGo(kind abi.Type, structs map[string]*tmplStruct) string {
 	}
 }
 
-// bindStructTypeJava converts a Solidity tuple type to a Java one and records the mapping
-// in the given map.
-// Notably, this function will resolve and record nested struct recursively.
-func bindStructTypeJava(kind abi.Type, structs map[string]*tmplStruct) string {
-	switch kind.T {
-	case abi.TupleTy:
-		// We compose a raw struct name and a canonical parameter expression
-		// together here. The reason is before solidity v0.5.11, kind.TupleRawName
-		// is empty, so we use canonical parameter expression to distinguish
-		// different struct definition. From the consideration of backward
-		// compatibility, we concat these two together so that if kind.TupleRawName
-		// is not empty, it can have unique id.
-		id := kind.TupleRawName + kind.String()
-		if s, exist := structs[id]; exist {
-			return s.Name
-		}
-		var fields []*tmplField
-		for i, elem := range kind.TupleElems {
-			field := bindStructTypeJava(*elem, structs)
-			fields = append(fields, &tmplField{Type: field, Name: decapitalise(kind.TupleRawNames[i]), SolKind: *elem})
-		}
-		name := kind.TupleRawName
-		if name == "" {
-			name = fmt.Sprintf("Class%d", len(structs))
-		}
-		structs[id] = &tmplStruct{
-			Name:   name,
-			Fields: fields,
-		}
-		return name
-	case abi.ArrayTy, abi.SliceTy:
-		return pluralizeJavaType(bindStructTypeJava(*kind.Elem, structs))
-	default:
-		return bindBasicTypeJava(kind)
-	}
-}
-
 // namedType is a set of functions that transform language specific types to
 // named versions that may be used inside method names.
 var namedType = map[Lang]func(string, abi.Type) string{
-	LangGo:   func(string, abi.Type) string { panic("this shouldn't be needed") },
-	LangJava: namedTypeJava,
-}
-
-// namedTypeJava converts some primitive data types to named variants that can
-// be used as parts of method names.
-func namedTypeJava(javaKind string, solKind abi.Type) string {
-	switch javaKind {
-	case "byte[]":
-		return "Binary"
-	case "boolean":
-		return "Bool"
-	default:
-		parts := regexp.MustCompile(`(u)?int([0-9]*)(\[[0-9]*\])?`).FindStringSubmatch(solKind.String())
-		if len(parts) != 4 {
-			return javaKind
-		}
-		switch parts[2] {
-		case "8", "16", "32", "64":
-			if parts[3] == "" {
-				return capitalise(fmt.Sprintf("%sint%s", parts[1], parts[2]))
-			}
-			return capitalise(fmt.Sprintf("%sint%ss", parts[1], parts[2]))
-
-		default:
-			return javaKind
-		}
-	}
+	LangGo: func(string, abi.Type) string { panic("this shouldn't be needed") },
 }
 
 // alias returns an alias of the given string based on the aliasing rules
@@ -556,8 +426,7 @@ func alias(aliases map[string]string, n string) string {
 // methodNormalizer is a name transformer that modifies Solidity method names to
 // conform to target language naming conventions.
 var methodNormalizer = map[Lang]func(string) string{
-	LangGo:   abi.ToCamelCase,
-	LangJava: decapitalise,
+	LangGo: abi.ToCamelCase,
 }
 
 // capitalise makes a camel-case string which starts with an upper case character.

accounts/abi/bind/template.go

@@ -75,8 +75,7 @@ type tmplStruct struct {
 // tmplSource is language to template mapping containing all the supported
 // programming languages the package can generate to.
 var tmplSource = map[Lang]string{
-	LangGo:   tmplSourceGo,
-	LangJava: tmplSourceJava,
+	LangGo: tmplSourceGo,
 }
 
 // tmplSourceGo is the Go source template that the generated Go contract binding
@@ -110,6 +109,7 @@ var (
 	_ = common.Big1
 	_ = types.BloomLookup
 	_ = event.NewSubscription
+	_ = abi.ConvertType
 )
 
 {{$structs := .Structs}}
@@ -268,11 +268,11 @@ var (
 
 	// bind{{.Type}} binds a generic wrapper to an already deployed contract.
 	func bind{{.Type}}(address common.Address, caller bind.ContractCaller, transactor bind.ContractTransactor, filterer bind.ContractFilterer) (*bind.BoundContract, error) {
-	  parsed, err := abi.JSON(strings.NewReader({{.Type}}ABI))
+	  parsed, err := {{.Type}}MetaData.GetAbi()
 	  if err != nil {
 	    return nil, err
 	  }
-	  return bind.NewBoundContract(address, parsed, caller, transactor, filterer), nil
+	  return bind.NewBoundContract(address, *parsed, caller, transactor, filterer), nil
 	}
 
 	// Call invokes the (constant) contract method with params as input values and
@@ -569,140 +569,3 @@ var (
  	{{end}}
 {{end}}
 `
-
-// tmplSourceJava is the Java source template that the generated Java contract binding
-// is based on.
-const tmplSourceJava = `
-// This file is an automatically generated Java binding. Do not modify as any
-// change will likely be lost upon the next re-generation!
-
-package {{.Package}};
-
-import org.ethereum.geth.*;
-import java.util.*;
-
-{{$structs := .Structs}}
-{{range $contract := .Contracts}}
-{{if not .Library}}public {{end}}class {{.Type}} {
-	// ABI is the input ABI used to generate the binding from.
-	public final static String ABI = "{{.InputABI}}";
-	{{if $contract.FuncSigs}}
-		// {{.Type}}FuncSigs maps the 4-byte function signature to its string representation.
-		public final static Map<String, String> {{.Type}}FuncSigs;
-		static {
-			Hashtable<String, String> temp = new Hashtable<String, String>();
-			{{range $strsig, $binsig := .FuncSigs}}temp.put("{{$binsig}}", "{{$strsig}}");
-			{{end}}
-			{{.Type}}FuncSigs = Collections.unmodifiableMap(temp);
-		}
-	{{end}}
-	{{if .InputBin}}
-	// BYTECODE is the compiled bytecode used for deploying new contracts.
-	public final static String BYTECODE = "0x{{.InputBin}}";
-
-	// deploy deploys a new Ethereum contract, binding an instance of {{.Type}} to it.
-	public static {{.Type}} deploy(TransactOpts auth, EthereumClient client{{range .Constructor.Inputs}}, {{bindtype .Type $structs}} {{.Name}}{{end}}) throws Exception {
-		Interfaces args = Geth.newInterfaces({{(len .Constructor.Inputs)}});
-		String bytecode = BYTECODE;
-		{{if .Libraries}}
-
-		// "link" contract to dependent libraries by deploying them first.
-		{{range $pattern, $name := .Libraries}}
-		{{capitalise $name}} {{decapitalise $name}}Inst = {{capitalise $name}}.deploy(auth, client);
-		bytecode = bytecode.replace("__${{$pattern}}$__", {{decapitalise $name}}Inst.Address.getHex().substring(2));
-		{{end}}
-		{{end}}
-		{{range $index, $element := .Constructor.Inputs}}Interface arg{{$index}} = Geth.newInterface();arg{{$index}}.set{{namedtype (bindtype .Type $structs) .Type}}({{.Name}});args.set({{$index}},arg{{$index}});
-		{{end}}
-		return new {{.Type}}(Geth.deployContract(auth, ABI, Geth.decodeFromHex(bytecode), client, args));
-	}
-
-	// Internal constructor used by contract deployment.
-	private {{.Type}}(BoundContract deployment) {
-		this.Address  = deployment.getAddress();
-		this.Deployer = deployment.getDeployer();
-		this.Contract = deployment;
-	}
-	{{end}}
-
-	// Ethereum address where this contract is located at.
-	public final Address Address;
-
-	// Ethereum transaction in which this contract was deployed (if known!).
-	public final Transaction Deployer;
-
-	// Contract instance bound to a blockchain address.
-	private final BoundContract Contract;
-
-	// Creates a new instance of {{.Type}}, bound to a specific deployed contract.
-	public {{.Type}}(Address address, EthereumClient client) throws Exception {
-		this(Geth.bindContract(address, ABI, client));
-	}
-
-	{{range .Calls}}
-	{{if gt (len .Normalized.Outputs) 1}}
-	// {{capitalise .Normalized.Name}}Results is the output of a call to {{.Normalized.Name}}.
-	public class {{capitalise .Normalized.Name}}Results {
-		{{range $index, $item := .Normalized.Outputs}}public {{bindtype .Type $structs}} {{if ne .Name ""}}{{.Name}}{{else}}Return{{$index}}{{end}};
-		{{end}}
-	}
-	{{end}}
-
-	// {{.Normalized.Name}} is a free data retrieval call binding the contract method 0x{{printf "%x" .Original.ID}}.
-	//
-	// Solidity: {{.Original.String}}
-	public {{if gt (len .Normalized.Outputs) 1}}{{capitalise .Normalized.Name}}Results{{else if eq (len .Normalized.Outputs) 0}}void{{else}}{{range .Normalized.Outputs}}{{bindtype .Type $structs}}{{end}}{{end}} {{.Normalized.Name}}(CallOpts opts{{range .Normalized.Inputs}}, {{bindtype .Type $structs}} {{.Name}}{{end}}) throws Exception {
-		Interfaces args = Geth.newInterfaces({{(len .Normalized.Inputs)}});
-		{{range $index, $item := .Normalized.Inputs}}Interface arg{{$index}} = Geth.newInterface();arg{{$index}}.set{{namedtype (bindtype .Type $structs) .Type}}({{.Name}});args.set({{$index}},arg{{$index}});
-		{{end}}
-
-		Interfaces results = Geth.newInterfaces({{(len .Normalized.Outputs)}});
-		{{range $index, $item := .Normalized.Outputs}}Interface result{{$index}} = Geth.newInterface(); result{{$index}}.setDefault{{namedtype (bindtype .Type $structs) .Type}}(); results.set({{$index}}, result{{$index}});
-		{{end}}
-
-		if (opts == null) {
-			opts = Geth.newCallOpts();
-		}
-		this.Contract.call(opts, results, "{{.Original.Name}}", args);
-		{{if gt (len .Normalized.Outputs) 1}}
-			{{capitalise .Normalized.Name}}Results result = new {{capitalise .Normalized.Name}}Results();
-			{{range $index, $item := .Normalized.Outputs}}result.{{if ne .Name ""}}{{.Name}}{{else}}Return{{$index}}{{end}} = results.get({{$index}}).get{{namedtype (bindtype .Type $structs) .Type}}();
-			{{end}}
-			return result;
-		{{else}}{{range .Normalized.Outputs}}return results.get(0).get{{namedtype (bindtype .Type $structs) .Type}}();{{end}}
-		{{end}}
-	}
-	{{end}}
-
-	{{range .Transacts}}
-	// {{.Normalized.Name}} is a paid mutator transaction binding the contract method 0x{{printf "%x" .Original.ID}}.
-	//
-	// Solidity: {{.Original.String}}
-	public Transaction {{.Normalized.Name}}(TransactOpts opts{{range .Normalized.Inputs}}, {{bindtype .Type $structs}} {{.Name}}{{end}}) throws Exception {
-		Interfaces args = Geth.newInterfaces({{(len .Normalized.Inputs)}});
-		{{range $index, $item := .Normalized.Inputs}}Interface arg{{$index}} = Geth.newInterface();arg{{$index}}.set{{namedtype (bindtype .Type $structs) .Type}}({{.Name}});args.set({{$index}},arg{{$index}});
-		{{end}}
-		return this.Contract.transact(opts, "{{.Original.Name}}"	, args);
-	}
-	{{end}}
-
-    {{if .Fallback}}
-	// Fallback is a paid mutator transaction binding the contract fallback function.
-	//
-	// Solidity: {{.Fallback.Original.String}}
-	public Transaction Fallback(TransactOpts opts, byte[] calldata) throws Exception { 
-		return this.Contract.rawTransact(opts, calldata);
-	}
-    {{end}}
-
-    {{if .Receive}}
-	// Receive is a paid mutator transaction binding the contract receive function.
-	//
-	// Solidity: {{.Receive.Original.String}}
-	public Transaction Receive(TransactOpts opts) throws Exception { 
-		return this.Contract.rawTransact(opts, null);
-	}
-    {{end}}
-}
-{{end}}
-`

accounts/abi/error_handling.go

@@ -23,7 +23,15 @@ import (
 )
 
 var (
-	errBadBool = errors.New("abi: improperly encoded boolean value")
+	errBadBool   = errors.New("abi: improperly encoded boolean value")
+	errBadUint8  = errors.New("abi: improperly encoded uint8 value")
+	errBadUint16 = errors.New("abi: improperly encoded uint16 value")
+	errBadUint32 = errors.New("abi: improperly encoded uint32 value")
+	errBadUint64 = errors.New("abi: improperly encoded uint64 value")
+	errBadInt8   = errors.New("abi: improperly encoded int8 value")
+	errBadInt16  = errors.New("abi: improperly encoded int16 value")
+	errBadInt32  = errors.New("abi: improperly encoded int32 value")
+	errBadInt64  = errors.New("abi: improperly encoded int64 value")
 )
 
 // formatSliceString formats the reflection kind with the given slice size
@@ -73,7 +81,6 @@ func typeCheck(t Type, value reflect.Value) error {
 	} else {
 		return nil
 	}
-
 }
 
 // typeErr returns a formatted type casting error.

accounts/abi/event_test.go

@@ -161,7 +161,6 @@ func TestEventMultiValueWithArrayUnpack(t *testing.T) {
 }
 
 func TestEventTupleUnpack(t *testing.T) {
-
 	type EventTransfer struct {
 		Value *big.Int
 	}

accounts/abi/reflect.go

@@ -25,16 +25,19 @@ import (
 )
 
 // ConvertType converts an interface of a runtime type into a interface of the
-// given type
-// e.g. turn
-// var fields []reflect.StructField
-// fields = append(fields, reflect.StructField{
-// 		Name: "X",
-//		Type: reflect.TypeOf(new(big.Int)),
-//		Tag:  reflect.StructTag("json:\"" + "x" + "\""),
-// }
-// into
-// type TupleT struct { X *big.Int }
+// given type, e.g. turn this code:
+//
+//	var fields []reflect.StructField
+//
+//	fields = append(fields, reflect.StructField{
+//			Name: "X",
+//			Type: reflect.TypeOf(new(big.Int)),
+//			Tag:  reflect.StructTag("json:\"" + "x" + "\""),
+//	}
+//
+// into:
+//
+//	type TupleT struct { X *big.Int }
 func ConvertType(in interface{}, proto interface{}) interface{} {
 	protoType := reflect.TypeOf(proto)
 	if reflect.TypeOf(in).ConvertibleTo(protoType) {
@@ -99,7 +102,7 @@ func mustArrayToByteSlice(value reflect.Value) reflect.Value {
 func set(dst, src reflect.Value) error {
 	dstType, srcType := dst.Type(), src.Type()
 	switch {
-	case dstType.Kind() == reflect.Interface && dst.Elem().IsValid():
+	case dstType.Kind() == reflect.Interface && dst.Elem().IsValid() && (dst.Elem().Type().Kind() == reflect.Ptr || dst.Elem().CanSet()):
 		return set(dst.Elem(), src)
 	case dstType.Kind() == reflect.Ptr && dstType.Elem() != reflect.TypeOf(big.Int{}):
 		return set(dst.Elem(), src)
@@ -170,11 +173,13 @@ func setStruct(dst, src reflect.Value) error {
 }
 
 // mapArgNamesToStructFields maps a slice of argument names to struct fields.
-// first round: for each Exportable field that contains a `abi:""` tag
-//   and this field name exists in the given argument name list, pair them together.
-// second round: for each argument name that has not been already linked,
-//   find what variable is expected to be mapped into, if it exists and has not been
-//   used, pair them.
+//
+// first round: for each Exportable field that contains a `abi:""` tag and this field name
+// exists in the given argument name list, pair them together.
+//
+// second round: for each argument name that has not been already linked, find what
+// variable is expected to be mapped into, if it exists and has not been used, pair them.
+//
 // Note this function assumes the given value is a struct value.
 func mapArgNamesToStructFields(argNames []string, value reflect.Value) (map[string]string, error) {
 	typ := value.Type()
@@ -220,7 +225,6 @@ func mapArgNamesToStructFields(argNames []string, value reflect.Value) (map[stri
 
 	// second round ~~~
 	for _, argName := range argNames {
-
 		structFieldName := ToCamelCase(argName)
 
 		if structFieldName == "" {

accounts/abi/reflect_test.go

@@ -32,7 +32,7 @@ type reflectTest struct {
 
 var reflectTests = []reflectTest{
 	{
-		name: "OneToOneCorrespondance",
+		name: "OneToOneCorrespondence",
 		args: []string{"fieldA"},
 		struc: struct {
 			FieldA int `abi:"fieldA"`

accounts/abi/selector_parser.go

@@ -166,7 +166,7 @@ func ParseSelector(unescapedSelector string) (SelectorMarshaling, error) {
 		return SelectorMarshaling{}, fmt.Errorf("failed to parse selector '%s': unexpected string '%s'", unescapedSelector, rest)
 	}
 
-	// Reassemble the fake ABI and constuct the JSON
+	// Reassemble the fake ABI and construct the JSON
 	fakeArgs, err := assembleArgs(args)
 	if err != nil {
 		return SelectorMarshaling{}, fmt.Errorf("failed to parse selector: %v", err)

accounts/abi/type.go

@@ -154,6 +154,9 @@ func NewType(t string, internalType string, components []ArgumentMarshaling) (ty
 		if varSize == 0 {
 			typ.T = BytesTy
 		} else {
+			if varSize > 32 {
+				return Type{}, fmt.Errorf("unsupported arg type: %s", t)
+			}
 			typ.T = FixedBytesTy
 			typ.Size = varSize
 		}

accounts/abi/type_test.go

@@ -366,3 +366,10 @@ func TestGetTypeSize(t *testing.T) {
 		}
 	}
 }
+
+func TestNewFixedBytesOver32(t *testing.T) {
+	_, err := NewType("bytes4096", "", nil)
+	if err == nil {
+		t.Errorf("fixed bytes with size over 32 is not spec'd")
+	}
+}

accounts/abi/unpack.go

@@ -19,6 +19,7 @@ package abi
 import (
 	"encoding/binary"
 	"fmt"
+	"math"
 	"math/big"
 	"reflect"
 
@@ -33,43 +34,72 @@ var (
 )
 
 // ReadInteger reads the integer based on its kind and returns the appropriate value.
-func ReadInteger(typ Type, b []byte) interface{} {
+func ReadInteger(typ Type, b []byte) (interface{}, error) {
+	ret := new(big.Int).SetBytes(b)
+
 	if typ.T == UintTy {
+		u64, isu64 := ret.Uint64(), ret.IsUint64()
 		switch typ.Size {
 		case 8:
-			return b[len(b)-1]
+			if !isu64 || u64 > math.MaxUint8 {
+				return nil, errBadUint8
+			}
+			return byte(u64), nil
 		case 16:
-			return binary.BigEndian.Uint16(b[len(b)-2:])
+			if !isu64 || u64 > math.MaxUint16 {
+				return nil, errBadUint16
+			}
+			return uint16(u64), nil
 		case 32:
-			return binary.BigEndian.Uint32(b[len(b)-4:])
+			if !isu64 || u64 > math.MaxUint32 {
+				return nil, errBadUint32
+			}
+			return uint32(u64), nil
 		case 64:
-			return binary.BigEndian.Uint64(b[len(b)-8:])
+			if !isu64 {
+				return nil, errBadUint64
+			}
+			return u64, nil
 		default:
 			// the only case left for unsigned integer is uint256.
-			return new(big.Int).SetBytes(b)
+			return ret, nil
 		}
 	}
+
+	// big.SetBytes can't tell if a number is negative or positive in itself.
+	// On EVM, if the returned number > max int256, it is negative.
+	// A number is > max int256 if the bit at position 255 is set.
+	if ret.Bit(255) == 1 {
+		ret.Add(MaxUint256, new(big.Int).Neg(ret))
+		ret.Add(ret, common.Big1)
+		ret.Neg(ret)
+	}
+	i64, isi64 := ret.Int64(), ret.IsInt64()
 	switch typ.Size {
 	case 8:
-		return int8(b[len(b)-1])
+		if !isi64 || i64 < math.MinInt8 || i64 > math.MaxInt8 {
+			return nil, errBadInt8
+		}
+		return int8(i64), nil
 	case 16:
-		return int16(binary.BigEndian.Uint16(b[len(b)-2:]))
+		if !isi64 || i64 < math.MinInt16 || i64 > math.MaxInt16 {
+			return nil, errBadInt16
+		}
+		return int16(i64), nil
 	case 32:
-		return int32(binary.BigEndian.Uint32(b[len(b)-4:]))
+		if !isi64 || i64 < math.MinInt32 || i64 > math.MaxInt32 {
+			return nil, errBadInt32
+		}
+		return int32(i64), nil
 	case 64:
-		return int64(binary.BigEndian.Uint64(b[len(b)-8:]))
+		if !isi64 {
+			return nil, errBadInt64
+		}
+		return i64, nil
 	default:
 		// the only case left for integer is int256
-		// big.SetBytes can't tell if a number is negative or positive in itself.
-		// On EVM, if the returned number > max int256, it is negative.
-		// A number is > max int256 if the bit at position 255 is set.
-		ret := new(big.Int).SetBytes(b)
-		if ret.Bit(255) == 1 {
-			ret.Add(MaxUint256, new(big.Int).Neg(ret))
-			ret.Add(ret, common.Big1)
-			ret.Neg(ret)
-		}
-		return ret
+
+		return ret, nil
 	}
 }
 
@@ -115,7 +145,6 @@ func ReadFixedBytes(t Type, word []byte) (interface{}, error) {
 
 	reflect.Copy(array, reflect.ValueOf(word[0:t.Size]))
 	return array.Interface(), nil
-
 }
 
 // forEachUnpack iteratively unpack elements.
@@ -124,7 +153,7 @@ func forEachUnpack(t Type, output []byte, start, size int) (interface{}, error)
 		return nil, fmt.Errorf("cannot marshal input to array, size is negative (%d)", size)
 	}
 	if start+32*size > len(output) {
-		return nil, fmt.Errorf("abi: cannot marshal in to go array: offset %d would go over slice boundary (len=%d)", len(output), start+32*size)
+		return nil, fmt.Errorf("abi: cannot marshal into go array: offset %d would go over slice boundary (len=%d)", len(output), start+32*size)
 	}
 
 	// this value will become our slice or our array, depending on the type
@@ -163,6 +192,9 @@ func forTupleUnpack(t Type, output []byte) (interface{}, error) {
 	virtualArgs := 0
 	for index, elem := range t.TupleElems {
 		marshalledValue, err := toGoType((index+virtualArgs)*32, *elem, output)
+		if err != nil {
+			return nil, err
+		}
 		if elem.T == ArrayTy && !isDynamicType(*elem) {
 			// If we have a static array, like [3]uint256, these are coded as
 			// just like uint256,uint256,uint256.
@@ -180,9 +212,6 @@ func forTupleUnpack(t Type, output []byte) (interface{}, error) {
 			// coded as just like uint256,bool,uint256
 			virtualArgs += getTypeSize(*elem)/32 - 1
 		}
-		if err != nil {
-			return nil, err
-		}
 		retval.Field(index).Set(reflect.ValueOf(marshalledValue))
 	}
 	return retval.Interface(), nil
@@ -235,7 +264,7 @@ func toGoType(index int, t Type, output []byte) (interface{}, error) {
 	case StringTy: // variable arrays are written at the end of the return bytes
 		return string(output[begin : begin+length]), nil
 	case IntTy, UintTy:
-		return ReadInteger(t, returnOutput), nil
+		return ReadInteger(t, returnOutput)
 	case BoolTy:
 		return readBool(returnOutput)
 	case AddressTy:

accounts/abi/unpack_test.go

@@ -20,6 +20,7 @@ import (
 	"bytes"
 	"encoding/hex"
 	"fmt"
+	"math"
 	"math/big"
 	"reflect"
 	"strconv"
@@ -352,6 +353,11 @@ func TestMethodMultiReturn(t *testing.T) {
 		&[]interface{}{&expected.Int, &expected.String},
 		"",
 		"Can unpack into a slice",
+	}, {
+		&[]interface{}{&bigint, ""},
+		&[]interface{}{&expected.Int, expected.String},
+		"",
+		"Can unpack into a slice without indirection",
 	}, {
 		&[2]interface{}{&bigint, new(string)},
 		&[2]interface{}{&expected.Int, &expected.String},
@@ -938,3 +944,164 @@ func TestOOMMaliciousInput(t *testing.T) {
 		}
 	}
 }
+
+func TestPackAndUnpackIncompatibleNumber(t *testing.T) {
+	var encodeABI Arguments
+	uint256Ty, err := NewType("uint256", "", nil)
+	if err != nil {
+		panic(err)
+	}
+	encodeABI = Arguments{
+		{Type: uint256Ty},
+	}
+
+	maxU64, ok := new(big.Int).SetString(strconv.FormatUint(math.MaxUint64, 10), 10)
+	if !ok {
+		panic("bug")
+	}
+	maxU64Plus1 := new(big.Int).Add(maxU64, big.NewInt(1))
+	cases := []struct {
+		decodeType  string
+		inputValue  *big.Int
+		err         error
+		expectValue interface{}
+	}{
+		{
+			decodeType: "uint8",
+			inputValue: big.NewInt(math.MaxUint8 + 1),
+			err:        errBadUint8,
+		},
+		{
+			decodeType:  "uint8",
+			inputValue:  big.NewInt(math.MaxUint8),
+			err:         nil,
+			expectValue: uint8(math.MaxUint8),
+		},
+		{
+			decodeType: "uint16",
+			inputValue: big.NewInt(math.MaxUint16 + 1),
+			err:        errBadUint16,
+		},
+		{
+			decodeType:  "uint16",
+			inputValue:  big.NewInt(math.MaxUint16),
+			err:         nil,
+			expectValue: uint16(math.MaxUint16),
+		},
+		{
+			decodeType: "uint32",
+			inputValue: big.NewInt(math.MaxUint32 + 1),
+			err:        errBadUint32,
+		},
+		{
+			decodeType:  "uint32",
+			inputValue:  big.NewInt(math.MaxUint32),
+			err:         nil,
+			expectValue: uint32(math.MaxUint32),
+		},
+		{
+			decodeType: "uint64",
+			inputValue: maxU64Plus1,
+			err:        errBadUint64,
+		},
+		{
+			decodeType:  "uint64",
+			inputValue:  maxU64,
+			err:         nil,
+			expectValue: uint64(math.MaxUint64),
+		},
+		{
+			decodeType:  "uint256",
+			inputValue:  maxU64Plus1,
+			err:         nil,
+			expectValue: maxU64Plus1,
+		},
+		{
+			decodeType: "int8",
+			inputValue: big.NewInt(math.MaxInt8 + 1),
+			err:        errBadInt8,
+		},
+		{
+			decodeType: "int8",
+			inputValue: big.NewInt(math.MinInt8 - 1),
+			err:        errBadInt8,
+		},
+		{
+			decodeType:  "int8",
+			inputValue:  big.NewInt(math.MaxInt8),
+			err:         nil,
+			expectValue: int8(math.MaxInt8),
+		},
+		{
+			decodeType: "int16",
+			inputValue: big.NewInt(math.MaxInt16 + 1),
+			err:        errBadInt16,
+		},
+		{
+			decodeType: "int16",
+			inputValue: big.NewInt(math.MinInt16 - 1),
+			err:        errBadInt16,
+		},
+		{
+			decodeType:  "int16",
+			inputValue:  big.NewInt(math.MaxInt16),
+			err:         nil,
+			expectValue: int16(math.MaxInt16),
+		},
+		{
+			decodeType: "int32",
+			inputValue: big.NewInt(math.MaxInt32 + 1),
+			err:        errBadInt32,
+		},
+		{
+			decodeType: "int32",
+			inputValue: big.NewInt(math.MinInt32 - 1),
+			err:        errBadInt32,
+		},
+		{
+			decodeType:  "int32",
+			inputValue:  big.NewInt(math.MaxInt32),
+			err:         nil,
+			expectValue: int32(math.MaxInt32),
+		},
+		{
+			decodeType: "int64",
+			inputValue: new(big.Int).Add(big.NewInt(math.MaxInt64), big.NewInt(1)),
+			err:        errBadInt64,
+		},
+		{
+			decodeType: "int64",
+			inputValue: new(big.Int).Sub(big.NewInt(math.MinInt64), big.NewInt(1)),
+			err:        errBadInt64,
+		},
+		{
+			decodeType:  "int64",
+			inputValue:  big.NewInt(math.MaxInt64),
+			err:         nil,
+			expectValue: int64(math.MaxInt64),
+		},
+	}
+	for i, testCase := range cases {
+		packed, err := encodeABI.Pack(testCase.inputValue)
+		if err != nil {
+			panic(err)
+		}
+		ty, err := NewType(testCase.decodeType, "", nil)
+		if err != nil {
+			panic(err)
+		}
+		decodeABI := Arguments{
+			{Type: ty},
+		}
+		decoded, err := decodeABI.Unpack(packed)
+		if err != testCase.err {
+			t.Fatalf("Expected error %v, actual error %v. case %d", testCase.err, err, i)
+		}
+		if err != nil {
+			continue
+		}
+		if !reflect.DeepEqual(decoded[0], testCase.expectValue) {
+			t.Fatalf("Expected value %v, actual value %v", testCase.expectValue, decoded[0])
+		}
+	}
+}

accounts/abi/utils.go

@@ -21,15 +21,14 @@ import "fmt"
 // ResolveNameConflict returns the next available name for a given thing.
 // This helper can be used for lots of purposes:
 //
-// - In solidity function overloading is supported, this function can fix
-//   the name conflicts of overloaded functions.
-// - In golang binding generation, the parameter(in function, event, error,
-//	 and struct definition) name will be converted to camelcase style which
-//	 may eventually lead to name conflicts.
+//   - In solidity function overloading is supported, this function can fix
+//     the name conflicts of overloaded functions.
+//   - In golang binding generation, the parameter(in function, event, error,
+//     and struct definition) name will be converted to camelcase style which
+//     may eventually lead to name conflicts.
 //
-// Name conflicts are mostly resolved by adding number suffix.
-// 	 e.g. if the abi contains Methods send, send1
-//   ResolveNameConflict would return send2 for input send.
+// Name conflicts are mostly resolved by adding number suffix. e.g. if the abi contains
+// Methods "send" and "send1", ResolveNameConflict would return "send2" for input "send".
 func ResolveNameConflict(rawName string, used func(string) bool) string {
 	name := rawName
 	ok := used(name)

accounts/accounts.go

@@ -177,7 +177,8 @@ type Backend interface {
 // safely used to calculate a signature from.
 //
 // The hash is calculated as
-//   keccak256("\x19Ethereum Signed Message:\n"${message length}${message}).
+//
+//	keccak256("\x19Ethereum Signed Message:\n"${message length}${message}).
 //
 // This gives context to the signed message and prevents signing of transactions.
 func TextHash(data []byte) []byte {
@@ -189,7 +190,8 @@ func TextHash(data []byte) []byte {
 // safely used to calculate a signature from.
 //
 // The hash is calculated as
-//   keccak256("\x19Ethereum Signed Message:\n"${message length}${message}).
+//
+//	keccak256("\x19Ethereum Signed Message:\n"${message length}${message}).
 //
 // This gives context to the signed message and prevents signing of transactions.
 func TextAndHash(data []byte) ([]byte, string) {

accounts/hd.go

@@ -46,7 +46,7 @@ var LegacyLedgerBaseDerivationPath = DerivationPath{0x80000000 + 44, 0x80000000
 // The BIP-32 spec https://github.com/bitcoin/bips/blob/master/bip-0032.mediawiki
 // defines derivation paths to be of the form:
 //
-//   m / purpose' / coin_type' / account' / change / address_index
+//	m / purpose' / coin_type' / account' / change / address_index
 //
 // The BIP-44 spec https://github.com/bitcoin/bips/blob/master/bip-0044.mediawiki
 // defines that the `purpose` be 44' (or 0x8000002C) for crypto currencies, and

accounts/keystore/account_cache.go

@@ -27,7 +27,7 @@ import (
 	"sync"
 	"time"
 
-	mapset "github.com/deckarep/golang-set"
+	mapset "github.com/deckarep/golang-set/v2"
 	"github.com/ethereum/go-ethereum/accounts"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/log"
@@ -79,7 +79,7 @@ func newAccountCache(keydir string) (*accountCache, chan struct{}) {
 		keydir: keydir,
 		byAddr: make(map[common.Address][]accounts.Account),
 		notify: make(chan struct{}, 1),
-		fileC:  fileCache{all: mapset.NewThreadUnsafeSet()},
+		fileC:  fileCache{all: mapset.NewThreadUnsafeSet[string]()},
 	}
 	ac.watcher = newWatcher(ac)
 	return ac, ac.notify
@@ -146,6 +146,14 @@ func (ac *accountCache) deleteByFile(path string) {
 	}
 }
 
+// watcherStarted returns true if the watcher loop started running (even if it
+// has since also ended).
+func (ac *accountCache) watcherStarted() bool {
+	ac.mu.Lock()
+	defer ac.mu.Unlock()
+	return ac.watcher.running || ac.watcher.runEnded
+}
+
 func removeAccount(slice []accounts.Account, elem accounts.Account) []accounts.Account {
 	for i := range slice {
 		if slice[i] == elem {
@@ -275,16 +283,15 @@ func (ac *accountCache) scanAccounts() error {
 	// Process all the file diffs
 	start := time.Now()
 
-	for _, p := range creates.ToSlice() {
-		if a := readAccount(p.(string)); a != nil {
+	for _, path := range creates.ToSlice() {
+		if a := readAccount(path); a != nil {
 			ac.add(*a)
 		}
 	}
-	for _, p := range deletes.ToSlice() {
-		ac.deleteByFile(p.(string))
+	for _, path := range deletes.ToSlice() {
+		ac.deleteByFile(path)
 	}
-	for _, p := range updates.ToSlice() {
-		path := p.(string)
+	for _, path := range updates.ToSlice() {
 		ac.deleteByFile(path)
 		if a := readAccount(path); a != nil {
 			ac.add(*a)

accounts/keystore/account_cache_test.go

@@ -50,6 +50,38 @@ var (
 	}
 )
 
+// waitWatcherStarts waits up to 1s for the keystore watcher to start.
+func waitWatcherStart(ks *KeyStore) bool {
+	// On systems where file watch is not supported, just return "ok".
+	if !ks.cache.watcher.enabled() {
+		return true
+	}
+	// The watcher should start, and then exit.
+	for t0 := time.Now(); time.Since(t0) < 1*time.Second; time.Sleep(100 * time.Millisecond) {
+		if ks.cache.watcherStarted() {
+			return true
+		}
+	}
+	return false
+}
+
+func waitForAccounts(wantAccounts []accounts.Account, ks *KeyStore) error {
+	var list []accounts.Account
+	for t0 := time.Now(); time.Since(t0) < 5*time.Second; time.Sleep(200 * time.Millisecond) {
+		list = ks.Accounts()
+		if reflect.DeepEqual(list, wantAccounts) {
+			// ks should have also received change notifications
+			select {
+			case <-ks.changes:
+			default:
+				return fmt.Errorf("wasn't notified of new accounts")
+			}
+			return nil
+		}
+	}
+	return fmt.Errorf("\ngot  %v\nwant %v", list, wantAccounts)
+}
+
 func TestWatchNewFile(t *testing.T) {
 	t.Parallel()
 
@@ -57,8 +89,9 @@ func TestWatchNewFile(t *testing.T) {
 
 	// Ensure the watcher is started before adding any files.
 	ks.Accounts()
-	time.Sleep(1000 * time.Millisecond)
-
+	if !waitWatcherStart(ks) {
+		t.Fatal("keystore watcher didn't start in time")
+	}
 	// Move in the files.
 	wantAccounts := make([]accounts.Account, len(cachetestAccounts))
 	for i := range cachetestAccounts {
@@ -72,37 +105,24 @@ func TestWatchNewFile(t *testing.T) {
 	}
 
 	// ks should see the accounts.
-	var list []accounts.Account
-	for d := 200 * time.Millisecond; d < 5*time.Second; d *= 2 {
-		list = ks.Accounts()
-		if reflect.DeepEqual(list, wantAccounts) {
-			// ks should have also received change notifications
-			select {
-			case <-ks.changes:
-			default:
-				t.Fatalf("wasn't notified of new accounts")
-			}
-			return
-		}
-		time.Sleep(d)
+	if err := waitForAccounts(wantAccounts, ks); err != nil {
+		t.Error(err)
 	}
-	t.Errorf("got %s, want %s", spew.Sdump(list), spew.Sdump(wantAccounts))
 }
 
 func TestWatchNoDir(t *testing.T) {
 	t.Parallel()
-
 	// Create ks but not the directory that it watches.
-	rand.Seed(time.Now().UnixNano())
 	dir := filepath.Join(os.TempDir(), fmt.Sprintf("eth-keystore-watchnodir-test-%d-%d", os.Getpid(), rand.Int()))
 	ks := NewKeyStore(dir, LightScryptN, LightScryptP)
-
 	list := ks.Accounts()
 	if len(list) > 0 {
 		t.Error("initial account list not empty:", list)
 	}
-	time.Sleep(100 * time.Millisecond)
-
+	// The watcher should start, and then exit.
+	if !waitWatcherStart(ks) {
+		t.Fatal("keystore watcher didn't start in time")
+	}
 	// Create the directory and copy a key file into it.
 	os.MkdirAll(dir, 0700)
 	defer os.RemoveAll(dir)
@@ -295,31 +315,12 @@ func TestCacheFind(t *testing.T) {
 	}
 }
 
-func waitForAccounts(wantAccounts []accounts.Account, ks *KeyStore) error {
-	var list []accounts.Account
-	for d := 200 * time.Millisecond; d < 8*time.Second; d *= 2 {
-		list = ks.Accounts()
-		if reflect.DeepEqual(list, wantAccounts) {
-			// ks should have also received change notifications
-			select {
-			case <-ks.changes:
-			default:
-				return fmt.Errorf("wasn't notified of new accounts")
-			}
-			return nil
-		}
-		time.Sleep(d)
-	}
-	return fmt.Errorf("\ngot  %v\nwant %v", list, wantAccounts)
-}
-
 // TestUpdatedKeyfileContents tests that updating the contents of a keystore file
 // is noticed by the watcher, and the account cache is updated accordingly
 func TestUpdatedKeyfileContents(t *testing.T) {
 	t.Parallel()
 
-	// Create a temporary kesytore to test with
-	rand.Seed(time.Now().UnixNano())
+	// Create a temporary keystore to test with
 	dir := filepath.Join(os.TempDir(), fmt.Sprintf("eth-keystore-updatedkeyfilecontents-test-%d-%d", os.Getpid(), rand.Int()))
 	ks := NewKeyStore(dir, LightScryptN, LightScryptP)
 
@@ -327,8 +328,9 @@ func TestUpdatedKeyfileContents(t *testing.T) {
 	if len(list) > 0 {
 		t.Error("initial account list not empty:", list)
 	}
-	time.Sleep(100 * time.Millisecond)
-
+	if !waitWatcherStart(ks) {
+		t.Fatal("keystore watcher didn't start in time")
+	}
 	// Create the directory and copy a key file into it.
 	os.MkdirAll(dir, 0700)
 	defer os.RemoveAll(dir)
@@ -346,9 +348,8 @@ func TestUpdatedKeyfileContents(t *testing.T) {
 		t.Error(err)
 		return
 	}
-
 	// needed so that modTime of `file` is different to its current value after forceCopyFile
-	time.Sleep(1000 * time.Millisecond)
+	time.Sleep(time.Second)
 
 	// Now replace file contents
 	if err := forceCopyFile(file, cachetestAccounts[1].URL.Path); err != nil {
@@ -364,7 +365,7 @@ func TestUpdatedKeyfileContents(t *testing.T) {
 	}
 
 	// needed so that modTime of `file` is different to its current value after forceCopyFile
-	time.Sleep(1000 * time.Millisecond)
+	time.Sleep(time.Second)
 
 	// Now replace file contents again
 	if err := forceCopyFile(file, cachetestAccounts[2].URL.Path); err != nil {
@@ -380,7 +381,7 @@ func TestUpdatedKeyfileContents(t *testing.T) {
 	}
 
 	// needed so that modTime of `file` is different to its current value after os.WriteFile
-	time.Sleep(1000 * time.Millisecond)
+	time.Sleep(time.Second)
 
 	// Now replace file contents with crap
 	if err := os.WriteFile(file, []byte("foo"), 0600); err != nil {

accounts/keystore/file_cache.go

@@ -23,23 +23,23 @@ import (
 	"sync"
 	"time"
 
-	mapset "github.com/deckarep/golang-set"
+	mapset "github.com/deckarep/golang-set/v2"
 	"github.com/ethereum/go-ethereum/log"
 )
 
 // fileCache is a cache of files seen during scan of keystore.
 type fileCache struct {
-	all     mapset.Set // Set of all files from the keystore folder
-	lastMod time.Time  // Last time instance when a file was modified
+	all     mapset.Set[string] // Set of all files from the keystore folder
+	lastMod time.Time          // Last time instance when a file was modified
 	mu      sync.Mutex
 }
 
 // scan performs a new scan on the given directory, compares against the already
 // cached filenames, and returns file sets: creates, deletes, updates.
-func (fc *fileCache) scan(keyDir string) (mapset.Set, mapset.Set, mapset.Set, error) {
+func (fc *fileCache) scan(keyDir string) (mapset.Set[string], mapset.Set[string], mapset.Set[string], error) {
 	t0 := time.Now()
 
-	// List all the failes from the keystore folder
+	// List all the files from the keystore folder
 	files, err := os.ReadDir(keyDir)
 	if err != nil {
 		return nil, nil, nil, err
@@ -50,8 +50,8 @@ func (fc *fileCache) scan(keyDir string) (mapset.Set, mapset.Set, mapset.Set, er
 	defer fc.mu.Unlock()
 
 	// Iterate all the files and gather their metadata
-	all := mapset.NewThreadUnsafeSet()
-	mods := mapset.NewThreadUnsafeSet()
+	all := mapset.NewThreadUnsafeSet[string]()
+	mods := mapset.NewThreadUnsafeSet[string]()
 
 	var newLastMod time.Time
 	for _, fi := range files {
@@ -61,7 +61,7 @@ func (fc *fileCache) scan(keyDir string) (mapset.Set, mapset.Set, mapset.Set, er
 			log.Trace("Ignoring file on account scan", "path", path)
 			continue
 		}
-		// Gather the set of all and fresly modified files
+		// Gather the set of all and freshly modified files
 		all.Add(path)
 
 		info, err := fi.Info()

accounts/keystore/keystore.go

@@ -498,6 +498,14 @@ func (ks *KeyStore) ImportPreSaleKey(keyJSON []byte, passphrase string) (account
 	return a, nil
 }
 
+// isUpdating returns whether the event notification loop is running.
+// This method is mainly meant for tests.
+func (ks *KeyStore) isUpdating() bool {
+	ks.mu.RLock()
+	defer ks.mu.RUnlock()
+	return ks.updating
+}
+
 // zeroKey zeroes a private key in memory.
 func zeroKey(k *ecdsa.PrivateKey) {
 	b := k.D.Bits()

accounts/keystore/keystore_test.go

@@ -113,6 +113,7 @@ func TestSignWithPassphrase(t *testing.T) {
 }
 
 func TestTimedUnlock(t *testing.T) {
+	t.Parallel()
 	_, ks := tmpKeyStore(t, true)
 
 	pass := "foo"
@@ -147,6 +148,7 @@ func TestTimedUnlock(t *testing.T) {
 }
 
 func TestOverrideUnlock(t *testing.T) {
+	t.Parallel()
 	_, ks := tmpKeyStore(t, false)
 
 	pass := "foo"
@@ -187,6 +189,7 @@ func TestOverrideUnlock(t *testing.T) {
 
 // This test should fail under -race if signing races the expiration goroutine.
 func TestSignRace(t *testing.T) {
+	t.Parallel()
 	_, ks := tmpKeyStore(t, false)
 
 	// Create a test account.
@@ -211,19 +214,33 @@ func TestSignRace(t *testing.T) {
 	t.Errorf("Account did not lock within the timeout")
 }
 
+// waitForKsUpdating waits until the updating-status of the ks reaches the
+// desired wantStatus.
+// It waits for a maximum time of maxTime, and returns false if it does not
+// finish in time
+func waitForKsUpdating(t *testing.T, ks *KeyStore, wantStatus bool, maxTime time.Duration) bool {
+	t.Helper()
+	// Wait max 250 ms, then return false
+	for t0 := time.Now(); time.Since(t0) < maxTime; {
+		if ks.isUpdating() == wantStatus {
+			return true
+		}
+		time.Sleep(25 * time.Millisecond)
+	}
+	return false
+}
+
 // Tests that the wallet notifier loop starts and stops correctly based on the
 // addition and removal of wallet event subscriptions.
 func TestWalletNotifierLifecycle(t *testing.T) {
-	// Create a temporary kesytore to test with
+	t.Parallel()
+	// Create a temporary keystore to test with
 	_, ks := tmpKeyStore(t, false)
 
 	// Ensure that the notification updater is not running yet
 	time.Sleep(250 * time.Millisecond)
-	ks.mu.RLock()
-	updating := ks.updating
-	ks.mu.RUnlock()
 
-	if updating {
+	if ks.isUpdating() {
 		t.Errorf("wallet notifier running without subscribers")
 	}
 	// Subscribe to the wallet feed and ensure the updater boots up
@@ -233,38 +250,26 @@ func TestWalletNotifierLifecycle(t *testing.T) {
 	for i := 0; i < len(subs); i++ {
 		// Create a new subscription
 		subs[i] = ks.Subscribe(updates)
-
-		// Ensure the notifier comes online
-		time.Sleep(250 * time.Millisecond)
-		ks.mu.RLock()
-		updating = ks.updating
-		ks.mu.RUnlock()
-
-		if !updating {
+		if !waitForKsUpdating(t, ks, true, 250*time.Millisecond) {
 			t.Errorf("sub %d: wallet notifier not running after subscription", i)
 		}
 	}
-	// Unsubscribe and ensure the updater terminates eventually
-	for i := 0; i < len(subs); i++ {
+	// Close all but one sub
+	for i := 0; i < len(subs)-1; i++ {
 		// Close an existing subscription
 		subs[i].Unsubscribe()
-
-		// Ensure the notifier shuts down at and only at the last close
-		for k := 0; k < int(walletRefreshCycle/(250*time.Millisecond))+2; k++ {
-			ks.mu.RLock()
-			updating = ks.updating
-			ks.mu.RUnlock()
-
-			if i < len(subs)-1 && !updating {
-				t.Fatalf("sub %d: event notifier stopped prematurely", i)
-			}
-			if i == len(subs)-1 && !updating {
-				return
-			}
-			time.Sleep(250 * time.Millisecond)
-		}
 	}
-	t.Errorf("wallet notifier didn't terminate after unsubscribe")
+	// Check that it is still running
+	time.Sleep(250 * time.Millisecond)
+
+	if !ks.isUpdating() {
+		t.Fatal("event notifier stopped prematurely")
+	}
+	// Unsubscribe the last one and ensure the updater terminates eventually.
+	subs[len(subs)-1].Unsubscribe()
+	if !waitForKsUpdating(t, ks, false, 4*time.Second) {
+		t.Errorf("wallet notifier didn't terminate after unsubscribe")
+	}
 }
 
 type walletEvent struct {
@@ -377,7 +382,6 @@ func TestImportExport(t *testing.T) {
 	if _, err = ks2.Import(json, "new", "new"); err == nil {
 		t.Errorf("importing a key twice succeeded")
 	}
-
 }
 
 // TestImportRace tests the keystore on races.
@@ -402,7 +406,6 @@ func TestImportRace(t *testing.T) {
 			if _, err := ks2.Import(json, "new", "new"); err != nil {
 				atomic.AddUint32(&atom, 1)
 			}
-
 		}()
 	}
 	wg.Wait()

accounts/keystore/passphrase.go

@@ -138,7 +138,6 @@ func (ks keyStorePassphrase) JoinPath(filename string) string {
 
 // Encryptdata encrypts the data given as 'data' with the password 'auth'.
 func EncryptDataV3(data, auth []byte, scryptN, scryptP int) (CryptoJSON, error) {
-
 	salt := make([]byte, 32)
 	if _, err := io.ReadFull(rand.Reader, salt); err != nil {
 		panic("reading from crypto/rand failed: " + err.Error())
@@ -341,7 +340,6 @@ func getKDFKey(cryptoJSON CryptoJSON, auth string) ([]byte, error) {
 		r := ensureInt(cryptoJSON.KDFParams["r"])
 		p := ensureInt(cryptoJSON.KDFParams["p"])
 		return scrypt.Key(authArray, salt, n, r, p, dkLen)
-
 	} else if cryptoJSON.KDF == "pbkdf2" {
 		c := ensureInt(cryptoJSON.KDFParams["c"])
 		prf := cryptoJSON.KDFParams["prf"].(string)

accounts/keystore/watch.go

@@ -23,25 +23,27 @@ import (
 	"time"
 
 	"github.com/ethereum/go-ethereum/log"
-	"github.com/rjeczalik/notify"
+	"github.com/fsnotify/fsnotify"
 )
 
 type watcher struct {
 	ac       *accountCache
-	starting bool
-	running  bool
-	ev       chan notify.EventInfo
+	running  bool // set to true when runloop begins
+	runEnded bool // set to true when runloop ends
+	starting bool // set to true prior to runloop starting
 	quit     chan struct{}
 }
 
 func newWatcher(ac *accountCache) *watcher {
 	return &watcher{
 		ac:   ac,
-		ev:   make(chan notify.EventInfo, 10),
 		quit: make(chan struct{}),
 	}
 }
 
+// enabled returns false on systems not supported.
+func (*watcher) enabled() bool { return true }
+
 // starts the watcher loop in the background.
 // Start a watcher in the background if that's not already in progress.
 // The caller must hold w.ac.mu.
@@ -62,16 +64,24 @@ func (w *watcher) loop() {
 		w.ac.mu.Lock()
 		w.running = false
 		w.starting = false
+		w.runEnded = true
 		w.ac.mu.Unlock()
 	}()
 	logger := log.New("path", w.ac.keydir)
 
-	if err := notify.Watch(w.ac.keydir, w.ev, notify.All); err != nil {
-		logger.Trace("Failed to watch keystore folder", "err", err)
+	// Create new watcher.
+	watcher, err := fsnotify.NewWatcher()
+	if err != nil {
+		log.Error("Failed to start filesystem watcher", "err", err)
 		return
 	}
-	defer notify.Stop(w.ev)
-	logger.Trace("Started watching keystore folder")
+	defer watcher.Close()
+	if err := watcher.Add(w.ac.keydir); err != nil {
+		logger.Warn("Failed to watch keystore folder", "err", err)
+		return
+	}
+
+	logger.Trace("Started watching keystore folder", "folder", w.ac.keydir)
 	defer logger.Trace("Stopped watching keystore folder")
 
 	w.ac.mu.Lock()
@@ -95,12 +105,24 @@ func (w *watcher) loop() {
 		select {
 		case <-w.quit:
 			return
-		case <-w.ev:
+		case _, ok := <-watcher.Events:
+			if !ok {
+				return
+			}
 			// Trigger the scan (with delay), if not already triggered
 			if !rescanTriggered {
 				debounce.Reset(debounceDuration)
 				rescanTriggered = true
 			}
+			// The fsnotify library does provide more granular event-info, it
+			// would be possible to refresh individual affected files instead
+			// of scheduling a full rescan. For most cases though, the
+			// full rescan is quick and obviously simplest.
+		case err, ok := <-watcher.Errors:
+			if !ok {
+				return
+			}
+			log.Info("Filsystem watcher error", "err", err)
 		case <-debounce.C:
 			w.ac.scanAccounts()
 			rescanTriggered = false

accounts/keystore/watch_fallback.go

@@ -22,8 +22,14 @@
 
 package keystore
 
-type watcher struct{ running bool }
+type watcher struct {
+	running  bool
+	runEnded bool
+}
 
 func newWatcher(*accountCache) *watcher { return new(watcher) }
 func (*watcher) start()                 {}
 func (*watcher) close()                 {}
+
+// enabled returns false on systems not supported.
+func (*watcher) enabled() bool { return false }

accounts/manager.go

@@ -257,7 +257,7 @@ func merge(slice []Wallet, wallets ...Wallet) []Wallet {
 	return slice
 }
 
-// drop is the couterpart of merge, which looks up wallets from within the sorted
+// drop is the counterpart of merge, which looks up wallets from within the sorted
 // cache and removes the ones specified.
 func drop(slice []Wallet, wallets ...Wallet) []Wallet {
 	for _, wallet := range wallets {

accounts/scwallet/securechannel.go

@@ -178,7 +178,7 @@ func (s *SecureChannelSession) mutuallyAuthenticate() error {
 		return err
 	}
 	if response.Sw1 != 0x90 || response.Sw2 != 0x00 {
-		return fmt.Errorf("got unexpected response from MUTUALLY_AUTHENTICATE: 0x%x%x", response.Sw1, response.Sw2)
+		return fmt.Errorf("got unexpected response from MUTUALLY_AUTHENTICATE: %#x%x", response.Sw1, response.Sw2)
 	}
 
 	if len(response.Data) != scSecretLength {
@@ -261,7 +261,7 @@ func (s *SecureChannelSession) transmitEncrypted(cla, ins, p1, p2 byte, data []b
 	rapdu.deserialize(plainData)
 
 	if rapdu.Sw1 != sw1Ok {
-		return nil, fmt.Errorf("unexpected response status Cla=0x%x, Ins=0x%x, Sw=0x%x%x", cla, ins, rapdu.Sw1, rapdu.Sw2)
+		return nil, fmt.Errorf("unexpected response status Cla=%#x, Ins=%#x, Sw=%#x%x", cla, ins, rapdu.Sw1, rapdu.Sw2)
 	}
 
 	return rapdu, nil

accounts/scwallet/wallet.go

@@ -99,8 +99,8 @@ const (
 	P1DeriveKeyFromCurrent = uint8(0x10)
 	statusP1WalletStatus   = uint8(0x00)
 	statusP1Path           = uint8(0x01)
-	signP1PrecomputedHash  = uint8(0x01)
-	signP2OnlyBlock        = uint8(0x81)
+	signP1PrecomputedHash  = uint8(0x00)
+	signP2OnlyBlock        = uint8(0x00)
 	exportP1Any            = uint8(0x00)
 	exportP2Pubkey         = uint8(0x01)
 )
@@ -167,7 +167,7 @@ func transmit(card *pcsc.Card, command *commandAPDU) (*responseAPDU, error) {
 	}
 
 	if response.Sw1 != sw1Ok {
-		return nil, fmt.Errorf("unexpected insecure response status Cla=0x%x, Ins=0x%x, Sw=0x%x%x", command.Cla, command.Ins, response.Sw1, response.Sw2)
+		return nil, fmt.Errorf("unexpected insecure response status Cla=%#x, Ins=%#x, Sw=%#x%x", command.Cla, command.Ins, response.Sw1, response.Sw2)
 	}
 
 	return response, nil
@@ -879,6 +879,7 @@ func (s *Session) walletStatus() (*walletStatus, error) {
 }
 
 // derivationPath fetches the wallet's current derivation path from the card.
+//
 //lint:ignore U1000 needs to be added to the console interface
 func (s *Session) derivationPath() (accounts.DerivationPath, error) {
 	response, err := s.Channel.transmitEncrypted(claSCWallet, insStatus, statusP1Path, 0, nil)
@@ -994,6 +995,7 @@ func (s *Session) derive(path accounts.DerivationPath) (accounts.Account, error)
 }
 
 // keyExport contains information on an exported keypair.
+//
 //lint:ignore U1000 needs to be added to the console interface
 type keyExport struct {
 	PublicKey  []byte `asn1:"tag:0"`
@@ -1001,6 +1003,7 @@ type keyExport struct {
 }
 
 // publicKey returns the public key for the current derivation path.
+//
 //lint:ignore U1000 needs to be added to the console interface
 func (s *Session) publicKey() ([]byte, error) {
 	response, err := s.Channel.transmitEncrypted(claSCWallet, insExportKey, exportP1Any, exportP2Pubkey, nil)

accounts/url.go

@@ -92,10 +92,9 @@ func (u *URL) UnmarshalJSON(input []byte) error {
 
 // Cmp compares x and y and returns:
 //
-//   -1 if x <  y
-//    0 if x == y
-//   +1 if x >  y
-//
+//	-1 if x <  y
+//	 0 if x == y
+//	+1 if x >  y
 func (u URL) Cmp(url URL) int {
 	if u.Scheme == url.Scheme {
 		return strings.Compare(u.Path, url.Path)

accounts/usbwallet/hub.go

@@ -71,18 +71,28 @@ type Hub struct {
 // NewLedgerHub creates a new hardware wallet manager for Ledger devices.
 func NewLedgerHub() (*Hub, error) {
 	return newHub(LedgerScheme, 0x2c97, []uint16{
+
+		// Device definitions taken from
+		// https://github.com/LedgerHQ/ledger-live/blob/38012bc8899e0f07149ea9cfe7e64b2c146bc92b/libs/ledgerjs/packages/devices/src/index.ts
+
 		// Original product IDs
 		0x0000, /* Ledger Blue */
 		0x0001, /* Ledger Nano S */
 		0x0004, /* Ledger Nano X */
+		0x0005, /* Ledger Nano S Plus */
+		0x0006, /* Ledger Nano FTS */
 
-		// Upcoming product IDs: https://www.ledger.com/2019/05/17/windows-10-update-sunsetting-u2f-tunnel-transport-for-ledger-devices/
 		0x0015, /* HID + U2F + WebUSB Ledger Blue */
 		0x1015, /* HID + U2F + WebUSB Ledger Nano S */
 		0x4015, /* HID + U2F + WebUSB Ledger Nano X */
+		0x5015, /* HID + U2F + WebUSB Ledger Nano S Plus */
+		0x6015, /* HID + U2F + WebUSB Ledger Nano FTS */
+
 		0x0011, /* HID + WebUSB Ledger Blue */
 		0x1011, /* HID + WebUSB Ledger Nano S */
 		0x4011, /* HID + WebUSB Ledger Nano X */
+		0x5011, /* HID + WebUSB Ledger Nano S Plus */
+		0x6011, /* HID + WebUSB Ledger Nano FTS */
 	}, 0xffa0, 0, newLedgerDriver)
 }
 

accounts/usbwallet/ledger.go

@@ -59,6 +59,8 @@ const (
 	ledgerP1InitTransactionData     ledgerParam1 = 0x00 // First transaction data block for signing
 	ledgerP1ContTransactionData     ledgerParam1 = 0x80 // Subsequent transaction data block for signing
 	ledgerP2DiscardAddressChainCode ledgerParam2 = 0x00 // Do not return the chain code along with the address
+
+	ledgerEip155Size int = 3 // Size of the EIP-155 chain_id,r,s in unsigned transactions
 )
 
 // errLedgerReplyInvalidHeader is the error message returned by a Ledger data exchange
@@ -195,18 +197,18 @@ func (w *ledgerDriver) SignTypedMessage(path accounts.DerivationPath, domainHash
 //
 // The version retrieval protocol is defined as follows:
 //
-//   CLA | INS | P1 | P2 | Lc | Le
-//   ----+-----+----+----+----+---
-//    E0 | 06  | 00 | 00 | 00 | 04
+//	CLA | INS | P1 | P2 | Lc | Le
+//	----+-----+----+----+----+---
+//	 E0 | 06  | 00 | 00 | 00 | 04
 //
 // With no input data, and the output data being:
 //
-//   Description                                        | Length
-//   ---------------------------------------------------+--------
-//   Flags 01: arbitrary data signature enabled by user | 1 byte
-//   Application major version                          | 1 byte
-//   Application minor version                          | 1 byte
-//   Application patch version                          | 1 byte
+//	Description                                        | Length
+//	---------------------------------------------------+--------
+//	Flags 01: arbitrary data signature enabled by user | 1 byte
+//	Application major version                          | 1 byte
+//	Application minor version                          | 1 byte
+//	Application patch version                          | 1 byte
 func (w *ledgerDriver) ledgerVersion() ([3]byte, error) {
 	// Send the request and wait for the response
 	reply, err := w.ledgerExchange(ledgerOpGetConfiguration, 0, 0, nil)
@@ -227,32 +229,32 @@ func (w *ledgerDriver) ledgerVersion() ([3]byte, error) {
 //
 // The address derivation protocol is defined as follows:
 //
-//   CLA | INS | P1 | P2 | Lc  | Le
-//   ----+-----+----+----+-----+---
-//    E0 | 02  | 00 return address
-//               01 display address and confirm before returning
-//                  | 00: do not return the chain code
-//                  | 01: return the chain code
-//                       | var | 00
+//	CLA | INS | P1 | P2 | Lc  | Le
+//	----+-----+----+----+-----+---
+//	 E0 | 02  | 00 return address
+//	            01 display address and confirm before returning
+//	               | 00: do not return the chain code
+//	               | 01: return the chain code
+//	                    | var | 00
 //
 // Where the input data is:
 //
-//   Description                                      | Length
-//   -------------------------------------------------+--------
-//   Number of BIP 32 derivations to perform (max 10) | 1 byte
-//   First derivation index (big endian)              | 4 bytes
-//   ...                                              | 4 bytes
-//   Last derivation index (big endian)               | 4 bytes
+//	Description                                      | Length
+//	-------------------------------------------------+--------
+//	Number of BIP 32 derivations to perform (max 10) | 1 byte
+//	First derivation index (big endian)              | 4 bytes
+//	...                                              | 4 bytes
+//	Last derivation index (big endian)               | 4 bytes
 //
 // And the output data is:
 //
-//   Description             | Length
-//   ------------------------+-------------------
-//   Public Key length       | 1 byte
-//   Uncompressed Public Key | arbitrary
-//   Ethereum address length | 1 byte
-//   Ethereum address        | 40 bytes hex ascii
-//   Chain code if requested | 32 bytes
+//	Description             | Length
+//	------------------------+-------------------
+//	Public Key length       | 1 byte
+//	Uncompressed Public Key | arbitrary
+//	Ethereum address length | 1 byte
+//	Ethereum address        | 40 bytes hex ascii
+//	Chain code if requested | 32 bytes
 func (w *ledgerDriver) ledgerDerive(derivationPath []uint32) (common.Address, error) {
 	// Flatten the derivation path into the Ledger request
 	path := make([]byte, 1+4*len(derivationPath))
@@ -290,35 +292,35 @@ func (w *ledgerDriver) ledgerDerive(derivationPath []uint32) (common.Address, er
 //
 // The transaction signing protocol is defined as follows:
 //
-//   CLA | INS | P1 | P2 | Lc  | Le
-//   ----+-----+----+----+-----+---
-//    E0 | 04  | 00: first transaction data block
-//               80: subsequent transaction data block
-//                  | 00 | variable | variable
+//	CLA | INS | P1 | P2 | Lc  | Le
+//	----+-----+----+----+-----+---
+//	 E0 | 04  | 00: first transaction data block
+//	            80: subsequent transaction data block
+//	               | 00 | variable | variable
 //
 // Where the input for the first transaction block (first 255 bytes) is:
 //
-//   Description                                      | Length
-//   -------------------------------------------------+----------
-//   Number of BIP 32 derivations to perform (max 10) | 1 byte
-//   First derivation index (big endian)              | 4 bytes
-//   ...                                              | 4 bytes
-//   Last derivation index (big endian)               | 4 bytes
-//   RLP transaction chunk                            | arbitrary
+//	Description                                      | Length
+//	-------------------------------------------------+----------
+//	Number of BIP 32 derivations to perform (max 10) | 1 byte
+//	First derivation index (big endian)              | 4 bytes
+//	...                                              | 4 bytes
+//	Last derivation index (big endian)               | 4 bytes
+//	RLP transaction chunk                            | arbitrary
 //
 // And the input for subsequent transaction blocks (first 255 bytes) are:
 //
-//   Description           | Length
-//   ----------------------+----------
-//   RLP transaction chunk | arbitrary
+//	Description           | Length
+//	----------------------+----------
+//	RLP transaction chunk | arbitrary
 //
 // And the output data is:
 //
-//   Description | Length
-//   ------------+---------
-//   signature V | 1 byte
-//   signature R | 32 bytes
-//   signature S | 32 bytes
+//	Description | Length
+//	------------+---------
+//	signature V | 1 byte
+//	signature R | 32 bytes
+//	signature S | 32 bytes
 func (w *ledgerDriver) ledgerSign(derivationPath []uint32, tx *types.Transaction, chainID *big.Int) (common.Address, *types.Transaction, error) {
 	// Flatten the derivation path into the Ledger request
 	path := make([]byte, 1+4*len(derivationPath))
@@ -347,9 +349,15 @@ func (w *ledgerDriver) ledgerSign(derivationPath []uint32, tx *types.Transaction
 		op    = ledgerP1InitTransactionData
 		reply []byte
 	)
+
+	// Chunk size selection to mitigate an underlying RLP deserialization issue on the ledger app.
+	// https://github.com/LedgerHQ/app-ethereum/issues/409
+	chunk := 255
+	for ; len(payload)%chunk <= ledgerEip155Size; chunk-- {
+	}
+
 	for len(payload) > 0 {
 		// Calculate the size of the next data chunk
-		chunk := 255
 		if chunk > len(payload) {
 			chunk = len(payload)
 		}
@@ -392,30 +400,28 @@ func (w *ledgerDriver) ledgerSign(derivationPath []uint32, tx *types.Transaction
 //
 // The signing protocol is defined as follows:
 //
-//   CLA | INS | P1 | P2                          | Lc  | Le
-//   ----+-----+----+-----------------------------+-----+---
-//    E0 | 0C  | 00 | implementation version : 00 | variable | variable
+//	CLA | INS | P1 | P2                          | Lc  | Le
+//	----+-----+----+-----------------------------+-----+---
+//	 E0 | 0C  | 00 | implementation version : 00 | variable | variable
 //
 // Where the input is:
 //
-//   Description                                      | Length
-//   -------------------------------------------------+----------
-//   Number of BIP 32 derivations to perform (max 10) | 1 byte
-//   First derivation index (big endian)              | 4 bytes
-//   ...                                              | 4 bytes
-//   Last derivation index (big endian)               | 4 bytes
-//   domain hash                                      | 32 bytes
-//   message hash                                     | 32 bytes
-//
-//
+//	Description                                      | Length
+//	-------------------------------------------------+----------
+//	Number of BIP 32 derivations to perform (max 10) | 1 byte
+//	First derivation index (big endian)              | 4 bytes
+//	...                                              | 4 bytes
+//	Last derivation index (big endian)               | 4 bytes
+//	domain hash                                      | 32 bytes
+//	message hash                                     | 32 bytes
 //
 // And the output data is:
 //
-//   Description | Length
-//   ------------+---------
-//   signature V | 1 byte
-//   signature R | 32 bytes
-//   signature S | 32 bytes
+//	Description | Length
+//	------------+---------
+//	signature V | 1 byte
+//	signature R | 32 bytes
+//	signature S | 32 bytes
 func (w *ledgerDriver) ledgerSignTypedMessage(derivationPath []uint32, domainHash []byte, messageHash []byte) ([]byte, error) {
 	// Flatten the derivation path into the Ledger request
 	path := make([]byte, 1+4*len(derivationPath))
@@ -454,12 +460,12 @@ func (w *ledgerDriver) ledgerSignTypedMessage(derivationPath []uint32, domainHas
 //
 // The common transport header is defined as follows:
 //
-//  Description                           | Length
-//  --------------------------------------+----------
-//  Communication channel ID (big endian) | 2 bytes
-//  Command tag                           | 1 byte
-//  Packet sequence index (big endian)    | 2 bytes
-//  Payload                               | arbitrary
+//	Description                           | Length
+//	--------------------------------------+----------
+//	Communication channel ID (big endian) | 2 bytes
+//	Command tag                           | 1 byte
+//	Packet sequence index (big endian)    | 2 bytes
+//	Payload                               | arbitrary
 //
 // The Communication channel ID allows commands multiplexing over the same
 // physical link. It is not used for the time being, and should be set to 0101
@@ -473,15 +479,15 @@ func (w *ledgerDriver) ledgerSignTypedMessage(derivationPath []uint32, domainHas
 //
 // APDU Command payloads are encoded as follows:
 //
-//  Description              | Length
-//  -----------------------------------
-//  APDU length (big endian) | 2 bytes
-//  APDU CLA                 | 1 byte
-//  APDU INS                 | 1 byte
-//  APDU P1                  | 1 byte
-//  APDU P2                  | 1 byte
-//  APDU length              | 1 byte
-//  Optional APDU data       | arbitrary
+//	Description              | Length
+//	-----------------------------------
+//	APDU length (big endian) | 2 bytes
+//	APDU CLA                 | 1 byte
+//	APDU INS                 | 1 byte
+//	APDU P1                  | 1 byte
+//	APDU P2                  | 1 byte
+//	APDU length              | 1 byte
+//	Optional APDU data       | arbitrary
 func (w *ledgerDriver) ledgerExchange(opcode ledgerOpcode, p1 ledgerParam1, p2 ledgerParam2, data []byte) ([]byte, error) {
 	// Construct the message payload, possibly split into multiple chunks
 	apdu := make([]byte, 2, 7+len(data))

accounts/usbwallet/trezor.go

@@ -84,15 +84,15 @@ func (w *trezorDriver) Status() (string, error) {
 
 // Open implements usbwallet.driver, attempting to initialize the connection to
 // the Trezor hardware wallet. Initializing the Trezor is a two or three phase operation:
-//  * The first phase is to initialize the connection and read the wallet's
-//    features. This phase is invoked if the provided passphrase is empty. The
-//    device will display the pinpad as a result and will return an appropriate
-//    error to notify the user that a second open phase is needed.
-//  * The second phase is to unlock access to the Trezor, which is done by the
-//    user actually providing a passphrase mapping a keyboard keypad to the pin
-//    number of the user (shuffled according to the pinpad displayed).
-//  * If needed the device will ask for passphrase which will require calling
-//    open again with the actual passphrase (3rd phase)
+//   - The first phase is to initialize the connection and read the wallet's
+//     features. This phase is invoked if the provided passphrase is empty. The
+//     device will display the pinpad as a result and will return an appropriate
+//     error to notify the user that a second open phase is needed.
+//   - The second phase is to unlock access to the Trezor, which is done by the
+//     user actually providing a passphrase mapping a keyboard keypad to the pin
+//     number of the user (shuffled according to the pinpad displayed).
+//   - If needed the device will ask for passphrase which will require calling
+//     open again with the actual passphrase (3rd phase)
 func (w *trezorDriver) Open(device io.ReadWriter, passphrase string) error {
 	w.device, w.failure = device, nil
 
@@ -196,10 +196,10 @@ func (w *trezorDriver) trezorDerive(derivationPath []uint32) (common.Address, er
 	if _, err := w.trezorExchange(&trezor.EthereumGetAddress{AddressN: derivationPath}, address); err != nil {
 		return common.Address{}, err
 	}
-	if addr := address.GetAddressBin(); len(addr) > 0 { // Older firmwares use binary fomats
+	if addr := address.GetAddressBin(); len(addr) > 0 { // Older firmwares use binary formats
 		return common.BytesToAddress(addr), nil
 	}
-	if addr := address.GetAddressHex(); len(addr) > 0 { // Newer firmwares use hexadecimal fomats
+	if addr := address.GetAddressHex(); len(addr) > 0 { // Newer firmwares use hexadecimal formats
 		return common.HexToAddress(addr), nil
 	}
 	return common.Address{}, errors.New("missing derived address")

accounts/usbwallet/trezor/messages-common.pb.go

@@ -94,7 +94,7 @@ func (Failure_FailureType) EnumDescriptor() ([]byte, []int) {
 	return fileDescriptor_aaf30d059fdbc38d, []int{1, 0}
 }
 
-//*
+// *
 // Type of button request
 type ButtonRequest_ButtonRequestType int32
 
@@ -175,7 +175,7 @@ func (ButtonRequest_ButtonRequestType) EnumDescriptor() ([]byte, []int) {
 	return fileDescriptor_aaf30d059fdbc38d, []int{2, 0}
 }
 
-//*
+// *
 // Type of PIN request
 type PinMatrixRequest_PinMatrixRequestType int32
 
@@ -220,7 +220,7 @@ func (PinMatrixRequest_PinMatrixRequestType) EnumDescriptor() ([]byte, []int) {
 	return fileDescriptor_aaf30d059fdbc38d, []int{4, 0}
 }
 
-//*
+// *
 // Response: Success of the previous request
 // @end
 type Success struct {
@@ -262,7 +262,7 @@ func (m *Success) GetMessage() string {
 	return ""
 }
 
-//*
+// *
 // Response: Failure of the previous request
 // @end
 type Failure struct {
@@ -312,7 +312,7 @@ func (m *Failure) GetMessage() string {
 	return ""
 }
 
-//*
+// *
 // Response: Device is waiting for HW button press.
 // @auxstart
 // @next ButtonAck
@@ -363,7 +363,7 @@ func (m *ButtonRequest) GetData() string {
 	return ""
 }
 
-//*
+// *
 // Request: Computer agrees to wait for HW button press
 // @auxend
 type ButtonAck struct {
@@ -397,7 +397,7 @@ func (m *ButtonAck) XXX_DiscardUnknown() {
 
 var xxx_messageInfo_ButtonAck proto.InternalMessageInfo
 
-//*
+// *
 // Response: Device is asking computer to show PIN matrix and awaits PIN encoded using this matrix scheme
 // @auxstart
 // @next PinMatrixAck
@@ -440,7 +440,7 @@ func (m *PinMatrixRequest) GetType() PinMatrixRequest_PinMatrixRequestType {
 	return PinMatrixRequest_PinMatrixRequestType_Current
 }
 
-//*
+// *
 // Request: Computer responds with encoded PIN
 // @auxend
 type PinMatrixAck struct {
@@ -482,7 +482,7 @@ func (m *PinMatrixAck) GetPin() string {
 	return ""
 }
 
-//*
+// *
 // Response: Device awaits encryption passphrase
 // @auxstart
 // @next PassphraseAck
@@ -525,7 +525,7 @@ func (m *PassphraseRequest) GetOnDevice() bool {
 	return false
 }
 
-//*
+// *
 // Request: Send passphrase back
 // @next PassphraseStateRequest
 type PassphraseAck struct {
@@ -575,7 +575,7 @@ func (m *PassphraseAck) GetState() []byte {
 	return nil
 }
 
-//*
+// *
 // Response: Device awaits passphrase state
 // @next PassphraseStateAck
 type PassphraseStateRequest struct {
@@ -617,7 +617,7 @@ func (m *PassphraseStateRequest) GetState() []byte {
 	return nil
 }
 
-//*
+// *
 // Request: Send passphrase state back
 // @auxend
 type PassphraseStateAck struct {
@@ -651,7 +651,7 @@ func (m *PassphraseStateAck) XXX_DiscardUnknown() {
 
 var xxx_messageInfo_PassphraseStateAck proto.InternalMessageInfo
 
-//*
+// *
 // Structure representing BIP32 (hierarchical deterministic) node
 // Used for imports of private key into the device and exporting public key out of device
 // @embed

accounts/usbwallet/trezor/messages-ethereum.pb.go

@@ -21,7 +21,7 @@ var _ = math.Inf
 // proto package needs to be updated.
 const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package
 
-//*
+// *
 // Request: Ask device for public key corresponding to address_n path
 // @start
 // @next EthereumPublicKey
@@ -73,7 +73,7 @@ func (m *EthereumGetPublicKey) GetShowDisplay() bool {
 	return false
 }
 
-//*
+// *
 // Response: Contains public key derived from device private seed
 // @end
 type EthereumPublicKey struct {
@@ -123,7 +123,7 @@ func (m *EthereumPublicKey) GetXpub() string {
 	return ""
 }
 
-//*
+// *
 // Request: Ask device for Ethereum address corresponding to address_n path
 // @start
 // @next EthereumAddress
@@ -175,7 +175,7 @@ func (m *EthereumGetAddress) GetShowDisplay() bool {
 	return false
 }
 
-//*
+// *
 // Response: Contains an Ethereum address derived from device private seed
 // @end
 type EthereumAddress struct {
@@ -225,7 +225,7 @@ func (m *EthereumAddress) GetAddressHex() string {
 	return ""
 }
 
-//*
+// *
 // Request: Ask device to sign transaction
 // All fields are optional from the protocol's point of view. Each field defaults to value `0` if missing.
 // Note: the first at most 1024 bytes of data MUST be transmitted as part of this message.
@@ -351,7 +351,7 @@ func (m *EthereumSignTx) GetTxType() uint32 {
 	return 0
 }
 
-//*
+// *
 // Response: Device asks for more data from transaction payload, or returns the signature.
 // If data_length is set, device awaits that many more bytes of payload.
 // Otherwise, the signature_* fields contain the computed transaction signature. All three fields will be present.
@@ -420,7 +420,7 @@ func (m *EthereumTxRequest) GetSignatureS() []byte {
 	return nil
 }
 
-//*
+// *
 // Request: Transaction payload data.
 // @next EthereumTxRequest
 type EthereumTxAck struct {
@@ -462,7 +462,7 @@ func (m *EthereumTxAck) GetDataChunk() []byte {
 	return nil
 }
 
-//*
+// *
 // Request: Ask device to sign message
 // @start
 // @next EthereumMessageSignature
@@ -514,7 +514,7 @@ func (m *EthereumSignMessage) GetMessage() []byte {
 	return nil
 }
 
-//*
+// *
 // Response: Signed message
 // @end
 type EthereumMessageSignature struct {
@@ -572,7 +572,7 @@ func (m *EthereumMessageSignature) GetAddressHex() string {
 	return ""
 }
 
-//*
+// *
 // Request: Ask device to verify message
 // @start
 // @next Success

accounts/usbwallet/trezor/messages-management.pb.go

@@ -21,7 +21,7 @@ var _ = math.Inf
 // proto package needs to be updated.
 const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package
 
-//*
+// *
 // Structure representing passphrase source
 type ApplySettings_PassphraseSourceType int32
 
@@ -66,7 +66,7 @@ func (ApplySettings_PassphraseSourceType) EnumDescriptor() ([]byte, []int) {
 	return fileDescriptor_0c720c20d27aa029, []int{4, 0}
 }
 
-//*
+// *
 // Type of recovery procedure. These should be used as bitmask, e.g.,
 // `RecoveryDeviceType_ScrambledWords | RecoveryDeviceType_Matrix`
 // listing every method supported by the host computer.
@@ -114,7 +114,7 @@ func (RecoveryDevice_RecoveryDeviceType) EnumDescriptor() ([]byte, []int) {
 	return fileDescriptor_0c720c20d27aa029, []int{17, 0}
 }
 
-//*
+// *
 // Type of Recovery Word request
 type WordRequest_WordRequestType int32
 
@@ -159,7 +159,7 @@ func (WordRequest_WordRequestType) EnumDescriptor() ([]byte, []int) {
 	return fileDescriptor_0c720c20d27aa029, []int{18, 0}
 }
 
-//*
+// *
 // Request: Reset device to default state and ask for device details
 // @start
 // @next Features
@@ -210,7 +210,7 @@ func (m *Initialize) GetSkipPassphrase() bool {
 	return false
 }
 
-//*
+// *
 // Request: Ask for device details (no device reset)
 // @start
 // @next Features
@@ -245,7 +245,7 @@ func (m *GetFeatures) XXX_DiscardUnknown() {
 
 var xxx_messageInfo_GetFeatures proto.InternalMessageInfo
 
-//*
+// *
 // Response: Reports various information about the device
 // @end
 type Features struct {
@@ -495,7 +495,7 @@ func (m *Features) GetNoBackup() bool {
 	return false
 }
 
-//*
+// *
 // Request: clear session (removes cached PIN, passphrase, etc).
 // @start
 // @next Success
@@ -530,7 +530,7 @@ func (m *ClearSession) XXX_DiscardUnknown() {
 
 var xxx_messageInfo_ClearSession proto.InternalMessageInfo
 
-//*
+// *
 // Request: change language and/or label of the device
 // @start
 // @next Success
@@ -622,7 +622,7 @@ func (m *ApplySettings) GetDisplayRotation() uint32 {
 	return 0
 }
 
-//*
+// *
 // Request: set flags of the device
 // @start
 // @next Success
@@ -666,7 +666,7 @@ func (m *ApplyFlags) GetFlags() uint32 {
 	return 0
 }
 
-//*
+// *
 // Request: Starts workflow for setting/changing/removing the PIN
 // @start
 // @next Success
@@ -710,7 +710,7 @@ func (m *ChangePin) GetRemove() bool {
 	return false
 }
 
-//*
+// *
 // Request: Test if the device is alive, device sends back the message in Success response
 // @start
 // @next Success
@@ -777,7 +777,7 @@ func (m *Ping) GetPassphraseProtection() bool {
 	return false
 }
 
-//*
+// *
 // Request: Abort last operation that required user interaction
 // @start
 // @next Failure
@@ -812,7 +812,7 @@ func (m *Cancel) XXX_DiscardUnknown() {
 
 var xxx_messageInfo_Cancel proto.InternalMessageInfo
 
-//*
+// *
 // Request: Request a sample of random data generated by hardware RNG. May be used for testing.
 // @start
 // @next Entropy
@@ -856,7 +856,7 @@ func (m *GetEntropy) GetSize() uint32 {
 	return 0
 }
 
-//*
+// *
 // Response: Reply with random data generated by internal RNG
 // @end
 type Entropy struct {
@@ -898,7 +898,7 @@ func (m *Entropy) GetEntropy() []byte {
 	return nil
 }
 
-//*
+// *
 // Request: Request device to wipe all sensitive data and settings
 // @start
 // @next Success
@@ -934,7 +934,7 @@ func (m *WipeDevice) XXX_DiscardUnknown() {
 
 var xxx_messageInfo_WipeDevice proto.InternalMessageInfo
 
-//*
+// *
 // Request: Load seed and related internal settings from the computer
 // @start
 // @next Success
@@ -1036,7 +1036,7 @@ func (m *LoadDevice) GetU2FCounter() uint32 {
 	return 0
 }
 
-//*
+// *
 // Request: Ask device to do initialization involving user interaction
 // @start
 // @next EntropyRequest
@@ -1147,7 +1147,7 @@ func (m *ResetDevice) GetNoBackup() bool {
 	return false
 }
 
-//*
+// *
 // Request: Perform backup of the device seed if not backed up using ResetDevice
 // @start
 // @next Success
@@ -1182,7 +1182,7 @@ func (m *BackupDevice) XXX_DiscardUnknown() {
 
 var xxx_messageInfo_BackupDevice proto.InternalMessageInfo
 
-//*
+// *
 // Response: Ask for additional entropy from host computer
 // @next EntropyAck
 type EntropyRequest struct {
@@ -1216,7 +1216,7 @@ func (m *EntropyRequest) XXX_DiscardUnknown() {
 
 var xxx_messageInfo_EntropyRequest proto.InternalMessageInfo
 
-//*
+// *
 // Request: Provide additional entropy for seed generation function
 // @next Success
 type EntropyAck struct {
@@ -1258,7 +1258,7 @@ func (m *EntropyAck) GetEntropy() []byte {
 	return nil
 }
 
-//*
+// *
 // Request: Start recovery workflow asking user for specific words of mnemonic
 // Used to recovery device safely even on untrusted computer.
 // @start
@@ -1369,7 +1369,7 @@ func (m *RecoveryDevice) GetDryRun() bool {
 	return false
 }
 
-//*
+// *
 // Response: Device is waiting for user to enter word of the mnemonic
 // Its position is shown only on device's internal display.
 // @next WordAck
@@ -1412,7 +1412,7 @@ func (m *WordRequest) GetType() WordRequest_WordRequestType {
 	return WordRequest_WordRequestType_Plain
 }
 
-//*
+// *
 // Request: Computer replies with word from the mnemonic
 // @next WordRequest
 // @next Success
@@ -1456,7 +1456,7 @@ func (m *WordAck) GetWord() string {
 	return ""
 }
 
-//*
+// *
 // Request: Set U2F counter
 // @start
 // @next Success

accounts/usbwallet/trezor/messages.pb.go

@@ -22,7 +22,7 @@ var _ = math.Inf
 // proto package needs to be updated.
 const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package
 
-//*
+// *
 // Mapping between TREZOR wire identifier (uint) and a protobuf message
 type MessageType int32
 

accounts/usbwallet/wallet.go

@@ -380,7 +380,7 @@ func (w *wallet) selfDerive() {
 					// of legacy-ledger, the first account on the legacy-path will
 					// be shown to the user, even if we don't actively track it
 					if i < len(nextAddrs)-1 {
-						w.log.Info("Skipping trakcking first account on legacy path, use personal.deriveAccount(<url>,<path>, false) to track",
+						w.log.Info("Skipping tracking first account on legacy path, use personal.deriveAccount(<url>,<path>, false) to track",
 							"path", path, "address", nextAddrs[i])
 						break
 					}
@@ -526,7 +526,6 @@ func (w *wallet) signHash(account accounts.Account, hash []byte) ([]byte, error)
 
 // SignData signs keccak256(data). The mimetype parameter describes the type of data being signed
 func (w *wallet) SignData(account accounts.Account, mimeType string, data []byte) ([]byte, error) {
-
 	// Unless we are doing 712 signing, simply dispatch to signHash
 	if !(mimeType == accounts.MimetypeTypedData && len(data) == 66 && data[0] == 0x19 && data[1] == 0x01) {
 		return w.signHash(account, crypto.Keccak256(data))

appveyor.yml

@@ -26,7 +26,7 @@ for:
       - go run build/ci.go lint
       - go run build/ci.go install -dlgo
     test_script:
-      - go run build/ci.go test -dlgo -coverage
+      - go run build/ci.go test -dlgo
 
   # linux/386 is disabled.
   - matrix:
@@ -54,4 +54,4 @@ for:
       - go run build/ci.go archive -arch %GETH_ARCH% -type zip -signer WINDOWS_SIGNING_KEY -upload gethstore/builds
       - go run build/ci.go nsis -arch %GETH_ARCH% -signer WINDOWS_SIGNING_KEY -upload gethstore/builds
     test_script:
-      - go run build/ci.go test -dlgo -arch %GETH_ARCH% -cc %GETH_CC% -coverage
+      - go run build/ci.go test -dlgo -arch %GETH_ARCH% -cc %GETH_CC%

beacon/engine/errors.go

@@ -14,7 +14,7 @@
 // You should have received a copy of the GNU Lesser General Public License
 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
 
-package beacon
+package engine
 
 import (
 	"github.com/ethereum/go-ethereum/common"
@@ -74,12 +74,12 @@ var (
 	//   - newPayloadV1: if the payload was accepted, but not processed (side chain)
 	ACCEPTED = "ACCEPTED"
 
-	INVALIDBLOCKHASH = "INVALID_BLOCK_HASH"
-
 	GenericServerError       = &EngineAPIError{code: -32000, msg: "Server error"}
 	UnknownPayload           = &EngineAPIError{code: -38001, msg: "Unknown payload"}
 	InvalidForkChoiceState   = &EngineAPIError{code: -38002, msg: "Invalid forkchoice state"}
 	InvalidPayloadAttributes = &EngineAPIError{code: -38003, msg: "Invalid payload attributes"}
+	TooLargeRequest          = &EngineAPIError{code: -38004, msg: "Too large request"}
+	InvalidParams            = &EngineAPIError{code: -32602, msg: "Invalid parameters"}
 
 	STATUS_INVALID         = ForkChoiceResponse{PayloadStatus: PayloadStatusV1{Status: INVALID}, PayloadID: nil}
 	STATUS_SYNCING         = ForkChoiceResponse{PayloadStatus: PayloadStatusV1{Status: SYNCING}, PayloadID: nil}

beacon/engine/gen_blockparams.go

@@ -0,0 +1,60 @@
+// Code generated by github.com/fjl/gencodec. DO NOT EDIT.
+
+package engine
+
+import (
+	"encoding/json"
+	"errors"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/core/types"
+)
+
+var _ = (*payloadAttributesMarshaling)(nil)
+
+// MarshalJSON marshals as JSON.
+func (p PayloadAttributes) MarshalJSON() ([]byte, error) {
+	type PayloadAttributes struct {
+		Timestamp             hexutil.Uint64      `json:"timestamp"             gencodec:"required"`
+		Random                common.Hash         `json:"prevRandao"            gencodec:"required"`
+		SuggestedFeeRecipient common.Address      `json:"suggestedFeeRecipient" gencodec:"required"`
+		Withdrawals           []*types.Withdrawal `json:"withdrawals"`
+	}
+	var enc PayloadAttributes
+	enc.Timestamp = hexutil.Uint64(p.Timestamp)
+	enc.Random = p.Random
+	enc.SuggestedFeeRecipient = p.SuggestedFeeRecipient
+	enc.Withdrawals = p.Withdrawals
+	return json.Marshal(&enc)
+}
+
+// UnmarshalJSON unmarshals from JSON.
+func (p *PayloadAttributes) UnmarshalJSON(input []byte) error {
+	type PayloadAttributes struct {
+		Timestamp             *hexutil.Uint64     `json:"timestamp"             gencodec:"required"`
+		Random                *common.Hash        `json:"prevRandao"            gencodec:"required"`
+		SuggestedFeeRecipient *common.Address     `json:"suggestedFeeRecipient" gencodec:"required"`
+		Withdrawals           []*types.Withdrawal `json:"withdrawals"`
+	}
+	var dec PayloadAttributes
+	if err := json.Unmarshal(input, &dec); err != nil {
+		return err
+	}
+	if dec.Timestamp == nil {
+		return errors.New("missing required field 'timestamp' for PayloadAttributes")
+	}
+	p.Timestamp = uint64(*dec.Timestamp)
+	if dec.Random == nil {
+		return errors.New("missing required field 'prevRandao' for PayloadAttributes")
+	}
+	p.Random = *dec.Random
+	if dec.SuggestedFeeRecipient == nil {
+		return errors.New("missing required field 'suggestedFeeRecipient' for PayloadAttributes")
+	}
+	p.SuggestedFeeRecipient = *dec.SuggestedFeeRecipient
+	if dec.Withdrawals != nil {
+		p.Withdrawals = dec.Withdrawals
+	}
+	return nil
+}

beacon/engine/gen_ed.go

@@ -1,6 +1,6 @@
 // Code generated by github.com/fjl/gencodec. DO NOT EDIT.
 
-package beacon
+package engine
 
 import (
 	"encoding/json"
@@ -9,29 +9,31 @@ import (
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/core/types"
 )
 
 var _ = (*executableDataMarshaling)(nil)
 
 // MarshalJSON marshals as JSON.
-func (e ExecutableDataV1) MarshalJSON() ([]byte, error) {
-	type ExecutableDataV1 struct {
-		ParentHash    common.Hash     `json:"parentHash"    gencodec:"required"`
-		FeeRecipient  common.Address  `json:"feeRecipient"  gencodec:"required"`
-		StateRoot     common.Hash     `json:"stateRoot"     gencodec:"required"`
-		ReceiptsRoot  common.Hash     `json:"receiptsRoot"  gencodec:"required"`
-		LogsBloom     hexutil.Bytes   `json:"logsBloom"     gencodec:"required"`
-		Random        common.Hash     `json:"prevRandao"    gencodec:"required"`
-		Number        hexutil.Uint64  `json:"blockNumber"   gencodec:"required"`
-		GasLimit      hexutil.Uint64  `json:"gasLimit"      gencodec:"required"`
-		GasUsed       hexutil.Uint64  `json:"gasUsed"       gencodec:"required"`
-		Timestamp     hexutil.Uint64  `json:"timestamp"     gencodec:"required"`
-		ExtraData     hexutil.Bytes   `json:"extraData"     gencodec:"required"`
-		BaseFeePerGas *hexutil.Big    `json:"baseFeePerGas" gencodec:"required"`
-		BlockHash     common.Hash     `json:"blockHash"     gencodec:"required"`
-		Transactions  []hexutil.Bytes `json:"transactions"  gencodec:"required"`
+func (e ExecutableData) MarshalJSON() ([]byte, error) {
+	type ExecutableData struct {
+		ParentHash    common.Hash         `json:"parentHash"    gencodec:"required"`
+		FeeRecipient  common.Address      `json:"feeRecipient"  gencodec:"required"`
+		StateRoot     common.Hash         `json:"stateRoot"     gencodec:"required"`
+		ReceiptsRoot  common.Hash         `json:"receiptsRoot"  gencodec:"required"`
+		LogsBloom     hexutil.Bytes       `json:"logsBloom"     gencodec:"required"`
+		Random        common.Hash         `json:"prevRandao"    gencodec:"required"`
+		Number        hexutil.Uint64      `json:"blockNumber"   gencodec:"required"`
+		GasLimit      hexutil.Uint64      `json:"gasLimit"      gencodec:"required"`
+		GasUsed       hexutil.Uint64      `json:"gasUsed"       gencodec:"required"`
+		Timestamp     hexutil.Uint64      `json:"timestamp"     gencodec:"required"`
+		ExtraData     hexutil.Bytes       `json:"extraData"     gencodec:"required"`
+		BaseFeePerGas *hexutil.Big        `json:"baseFeePerGas" gencodec:"required"`
+		BlockHash     common.Hash         `json:"blockHash"     gencodec:"required"`
+		Transactions  []hexutil.Bytes     `json:"transactions"  gencodec:"required"`
+		Withdrawals   []*types.Withdrawal `json:"withdrawals"`
 	}
-	var enc ExecutableDataV1
+	var enc ExecutableData
 	enc.ParentHash = e.ParentHash
 	enc.FeeRecipient = e.FeeRecipient
 	enc.StateRoot = e.StateRoot
@@ -51,89 +53,94 @@ func (e ExecutableDataV1) MarshalJSON() ([]byte, error) {
 			enc.Transactions[k] = v
 		}
 	}
+	enc.Withdrawals = e.Withdrawals
 	return json.Marshal(&enc)
 }
 
 // UnmarshalJSON unmarshals from JSON.
-func (e *ExecutableDataV1) UnmarshalJSON(input []byte) error {
-	type ExecutableDataV1 struct {
-		ParentHash    *common.Hash    `json:"parentHash"    gencodec:"required"`
-		FeeRecipient  *common.Address `json:"feeRecipient"  gencodec:"required"`
-		StateRoot     *common.Hash    `json:"stateRoot"     gencodec:"required"`
-		ReceiptsRoot  *common.Hash    `json:"receiptsRoot"  gencodec:"required"`
-		LogsBloom     *hexutil.Bytes  `json:"logsBloom"     gencodec:"required"`
-		Random        *common.Hash    `json:"prevRandao"    gencodec:"required"`
-		Number        *hexutil.Uint64 `json:"blockNumber"   gencodec:"required"`
-		GasLimit      *hexutil.Uint64 `json:"gasLimit"      gencodec:"required"`
-		GasUsed       *hexutil.Uint64 `json:"gasUsed"       gencodec:"required"`
-		Timestamp     *hexutil.Uint64 `json:"timestamp"     gencodec:"required"`
-		ExtraData     *hexutil.Bytes  `json:"extraData"     gencodec:"required"`
-		BaseFeePerGas *hexutil.Big    `json:"baseFeePerGas" gencodec:"required"`
-		BlockHash     *common.Hash    `json:"blockHash"     gencodec:"required"`
-		Transactions  []hexutil.Bytes `json:"transactions"  gencodec:"required"`
+func (e *ExecutableData) UnmarshalJSON(input []byte) error {
+	type ExecutableData struct {
+		ParentHash    *common.Hash        `json:"parentHash"    gencodec:"required"`
+		FeeRecipient  *common.Address     `json:"feeRecipient"  gencodec:"required"`
+		StateRoot     *common.Hash        `json:"stateRoot"     gencodec:"required"`
+		ReceiptsRoot  *common.Hash        `json:"receiptsRoot"  gencodec:"required"`
+		LogsBloom     *hexutil.Bytes      `json:"logsBloom"     gencodec:"required"`
+		Random        *common.Hash        `json:"prevRandao"    gencodec:"required"`
+		Number        *hexutil.Uint64     `json:"blockNumber"   gencodec:"required"`
+		GasLimit      *hexutil.Uint64     `json:"gasLimit"      gencodec:"required"`
+		GasUsed       *hexutil.Uint64     `json:"gasUsed"       gencodec:"required"`
+		Timestamp     *hexutil.Uint64     `json:"timestamp"     gencodec:"required"`
+		ExtraData     *hexutil.Bytes      `json:"extraData"     gencodec:"required"`
+		BaseFeePerGas *hexutil.Big        `json:"baseFeePerGas" gencodec:"required"`
+		BlockHash     *common.Hash        `json:"blockHash"     gencodec:"required"`
+		Transactions  []hexutil.Bytes     `json:"transactions"  gencodec:"required"`
+		Withdrawals   []*types.Withdrawal `json:"withdrawals"`
 	}
-	var dec ExecutableDataV1
+	var dec ExecutableData
 	if err := json.Unmarshal(input, &dec); err != nil {
 		return err
 	}
 	if dec.ParentHash == nil {
-		return errors.New("missing required field 'parentHash' for ExecutableDataV1")
+		return errors.New("missing required field 'parentHash' for ExecutableData")
 	}
 	e.ParentHash = *dec.ParentHash
 	if dec.FeeRecipient == nil {
-		return errors.New("missing required field 'feeRecipient' for ExecutableDataV1")
+		return errors.New("missing required field 'feeRecipient' for ExecutableData")
 	}
 	e.FeeRecipient = *dec.FeeRecipient
 	if dec.StateRoot == nil {
-		return errors.New("missing required field 'stateRoot' for ExecutableDataV1")
+		return errors.New("missing required field 'stateRoot' for ExecutableData")
 	}
 	e.StateRoot = *dec.StateRoot
 	if dec.ReceiptsRoot == nil {
-		return errors.New("missing required field 'receiptsRoot' for ExecutableDataV1")
+		return errors.New("missing required field 'receiptsRoot' for ExecutableData")
 	}
 	e.ReceiptsRoot = *dec.ReceiptsRoot
 	if dec.LogsBloom == nil {
-		return errors.New("missing required field 'logsBloom' for ExecutableDataV1")
+		return errors.New("missing required field 'logsBloom' for ExecutableData")
 	}
 	e.LogsBloom = *dec.LogsBloom
 	if dec.Random == nil {
-		return errors.New("missing required field 'prevRandao' for ExecutableDataV1")
+		return errors.New("missing required field 'prevRandao' for ExecutableData")
 	}
 	e.Random = *dec.Random
 	if dec.Number == nil {
-		return errors.New("missing required field 'blockNumber' for ExecutableDataV1")
+		return errors.New("missing required field 'blockNumber' for ExecutableData")
 	}
 	e.Number = uint64(*dec.Number)
 	if dec.GasLimit == nil {
-		return errors.New("missing required field 'gasLimit' for ExecutableDataV1")
+		return errors.New("missing required field 'gasLimit' for ExecutableData")
 	}
 	e.GasLimit = uint64(*dec.GasLimit)
 	if dec.GasUsed == nil {
-		return errors.New("missing required field 'gasUsed' for ExecutableDataV1")
+		return errors.New("missing required field 'gasUsed' for ExecutableData")
 	}
 	e.GasUsed = uint64(*dec.GasUsed)
 	if dec.Timestamp == nil {
-		return errors.New("missing required field 'timestamp' for ExecutableDataV1")
+		return errors.New("missing required field 'timestamp' for ExecutableData")
 	}
 	e.Timestamp = uint64(*dec.Timestamp)
 	if dec.ExtraData == nil {
-		return errors.New("missing required field 'extraData' for ExecutableDataV1")
+		return errors.New("missing required field 'extraData' for ExecutableData")
 	}
 	e.ExtraData = *dec.ExtraData
 	if dec.BaseFeePerGas == nil {
-		return errors.New("missing required field 'baseFeePerGas' for ExecutableDataV1")
+		return errors.New("missing required field 'baseFeePerGas' for ExecutableData")
 	}
 	e.BaseFeePerGas = (*big.Int)(dec.BaseFeePerGas)
 	if dec.BlockHash == nil {
-		return errors.New("missing required field 'blockHash' for ExecutableDataV1")
+		return errors.New("missing required field 'blockHash' for ExecutableData")
 	}
 	e.BlockHash = *dec.BlockHash
 	if dec.Transactions == nil {
-		return errors.New("missing required field 'transactions' for ExecutableDataV1")
+		return errors.New("missing required field 'transactions' for ExecutableData")
 	}
 	e.Transactions = make([][]byte, len(dec.Transactions))
 	for k, v := range dec.Transactions {
 		e.Transactions[k] = v
 	}
+	if dec.Withdrawals != nil {
+		e.Withdrawals = dec.Withdrawals
+	}
 	return nil
 }

beacon/engine/gen_epe.go

@@ -0,0 +1,46 @@
+// Code generated by github.com/fjl/gencodec. DO NOT EDIT.
+
+package engine
+
+import (
+	"encoding/json"
+	"errors"
+	"math/big"
+
+	"github.com/ethereum/go-ethereum/common/hexutil"
+)
+
+var _ = (*executionPayloadEnvelopeMarshaling)(nil)
+
+// MarshalJSON marshals as JSON.
+func (e ExecutionPayloadEnvelope) MarshalJSON() ([]byte, error) {
+	type ExecutionPayloadEnvelope struct {
+		ExecutionPayload *ExecutableData `json:"executionPayload"  gencodec:"required"`
+		BlockValue       *hexutil.Big    `json:"blockValue"  gencodec:"required"`
+	}
+	var enc ExecutionPayloadEnvelope
+	enc.ExecutionPayload = e.ExecutionPayload
+	enc.BlockValue = (*hexutil.Big)(e.BlockValue)
+	return json.Marshal(&enc)
+}
+
+// UnmarshalJSON unmarshals from JSON.
+func (e *ExecutionPayloadEnvelope) UnmarshalJSON(input []byte) error {
+	type ExecutionPayloadEnvelope struct {
+		ExecutionPayload *ExecutableData `json:"executionPayload"  gencodec:"required"`
+		BlockValue       *hexutil.Big    `json:"blockValue"  gencodec:"required"`
+	}
+	var dec ExecutionPayloadEnvelope
+	if err := json.Unmarshal(input, &dec); err != nil {
+		return err
+	}
+	if dec.ExecutionPayload == nil {
+		return errors.New("missing required field 'executionPayload' for ExecutionPayloadEnvelope")
+	}
+	e.ExecutionPayload = dec.ExecutionPayload
+	if dec.BlockValue == nil {
+		return errors.New("missing required field 'blockValue' for ExecutionPayloadEnvelope")
+	}
+	e.BlockValue = (*big.Int)(dec.BlockValue)
+	return nil
+}

beacon/engine/types.go

@@ -14,7 +14,7 @@
 // You should have received a copy of the GNU Lesser General Public License
 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
 
-package beacon
+package engine
 
 import (
 	"fmt"
@@ -26,38 +26,41 @@ import (
 	"github.com/ethereum/go-ethereum/trie"
 )
 
-//go:generate go run github.com/fjl/gencodec -type PayloadAttributesV1 -field-override payloadAttributesMarshaling -out gen_blockparams.go
+//go:generate go run github.com/fjl/gencodec -type PayloadAttributes -field-override payloadAttributesMarshaling -out gen_blockparams.go
 
-// PayloadAttributesV1 structure described at https://github.com/ethereum/execution-apis/pull/74
-type PayloadAttributesV1 struct {
-	Timestamp             uint64         `json:"timestamp"     gencodec:"required"`
-	Random                common.Hash    `json:"prevRandao"        gencodec:"required"`
-	SuggestedFeeRecipient common.Address `json:"suggestedFeeRecipient"  gencodec:"required"`
+// PayloadAttributes describes the environment context in which a block should
+// be built.
+type PayloadAttributes struct {
+	Timestamp             uint64              `json:"timestamp"             gencodec:"required"`
+	Random                common.Hash         `json:"prevRandao"            gencodec:"required"`
+	SuggestedFeeRecipient common.Address      `json:"suggestedFeeRecipient" gencodec:"required"`
+	Withdrawals           []*types.Withdrawal `json:"withdrawals"`
 }
 
-// JSON type overrides for PayloadAttributesV1.
+// JSON type overrides for PayloadAttributes.
 type payloadAttributesMarshaling struct {
 	Timestamp hexutil.Uint64
 }
 
-//go:generate go run github.com/fjl/gencodec -type ExecutableDataV1 -field-override executableDataMarshaling -out gen_ed.go
+//go:generate go run github.com/fjl/gencodec -type ExecutableData -field-override executableDataMarshaling -out gen_ed.go
 
-// ExecutableDataV1 structure described at https://github.com/ethereum/execution-apis/src/engine/specification.md
-type ExecutableDataV1 struct {
-	ParentHash    common.Hash    `json:"parentHash"    gencodec:"required"`
-	FeeRecipient  common.Address `json:"feeRecipient"  gencodec:"required"`
-	StateRoot     common.Hash    `json:"stateRoot"     gencodec:"required"`
-	ReceiptsRoot  common.Hash    `json:"receiptsRoot"  gencodec:"required"`
-	LogsBloom     []byte         `json:"logsBloom"     gencodec:"required"`
-	Random        common.Hash    `json:"prevRandao"    gencodec:"required"`
-	Number        uint64         `json:"blockNumber"   gencodec:"required"`
-	GasLimit      uint64         `json:"gasLimit"      gencodec:"required"`
-	GasUsed       uint64         `json:"gasUsed"       gencodec:"required"`
-	Timestamp     uint64         `json:"timestamp"     gencodec:"required"`
-	ExtraData     []byte         `json:"extraData"     gencodec:"required"`
-	BaseFeePerGas *big.Int       `json:"baseFeePerGas" gencodec:"required"`
-	BlockHash     common.Hash    `json:"blockHash"     gencodec:"required"`
-	Transactions  [][]byte       `json:"transactions"  gencodec:"required"`
+// ExecutableData is the data necessary to execute an EL payload.
+type ExecutableData struct {
+	ParentHash    common.Hash         `json:"parentHash"    gencodec:"required"`
+	FeeRecipient  common.Address      `json:"feeRecipient"  gencodec:"required"`
+	StateRoot     common.Hash         `json:"stateRoot"     gencodec:"required"`
+	ReceiptsRoot  common.Hash         `json:"receiptsRoot"  gencodec:"required"`
+	LogsBloom     []byte              `json:"logsBloom"     gencodec:"required"`
+	Random        common.Hash         `json:"prevRandao"    gencodec:"required"`
+	Number        uint64              `json:"blockNumber"   gencodec:"required"`
+	GasLimit      uint64              `json:"gasLimit"      gencodec:"required"`
+	GasUsed       uint64              `json:"gasUsed"       gencodec:"required"`
+	Timestamp     uint64              `json:"timestamp"     gencodec:"required"`
+	ExtraData     []byte              `json:"extraData"     gencodec:"required"`
+	BaseFeePerGas *big.Int            `json:"baseFeePerGas" gencodec:"required"`
+	BlockHash     common.Hash         `json:"blockHash"     gencodec:"required"`
+	Transactions  [][]byte            `json:"transactions"  gencodec:"required"`
+	Withdrawals   []*types.Withdrawal `json:"withdrawals"`
 }
 
 // JSON type overrides for executableData.
@@ -72,6 +75,18 @@ type executableDataMarshaling struct {
 	Transactions  []hexutil.Bytes
 }
 
+//go:generate go run github.com/fjl/gencodec -type ExecutionPayloadEnvelope -field-override executionPayloadEnvelopeMarshaling -out gen_epe.go
+
+type ExecutionPayloadEnvelope struct {
+	ExecutionPayload *ExecutableData `json:"executionPayload"  gencodec:"required"`
+	BlockValue       *big.Int        `json:"blockValue"  gencodec:"required"`
+}
+
+// JSON type overrides for ExecutionPayloadEnvelope.
+type executionPayloadEnvelopeMarshaling struct {
+	BlockValue *hexutil.Big
+}
+
 type PayloadStatusV1 struct {
 	Status          string       `json:"status"`
 	LatestValidHash *common.Hash `json:"latestValidHash"`
@@ -136,11 +151,15 @@ func decodeTransactions(enc [][]byte) ([]*types.Transaction, error) {
 
 // ExecutableDataToBlock constructs a block from executable data.
 // It verifies that the following fields:
-// 		len(extraData) <= 32
-// 		uncleHash = emptyUncleHash
-// 		difficulty = 0
-// and that the blockhash of the constructed block matches the parameters.
-func ExecutableDataToBlock(params ExecutableDataV1) (*types.Block, error) {
+//
+//	len(extraData) <= 32
+//	uncleHash = emptyUncleHash
+//	difficulty = 0
+//
+// and that the blockhash of the constructed block matches the parameters. Nil
+// Withdrawals value will propagate through the returned block. Empty
+// Withdrawals value must be passed via non-nil, length 0 value in params.
+func ExecutableDataToBlock(params ExecutableData) (*types.Block, error) {
 	txs, err := decodeTransactions(params.Transactions)
 	if err != nil {
 		return nil, err
@@ -155,34 +174,43 @@ func ExecutableDataToBlock(params ExecutableDataV1) (*types.Block, error) {
 	if params.BaseFeePerGas != nil && (params.BaseFeePerGas.Sign() == -1 || params.BaseFeePerGas.BitLen() > 256) {
 		return nil, fmt.Errorf("invalid baseFeePerGas: %v", params.BaseFeePerGas)
 	}
-	header := &types.Header{
-		ParentHash:  params.ParentHash,
-		UncleHash:   types.EmptyUncleHash,
-		Coinbase:    params.FeeRecipient,
-		Root:        params.StateRoot,
-		TxHash:      types.DeriveSha(types.Transactions(txs), trie.NewStackTrie(nil)),
-		ReceiptHash: params.ReceiptsRoot,
-		Bloom:       types.BytesToBloom(params.LogsBloom),
-		Difficulty:  common.Big0,
-		Number:      new(big.Int).SetUint64(params.Number),
-		GasLimit:    params.GasLimit,
-		GasUsed:     params.GasUsed,
-		Time:        params.Timestamp,
-		BaseFee:     params.BaseFeePerGas,
-		Extra:       params.ExtraData,
-		MixDigest:   params.Random,
+	// Only set withdrawalsRoot if it is non-nil. This allows CLs to use
+	// ExecutableData before withdrawals are enabled by marshaling
+	// Withdrawals as the json null value.
+	var withdrawalsRoot *common.Hash
+	if params.Withdrawals != nil {
+		h := types.DeriveSha(types.Withdrawals(params.Withdrawals), trie.NewStackTrie(nil))
+		withdrawalsRoot = &h
 	}
-	block := types.NewBlockWithHeader(header).WithBody(txs, nil /* uncles */)
+	header := &types.Header{
+		ParentHash:      params.ParentHash,
+		UncleHash:       types.EmptyUncleHash,
+		Coinbase:        params.FeeRecipient,
+		Root:            params.StateRoot,
+		TxHash:          types.DeriveSha(types.Transactions(txs), trie.NewStackTrie(nil)),
+		ReceiptHash:     params.ReceiptsRoot,
+		Bloom:           types.BytesToBloom(params.LogsBloom),
+		Difficulty:      common.Big0,
+		Number:          new(big.Int).SetUint64(params.Number),
+		GasLimit:        params.GasLimit,
+		GasUsed:         params.GasUsed,
+		Time:            params.Timestamp,
+		BaseFee:         params.BaseFeePerGas,
+		Extra:           params.ExtraData,
+		MixDigest:       params.Random,
+		WithdrawalsHash: withdrawalsRoot,
+	}
+	block := types.NewBlockWithHeader(header).WithBody(txs, nil /* uncles */).WithWithdrawals(params.Withdrawals)
 	if block.Hash() != params.BlockHash {
 		return nil, fmt.Errorf("blockhash mismatch, want %x, got %x", params.BlockHash, block.Hash())
 	}
 	return block, nil
 }
 
-// BlockToExecutableData constructs the executableDataV1 structure by filling the
+// BlockToExecutableData constructs the ExecutableData structure by filling the
 // fields from the given block. It assumes the given block is post-merge block.
-func BlockToExecutableData(block *types.Block) *ExecutableDataV1 {
-	return &ExecutableDataV1{
+func BlockToExecutableData(block *types.Block, fees *big.Int) *ExecutionPayloadEnvelope {
+	data := &ExecutableData{
 		BlockHash:     block.Hash(),
 		ParentHash:    block.ParentHash(),
 		FeeRecipient:  block.Coinbase(),
@@ -197,5 +225,13 @@ func BlockToExecutableData(block *types.Block) *ExecutableDataV1 {
 		Transactions:  encodeTransactions(block.Transactions()),
 		Random:        block.MixDigest(),
 		ExtraData:     block.Extra(),
+		Withdrawals:   block.Withdrawals(),
 	}
+	return &ExecutionPayloadEnvelope{ExecutionPayload: data, BlockValue: fees}
+}
+
+// ExecutionPayloadBodyV1 is used in the response to GetPayloadBodiesByHashV1 and GetPayloadBodiesByRangeV1
+type ExecutionPayloadBodyV1 struct {
+	TransactionData []hexutil.Bytes     `json:"transactions"`
+	Withdrawals     []*types.Withdrawal `json:"withdrawals"`
 }

build/checksums.txt

@@ -1,58 +1,46 @@
 # This file contains sha256 checksums of optional build dependencies.
 
-efd43e0f1402e083b73a03d444b7b6576bb4c539ac46208b63a916b69aca4088  go1.18.1.src.tar.gz
-3703e9a0db1000f18c0c7b524f3d378aac71219b4715a6a4c5683eb639f41a4d  go1.18.1.darwin-amd64.tar.gz
-6d5641a06edba8cd6d425fb0adad06bad80e2afe0fa91b4aa0e5aed1bc78f58e  go1.18.1.darwin-arm64.tar.gz
-b9a9063d4265d8ccc046c9b314194d6eadc47e56d0d637db81e98e68aad45035  go1.18.1.freebsd-386.tar.gz
-2bc1c138d645e37dbbc63517dd1cf1bf33fc4cb95f442a6384df0418b5134e9f  go1.18.1.freebsd-amd64.tar.gz
-9a8df5dde9058f08ac01ecfaae42534610db398e487138788c01da26a0d41ff9  go1.18.1.linux-386.tar.gz
-b3b815f47ababac13810fc6021eb73d65478e0b2db4b09d348eefad9581a2334  go1.18.1.linux-amd64.tar.gz
-56a91851c97fb4697077abbca38860f735c32b38993ff79b088dac46e4735633  go1.18.1.linux-arm64.tar.gz
-9edc01c8e7db64e9ceeffc8258359e027812886ceca3444e83c4eb96ddb068ee  go1.18.1.linux-armv6l.tar.gz
-33db623d1eecf362fe365107c12efc90eff0b9609e0b3345e258388019cb552a  go1.18.1.linux-ppc64le.tar.gz
-5d9301324148ed4dbfaa0800da43a843ffd65c834ee73fcf087255697c925f74  go1.18.1.linux-s390x.tar.gz
-49ae65551acbfaa57b52fbefa0350b2072512ae3103b8cf1a919a02626dbc743  go1.18.1.windows-386.zip
-c30bc3f1f7314a953fe208bd9cd5e24bd9403392a6c556ced3677f9f70f71fe1  go1.18.1.windows-amd64.zip
-2c4a8265030eac37f906634f5c13c22c3d0ea725f2488e1bca005c6b981653d7  go1.18.1.windows-arm64.zip
+e447b498cde50215c4f7619e5124b0fc4e25fb5d16ea47271c47f278e7aa763a  go1.20.3.src.tar.gz
+c1e1161d6d859deb576e6cfabeb40e3d042ceb1c6f444f617c3c9d76269c3565  go1.20.3.darwin-amd64.tar.gz
+86b0ed0f2b2df50fa8036eea875d1cf2d76cefdacf247c44639a1464b7e36b95  go1.20.3.darwin-arm64.tar.gz
+340e80abd047c597fdc0f50a6cc59617f06c297d62f7fc77f4a0164e2da6f7aa  go1.20.3.freebsd-386.tar.gz
+2169fcd8b6c94c5fbe07c0b470ccfb6001d343f6548ad49f3d9ab78e3b5753c7  go1.20.3.freebsd-amd64.tar.gz
+e12384311403f1389d14cc1c1295bfb4e0dd5ab919403b80da429f671a223507  go1.20.3.linux-386.tar.gz
+979694c2c25c735755bf26f4f45e19e64e4811d661dd07b8c010f7a8e18adfca  go1.20.3.linux-amd64.tar.gz
+eb186529f13f901e7a2c4438a05c2cd90d74706aaa0a888469b2a4a617b6ee54  go1.20.3.linux-arm64.tar.gz
+b421e90469a83671641f81b6e20df6500f033e9523e89cbe7b7223704dd1035c  go1.20.3.linux-armv6l.tar.gz
+943c89aa1624ea544a022b31e3d6e16a037200e436370bdd5fd67f3fa60be282  go1.20.3.linux-ppc64le.tar.gz
+126cf823a5634ef2544b866db107b9d351d3ea70d9e240b0bdcfb46f4dcae54b  go1.20.3.linux-s390x.tar.gz
+37e9146e1f9d681cfcaa6fee6c7b890c44c64bc50228c9588f3c4231346d33bd  go1.20.3.windows-386.zip
+143a2837821c7dbacf7744cbb1a8421c1f48307c6fdfaeffc5f8c2f69e1b7932  go1.20.3.windows-amd64.zip
+158cb159e00bc979f473e0f5b5a561613129c5e51067967b72b8e072e5a4db81  go1.20.3.windows-arm64.zip
 
-03c181fc1bb29ea3e73cbb23399c43b081063833a7cf7554b94e5a98308df53e  golangci-lint-1.45.2-linux-riscv64.deb
-08a50bbbf451ede6d5354179eb3e14a5634e156dfa92cb9a2606f855a637e35b  golangci-lint-1.45.2-linux-ppc64le.rpm
-0d12f6ec1296b5a70e392aa88cd2295cceef266165eb7028e675f455515dd1c9  golangci-lint-1.45.2-linux-armv7.deb
-10f2846e2e50e4ea8ae426ee62dcd2227b23adddd8e991aa3c065927ac948735  golangci-lint-1.45.2-linux-ppc64le.deb
-1463049b744871168095e3e8f687247d6040eeb895955b869889ea151e0603ab  golangci-lint-1.45.2-linux-arm64.tar.gz
-15720f9c4c6f9324af695f081dc189adc7751b255759e78d7b2df1d7e9192533  golangci-lint-1.45.2-linux-amd64.deb
-166d922e4d3cfe3d47786c590154a9c8ea689dff0aa92b73d2f5fc74fc570c29  golangci-lint-1.45.2-linux-arm64.rpm
-1a3754c69f7cc19ab89cbdcc2550da4cf9abb3120383c6b3bd440c1ec22da2e6  golangci-lint-1.45.2-freebsd-386.tar.gz
-1dec0aa46d4f0d241863b573f70129bdf1de9c595cf51172a840a588a4cd9fc5  golangci-lint-1.45.2-windows-amd64.zip
-3198453806517c1ad988229f5e758ef850e671203f46d6905509df5bdf4dc24b  golangci-lint-1.45.2-freebsd-armv7.tar.gz
-46a3cd1749d7b98adc2dc01510ddbe21abe42689c8a53fb0e81662713629f215  golangci-lint-1.45.2-linux-386.deb
-4e28bfb593d464b9e160f2acd5b71993836a183270bf8299b78ad31f7a168c0d  golangci-lint-1.45.2-linux-arm64.deb
-5157a58c8f9ab85c33af2e46f0d7c57a3b1e8953b81d61130e292e09f545cfab  golangci-lint-1.45.2-linux-mips64le.tar.gz
-518cd027644129fbf8ec4f02bd6f9ad7278aae826f92b63c80d4d0819ddde49a  golangci-lint-1.45.2-linux-armv6.rpm
-595ad6c6dade4c064351bc309f411703e457f8ffbb7a1806b3d8ee713333427f  golangci-lint-1.45.2-linux-amd64.tar.gz
-6994d6c80f0730751090986184a3481b4be2e6b6e84416238a2b857910045a4f  golangci-lint-1.45.2-windows-arm64.zip
-6c81652fc340118811b487f713c441fc6f527800bf5fd11b8929d08124efa015  golangci-lint-1.45.2-linux-armv7.tar.gz
-726cb045559b7518bafdd3459de70a0647c087eb1b4634627a4b2e95b1258580  golangci-lint-1.45.2-freebsd-amd64.tar.gz
-77df3774cdfda49b956d4a0e676da9a9b883f496ee37293c530770fef6b1d24e  golangci-lint-1.45.2-linux-mips64.deb
-7a9840f279a7d5d405bb434e101c2290964b3729630ac2add29280b962b7b9a5  golangci-lint-1.45.2-windows-armv6.zip
-7d4bf9a5d80ec467aaaf66e78dbdcab567bbc6ba8151334c714eee58766aae32  golangci-lint-1.45.2-windows-armv7.zip
-7e5f8821d39bb11d273b0841b34355f56bd5a45a2d5179f0d09e614e0efc0482  golangci-lint-1.45.2-linux-s390x.rpm
-828de1bde796b23d8656b17a8885fbd879ef612795d62d1e4618126b419728b5  golangci-lint-1.45.2-linux-mips64.rpm
-879a52107a797678a03c175cc7cf441411a14a01f66dc87f70bdfa304a4129a6  golangci-lint-1.45.2-windows-386.zip
-87b6c7e3a3769f7d9abeb3bb82119b3c91e3c975300f6834fdeef8b2e37c98ff  golangci-lint-1.45.2-linux-amd64.rpm
-8b605c6d686c8af53ecc4ef39544541eeb1644d34cc10f9ffc5087808210c4ff  golangci-lint-1.45.2-linux-s390x.deb
-9427dbf51d0ac6f73a0f992838bd40c817470cc5bf6c8e2e2bea6fac46d7af6e  golangci-lint-1.45.2-linux-ppc64le.tar.gz
-995e509e895ca6a64ffc7395ac884d5961bdec98423cb896b17f345a9b4a19cf  golangci-lint-1.45.2-darwin-amd64.tar.gz
-a3f36278f2ea5516341e9071a2df6e65df272be80230b5406a12b72c6d425bee  golangci-lint-1.45.2-linux-armv7.rpm
-a5e12c50c23e87ac1deffc872f92ae85427b1198604969399805ae47cfe43f08  golangci-lint-1.45.2-linux-riscv64.tar.gz
-aa8fa1be0729dbc2fbc4e01e82027097613eee74bd686ebef20f860b01fff8b3  golangci-lint-1.45.2-freebsd-armv6.tar.gz
-c2b9669decc1b638cf2ee9060571af4e255f6dfcbb225c293e3a7ee4bb2c7217  golangci-lint-1.45.2-darwin-arm64.tar.gz
-dfa8bdaf0387aec1cd5c1aa8857f67b2bbdfc2e42efce540c8fb9bbe3e8af302  golangci-lint-1.45.2-linux-armv6.tar.gz
-eb8b8539dd017eee5c131ea9b875893ab2cebeeca41e8c6624907fb02224d643  golangci-lint-1.45.2-linux-386.rpm
-ed6c7e17a857f30d715c5302fa250d95936936b277024bffea201187a257d7a7  golangci-lint-1.45.2-linux-armv6.deb
-ef4d0154ace4001f01b288baeb118176242efb4fd163e178763e3213b77ef30b  golangci-lint-1.45.2-linux-mips64le.deb
-ef7002a2229f5ff5ba201a715fcf877664ea88decbe58e69d163293913024955  golangci-lint-1.45.2-linux-s390x.tar.gz
-f13ecbd09228632e6bbe91a8324bd675c406eed22eb6d2c1e8192eed9ec4f914  golangci-lint-1.45.2-linux-386.tar.gz
-f4cd9cfb09252f51699407277512263cae8409b665dd764f55a34738d0e89edc  golangci-lint-1.45.2-linux-riscv64.rpm
-fb1945dc59d37c9d14bf0a4aea11ea8651fa0e1d582ea80c4c44d0a536c08893  golangci-lint-1.45.2-linux-mips64.tar.gz
-fe542c22738010f453c735a3c410decfd3784d1bd394b395c298ee298fc4c606  golangci-lint-1.45.2-linux-mips64le.rpm
+fba08acc4027f69f07cef48fbff70b8a7ecdfaa1c2aba9ad3fb31d60d9f5d4bc  golangci-lint-1.51.1-darwin-amd64.tar.gz
+75b8f0ff3a4e68147156be4161a49d4576f1be37a0b506473f8c482140c1e7f2  golangci-lint-1.51.1-darwin-arm64.tar.gz
+e06b3459aaed356e1667580be00b05f41f3b2e29685d12cdee571c23e1edb414  golangci-lint-1.51.1-freebsd-386.tar.gz
+623ce2d0fa4d35cc2e8d69fa7334227ab592380962a13b4d9cdc77cf41db2008  golangci-lint-1.51.1-freebsd-amd64.tar.gz
+131365feb0584cc2736c43192fa673ca50e5b6b765456990cb379ecfb787e568  golangci-lint-1.51.1-freebsd-armv6.tar.gz
+98fb627927cbb654f5bf85dcffc5f646666b2ce96ea0fed977c9fb28abd51532  golangci-lint-1.51.1-freebsd-armv7.tar.gz
+b36a99702fa762c15840261bc0fb41b4b1b16b8b19b8c0941bae98c85bb0f8b8  golangci-lint-1.51.1-linux-386.tar.gz
+17aeb26c76820c22efa0e1838b0ab93e90cfedef43fbfc9a2f33f27eb9e5e070  golangci-lint-1.51.1-linux-amd64.tar.gz
+9744bc34e7b8d82ca788b667bfb7155a39b4be9aef43bf9f10318b1372cea338  golangci-lint-1.51.1-linux-arm64.tar.gz
+0dda8dbeb2ff7455a044ec8e347f2fc6d655d2e99d281b3b95e88167031c673d  golangci-lint-1.51.1-linux-armv6.tar.gz
+0512f311b11d43b8b22989d929f0fe8a2e1e5ebe497f1eb0ff73a0fc3d188fd1  golangci-lint-1.51.1-linux-armv7.tar.gz
+d767108dcf84a8eaa844df3454cb0f75a492f4e7102ecc2b0a3545cfe073a566  golangci-lint-1.51.1-linux-loong64.tar.gz
+3bd56c54daec16585b2668e0dfabb27af2c2b38cc0fdb46923e2521e1634846b  golangci-lint-1.51.1-linux-mips64.tar.gz
+f72f5adfa2219e15d2414c9a2966f86e74556cf17a85c727a7fb7770a16cf814  golangci-lint-1.51.1-linux-mips64le.tar.gz
+e605521dac98096d8737e1997c954f41f1d0d8275b8731f62783d410c23574b9  golangci-lint-1.51.1-linux-ppc64le.tar.gz
+2f683217b814339e74d61ca700922d8407f15addd6d4c5e8b156fbab79f26a87  golangci-lint-1.51.1-linux-riscv64.tar.gz
+d98528292b65971a3594e5880530e7624597dc9806fcfccdfbe39be411713d63  golangci-lint-1.51.1-linux-s390x.tar.gz
+9bb2d0fe9e692ed0aea4f2537e3e6862b2f6768fe2849a84f4a6ad09da9fd971  golangci-lint-1.51.1-netbsd-386.tar.gz
+34cafdcd11ae73ae88d66c33eb8449f5c976fc3e37b44774dbe9c71caa95e592  golangci-lint-1.51.1-netbsd-amd64.tar.gz
+f8b4e1e47ac17caafe8a5f32f975a2b6a7cb14c27c0f73c1fb15c20ca91c2e03  golangci-lint-1.51.1-netbsd-armv6.tar.gz
+c4f58b7e227b9fd41f0e9310dc83f4a4e7d026598e2f6e95b78761081a6d9bd2  golangci-lint-1.51.1-netbsd-armv7.tar.gz
+6710e2f5375dc75521c1a17980a6cbbe6ff76c2f8b852964a8af558899a97cf5  golangci-lint-1.51.1-windows-386.zip
+722d7b87b9cdda0a3835d5030b3fc5385c2eba4c107f63f6391cfb2ac35f051d  golangci-lint-1.51.1-windows-amd64.zip
+eb57f9bcb56646f2e3d6ccaf02ec227815fb05077b2e0b1bf9e755805acdc2b9  golangci-lint-1.51.1-windows-arm64.zip
+bce02f7232723cb727755ee11f168a700a00896a25d37f87c4b173bce55596b4  golangci-lint-1.51.1-windows-armv6.zip
+cf6403f84707ce8c98664736772271bc8874f2e760c2fd0f00cf3e85963507e9  golangci-lint-1.51.1-windows-armv7.zip
+
+# This is the builder on PPA that will build Go itself (inception-y), don't modify!
+d7f0013f82e6d7f862cc6cb5c8cdb48eef5f2e239b35baa97e2f1a7466043767  go1.19.6.src.tar.gz

build/ci.go

@@ -24,24 +24,20 @@ Usage: go run build/ci.go <command> <command flags/arguments>
 
 Available commands are:
 
-   install    [ -arch architecture ] [ -cc compiler ] [ packages... ]                          -- builds packages and executables
-   test       [ -coverage ] [ packages... ]                                                    -- runs the tests
-   lint                                                                                        -- runs certain pre-selected linters
-   archive    [ -arch architecture ] [ -type zip|tar ] [ -signer key-envvar ] [ -signify key-envvar ] [ -upload dest ] -- archives build artifacts
-   importkeys                                                                                  -- imports signing keys from env
-   debsrc     [ -signer key-id ] [ -upload dest ]                                              -- creates a debian source package
-   nsis                                                                                        -- creates a Windows NSIS installer
-   aar        [ -local ] [ -sign key-id ] [-deploy repo] [ -upload dest ]                      -- creates an Android archive
-   xcode      [ -local ] [ -sign key-id ] [-deploy repo] [ -upload dest ]                      -- creates an iOS XCode framework
-   purge      [ -store blobstore ] [ -days threshold ]                                         -- purges old archives from the blobstore
+	install    [ -arch architecture ] [ -cc compiler ] [ packages... ]                          -- builds packages and executables
+	test       [ -coverage ] [ packages... ]                                                    -- runs the tests
+	lint                                                                                        -- runs certain pre-selected linters
+	archive    [ -arch architecture ] [ -type zip|tar ] [ -signer key-envvar ] [ -signify key-envvar ] [ -upload dest ] -- archives build artifacts
+	importkeys                                                                                  -- imports signing keys from env
+	debsrc     [ -signer key-id ] [ -upload dest ]                                              -- creates a debian source package
+	nsis                                                                                        -- creates a Windows NSIS installer
+	purge      [ -store blobstore ] [ -days threshold ]                                         -- purges old archives from the blobstore
 
 For all commands, -n prevents execution of external programs (dry run mode).
-
 */
 package main
 
 import (
-	"bufio"
 	"bytes"
 	"encoding/base64"
 	"flag"
@@ -51,7 +47,6 @@ import (
 	"os/exec"
 	"path"
 	"path/filepath"
-	"regexp"
 	"runtime"
 	"strconv"
 	"strings"
@@ -78,7 +73,6 @@ var (
 		executablePath("bootnode"),
 		executablePath("evm"),
 		executablePath("geth"),
-		executablePath("puppeth"),
 		executablePath("rlpdump"),
 		executablePath("clef"),
 	}
@@ -101,10 +95,6 @@ var (
 			BinaryName:  "geth",
 			Description: "Ethereum CLI client.",
 		},
-		{
-			BinaryName:  "puppeth",
-			Description: "Ethereum private network manager.",
-		},
 		{
 			BinaryName:  "rlpdump",
 			Description: "Developer utility tool that prints RLP structures.",
@@ -130,15 +120,15 @@ var (
 	// Distros for which packages are created.
 	// Note: vivid is unsupported because there is no golang-1.6 package for it.
 	// Note: the following Ubuntu releases have been officially deprecated on Launchpad:
-	//   wily, yakkety, zesty, artful, cosmic, disco, eoan, groovy, hirsuite
+	//   wily, yakkety, zesty, artful, cosmic, disco, eoan, groovy, hirsuite, impish
 	debDistroGoBoots = map[string]string{
 		"trusty":  "golang-1.11", // EOL: 04/2024
 		"xenial":  "golang-go",   // EOL: 04/2026
 		"bionic":  "golang-go",   // EOL: 04/2028
 		"focal":   "golang-go",   // EOL: 04/2030
-		"impish":  "golang-go",   // EOL: 07/2022
 		"jammy":   "golang-go",   // EOL: 04/2032
-		//"kinetic": "golang-go",   //  EOL: 07/2023
+		"kinetic": "golang-go",   // EOL: 07/2023
+		"lunar":   "golang-go",   // EOL: 01/2024
 	}
 
 	debGoBootPaths = map[string]string{
@@ -146,10 +136,18 @@ var (
 		"golang-go":   "/usr/lib/go",
 	}
 
-	// This is the version of go that will be downloaded by
+	// This is the version of Go that will be downloaded by
 	//
 	//     go run ci.go install -dlgo
-	dlgoVersion = "1.18.1"
+	dlgoVersion = "1.20.3"
+
+	// This is the version of Go that will be used to bootstrap the PPA builder.
+	//
+	// This version is fine to be old and full of security holes, we just use it
+	// to build the latest Go. Don't change it. If it ever becomes insufficient,
+	// we need to switch over to a recursive builder to jumpt across supported
+	// versions.
+	gobootVersion = "1.19.6"
 )
 
 var GOBIN, _ = filepath.Abs(filepath.Join("build", "bin"))
@@ -185,10 +183,6 @@ func main() {
 		doDebianSource(os.Args[2:])
 	case "nsis":
 		doWindowsInstaller(os.Args[2:])
-	case "aar":
-		doAndroidArchive(os.Args[2:])
-	case "xcode":
-		doXCodeFramework(os.Args[2:])
 	case "purge":
 		doPurge(os.Args[2:])
 	default:
@@ -200,9 +194,10 @@ func main() {
 
 func doInstall(cmdline []string) {
 	var (
-		dlgo = flag.Bool("dlgo", false, "Download Go and build with it")
-		arch = flag.String("arch", "", "Architecture to cross build for")
-		cc   = flag.String("cc", "", "C compiler to cross build with")
+		dlgo       = flag.Bool("dlgo", false, "Download Go and build with it")
+		arch       = flag.String("arch", "", "Architecture to cross build for")
+		cc         = flag.String("cc", "", "C compiler to cross build with")
+		staticlink = flag.Bool("static", false, "Create statically-linked executable")
 	)
 	flag.CommandLine.Parse(cmdline)
 
@@ -213,9 +208,12 @@ func doInstall(cmdline []string) {
 		tc.Root = build.DownloadGo(csdb, dlgoVersion)
 	}
 
+	// Disable CLI markdown doc generation in release builds.
+	buildTags := []string{"urfave_cli_no_docs"}
+
 	// Configure the build.
 	env := build.Env()
-	gobuild := tc.Go("build", buildFlags(env)...)
+	gobuild := tc.Go("build", buildFlags(env, *staticlink, buildTags)...)
 
 	// arm64 CI builders are memory-constrained and can't handle concurrent builds,
 	// better disable it. This check isn't the best, it should probably
@@ -248,25 +246,35 @@ func doInstall(cmdline []string) {
 }
 
 // buildFlags returns the go tool flags for building.
-func buildFlags(env build.Environment) (flags []string) {
+func buildFlags(env build.Environment, staticLinking bool, buildTags []string) (flags []string) {
 	var ld []string
 	if env.Commit != "" {
-		ld = append(ld, "-X", "main.gitCommit="+env.Commit)
-		ld = append(ld, "-X", "main.gitDate="+env.Date)
+		ld = append(ld, "-X", "github.com/ethereum/go-ethereum/internal/version.gitCommit="+env.Commit)
+		ld = append(ld, "-X", "github.com/ethereum/go-ethereum/internal/version.gitDate="+env.Date)
 	}
 	// Strip DWARF on darwin. This used to be required for certain things,
 	// and there is no downside to this, so we just keep doing it.
 	if runtime.GOOS == "darwin" {
 		ld = append(ld, "-s")
 	}
-	// Enforce the stacksize to 8M, which is the case on most platforms apart from
-	// alpine Linux.
 	if runtime.GOOS == "linux" {
-		ld = append(ld, "-extldflags", "-Wl,-z,stack-size=0x800000")
+		// Enforce the stacksize to 8M, which is the case on most platforms apart from
+		// alpine Linux.
+		extld := []string{"-Wl,-z,stack-size=0x800000"}
+		if staticLinking {
+			extld = append(extld, "-static")
+			// Under static linking, use of certain glibc features must be
+			// disabled to avoid shared library dependencies.
+			buildTags = append(buildTags, "osusergo", "netgo")
+		}
+		ld = append(ld, "-extldflags", "'"+strings.Join(extld, " ")+"'")
 	}
 	if len(ld) > 0 {
 		flags = append(flags, "-ldflags", strings.Join(ld, " "))
 	}
+	if len(buildTags) > 0 {
+		flags = append(flags, "-tags", strings.Join(buildTags, ","))
+	}
 	return flags
 }
 
@@ -333,7 +341,7 @@ func doLint(cmdline []string) {
 
 // downloadLinter downloads and unpacks golangci-lint.
 func downloadLinter(cachedir string) string {
-	const version = "1.45.2"
+	const version = "1.51.1"
 
 	csdb := build.MustLoadChecksums("build/checksums.txt")
 	arch := runtime.GOARCH
@@ -457,10 +465,6 @@ func maybeSkipArchive(env build.Environment) {
 		log.Printf("skipping archive creation because this is a PR build")
 		os.Exit(0)
 	}
-	if env.IsCronJob {
-		log.Printf("skipping archive creation because this is a cron job")
-		os.Exit(0)
-	}
 	if env.Branch != "master" && !strings.HasPrefix(env.Tag, "v1.") {
 		log.Printf("skipping archive creation because branch %q, tag %q is not on the inclusion list", env.Branch, env.Tag)
 		os.Exit(0)
@@ -594,7 +598,7 @@ func doDocker(cmdline []string) {
 			}
 			if mismatch {
 				// Build numbers mismatching, retry in a short time to
-				// avoid concurrent failes in both publisher images. If
+				// avoid concurrent fails in both publisher images. If
 				// however the retry failed too, it means the concurrent
 				// builder is still crunching, let that do the publish.
 				if i == 0 {
@@ -655,10 +659,11 @@ func doDebianSource(cmdline []string) {
 		gpg.Stdin = bytes.NewReader(key)
 		build.MustRun(gpg)
 	}
-
-	// Download and verify the Go source package.
-	gobundle := downloadGoSources(*cachedir)
-
+	// Download and verify the Go source packages.
+	var (
+		gobootbundle = downloadGoBootstrapSources(*cachedir)
+		gobundle     = downloadGoSources(*cachedir)
+	)
 	// Download all the dependencies needed to build the sources and run the ci script
 	srcdepfetch := tc.Go("mod", "download")
 	srcdepfetch.Env = append(srcdepfetch.Env, "GOPATH="+filepath.Join(*workdir, "modgopath"))
@@ -675,12 +680,19 @@ func doDebianSource(cmdline []string) {
 			meta := newDebMetadata(distro, goboot, *signer, env, now, pkg.Name, pkg.Version, pkg.Executables)
 			pkgdir := stageDebianSource(*workdir, meta)
 
-			// Add Go source code
+			// Add bootstrapper Go source code
+			if err := build.ExtractArchive(gobootbundle, pkgdir); err != nil {
+				log.Fatalf("Failed to extract bootstrapper Go sources: %v", err)
+			}
+			if err := os.Rename(filepath.Join(pkgdir, "go"), filepath.Join(pkgdir, ".goboot")); err != nil {
+				log.Fatalf("Failed to rename bootstrapper Go source folder: %v", err)
+			}
+			// Add builder Go source code
 			if err := build.ExtractArchive(gobundle, pkgdir); err != nil {
-				log.Fatalf("Failed to extract Go sources: %v", err)
+				log.Fatalf("Failed to extract builder Go sources: %v", err)
 			}
 			if err := os.Rename(filepath.Join(pkgdir, "go"), filepath.Join(pkgdir, ".go")); err != nil {
-				log.Fatalf("Failed to rename Go source folder: %v", err)
+				log.Fatalf("Failed to rename builder Go source folder: %v", err)
 			}
 			// Add all dependency modules in compressed form
 			os.MkdirAll(filepath.Join(pkgdir, ".mod", "cache"), 0755)
@@ -709,6 +721,19 @@ func doDebianSource(cmdline []string) {
 	}
 }
 
+// downloadGoBootstrapSources downloads the Go source tarball that will be used
+// to bootstrap the builder Go.
+func downloadGoBootstrapSources(cachedir string) string {
+	csdb := build.MustLoadChecksums("build/checksums.txt")
+	file := fmt.Sprintf("go%s.src.tar.gz", gobootVersion)
+	url := "https://dl.google.com/go/" + file
+	dst := filepath.Join(cachedir, file)
+	if err := csdb.DownloadFile(url, dst); err != nil {
+		log.Fatal(err)
+	}
+	return dst
+}
+
 // downloadGoSources downloads the Go source tarball.
 func downloadGoSources(cachedir string) string {
 	csdb := build.MustLoadChecksums("build/checksums.txt")
@@ -968,7 +993,10 @@ func doWindowsInstaller(cmdline []string) {
 	if env.Commit != "" {
 		version[2] += "-" + env.Commit[:8]
 	}
-	installer, _ := filepath.Abs("geth-" + archiveBasename(*arch, params.ArchiveVersion(env.Commit)) + ".exe")
+	installer, err := filepath.Abs("geth-" + archiveBasename(*arch, params.ArchiveVersion(env.Commit)) + ".exe")
+	if err != nil {
+		log.Fatalf("Failed to convert installer file path: %v", err)
+	}
 	build.MustRunCommand("makensis.exe",
 		"/DOUTPUTFILE="+installer,
 		"/DMAJORVERSION="+version[0],
@@ -983,236 +1011,6 @@ func doWindowsInstaller(cmdline []string) {
 	}
 }
 
-// Android archives
-
-func doAndroidArchive(cmdline []string) {
-	var (
-		local   = flag.Bool("local", false, `Flag whether we're only doing a local build (skip Maven artifacts)`)
-		signer  = flag.String("signer", "", `Environment variable holding the signing key (e.g. ANDROID_SIGNING_KEY)`)
-		signify = flag.String("signify", "", `Environment variable holding the signify signing key (e.g. ANDROID_SIGNIFY_KEY)`)
-		deploy  = flag.String("deploy", "", `Destination to deploy the archive (usually "https://oss.sonatype.org")`)
-		upload  = flag.String("upload", "", `Destination to upload the archive (usually "gethstore/builds")`)
-	)
-	flag.CommandLine.Parse(cmdline)
-	env := build.Env()
-	tc := new(build.GoToolchain)
-
-	// Sanity check that the SDK and NDK are installed and set
-	if os.Getenv("ANDROID_HOME") == "" {
-		log.Fatal("Please ensure ANDROID_HOME points to your Android SDK")
-	}
-
-	// Build gomobile.
-	install := tc.Install(GOBIN, "golang.org/x/mobile/cmd/gomobile@latest", "golang.org/x/mobile/cmd/gobind@latest")
-	install.Env = append(install.Env)
-	build.MustRun(install)
-
-	// Ensure all dependencies are available. This is required to make
-	// gomobile bind work because it expects go.sum to contain all checksums.
-	build.MustRun(tc.Go("mod", "download"))
-
-	// Build the Android archive and Maven resources
-	build.MustRun(gomobileTool("bind", "-ldflags", "-s -w", "--target", "android", "--javapkg", "org.ethereum", "-v", "github.com/ethereum/go-ethereum/mobile"))
-
-	if *local {
-		// If we're building locally, copy bundle to build dir and skip Maven
-		os.Rename("geth.aar", filepath.Join(GOBIN, "geth.aar"))
-		os.Rename("geth-sources.jar", filepath.Join(GOBIN, "geth-sources.jar"))
-		return
-	}
-	meta := newMavenMetadata(env)
-	build.Render("build/mvn.pom", meta.Package+".pom", 0755, meta)
-
-	// Skip Maven deploy and Azure upload for PR builds
-	maybeSkipArchive(env)
-
-	// Sign and upload the archive to Azure
-	archive := "geth-" + archiveBasename("android", params.ArchiveVersion(env.Commit)) + ".aar"
-	os.Rename("geth.aar", archive)
-
-	if err := archiveUpload(archive, *upload, *signer, *signify); err != nil {
-		log.Fatal(err)
-	}
-	// Sign and upload all the artifacts to Maven Central
-	os.Rename(archive, meta.Package+".aar")
-	if *signer != "" && *deploy != "" {
-		// Import the signing key into the local GPG instance
-		key := getenvBase64(*signer)
-		gpg := exec.Command("gpg", "--import")
-		gpg.Stdin = bytes.NewReader(key)
-		build.MustRun(gpg)
-		keyID, err := build.PGPKeyID(string(key))
-		if err != nil {
-			log.Fatal(err)
-		}
-		// Upload the artifacts to Sonatype and/or Maven Central
-		repo := *deploy + "/service/local/staging/deploy/maven2"
-		if meta.Develop {
-			repo = *deploy + "/content/repositories/snapshots"
-		}
-		build.MustRunCommand("mvn", "gpg:sign-and-deploy-file", "-e", "-X",
-			"-settings=build/mvn.settings", "-Durl="+repo, "-DrepositoryId=ossrh",
-			"-Dgpg.keyname="+keyID,
-			"-DpomFile="+meta.Package+".pom", "-Dfile="+meta.Package+".aar")
-	}
-}
-
-func gomobileTool(subcmd string, args ...string) *exec.Cmd {
-	cmd := exec.Command(filepath.Join(GOBIN, "gomobile"), subcmd)
-	cmd.Args = append(cmd.Args, args...)
-	cmd.Env = []string{
-		"PATH=" + GOBIN + string(os.PathListSeparator) + os.Getenv("PATH"),
-	}
-	for _, e := range os.Environ() {
-		if strings.HasPrefix(e, "GOPATH=") || strings.HasPrefix(e, "PATH=") || strings.HasPrefix(e, "GOBIN=") {
-			continue
-		}
-		cmd.Env = append(cmd.Env, e)
-	}
-	cmd.Env = append(cmd.Env, "GOBIN="+GOBIN)
-	return cmd
-}
-
-type mavenMetadata struct {
-	Version      string
-	Package      string
-	Develop      bool
-	Contributors []mavenContributor
-}
-
-type mavenContributor struct {
-	Name  string
-	Email string
-}
-
-func newMavenMetadata(env build.Environment) mavenMetadata {
-	// Collect the list of authors from the repo root
-	contribs := []mavenContributor{}
-	if authors, err := os.Open("AUTHORS"); err == nil {
-		defer authors.Close()
-
-		scanner := bufio.NewScanner(authors)
-		for scanner.Scan() {
-			// Skip any whitespace from the authors list
-			line := strings.TrimSpace(scanner.Text())
-			if line == "" || line[0] == '#' {
-				continue
-			}
-			// Split the author and insert as a contributor
-			re := regexp.MustCompile("([^<]+) <(.+)>")
-			parts := re.FindStringSubmatch(line)
-			if len(parts) == 3 {
-				contribs = append(contribs, mavenContributor{Name: parts[1], Email: parts[2]})
-			}
-		}
-	}
-	// Render the version and package strings
-	version := params.Version
-	if isUnstableBuild(env) {
-		version += "-SNAPSHOT"
-	}
-	return mavenMetadata{
-		Version:      version,
-		Package:      "geth-" + version,
-		Develop:      isUnstableBuild(env),
-		Contributors: contribs,
-	}
-}
-
-// XCode frameworks
-
-func doXCodeFramework(cmdline []string) {
-	var (
-		local   = flag.Bool("local", false, `Flag whether we're only doing a local build (skip Maven artifacts)`)
-		signer  = flag.String("signer", "", `Environment variable holding the signing key (e.g. IOS_SIGNING_KEY)`)
-		signify = flag.String("signify", "", `Environment variable holding the signify signing key (e.g. IOS_SIGNIFY_KEY)`)
-		deploy  = flag.String("deploy", "", `Destination to deploy the archive (usually "trunk")`)
-		upload  = flag.String("upload", "", `Destination to upload the archives (usually "gethstore/builds")`)
-	)
-	flag.CommandLine.Parse(cmdline)
-	env := build.Env()
-	tc := new(build.GoToolchain)
-
-	// Build gomobile.
-	build.MustRun(tc.Install(GOBIN, "golang.org/x/mobile/cmd/gomobile", "golang.org/x/mobile/cmd/gobind"))
-
-	// Build the iOS XCode framework
-	bind := gomobileTool("bind", "-ldflags", "-s -w", "--target", "ios", "-v", "github.com/ethereum/go-ethereum/mobile")
-
-	if *local {
-		// If we're building locally, use the build folder and stop afterwards
-		bind.Dir = GOBIN
-		build.MustRun(bind)
-		return
-	}
-
-	// Create the archive.
-	maybeSkipArchive(env)
-	archive := "geth-" + archiveBasename("ios", params.ArchiveVersion(env.Commit))
-	if err := os.MkdirAll(archive, 0755); err != nil {
-		log.Fatal(err)
-	}
-	bind.Dir, _ = filepath.Abs(archive)
-	build.MustRun(bind)
-	build.MustRunCommand("tar", "-zcvf", archive+".tar.gz", archive)
-
-	// Sign and upload the framework to Azure
-	if err := archiveUpload(archive+".tar.gz", *upload, *signer, *signify); err != nil {
-		log.Fatal(err)
-	}
-	// Prepare and upload a PodSpec to CocoaPods
-	if *deploy != "" {
-		meta := newPodMetadata(env, archive)
-		build.Render("build/pod.podspec", "Geth.podspec", 0755, meta)
-		build.MustRunCommand("pod", *deploy, "push", "Geth.podspec", "--allow-warnings")
-	}
-}
-
-type podMetadata struct {
-	Version      string
-	Commit       string
-	Archive      string
-	Contributors []podContributor
-}
-
-type podContributor struct {
-	Name  string
-	Email string
-}
-
-func newPodMetadata(env build.Environment, archive string) podMetadata {
-	// Collect the list of authors from the repo root
-	contribs := []podContributor{}
-	if authors, err := os.Open("AUTHORS"); err == nil {
-		defer authors.Close()
-
-		scanner := bufio.NewScanner(authors)
-		for scanner.Scan() {
-			// Skip any whitespace from the authors list
-			line := strings.TrimSpace(scanner.Text())
-			if line == "" || line[0] == '#' {
-				continue
-			}
-			// Split the author and insert as a contributor
-			re := regexp.MustCompile("([^<]+) <(.+)>")
-			parts := re.FindStringSubmatch(line)
-			if len(parts) == 3 {
-				contribs = append(contribs, podContributor{Name: parts[1], Email: parts[2]})
-			}
-		}
-	}
-	version := params.Version
-	if isUnstableBuild(env) {
-		version += "-unstable." + env.Buildnum
-	}
-	return podMetadata{
-		Archive:      archive,
-		Version:      version,
-		Commit:       env.Commit,
-		Contributors: contribs,
-	}
-}
-
 // Binary distribution cleanups
 
 func doPurge(cmdline []string) {

build/deb/ethereum/completions/bash/geth

@@ -0,0 +1,16 @@
+_geth_bash_autocomplete() {
+  if [[ "${COMP_WORDS[0]}" != "source" ]]; then
+    local cur opts base
+    COMPREPLY=()
+    cur="${COMP_WORDS[COMP_CWORD]}"
+    if [[ "$cur" == "-"* ]]; then
+      opts=$( ${COMP_WORDS[@]:0:$COMP_CWORD} ${cur} --generate-bash-completion )
+    else
+      opts=$( ${COMP_WORDS[@]:0:$COMP_CWORD} --generate-bash-completion )
+    fi
+    COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) )
+    return 0
+  fi
+}
+
+complete -o bashdefault -o default -o nospace -F _geth_bash_autocomplete geth

build/deb/ethereum/completions/zsh/_geth

@@ -0,0 +1,18 @@
+_geth_zsh_autocomplete() {
+  local -a opts
+  local cur
+  cur=${words[-1]}
+  if [[ "$cur" == "-"* ]]; then
+    opts=("${(@f)$(${words[@]:0:#words[@]-1} ${cur} --generate-bash-completion)}")
+  else
+    opts=("${(@f)$(${words[@]:0:#words[@]-1} --generate-bash-completion)}")
+  fi
+
+  if [[ "${opts[1]}" != "" ]]; then
+    _describe 'values' opts
+  else
+    _files
+  fi
+}
+
+compdef _geth_zsh_autocomplete geth

build/deb/ethereum/deb.install

@@ -1 +1,5 @@
 build/bin/{{.BinaryName}} usr/bin
+{{- if eq .BinaryName "geth" }}
+build/deb/ethereum/completions/bash/geth etc/bash_completion.d
+build/deb/ethereum/completions/zsh/_geth usr/share/zsh/vendor-completions
+{{end -}}

build/deb/ethereum/deb.rules

@@ -16,7 +16,11 @@ override_dh_auto_build:
 	# We can't download a fresh Go within Launchpad, so we're shipping and building
 	# one on the fly. However, we can't build it inside the go-ethereum folder as
 	# bootstrapping clashes with go modules, so build in a sibling folder.
-	(mv .go ../ && cd ../.go/src && ./make.bash)
+	#
+	# We're also shipping the bootstrapper as of Go 1.20 as it had minimum version
+	# requirements opposed to older versions of Go.
+	(mv .goboot ../ && cd ../.goboot/src && ./make.bash)
+	(mv .go ../ && cd ../.go/src && GOROOT_BOOTSTRAP=`pwd`/../../.goboot ./make.bash)
 
 	# We can't download external go modules within Launchpad, so we're shipping the
 	# entire dependency source cache with go-ethereum.

build/mvn.pom

@@ -1,57 +0,0 @@
-<project xmlns="http://maven.apache.org/POM/4.0.0"
-  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
-                      http://maven.apache.org/xsd/maven-4.0.0.xsd">
-  <modelVersion>4.0.0</modelVersion>
-
-  <groupId>org.ethereum</groupId>
-  <artifactId>geth</artifactId>
-  <version>{{.Version}}</version>
-  <packaging>aar</packaging>
-
-  <name>Android Ethereum Client</name>
-  <description>Android port of the go-ethereum libraries and node</description>
-  <url>https://github.com/ethereum/go-ethereum</url>
-  <inceptionYear>2015</inceptionYear>
-
-  <licenses>
-    <license>
-      <name>GNU Lesser General Public License, Version 3.0</name>
-      <url>https://www.gnu.org/licenses/lgpl-3.0.en.html</url>
-      <distribution>repo</distribution>
-    </license>
-  </licenses>
-
-  <organization>
-    <name>Ethereum</name>
-    <url>https://ethereum.org</url>
-  </organization>
-
-  <developers>
-    <developer>
-      <id>karalabe</id>
-      <name>Péter Szilágyi</name>
-      <email>peterke@gmail.com</email>
-      <url>https://github.com/karalabe</url>
-      <properties>
-        <picUrl>https://www.gravatar.com/avatar/2ecbf0f5b4b79eebf8c193e5d324357f?s=256</picUrl>
-      </properties>
-    </developer>
-  </developers>
-
-  <contributors>{{range .Contributors}}
-    <contributor>
-      <name>{{.Name}}</name>
-      <email>{{.Email}}</email>
-    </contributor>{{end}}
-  </contributors>
-
-  <issueManagement>
-    <system>GitHub Issues</system>
-    <url>https://github.com/ethereum/go-ethereum/issues/</url>
-  </issueManagement>
-
-  <scm>
-    <url>https://github.com/ethereum/go-ethereum</url>
-  </scm>
-</project>

build/mvn.settings

@@ -1,24 +0,0 @@
-<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
-  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-  xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0
-                      http://maven.apache.org/xsd/settings-1.0.0.xsd">
-  <servers>
-    <server>
-      <id>ossrh</id>
-      <username>${env.ANDROID_SONATYPE_USERNAME}</username>
-      <password>${env.ANDROID_SONATYPE_PASSWORD}</password>
-    </server>
-  </servers>
-  <profiles>
-    <profile>
-      <id>ossrh</id>
-      <activation>
-        <activeByDefault>true</activeByDefault>
-      </activation>
-      <properties>
-        <gpg.executable>gpg</gpg.executable>
-        <gpg.passphrase></gpg.passphrase>
-      </properties>
-    </profile>
-  </profiles>
-</settings>

build/pod.podspec

@@ -1,22 +0,0 @@
-Pod::Spec.new do |spec|
-  spec.name         = 'Geth'
-  spec.version      = '{{.Version}}'
-  spec.license      = { :type => 'GNU Lesser General Public License, Version 3.0' }
-  spec.homepage     = 'https://github.com/ethereum/go-ethereum'
-  spec.authors      = { {{range .Contributors}}
-		'{{.Name}}' => '{{.Email}}',{{end}}
-	}
-  spec.summary      = 'iOS Ethereum Client'
-  spec.source       = { :git => 'https://github.com/ethereum/go-ethereum.git', :commit => '{{.Commit}}' }
-
-	spec.platform = :ios
-  spec.ios.deployment_target  = '9.0'
-	spec.ios.vendored_frameworks = 'Frameworks/Geth.framework'
-
-	spec.prepare_command = <<-CMD
-    curl https://gethstore.blob.core.windows.net/builds/{{.Archive}}.tar.gz | tar -xvz
-    mkdir Frameworks
-    mv {{.Archive}}/Geth.framework Frameworks
-    rm -rf {{.Archive}}
-  CMD
-end

build/tools/tools.go

@@ -24,8 +24,4 @@ import (
 	_ "github.com/fjl/gencodec"
 	_ "github.com/golang/protobuf/protoc-gen-go"
 	_ "golang.org/x/tools/cmd/stringer"
-
-	// Tool imports for mobile build.
-	_ "golang.org/x/mobile/cmd/gobind"
-	_ "golang.org/x/mobile/cmd/gomobile"
 )

build/update-license.go

@@ -342,7 +342,10 @@ func isGenerated(file string) bool {
 	}
 	defer fd.Close()
 	buf := make([]byte, 2048)
-	n, _ := fd.Read(buf)
+	n, err := fd.Read(buf)
+	if err != nil {
+		return false
+	}
 	buf = buf[:n]
 	for _, l := range bytes.Split(buf, []byte("\n")) {
 		if bytes.HasPrefix(l, []byte("// Code generated")) {

cmd/abigen/main.go

@@ -30,58 +30,54 @@ import (
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/internal/flags"
 	"github.com/ethereum/go-ethereum/log"
-	"gopkg.in/urfave/cli.v1"
+	"github.com/urfave/cli/v2"
 )
 
 var (
-	// Git SHA1 commit hash of the release (set via linker flags)
-	gitCommit = ""
-	gitDate   = ""
-
-	app *cli.App
-
 	// Flags needed by abigen
-	abiFlag = cli.StringFlag{
+	abiFlag = &cli.StringFlag{
 		Name:  "abi",
 		Usage: "Path to the Ethereum contract ABI json to bind, - for STDIN",
 	}
-	binFlag = cli.StringFlag{
+	binFlag = &cli.StringFlag{
 		Name:  "bin",
 		Usage: "Path to the Ethereum contract bytecode (generate deploy method)",
 	}
-	typeFlag = cli.StringFlag{
+	typeFlag = &cli.StringFlag{
 		Name:  "type",
 		Usage: "Struct name for the binding (default = package name)",
 	}
-	jsonFlag = cli.StringFlag{
+	jsonFlag = &cli.StringFlag{
 		Name:  "combined-json",
 		Usage: "Path to the combined-json file generated by compiler, - for STDIN",
 	}
-	excFlag = cli.StringFlag{
+	excFlag = &cli.StringFlag{
 		Name:  "exc",
 		Usage: "Comma separated types to exclude from binding",
 	}
-	pkgFlag = cli.StringFlag{
+	pkgFlag = &cli.StringFlag{
 		Name:  "pkg",
 		Usage: "Package name to generate the binding into",
 	}
-	outFlag = cli.StringFlag{
+	outFlag = &cli.StringFlag{
 		Name:  "out",
 		Usage: "Output file for the generated binding (default = stdout)",
 	}
-	langFlag = cli.StringFlag{
+	langFlag = &cli.StringFlag{
 		Name:  "lang",
-		Usage: "Destination language for the bindings (go, java, objc)",
+		Usage: "Destination language for the bindings (go)",
 		Value: "go",
 	}
-	aliasFlag = cli.StringFlag{
+	aliasFlag = &cli.StringFlag{
 		Name:  "alias",
 		Usage: "Comma separated aliases for function and event renaming, e.g. original1=alias1, original2=alias2",
 	}
 )
 
+var app = flags.NewApp("Ethereum ABI wrapper code generator")
+
 func init() {
-	app = flags.NewApp(gitCommit, gitDate, "ethereum checkpoint helper tool")
+	app.Name = "abigen"
 	app.Flags = []cli.Flag{
 		abiFlag,
 		binFlag,
@@ -93,26 +89,21 @@ func init() {
 		langFlag,
 		aliasFlag,
 	}
-	app.Action = utils.MigrateFlags(abigen)
-	cli.CommandHelpTemplate = flags.OriginCommandHelpTemplate
+	app.Action = abigen
 }
 
 func abigen(c *cli.Context) error {
 	utils.CheckExclusive(c, abiFlag, jsonFlag) // Only one source can be selected.
-	if c.GlobalString(pkgFlag.Name) == "" {
+
+	if c.String(pkgFlag.Name) == "" {
 		utils.Fatalf("No destination package specified (--pkg)")
 	}
 	var lang bind.Lang
-	switch c.GlobalString(langFlag.Name) {
+	switch c.String(langFlag.Name) {
 	case "go":
 		lang = bind.LangGo
-	case "java":
-		lang = bind.LangJava
-	case "objc":
-		lang = bind.LangObjC
-		utils.Fatalf("Objc binding generation is uncompleted")
 	default:
-		utils.Fatalf("Unsupported destination language \"%s\" (--lang)", c.GlobalString(langFlag.Name))
+		utils.Fatalf("Unsupported destination language \"%s\" (--lang)", c.String(langFlag.Name))
 	}
 	// If the entire solidity code was specified, build and bind based on that
 	var (
@@ -123,13 +114,13 @@ func abigen(c *cli.Context) error {
 		libs    = make(map[string]string)
 		aliases = make(map[string]string)
 	)
-	if c.GlobalString(abiFlag.Name) != "" {
+	if c.String(abiFlag.Name) != "" {
 		// Load up the ABI, optional bytecode and type name from the parameters
 		var (
 			abi []byte
 			err error
 		)
-		input := c.GlobalString(abiFlag.Name)
+		input := c.String(abiFlag.Name)
 		if input == "-" {
 			abi, err = io.ReadAll(os.Stdin)
 		} else {
@@ -141,7 +132,7 @@ func abigen(c *cli.Context) error {
 		abis = append(abis, string(abi))
 
 		var bin []byte
-		if binFile := c.GlobalString(binFlag.Name); binFile != "" {
+		if binFile := c.String(binFlag.Name); binFile != "" {
 			if bin, err = os.ReadFile(binFile); err != nil {
 				utils.Fatalf("Failed to read input bytecode: %v", err)
 			}
@@ -151,22 +142,25 @@ func abigen(c *cli.Context) error {
 		}
 		bins = append(bins, string(bin))
 
-		kind := c.GlobalString(typeFlag.Name)
+		kind := c.String(typeFlag.Name)
 		if kind == "" {
-			kind = c.GlobalString(pkgFlag.Name)
+			kind = c.String(pkgFlag.Name)
 		}
 		types = append(types, kind)
 	} else {
 		// Generate the list of types to exclude from binding
-		exclude := make(map[string]bool)
-		for _, kind := range strings.Split(c.GlobalString(excFlag.Name), ",") {
-			exclude[strings.ToLower(kind)] = true
+		var exclude *nameFilter
+		if c.IsSet(excFlag.Name) {
+			var err error
+			if exclude, err = newNameFilter(strings.Split(c.String(excFlag.Name), ",")...); err != nil {
+				utils.Fatalf("Failed to parse excludes: %v", err)
+			}
 		}
 		var contracts map[string]*compiler.Contract
 
-		if c.GlobalIsSet(jsonFlag.Name) {
+		if c.IsSet(jsonFlag.Name) {
 			var (
-				input      = c.GlobalString(jsonFlag.Name)
+				input      = c.String(jsonFlag.Name)
 				jsonOutput []byte
 				err        error
 			)
@@ -185,7 +179,11 @@ func abigen(c *cli.Context) error {
 		}
 		// Gather all non-excluded contract for binding
 		for name, contract := range contracts {
-			if exclude[strings.ToLower(name)] {
+			// fully qualified name is of the form <solFilePath>:<type>
+			nameParts := strings.Split(name, ":")
+			typeName := nameParts[len(nameParts)-1]
+			if exclude != nil && exclude.Matches(name) {
+				fmt.Fprintf(os.Stderr, "excluding: %v\n", name)
 				continue
 			}
 			abi, err := json.Marshal(contract.Info.AbiDefinition) // Flatten the compiler parse
@@ -195,40 +193,39 @@ func abigen(c *cli.Context) error {
 			abis = append(abis, string(abi))
 			bins = append(bins, contract.Code)
 			sigs = append(sigs, contract.Hashes)
-			nameParts := strings.Split(name, ":")
-			types = append(types, nameParts[len(nameParts)-1])
+			types = append(types, typeName)
 
 			// Derive the library placeholder which is a 34 character prefix of the
 			// hex encoding of the keccak256 hash of the fully qualified library name.
 			// Note that the fully qualified library name is the path of its source
 			// file and the library name separated by ":".
 			libPattern := crypto.Keccak256Hash([]byte(name)).String()[2:36] // the first 2 chars are 0x
-			libs[libPattern] = nameParts[len(nameParts)-1]
+			libs[libPattern] = typeName
 		}
 	}
 	// Extract all aliases from the flags
-	if c.GlobalIsSet(aliasFlag.Name) {
+	if c.IsSet(aliasFlag.Name) {
 		// We support multi-versions for aliasing
 		// e.g.
 		//      foo=bar,foo2=bar2
 		//      foo:bar,foo2:bar2
 		re := regexp.MustCompile(`(?:(\w+)[:=](\w+))`)
-		submatches := re.FindAllStringSubmatch(c.GlobalString(aliasFlag.Name), -1)
+		submatches := re.FindAllStringSubmatch(c.String(aliasFlag.Name), -1)
 		for _, match := range submatches {
 			aliases[match[1]] = match[2]
 		}
 	}
 	// Generate the contract binding
-	code, err := bind.Bind(types, abis, bins, sigs, c.GlobalString(pkgFlag.Name), lang, libs, aliases)
+	code, err := bind.Bind(types, abis, bins, sigs, c.String(pkgFlag.Name), lang, libs, aliases)
 	if err != nil {
 		utils.Fatalf("Failed to generate ABI binding: %v", err)
 	}
 	// Either flush it out to a file or display on the standard output
-	if !c.GlobalIsSet(outFlag.Name) {
+	if !c.IsSet(outFlag.Name) {
 		fmt.Printf("%s\n", code)
 		return nil
 	}
-	if err := os.WriteFile(c.GlobalString(outFlag.Name), []byte(code), 0600); err != nil {
+	if err := os.WriteFile(c.String(outFlag.Name), []byte(code), 0600); err != nil {
 		utils.Fatalf("Failed to write ABI binding: %v", err)
 	}
 	return nil

cmd/abigen/namefilter.go

@@ -0,0 +1,58 @@
+package main
+
+import (
+	"fmt"
+	"strings"
+)
+
+type nameFilter struct {
+	fulls map[string]bool // path/to/contract.sol:Type
+	files map[string]bool // path/to/contract.sol:*
+	types map[string]bool // *:Type
+}
+
+func newNameFilter(patterns ...string) (*nameFilter, error) {
+	f := &nameFilter{
+		fulls: make(map[string]bool),
+		files: make(map[string]bool),
+		types: make(map[string]bool),
+	}
+	for _, pattern := range patterns {
+		if err := f.add(pattern); err != nil {
+			return nil, err
+		}
+	}
+	return f, nil
+}
+
+func (f *nameFilter) add(pattern string) error {
+	ft := strings.Split(pattern, ":")
+	if len(ft) != 2 {
+		// filenames and types must not include ':' symbol
+		return fmt.Errorf("invalid pattern: %s", pattern)
+	}
+
+	file, typ := ft[0], ft[1]
+	if file == "*" {
+		f.types[typ] = true
+		return nil
+	} else if typ == "*" {
+		f.files[file] = true
+		return nil
+	}
+	f.fulls[pattern] = true
+	return nil
+}
+
+func (f *nameFilter) Matches(name string) bool {
+	ft := strings.Split(name, ":")
+	if len(ft) != 2 {
+		// If contract names are always of the fully-qualified form
+		// <filePath>:<type>, then this case will never happen.
+		return false
+	}
+
+	file, typ := ft[0], ft[1]
+	// full paths > file paths > types
+	return f.fulls[name] || f.files[file] || f.types[typ]
+}

cmd/abigen/namefilter_test.go

@@ -0,0 +1,38 @@
+package main
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestNameFilter(t *testing.T) {
+	_, err := newNameFilter("Foo")
+	require.Error(t, err)
+	_, err = newNameFilter("too/many:colons:Foo")
+	require.Error(t, err)
+
+	f, err := newNameFilter("a/path:A", "*:B", "c/path:*")
+	require.NoError(t, err)
+
+	for _, tt := range []struct {
+		name  string
+		match bool
+	}{
+		{"a/path:A", true},
+		{"unknown/path:A", false},
+		{"a/path:X", false},
+		{"unknown/path:X", false},
+		{"any/path:B", true},
+		{"c/path:X", true},
+		{"c/path:foo:B", false},
+	} {
+		match := f.Matches(tt.name)
+		if tt.match {
+			assert.True(t, match, "expected match")
+		} else {
+			assert.False(t, match, "expected no match")
+		}
+	}
+}

cmd/bootnode/main.go

@@ -40,7 +40,7 @@ func main() {
 		writeAddr   = flag.Bool("writeaddress", false, "write out the node's public key and quit")
 		nodeKeyFile = flag.String("nodekey", "", "private key filename")
 		nodeKeyHex  = flag.String("nodekeyhex", "", "private key as hex (for testing)")
-		natdesc     = flag.String("nat", "none", "port mapping mechanism (any|none|upnp|pmp|extip:<IP>)")
+		natdesc     = flag.String("nat", "none", "port mapping mechanism (any|none|upnp|pmp|pmp:<IP>|extip:<IP>)")
 		netrestrict = flag.String("netrestrict", "", "restrict network communication to the given IP networks (CIDR masks)")
 		runv5       = flag.Bool("v5", false, "run a v5 topic discovery bootnode")
 		verbosity   = flag.Int("verbosity", int(log.LvlInfo), "log verbosity (0-5)")

cmd/checkpoint-admin/README.md

@@ -46,7 +46,7 @@ Deploy checkpoint oracle contract. `--signers` indicates the specified trusted s
 checkpoint-admin deploy --rpc <NODE_RPC_ENDPOINT> --clef <CLEF_ENDPOINT> --signer <SIGNER_TO_SIGN_TX> --signers <TRUSTED_SIGNER_LIST> --threshold 1
 ```
 
-It is worth noting that checkpoint-admin only supports clef as a signer for transactions and plain text(checkpoint). For more clef usage, please see the clef [tutorial](https://geth.ethereum.org/docs/clef/tutorial) .
+It is worth noting that checkpoint-admin only supports clef as a signer for transactions and plain text(checkpoint). For more clef usage, please see the clef [tutorial](https://geth.ethereum.org/docs/tools/clef/tutorial) .
 
 #### Sign
 
@@ -86,7 +86,7 @@ checkpoint-admin status --rpc <NODE_RPC_ENDPOINT>
 
 ### Enable checkpoint oracle in your private network
 
-Currently, only the Ethereum mainnet and the default supported test networks (ropsten, rinkeby, goerli) activate this feature. If you want to activate this feature in your private network, you can overwrite the relevant checkpoint oracle settings through the configuration file after deploying the oracle contract.
+Currently, only the Ethereum mainnet and the default supported test networks (rinkeby, goerli) activate this feature. If you want to activate this feature in your private network, you can overwrite the relevant checkpoint oracle settings through the configuration file after deploying the oracle contract.
 
 * Get your node configuration file `geth dumpconfig OTHER_COMMAND_LINE_OPTIONS > config.toml`
 * Edit the configuration file and add the following information

cmd/checkpoint-admin/common.go

@@ -28,12 +28,12 @@ import (
 	"github.com/ethereum/go-ethereum/ethclient"
 	"github.com/ethereum/go-ethereum/params"
 	"github.com/ethereum/go-ethereum/rpc"
-	"gopkg.in/urfave/cli.v1"
+	"github.com/urfave/cli/v2"
 )
 
 // newClient creates a client with specified remote URL.
 func newClient(ctx *cli.Context) *ethclient.Client {
-	client, err := ethclient.Dial(ctx.GlobalString(nodeURLFlag.Name))
+	client, err := ethclient.Dial(ctx.String(nodeURLFlag.Name))
 	if err != nil {
 		utils.Fatalf("Failed to connect to Ethereum node: %v", err)
 	}
@@ -64,9 +64,9 @@ func getContractAddr(client *rpc.Client) common.Address {
 func getCheckpoint(ctx *cli.Context, client *rpc.Client) *params.TrustedCheckpoint {
 	var checkpoint *params.TrustedCheckpoint
 
-	if ctx.GlobalIsSet(indexFlag.Name) {
+	if ctx.IsSet(indexFlag.Name) {
 		var result [3]string
-		index := uint64(ctx.GlobalInt64(indexFlag.Name))
+		index := uint64(ctx.Int64(indexFlag.Name))
 		if err := client.Call(&result, "les_getCheckpoint", index); err != nil {
 			utils.Fatalf("Failed to get local checkpoint %v, please ensure the les API is exposed", err)
 		}

cmd/checkpoint-admin/exec.go

@@ -36,10 +36,10 @@ import (
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/params"
 	"github.com/ethereum/go-ethereum/rpc"
-	"gopkg.in/urfave/cli.v1"
+	"github.com/urfave/cli/v2"
 )
 
-var commandDeploy = cli.Command{
+var commandDeploy = &cli.Command{
 	Name:  "deploy",
 	Usage: "Deploy a new checkpoint oracle contract",
 	Flags: []cli.Flag{
@@ -49,10 +49,10 @@ var commandDeploy = cli.Command{
 		signersFlag,
 		thresholdFlag,
 	},
-	Action: utils.MigrateFlags(deploy),
+	Action: deploy,
 }
 
-var commandSign = cli.Command{
+var commandSign = &cli.Command{
 	Name:  "sign",
 	Usage: "Sign the checkpoint with the specified key",
 	Flags: []cli.Flag{
@@ -63,10 +63,10 @@ var commandSign = cli.Command{
 		hashFlag,
 		oracleFlag,
 	},
-	Action: utils.MigrateFlags(sign),
+	Action: sign,
 }
 
-var commandPublish = cli.Command{
+var commandPublish = &cli.Command{
 	Name:  "publish",
 	Usage: "Publish a checkpoint into the oracle",
 	Flags: []cli.Flag{
@@ -76,7 +76,7 @@ var commandPublish = cli.Command{
 		indexFlag,
 		signaturesFlag,
 	},
-	Action: utils.MigrateFlags(publish),
+	Action: publish,
 }
 
 // deploy deploys the checkpoint registrar contract.
@@ -132,7 +132,7 @@ func sign(ctx *cli.Context) error {
 		node   *rpc.Client
 		oracle *checkpointoracle.CheckpointOracle
 	)
-	if !ctx.GlobalIsSet(nodeURLFlag.Name) {
+	if !ctx.IsSet(nodeURLFlag.Name) {
 		// Offline mode signing
 		offline = true
 		if !ctx.IsSet(hashFlag.Name) {
@@ -151,7 +151,7 @@ func sign(ctx *cli.Context) error {
 		address = common.HexToAddress(ctx.String(oracleFlag.Name))
 	} else {
 		// Interactive mode signing, retrieve the data from the remote node
-		node = newRPCClient(ctx.GlobalString(nodeURLFlag.Name))
+		node = newRPCClient(ctx.String(nodeURLFlag.Name))
 
 		checkpoint := getCheckpoint(ctx, node)
 		chash, cindex, address = checkpoint.Hash(), checkpoint.SectionIndex, getContractAddr(node)
@@ -265,7 +265,7 @@ func publish(ctx *cli.Context) error {
 	}
 	// Retrieve the checkpoint we want to sign to sort the signatures
 	var (
-		client       = newRPCClient(ctx.GlobalString(nodeURLFlag.Name))
+		client       = newRPCClient(ctx.String(nodeURLFlag.Name))
 		addr, oracle = newContract(client)
 		checkpoint   = getCheckpoint(ctx, client)
 		sighash      = sighash(checkpoint.SectionIndex, addr, checkpoint.Hash())

cmd/checkpoint-admin/main.go

@@ -25,20 +25,13 @@ import (
 	"github.com/ethereum/go-ethereum/common/fdlimit"
 	"github.com/ethereum/go-ethereum/internal/flags"
 	"github.com/ethereum/go-ethereum/log"
-	"gopkg.in/urfave/cli.v1"
+	"github.com/urfave/cli/v2"
 )
 
-var (
-	// Git SHA1 commit hash of the release (set via linker flags)
-	gitCommit = ""
-	gitDate   = ""
-)
-
-var app *cli.App
+var app = flags.NewApp("ethereum checkpoint helper tool")
 
 func init() {
-	app = flags.NewApp(gitCommit, gitDate, "ethereum checkpoint helper tool")
-	app.Commands = []cli.Command{
+	app.Commands = []*cli.Command{
 		commandStatus,
 		commandDeploy,
 		commandSign,
@@ -48,46 +41,45 @@ func init() {
 		oracleFlag,
 		nodeURLFlag,
 	}
-	cli.CommandHelpTemplate = flags.OriginCommandHelpTemplate
 }
 
 // Commonly used command line flags.
 var (
-	indexFlag = cli.Int64Flag{
+	indexFlag = &cli.Int64Flag{
 		Name:  "index",
 		Usage: "Checkpoint index (query latest from remote node if not specified)",
 	}
-	hashFlag = cli.StringFlag{
+	hashFlag = &cli.StringFlag{
 		Name:  "hash",
 		Usage: "Checkpoint hash (query latest from remote node if not specified)",
 	}
-	oracleFlag = cli.StringFlag{
+	oracleFlag = &cli.StringFlag{
 		Name:  "oracle",
 		Usage: "Checkpoint oracle address (query from remote node if not specified)",
 	}
-	thresholdFlag = cli.Int64Flag{
+	thresholdFlag = &cli.Int64Flag{
 		Name:  "threshold",
 		Usage: "Minimal number of signatures required to approve a checkpoint",
 	}
-	nodeURLFlag = cli.StringFlag{
+	nodeURLFlag = &cli.StringFlag{
 		Name:  "rpc",
 		Value: "http://localhost:8545",
 		Usage: "The rpc endpoint of a local or remote geth node",
 	}
-	clefURLFlag = cli.StringFlag{
+	clefURLFlag = &cli.StringFlag{
 		Name:  "clef",
 		Value: "http://localhost:8550",
 		Usage: "The rpc endpoint of clef",
 	}
-	signerFlag = cli.StringFlag{
+	signerFlag = &cli.StringFlag{
 		Name:  "signer",
 		Usage: "Signer address for clef signing",
 	}
-	signersFlag = cli.StringFlag{
+	signersFlag = &cli.StringFlag{
 		Name:  "signers",
 		Usage: "Comma separated accounts of trusted checkpoint signers",
 	}
-	signaturesFlag = cli.StringFlag{
+	signaturesFlag = &cli.StringFlag{
 		Name:  "signatures",
 		Usage: "Comma separated checkpoint signatures to submit",
 	}

cmd/checkpoint-admin/status.go

@@ -19,24 +19,23 @@ package main
 import (
 	"fmt"
 
-	"github.com/ethereum/go-ethereum/cmd/utils"
 	"github.com/ethereum/go-ethereum/common"
-	"gopkg.in/urfave/cli.v1"
+	"github.com/urfave/cli/v2"
 )
 
-var commandStatus = cli.Command{
+var commandStatus = &cli.Command{
 	Name:  "status",
 	Usage: "Fetches the signers and checkpoint status of the oracle contract",
 	Flags: []cli.Flag{
 		nodeURLFlag,
 	},
-	Action: utils.MigrateFlags(status),
+	Action: status,
 }
 
 // status fetches the admin list of specified registrar contract.
 func status(ctx *cli.Context) error {
 	// Create a wrapper around the checkpoint oracle contract
-	addr, oracle := newContract(newRPCClient(ctx.GlobalString(nodeURLFlag.Name)))
+	addr, oracle := newContract(newRPCClient(ctx.String(nodeURLFlag.Name)))
 	fmt.Printf("Oracle => %s\n", addr.Hex())
 	fmt.Println()
 

cmd/clef/README.md

@@ -29,7 +29,7 @@ GLOBAL OPTIONS:
    --loglevel value        log level to emit to the screen (default: 4)
    --keystore value        Directory for the keystore (default: "$HOME/.ethereum/keystore")
    --configdir value       Directory for Clef configuration (default: "$HOME/.clef")
-   --chainid value         Chain id to use for signing (1=mainnet, 3=Ropsten, 4=Rinkeby, 5=Goerli) (default: 1)
+   --chainid value         Chain id to use for signing (1=mainnet, 4=Rinkeby, 5=Goerli) (default: 1)
    --lightkdf              Reduce key-derivation RAM & CPU usage at some expense of KDF strength
    --nousb                 Disables monitoring for and managing USB hardware wallets
    --pcscdpath value       Path to the smartcard daemon (pcscd) socket file (default: "/run/pcscd/pcscd.comm")

cmd/clef/consolecmd_test.go

@@ -0,0 +1,117 @@
+// Copyright 2022 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+// TestImportRaw tests clef --importraw
+func TestImportRaw(t *testing.T) {
+	keyPath := filepath.Join(os.TempDir(), fmt.Sprintf("%v-tempkey.test", t.Name()))
+	os.WriteFile(keyPath, []byte("0102030405060708090a0102030405060708090a0102030405060708090a0102"), 0777)
+	t.Cleanup(func() { os.Remove(keyPath) })
+
+	t.Parallel()
+	t.Run("happy-path", func(t *testing.T) {
+		// Run clef importraw
+		clef := runClef(t, "--suppress-bootwarn", "--lightkdf", "importraw", keyPath)
+		clef.input("myverylongpassword").input("myverylongpassword")
+		if out := string(clef.Output()); !strings.Contains(out,
+			"Key imported:\n  Address 0x9160DC9105f7De5dC5E7f3d97ef11DA47269BdA6") {
+			t.Logf("Output\n%v", out)
+			t.Error("Failure")
+		}
+	})
+	// tests clef --importraw with mismatched passwords.
+	t.Run("pw-mismatch", func(t *testing.T) {
+		// Run clef importraw
+		clef := runClef(t, "--suppress-bootwarn", "--lightkdf", "importraw", keyPath)
+		clef.input("myverylongpassword1").input("myverylongpassword2").WaitExit()
+		if have, want := clef.StderrText(), "Passwords do not match\n"; have != want {
+			t.Errorf("have %q, want %q", have, want)
+		}
+	})
+	// tests clef --importraw with a too short password.
+	t.Run("short-pw", func(t *testing.T) {
+		// Run clef importraw
+		clef := runClef(t, "--suppress-bootwarn", "--lightkdf", "importraw", keyPath)
+		clef.input("shorty").input("shorty").WaitExit()
+		if have, want := clef.StderrText(),
+			"password requirements not met: password too short (<10 characters)\n"; have != want {
+			t.Errorf("have %q, want %q", have, want)
+		}
+	})
+}
+
+// TestListAccounts tests clef --list-accounts
+func TestListAccounts(t *testing.T) {
+	keyPath := filepath.Join(os.TempDir(), fmt.Sprintf("%v-tempkey.test", t.Name()))
+	os.WriteFile(keyPath, []byte("0102030405060708090a0102030405060708090a0102030405060708090a0102"), 0777)
+	t.Cleanup(func() { os.Remove(keyPath) })
+
+	t.Parallel()
+	t.Run("no-accounts", func(t *testing.T) {
+		clef := runClef(t, "--suppress-bootwarn", "--lightkdf", "list-accounts")
+		if out := string(clef.Output()); !strings.Contains(out, "The keystore is empty.") {
+			t.Logf("Output\n%v", out)
+			t.Error("Failure")
+		}
+	})
+	t.Run("one-account", func(t *testing.T) {
+		// First, we need to import
+		clef := runClef(t, "--suppress-bootwarn", "--lightkdf", "importraw", keyPath)
+		clef.input("myverylongpassword").input("myverylongpassword").WaitExit()
+		// Secondly, do a listing, using the same datadir
+		clef = runWithKeystore(t, clef.Datadir, "--suppress-bootwarn", "--lightkdf", "list-accounts")
+		if out := string(clef.Output()); !strings.Contains(out, "0x9160DC9105f7De5dC5E7f3d97ef11DA47269BdA6 (keystore:") {
+			t.Logf("Output\n%v", out)
+			t.Error("Failure")
+		}
+	})
+}
+
+// TestListWallets tests clef --list-wallets
+func TestListWallets(t *testing.T) {
+	keyPath := filepath.Join(os.TempDir(), fmt.Sprintf("%v-tempkey.test", t.Name()))
+	os.WriteFile(keyPath, []byte("0102030405060708090a0102030405060708090a0102030405060708090a0102"), 0777)
+	t.Cleanup(func() { os.Remove(keyPath) })
+
+	t.Parallel()
+	t.Run("no-accounts", func(t *testing.T) {
+		clef := runClef(t, "--suppress-bootwarn", "--lightkdf", "list-wallets")
+		if out := string(clef.Output()); !strings.Contains(out, "There are no wallets.") {
+			t.Logf("Output\n%v", out)
+			t.Error("Failure")
+		}
+	})
+	t.Run("one-account", func(t *testing.T) {
+		// First, we need to import
+		clef := runClef(t, "--suppress-bootwarn", "--lightkdf", "importraw", keyPath)
+		clef.input("myverylongpassword").input("myverylongpassword").WaitExit()
+		// Secondly, do a listing, using the same datadir
+		clef = runWithKeystore(t, clef.Datadir, "--suppress-bootwarn", "--lightkdf", "list-wallets")
+		if out := string(clef.Output()); !strings.Contains(out, "Account 0: 0x9160DC9105f7De5dC5E7f3d97ef11DA47269BdA6") {
+			t.Logf("Output\n%v", out)
+			t.Error("Failure")
+		}
+	})
+}

cmd/clef/main.go

@@ -23,6 +23,7 @@ import (
 	"crypto/sha256"
 	"encoding/hex"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"math/big"
@@ -30,7 +31,6 @@ import (
 	"os/signal"
 	"path/filepath"
 	"runtime"
-	"sort"
 	"strings"
 	"time"
 
@@ -55,7 +55,7 @@ import (
 	"github.com/ethereum/go-ethereum/signer/storage"
 	"github.com/mattn/go-colorable"
 	"github.com/mattn/go-isatty"
-	"gopkg.in/urfave/cli.v1"
+	"github.com/urfave/cli/v2"
 )
 
 const legalWarning = `
@@ -73,70 +73,70 @@ PURPOSE. See the GNU General Public License for more details.
 `
 
 var (
-	logLevelFlag = cli.IntFlag{
+	logLevelFlag = &cli.IntFlag{
 		Name:  "loglevel",
-		Value: 4,
+		Value: 3,
 		Usage: "log level to emit to the screen",
 	}
-	advancedMode = cli.BoolFlag{
+	advancedMode = &cli.BoolFlag{
 		Name:  "advanced",
 		Usage: "If enabled, issues warnings instead of rejections for suspicious requests. Default off",
 	}
-	acceptFlag = cli.BoolFlag{
+	acceptFlag = &cli.BoolFlag{
 		Name:  "suppress-bootwarn",
 		Usage: "If set, does not show the warning during boot",
 	}
-	keystoreFlag = cli.StringFlag{
+	keystoreFlag = &cli.StringFlag{
 		Name:  "keystore",
 		Value: filepath.Join(node.DefaultDataDir(), "keystore"),
 		Usage: "Directory for the keystore",
 	}
-	configdirFlag = cli.StringFlag{
+	configdirFlag = &cli.StringFlag{
 		Name:  "configdir",
 		Value: DefaultConfigDir(),
 		Usage: "Directory for Clef configuration",
 	}
-	chainIdFlag = cli.Int64Flag{
+	chainIdFlag = &cli.Int64Flag{
 		Name:  "chainid",
 		Value: params.MainnetChainConfig.ChainID.Int64(),
-		Usage: "Chain id to use for signing (1=mainnet, 3=Ropsten, 4=Rinkeby, 5=Goerli)",
+		Usage: "Chain id to use for signing (1=mainnet, 4=Rinkeby, 5=Goerli)",
 	}
-	rpcPortFlag = cli.IntFlag{
-		Name:  "http.port",
-		Usage: "HTTP-RPC server listening port",
-		Value: node.DefaultHTTPPort + 5,
+	rpcPortFlag = &cli.IntFlag{
+		Name:     "http.port",
+		Usage:    "HTTP-RPC server listening port",
+		Value:    node.DefaultHTTPPort + 5,
+		Category: flags.APICategory,
 	}
-	signerSecretFlag = cli.StringFlag{
+	signerSecretFlag = &cli.StringFlag{
 		Name:  "signersecret",
 		Usage: "A file containing the (encrypted) master seed to encrypt Clef data, e.g. keystore credentials and ruleset hash",
 	}
-	customDBFlag = cli.StringFlag{
+	customDBFlag = &cli.StringFlag{
 		Name:  "4bytedb-custom",
 		Usage: "File used for writing new 4byte-identifiers submitted via API",
 		Value: "./4byte-custom.json",
 	}
-	auditLogFlag = cli.StringFlag{
+	auditLogFlag = &cli.StringFlag{
 		Name:  "auditlog",
 		Usage: "File used to emit audit logs. Set to \"\" to disable",
 		Value: "audit.log",
 	}
-	ruleFlag = cli.StringFlag{
+	ruleFlag = &cli.StringFlag{
 		Name:  "rules",
 		Usage: "Path to the rule file to auto-authorize requests with",
 	}
-	stdiouiFlag = cli.BoolFlag{
+	stdiouiFlag = &cli.BoolFlag{
 		Name: "stdio-ui",
 		Usage: "Use STDIN/STDOUT as a channel for an external UI. " +
 			"This means that an STDIN/STDOUT is used for RPC-communication with a e.g. a graphical user " +
 			"interface, and can be used when Clef is started by an external process.",
 	}
-	testFlag = cli.BoolFlag{
+	testFlag = &cli.BoolFlag{
 		Name:  "stdio-ui-test",
 		Usage: "Mechanism to test interface between Clef and UI. Requires 'stdio-ui'.",
 	}
-	app         = cli.NewApp()
-	initCommand = cli.Command{
-		Action:    utils.MigrateFlags(initializeSecrets),
+	initCommand = &cli.Command{
+		Action:    initializeSecrets,
 		Name:      "init",
 		Usage:     "Initialize the signer, generate secret storage",
 		ArgsUsage: "",
@@ -148,8 +148,8 @@ var (
 The init command generates a master seed which Clef can use to store credentials and data needed for
 the rule-engine to work.`,
 	}
-	attestCommand = cli.Command{
-		Action:    utils.MigrateFlags(attestFile),
+	attestCommand = &cli.Command{
+		Action:    attestFile,
 		Name:      "attest",
 		Usage:     "Attest that a js-file is to be used",
 		ArgsUsage: "<sha256sum>",
@@ -165,8 +165,8 @@ incoming requests.
 Whenever you make an edit to the rule file, you need to use attestation to tell
 Clef that the file is 'safe' to execute.`,
 	}
-	setCredentialCommand = cli.Command{
-		Action:    utils.MigrateFlags(setCredential),
+	setCredentialCommand = &cli.Command{
+		Action:    setCredential,
 		Name:      "setpw",
 		Usage:     "Store a credential for a keystore file",
 		ArgsUsage: "<address>",
@@ -178,8 +178,8 @@ Clef that the file is 'safe' to execute.`,
 		Description: `
 The setpw command stores a password for a given address (keyfile).
 `}
-	delCredentialCommand = cli.Command{
-		Action:    utils.MigrateFlags(removeCredential),
+	delCredentialCommand = &cli.Command{
+		Action:    removeCredential,
 		Name:      "delpw",
 		Usage:     "Remove a credential for a keystore file",
 		ArgsUsage: "<address>",
@@ -191,8 +191,8 @@ The setpw command stores a password for a given address (keyfile).
 		Description: `
 The delpw command removes a password for a given address (keyfile).
 `}
-	newAccountCommand = cli.Command{
-		Action:    utils.MigrateFlags(newAccount),
+	newAccountCommand = &cli.Command{
+		Action:    newAccount,
 		Name:      "newaccount",
 		Usage:     "Create a new account",
 		ArgsUsage: "",
@@ -204,51 +204,64 @@ The delpw command removes a password for a given address (keyfile).
 		},
 		Description: `
 The newaccount command creates a new keystore-backed account. It is a convenience-method
-which can be used in lieu of an external UI.`,
-	}
-
-	gendocCommand = cli.Command{
+which can be used in lieu of an external UI.
+`}
+	gendocCommand = &cli.Command{
 		Action: GenDoc,
 		Name:   "gendoc",
 		Usage:  "Generate documentation about json-rpc format",
 		Description: `
 The gendoc generates example structures of the json-rpc communication types.
 `}
-)
-
-// AppHelpFlagGroups is the application flags, grouped by functionality.
-var AppHelpFlagGroups = []flags.FlagGroup{
-	{
-		Name: "FLAGS",
+	listAccountsCommand = &cli.Command{
+		Action: listAccounts,
+		Name:   "list-accounts",
+		Usage:  "List accounts in the keystore",
 		Flags: []cli.Flag{
 			logLevelFlag,
 			keystoreFlag,
-			configdirFlag,
-			chainIdFlag,
 			utils.LightKDFFlag,
-			utils.NoUSBFlag,
-			utils.SmartCardDaemonPathFlag,
-			utils.HTTPListenAddrFlag,
-			utils.HTTPVirtualHostsFlag,
-			utils.IPCDisabledFlag,
-			utils.IPCPathFlag,
-			utils.HTTPEnabledFlag,
-			rpcPortFlag,
-			signerSecretFlag,
-			customDBFlag,
-			auditLogFlag,
-			ruleFlag,
-			stdiouiFlag,
-			testFlag,
-			advancedMode,
 			acceptFlag,
 		},
-	},
-}
+		Description: `
+	Lists the accounts in the keystore.
+	`}
+	listWalletsCommand = &cli.Command{
+		Action: listWallets,
+		Name:   "list-wallets",
+		Usage:  "List wallets known to Clef",
+		Flags: []cli.Flag{
+			logLevelFlag,
+			keystoreFlag,
+			utils.LightKDFFlag,
+			acceptFlag,
+		},
+		Description: `
+	Lists the wallets known to Clef.
+	`}
+	importRawCommand = &cli.Command{
+		Action:    accountImport,
+		Name:      "importraw",
+		Usage:     "Import a hex-encoded private key.",
+		ArgsUsage: "<keyfile>",
+		Flags: []cli.Flag{
+			logLevelFlag,
+			keystoreFlag,
+			utils.LightKDFFlag,
+			acceptFlag,
+		},
+		Description: `
+Imports an unencrypted private key from <keyfile> and creates a new account.
+Prints the address.
+The keyfile is assumed to contain an unencrypted private key in hexadecimal format.
+The account is saved in encrypted format, you are prompted for a password.
+`}
+)
+
+var app = flags.NewApp("Manage Ethereum account operations")
 
 func init() {
 	app.Name = "Clef"
-	app.Usage = "Manage Ethereum account operations"
 	app.Flags = []cli.Flag{
 		logLevelFlag,
 		keystoreFlag,
@@ -273,46 +286,15 @@ func init() {
 		acceptFlag,
 	}
 	app.Action = signer
-	app.Commands = []cli.Command{initCommand,
+	app.Commands = []*cli.Command{initCommand,
 		attestCommand,
 		setCredentialCommand,
 		delCredentialCommand,
 		newAccountCommand,
-		gendocCommand}
-	cli.CommandHelpTemplate = flags.CommandHelpTemplate
-	// Override the default app help template
-	cli.AppHelpTemplate = flags.ClefAppHelpTemplate
-
-	// Override the default app help printer, but only for the global app help
-	originalHelpPrinter := cli.HelpPrinter
-	cli.HelpPrinter = func(w io.Writer, tmpl string, data interface{}) {
-		if tmpl == flags.ClefAppHelpTemplate {
-			// Render out custom usage screen
-			originalHelpPrinter(w, tmpl, flags.HelpData{App: data, FlagGroups: AppHelpFlagGroups})
-		} else if tmpl == flags.CommandHelpTemplate {
-			// Iterate over all command specific flags and categorize them
-			categorized := make(map[string][]cli.Flag)
-			for _, flag := range data.(cli.Command).Flags {
-				if _, ok := categorized[flag.String()]; !ok {
-					categorized[flags.FlagCategory(flag, AppHelpFlagGroups)] = append(categorized[flags.FlagCategory(flag, AppHelpFlagGroups)], flag)
-				}
-			}
-
-			// sort to get a stable ordering
-			sorted := make([]flags.FlagGroup, 0, len(categorized))
-			for cat, flgs := range categorized {
-				sorted = append(sorted, flags.FlagGroup{Name: cat, Flags: flgs})
-			}
-			sort.Sort(flags.ByCategory(sorted))
-
-			// add sorted array to data and render with default printer
-			originalHelpPrinter(w, tmpl, map[string]interface{}{
-				"cmd":              data,
-				"categorizedFlags": sorted,
-			})
-		} else {
-			originalHelpPrinter(w, tmpl, data)
-		}
+		importRawCommand,
+		gendocCommand,
+		listAccountsCommand,
+		listWalletsCommand,
 	}
 }
 
@@ -329,7 +311,7 @@ func initializeSecrets(c *cli.Context) error {
 		return err
 	}
 	// Ensure the master key does not yet exist, we're not willing to overwrite
-	configDir := c.GlobalString(configdirFlag.Name)
+	configDir := c.String(configdirFlag.Name)
 	if err := os.Mkdir(configDir, 0700); err != nil && !os.IsExist(err) {
 		return err
 	}
@@ -347,7 +329,7 @@ func initializeSecrets(c *cli.Context) error {
 		return fmt.Errorf("failed to read enough random")
 	}
 	n, p := keystore.StandardScryptN, keystore.StandardScryptP
-	if c.GlobalBool(utils.LightKDFFlag.Name) {
+	if c.Bool(utils.LightKDFFlag.Name) {
 		n, p = keystore.LightScryptN, keystore.LightScryptP
 	}
 	text := "The master seed of clef will be locked with a password.\nPlease specify a password. Do not forget this password!"
@@ -390,8 +372,9 @@ You should treat 'masterseed.json' with utmost secrecy and make a backup of it!
 `)
 	return nil
 }
+
 func attestFile(ctx *cli.Context) error {
-	if len(ctx.Args()) < 1 {
+	if ctx.NArg() < 1 {
 		utils.Fatalf("This command requires an argument.")
 	}
 	if err := initialize(ctx); err != nil {
@@ -402,7 +385,7 @@ func attestFile(ctx *cli.Context) error {
 	if err != nil {
 		utils.Fatalf(err.Error())
 	}
-	configDir := ctx.GlobalString(configdirFlag.Name)
+	configDir := ctx.String(configdirFlag.Name)
 	vaultLocation := filepath.Join(configDir, common.Bytes2Hex(crypto.Keccak256([]byte("vault"), stretchedKey)[:10]))
 	confKey := crypto.Keccak256([]byte("config"), stretchedKey)
 
@@ -414,8 +397,24 @@ func attestFile(ctx *cli.Context) error {
 	return nil
 }
 
+func initInternalApi(c *cli.Context) (*core.UIServerAPI, core.UIClientAPI, error) {
+	if err := initialize(c); err != nil {
+		return nil, nil, err
+	}
+	var (
+		ui                        = core.NewCommandlineUI()
+		pwStorage storage.Storage = &storage.NoStorage{}
+		ksLoc                     = c.String(keystoreFlag.Name)
+		lightKdf                  = c.Bool(utils.LightKDFFlag.Name)
+	)
+	am := core.StartClefAccountManager(ksLoc, true, lightKdf, "")
+	api := core.NewSignerAPI(am, 0, true, ui, nil, false, pwStorage)
+	internalApi := core.NewUIServerAPI(api)
+	return internalApi, ui, nil
+}
+
 func setCredential(ctx *cli.Context) error {
-	if len(ctx.Args()) < 1 {
+	if ctx.NArg() < 1 {
 		utils.Fatalf("This command requires an address to be passed as an argument")
 	}
 	if err := initialize(ctx); err != nil {
@@ -433,7 +432,7 @@ func setCredential(ctx *cli.Context) error {
 	if err != nil {
 		utils.Fatalf(err.Error())
 	}
-	configDir := ctx.GlobalString(configdirFlag.Name)
+	configDir := ctx.String(configdirFlag.Name)
 	vaultLocation := filepath.Join(configDir, common.Bytes2Hex(crypto.Keccak256([]byte("vault"), stretchedKey)[:10]))
 	pwkey := crypto.Keccak256([]byte("credentials"), stretchedKey)
 
@@ -445,7 +444,7 @@ func setCredential(ctx *cli.Context) error {
 }
 
 func removeCredential(ctx *cli.Context) error {
-	if len(ctx.Args()) < 1 {
+	if ctx.NArg() < 1 {
 		utils.Fatalf("This command requires an address to be passed as an argument")
 	}
 	if err := initialize(ctx); err != nil {
@@ -461,7 +460,7 @@ func removeCredential(ctx *cli.Context) error {
 	if err != nil {
 		utils.Fatalf(err.Error())
 	}
-	configDir := ctx.GlobalString(configdirFlag.Name)
+	configDir := ctx.String(configdirFlag.Name)
 	vaultLocation := filepath.Join(configDir, common.Bytes2Hex(crypto.Keccak256([]byte("vault"), stretchedKey)[:10]))
 	pwkey := crypto.Keccak256([]byte("credentials"), stretchedKey)
 
@@ -472,41 +471,16 @@ func removeCredential(ctx *cli.Context) error {
 	return nil
 }
 
-func newAccount(c *cli.Context) error {
-	if err := initialize(c); err != nil {
-		return err
-	}
-	// The newaccount is meant for users using the CLI, since 'real' external
-	// UIs can use the UI-api instead. So we'll just use the native CLI UI here.
-	var (
-		ui                        = core.NewCommandlineUI()
-		pwStorage storage.Storage = &storage.NoStorage{}
-		ksLoc                     = c.GlobalString(keystoreFlag.Name)
-		lightKdf                  = c.GlobalBool(utils.LightKDFFlag.Name)
-	)
-	log.Info("Starting clef", "keystore", ksLoc, "light-kdf", lightKdf)
-	am := core.StartClefAccountManager(ksLoc, true, lightKdf, "")
-	// This gives is us access to the external API
-	apiImpl := core.NewSignerAPI(am, 0, true, ui, nil, false, pwStorage)
-	// This gives us access to the internal API
-	internalApi := core.NewUIServerAPI(apiImpl)
-	addr, err := internalApi.New(context.Background())
-	if err == nil {
-		fmt.Printf("Generated account %v\n", addr.String())
-	}
-	return err
-}
-
 func initialize(c *cli.Context) error {
 	// Set up the logger to print everything
 	logOutput := os.Stdout
-	if c.GlobalBool(stdiouiFlag.Name) {
+	if c.Bool(stdiouiFlag.Name) {
 		logOutput = os.Stderr
 		// If using the stdioui, we can't do the 'confirm'-flow
-		if !c.GlobalBool(acceptFlag.Name) {
+		if !c.Bool(acceptFlag.Name) {
 			fmt.Fprint(logOutput, legalWarning)
 		}
-	} else if !c.GlobalBool(acceptFlag.Name) {
+	} else if !c.Bool(acceptFlag.Name) {
 		if !confirm(legalWarning) {
 			return fmt.Errorf("aborted by user")
 		}
@@ -522,6 +496,108 @@ func initialize(c *cli.Context) error {
 	return nil
 }
 
+func newAccount(c *cli.Context) error {
+	internalApi, _, err := initInternalApi(c)
+	if err != nil {
+		return err
+	}
+	addr, err := internalApi.New(context.Background())
+	if err == nil {
+		fmt.Printf("Generated account %v\n", addr.String())
+	}
+	return err
+}
+
+func listAccounts(c *cli.Context) error {
+	internalApi, _, err := initInternalApi(c)
+	if err != nil {
+		return err
+	}
+	accs, err := internalApi.ListAccounts(context.Background())
+	if err != nil {
+		return err
+	}
+	if len(accs) == 0 {
+		fmt.Println("\nThe keystore is empty.")
+	}
+	fmt.Println()
+	for _, account := range accs {
+		fmt.Printf("%v (%v)\n", account.Address, account.URL)
+	}
+	return err
+}
+
+func listWallets(c *cli.Context) error {
+	internalApi, _, err := initInternalApi(c)
+	if err != nil {
+		return err
+	}
+	wallets := internalApi.ListWallets()
+	if len(wallets) == 0 {
+		fmt.Println("\nThere are no wallets.")
+	}
+	fmt.Println()
+	for i, wallet := range wallets {
+		fmt.Printf("- Wallet %d at %v (%v %v)\n", i, wallet.URL, wallet.Status, wallet.Failure)
+		for j, acc := range wallet.Accounts {
+			fmt.Printf("  -Account %d: %v (%v)\n", j, acc.Address, acc.URL)
+		}
+		fmt.Println()
+	}
+	return nil
+}
+
+// accountImport imports a raw hexadecimal private key via CLI.
+func accountImport(c *cli.Context) error {
+	if c.Args().Len() != 1 {
+		return errors.New("<keyfile> must be given as first argument.")
+	}
+	internalApi, ui, err := initInternalApi(c)
+	if err != nil {
+		return err
+	}
+	pKey, err := crypto.LoadECDSA(c.Args().First())
+	if err != nil {
+		return err
+	}
+	readPw := func(prompt string) (string, error) {
+		resp, err := ui.OnInputRequired(core.UserInputRequest{
+			Title:      "Password",
+			Prompt:     prompt,
+			IsPassword: true,
+		})
+		if err != nil {
+			return "", err
+		}
+		return resp.Text, nil
+	}
+	first, err := readPw("Please enter a password for the imported account")
+	if err != nil {
+		return err
+	}
+	second, err := readPw("Please repeat the password you just entered")
+	if err != nil {
+		return err
+	}
+	if first != second {
+		return errors.New("Passwords do not match")
+	}
+	acc, err := internalApi.ImportRawKey(hex.EncodeToString(crypto.FromECDSA(pKey)), first)
+	if err != nil {
+		return err
+	}
+	ui.ShowInfo(fmt.Sprintf(`Key imported:
+  Address %v
+  Keystore file: %v
+
+The key is now encrypted; losing the password will result in permanently losing 
+access to the key and all associated funds!
+
+Make sure to backup keystore and passwords in a safe location.`,
+		acc.Address, acc.URL.Path))
+	return nil
+}
+
 // ipcEndpoint resolves an IPC endpoint based on a configured value, taking into
 // account the set data folders as well as the designated platform we're currently
 // running on.
@@ -545,8 +621,8 @@ func ipcEndpoint(ipcPath, datadir string) string {
 
 func signer(c *cli.Context) error {
 	// If we have some unrecognized command, bail out
-	if args := c.Args(); len(args) > 0 {
-		return fmt.Errorf("invalid command: %q", args[0])
+	if c.NArg() > 0 {
+		return fmt.Errorf("invalid command: %q", c.Args().First())
 	}
 	if err := initialize(c); err != nil {
 		return err
@@ -554,7 +630,7 @@ func signer(c *cli.Context) error {
 	var (
 		ui core.UIClientAPI
 	)
-	if c.GlobalBool(stdiouiFlag.Name) {
+	if c.Bool(stdiouiFlag.Name) {
 		log.Info("Using stdin/stdout as UI-channel")
 		ui = core.NewStdIOUI()
 	} else {
@@ -562,7 +638,7 @@ func signer(c *cli.Context) error {
 		ui = core.NewCommandlineUI()
 	}
 	// 4bytedb data
-	fourByteLocal := c.GlobalString(customDBFlag.Name)
+	fourByteLocal := c.String(customDBFlag.Name)
 	db, err := fourbyte.NewWithFile(fourByteLocal)
 	if err != nil {
 		utils.Fatalf(err.Error())
@@ -574,7 +650,7 @@ func signer(c *cli.Context) error {
 		api       core.ExternalAPI
 		pwStorage storage.Storage = &storage.NoStorage{}
 	)
-	configDir := c.GlobalString(configdirFlag.Name)
+	configDir := c.String(configdirFlag.Name)
 	if stretchedKey, err := readMasterKey(c, ui); err != nil {
 		log.Warn("Failed to open master, rules disabled", "err", err)
 	} else {
@@ -591,7 +667,7 @@ func signer(c *cli.Context) error {
 		configStorage := storage.NewAESEncryptedStorage(filepath.Join(vaultLocation, "config.json"), confkey)
 
 		// Do we have a rule-file?
-		if ruleFile := c.GlobalString(ruleFlag.Name); ruleFile != "" {
+		if ruleFile := c.String(ruleFlag.Name); ruleFile != "" {
 			ruleJS, err := os.ReadFile(ruleFile)
 			if err != nil {
 				log.Warn("Could not load rules, disabling", "file", ruleFile, "err", err)
@@ -615,12 +691,12 @@ func signer(c *cli.Context) error {
 		}
 	}
 	var (
-		chainId  = c.GlobalInt64(chainIdFlag.Name)
-		ksLoc    = c.GlobalString(keystoreFlag.Name)
-		lightKdf = c.GlobalBool(utils.LightKDFFlag.Name)
-		advanced = c.GlobalBool(advancedMode.Name)
-		nousb    = c.GlobalBool(utils.NoUSBFlag.Name)
-		scpath   = c.GlobalString(utils.SmartCardDaemonPathFlag.Name)
+		chainId  = c.Int64(chainIdFlag.Name)
+		ksLoc    = c.String(keystoreFlag.Name)
+		lightKdf = c.Bool(utils.LightKDFFlag.Name)
+		advanced = c.Bool(advancedMode.Name)
+		nousb    = c.Bool(utils.NoUSBFlag.Name)
+		scpath   = c.String(utils.SmartCardDaemonPathFlag.Name)
 	)
 	log.Info("Starting signer", "chainid", chainId, "keystore", ksLoc,
 		"light-kdf", lightKdf, "advanced", advanced)
@@ -631,8 +707,9 @@ func signer(c *cli.Context) error {
 	// it with the UI.
 	ui.RegisterUIServer(core.NewUIServerAPI(apiImpl))
 	api = apiImpl
+
 	// Audit logging
-	if logfile := c.GlobalString(auditLogFlag.Name); logfile != "" {
+	if logfile := c.String(auditLogFlag.Name); logfile != "" {
 		api, err = core.NewAuditLogger(logfile, api)
 		if err != nil {
 			utils.Fatalf(err.Error())
@@ -647,16 +724,15 @@ func signer(c *cli.Context) error {
 	rpcAPI := []rpc.API{
 		{
 			Namespace: "account",
-			Public:    true,
 			Service:   api,
-			Version:   "1.0"},
+		},
 	}
-	if c.GlobalBool(utils.HTTPEnabledFlag.Name) {
-		vhosts := utils.SplitAndTrim(c.GlobalString(utils.HTTPVirtualHostsFlag.Name))
-		cors := utils.SplitAndTrim(c.GlobalString(utils.HTTPCORSDomainFlag.Name))
+	if c.Bool(utils.HTTPEnabledFlag.Name) {
+		vhosts := utils.SplitAndTrim(c.String(utils.HTTPVirtualHostsFlag.Name))
+		cors := utils.SplitAndTrim(c.String(utils.HTTPCORSDomainFlag.Name))
 
 		srv := rpc.NewServer()
-		err := node.RegisterApis(rpcAPI, []string{"account"}, srv, false)
+		err := node.RegisterApis(rpcAPI, []string{"account"}, srv)
 		if err != nil {
 			utils.Fatalf("Could not register API: %w", err)
 		}
@@ -666,7 +742,7 @@ func signer(c *cli.Context) error {
 		port := c.Int(rpcPortFlag.Name)
 
 		// start http server
-		httpEndpoint := fmt.Sprintf("%s:%d", c.GlobalString(utils.HTTPListenAddrFlag.Name), port)
+		httpEndpoint := fmt.Sprintf("%s:%d", c.String(utils.HTTPListenAddrFlag.Name), port)
 		httpServer, addr, err := node.StartHTTPEndpoint(httpEndpoint, rpc.DefaultHTTPTimeouts, handler)
 		if err != nil {
 			utils.Fatalf("Could not start RPC api: %v", err)
@@ -680,8 +756,8 @@ func signer(c *cli.Context) error {
 			log.Info("HTTP endpoint closed", "url", extapiURL)
 		}()
 	}
-	if !c.GlobalBool(utils.IPCDisabledFlag.Name) {
-		givenPath := c.GlobalString(utils.IPCPathFlag.Name)
+	if !c.Bool(utils.IPCDisabledFlag.Name) {
+		givenPath := c.String(utils.IPCPathFlag.Name)
 		ipcapiURL = ipcEndpoint(filepath.Join(givenPath, "clef.ipc"), configDir)
 		listener, _, err := rpc.StartIPCEndpoint(ipcapiURL, rpcAPI)
 		if err != nil {
@@ -693,8 +769,7 @@ func signer(c *cli.Context) error {
 			log.Info("IPC endpoint closed", "url", ipcapiURL)
 		}()
 	}
-
-	if c.GlobalBool(testFlag.Name) {
+	if c.Bool(testFlag.Name) {
 		log.Info("Performing UI test")
 		go testExternalUI(apiImpl)
 	}
@@ -704,8 +779,7 @@ func signer(c *cli.Context) error {
 			"extapi_version": core.ExternalAPIVersion,
 			"extapi_http":    extapiURL,
 			"extapi_ipc":     ipcapiURL,
-		},
-	})
+		}})
 
 	abortChan := make(chan os.Signal, 1)
 	signal.Notify(abortChan, os.Interrupt)
@@ -720,7 +794,7 @@ func signer(c *cli.Context) error {
 // persistence requirements.
 func DefaultConfigDir() string {
 	// Try to place the data folder in the user's home dir
-	home := utils.HomeDir()
+	home := flags.HomeDir()
 	if home != "" {
 		if runtime.GOOS == "darwin" {
 			return filepath.Join(home, "Library", "Signer")
@@ -740,10 +814,10 @@ func DefaultConfigDir() string {
 func readMasterKey(ctx *cli.Context, ui core.UIClientAPI) ([]byte, error) {
 	var (
 		file      string
-		configDir = ctx.GlobalString(configdirFlag.Name)
+		configDir = ctx.String(configdirFlag.Name)
 	)
-	if ctx.GlobalIsSet(signerSecretFlag.Name) {
-		file = ctx.GlobalString(signerSecretFlag.Name)
+	if ctx.IsSet(signerSecretFlag.Name) {
+		file = ctx.String(signerSecretFlag.Name)
 	} else {
 		file = filepath.Join(configDir, "masterseed.json")
 	}
@@ -817,7 +891,6 @@ func confirm(text string) bool {
 }
 
 func testExternalUI(api *core.SignerAPI) {
-
 	ctx := context.WithValue(context.Background(), "remote", "clef binary")
 	ctx = context.WithValue(ctx, "scheme", "in-proc")
 	ctx = context.WithValue(ctx, "local", "main")
@@ -917,7 +990,6 @@ func testExternalUI(api *core.SignerAPI) {
 		expectDeny("signdata - text", err)
 	}
 	{ // Sign transaction
-
 		api.UI.ShowInfo("Please reject next transaction")
 		time.Sleep(delay)
 		data := hexutil.Bytes([]byte{})
@@ -960,7 +1032,6 @@ func testExternalUI(api *core.SignerAPI) {
 	}
 	result := fmt.Sprintf("Tests completed. %d errors:\n%s\n", len(errs), strings.Join(errs, "\n"))
 	api.UI.ShowInfo(result)
-
 }
 
 type encryptedSeedStorage struct {
@@ -996,8 +1067,7 @@ func decryptSeed(keyjson []byte, auth string) ([]byte, error) {
 }
 
 // GenDoc outputs examples of all structures used in json-rpc communication
-func GenDoc(ctx *cli.Context) {
-
+func GenDoc(ctx *cli.Context) error {
 	var (
 		a    = common.HexToAddress("0xdeadbeef000000000000000000000000deadbeef")
 		b    = common.HexToAddress("0x1111111122222222222233333333334444444444")
@@ -1107,7 +1177,6 @@ func GenDoc(ctx *cli.Context) {
 		var tx types.Transaction
 		tx.UnmarshalBinary(rlpdata)
 		add("OnApproved - SignTransactionResult", desc, &ethapi.SignTransactionResult{Raw: rlpdata, Tx: &tx})
-
 	}
 	{ // User input
 		add("UserInputRequest", "Sent when clef needs the user to provide data. If 'password' is true, the input field should be treated accordingly (echo-free)",
@@ -1146,4 +1215,5 @@ These data types are defined in the channel between clef and the UI`)
 	for _, elem := range output {
 		fmt.Println(elem)
 	}
+	return nil
 }

cmd/clef/run_test.go

@@ -0,0 +1,109 @@
+// Copyright 2022 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"fmt"
+	"os"
+	"testing"
+
+	"github.com/docker/docker/pkg/reexec"
+	"github.com/ethereum/go-ethereum/internal/cmdtest"
+)
+
+const registeredName = "clef-test"
+
+type testproc struct {
+	*cmdtest.TestCmd
+
+	// template variables for expect
+	Datadir   string
+	Etherbase string
+}
+
+func init() {
+	reexec.Register(registeredName, func() {
+		if err := app.Run(os.Args); err != nil {
+			fmt.Fprintln(os.Stderr, err)
+			os.Exit(1)
+		}
+		os.Exit(0)
+	})
+}
+
+func TestMain(m *testing.M) {
+	// check if we have been reexec'd
+	if reexec.Init() {
+		return
+	}
+	os.Exit(m.Run())
+}
+
+// runClef spawns clef with the given command line args and adds keystore arg.
+// This method creates a temporary  keystore folder which will be removed after
+// the test exits.
+func runClef(t *testing.T, args ...string) *testproc {
+	ddir, err := os.MkdirTemp("", "cleftest-*")
+	if err != nil {
+		return nil
+	}
+	t.Cleanup(func() {
+		os.RemoveAll(ddir)
+	})
+	return runWithKeystore(t, ddir, args...)
+}
+
+// runWithKeystore spawns clef with the given command line args and adds keystore arg.
+// This method does _not_ create the keystore folder, but it _does_ add the arg
+// to the args.
+func runWithKeystore(t *testing.T, keystore string, args ...string) *testproc {
+	args = append([]string{"--keystore", keystore}, args...)
+	tt := &testproc{Datadir: keystore}
+	tt.TestCmd = cmdtest.NewTestCmd(t, tt)
+	// Boot "clef". This actually runs the test binary but the TestMain
+	// function will prevent any tests from running.
+	tt.Run(registeredName, args...)
+	return tt
+}
+
+func (proc *testproc) input(text string) *testproc {
+	proc.TestCmd.InputLine(text)
+	return proc
+}
+
+/*
+// waitForEndpoint waits for the rpc endpoint to appear, or
+// aborts after 3 seconds.
+func (proc *testproc) waitForEndpoint(t *testing.T) *testproc {
+	t.Helper()
+	timeout := 3 * time.Second
+	ipc := filepath.Join(proc.Datadir, "clef.ipc")
+
+	start := time.Now()
+	for time.Since(start) < timeout {
+		if _, err := os.Stat(ipc); !errors.Is(err, os.ErrNotExist) {
+			t.Logf("endpoint %v opened", ipc)
+			return proc
+		}
+		time.Sleep(200 * time.Millisecond)
+	}
+	t.Logf("stderr: \n%v", proc.StderrText())
+	t.Logf("stdout: \n%v", proc.Output())
+	t.Fatal("endpoint", ipc, "did not open within", timeout)
+	return proc
+}
+*/

cmd/devp2p/README.md

@@ -44,7 +44,7 @@ set to standard output. The following filters are supported:
 - `-limit <N>` limits the output set to N entries, taking the top N nodes by score
 - `-ip <CIDR>` filters nodes by IP subnet
 - `-min-age <duration>` filters nodes by 'first seen' time
-- `-eth-network <mainnet/rinkeby/goerli/ropsten>` filters nodes by "eth" ENR entry
+- `-eth-network <mainnet/rinkeby/goerli/sepolia>` filters nodes by "eth" ENR entry
 - `-les-server` filters nodes by LES server support
 - `-snap` filters nodes by snap protocol support
 
@@ -135,6 +135,6 @@ replacing `<enode>` with the enode of the geth node:
 ```
 
 [eth]: https://github.com/ethereum/devp2p/blob/master/caps/eth.md
-[dns-tutorial]: https://geth.ethereum.org/docs/developers/dns-discovery-setup
+[dns-tutorial]: https://geth.ethereum.org/docs/developers/geth-developer/dns-discovery-setup
 [discv4]: https://github.com/ethereum/devp2p/tree/master/discv4.md
 [discv5]: https://github.com/ethereum/devp2p/tree/master/discv5/discv5.md

cmd/devp2p/crawl.go

@@ -17,6 +17,8 @@
 package main
 
 import (
+	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/ethereum/go-ethereum/log"
@@ -34,8 +36,17 @@ type crawler struct {
 
 	// settings
 	revalidateInterval time.Duration
+	mu                 sync.RWMutex
 }
 
+const (
+	nodeRemoved = iota
+	nodeSkipRecent
+	nodeSkipIncompat
+	nodeAdded
+	nodeUpdated
+)
+
 type resolver interface {
 	RequestENR(*enode.Node) (*enode.Node, error)
 }
@@ -59,23 +70,59 @@ func newCrawler(input nodeSet, disc resolver, iters ...enode.Iterator) *crawler
 	return c
 }
 
-func (c *crawler) run(timeout time.Duration) nodeSet {
+func (c *crawler) run(timeout time.Duration, nthreads int) nodeSet {
 	var (
 		timeoutTimer = time.NewTimer(timeout)
 		timeoutCh    <-chan time.Time
+		statusTicker = time.NewTicker(time.Second * 8)
 		doneCh       = make(chan enode.Iterator, len(c.iters))
 		liveIters    = len(c.iters)
 	)
+	if nthreads < 1 {
+		nthreads = 1
+	}
 	defer timeoutTimer.Stop()
+	defer statusTicker.Stop()
 	for _, it := range c.iters {
 		go c.runIterator(doneCh, it)
 	}
+	var (
+		added   uint64
+		updated uint64
+		skipped uint64
+		recent  uint64
+		removed uint64
+		wg      sync.WaitGroup
+	)
+	wg.Add(nthreads)
+	for i := 0; i < nthreads; i++ {
+		go func() {
+			defer wg.Done()
+			for {
+				select {
+				case n := <-c.ch:
+					switch c.updateNode(n) {
+					case nodeSkipIncompat:
+						atomic.AddUint64(&skipped, 1)
+					case nodeSkipRecent:
+						atomic.AddUint64(&recent, 1)
+					case nodeRemoved:
+						atomic.AddUint64(&removed, 1)
+					case nodeAdded:
+						atomic.AddUint64(&added, 1)
+					default:
+						atomic.AddUint64(&updated, 1)
+					}
+				case <-c.closed:
+					return
+				}
+			}
+		}()
+	}
 
 loop:
 	for {
 		select {
-		case n := <-c.ch:
-			c.updateNode(n)
 		case it := <-doneCh:
 			if it == c.inputIter {
 				// Enable timeout when we're done revalidating the input nodes.
@@ -89,6 +136,13 @@ loop:
 			}
 		case <-timeoutCh:
 			break loop
+		case <-statusTicker.C:
+			log.Info("Crawling in progress",
+				"added", atomic.LoadUint64(&added),
+				"updated", atomic.LoadUint64(&updated),
+				"removed", atomic.LoadUint64(&removed),
+				"ignored(recent)", atomic.LoadUint64(&recent),
+				"ignored(incompatible)", atomic.LoadUint64(&skipped))
 		}
 	}
 
@@ -99,6 +153,7 @@ loop:
 	for ; liveIters > 0; liveIters-- {
 		<-doneCh
 	}
+	wg.Wait()
 	return c.output
 }
 
@@ -113,22 +168,26 @@ func (c *crawler) runIterator(done chan<- enode.Iterator, it enode.Iterator) {
 	}
 }
 
-func (c *crawler) updateNode(n *enode.Node) {
+// updateNode updates the info about the given node, and returns a status
+// about what changed
+func (c *crawler) updateNode(n *enode.Node) int {
+	c.mu.RLock()
 	node, ok := c.output[n.ID()]
+	c.mu.RUnlock()
 
 	// Skip validation of recently-seen nodes.
 	if ok && time.Since(node.LastCheck) < c.revalidateInterval {
-		return
+		return nodeSkipRecent
 	}
 
 	// Request the node record.
-	nn, err := c.disc.RequestENR(n)
+	status := nodeUpdated
 	node.LastCheck = truncNow()
-	if err != nil {
+	if nn, err := c.disc.RequestENR(n); err != nil {
 		if node.Score == 0 {
 			// Node doesn't implement EIP-868.
 			log.Debug("Skipping node", "id", n.ID())
-			return
+			return nodeSkipIncompat
 		}
 		node.Score /= 2
 	} else {
@@ -137,18 +196,21 @@ func (c *crawler) updateNode(n *enode.Node) {
 		node.Score++
 		if node.FirstResponse.IsZero() {
 			node.FirstResponse = node.LastCheck
+			status = nodeAdded
 		}
 		node.LastResponse = node.LastCheck
 	}
-
 	// Store/update node in output set.
+	c.mu.Lock()
+	defer c.mu.Unlock()
 	if node.Score <= 0 {
-		log.Info("Removing node", "id", n.ID())
+		log.Debug("Removing node", "id", n.ID())
 		delete(c.output, n.ID())
-	} else {
-		log.Info("Updating node", "id", n.ID(), "seq", n.Seq(), "score", node.Score)
-		c.output[n.ID()] = node
+		return nodeRemoved
 	}
+	log.Debug("Updating node", "id", n.ID(), "seq", n.Seq(), "score", node.Score)
+	c.output[n.ID()] = node
+	return status
 }
 
 func truncNow() time.Time {

cmd/devp2p/discv4cmd.go

@@ -19,23 +19,25 @@ package main
 import (
 	"fmt"
 	"net"
+	"strconv"
 	"strings"
 	"time"
 
 	"github.com/ethereum/go-ethereum/cmd/devp2p/internal/v4test"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/internal/flags"
 	"github.com/ethereum/go-ethereum/p2p/discover"
 	"github.com/ethereum/go-ethereum/p2p/enode"
 	"github.com/ethereum/go-ethereum/params"
-	"gopkg.in/urfave/cli.v1"
+	"github.com/urfave/cli/v2"
 )
 
 var (
-	discv4Command = cli.Command{
+	discv4Command = &cli.Command{
 		Name:  "discv4",
 		Usage: "Node Discovery v4 tools",
-		Subcommands: []cli.Command{
+		Subcommands: []*cli.Command{
 			discv4PingCommand,
 			discv4RequestRecordCommand,
 			discv4ResolveCommand,
@@ -44,39 +46,41 @@ var (
 			discv4TestCommand,
 		},
 	}
-	discv4PingCommand = cli.Command{
+	discv4PingCommand = &cli.Command{
 		Name:      "ping",
 		Usage:     "Sends ping to a node",
 		Action:    discv4Ping,
 		ArgsUsage: "<node>",
+		Flags:     discoveryNodeFlags,
 	}
-	discv4RequestRecordCommand = cli.Command{
+	discv4RequestRecordCommand = &cli.Command{
 		Name:      "requestenr",
 		Usage:     "Requests a node record using EIP-868 enrRequest",
 		Action:    discv4RequestRecord,
 		ArgsUsage: "<node>",
+		Flags:     discoveryNodeFlags,
 	}
-	discv4ResolveCommand = cli.Command{
+	discv4ResolveCommand = &cli.Command{
 		Name:      "resolve",
 		Usage:     "Finds a node in the DHT",
 		Action:    discv4Resolve,
 		ArgsUsage: "<node>",
-		Flags:     []cli.Flag{bootnodesFlag},
+		Flags:     discoveryNodeFlags,
 	}
-	discv4ResolveJSONCommand = cli.Command{
+	discv4ResolveJSONCommand = &cli.Command{
 		Name:      "resolve-json",
 		Usage:     "Re-resolves nodes in a nodes.json file",
 		Action:    discv4ResolveJSON,
-		Flags:     []cli.Flag{bootnodesFlag},
+		Flags:     discoveryNodeFlags,
 		ArgsUsage: "<nodes.json file>",
 	}
-	discv4CrawlCommand = cli.Command{
+	discv4CrawlCommand = &cli.Command{
 		Name:   "crawl",
 		Usage:  "Updates a nodes.json file with random nodes found in the DHT",
 		Action: discv4Crawl,
-		Flags:  []cli.Flag{bootnodesFlag, crawlTimeoutFlag},
+		Flags:  flags.Merge(discoveryNodeFlags, []cli.Flag{crawlTimeoutFlag, crawlParallelismFlag}),
 	}
-	discv4TestCommand = cli.Command{
+	discv4TestCommand = &cli.Command{
 		Name:   "test",
 		Usage:  "Runs tests against a node",
 		Action: discv4Test,
@@ -91,34 +95,51 @@ var (
 )
 
 var (
-	bootnodesFlag = cli.StringFlag{
+	bootnodesFlag = &cli.StringFlag{
 		Name:  "bootnodes",
 		Usage: "Comma separated nodes used for bootstrapping",
 	}
-	nodekeyFlag = cli.StringFlag{
+	nodekeyFlag = &cli.StringFlag{
 		Name:  "nodekey",
 		Usage: "Hex-encoded node key",
 	}
-	nodedbFlag = cli.StringFlag{
+	nodedbFlag = &cli.StringFlag{
 		Name:  "nodedb",
 		Usage: "Nodes database location",
 	}
-	listenAddrFlag = cli.StringFlag{
+	listenAddrFlag = &cli.StringFlag{
 		Name:  "addr",
 		Usage: "Listening address",
 	}
-	crawlTimeoutFlag = cli.DurationFlag{
+	extAddrFlag = &cli.StringFlag{
+		Name:  "extaddr",
+		Usage: "UDP endpoint announced in ENR. You can provide a bare IP address or IP:port as the value of this flag.",
+	}
+	crawlTimeoutFlag = &cli.DurationFlag{
 		Name:  "timeout",
 		Usage: "Time limit for the crawl.",
 		Value: 30 * time.Minute,
 	}
-	remoteEnodeFlag = cli.StringFlag{
-		Name:   "remote",
-		Usage:  "Enode of the remote node under test",
-		EnvVar: "REMOTE_ENODE",
+	crawlParallelismFlag = &cli.IntFlag{
+		Name:  "parallel",
+		Usage: "How many parallel discoveries to attempt.",
+		Value: 16,
+	}
+	remoteEnodeFlag = &cli.StringFlag{
+		Name:    "remote",
+		Usage:   "Enode of the remote node under test",
+		EnvVars: []string{"REMOTE_ENODE"},
 	}
 )
 
+var discoveryNodeFlags = []cli.Flag{
+	bootnodesFlag,
+	nodekeyFlag,
+	nodedbFlag,
+	listenAddrFlag,
+	extAddrFlag,
+}
+
 func discv4Ping(ctx *cli.Context) error {
 	n := getNodeArg(ctx)
 	disc := startV4(ctx)
@@ -179,7 +200,7 @@ func discv4ResolveJSON(ctx *cli.Context) error {
 	defer disc.Close()
 	c := newCrawler(inputSet, disc, enode.IterNodes(nodeargs))
 	c.revalidateInterval = 0
-	output := c.run(0)
+	output := c.run(0, 1)
 	writeNodesJSON(nodesFile, output)
 	return nil
 }
@@ -198,7 +219,7 @@ func discv4Crawl(ctx *cli.Context) error {
 	defer disc.Close()
 	c := newCrawler(inputSet, disc, disc.RandomNodes())
 	c.revalidateInterval = 10 * time.Minute
-	output := c.run(ctx.Duration(crawlTimeoutFlag.Name))
+	output := c.run(ctx.Duration(crawlTimeoutFlag.Name), ctx.Int(crawlParallelismFlag.Name))
 	writeNodesJSON(nodesFile, output)
 	return nil
 }
@@ -218,7 +239,7 @@ func discv4Test(ctx *cli.Context) error {
 // startV4 starts an ephemeral discovery V4 node.
 func startV4(ctx *cli.Context) *discover.UDPv4 {
 	ln, config := makeDiscoveryConfig(ctx)
-	socket := listen(ln, ctx.String(listenAddrFlag.Name))
+	socket := listen(ctx, ln)
 	disc, err := discover.ListenV4(socket, ln, config)
 	if err != nil {
 		exit(err)
@@ -256,7 +277,28 @@ func makeDiscoveryConfig(ctx *cli.Context) (*enode.LocalNode, discover.Config) {
 	return ln, cfg
 }
 
-func listen(ln *enode.LocalNode, addr string) *net.UDPConn {
+func parseExtAddr(spec string) (ip net.IP, port int, ok bool) {
+	ip = net.ParseIP(spec)
+	if ip != nil {
+		return ip, 0, true
+	}
+	host, portstr, err := net.SplitHostPort(spec)
+	if err != nil {
+		return nil, 0, false
+	}
+	ip = net.ParseIP(host)
+	if ip == nil {
+		return nil, 0, false
+	}
+	port, err = strconv.Atoi(portstr)
+	if err != nil {
+		return nil, 0, false
+	}
+	return ip, port, true
+}
+
+func listen(ctx *cli.Context, ln *enode.LocalNode) *net.UDPConn {
+	addr := ctx.String(listenAddrFlag.Name)
 	if addr == "" {
 		addr = "0.0.0.0:0"
 	}
@@ -264,6 +306,8 @@ func listen(ln *enode.LocalNode, addr string) *net.UDPConn {
 	if err != nil {
 		exit(err)
 	}
+
+	// Configure UDP endpoint in ENR from listener address.
 	usocket := socket.(*net.UDPConn)
 	uaddr := socket.LocalAddr().(*net.UDPAddr)
 	if uaddr.IP.IsUnspecified() {
@@ -272,6 +316,22 @@ func listen(ln *enode.LocalNode, addr string) *net.UDPConn {
 		ln.SetFallbackIP(uaddr.IP)
 	}
 	ln.SetFallbackUDP(uaddr.Port)
+
+	// If an ENR endpoint is set explicitly on the command-line, override
+	// the information from the listening address. Note this is careful not
+	// to set the UDP port if the external address doesn't have it.
+	extAddr := ctx.String(extAddrFlag.Name)
+	if extAddr != "" {
+		ip, port, ok := parseExtAddr(extAddr)
+		if !ok {
+			exit(fmt.Errorf("-%s: invalid external address %q", extAddrFlag.Name, extAddr))
+		}
+		ln.SetStaticIP(ip)
+		if port != 0 {
+			ln.SetFallbackUDP(port)
+		}
+	}
+
 	return usocket
 }
 

cmd/devp2p/discv5cmd.go

@@ -22,15 +22,16 @@ import (
 
 	"github.com/ethereum/go-ethereum/cmd/devp2p/internal/v5test"
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/internal/flags"
 	"github.com/ethereum/go-ethereum/p2p/discover"
-	"gopkg.in/urfave/cli.v1"
+	"github.com/urfave/cli/v2"
 )
 
 var (
-	discv5Command = cli.Command{
+	discv5Command = &cli.Command{
 		Name:  "discv5",
 		Usage: "Node Discovery v5 tools",
-		Subcommands: []cli.Command{
+		Subcommands: []*cli.Command{
 			discv5PingCommand,
 			discv5ResolveCommand,
 			discv5CrawlCommand,
@@ -38,24 +39,27 @@ var (
 			discv5ListenCommand,
 		},
 	}
-	discv5PingCommand = cli.Command{
+	discv5PingCommand = &cli.Command{
 		Name:   "ping",
 		Usage:  "Sends ping to a node",
 		Action: discv5Ping,
+		Flags:  discoveryNodeFlags,
 	}
-	discv5ResolveCommand = cli.Command{
+	discv5ResolveCommand = &cli.Command{
 		Name:   "resolve",
 		Usage:  "Finds a node in the DHT",
 		Action: discv5Resolve,
-		Flags:  []cli.Flag{bootnodesFlag},
+		Flags:  discoveryNodeFlags,
 	}
-	discv5CrawlCommand = cli.Command{
+	discv5CrawlCommand = &cli.Command{
 		Name:   "crawl",
 		Usage:  "Updates a nodes.json file with random nodes found in the DHT",
 		Action: discv5Crawl,
-		Flags:  []cli.Flag{bootnodesFlag, crawlTimeoutFlag},
+		Flags: flags.Merge(discoveryNodeFlags, []cli.Flag{
+			crawlTimeoutFlag,
+		}),
 	}
-	discv5TestCommand = cli.Command{
+	discv5TestCommand = &cli.Command{
 		Name:   "test",
 		Usage:  "Runs protocol tests against a node",
 		Action: discv5Test,
@@ -66,16 +70,11 @@ var (
 			testListen2Flag,
 		},
 	}
-	discv5ListenCommand = cli.Command{
+	discv5ListenCommand = &cli.Command{
 		Name:   "listen",
 		Usage:  "Runs a node",
 		Action: discv5Listen,
-		Flags: []cli.Flag{
-			bootnodesFlag,
-			nodekeyFlag,
-			nodedbFlag,
-			listenAddrFlag,
-		},
+		Flags:  discoveryNodeFlags,
 	}
 )
 
@@ -111,7 +110,7 @@ func discv5Crawl(ctx *cli.Context) error {
 	defer disc.Close()
 	c := newCrawler(inputSet, disc, disc.RandomNodes())
 	c.revalidateInterval = 10 * time.Minute
-	output := c.run(ctx.Duration(crawlTimeoutFlag.Name))
+	output := c.run(ctx.Duration(crawlTimeoutFlag.Name), ctx.Int(crawlParallelismFlag.Name))
 	writeNodesJSON(nodesFile, output)
 	return nil
 }
@@ -137,7 +136,7 @@ func discv5Listen(ctx *cli.Context) error {
 // startV5 starts an ephemeral discovery v5 node.
 func startV5(ctx *cli.Context) *discover.UDPv5 {
 	ln, config := makeDiscoveryConfig(ctx)
-	socket := listen(ln, ctx.String(listenAddrFlag.Name))
+	socket := listen(ctx, ln)
 	disc, err := discover.ListenV5(socket, ln, config)
 	if err != nil {
 		exit(err)

cmd/devp2p/dns_cloudflare.go

@@ -24,16 +24,16 @@ import (
 	"github.com/cloudflare/cloudflare-go"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/p2p/dnsdisc"
-	"gopkg.in/urfave/cli.v1"
+	"github.com/urfave/cli/v2"
 )
 
 var (
-	cloudflareTokenFlag = cli.StringFlag{
-		Name:   "token",
-		Usage:  "CloudFlare API token",
-		EnvVar: "CLOUDFLARE_API_TOKEN",
+	cloudflareTokenFlag = &cli.StringFlag{
+		Name:    "token",
+		Usage:   "CloudFlare API token",
+		EnvVars: []string{"CLOUDFLARE_API_TOKEN"},
 	}
-	cloudflareZoneIDFlag = cli.StringFlag{
+	cloudflareZoneIDFlag = &cli.StringFlag{
 		Name:  "zoneid",
 		Usage: "CloudFlare Zone ID (optional)",
 	}
@@ -126,41 +126,51 @@ func (c *cloudflareClient) uploadRecords(name string, records map[string]string)
 	}
 
 	// Iterate over the new records and inject anything missing.
+	log.Info("Updating DNS entries")
+	created := 0
+	updated := 0
+	skipped := 0
 	for path, val := range records {
 		old, exists := existing[path]
 		if !exists {
 			// Entry is unknown, push a new one to Cloudflare.
-			log.Info(fmt.Sprintf("Creating %s = %q", path, val))
+			log.Debug(fmt.Sprintf("Creating %s = %q", path, val))
+			created++
 			ttl := rootTTL
 			if path != name {
 				ttl = treeNodeTTLCloudflare // Max TTL permitted by Cloudflare
-
 			}
 			record := cloudflare.DNSRecord{Type: "TXT", Name: path, Content: val, TTL: ttl}
 			_, err = c.CreateDNSRecord(context.Background(), c.zoneID, record)
 		} else if old.Content != val {
 			// Entry already exists, only change its content.
 			log.Info(fmt.Sprintf("Updating %s from %q to %q", path, old.Content, val))
+			updated++
 			old.Content = val
 			err = c.UpdateDNSRecord(context.Background(), c.zoneID, old.ID, old)
 		} else {
+			skipped++
 			log.Debug(fmt.Sprintf("Skipping %s = %q", path, val))
 		}
 		if err != nil {
 			return fmt.Errorf("failed to publish %s: %v", path, err)
 		}
 	}
-
+	log.Info("Updated DNS entries", "new", created, "updated", updated, "untouched", skipped)
 	// Iterate over the old records and delete anything stale.
+	deleted := 0
+	log.Info("Deleting stale DNS entries")
 	for path, entry := range existing {
 		if _, ok := records[path]; ok {
 			continue
 		}
 		// Stale entry, nuke it.
-		log.Info(fmt.Sprintf("Deleting %s = %q", path, entry.Content))
+		log.Debug(fmt.Sprintf("Deleting %s = %q", path, entry.Content))
+		deleted++
 		if err := c.DeleteDNSRecord(context.Background(), c.zoneID, entry.ID); err != nil {
 			return fmt.Errorf("failed to delete %s: %v", path, err)
 		}
 	}
+	log.Info("Deleted stale DNS entries", "count", deleted)
 	return nil
 }

cmd/devp2p/dns_route53.go

@@ -32,7 +32,7 @@ import (
 	"github.com/aws/aws-sdk-go-v2/service/route53/types"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/p2p/dnsdisc"
-	"gopkg.in/urfave/cli.v1"
+	"github.com/urfave/cli/v2"
 )
 
 const (
@@ -45,21 +45,21 @@ const (
 )
 
 var (
-	route53AccessKeyFlag = cli.StringFlag{
-		Name:   "access-key-id",
-		Usage:  "AWS Access Key ID",
-		EnvVar: "AWS_ACCESS_KEY_ID",
+	route53AccessKeyFlag = &cli.StringFlag{
+		Name:    "access-key-id",
+		Usage:   "AWS Access Key ID",
+		EnvVars: []string{"AWS_ACCESS_KEY_ID"},
 	}
-	route53AccessSecretFlag = cli.StringFlag{
-		Name:   "access-key-secret",
-		Usage:  "AWS Access Key Secret",
-		EnvVar: "AWS_SECRET_ACCESS_KEY",
+	route53AccessSecretFlag = &cli.StringFlag{
+		Name:    "access-key-secret",
+		Usage:   "AWS Access Key Secret",
+		EnvVars: []string{"AWS_SECRET_ACCESS_KEY"},
 	}
-	route53ZoneIDFlag = cli.StringFlag{
+	route53ZoneIDFlag = &cli.StringFlag{
 		Name:  "zone-id",
 		Usage: "Route53 Zone ID",
 	}
-	route53RegionFlag = cli.StringFlag{
+	route53RegionFlag = &cli.StringFlag{
 		Name:  "aws-region",
 		Usage: "AWS Region",
 		Value: "eu-central-1",
@@ -221,7 +221,13 @@ func (c *route53Client) computeChanges(name string, records map[string]string, e
 	}
 	records = lrecords
 
-	var changes []types.Change
+	var (
+		changes []types.Change
+		inserts int
+		upserts int
+		skips   int
+	)
+
 	for path, newValue := range records {
 		prevRecords, exists := existing[path]
 		prevValue := strings.Join(prevRecords.values, "")
@@ -237,20 +243,30 @@ func (c *route53Client) computeChanges(name string, records map[string]string, e
 
 		if !exists {
 			// Entry is unknown, push a new one
-			log.Info(fmt.Sprintf("Creating %s = %s", path, newValue))
+			log.Debug(fmt.Sprintf("Creating %s = %s", path, newValue))
 			changes = append(changes, newTXTChange("CREATE", path, ttl, newValue))
+			inserts++
 		} else if prevValue != newValue || prevRecords.ttl != ttl {
 			// Entry already exists, only change its content.
 			log.Info(fmt.Sprintf("Updating %s from %s to %s", path, prevValue, newValue))
 			changes = append(changes, newTXTChange("UPSERT", path, ttl, newValue))
+			upserts++
 		} else {
 			log.Debug(fmt.Sprintf("Skipping %s = %s", path, newValue))
+			skips++
 		}
 	}
 
 	// Iterate over the old records and delete anything stale.
-	changes = append(changes, makeDeletionChanges(existing, records)...)
+	deletions := makeDeletionChanges(existing, records)
+	changes = append(changes, deletions...)
 
+	log.Info("Computed DNS changes",
+		"changes", len(changes),
+		"inserts", inserts,
+		"skips", skips,
+		"deleted", len(deletions),
+		"upserts", upserts)
 	// Ensure changes are in the correct order.
 	sortChanges(changes)
 	return changes
@@ -263,7 +279,7 @@ func makeDeletionChanges(records map[string]recordSet, keep map[string]string) [
 		if _, ok := keep[path]; ok {
 			continue
 		}
-		log.Info(fmt.Sprintf("Deleting %s = %s", path, strings.Join(set.values, "")))
+		log.Debug(fmt.Sprintf("Deleting %s = %s", path, strings.Join(set.values, "")))
 		changes = append(changes, newTXTChange("DELETE", path, set.ttl, set.values...))
 	}
 	return changes
@@ -329,8 +345,9 @@ func (c *route53Client) collectRecords(name string) (map[string]recordSet, error
 	var req route53.ListResourceRecordSetsInput
 	req.HostedZoneId = &c.zoneID
 	existing := make(map[string]recordSet)
+	log.Info("Loading existing TXT records", "name", name, "zone", c.zoneID)
 	for page := 0; ; page++ {
-		log.Info("Loading existing TXT records", "name", name, "zone", c.zoneID, "page", page)
+		log.Debug("Loading existing TXT records", "name", name, "zone", c.zoneID, "page", page)
 		resp, err := c.api.ListResourceRecordSets(context.TODO(), &req)
 		if err != nil {
 			return existing, err
@@ -360,7 +377,7 @@ func (c *route53Client) collectRecords(name string) (map[string]recordSet, error
 		req.StartRecordName = resp.NextRecordName
 		req.StartRecordType = resp.NextRecordType
 	}
-
+	log.Info("Loaded existing TXT records", "name", name, "zone", c.zoneID, "records", len(existing))
 	return existing, nil
 }
 

cmd/devp2p/dnscmd.go

@@ -29,14 +29,14 @@ import (
 	"github.com/ethereum/go-ethereum/console/prompt"
 	"github.com/ethereum/go-ethereum/p2p/dnsdisc"
 	"github.com/ethereum/go-ethereum/p2p/enode"
-	"gopkg.in/urfave/cli.v1"
+	"github.com/urfave/cli/v2"
 )
 
 var (
-	dnsCommand = cli.Command{
+	dnsCommand = &cli.Command{
 		Name:  "dns",
 		Usage: "DNS Discovery Commands",
-		Subcommands: []cli.Command{
+		Subcommands: []*cli.Command{
 			dnsSyncCommand,
 			dnsSignCommand,
 			dnsTXTCommand,
@@ -45,34 +45,34 @@ var (
 			dnsRoute53NukeCommand,
 		},
 	}
-	dnsSyncCommand = cli.Command{
+	dnsSyncCommand = &cli.Command{
 		Name:      "sync",
 		Usage:     "Download a DNS discovery tree",
 		ArgsUsage: "<url> [ <directory> ]",
 		Action:    dnsSync,
 		Flags:     []cli.Flag{dnsTimeoutFlag},
 	}
-	dnsSignCommand = cli.Command{
+	dnsSignCommand = &cli.Command{
 		Name:      "sign",
 		Usage:     "Sign a DNS discovery tree",
 		ArgsUsage: "<tree-directory> <key-file>",
 		Action:    dnsSign,
 		Flags:     []cli.Flag{dnsDomainFlag, dnsSeqFlag},
 	}
-	dnsTXTCommand = cli.Command{
+	dnsTXTCommand = &cli.Command{
 		Name:      "to-txt",
 		Usage:     "Create a DNS TXT records for a discovery tree",
 		ArgsUsage: "<tree-directory> <output-file>",
 		Action:    dnsToTXT,
 	}
-	dnsCloudflareCommand = cli.Command{
+	dnsCloudflareCommand = &cli.Command{
 		Name:      "to-cloudflare",
 		Usage:     "Deploy DNS TXT records to CloudFlare",
 		ArgsUsage: "<tree-directory>",
 		Action:    dnsToCloudflare,
 		Flags:     []cli.Flag{cloudflareTokenFlag, cloudflareZoneIDFlag},
 	}
-	dnsRoute53Command = cli.Command{
+	dnsRoute53Command = &cli.Command{
 		Name:      "to-route53",
 		Usage:     "Deploy DNS TXT records to Amazon Route53",
 		ArgsUsage: "<tree-directory>",
@@ -84,7 +84,7 @@ var (
 			route53RegionFlag,
 		},
 	}
-	dnsRoute53NukeCommand = cli.Command{
+	dnsRoute53NukeCommand = &cli.Command{
 		Name:      "nuke-route53",
 		Usage:     "Deletes DNS TXT records of a subdomain on Amazon Route53",
 		ArgsUsage: "<domain>",
@@ -99,15 +99,15 @@ var (
 )
 
 var (
-	dnsTimeoutFlag = cli.DurationFlag{
+	dnsTimeoutFlag = &cli.DurationFlag{
 		Name:  "timeout",
 		Usage: "Timeout for DNS lookups",
 	}
-	dnsDomainFlag = cli.StringFlag{
+	dnsDomainFlag = &cli.StringFlag{
 		Name:  "domain",
 		Usage: "Domain name of the tree",
 	}
-	dnsSeqFlag = cli.UintFlag{
+	dnsSeqFlag = &cli.UintFlag{
 		Name:  "seq",
 		Usage: "New sequence number of the tree",
 	}

cmd/devp2p/enrcmd.go

@@ -30,12 +30,12 @@ import (
 	"github.com/ethereum/go-ethereum/p2p/enode"
 	"github.com/ethereum/go-ethereum/p2p/enr"
 	"github.com/ethereum/go-ethereum/rlp"
-	"gopkg.in/urfave/cli.v1"
+	"github.com/urfave/cli/v2"
 )
 
-var fileFlag = cli.StringFlag{Name: "file"}
+var fileFlag = &cli.StringFlag{Name: "file"}
 
-var enrdumpCommand = cli.Command{
+var enrdumpCommand = &cli.Command{
 	Name:   "enrdump",
 	Usage:  "Pretty-prints node records",
 	Action: enrdump,
@@ -62,7 +62,7 @@ func enrdump(ctx *cli.Context) error {
 		}
 		source = string(b)
 	} else if ctx.NArg() == 1 {
-		source = ctx.Args()[0]
+		source = ctx.Args().First()
 	} else {
 		return fmt.Errorf("need record as argument")
 	}

cmd/devp2p/internal/ethtest/chain.go

@@ -76,7 +76,7 @@ func (c *Chain) RootAt(height int) common.Hash {
 
 // ForkID gets the fork id of the chain.
 func (c *Chain) ForkID() forkid.ID {
-	return forkid.NewID(c.chainConfig, c.blocks[0].Hash(), uint64(c.Len()))
+	return forkid.NewID(c.chainConfig, c.blocks[0].Hash(), uint64(c.Len()), c.blocks[0].Time())
 }
 
 // Shorten returns a copy chain of a desired height from the imported
@@ -96,12 +96,12 @@ func (c *Chain) Head() *types.Block {
 	return c.blocks[c.Len()-1]
 }
 
-func (c *Chain) GetHeaders(req GetBlockHeaders) (BlockHeaders, error) {
+func (c *Chain) GetHeaders(req *GetBlockHeaders) ([]*types.Header, error) {
 	if req.Amount < 1 {
 		return nil, fmt.Errorf("no block headers requested")
 	}
 
-	headers := make(BlockHeaders, req.Amount)
+	headers := make([]*types.Header, req.Amount)
 	var blockNumber uint64
 
 	// range over blocks to check if our chain has the requested header
@@ -119,7 +119,6 @@ func (c *Chain) GetHeaders(req GetBlockHeaders) (BlockHeaders, error) {
 		for i := 1; i < int(req.Amount); i++ {
 			blockNumber -= (1 - req.Skip)
 			headers[i] = c.blocks[blockNumber].Header()
-
 		}
 
 		return headers, nil
@@ -140,7 +139,7 @@ func loadChain(chainfile string, genesis string) (*Chain, error) {
 	if err != nil {
 		return nil, err
 	}
-	gblock := gen.ToBlock(nil)
+	gblock := gen.ToBlock()
 
 	blocks, err := blocksFromFile(chainfile, gblock)
 	if err != nil {