Merge pull request #10003 from filecoin-project/deprecate-snapcraft-appimage

_build_: Remove AppImage and Snapcraft build automation

.circleci/config.yml

@@ -355,56 +355,6 @@ jobs:
             - run: ./scripts/generate-checksums.sh
             - run: ./scripts/publish-checksums.sh
 
-  build-appimage:
-    machine:
-      image: ubuntu-2004:202111-02
-    steps:
-      - checkout
-      - attach_workspace:
-          at: /tmp/workspace
-      - run:
-          name: Update Go
-          command: |
-            sudo rm -rf /usr/local/go && \
-            curl -L https://golang.org/dl/go`cat GO_VERSION_MIN`.linux-amd64.tar.gz -o /tmp/go.tar.gz && \
-            sudo tar -C /usr/local -xvf /tmp/go.tar.gz
-      - run: go version
-      - run:
-          name: install appimage-builder
-          command: |
-            # appimage-builder requires /dev/snd to exist. It creates containers during the testing phase
-            # that pass sound devices from the host to the testing container. (hard coded!)
-            # https://github.com/AppImageCrafters/appimage-builder/blob/master/appimagebuilder/modules/test/execution_test.py#L54
-            # Circleci doesn't provide a working sound device; this is enough to fake it.
-            if [ ! -e /dev/snd ]
-            then
-              sudo mkdir /dev/snd
-              sudo mknod /dev/snd/ControlC0 c 1 2
-            fi
-
-            # docs: https://appimage-builder.readthedocs.io/en/latest/intro/install.html
-            sudo apt update
-            sudo apt install -y python3-pip python3-setuptools patchelf desktop-file-utils libgdk-pixbuf2.0-dev fakeroot strace
-            sudo curl -Lo /usr/local/bin/appimagetool https://github.com/AppImage/AppImageKit/releases/download/continuous/appimagetool-x86_64.AppImage
-            sudo chmod +x /usr/local/bin/appimagetool
-            sudo pip3 install appimage-builder
-      - run:
-          name: install lotus dependencies
-          command: sudo apt install ocl-icd-opencl-dev libhwloc-dev
-      - run:
-          name: build appimage
-          command: |
-            sed -i "s/version: latest/version: ${CIRCLE_TAG:-latest}/" AppImageBuilder.yml
-            make appimage
-      - run: |
-            mkdir -p /tmp/workspace/appimage && \
-            mv Lotus-*.AppImage /tmp/workspace/appimage/
-      - persist_to_workspace:
-          root: /tmp/workspace
-          paths:
-            - appimage
-
-
   gofmt:
     executor: golang
     working_directory: ~/lotus
@@ -467,71 +417,6 @@ jobs:
             golangci-lint run -v --timeout 10m \
               --concurrency 4 << parameters.args >>
 
-  publish:
-    description: publish binary artifacts
-    executor: ubuntu
-    parameters:
-      linux:
-        default: false
-        description: publish linux binaries?
-        type: boolean
-      appimage:
-        default: false
-        description: publish appimage binaries?
-        type: boolean
-    steps:
-      - run:
-          name: Install git jq curl
-          command: apt update && apt install -y git jq curl sudo
-      - checkout
-      - git_fetch_all_tags
-      - install_ipfs
-      - attach_workspace:
-          at: /tmp/workspace
-      - when:
-          condition: << parameters.linux >>
-          steps:
-            - run: ./scripts/build-arch-bundle.sh linux
-            - run: ./scripts/publish-arch-release.sh linux
-      - when:
-          condition: << parameters.appimage >>
-          steps:
-            - run: ./scripts/build-appimage-bundle.sh
-            - run: ./scripts/publish-arch-release.sh appimage
-
-  publish-snapcraft:
-    description: build and push snapcraft
-    machine:
-      image: ubuntu-2004:202104-01
-    resource_class: 2xlarge
-    parameters:
-      channel:
-        type: string
-        default: "edge"
-        description: snapcraft channel
-      snap-name:
-        type: string
-        default: 'lotus-filecoin'
-        description: name of snap in snap store
-    steps:
-      - checkout
-      - run:
-          name: Install snapcraft
-          command: sudo snap install snapcraft --classic
-      - run:
-          name: Build << parameters.snap-name >> snap
-          command: |
-            if [ "<< parameters.snap-name >>" != 'lotus-filecoin' ]; then
-              cat snap/snapcraft.yaml | sed 's/lotus-filecoin/lotus/' > edited-snapcraft.yaml
-              mv edited-snapcraft.yaml snap/snapcraft.yaml
-            fi
-
-            snapcraft --use-lxd --debug
-      - run:
-          name: Publish snap to << parameters.channel >> channel
-          shell: /bin/bash -o pipefail
-          command: |
-            snapcraft upload *.snap --release << parameters.channel >>
   build-docker:
     description: >
       Publish to Dockerhub
@@ -1128,71 +1013,6 @@ workflows:
             branches:
               only:
                 - /^release\/v\d+\.\d+\.\d+(-rc\d+)?$/
-      - build-appimage:
-          name: "Build AppImage"
-          filters:
-            branches:
-              only:
-                - /^release\/v\d+\.\d+\.\d+(-rc\d+)?$/
-            tags:
-              only:
-                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
-      - publish:
-          name: "Publish AppImage"
-          appimage: true
-          requires:
-            - "Build AppImage"
-          filters:
-            branches:
-              ignore:
-                - /.*/
-            tags:
-              only:
-                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
-      - publish-snapcraft:
-          name: "Publish Snapcraft (lotus / stable)"
-          channel: stable
-          snap-name: lotus
-          filters:
-            branches:
-              ignore:
-                - /.*/
-            tags:
-              only:
-                - /^v\d+\.\d+\.\d+$/
-      - publish-snapcraft:
-          name: "Publish Snapcraft (lotus / candidate)"
-          channel: candidate
-          snap-name: lotus
-          filters:
-            branches:
-              ignore:
-                - /.*/
-            tags:
-              only:
-                - /^v\d+\.\d+\.\d+-rc\d+$/
-      - publish-snapcraft:
-          name: "Publish Snapcraft (lotus-filecoin / stable)"
-          channel: stable
-          snap-name: lotus-filecoin
-          filters:
-            branches:
-              ignore:
-                - /.*/
-            tags:
-              only:
-                - /^v\d+\.\d+\.\d+$/
-      - publish-snapcraft:
-          name: "Publish Snapcraft (lotus-filecoin / candidate)"
-          channel: candidate
-          snap-name: lotus-filecoin
-          filters:
-            branches:
-              ignore:
-                - /.*/
-            tags:
-              only:
-                - /^v\d+\.\d+\.\d+-rc\d+$/
       - build-docker:
           name: "Docker push (lotus-all-in-one / stable / mainnet)"
           image: lotus-all-in-one
@@ -1428,14 +1248,6 @@ workflows:
               only:
                 - master
     jobs:
-      - publish-snapcraft:
-          name: "Publish Snapcraft (lotus / edge)"
-          channel: edge
-          snap-name: lotus
-      - publish-snapcraft:
-          name: "Publish Snapcraft (lotus-filecoin / edge)"
-          channel: edge
-          snap-name: lotus-filecoin
       - build-docker:
           name: "Docker (lotus-all-in-one / nightly / mainnet)"
           image: lotus-all-in-one

.circleci/gen.go

@@ -107,13 +107,11 @@ func main() {
 	// form the input data.
 	type data struct {
 		Networks   []string
-		SnapNames  []string
 		ItestFiles []string
 		UnitSuites map[string]string
 	}
 	in := data{
 		Networks:   []string{"mainnet", "butterflynet", "calibnet", "debug"},
-		SnapNames:  []string{"lotus", "lotus-filecoin"},
 		ItestFiles: itests,
 		UnitSuites: func() map[string]string {
 			ret := make(map[string]string)

.circleci/template.yml

@@ -355,56 +355,6 @@ jobs:
             - run: ./scripts/generate-checksums.sh
             - run: ./scripts/publish-checksums.sh
 
-  build-appimage:
-    machine:
-      image: ubuntu-2004:202111-02
-    steps:
-      - checkout
-      - attach_workspace:
-          at: /tmp/workspace
-      - run:
-          name: Update Go
-          command: |
-            sudo rm -rf /usr/local/go && \
-            curl -L https://golang.org/dl/go`cat GO_VERSION_MIN`.linux-amd64.tar.gz -o /tmp/go.tar.gz && \
-            sudo tar -C /usr/local -xvf /tmp/go.tar.gz
-      - run: go version
-      - run:
-          name: install appimage-builder
-          command: |
-            # appimage-builder requires /dev/snd to exist. It creates containers during the testing phase
-            # that pass sound devices from the host to the testing container. (hard coded!)
-            # https://github.com/AppImageCrafters/appimage-builder/blob/master/appimagebuilder/modules/test/execution_test.py#L54
-            # Circleci doesn't provide a working sound device; this is enough to fake it.
-            if [ ! -e /dev/snd ]
-            then
-              sudo mkdir /dev/snd
-              sudo mknod /dev/snd/ControlC0 c 1 2
-            fi
-
-            # docs: https://appimage-builder.readthedocs.io/en/latest/intro/install.html
-            sudo apt update
-            sudo apt install -y python3-pip python3-setuptools patchelf desktop-file-utils libgdk-pixbuf2.0-dev fakeroot strace
-            sudo curl -Lo /usr/local/bin/appimagetool https://github.com/AppImage/AppImageKit/releases/download/continuous/appimagetool-x86_64.AppImage
-            sudo chmod +x /usr/local/bin/appimagetool
-            sudo pip3 install appimage-builder
-      - run:
-          name: install lotus dependencies
-          command: sudo apt install ocl-icd-opencl-dev libhwloc-dev
-      - run:
-          name: build appimage
-          command: |
-            sed -i "s/version: latest/version: ${CIRCLE_TAG:-latest}/" AppImageBuilder.yml
-            make appimage
-      - run: |
-            mkdir -p /tmp/workspace/appimage && \
-            mv Lotus-*.AppImage /tmp/workspace/appimage/
-      - persist_to_workspace:
-          root: /tmp/workspace
-          paths:
-            - appimage
-
-
   gofmt:
     executor: golang
     working_directory: ~/lotus
@@ -467,71 +417,6 @@ jobs:
             golangci-lint run -v --timeout 10m \
               --concurrency 4 << parameters.args >>
 
-  publish:
-    description: publish binary artifacts
-    executor: ubuntu
-    parameters:
-      linux:
-        default: false
-        description: publish linux binaries?
-        type: boolean
-      appimage:
-        default: false
-        description: publish appimage binaries?
-        type: boolean
-    steps:
-      - run:
-          name: Install git jq curl
-          command: apt update && apt install -y git jq curl sudo
-      - checkout
-      - git_fetch_all_tags
-      - install_ipfs
-      - attach_workspace:
-          at: /tmp/workspace
-      - when:
-          condition: << parameters.linux >>
-          steps:
-            - run: ./scripts/build-arch-bundle.sh linux
-            - run: ./scripts/publish-arch-release.sh linux
-      - when:
-          condition: << parameters.appimage >>
-          steps:
-            - run: ./scripts/build-appimage-bundle.sh
-            - run: ./scripts/publish-arch-release.sh appimage
-
-  publish-snapcraft:
-    description: build and push snapcraft
-    machine:
-      image: ubuntu-2004:202104-01
-    resource_class: 2xlarge
-    parameters:
-      channel:
-        type: string
-        default: "edge"
-        description: snapcraft channel
-      snap-name:
-        type: string
-        default: 'lotus-filecoin'
-        description: name of snap in snap store
-    steps:
-      - checkout
-      - run:
-          name: Install snapcraft
-          command: sudo snap install snapcraft --classic
-      - run:
-          name: Build << parameters.snap-name >> snap
-          command: |
-            if [ "<< parameters.snap-name >>" != 'lotus-filecoin' ]; then
-              cat snap/snapcraft.yaml | sed 's/lotus-filecoin/lotus/' > edited-snapcraft.yaml
-              mv edited-snapcraft.yaml snap/snapcraft.yaml
-            fi
-
-            snapcraft --use-lxd --debug
-      - run:
-          name: Publish snap to << parameters.channel >> channel
-          shell: /bin/bash -o pipefail
-          command: |
-            snapcraft upload *.snap --release << parameters.channel >>
   build-docker:
     description: >
       Publish to Dockerhub
@@ -743,51 +628,6 @@ workflows:
             branches:
               only:
                 - /^release\/v\d+\.\d+\.\d+(-rc\d+)?$/
-      - build-appimage:
-          name: "Build AppImage"
-          filters:
-            branches:
-              only:
-                - /^release\/v\d+\.\d+\.\d+(-rc\d+)?$/
-            tags:
-              only:
-                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
-      - publish:
-          name: "Publish AppImage"
-          appimage: true
-          requires:
-            - "Build AppImage"
-          filters:
-            branches:
-              ignore:
-                - /.*/
-            tags:
-              only:
-                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
-      [[- range .SnapNames]]
-      - publish-snapcraft:
-          name: "Publish Snapcraft ([[.]] / stable)"
-          channel: stable
-          snap-name: [[.]]
-          filters:
-            branches:
-              ignore:
-                - /.*/
-            tags:
-              only:
-                - /^v\d+\.\d+\.\d+$/
-      - publish-snapcraft:
-          name: "Publish Snapcraft ([[.]] / candidate)"
-          channel: candidate
-          snap-name: [[.]]
-          filters:
-            branches:
-              ignore:
-                - /.*/
-            tags:
-              only:
-                - /^v\d+\.\d+\.\d+-rc\d+$/
-      [[- end]]
       [[- range .Networks]]
       - build-docker:
           name: "Docker push (lotus-all-in-one / stable / [[.]])"
@@ -890,12 +730,6 @@ workflows:
               only:
                 - master
     jobs:
-      [[- range .SnapNames]]
-      - publish-snapcraft:
-          name: "Publish Snapcraft ([[.]] / edge)"
-          channel: edge
-          snap-name: [[.]]
-      [[- end]]
       [[- range .Networks]]
       - build-docker:
           name: "Docker (lotus-all-in-one / nightly / [[.]])"

scripts/build-appimage-bundle.sh

@@ -1,26 +0,0 @@
-#!/usr/bin/env bash
-set -ex
-
-REQUIRED=(
-    "ipfs"
-    "sha512sum"
-)
-for REQUIRE in "${REQUIRED[@]}"
-do
-    command -v "${REQUIRE}" >/dev/null 2>&1 || echo >&2 "'${REQUIRE}' must be installed"
-done
-
-mkdir bundle
-pushd bundle
-
-export IPFS_PATH=`mktemp -d`
-ipfs init
-ipfs daemon &
-PID="$!"
-trap "kill -9 ${PID}" EXIT
-sleep 30
-
-cp "/tmp/workspace/appimage/Lotus-${CIRCLE_TAG}-x86_64.AppImage" .
-sha512sum "Lotus-${CIRCLE_TAG}-x86_64.AppImage" > "Lotus-${CIRCLE_TAG}-x86_64.AppImage.sha512"
-ipfs add -q "Lotus-${CIRCLE_TAG}-x86_64.AppImage" > "Lotus-${CIRCLE_TAG}-x86_64.AppImage.cid"
-popd

scripts/build-arch-bundle.sh

@@ -1,46 +0,0 @@
-#!/usr/bin/env bash
-set -ex
-
-ARCH=$1
-
-REQUIRED=(
-    "ipfs"
-    "sha512sum"
-)
-for REQUIRE in "${REQUIRED[@]}"
-do
-    command -v "${REQUIRE}" >/dev/null 2>&1 || echo >&2 "'${REQUIRE}' must be installed"
-done
-
-mkdir bundle
-pushd bundle
-
-BINARIES=(
-    "lotus"
-    "lotus-miner"
-    "lotus-worker"
-)
-
-export IPFS_PATH=`mktemp -d`
-ipfs init
-ipfs daemon &
-PID="$!"
-trap "kill -9 ${PID}" EXIT
-sleep 30
-
-mkdir -p "${ARCH}/lotus"
-pushd "${ARCH}"
-for BINARY in "${BINARIES[@]}"
-do
-    cp "../../${ARCH}/${BINARY}" "lotus/"
-    chmod +x "lotus/${BINARY}"
-done
-
-tar -zcvf "../lotus_${CIRCLE_TAG}_${ARCH}-amd64.tar.gz" lotus
-popd
-rm -rf "${ARCH}"
-
-sha512sum "lotus_${CIRCLE_TAG}_${ARCH}-amd64.tar.gz" > "lotus_${CIRCLE_TAG}_${ARCH}-amd64.tar.gz.sha512"
-
-ipfs add -q "lotus_${CIRCLE_TAG}_${ARCH}-amd64.tar.gz" > "lotus_${CIRCLE_TAG}_${ARCH}-amd64.tar.gz.cid"
-popd

scripts/publish-arch-release.sh

@@ -1,121 +0,0 @@
-#!/usr/bin/env bash
-set -e
-
-ARCH=$1
-
-pushd bundle
-
-# make sure we have a token set, api requests won't work otherwise
-if [ -z "${GITHUB_TOKEN}" ]; then
-  echo "\${GITHUB_TOKEN} not set, publish failed"
-  exit 1
-fi
-
-REQUIRED=(
-    "jq"
-    "curl"
-)
-for REQUIRE in "${REQUIRED[@]}"
-do
-    command -v "${REQUIRE}" >/dev/null 2>&1 || echo >&2 "'${REQUIRE}' must be installed"
-done
-
-#see if the release already exists by tag
-RELEASE_RESPONSE=`
-  curl \
-    --header "Authorization: token ${GITHUB_TOKEN}" \
-    "https://api.github.com/repos/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}/releases/tags/${CIRCLE_TAG}"
-`
-RELEASE_ID=`echo "${RELEASE_RESPONSE}" | jq '.id'`
-
-if [ "${RELEASE_ID}" = "null" ]; then
-  echo "creating release"
-
-  COND_CREATE_DISCUSSION=""
-  PRERELEASE=true
-  if [[ ${CIRCLE_TAG} =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
-    COND_CREATE_DISCUSSION="\"discussion_category_name\": \"announcement\","
-    PRERELEASE=false
-  fi
-
-  RELEASE_DATA="{
-    \"tag_name\": \"${CIRCLE_TAG}\",
-    \"target_commitish\": \"${CIRCLE_SHA1}\",
-    ${COND_CREATE_DISCUSSION}
-    \"name\": \"${CIRCLE_TAG}\",
-    \"body\": \"\",
-    \"prerelease\": ${PRERELEASE}
-  }"
-
-  # create it if it doesn't exist yet
-  RELEASE_RESPONSE=`
-    curl \
-        --request POST \
-        --header "Authorization: token ${GITHUB_TOKEN}" \
-        --header "Content-Type: application/json" \
-        --data "${RELEASE_DATA}" \
-        "https://api.github.com/repos/$CIRCLE_PROJECT_USERNAME/${CIRCLE_PROJECT_REPONAME}/releases"
-  `
-else
-  echo "release already exists"
-fi
-
-RELEASE_UPLOAD_URL=`echo "${RELEASE_RESPONSE}" | jq -r '.upload_url' | cut -d'{' -f1`
-echo "Preparing to send artifacts to ${RELEASE_UPLOAD_URL}"
-
-if [ $ARCH = 'linux' ]; then
-artifacts=(
-  "lotus_${CIRCLE_TAG}_linux-amd64.tar.gz"
-  "lotus_${CIRCLE_TAG}_linux-amd64.tar.gz.cid"
-  "lotus_${CIRCLE_TAG}_linux-amd64.tar.gz.sha512"
-)
-elif [ $ARCH = 'darwin' ]; then
-artifacts=(
-  "lotus_${CIRCLE_TAG}_darwin-amd64.tar.gz"
-  "lotus_${CIRCLE_TAG}_darwin-amd64.tar.gz.cid"
-  "lotus_${CIRCLE_TAG}_darwin-amd64.tar.gz.sha512"
-)
-elif [ $ARCH = 'appimage' ]; then
-artifacts=(
-  "Lotus-${CIRCLE_TAG}-x86_64.AppImage"
-  "Lotus-${CIRCLE_TAG}-x86_64.AppImage.cid"
-  "Lotus-${CIRCLE_TAG}-x86_64.AppImage.sha512"
-)
-else
-  echo "$1 is not a supported architecture to publish a release for" 1>&2
-  exit 1
-fi
-
-for RELEASE_FILE in "${artifacts[@]}"
-do
-  echo "Uploading ${RELEASE_FILE}..."
-  curl \
-    --request POST \
-    --fail \
-    --header "Authorization: token ${GITHUB_TOKEN}" \
-    --header "Content-Type: application/octet-stream" \
-    --data-binary "@${RELEASE_FILE}" \
-    "$RELEASE_UPLOAD_URL?name=$(basename "${RELEASE_FILE}")"
-
-  echo "Uploaded ${RELEASE_FILE}"
-done
-
-popd
-
-miscellaneous=(
-  "README.md"
-  "LICENSE-MIT"
-  "LICENSE-APACHE"
-)
-for MISC in "${miscellaneous[@]}"
-do
-  echo "Uploading release bundle: ${MISC}"
-  curl \
-    --request POST \
-    --header "Authorization: token ${GITHUB_TOKEN}" \
-    --header "Content-Type: application/octet-stream" \
-    --data-binary "@${MISC}" \
-    "$RELEASE_UPLOAD_URL?name=$(basename "${MISC}")"
-
-  echo "Release bundle uploaded: ${MISC}"
-done