Produce DEB files for amd64 for openCL and cuda (#11885)
* service changes * better postinst * better postinst * fix: curio: Update pgx imports, fix db_storage alloc * feat: curioweb: Improve task_history indexes (#11911) * ready to test deb making * deb files build * it works * fullname * last bit of cleanup * doc update * future plans * lint * code comments --------- Co-authored-by: Łukasz Magiera <magik6k@gmail.com> Co-authored-by: Łukasz Magiera <magik6k@users.noreply.github.com>
This commit is contained in:
Andrew Jackson (Ajax)
GitHub
parent
7e005be751
commit
f5dd7665f6
@ -63,8 +63,9 @@ var runCmd = &cli.Command{
|
||||
},
|
||||
&cli.StringSliceFlag{
|
||||
Name: "layers",
|
||||
Aliases: []string{"l", "layer"},
|
||||
Usage: "list of layers to be interpreted (atop defaults). Default: base",
|
||||
EnvVars: []string{"CURIO_LAYERS"},
|
||||
Aliases: []string{"l", "layer"},
|
||||
},
|
||||
},
|
||||
Action: func(cctx *cli.Context) (err error) {
|
||||
|
||||
@ -246,7 +246,7 @@ OPTIONS:
|
||||
--manage-fdlimit manage open file limit (default: true)
|
||||
--storage-json value path to json file containing storage config (default: "~/.curio/storage.json")
|
||||
--journal value path to journal files (default: "~/.curio/")
|
||||
--layers value, -l value, --layer value [ --layers value, -l value, --layer value ] list of layers to be interpreted (atop defaults). Default: base
|
||||
--layers value, -l value, --layer value [ --layers value, -l value, --layer value ] list of layers to be interpreted (atop defaults). Default: base [$CURIO_LAYERS]
|
||||
--help, -h show help
|
||||
```
|
||||
|
||||
|
||||
2
go.mod
2
go.mod
@ -19,6 +19,7 @@ require (
|
||||
github.com/buger/goterm v1.0.3
|
||||
github.com/charmbracelet/lipgloss v0.10.0
|
||||
github.com/chzyer/readline v1.5.1
|
||||
github.com/codeskyblue/go-sh v0.0.0-20200712050446-30169cf553fe
|
||||
github.com/containerd/cgroups v1.1.0
|
||||
github.com/coreos/go-systemd/v22 v22.5.0
|
||||
github.com/detailyang/go-fallocate v0.0.0-20180908115635-432fa640bd2e
|
||||
@ -190,6 +191,7 @@ require (
|
||||
github.com/cespare/xxhash v1.1.0 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.2.0 // indirect
|
||||
github.com/cilium/ebpf v0.9.1 // indirect
|
||||
github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0 // indirect
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
|
||||
github.com/crackcomm/go-gitignore v0.0.0-20231225121904-e25f5bc08668 // indirect
|
||||
github.com/cskr/pubsub v1.0.2 // indirect
|
||||
|
||||
4
go.sum
4
go.sum
@ -191,6 +191,10 @@ github.com/cockroachdb/cockroach-go/v2 v2.2.0/go.mod h1:u3MiKYGupPPjkn3ozknpMUpx
|
||||
github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
|
||||
github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI=
|
||||
github.com/codegangsta/cli v1.20.0/go.mod h1:/qJNoX69yVSKu5o4jLyXAENLRyk1uhi7zkbQ3slBdOA=
|
||||
github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0 h1:sDMmm+q/3+BukdIpxwO365v/Rbspp2Nt5XntgQRXq8Q=
|
||||
github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM=
|
||||
github.com/codeskyblue/go-sh v0.0.0-20200712050446-30169cf553fe h1:69JI97HlzP+PH5Mi1thcGlDoBr6PS2Oe+l3mNmAkbs4=
|
||||
github.com/codeskyblue/go-sh v0.0.0-20200712050446-30169cf553fe/go.mod h1:VQx0hjo2oUeQkQUET7wRwradO6f+fN5jzXgB/zROxxE=
|
||||
github.com/containerd/cgroups v0.0.0-20201119153540-4cbc285b3327/go.mod h1:ZJeTFisyysqgcCdecO57Dj79RfL0LNeGiFUqLYQRYLE=
|
||||
github.com/containerd/cgroups v1.1.0 h1:v8rEWFl6EoqHB+swVNjVoCJE8o3jX7e8nqBGPLaDFBM=
|
||||
github.com/containerd/cgroups v1.1.0/go.mod h1:6ppBcbh/NOOUU+dMKrykgaBnK9lCIBxHqJDGwsa1mIw=
|
||||
|
||||
5
scripts/curio-apt/DEBIAN/changelog
Normal file
5
scripts/curio-apt/DEBIAN/changelog
Normal file
@ -0,0 +1,5 @@
|
||||
curio (0.0.0.1) UNRELEASED; urgency=medium
|
||||
|
||||
* Initial Release.
|
||||
|
||||
-- andy <support@curiostorage.org> Mon, 29 Apr 2024 21:25:39 +0000
|
||||
20
scripts/curio-apt/DEBIAN/control
Normal file
20
scripts/curio-apt/DEBIAN/control
Normal file
@ -0,0 +1,20 @@
|
||||
Source: curio
|
||||
Section: Network
|
||||
Priority: optional
|
||||
Maintainer: Andrew Jackson <support@curiostorage.org>
|
||||
Build-Depends: debhelper-compat (= 13)
|
||||
Standards-Version: 4.6.0
|
||||
Homepage: github.com/filecoin-project/lotus
|
||||
Rules-Requires-Root: no
|
||||
Package: $PACKAGE
|
||||
Version: $VERSION
|
||||
Architecture: amd64
|
||||
Depends: hwloc, mesa-opencl-icd
|
||||
Description: A Filecoin Storage Provider implementation.
|
||||
This Filecoin Storage Provider implementation is a fork of the Lotus project.
|
||||
It improves upon the previous by solving more complete problem sets including
|
||||
avoiding single points of failure, improving performance, and providing a
|
||||
more robust and scalable solution.
|
||||
It needs to be run in conjunction with the Filecoin network daemon and requires
|
||||
setup of a Miner-ID and a Wallet-ID. See documentation on the website for more
|
||||
information: curiostorage.org
|
||||
10
scripts/curio-apt/DEBIAN/copyright
Normal file
10
scripts/curio-apt/DEBIAN/copyright
Normal file
@ -0,0 +1,10 @@
|
||||
Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
|
||||
Upstream-Name: curio
|
||||
Upstream-Contact: support@curiostorage.org
|
||||
Source: http://mit-license.org
|
||||
|
||||
Files: *
|
||||
Copyright: 2024 Andrew Jackson support@curiostorage.org
|
||||
2024 Lukasz Magiera
|
||||
License: MIT
|
||||
|
||||
23
scripts/curio-apt/DEBIAN/postinst
Executable file
23
scripts/curio-apt/DEBIAN/postinst
Executable file
@ -0,0 +1,23 @@
|
||||
#!/bin/sh
|
||||
set -e
|
||||
|
||||
case "$1" in
|
||||
configure)
|
||||
# Reload systemd after installation or upgrade
|
||||
systemctl daemon-reload
|
||||
|
||||
if [ -z "$2" ]; then
|
||||
# This is an initial installation
|
||||
systemctl enable curio.service
|
||||
if [ ! -f "/etc/curio.env" ]; then
|
||||
echo "CURIO_LAYERS=gui,post\nCURIO_ALL_REMAINING_FIELDS_ARE_OPTIONAL=true\nCURIO_DB_HOST=yugabyte\nCURIO_DB_USER=yugabyte\nCURIO_DB_PASSWORD=yugabyte\nCURIO_DB_PORT=5433\nCURIO_DB_NAME=yugabyte\nCURIO_REPO_PATH=~/.curio" >/etc/curio.env
|
||||
echo "setup /etc/curio.env then run: systemctl start curio.service"
|
||||
fi
|
||||
else
|
||||
# This is an upgrade
|
||||
systemctl restart curio.service
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
|
||||
exit 0
|
||||
109
scripts/curio-apt/make_debs.go
Normal file
109
scripts/curio-apt/make_debs.go
Normal file
@ -0,0 +1,109 @@
|
||||
// Run from lotus root.
|
||||
// requires packages: dpkg-dev
|
||||
// Usage:
|
||||
// ~/GitHub/lotus$ go run scripts/curio-apt/make_debs.go 0.9.7 ~/apt-private.asc
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path"
|
||||
"strings"
|
||||
|
||||
"github.com/codeskyblue/go-sh"
|
||||
)
|
||||
|
||||
var version string
|
||||
|
||||
func main() {
|
||||
if len(os.Args) < 3 || strings.EqualFold(os.Args[1], "help") {
|
||||
fmt.Println("Usage: make_debs <version> path_to_private_key.asc")
|
||||
fmt.Println("Run this from the root of the lotus repo as it runs 'make'.")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
version = os.Args[1]
|
||||
|
||||
// Import the key (repeat imports are OK)
|
||||
OrPanic(sh.Command("gpg", "--import", os.Args[2]).Run())
|
||||
|
||||
base, err := os.MkdirTemp(os.TempDir(), "curio-apt")
|
||||
OrPanic(err)
|
||||
|
||||
part2(base, "curio-cuda", "")
|
||||
part2(base, "curio-opencl", "FFI_USE_OPENCL=1")
|
||||
fmt.Println("Done. DEB files are in ", base)
|
||||
}
|
||||
|
||||
func part2(base, product, extra string) {
|
||||
// copy scripts/curio-apt/debian to dir/debian
|
||||
dir := path.Join(base, product)
|
||||
err := os.MkdirAll(path.Join(dir, "DEBIAN"), 0755)
|
||||
OrPanic(err)
|
||||
|
||||
OrPanic(sh.Command("cp", "-r", "scripts/curio-apt/DEBIAN", dir).Run())
|
||||
sess := sh.NewSession()
|
||||
for _, env := range strings.Split(extra, " ") {
|
||||
if len(env) == 0 {
|
||||
continue
|
||||
}
|
||||
v := strings.Split(env, "=")
|
||||
sess.SetEnv(v[0], v[1])
|
||||
}
|
||||
fmt.Println("making")
|
||||
|
||||
// This ENV is only for fixing this script. It will result in a bad build.
|
||||
if os.Getenv("CURIO_DEB_NOBUILD") != "1" {
|
||||
// FUTURE: Use cross-compilation to cover more arch and run anywhere.
|
||||
// FUTURE: Use RUST & Go PGO.
|
||||
OrPanic(sess.Command("make", "clean", "all").Run())
|
||||
}
|
||||
|
||||
// strip binaries
|
||||
OrPanic(sh.Command("strip", "curio").Run())
|
||||
OrPanic(sh.Command("strip", "sptool").Run())
|
||||
|
||||
fmt.Println("copying")
|
||||
{
|
||||
base := path.Join(dir, "usr", "local", "bin")
|
||||
OrPanic(os.MkdirAll(base, 0755))
|
||||
OrPanic(copyFile("curio", path.Join(base, "curio")))
|
||||
OrPanic(copyFile("sptool", path.Join(base, "sptool")))
|
||||
}
|
||||
// fix the debian/control "package" and "version" fields
|
||||
f, err := os.ReadFile(path.Join(dir, "DEBIAN", "control"))
|
||||
OrPanic(err)
|
||||
f = []byte(strings.ReplaceAll(string(f), "$PACKAGE", product))
|
||||
f = []byte(strings.ReplaceAll(string(f), "$VERSION", version))
|
||||
OrPanic(os.WriteFile(path.Join(dir, "DEBIAN", "control"), f, 0644))
|
||||
fullname := product + "-" + version + "_amd64.deb"
|
||||
|
||||
// Option 1: piece by piece. Maybe could work, but it is complex.
|
||||
// Build a .changes file
|
||||
//OrPanic(sh.Command("dpkg-genchanges", "-b", "-u.").SetDir(dir).Run())
|
||||
// Sign the .changes file
|
||||
//OrPanic(sh.Command("debsign", "--sign=origin", "--default", path.Join(dir, "..", "*.changes")).Run())
|
||||
// Build the .deb file
|
||||
//OrPanic(sh.Command("dpkg-deb", "--build", ".").SetDir(dir).Run())
|
||||
|
||||
// Option 2: The following command should sign the deb file.
|
||||
// FAIL B/C wants to build.
|
||||
//sh.Command("dpkg-buildpackage", "--build=binary").SetDir(dir).Run()
|
||||
|
||||
// Option 3: Use new helpler commands outside of regular DEB stuff.
|
||||
OrPanic(sh.NewSession().SetDir(base).Command("dpkg-deb", "-Z", "xz", "--build", product, fullname).Run())
|
||||
|
||||
// Sign the DEB we built.
|
||||
OrPanic(sh.NewSession().SetDir(base).Command(
|
||||
"dpkg-sig", "--sign", "builder", "-k", "B751F6AC4FA6D98F", fullname).Run())
|
||||
}
|
||||
|
||||
func copyFile(src, dest string) error {
|
||||
return sh.Command("cp", src, dest).Run()
|
||||
}
|
||||
|
||||
func OrPanic(err error) {
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
@ -1,12 +1,15 @@
|
||||
[Unit]
|
||||
Description=Curio
|
||||
After=network.target
|
||||
After=lotus-daemon.service
|
||||
|
||||
[Service]
|
||||
ExecStart=/usr/local/bin/curio run
|
||||
Environment=GOLOG_FILE="/var/log/curio/curio.log"
|
||||
Environment=GOLOG_LOG_FMT="json"
|
||||
LimitNOFILE=1000000
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
EnvironmentFile=/etc/curio.env
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
|
||||
Loading…
Reference in New Issue
Block a user