cerc-io/lotus
16
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

v1.27.0-a #10

Closed
jonathanface wants to merge 473 commits from v1.27.0-a into master
pull from: v1.27.0-a
merge into: cerc-io:master
Conversation 0 Commits 473 Files Changed 452 +56 -24
5 changed files with 56 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit b9c06b64d7 - Show all commits

10
curiosrc/gc/storage_endpoint_gc.go
View File

@ -197,16 +197,14 @@ func (s *StorageEndpointGC) Do(taskID harmonytask.TaskID, stillOwned func() bool
// Remove dead URLs from storage_path entries and handle path cleanup
for _, du := range deadURLs {
// Fetch the current URLs for the storage path
var currentPath struct {
URLs string
}
err = tx.Get(&currentPath, "SELECT urls FROM storage_path WHERE storage_id = $1", du.StorageID)
var URLs string
err = tx.QueryRow("SELECT urls FROM storage_path WHERE storage_id = $1", du.StorageID).Scan(&URLs)
if err != nil {
return false, xerrors.Errorf("fetching storage paths: %w", err)
}
// Filter out the dead URL using lo.Reject and prepare the updated list
urls := strings.Split(currentPath.URLs, paths.URLSeparator)
urls := strings.Split(URLs, paths.URLSeparator)
urls = lo.Reject(urls, func(u string, _ int) bool {
return u == du.URL
})
@ -214,7 +212,7 @@ func (s *StorageEndpointGC) Do(taskID harmonytask.TaskID, stillOwned func() bool
log.Debugw("filtered urls", "urls", urls, "dead_url", du.URL, "storage_id", du.StorageID)
if os.Getenv("CURIO_STORAGE_META_GC_DRYRUN") != "no" { // todo drop this after testing
log.Debugw("dryrun: not updating storage path", "storage_id", du.StorageID, "urls", urls, "dead_url", du.URL, "current_urls", currentPath.URLs, "dead_urls", deadURLs)
log.Debugw("dryrun: not updating storage path", "storage_id", du.StorageID, "urls", urls, "dead_url", du.URL, "current_urls", URLs, "dead_urls", deadURLs)
continue
}

7
go.mod
View File

@ -159,7 +159,7 @@ require (
go.uber.org/fx v1.20.1
go.uber.org/multierr v1.11.0
go.uber.org/zap v1.27.0
golang.org/x/crypto v0.19.0
golang.org/x/crypto v0.20.0
golang.org/x/exp v0.0.0-20240213143201-ec583247a57a
golang.org/x/net v0.21.0
golang.org/x/sync v0.6.0
@ -249,8 +249,8 @@ require (
github.com/ipld/go-ipld-adl-hamt v0.0.0-20220616142416-9004dbd839e0 // indirect
github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
github.com/jackc/puddle/v2 v2.2.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9 // indirect
github.com/jackc/puddle/v2 v2.2.1 // indirect
github.com/jackpal/go-nat-pmp v1.0.2 // indirect
github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c // indirect
github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect
@ -318,6 +318,7 @@ require (
github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
github.com/yugabyte/pgx/v5 v5.5.3-yb-2 // indirect
github.com/zondax/hid v0.9.2 // indirect
github.com/zondax/ledger-go v0.14.3 // indirect
go.opentelemetry.io/otel/metric v1.21.0 // indirect

8
go.sum
View File

@ -884,10 +884,14 @@ github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsI
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk=
github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9 h1:L0QtFUgDarD7Fpv9jeVMgy/+Ec0mtnmYuImjTz6dtDA=
github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
github.com/jackc/pgx/v5 v5.4.1 h1:oKfB/FhuVtit1bBM3zNRRsZ925ZkMN3HXL+LgLUM9lE=
github.com/jackc/pgx/v5 v5.4.1/go.mod h1:q6iHT8uDNXWiFNOlRqJzBTaSH3+2xCXkokxHZC5qWFY=
github.com/jackc/puddle/v2 v2.2.0 h1:RdcDk92EJBuBS55nQMMYFXTxwstHug4jkhT5pq8VxPk=
github.com/jackc/puddle/v2 v2.2.0/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk=
github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
github.com/jackpal/gateway v1.0.5/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA=
github.com/jackpal/go-nat-pmp v1.0.1/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc=
github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus=
@ -1711,6 +1715,8 @@ github.com/xorcare/golden v0.6.1-0.20191112154924-b87f686d7542/go.mod h1:7T39/ZM
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU=
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8=
github.com/yugabyte/pgx/v5 v5.5.3-yb-2 h1:SDk2waZb2o6dSLYqk+vq0Ur2jnIv+X2A+P+QPR1UThU=
github.com/yugabyte/pgx/v5 v5.5.3-yb-2/go.mod h1:2SxizGfDY7UDCRTtbI/xd98C/oGN7S/3YoGF8l9gx/c=
github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@ -1840,6 +1846,8 @@ golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw
golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo=
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
golang.org/x/crypto v0.20.0 h1:jmAMJJZXr5KiCw05dfYK9QnqaqKLYXijU23lsEdcQqg=
golang.org/x/crypto v0.20.0/go.mod h1:Xwo95rrVNIoSMx9wa1JroENMToLWn3RNVrTBpLHgZPQ=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=

6
lib/harmony/harmonydb/harmonydb.go
View File

@ -15,9 +15,9 @@ import (
"time"
logging "github.com/ipfs/go-log/v2"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgconn"
"github.com/jackc/pgx/v5/pgxpool"
"github.com/yugabyte/pgx/v5"
"github.com/yugabyte/pgx/v5/pgconn"
"github.com/yugabyte/pgx/v5/pgxpool"
"golang.org/x/xerrors"
"github.com/filecoin-project/lotus/node/config"

49
lib/harmony/harmonydb/userfuncs.go
View File

@ -3,17 +3,18 @@ package harmonydb
import (
"context"
"errors"
"fmt"
"runtime"
"time"
"github.com/georgysavva/scany/v2/pgxscan"
"github.com/georgysavva/scany/v2/dbscan"
"github.com/jackc/pgerrcode"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgconn"
"github.com/samber/lo"
"github.com/yugabyte/pgx/v5"
"github.com/yugabyte/pgx/v5/pgconn"
)
var errTx = errors.New("Cannot use a non-transaction func in a transaction")
var errTx = errors.New("cannot use a non-transaction func in a transaction")
// rawStringOnly is _intentionally_private_ to force only basic strings in SQL queries.
// In any package, raw strings will satisfy compilation. Ex:
@ -42,7 +43,7 @@ type Qry interface {
Values() ([]any, error)
}
// Query offers Next/Err/Close/Scan/Values/StructScan
// Query offers Next/Err/Close/Scan/Values
type Query struct {
Qry
}
@ -69,8 +70,12 @@ func (db *DB) Query(ctx context.Context, sql rawStringOnly, arguments ...any) (*
q, err := db.pgx.Query(ctx, string(sql), arguments...)
return &Query{q}, err
}
// StructScan allows scanning a single row into a struct.
// This improves efficiency of processing large result sets
// by avoiding the need to allocate a slice of structs.
func (q *Query) StructScan(s any) error {
return pgxscan.ScanRow(s, q.Qry.(pgx.Rows))
return dbscan.ScanRow(s, dbscanRows{q.Qry.(pgx.Rows)})
}
type Row interface {
@ -95,6 +100,20 @@ func (db *DB) QueryRow(ctx context.Context, sql rawStringOnly, arguments ...any)
return db.pgx.QueryRow(ctx, string(sql), arguments...)
}
type dbscanRows struct {
pgx.Rows
}
func (d dbscanRows) Close() error {
d.Rows.Close()
return nil
}
func (d dbscanRows) Columns() ([]string, error) {
return lo.Map(d.Rows.FieldDescriptions(), func(fd pgconn.FieldDescription, _ int) string {
return fd.Name
}), nil
}
/*
Select multiple rows into a slice using name matching
Ex:
@ -113,7 +132,12 @@ func (db *DB) Select(ctx context.Context, sliceOfStructPtr any, sql rawStringOnl
if db.usedInTransaction() {
return errTx
}
return pgxscan.Select(ctx, db.pgx, sliceOfStructPtr, string(sql), arguments...)
rows, err := db.pgx.Query(ctx, string(sql), arguments...)
if err != nil {
return err
}
defer rows.Close()
return dbscan.ScanAll(sliceOfStructPtr, dbscanRows{rows})
}
type Tx struct {
@ -233,11 +257,12 @@ func (t *Tx) QueryRow(sql rawStringOnly, arguments ...any) Row {
// Select in a transaction.
func (t *Tx) Select(sliceOfStructPtr any, sql rawStringOnly, arguments ...any) error {
return pgxscan.Select(t.ctx, t.Tx, sliceOfStructPtr, string(sql), arguments...)
}
func (t *Tx) Get(s any, sql rawStringOnly, arguments ...any) error {
return pgxscan.Get(t.ctx, t.Tx, s, string(sql), arguments...)
rows, err := t.Query(sql, arguments...)
if err != nil {
return fmt.Errorf("scany: query multiple result rows: %w", err)
}
defer rows.Close()
return dbscan.ScanAll(sliceOfStructPtr, dbscanRows{rows.Qry.(pgx.Rows)})
}
func IsErrUniqueContraint(err error) bool {