cerc-io/lotus
16
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

harmony codename and long itest timeout

Browse Source
This commit is contained in:
Andrew Jackson (Ajax) 2023-07-14 11:22:01 -05:00
parent bde2dbdc14
commit b9c615ace5
15 changed files with 86 additions and 86 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
.circleci/config.yml
View File

@ -178,9 +178,9 @@ jobs:
SKIP_CONFORMANCE: "1"
LOTUS_SRC_DIR: /home/circleci/project
command: |
dockerize -wait tcp://yugabyte:5433 -timeout 1m
mkdir -p /tmp/test-reports/<< parameters.suite >>
mkdir -p /tmp/test-artifacts
dockerize -wait tcp://yugabyte:5433 -timeout 3m
gotestsum \
--format standard-verbose \
--junitfile /tmp/test-reports/<< parameters.suite >>/junit.xml \
@ -557,12 +557,6 @@ workflows:
- build
suite: itest-cli
target: "./itests/cli_test.go"
- test:
name: test-itest-clusterdb
requires:
- build
suite: itest-clusterdb
target: "./itests/clusterdb_test.go"
- test:
name: test-itest-deadlines
requires:
@ -780,6 +774,12 @@ workflows:
- build
suite: itest-get_messages_in_ts
target: "./itests/get_messages_in_ts_test.go"
- test:
name: test-itest-harmonydb
requires:
- build
suite: itest-harmonydb
target: "./itests/harmonydb_test.go"
- test:
name: test-itest-lite_migration
requires:

2
.circleci/template.yml
View File

@ -178,9 +178,9 @@ jobs:
SKIP_CONFORMANCE: "1"
LOTUS_SRC_DIR: /home/circleci/project
command: |
dockerize -wait tcp://yugabyte:5433 -timeout 1m
mkdir -p /tmp/test-reports/<< parameters.suite >>
mkdir -p /tmp/test-artifacts
dockerize -wait tcp://yugabyte:5433 -timeout 3m
gotestsum \
--format standard-verbose \
--junitfile /tmp/test-reports/<< parameters.suite >>/junit.xml \

14
documentation/en/default-lotus-miner-config.toml
View File

@ -894,43 +894,43 @@
#GCInterval = "1m0s"
[ClusterDB]
[HarmonyDB]
# HOSTS is a list of hostnames to nodes running YugabyteDB
# in a cluster. Only 1 is required
#
# type: []string
# env var: LOTUS_CLUSTERDB_HOSTS
# env var: LOTUS_HARMONYDB_HOSTS
#Hosts = ["127.0.0.1"]
# The Yugabyte server's username with full credentials to operate on Lotus' Database. Blank for default.
#
# type: string
# env var: LOTUS_CLUSTERDB_USERNAME
# env var: LOTUS_HARMONYDB_USERNAME
#Username = "yugabyte"
# The password for the related username. Blank for default.
#
# type: string
# env var: LOTUS_CLUSTERDB_PASSWORD
# env var: LOTUS_HARMONYDB_PASSWORD
#Password = "yugabyte"
# The database (logical partition) within Yugabyte. Blank for default.
#
# type: string
# env var: LOTUS_CLUSTERDB_DATABASE
# env var: LOTUS_HARMONYDB_DATABASE
#Database = "yugabyte"
# The port to find Yugabyte. Blank for default.
#
# type: string
# env var: LOTUS_CLUSTERDB_PORT
# env var: LOTUS_HARMONYDB_PORT
#Port = "5433"
# ITest is for optimized integration testing and not
# for production. Blank for default production configuration.
#
# type: string
# env var: LOTUS_CLUSTERDB_ITEST
# env var: LOTUS_HARMONYDB_ITEST
#ITest = ""

10
itests/clusterdb_test.go → itests/harmonydb_test.go
View File

@ -8,7 +8,7 @@ import (
"testing"
"github.com/filecoin-project/lotus/itests/kit"
"github.com/filecoin-project/lotus/lib/sturdy/clusterdb"
"github.com/filecoin-project/lotus/lib/harmony/harmonydb"
"github.com/filecoin-project/lotus/node/impl"
)
@ -26,7 +26,7 @@ func TestCrud(t *testing.T) {
defer cancel()
withSetup(t, func(miner *kit.TestMiner) {
cdb := miner.BaseAPI.(*impl.StorageMinerAPI).ClusterDB
cdb := miner.BaseAPI.(*impl.StorageMinerAPI).HarmonyDB
err := cdb.Exec(ctx, `
INSERT INTO
itest_scratch (some_int, content)
@ -64,11 +64,11 @@ func TestTransaction(t *testing.T) {
defer cancel()
withSetup(t, func(miner *kit.TestMiner) {
cdb := miner.BaseAPI.(*impl.StorageMinerAPI).ClusterDB
cdb := miner.BaseAPI.(*impl.StorageMinerAPI).HarmonyDB
if err := cdb.Exec(ctx, "INSERT INTO itest_scratch (some_int) VALUES (4), (5), (6)"); err != nil {
t.Fatal("E0", err)
}
cdb.BeginTransaction(ctx, func(tx *clusterdb.Transaction) (commit bool) {
cdb.BeginTransaction(ctx, func(tx *harmonydb.Transaction) (commit bool) {
if err := tx.Exec(ctx, "INSERT INTO itest_scratch (some_int) VALUES (7), (8), (9)"); err != nil {
t.Fatal("E1", err)
}
@ -119,7 +119,7 @@ func TestPartialWalk(t *testing.T) {
defer cancel()
withSetup(t, func(miner *kit.TestMiner) {
cdb := miner.BaseAPI.(*impl.StorageMinerAPI).ClusterDB
cdb := miner.BaseAPI.(*impl.StorageMinerAPI).HarmonyDB
if err := cdb.Exec(ctx, `
INSERT INTO
itest_scratch (content, some_int)

10
itests/kit/ensemble.go
View File

@ -49,7 +49,7 @@ import (
"github.com/filecoin-project/lotus/cmd/lotus-worker/sealworker"
"github.com/filecoin-project/lotus/gateway"
"github.com/filecoin-project/lotus/genesis"
"github.com/filecoin-project/lotus/lib/sturdy/clusterdb"
"github.com/filecoin-project/lotus/lib/harmony/harmonydb"
"github.com/filecoin-project/lotus/markets/idxprov"
"github.com/filecoin-project/lotus/markets/idxprov/idxprov_test"
lotusminer "github.com/filecoin-project/lotus/miner"
@ -359,7 +359,7 @@ func (n *Ensemble) Start() *Ensemble {
n.mn = mocknet.New()
}
sharedITestID := clusterdb.ITestNewID()
sharedITestID := harmonydb.ITestNewID()
// ---------------------
// FULL NODES
@ -727,7 +727,7 @@ func (n *Ensemble) Start() *Ensemble {
// upgrades
node.Override(new(stmgr.UpgradeSchedule), n.options.upgradeSchedule),
node.Override(new(clusterdb.ITestID), sharedITestID),
node.Override(new(harmonydb.ITestID), sharedITestID),
}
if m.options.subsystems.Has(SMarkets) {
@ -775,7 +775,7 @@ func (n *Ensemble) Start() *Ensemble {
n.t.Cleanup(func() { _ = stop(context.Background()) })
n.t.Cleanup(func() {
m.StorageMiner.(*impl.StorageMinerAPI).ClusterDB.ITestDeleteAll()
m.StorageMiner.(*impl.StorageMinerAPI).HarmonyDB.ITestDeleteAll()
})
m.BaseAPI = m.StorageMiner
@ -833,7 +833,7 @@ func (n *Ensemble) Start() *Ensemble {
auth := http.Header(nil)
// FUTURE: Use m.MinerNode.(BaseAPI).(impl.StorageMinerAPI).ClusterDB to setup.
// FUTURE: Use m.MinerNode.(BaseAPI).(impl.StorageMinerAPI).HarmonyDB to setup.
remote := paths.NewRemote(localStore, m.MinerNode, auth, 20, &paths.DefaultPartialFileHandler{})
store := m.options.workerStorageOpt(remote)

4
lib/sturdy/clusterdb/doc.go → lib/harmony/harmonydb/doc.go
View File

@ -1,5 +1,5 @@
/*
# Clusterdb provides database abstractions over SP-wide Postgres-compatible instance(s).
# HarmonyDB provides database abstractions over SP-wide Postgres-compatible instance(s).
# Features
@ -32,4 +32,4 @@ Write SQL with context, raw strings, and args:
Note: Scan() is column-oriented, while Select() & StructScan() is field name/tag oriented.
*/
package clusterdb
package harmonydb

10
lib/sturdy/clusterdb/clusterdb.go → lib/harmony/harmonydb/harmonydb.go
View File

@ -1,4 +1,4 @@
package clusterdb
package harmonydb
import (
"context"
@ -35,13 +35,13 @@ type DB struct {
log func(string)
}
var logger = logging.Logger("clusterdb")
var logger = logging.Logger("harmonydb")
// NewFromConfig is a convenience function.
// In usage:
//
// db, err := NewFromConfig(config.ClusterDB) // in binary init
func NewFromConfig(cfg config.ClusterDB) (*DB, error) {
// db, err := NewFromConfig(config.HarmonyDB) // in binary init
func NewFromConfig(cfg config.HarmonyDB) (*DB, error) {
return New(
cfg.Hosts,
cfg.Username,
@ -53,7 +53,7 @@ func NewFromConfig(cfg config.ClusterDB) (*DB, error) {
)
}
func NewFromConfigWithITestID(cfg config.ClusterDB) func(id ITestID) (*DB, error) {
func NewFromConfigWithITestID(cfg config.HarmonyDB) func(id ITestID) (*DB, error) {
return func(id ITestID) (*DB, error) {
return New(
cfg.Hosts,

4
lib/sturdy/clusterdb/metrics.go → lib/harmony/harmonydb/metrics.go
View File

@ -1,4 +1,4 @@
package clusterdb
package harmonydb
import (
"github.com/prometheus/client_golang/prometheus"
@ -11,7 +11,7 @@ import (
var (
dbTag, _ = tag.NewKey("db_name")
pre = "clusterdb/base/"
pre = "harmonydb/base/"
waitsBuckets = []float64{0, 10, 20, 30, 50, 80, 130, 210, 340, 550, 890}
whichHostBuckets = []float64{0, 1, 2, 3, 4, 5}
)

0
lib/sturdy/clusterdb/sql/20230706.sql → lib/harmony/harmonydb/sql/20230706.sql
View File

4
lib/sturdy/clusterdb/userfuncs.go → lib/harmony/harmonydb/userfuncs.go
View File

@ -1,4 +1,4 @@
package clusterdb
package harmonydb
import (
"context"
@ -19,7 +19,7 @@ type Intf interface {
// rawStringOnly is _intentionally_private_ to force only basic strings in SQL queries.
// In any package, raw strings will satisfy compilation. Ex:
//
// clusterDB.Exec("INSERT INTO version (number) VALUES (1)")
// harmonydb.Exec("INSERT INTO version (number) VALUES (1)")
//
// This prevents SQL injection attacks where the input contains query fragments.
type rawStringOnly string

6
node/builder_miner.go
View File

@ -19,7 +19,7 @@ import (
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/gen"
"github.com/filecoin-project/lotus/chain/gen/slashfilter"
"github.com/filecoin-project/lotus/lib/sturdy/clusterdb"
"github.com/filecoin-project/lotus/lib/harmony/harmonydb"
"github.com/filecoin-project/lotus/markets/dagstore"
"github.com/filecoin-project/lotus/markets/dealfilter"
"github.com/filecoin-project/lotus/markets/idxprov"
@ -232,8 +232,8 @@ func ConfigStorageMiner(c interface{}) Option {
Override(new(config.SealerConfig), cfg.Storage),
Override(new(config.ProvingConfig), cfg.Proving),
Override(new(*ctladdr.AddressSelector), modules.AddressSelector(&cfg.Addresses)),
Override(new(*clusterdb.DB), clusterdb.NewFromConfigWithITestID(cfg.ClusterDB)),
Override(new(clusterdb.ITestID), clusterdb.ITestID("")),
Override(new(*harmonydb.DB), harmonydb.NewFromConfigWithITestID(cfg.HarmonyDB)),
Override(new(harmonydb.ITestID), harmonydb.ITestID("")),
)
}

2
node/config/def.go
View File

@ -269,7 +269,7 @@ func DefaultStorageMiner() *StorageMiner {
MaxConcurrentUnseals: 5,
GCInterval: Duration(1 * time.Minute),
},
ClusterDB: ClusterDB{
HarmonyDB: HarmonyDB{
Hosts: []string{"127.0.0.1"},
Username: "yugabyte",
Password: "yugabyte",

84
node/config/doc_gen.go
View File

@ -117,46 +117,6 @@ without existing payment channels with available funds will fail instead
of automatically performing on-chain operations.`,
},
},
"ClusterDB": []DocField{
{
Name: "Hosts",
Type: "[]string",
Comment: `HOSTS is a list of hostnames to nodes running YugabyteDB
in a cluster. Only 1 is required`,
},
{
Name: "Username",
Type: "string",
Comment: `The Yugabyte server's username with full credentials to operate on Lotus' Database. Blank for default.`,
},
{
Name: "Password",
Type: "string",
Comment: `The password for the related username. Blank for default.`,
},
{
Name: "Database",
Type: "string",
Comment: `The database (logical partition) within Yugabyte. Blank for default.`,
},
{
Name: "Port",
Type: "string",
Comment: `The port to find Yugabyte. Blank for default.`,
},
{
Name: "ITest",
Type: "string",
Comment: `ITest is for optimized integration testing and not
for production. Blank for default production configuration.`,
},
},
"Common": []DocField{
{
Name: "API",
@ -508,6 +468,46 @@ Set to 0 to keep all mappings`,
Comment: ``,
},
},
"HarmonyDB": []DocField{
{
Name: "Hosts",
Type: "[]string",
Comment: `HOSTS is a list of hostnames to nodes running YugabyteDB
in a cluster. Only 1 is required`,
},
{
Name: "Username",
Type: "string",
Comment: `The Yugabyte server's username with full credentials to operate on Lotus' Database. Blank for default.`,
},
{
Name: "Password",
Type: "string",
Comment: `The password for the related username. Blank for default.`,
},
{
Name: "Database",
Type: "string",
Comment: `The database (logical partition) within Yugabyte. Blank for default.`,
},
{
Name: "Port",
Type: "string",
Comment: `The port to find Yugabyte. Blank for default.`,
},
{
Name: "ITest",
Type: "string",
Comment: `ITest is for optimized integration testing and not
for production. Blank for default production configuration.`,
},
},
"IndexConfig": []DocField{
{
Name: "EnableMsgIndex",
@ -1429,8 +1429,8 @@ HotstoreMaxSpaceTarget - HotstoreMaxSpaceSafetyBuffer`,
Comment: ``,
},
{
Name: "ClusterDB",
Type: "ClusterDB",
Name: "HarmonyDB",
Type: "HarmonyDB",
Comment: ``,
},

4
node/config/types.go
View File

@ -62,7 +62,7 @@ type StorageMiner struct {
Addresses MinerAddressConfig
DAGStore DAGStoreConfig
ClusterDB ClusterDB
HarmonyDB HarmonyDB
}
type DAGStoreConfig struct {
@ -735,7 +735,7 @@ type IndexConfig struct {
EnableMsgIndex bool
}
type ClusterDB struct {
type HarmonyDB struct {
// HOSTS is a list of hostnames to nodes running YugabyteDB
// in a cluster. Only 1 is required
Hosts []string

4
node/impl/storminer.go
View File

@ -45,7 +45,7 @@ import (
"github.com/filecoin-project/lotus/chain/actors/builtin"
"github.com/filecoin-project/lotus/chain/gen"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/lib/sturdy/clusterdb"
"github.com/filecoin-project/lotus/lib/harmony/harmonydb"
mktsdagstore "github.com/filecoin-project/lotus/markets/dagstore"
"github.com/filecoin-project/lotus/markets/storageadapter"
"github.com/filecoin-project/lotus/miner"
@ -124,7 +124,7 @@ type StorageMinerAPI struct {
GetExpectedSealDurationFunc dtypes.GetExpectedSealDurationFunc `optional:"true"`
SetExpectedSealDurationFunc dtypes.SetExpectedSealDurationFunc `optional:"true"`
ClusterDB *clusterdb.DB
HarmonyDB *harmonydb.DB
}
var _ api.StorageMiner = &StorageMinerAPI{}