From b9c615ace5605b83c9d9c6b8820f150c8cc2c93b Mon Sep 17 00:00:00 2001 From: "Andrew Jackson (Ajax)" Date: Fri, 14 Jul 2023 11:22:01 -0500 Subject: [PATCH] harmony codename and long itest timeout --- .circleci/config.yml | 14 ++-- .circleci/template.yml | 2 +- .../en/default-lotus-miner-config.toml | 14 ++-- .../{clusterdb_test.go => harmonydb_test.go} | 10 +-- itests/kit/ensemble.go | 10 +-- .../clusterdb => harmony/harmonydb}/doc.go | 4 +- .../harmonydb/harmonydb.go} | 10 +-- .../harmonydb}/metrics.go | 4 +- .../harmonydb}/sql/20230706.sql | 0 .../harmonydb}/userfuncs.go | 4 +- node/builder_miner.go | 6 +- node/config/def.go | 2 +- node/config/doc_gen.go | 84 +++++++++---------- node/config/types.go | 4 +- node/impl/storminer.go | 4 +- 15 files changed, 86 insertions(+), 86 deletions(-) rename itests/{clusterdb_test.go => harmonydb_test.go} (92%) rename lib/{sturdy/clusterdb => harmony/harmonydb}/doc.go (93%) rename lib/{sturdy/clusterdb/clusterdb.go => harmony/harmonydb/harmonydb.go} (96%) rename lib/{sturdy/clusterdb => harmony/harmonydb}/metrics.go (97%) rename lib/{sturdy/clusterdb => harmony/harmonydb}/sql/20230706.sql (100%) rename lib/{sturdy/clusterdb => harmony/harmonydb}/userfuncs.go (98%) diff --git a/.circleci/config.yml b/.circleci/config.yml index 81714ae26..8508d8196 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -178,9 +178,9 @@ jobs: SKIP_CONFORMANCE: "1" LOTUS_SRC_DIR: /home/circleci/project command: | - dockerize -wait tcp://yugabyte:5433 -timeout 1m mkdir -p /tmp/test-reports/<< parameters.suite >> mkdir -p /tmp/test-artifacts + dockerize -wait tcp://yugabyte:5433 -timeout 3m gotestsum \ --format standard-verbose \ --junitfile /tmp/test-reports/<< parameters.suite >>/junit.xml \ @@ -557,12 +557,6 @@ workflows: - build suite: itest-cli target: "./itests/cli_test.go" - - test: - name: test-itest-clusterdb - requires: - - build - suite: itest-clusterdb - target: "./itests/clusterdb_test.go" - test: name: test-itest-deadlines requires: @@ -780,6 +774,12 @@ workflows: - build suite: itest-get_messages_in_ts target: "./itests/get_messages_in_ts_test.go" + - test: + name: test-itest-harmonydb + requires: + - build + suite: itest-harmonydb + target: "./itests/harmonydb_test.go" - test: name: test-itest-lite_migration requires: diff --git a/.circleci/template.yml b/.circleci/template.yml index 2b24cbf01..8e340f70a 100644 --- a/.circleci/template.yml +++ b/.circleci/template.yml @@ -178,9 +178,9 @@ jobs: SKIP_CONFORMANCE: "1" LOTUS_SRC_DIR: /home/circleci/project command: | - dockerize -wait tcp://yugabyte:5433 -timeout 1m mkdir -p /tmp/test-reports/<< parameters.suite >> mkdir -p /tmp/test-artifacts + dockerize -wait tcp://yugabyte:5433 -timeout 3m gotestsum \ --format standard-verbose \ --junitfile /tmp/test-reports/<< parameters.suite >>/junit.xml \ diff --git a/documentation/en/default-lotus-miner-config.toml b/documentation/en/default-lotus-miner-config.toml index 1ff115b07..0efb22ff0 100644 --- a/documentation/en/default-lotus-miner-config.toml +++ b/documentation/en/default-lotus-miner-config.toml @@ -894,43 +894,43 @@ #GCInterval = "1m0s" -[ClusterDB] +[HarmonyDB] # HOSTS is a list of hostnames to nodes running YugabyteDB # in a cluster. Only 1 is required # # type: []string - # env var: LOTUS_CLUSTERDB_HOSTS + # env var: LOTUS_HARMONYDB_HOSTS #Hosts = ["127.0.0.1"] # The Yugabyte server's username with full credentials to operate on Lotus' Database. Blank for default. # # type: string - # env var: LOTUS_CLUSTERDB_USERNAME + # env var: LOTUS_HARMONYDB_USERNAME #Username = "yugabyte" # The password for the related username. Blank for default. # # type: string - # env var: LOTUS_CLUSTERDB_PASSWORD + # env var: LOTUS_HARMONYDB_PASSWORD #Password = "yugabyte" # The database (logical partition) within Yugabyte. Blank for default. # # type: string - # env var: LOTUS_CLUSTERDB_DATABASE + # env var: LOTUS_HARMONYDB_DATABASE #Database = "yugabyte" # The port to find Yugabyte. Blank for default. # # type: string - # env var: LOTUS_CLUSTERDB_PORT + # env var: LOTUS_HARMONYDB_PORT #Port = "5433" # ITest is for optimized integration testing and not # for production. Blank for default production configuration. # # type: string - # env var: LOTUS_CLUSTERDB_ITEST + # env var: LOTUS_HARMONYDB_ITEST #ITest = "" diff --git a/itests/clusterdb_test.go b/itests/harmonydb_test.go similarity index 92% rename from itests/clusterdb_test.go rename to itests/harmonydb_test.go index dd9dd8f5b..150b2bd9e 100644 --- a/itests/clusterdb_test.go +++ b/itests/harmonydb_test.go @@ -8,7 +8,7 @@ import ( "testing" "github.com/filecoin-project/lotus/itests/kit" - "github.com/filecoin-project/lotus/lib/sturdy/clusterdb" + "github.com/filecoin-project/lotus/lib/harmony/harmonydb" "github.com/filecoin-project/lotus/node/impl" ) @@ -26,7 +26,7 @@ func TestCrud(t *testing.T) { defer cancel() withSetup(t, func(miner *kit.TestMiner) { - cdb := miner.BaseAPI.(*impl.StorageMinerAPI).ClusterDB + cdb := miner.BaseAPI.(*impl.StorageMinerAPI).HarmonyDB err := cdb.Exec(ctx, ` INSERT INTO itest_scratch (some_int, content) @@ -64,11 +64,11 @@ func TestTransaction(t *testing.T) { defer cancel() withSetup(t, func(miner *kit.TestMiner) { - cdb := miner.BaseAPI.(*impl.StorageMinerAPI).ClusterDB + cdb := miner.BaseAPI.(*impl.StorageMinerAPI).HarmonyDB if err := cdb.Exec(ctx, "INSERT INTO itest_scratch (some_int) VALUES (4), (5), (6)"); err != nil { t.Fatal("E0", err) } - cdb.BeginTransaction(ctx, func(tx *clusterdb.Transaction) (commit bool) { + cdb.BeginTransaction(ctx, func(tx *harmonydb.Transaction) (commit bool) { if err := tx.Exec(ctx, "INSERT INTO itest_scratch (some_int) VALUES (7), (8), (9)"); err != nil { t.Fatal("E1", err) } @@ -119,7 +119,7 @@ func TestPartialWalk(t *testing.T) { defer cancel() withSetup(t, func(miner *kit.TestMiner) { - cdb := miner.BaseAPI.(*impl.StorageMinerAPI).ClusterDB + cdb := miner.BaseAPI.(*impl.StorageMinerAPI).HarmonyDB if err := cdb.Exec(ctx, ` INSERT INTO itest_scratch (content, some_int) diff --git a/itests/kit/ensemble.go b/itests/kit/ensemble.go index da9df08d1..dfbc412a1 100644 --- a/itests/kit/ensemble.go +++ b/itests/kit/ensemble.go @@ -49,7 +49,7 @@ import ( "github.com/filecoin-project/lotus/cmd/lotus-worker/sealworker" "github.com/filecoin-project/lotus/gateway" "github.com/filecoin-project/lotus/genesis" - "github.com/filecoin-project/lotus/lib/sturdy/clusterdb" + "github.com/filecoin-project/lotus/lib/harmony/harmonydb" "github.com/filecoin-project/lotus/markets/idxprov" "github.com/filecoin-project/lotus/markets/idxprov/idxprov_test" lotusminer "github.com/filecoin-project/lotus/miner" @@ -359,7 +359,7 @@ func (n *Ensemble) Start() *Ensemble { n.mn = mocknet.New() } - sharedITestID := clusterdb.ITestNewID() + sharedITestID := harmonydb.ITestNewID() // --------------------- // FULL NODES @@ -727,7 +727,7 @@ func (n *Ensemble) Start() *Ensemble { // upgrades node.Override(new(stmgr.UpgradeSchedule), n.options.upgradeSchedule), - node.Override(new(clusterdb.ITestID), sharedITestID), + node.Override(new(harmonydb.ITestID), sharedITestID), } if m.options.subsystems.Has(SMarkets) { @@ -775,7 +775,7 @@ func (n *Ensemble) Start() *Ensemble { n.t.Cleanup(func() { _ = stop(context.Background()) }) n.t.Cleanup(func() { - m.StorageMiner.(*impl.StorageMinerAPI).ClusterDB.ITestDeleteAll() + m.StorageMiner.(*impl.StorageMinerAPI).HarmonyDB.ITestDeleteAll() }) m.BaseAPI = m.StorageMiner @@ -833,7 +833,7 @@ func (n *Ensemble) Start() *Ensemble { auth := http.Header(nil) - // FUTURE: Use m.MinerNode.(BaseAPI).(impl.StorageMinerAPI).ClusterDB to setup. + // FUTURE: Use m.MinerNode.(BaseAPI).(impl.StorageMinerAPI).HarmonyDB to setup. remote := paths.NewRemote(localStore, m.MinerNode, auth, 20, &paths.DefaultPartialFileHandler{}) store := m.options.workerStorageOpt(remote) diff --git a/lib/sturdy/clusterdb/doc.go b/lib/harmony/harmonydb/doc.go similarity index 93% rename from lib/sturdy/clusterdb/doc.go rename to lib/harmony/harmonydb/doc.go index 6e86aacd7..ac60a0260 100644 --- a/lib/sturdy/clusterdb/doc.go +++ b/lib/harmony/harmonydb/doc.go @@ -1,5 +1,5 @@ /* -# Clusterdb provides database abstractions over SP-wide Postgres-compatible instance(s). +# HarmonyDB provides database abstractions over SP-wide Postgres-compatible instance(s). # Features @@ -32,4 +32,4 @@ Write SQL with context, raw strings, and args: Note: Scan() is column-oriented, while Select() & StructScan() is field name/tag oriented. */ -package clusterdb +package harmonydb diff --git a/lib/sturdy/clusterdb/clusterdb.go b/lib/harmony/harmonydb/harmonydb.go similarity index 96% rename from lib/sturdy/clusterdb/clusterdb.go rename to lib/harmony/harmonydb/harmonydb.go index 13a5db209..fdf57b846 100644 --- a/lib/sturdy/clusterdb/clusterdb.go +++ b/lib/harmony/harmonydb/harmonydb.go @@ -1,4 +1,4 @@ -package clusterdb +package harmonydb import ( "context" @@ -35,13 +35,13 @@ type DB struct { log func(string) } -var logger = logging.Logger("clusterdb") +var logger = logging.Logger("harmonydb") // NewFromConfig is a convenience function. // In usage: // -// db, err := NewFromConfig(config.ClusterDB) // in binary init -func NewFromConfig(cfg config.ClusterDB) (*DB, error) { +// db, err := NewFromConfig(config.HarmonyDB) // in binary init +func NewFromConfig(cfg config.HarmonyDB) (*DB, error) { return New( cfg.Hosts, cfg.Username, @@ -53,7 +53,7 @@ func NewFromConfig(cfg config.ClusterDB) (*DB, error) { ) } -func NewFromConfigWithITestID(cfg config.ClusterDB) func(id ITestID) (*DB, error) { +func NewFromConfigWithITestID(cfg config.HarmonyDB) func(id ITestID) (*DB, error) { return func(id ITestID) (*DB, error) { return New( cfg.Hosts, diff --git a/lib/sturdy/clusterdb/metrics.go b/lib/harmony/harmonydb/metrics.go similarity index 97% rename from lib/sturdy/clusterdb/metrics.go rename to lib/harmony/harmonydb/metrics.go index 430de54f2..fde11279f 100644 --- a/lib/sturdy/clusterdb/metrics.go +++ b/lib/harmony/harmonydb/metrics.go @@ -1,4 +1,4 @@ -package clusterdb +package harmonydb import ( "github.com/prometheus/client_golang/prometheus" @@ -11,7 +11,7 @@ import ( var ( dbTag, _ = tag.NewKey("db_name") - pre = "clusterdb/base/" + pre = "harmonydb/base/" waitsBuckets = []float64{0, 10, 20, 30, 50, 80, 130, 210, 340, 550, 890} whichHostBuckets = []float64{0, 1, 2, 3, 4, 5} ) diff --git a/lib/sturdy/clusterdb/sql/20230706.sql b/lib/harmony/harmonydb/sql/20230706.sql similarity index 100% rename from lib/sturdy/clusterdb/sql/20230706.sql rename to lib/harmony/harmonydb/sql/20230706.sql diff --git a/lib/sturdy/clusterdb/userfuncs.go b/lib/harmony/harmonydb/userfuncs.go similarity index 98% rename from lib/sturdy/clusterdb/userfuncs.go rename to lib/harmony/harmonydb/userfuncs.go index 98ceddb08..8eebdd607 100644 --- a/lib/sturdy/clusterdb/userfuncs.go +++ b/lib/harmony/harmonydb/userfuncs.go @@ -1,4 +1,4 @@ -package clusterdb +package harmonydb import ( "context" @@ -19,7 +19,7 @@ type Intf interface { // rawStringOnly is _intentionally_private_ to force only basic strings in SQL queries. // In any package, raw strings will satisfy compilation. Ex: // -// clusterDB.Exec("INSERT INTO version (number) VALUES (1)") +// harmonydb.Exec("INSERT INTO version (number) VALUES (1)") // // This prevents SQL injection attacks where the input contains query fragments. type rawStringOnly string diff --git a/node/builder_miner.go b/node/builder_miner.go index 366c89fa4..3b3fa8d48 100644 --- a/node/builder_miner.go +++ b/node/builder_miner.go @@ -19,7 +19,7 @@ import ( "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/gen" "github.com/filecoin-project/lotus/chain/gen/slashfilter" - "github.com/filecoin-project/lotus/lib/sturdy/clusterdb" + "github.com/filecoin-project/lotus/lib/harmony/harmonydb" "github.com/filecoin-project/lotus/markets/dagstore" "github.com/filecoin-project/lotus/markets/dealfilter" "github.com/filecoin-project/lotus/markets/idxprov" @@ -232,8 +232,8 @@ func ConfigStorageMiner(c interface{}) Option { Override(new(config.SealerConfig), cfg.Storage), Override(new(config.ProvingConfig), cfg.Proving), Override(new(*ctladdr.AddressSelector), modules.AddressSelector(&cfg.Addresses)), - Override(new(*clusterdb.DB), clusterdb.NewFromConfigWithITestID(cfg.ClusterDB)), - Override(new(clusterdb.ITestID), clusterdb.ITestID("")), + Override(new(*harmonydb.DB), harmonydb.NewFromConfigWithITestID(cfg.HarmonyDB)), + Override(new(harmonydb.ITestID), harmonydb.ITestID("")), ) } diff --git a/node/config/def.go b/node/config/def.go index 5075355dc..47d314fc0 100644 --- a/node/config/def.go +++ b/node/config/def.go @@ -269,7 +269,7 @@ func DefaultStorageMiner() *StorageMiner { MaxConcurrentUnseals: 5, GCInterval: Duration(1 * time.Minute), }, - ClusterDB: ClusterDB{ + HarmonyDB: HarmonyDB{ Hosts: []string{"127.0.0.1"}, Username: "yugabyte", Password: "yugabyte", diff --git a/node/config/doc_gen.go b/node/config/doc_gen.go index 9c3617a69..c513eecbc 100644 --- a/node/config/doc_gen.go +++ b/node/config/doc_gen.go @@ -117,46 +117,6 @@ without existing payment channels with available funds will fail instead of automatically performing on-chain operations.`, }, }, - "ClusterDB": []DocField{ - { - Name: "Hosts", - Type: "[]string", - - Comment: `HOSTS is a list of hostnames to nodes running YugabyteDB -in a cluster. Only 1 is required`, - }, - { - Name: "Username", - Type: "string", - - Comment: `The Yugabyte server's username with full credentials to operate on Lotus' Database. Blank for default.`, - }, - { - Name: "Password", - Type: "string", - - Comment: `The password for the related username. Blank for default.`, - }, - { - Name: "Database", - Type: "string", - - Comment: `The database (logical partition) within Yugabyte. Blank for default.`, - }, - { - Name: "Port", - Type: "string", - - Comment: `The port to find Yugabyte. Blank for default.`, - }, - { - Name: "ITest", - Type: "string", - - Comment: `ITest is for optimized integration testing and not -for production. Blank for default production configuration.`, - }, - }, "Common": []DocField{ { Name: "API", @@ -508,6 +468,46 @@ Set to 0 to keep all mappings`, Comment: ``, }, }, + "HarmonyDB": []DocField{ + { + Name: "Hosts", + Type: "[]string", + + Comment: `HOSTS is a list of hostnames to nodes running YugabyteDB +in a cluster. Only 1 is required`, + }, + { + Name: "Username", + Type: "string", + + Comment: `The Yugabyte server's username with full credentials to operate on Lotus' Database. Blank for default.`, + }, + { + Name: "Password", + Type: "string", + + Comment: `The password for the related username. Blank for default.`, + }, + { + Name: "Database", + Type: "string", + + Comment: `The database (logical partition) within Yugabyte. Blank for default.`, + }, + { + Name: "Port", + Type: "string", + + Comment: `The port to find Yugabyte. Blank for default.`, + }, + { + Name: "ITest", + Type: "string", + + Comment: `ITest is for optimized integration testing and not +for production. Blank for default production configuration.`, + }, + }, "IndexConfig": []DocField{ { Name: "EnableMsgIndex", @@ -1429,8 +1429,8 @@ HotstoreMaxSpaceTarget - HotstoreMaxSpaceSafetyBuffer`, Comment: ``, }, { - Name: "ClusterDB", - Type: "ClusterDB", + Name: "HarmonyDB", + Type: "HarmonyDB", Comment: ``, }, diff --git a/node/config/types.go b/node/config/types.go index 74630d9a3..e2d1e655d 100644 --- a/node/config/types.go +++ b/node/config/types.go @@ -62,7 +62,7 @@ type StorageMiner struct { Addresses MinerAddressConfig DAGStore DAGStoreConfig - ClusterDB ClusterDB + HarmonyDB HarmonyDB } type DAGStoreConfig struct { @@ -735,7 +735,7 @@ type IndexConfig struct { EnableMsgIndex bool } -type ClusterDB struct { +type HarmonyDB struct { // HOSTS is a list of hostnames to nodes running YugabyteDB // in a cluster. Only 1 is required Hosts []string diff --git a/node/impl/storminer.go b/node/impl/storminer.go index 89837baff..4932e0504 100644 --- a/node/impl/storminer.go +++ b/node/impl/storminer.go @@ -45,7 +45,7 @@ import ( "github.com/filecoin-project/lotus/chain/actors/builtin" "github.com/filecoin-project/lotus/chain/gen" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/lib/sturdy/clusterdb" + "github.com/filecoin-project/lotus/lib/harmony/harmonydb" mktsdagstore "github.com/filecoin-project/lotus/markets/dagstore" "github.com/filecoin-project/lotus/markets/storageadapter" "github.com/filecoin-project/lotus/miner" @@ -124,7 +124,7 @@ type StorageMinerAPI struct { GetExpectedSealDurationFunc dtypes.GetExpectedSealDurationFunc `optional:"true"` SetExpectedSealDurationFunc dtypes.SetExpectedSealDurationFunc `optional:"true"` - ClusterDB *clusterdb.DB + HarmonyDB *harmonydb.DB } var _ api.StorageMiner = &StorageMinerAPI{}