LaconicNetwork/cosmos-sdk
7
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 2 Wiki Activity

feat(schema/indexer)!: implement start indexing (#21636)

Browse Source 
Co-authored-by: Marko <marko@baricevic.me>
This commit is contained in:
Aaron Craelius
2024-09-23 16:09:01 +00:00
committed by GitHub
co-authored by Marko
parent f3a295a1a1
commit d273ae03da
17 changed files with 570 additions and 170 deletions
Download Patch File Download Diff File Expand all files Collapse all files
indexer/postgres/indexer.go
+12 -22
View File
@@ -3,8 +3,8 @@ package postgres
import (
"context"
"database/sql"
"encoding/json"
"errors"
"fmt"
"cosmossdk.io/schema/indexer"
"cosmossdk.io/schema/logutil"
@@ -21,8 +21,6 @@ type Config struct {
DisableRetainDeletions bool `json:"disable_retain_deletions"`
}
type SqlLogger = func(msg, sql string, params ...interface{})
type indexerImpl struct {
ctx context.Context
db *sql.DB
@@ -32,10 +30,17 @@ type indexerImpl struct {
logger logutil.Logger
}
func StartIndexer(params indexer.InitParams) (indexer.InitResult, error) {
config, err := decodeConfig(params.Config.Config)
if err != nil {
return indexer.InitResult{}, err
func init() {
indexer.Register("postgres", indexer.Initializer{
InitFunc: startIndexer,
ConfigType: Config{},
})
}
func startIndexer(params indexer.InitParams) (indexer.InitResult, error) {
config, ok := params.Config.Config.(Config)
if !ok {
return indexer.InitResult{}, fmt.Errorf("invalid config type, expected %T got %T", Config{}, params.Config.Config)
}
ctx := params.Context
@@ -89,18 +94,3 @@ func StartIndexer(params indexer.InitParams) (indexer.InitResult, error) {
View: idx,
}, nil
}
func decodeConfig(rawConfig map[string]interface{}) (*Config, error) {
bz, err := json.Marshal(rawConfig)
if err != nil {
return nil, err
}
var config Config
err = json.Unmarshal(bz, &config)
if err != nil {
return nil, err
}
return &config, nil
}
indexer/postgres/tests/config.go
-26
View File
@@ -1,26 +0,0 @@
package tests
import (
"encoding/json"
"cosmossdk.io/indexer/postgres"
"cosmossdk.io/schema/indexer"
)
func postgresConfigToIndexerConfig(cfg postgres.Config) (indexer.Config, error) {
cfgBz, err := json.Marshal(cfg)
if err != nil {
return indexer.Config{}, err
}
var cfgMap map[string]interface{}
err = json.Unmarshal(cfgBz, &cfgMap)
if err != nil {
return indexer.Config{}, err
}
return indexer.Config{
Type: "postgres",
Config: cfgMap,
}, nil
}
indexer/postgres/tests/init_schema_test.go
+13 -10
View File
@@ -33,17 +33,20 @@ func testInitSchema(t *testing.T, disableRetainDeletions bool, goldenFileName st
connectionUrl := createTestDB(t)
buf := &strings.Builder{}
cfg, err := postgresConfigToIndexerConfig(postgres.Config{
DatabaseURL: connectionUrl,
DisableRetainDeletions: disableRetainDeletions,
})
require.NoError(t, err)
res, err := postgres.StartIndexer(indexer.InitParams{
Config: cfg,
res, err := indexer.StartIndexing(indexer.IndexingOptions{
Config: indexer.IndexingConfig{
Target: map[string]indexer.Config{
"postgres": {
Type: "postgres",
Config: postgres.Config{
DatabaseURL: connectionUrl,
DisableRetainDeletions: disableRetainDeletions,
},
},
},
},
Context: context.Background(),
Logger: &prettyLogger{buf},
Logger: prettyLogger{buf},
})
require.NoError(t, err)
listener := res.Listener
indexer/postgres/tests/postgres_test.go
+18 -10
View File
@@ -52,24 +52,29 @@ func testPostgresIndexer(t *testing.T, retainDeletions bool) {
require.NoError(t, err)
})
cfg, err := postgresConfigToIndexerConfig(postgres.Config{
DatabaseURL: dbUrl,
DisableRetainDeletions: !retainDeletions,
})
require.NoError(t, err)
debugLog := &strings.Builder{}
pgIndexer, err := postgres.StartIndexer(indexer.InitParams{
Config: cfg,
res, err := indexer.StartIndexing(indexer.IndexingOptions{
Config: indexer.IndexingConfig{
Target: map[string]indexer.Config{
"postgres": {
Type: "postgres",
Config: postgres.Config{
DatabaseURL: dbUrl,
DisableRetainDeletions: !retainDeletions,
},
},
},
},
Context: ctx,
Logger: &prettyLogger{debugLog},
AddressCodec: addressutil.HexAddressCodec{},
})
require.NoError(t, err)
require.NoError(t, err)
sim, err := appdatasim.NewSimulator(appdatasim.Options{
Listener: pgIndexer.Listener,
Listener: res.Listener,
AppSchema: indexertesting.ExampleAppSchema,
StateSimOptions: statesim.Options{
CanRetainDeletions: retainDeletions,
@@ -77,6 +82,9 @@ func testPostgresIndexer(t *testing.T, retainDeletions bool) {
})
require.NoError(t, err)
pgIndexerView := res.IndexerInfos["postgres"].View
require.NotNil(t, pgIndexerView)
blockDataGen := sim.BlockDataGenN(10, 100)
numBlocks := 200
if testing.Short() {
@@ -93,7 +101,7 @@ func testPostgresIndexer(t *testing.T, retainDeletions bool) {
require.NoError(t, sim.ProcessBlockData(blockData), debugLog.String())
// compare the expected state in the simulator to the actual state in the indexer and expect the diff to be empty
require.Empty(t, appdatasim.DiffAppData(sim, pgIndexer.View), debugLog.String())
require.Empty(t, appdatasim.DiffAppData(sim, pgIndexerView), debugLog.String())
// reset the debug log after each successful block so that it doesn't get too long when debugging
debugLog.Reset()
indexer/postgres/tests/testdata/init_schema.txt
Vendored
+4
View File
@@ -1,3 +1,7 @@
INFO: Starting indexing
INFO: Starting indexer
target_name: postgres
type: postgres
DEBUG: Creating enum type
sql: CREATE TYPE "test_my_enum" AS ENUM ('a', 'b', 'c');
DEBUG: Creating enum type
indexer/postgres/tests/testdata/init_schema_no_retain_delete.txt
Vendored
+4
View File
@@ -1,3 +1,7 @@
INFO: Starting indexing
INFO: Starting indexer
target_name: postgres
type: postgres
DEBUG: Creating enum type
sql: CREATE TYPE "test_my_enum" AS ENUM ('a', 'b', 'c');
DEBUG: Creating enum type