From 9376db550849789195dc2f584a4469c24026ae92 Mon Sep 17 00:00:00 2001 From: Aaron Craelius Date: Thu, 18 Jul 2024 11:34:09 +0200 Subject: [PATCH] feat(indexer): postgres schema creation + CI config (#20701) Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> Co-authored-by: marbar3778 --- .github/dependabot.yml | 18 +++ .github/pr_labeler.yml | 2 + .github/workflows/test.yml | 36 ++++++ go.work.example | 1 + indexer/postgres/CHANGELOG.md | 37 ++++++ indexer/postgres/README.md | 41 ++++++ indexer/postgres/base_sql.go | 8 ++ indexer/postgres/column.go | 120 ++++++++++++++++++ indexer/postgres/conn.go | 14 ++ indexer/postgres/create_table.go | 96 ++++++++++++++ indexer/postgres/create_table_test.go | 94 ++++++++++++++ indexer/postgres/enum.go | 92 ++++++++++++++ indexer/postgres/enum_test.go | 16 +++ indexer/postgres/go.mod | 11 ++ indexer/postgres/go.sum | 2 + indexer/postgres/indexer.go | 80 ++++++++++++ .../internal/testdata/example_schema.go | 98 ++++++++++++++ indexer/postgres/module.go | 61 +++++++++ indexer/postgres/object.go | 44 +++++++ indexer/postgres/options.go | 10 ++ indexer/postgres/sonar-project.properties | 16 +++ indexer/postgres/tests/README.md | 3 + indexer/postgres/tests/go.mod | 33 +++++ indexer/postgres/tests/go.sum | 56 ++++++++ indexer/postgres/tests/init_schema_test.go | 89 +++++++++++++ .../postgres/tests/testdata/init_schema.txt | 56 ++++++++ .../testdata/init_schema_no_retain_delete.txt | 55 ++++++++ schema/field.go | 6 +- schema/object_type.go | 6 +- 29 files changed, 1195 insertions(+), 6 deletions(-) create mode 100644 indexer/postgres/CHANGELOG.md create mode 100644 indexer/postgres/README.md create mode 100644 indexer/postgres/base_sql.go create mode 100644 indexer/postgres/column.go create mode 100644 indexer/postgres/conn.go create mode 100644 indexer/postgres/create_table.go create mode 100644 indexer/postgres/create_table_test.go create mode 100644 indexer/postgres/enum.go create mode 100644 indexer/postgres/enum_test.go create mode 100644 indexer/postgres/go.mod create mode 100644 indexer/postgres/go.sum create mode 100644 indexer/postgres/indexer.go create mode 100644 indexer/postgres/internal/testdata/example_schema.go create mode 100644 indexer/postgres/module.go create mode 100644 indexer/postgres/object.go create mode 100644 indexer/postgres/options.go create mode 100644 indexer/postgres/sonar-project.properties create mode 100644 indexer/postgres/tests/README.md create mode 100644 indexer/postgres/tests/go.mod create mode 100644 indexer/postgres/tests/go.sum create mode 100644 indexer/postgres/tests/init_schema_test.go create mode 100644 indexer/postgres/tests/testdata/init_schema.txt create mode 100644 indexer/postgres/tests/testdata/init_schema_no_retain_delete.txt diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 7c4a08af17..7af739bbcf 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -106,6 +106,24 @@ updates: labels: - "A:automerge" - dependencies + - package-ecosystem: gomod + directory: "/indexer/postgres" + schedule: + interval: weekly + day: wednesday + time: "01:53" + labels: + - "A:automerge" + - dependencies + - package-ecosystem: gomod + directory: "/indexer/postgres/tests" + schedule: + interval: weekly + day: wednesday + time: "01:53" + labels: + - "A:automerge" + - dependencies - package-ecosystem: gomod directory: "/schema" schedule: diff --git a/.github/pr_labeler.yml b/.github/pr_labeler.yml index b935cf3195..8b710d5fd9 100644 --- a/.github/pr_labeler.yml +++ b/.github/pr_labeler.yml @@ -24,6 +24,8 @@ - orm/**/* "C:schema": - schema/**/* +"C:indexer/postgres": + - indexer/postgres/**/* "C:x/accounts": - x/accounts/**/* "C:x/accounts/multisig": diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9515c80f7b..6eea986e27 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -487,6 +487,42 @@ jobs: with: projectBaseDir: schema/ + test-indexer-postgres: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: "1.22" + cache: true + cache-dependency-path: indexer/postgres/tests/go.sum + - uses: technote-space/get-diff-action@v6.1.2 + id: git_diff + with: + PATTERNS: | + indexer/postgres/**/*.go + indexer/postgres/go.mod + indexer/postgres/go.sum + indexer/postgres/tests/go.mod + indexer/postgres/tests/go.sum + - name: tests + if: env.GIT_DIFF + run: | + cd indexer/postgres + go test -mod=readonly -timeout 30m -coverprofile=cov.out -covermode=atomic ./... + cd tests + go test -mod=readonly -timeout 30m -coverprofile=cov.out -covermode=atomic -coverpkg=cosmossdk.io/indexer/postgres ./... + cd .. + go run github.com/dylandreimerink/gocovmerge/cmd/gocovmerge@latest cov.out tests/cov.out > coverage.out + - name: sonarcloud + if: ${{ env.GIT_DIFF && !github.event.pull_request.draft && env.SONAR_TOKEN != null }} + uses: SonarSource/sonarcloud-github-action@master + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} + with: + projectBaseDir: indexer/postgres/ + test-simapp: runs-on: ubuntu-latest steps: diff --git a/go.work.example b/go.work.example index 5e0f392aad..035cbb3e34 100644 --- a/go.work.example +++ b/go.work.example @@ -9,6 +9,7 @@ use ( ./core/testing ./depinject ./errors + ./indexer/postgres ./log ./math ./orm diff --git a/indexer/postgres/CHANGELOG.md b/indexer/postgres/CHANGELOG.md new file mode 100644 index 0000000000..0c3c9d0385 --- /dev/null +++ b/indexer/postgres/CHANGELOG.md @@ -0,0 +1,37 @@ + + +# Changelog + +## [Unreleased] diff --git a/indexer/postgres/README.md b/indexer/postgres/README.md new file mode 100644 index 0000000000..bb8c480f66 --- /dev/null +++ b/indexer/postgres/README.md @@ -0,0 +1,41 @@ +# PostgreSQL Indexer + +The PostgreSQL indexer can fully index the current state for all modules that implement `cosmossdk.io/schema.HasModuleCodec`. +implement `cosmossdk.io/schema.HasModuleCodec`. + +## Table, Column and Enum Naming + +`ObjectType`s names are converted to table names prefixed with the module name and an underscore. i.e. the `ObjectType` `foo` in module `bar` will be stored in a table named `bar_foo`. + +Column names are identical to field names. All identifiers are quoted with double quotes so that they are case-sensitive and won't clash with any reserved names. + +Like, table names, enum types are prefixed with the module name and an underscore. + +## Schema Type Mapping + +The mapping of `cosmossdk.io/schema` `Kind`s to PostgreSQL types is as follows: + +| Kind | PostgreSQL Type | Notes | +|---------------------|----------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `StringKind` | `TEXT` | | +| `BoolKind` | `BOOLEAN` | | +| `BytesKind` | `BYTEA` | | +| `Int8Kind` | `SMALLINT` | | +| `Int16Kind` | `SMALLINT` | | +| `Int32Kind` | `INTEGER` | | +| `Int64Kind` | `BIGINT` | | +| `Uint8Kind` | `SMALLINT` | | +| `Uint16Kind` | `INTEGER` | | +| `Uint32Kind` | `BIGINT` | | +| `Uint64Kind` | `NUMERIC` | | +| `Float32Kind` | `REAL` | | +| `Float64Kind` | `DOUBLE PRECISION` | | +| `IntegerStringKind` | `NUMERIC` | | +| `DecimalStringKind` | `NUMERIC` | | +| `JSONKind` | `JSONB` | | +| `Bech32AddressKind` | `TEXT` | addresses are converted to strings with the specified address prefix | +| `TimeKind` | `BIGINT` and `TIMESTAMPTZ` | time types are stored as two columns, one with the `_nanos` suffix with full nanoseconds precision, and another as a `TIMESTAMPTZ` generated column with microsecond precision | +| `DurationKind` | `BIGINT` | durations are stored as a single column in nanoseconds | +| `EnumKind` | `_` | a custom enum type is created for each module prefixed with the module name it pertains to | + + diff --git a/indexer/postgres/base_sql.go b/indexer/postgres/base_sql.go new file mode 100644 index 0000000000..81e1ac7042 --- /dev/null +++ b/indexer/postgres/base_sql.go @@ -0,0 +1,8 @@ +package postgres + +// BaseSQL is the base SQL that is always included in the schema. +const BaseSQL = ` +CREATE OR REPLACE FUNCTION nanos_to_timestamptz(nanos bigint) RETURNS timestamptz AS $$ + SELECT to_timestamp(nanos / 1000000000) + (nanos / 1000000000) * INTERVAL '1 microsecond' +$$ LANGUAGE SQL IMMUTABLE; +` diff --git a/indexer/postgres/column.go b/indexer/postgres/column.go new file mode 100644 index 0000000000..f9692af137 --- /dev/null +++ b/indexer/postgres/column.go @@ -0,0 +1,120 @@ +package postgres + +import ( + "fmt" + "io" + + "cosmossdk.io/schema" +) + +// createColumnDefinition writes a column definition within a CREATE TABLE statement for the field. +func (tm *ObjectIndexer) createColumnDefinition(writer io.Writer, field schema.Field) error { + _, err := fmt.Fprintf(writer, "%q ", field.Name) + if err != nil { + return err + } + + simple := simpleColumnType(field.Kind) + if simple != "" { + _, err = fmt.Fprintf(writer, "%s", simple) + if err != nil { + return err + } + + return writeNullability(writer, field.Nullable) + } else { + switch field.Kind { + case schema.EnumKind: + _, err = fmt.Fprintf(writer, "%q", enumTypeName(tm.moduleName, field.EnumDefinition)) + if err != nil { + return err + } + case schema.TimeKind: + // for time fields, we generate two columns: + // - one with nanoseconds precision for lossless storage, suffixed with _nanos + // - one as a timestamptz (microsecond precision) for ease of use, that is GENERATED + nanosColName := fmt.Sprintf("%s_nanos", field.Name) + _, err = fmt.Fprintf(writer, "TIMESTAMPTZ GENERATED ALWAYS AS (nanos_to_timestamptz(%q)) STORED,\n\t", nanosColName) + if err != nil { + return err + } + + _, err = fmt.Fprintf(writer, `%q BIGINT`, nanosColName) + if err != nil { + return err + } + default: + return fmt.Errorf("unexpected kind: %v, this should have been handled earlier", field.Kind) + } + + return writeNullability(writer, field.Nullable) + } +} + +// writeNullability writes column nullability. +func writeNullability(writer io.Writer, nullable bool) error { + if nullable { + _, err := fmt.Fprintf(writer, " NULL,\n\t") + return err + } else { + _, err := fmt.Fprintf(writer, " NOT NULL,\n\t") + return err + } +} + +// simpleColumnType returns the postgres column type for the kind for simple types. +func simpleColumnType(kind schema.Kind) string { + //nolint:goconst // adding constants for these postgres type names would impede readability + switch kind { + case schema.StringKind: + return "TEXT" + case schema.BoolKind: + return "BOOLEAN" + case schema.BytesKind: + return "BYTEA" + case schema.Int8Kind: + return "SMALLINT" + case schema.Int16Kind: + return "SMALLINT" + case schema.Int32Kind: + return "INTEGER" + case schema.Int64Kind: + return "BIGINT" + case schema.Uint8Kind: + return "SMALLINT" + case schema.Uint16Kind: + return "INTEGER" + case schema.Uint32Kind: + return "BIGINT" + case schema.Uint64Kind: + return "NUMERIC" + case schema.IntegerStringKind: + return "NUMERIC" + case schema.DecimalStringKind: + return "NUMERIC" + case schema.Float32Kind: + return "REAL" + case schema.Float64Kind: + return "DOUBLE PRECISION" + case schema.JSONKind: + return "JSONB" + case schema.DurationKind: + return "BIGINT" + case schema.Bech32AddressKind: + return "TEXT" + default: + return "" + } +} + +// updatableColumnName is the name of the insertable/updatable column name for the field. +// This is the field name in most cases, except for time columns which are stored as nanos +// and then converted to timestamp generated columns. +func (tm *ObjectIndexer) updatableColumnName(field schema.Field) (name string, err error) { + name = field.Name + if field.Kind == schema.TimeKind { + name = fmt.Sprintf("%s_nanos", name) + } + name = fmt.Sprintf("%q", name) + return +} diff --git a/indexer/postgres/conn.go b/indexer/postgres/conn.go new file mode 100644 index 0000000000..de8c1cac6b --- /dev/null +++ b/indexer/postgres/conn.go @@ -0,0 +1,14 @@ +package postgres + +import ( + "context" + "database/sql" +) + +// DBConn is an interface that abstracts the *sql.DB, *sql.Tx and *sql.Conn types. +type DBConn interface { + ExecContext(ctx context.Context, query string, args ...interface{}) (sql.Result, error) + PrepareContext(ctx context.Context, query string) (*sql.Stmt, error) + QueryContext(ctx context.Context, query string, args ...interface{}) (*sql.Rows, error) + QueryRowContext(ctx context.Context, query string, args ...interface{}) *sql.Row +} diff --git a/indexer/postgres/create_table.go b/indexer/postgres/create_table.go new file mode 100644 index 0000000000..8f5f0e6ca2 --- /dev/null +++ b/indexer/postgres/create_table.go @@ -0,0 +1,96 @@ +package postgres + +import ( + "context" + "fmt" + "io" + "strings" +) + +// CreateTable creates the table for the object type. +func (tm *ObjectIndexer) CreateTable(ctx context.Context, conn DBConn) error { + buf := new(strings.Builder) + err := tm.CreateTableSql(buf) + if err != nil { + return err + } + + sqlStr := buf.String() + if tm.options.Logger != nil { + tm.options.Logger(fmt.Sprintf("Creating table %s", tm.TableName()), sqlStr) + } + _, err = conn.ExecContext(ctx, sqlStr) + return err +} + +// CreateTableSql generates a CREATE TABLE statement for the object type. +func (tm *ObjectIndexer) CreateTableSql(writer io.Writer) error { + _, err := fmt.Fprintf(writer, "CREATE TABLE IF NOT EXISTS %q (\n\t", tm.TableName()) + if err != nil { + return err + } + isSingleton := false + if len(tm.typ.KeyFields) == 0 { + isSingleton = true + _, err = fmt.Fprintf(writer, "_id INTEGER NOT NULL CHECK (_id = 1),\n\t") + if err != nil { + return err + } + } else { + for _, field := range tm.typ.KeyFields { + err = tm.createColumnDefinition(writer, field) + if err != nil { + return err + } + } + } + + for _, field := range tm.typ.ValueFields { + err = tm.createColumnDefinition(writer, field) + if err != nil { + return err + } + } + + // add _deleted column when we have RetainDeletions set and enabled + if !tm.options.DisableRetainDeletions && tm.typ.RetainDeletions { + _, err = fmt.Fprintf(writer, "_deleted BOOLEAN NOT NULL DEFAULT FALSE,\n\t") + if err != nil { + return err + } + } + + var pKeys []string + if !isSingleton { + for _, field := range tm.typ.KeyFields { + name, err := tm.updatableColumnName(field) + if err != nil { + return err + } + + pKeys = append(pKeys, name) + } + } else { + pKeys = []string{"_id"} + } + + _, err = fmt.Fprintf(writer, "PRIMARY KEY (%s)", strings.Join(pKeys, ", ")) + if err != nil { + return err + } + + _, err = fmt.Fprintf(writer, "\n);\n") + if err != nil { + return err + } + + // we GRANT SELECT on the table to PUBLIC so that the table is automatically available + // for querying using off-the-shelf tools like pg_graphql, Postgrest, Postgraphile, etc. + // without any login permissions + _, err = fmt.Fprintf(writer, "GRANT SELECT ON TABLE %q TO PUBLIC;", tm.TableName()) + if err != nil { + return err + } + + return nil +} diff --git a/indexer/postgres/create_table_test.go b/indexer/postgres/create_table_test.go new file mode 100644 index 0000000000..dec09d7aed --- /dev/null +++ b/indexer/postgres/create_table_test.go @@ -0,0 +1,94 @@ +package postgres + +import ( + "os" + + "cosmossdk.io/indexer/postgres/internal/testdata" + "cosmossdk.io/schema" +) + +func ExampleObjectIndexer_CreateTableSql_allKinds() { + exampleCreateTable(testdata.AllKindsObject) + // Output: + // CREATE TABLE IF NOT EXISTS "test_all_kinds" ( + // "id" BIGINT NOT NULL, + // "ts" TIMESTAMPTZ GENERATED ALWAYS AS (nanos_to_timestamptz("ts_nanos")) STORED, + // "ts_nanos" BIGINT NOT NULL, + // "string" TEXT NOT NULL, + // "bytes" BYTEA NOT NULL, + // "int8" SMALLINT NOT NULL, + // "uint8" SMALLINT NOT NULL, + // "int16" SMALLINT NOT NULL, + // "uint16" INTEGER NOT NULL, + // "int32" INTEGER NOT NULL, + // "uint32" BIGINT NOT NULL, + // "int64" BIGINT NOT NULL, + // "uint64" NUMERIC NOT NULL, + // "integer" NUMERIC NOT NULL, + // "decimal" NUMERIC NOT NULL, + // "bool" BOOLEAN NOT NULL, + // "time" TIMESTAMPTZ GENERATED ALWAYS AS (nanos_to_timestamptz("time_nanos")) STORED, + // "time_nanos" BIGINT NOT NULL, + // "duration" BIGINT NOT NULL, + // "float32" REAL NOT NULL, + // "float64" DOUBLE PRECISION NOT NULL, + // "bech32address" TEXT NOT NULL, + // "enum" "test_my_enum" NOT NULL, + // "json" JSONB NOT NULL, + // PRIMARY KEY ("id", "ts_nanos") + // ); + // GRANT SELECT ON TABLE "test_all_kinds" TO PUBLIC; +} + +func ExampleObjectIndexer_CreateTableSql_singleton() { + exampleCreateTable(testdata.SingletonObject) + // Output: + // CREATE TABLE IF NOT EXISTS "test_singleton" ( + // _id INTEGER NOT NULL CHECK (_id = 1), + // "foo" TEXT NOT NULL, + // "bar" INTEGER NULL, + // "an_enum" "test_my_enum" NOT NULL, + // PRIMARY KEY (_id) + // ); + // GRANT SELECT ON TABLE "test_singleton" TO PUBLIC; +} + +func ExampleObjectIndexer_CreateTableSql_vote() { + exampleCreateTable(testdata.VoteObject) + // Output: + // CREATE TABLE IF NOT EXISTS "test_vote" ( + // "proposal" BIGINT NOT NULL, + // "address" TEXT NOT NULL, + // "vote" "test_vote_type" NOT NULL, + // _deleted BOOLEAN NOT NULL DEFAULT FALSE, + // PRIMARY KEY ("proposal", "address") + // ); + // GRANT SELECT ON TABLE "test_vote" TO PUBLIC; +} + +func ExampleObjectIndexer_CreateTableSql_vote_no_retain_delete() { + exampleCreateTableOpt(testdata.VoteObject, true) + // Output: + // CREATE TABLE IF NOT EXISTS "test_vote" ( + // "proposal" BIGINT NOT NULL, + // "address" TEXT NOT NULL, + // "vote" "test_vote_type" NOT NULL, + // PRIMARY KEY ("proposal", "address") + // ); + // GRANT SELECT ON TABLE "test_vote" TO PUBLIC; +} + +func exampleCreateTable(objectType schema.ObjectType) { + exampleCreateTableOpt(objectType, false) +} + +func exampleCreateTableOpt(objectType schema.ObjectType, noRetainDelete bool) { + tm := NewObjectIndexer("test", objectType, Options{ + Logger: func(msg, sql string, params ...interface{}) {}, + DisableRetainDeletions: noRetainDelete, + }) + err := tm.CreateTableSql(os.Stdout) + if err != nil { + panic(err) + } +} diff --git a/indexer/postgres/enum.go b/indexer/postgres/enum.go new file mode 100644 index 0000000000..c438257d20 --- /dev/null +++ b/indexer/postgres/enum.go @@ -0,0 +1,92 @@ +package postgres + +import ( + "context" + "database/sql" + "fmt" + "io" + "strings" + + "cosmossdk.io/schema" +) + +// CreateEnumType creates an enum type in the database. +func (m *ModuleIndexer) CreateEnumType(ctx context.Context, conn DBConn, enum schema.EnumDefinition) error { + typeName := enumTypeName(m.moduleName, enum) + row := conn.QueryRowContext(ctx, "SELECT 1 FROM pg_type WHERE typname = $1", typeName) + var res interface{} + if err := row.Scan(&res); err != nil { + if err != sql.ErrNoRows { + return fmt.Errorf("failed to check if enum type %q exists: %v", typeName, err) //nolint:errorlint // using %v for go 1.12 compat + } + } else { + // the enum type already exists + return nil + } + + buf := new(strings.Builder) + err := CreateEnumTypeSql(buf, m.moduleName, enum) + if err != nil { + return err + } + + sqlStr := buf.String() + if m.options.Logger != nil { + m.options.Logger("Creating enum type", sqlStr) + } + _, err = conn.ExecContext(ctx, sqlStr) + return err +} + +// CreateEnumTypeSql generates a CREATE TYPE statement for the enum definition. +func CreateEnumTypeSql(writer io.Writer, moduleName string, enum schema.EnumDefinition) error { + _, err := fmt.Fprintf(writer, "CREATE TYPE %q AS ENUM (", enumTypeName(moduleName, enum)) + if err != nil { + return err + } + + for i, value := range enum.Values { + if i > 0 { + _, err = fmt.Fprintf(writer, ", ") + if err != nil { + return err + } + } + _, err = fmt.Fprintf(writer, "'%s'", value) + if err != nil { + return err + } + } + + _, err = fmt.Fprintf(writer, ");") + return err +} + +// enumTypeName returns the name of the enum type scoped to the module. +func enumTypeName(moduleName string, enum schema.EnumDefinition) string { + return fmt.Sprintf("%s_%s", moduleName, enum.Name) +} + +// createEnumTypesForFields creates enum types for all the fields that have enum kind in the module schema. +func (m *ModuleIndexer) createEnumTypesForFields(ctx context.Context, conn DBConn, fields []schema.Field) error { + for _, field := range fields { + if field.Kind != schema.EnumKind { + continue + } + + if _, ok := m.definedEnums[field.EnumDefinition.Name]; ok { + // if the enum type is already defined, skip + // we assume validation already happened + continue + } + + err := m.CreateEnumType(ctx, conn, field.EnumDefinition) + if err != nil { + return err + } + + m.definedEnums[field.EnumDefinition.Name] = field.EnumDefinition + } + + return nil +} diff --git a/indexer/postgres/enum_test.go b/indexer/postgres/enum_test.go new file mode 100644 index 0000000000..22d8870171 --- /dev/null +++ b/indexer/postgres/enum_test.go @@ -0,0 +1,16 @@ +package postgres + +import ( + "os" + + "cosmossdk.io/indexer/postgres/internal/testdata" +) + +func ExampleCreateEnumTypeSql() { + err := CreateEnumTypeSql(os.Stdout, "test", testdata.MyEnum) + if err != nil { + panic(err) + } + // Output: + // CREATE TYPE "test_my_enum" AS ENUM ('a', 'b', 'c'); +} diff --git a/indexer/postgres/go.mod b/indexer/postgres/go.mod new file mode 100644 index 0000000000..d85dbc4671 --- /dev/null +++ b/indexer/postgres/go.mod @@ -0,0 +1,11 @@ +module cosmossdk.io/indexer/postgres + +// NOTE: we are staying on an earlier version of golang to avoid problems building +// with older codebases. +go 1.12 + +// NOTE: cosmossdk.io/schema should be the only dependency here +// so there are no problems building this with any version of the SDK. +// This module should only use the golang standard library (database/sql) +// and cosmossdk.io/indexer/base. +require cosmossdk.io/schema v0.1.1 diff --git a/indexer/postgres/go.sum b/indexer/postgres/go.sum new file mode 100644 index 0000000000..6a92c3d3ec --- /dev/null +++ b/indexer/postgres/go.sum @@ -0,0 +1,2 @@ +cosmossdk.io/schema v0.1.1 h1:I0M6pgI7R10nq+/HCQfbO6BsGBZA8sQy+duR1Y3aKcA= +cosmossdk.io/schema v0.1.1/go.mod h1:RDAhxIeNB4bYqAlF4NBJwRrgtnciMcyyg0DOKnhNZQQ= diff --git a/indexer/postgres/indexer.go b/indexer/postgres/indexer.go new file mode 100644 index 0000000000..afcd8e0d8d --- /dev/null +++ b/indexer/postgres/indexer.go @@ -0,0 +1,80 @@ +package postgres + +import ( + "context" + "database/sql" + "fmt" + + "cosmossdk.io/schema/appdata" +) + +type Config struct { + // DatabaseURL is the PostgreSQL connection URL to use to connect to the database. + DatabaseURL string `json:"database_url"` + + // DatabaseDriver is the PostgreSQL database/sql driver to use. This defaults to "pgx". + DatabaseDriver string `json:"database_driver"` + + // DisableRetainDeletions disables the retain deletions functionality even if it is set in an object type schema. + DisableRetainDeletions bool `json:"disable_retain_deletions"` +} + +type SqlLogger = func(msg, sql string, params ...interface{}) + +func StartIndexer(ctx context.Context, logger SqlLogger, config Config) (appdata.Listener, error) { + if config.DatabaseURL == "" { + return appdata.Listener{}, fmt.Errorf("missing database URL") + } + + driver := config.DatabaseDriver + if driver == "" { + driver = "pgx" + } + + db, err := sql.Open(driver, config.DatabaseURL) + if err != nil { + return appdata.Listener{}, err + } + + tx, err := db.BeginTx(ctx, nil) + if err != nil { + return appdata.Listener{}, err + } + + // commit base schema + _, err = tx.Exec(BaseSQL) + if err != nil { + return appdata.Listener{}, err + } + + moduleIndexers := map[string]*ModuleIndexer{} + opts := Options{ + DisableRetainDeletions: config.DisableRetainDeletions, + Logger: logger, + } + + return appdata.Listener{ + InitializeModuleData: func(data appdata.ModuleInitializationData) error { + moduleName := data.ModuleName + modSchema := data.Schema + _, ok := moduleIndexers[moduleName] + if ok { + return fmt.Errorf("module %s already initialized", moduleName) + } + + mm := NewModuleIndexer(moduleName, modSchema, opts) + moduleIndexers[moduleName] = mm + + return mm.InitializeSchema(ctx, tx) + }, + Commit: func(data appdata.CommitData) error { + err = tx.Commit() + if err != nil { + return err + } + + tx, err = db.BeginTx(ctx, nil) + return err + }, + }, nil +} diff --git a/indexer/postgres/internal/testdata/example_schema.go b/indexer/postgres/internal/testdata/example_schema.go new file mode 100644 index 0000000000..ccdd39d96c --- /dev/null +++ b/indexer/postgres/internal/testdata/example_schema.go @@ -0,0 +1,98 @@ +package testdata + +import "cosmossdk.io/schema" + +var ExampleSchema schema.ModuleSchema + +var AllKindsObject schema.ObjectType + +func init() { + AllKindsObject = schema.ObjectType{ + Name: "all_kinds", + KeyFields: []schema.Field{ + { + Name: "id", + Kind: schema.Int64Kind, + }, + { + Name: "ts", + Kind: schema.TimeKind, + }, + }, + } + + for i := schema.InvalidKind + 1; i <= schema.MAX_VALID_KIND; i++ { + field := schema.Field{ + Name: i.String(), + Kind: i, + } + + switch i { + case schema.EnumKind: + field.EnumDefinition = MyEnum + case schema.Bech32AddressKind: + field.AddressPrefix = "foo" + default: + } + + AllKindsObject.ValueFields = append(AllKindsObject.ValueFields, field) + } + + ExampleSchema = schema.ModuleSchema{ + ObjectTypes: []schema.ObjectType{ + AllKindsObject, + SingletonObject, + VoteObject, + }, + } +} + +var SingletonObject = schema.ObjectType{ + Name: "singleton", + ValueFields: []schema.Field{ + { + Name: "foo", + Kind: schema.StringKind, + }, + { + Name: "bar", + Kind: schema.Int32Kind, + Nullable: true, + }, + { + Name: "an_enum", + Kind: schema.EnumKind, + EnumDefinition: MyEnum, + }, + }, +} + +var VoteObject = schema.ObjectType{ + Name: "vote", + KeyFields: []schema.Field{ + { + Name: "proposal", + Kind: schema.Int64Kind, + }, + { + Name: "address", + Kind: schema.Bech32AddressKind, + }, + }, + ValueFields: []schema.Field{ + { + Name: "vote", + Kind: schema.EnumKind, + EnumDefinition: schema.EnumDefinition{ + Name: "vote_type", + Values: []string{"yes", "no", "abstain"}, + }, + }, + }, + RetainDeletions: true, +} + +var MyEnum = schema.EnumDefinition{ + Name: "my_enum", + Values: []string{"a", "b", "c"}, +} diff --git a/indexer/postgres/module.go b/indexer/postgres/module.go new file mode 100644 index 0000000000..57564700b7 --- /dev/null +++ b/indexer/postgres/module.go @@ -0,0 +1,61 @@ +package postgres + +import ( + "context" + "fmt" + + "cosmossdk.io/schema" +) + +// ModuleIndexer manages the tables for a module. +type ModuleIndexer struct { + moduleName string + schema schema.ModuleSchema + tables map[string]*ObjectIndexer + definedEnums map[string]schema.EnumDefinition + options Options +} + +// NewModuleIndexer creates a new ModuleIndexer for the given module schema. +func NewModuleIndexer(moduleName string, modSchema schema.ModuleSchema, options Options) *ModuleIndexer { + return &ModuleIndexer{ + moduleName: moduleName, + schema: modSchema, + tables: map[string]*ObjectIndexer{}, + definedEnums: map[string]schema.EnumDefinition{}, + options: options, + } +} + +// InitializeSchema creates tables for all object types in the module schema and creates enum types. +func (m *ModuleIndexer) InitializeSchema(ctx context.Context, conn DBConn) error { + // create enum types + for _, typ := range m.schema.ObjectTypes { + err := m.createEnumTypesForFields(ctx, conn, typ.KeyFields) + if err != nil { + return err + } + + err = m.createEnumTypesForFields(ctx, conn, typ.ValueFields) + if err != nil { + return err + } + } + + // create tables for all object types + for _, typ := range m.schema.ObjectTypes { + tm := NewObjectIndexer(m.moduleName, typ, m.options) + m.tables[typ.Name] = tm + err := tm.CreateTable(ctx, conn) + if err != nil { + return fmt.Errorf("failed to create table for %s in module %s: %v", typ.Name, m.moduleName, err) //nolint:errorlint // using %v for go 1.12 compat + } + } + + return nil +} + +// ObjectIndexers returns the object indexers for the module. +func (m *ModuleIndexer) ObjectIndexers() map[string]*ObjectIndexer { + return m.tables +} diff --git a/indexer/postgres/object.go b/indexer/postgres/object.go new file mode 100644 index 0000000000..78bbfdf636 --- /dev/null +++ b/indexer/postgres/object.go @@ -0,0 +1,44 @@ +package postgres + +import ( + "fmt" + + "cosmossdk.io/schema" +) + +// ObjectIndexer is a helper struct that generates SQL for a given object type. +type ObjectIndexer struct { + moduleName string + typ schema.ObjectType + valueFields map[string]schema.Field + allFields map[string]schema.Field + options Options +} + +// NewObjectIndexer creates a new ObjectIndexer for the given object type. +func NewObjectIndexer(moduleName string, typ schema.ObjectType, options Options) *ObjectIndexer { + allFields := make(map[string]schema.Field) + valueFields := make(map[string]schema.Field) + + for _, field := range typ.KeyFields { + allFields[field.Name] = field + } + + for _, field := range typ.ValueFields { + valueFields[field.Name] = field + allFields[field.Name] = field + } + + return &ObjectIndexer{ + moduleName: moduleName, + typ: typ, + allFields: allFields, + valueFields: valueFields, + options: options, + } +} + +// TableName returns the name of the table for the object type scoped to its module. +func (tm *ObjectIndexer) TableName() string { + return fmt.Sprintf("%s_%s", tm.moduleName, tm.typ.Name) +} diff --git a/indexer/postgres/options.go b/indexer/postgres/options.go new file mode 100644 index 0000000000..be93d43b6c --- /dev/null +++ b/indexer/postgres/options.go @@ -0,0 +1,10 @@ +package postgres + +// Options are the options for module and object indexers. +type Options struct { + // DisableRetainDeletions disables retain deletions functionality even on object types that have it set. + DisableRetainDeletions bool + + // Logger is the logger for the indexer to use. + Logger SqlLogger +} diff --git a/indexer/postgres/sonar-project.properties b/indexer/postgres/sonar-project.properties new file mode 100644 index 0000000000..6d7366413a --- /dev/null +++ b/indexer/postgres/sonar-project.properties @@ -0,0 +1,16 @@ +sonar.projectKey=cosmos-sdk-indexer-postgres +sonar.organization=cosmos + +sonar.projectName=Cosmos SDK - Postgres Indexer +sonar.project.monorepo.enabled=true + +sonar.sources=. +sonar.exclusions=**/*_test.go,**/*.pb.go,**/*.pulsar.go,**/*.pb.gw.go +sonar.coverage.exclusions=**/*_test.go,**/testutil/**,**/*.pb.go,**/*.pb.gw.go,**/*.pulsar.go,test_helpers.go,docs/** +sonar.tests=. +sonar.test.inclusions=**/*_test.go +sonar.go.coverage.reportPaths=coverage.out + +sonar.sourceEncoding=UTF-8 +sonar.scm.provider=git +sonar.scm.forceReloadAll=true diff --git a/indexer/postgres/tests/README.md b/indexer/postgres/tests/README.md new file mode 100644 index 0000000000..a57c861711 --- /dev/null +++ b/indexer/postgres/tests/README.md @@ -0,0 +1,3 @@ +# PostgreSQL Indexer Tests + +The majority of tests for the PostgreSQL indexer are stored in this separate `tests` go module to keep the main indexer module free of dependencies on any particular PostgreSQL driver. This allows users to choose their own driver and integrate the indexer free of any dependency conflict concerns. \ No newline at end of file diff --git a/indexer/postgres/tests/go.mod b/indexer/postgres/tests/go.mod new file mode 100644 index 0000000000..d5a2930425 --- /dev/null +++ b/indexer/postgres/tests/go.mod @@ -0,0 +1,33 @@ +module cosmossdk.io/indexer/postgres/testing + +require ( + cosmossdk.io/indexer/postgres v0.0.0-00010101000000-000000000000 + cosmossdk.io/schema v0.1.1 + github.com/fergusstrange/embedded-postgres v1.27.0 + github.com/hashicorp/consul/sdk v0.16.1 + github.com/jackc/pgx/v5 v5.6.0 + github.com/stretchr/testify v1.9.0 + gotest.tools/v3 v3.5.1 +) + +require ( + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/google/go-cmp v0.6.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect + github.com/jackc/puddle/v2 v2.2.1 // indirect + github.com/kr/text v0.2.0 // indirect + github.com/lib/pq v1.10.4 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/rogpeppe/go-internal v1.12.0 // indirect + github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect + golang.org/x/crypto v0.23.0 // indirect + golang.org/x/sync v0.1.0 // indirect + golang.org/x/sys v0.20.0 // indirect + golang.org/x/text v0.15.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) + +replace cosmossdk.io/indexer/postgres => ../. + +go 1.22 diff --git a/indexer/postgres/tests/go.sum b/indexer/postgres/tests/go.sum new file mode 100644 index 0000000000..a4ba87b486 --- /dev/null +++ b/indexer/postgres/tests/go.sum @@ -0,0 +1,56 @@ +cosmossdk.io/schema v0.1.1 h1:I0M6pgI7R10nq+/HCQfbO6BsGBZA8sQy+duR1Y3aKcA= +cosmossdk.io/schema v0.1.1/go.mod h1:RDAhxIeNB4bYqAlF4NBJwRrgtnciMcyyg0DOKnhNZQQ= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/fergusstrange/embedded-postgres v1.27.0 h1:RAlpWL194IhEpPgeJceTM0ifMJKhiSVxBVIDYB1Jee8= +github.com/fergusstrange/embedded-postgres v1.27.0/go.mod h1:t/MLs0h9ukYM6FSt99R7InCHs1nW0ordoVCcnzmpTYw= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/hashicorp/consul/sdk v0.16.1 h1:V8TxTnImoPD5cj0U9Spl0TUxcytjcbbJeADFF07KdHg= +github.com/hashicorp/consul/sdk v0.16.1/go.mod h1:fSXvwxB2hmh1FMZCNl6PwX0Q/1wdWtHJcZ7Ea5tns0s= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.6.0 h1:SWJzexBzPL5jb0GEsrPMLIsi/3jOo7RHlzTjcAeDrPY= +github.com/jackc/pgx/v5 v5.6.0/go.mod h1:DNZ/vlrUnhWCoFGxHAG8U2ljioxukquj7utPDgtQdTw= +github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk= +github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/lib/pq v1.10.4 h1:SO9z7FRPzA03QhHKJrH5BXA6HU1rS4V2nIVrrNC1iYk= +github.com/lib/pq v1.10.4/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo= +github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos= +go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= +go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= +golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/v3 v3.5.1 h1:EENdUnS3pdur5nybKYIh2Vfgc8IUNBjxDPSjtiJcOzU= +gotest.tools/v3 v3.5.1/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU= diff --git a/indexer/postgres/tests/init_schema_test.go b/indexer/postgres/tests/init_schema_test.go new file mode 100644 index 0000000000..1afa6caea9 --- /dev/null +++ b/indexer/postgres/tests/init_schema_test.go @@ -0,0 +1,89 @@ +package tests + +import ( + "context" + "fmt" + "os" + "strings" + "testing" + + embeddedpostgres "github.com/fergusstrange/embedded-postgres" + "github.com/hashicorp/consul/sdk/freeport" + + // this is where we get our pgx database driver from + _ "github.com/jackc/pgx/v5/stdlib" + "github.com/stretchr/testify/require" + "gotest.tools/v3/golden" + + "cosmossdk.io/indexer/postgres" + "cosmossdk.io/indexer/postgres/internal/testdata" + "cosmossdk.io/schema/appdata" +) + +func TestInitSchema(t *testing.T) { + t.Run("default", func(t *testing.T) { + testInitSchema(t, false, "init_schema.txt") + }) + + t.Run("retain deletions disabled", func(t *testing.T) { + testInitSchema(t, true, "init_schema_no_retain_delete.txt") + }) +} + +func testInitSchema(t *testing.T, disableRetainDeletions bool, goldenFileName string) { + t.Helper() + connectionUrl := createTestDB(t) + + buf := &strings.Builder{} + logger := func(msg, sql string, params ...interface{}) { + _, err := fmt.Fprintln(buf, msg) + require.NoError(t, err) + _, err = fmt.Fprintln(buf, sql) + require.NoError(t, err) + if len(params) != 0 { + _, err = fmt.Fprintln(buf, "Params:", params) + require.NoError(t, err) + } + _, err = fmt.Fprintln(buf) + require.NoError(t, err) + } + listener, err := postgres.StartIndexer(context.Background(), logger, postgres.Config{ + DatabaseURL: connectionUrl, + DisableRetainDeletions: disableRetainDeletions, + }) + require.NoError(t, err) + + require.NotNil(t, listener.InitializeModuleData) + require.NoError(t, listener.InitializeModuleData(appdata.ModuleInitializationData{ + ModuleName: "test", + Schema: testdata.ExampleSchema, + })) + + require.NotNil(t, listener.Commit) + require.NoError(t, listener.Commit(appdata.CommitData{})) + + golden.Assert(t, buf.String(), goldenFileName) +} + +func createTestDB(t *testing.T) (connectionUrl string) { + t.Helper() + tempDir, err := os.MkdirTemp("", "postgres-indexer-test") + require.NoError(t, err) + t.Cleanup(func() { + require.NoError(t, os.RemoveAll(tempDir)) + }) + + dbPort := freeport.GetOne(t) + pgConfig := embeddedpostgres.DefaultConfig(). + Port(uint32(dbPort)). + DataPath(tempDir) + + connectionUrl = pgConfig.GetConnectionURL() + pg := embeddedpostgres.NewDatabase(pgConfig) + require.NoError(t, pg.Start()) + t.Cleanup(func() { + require.NoError(t, pg.Stop()) + }) + + return +} diff --git a/indexer/postgres/tests/testdata/init_schema.txt b/indexer/postgres/tests/testdata/init_schema.txt new file mode 100644 index 0000000000..e2a0a1730e --- /dev/null +++ b/indexer/postgres/tests/testdata/init_schema.txt @@ -0,0 +1,56 @@ +Creating enum type +CREATE TYPE "test_my_enum" AS ENUM ('a', 'b', 'c'); + +Creating enum type +CREATE TYPE "test_vote_type" AS ENUM ('yes', 'no', 'abstain'); + +Creating table test_all_kinds +CREATE TABLE IF NOT EXISTS "test_all_kinds" ( + "id" BIGINT NOT NULL, + "ts" TIMESTAMPTZ GENERATED ALWAYS AS (nanos_to_timestamptz("ts_nanos")) STORED, + "ts_nanos" BIGINT NOT NULL, + "string" TEXT NOT NULL, + "bytes" BYTEA NOT NULL, + "int8" SMALLINT NOT NULL, + "uint8" SMALLINT NOT NULL, + "int16" SMALLINT NOT NULL, + "uint16" INTEGER NOT NULL, + "int32" INTEGER NOT NULL, + "uint32" BIGINT NOT NULL, + "int64" BIGINT NOT NULL, + "uint64" NUMERIC NOT NULL, + "integer" NUMERIC NOT NULL, + "decimal" NUMERIC NOT NULL, + "bool" BOOLEAN NOT NULL, + "time" TIMESTAMPTZ GENERATED ALWAYS AS (nanos_to_timestamptz("time_nanos")) STORED, + "time_nanos" BIGINT NOT NULL, + "duration" BIGINT NOT NULL, + "float32" REAL NOT NULL, + "float64" DOUBLE PRECISION NOT NULL, + "bech32address" TEXT NOT NULL, + "enum" "test_my_enum" NOT NULL, + "json" JSONB NOT NULL, + PRIMARY KEY ("id", "ts_nanos") +); +GRANT SELECT ON TABLE "test_all_kinds" TO PUBLIC; + +Creating table test_singleton +CREATE TABLE IF NOT EXISTS "test_singleton" ( + _id INTEGER NOT NULL CHECK (_id = 1), + "foo" TEXT NOT NULL, + "bar" INTEGER NULL, + "an_enum" "test_my_enum" NOT NULL, + PRIMARY KEY (_id) +); +GRANT SELECT ON TABLE "test_singleton" TO PUBLIC; + +Creating table test_vote +CREATE TABLE IF NOT EXISTS "test_vote" ( + "proposal" BIGINT NOT NULL, + "address" TEXT NOT NULL, + "vote" "test_vote_type" NOT NULL, + _deleted BOOLEAN NOT NULL DEFAULT FALSE, + PRIMARY KEY ("proposal", "address") +); +GRANT SELECT ON TABLE "test_vote" TO PUBLIC; + diff --git a/indexer/postgres/tests/testdata/init_schema_no_retain_delete.txt b/indexer/postgres/tests/testdata/init_schema_no_retain_delete.txt new file mode 100644 index 0000000000..0d8cdad2cd --- /dev/null +++ b/indexer/postgres/tests/testdata/init_schema_no_retain_delete.txt @@ -0,0 +1,55 @@ +Creating enum type +CREATE TYPE "test_my_enum" AS ENUM ('a', 'b', 'c'); + +Creating enum type +CREATE TYPE "test_vote_type" AS ENUM ('yes', 'no', 'abstain'); + +Creating table test_all_kinds +CREATE TABLE IF NOT EXISTS "test_all_kinds" ( + "id" BIGINT NOT NULL, + "ts" TIMESTAMPTZ GENERATED ALWAYS AS (nanos_to_timestamptz("ts_nanos")) STORED, + "ts_nanos" BIGINT NOT NULL, + "string" TEXT NOT NULL, + "bytes" BYTEA NOT NULL, + "int8" SMALLINT NOT NULL, + "uint8" SMALLINT NOT NULL, + "int16" SMALLINT NOT NULL, + "uint16" INTEGER NOT NULL, + "int32" INTEGER NOT NULL, + "uint32" BIGINT NOT NULL, + "int64" BIGINT NOT NULL, + "uint64" NUMERIC NOT NULL, + "integer" NUMERIC NOT NULL, + "decimal" NUMERIC NOT NULL, + "bool" BOOLEAN NOT NULL, + "time" TIMESTAMPTZ GENERATED ALWAYS AS (nanos_to_timestamptz("time_nanos")) STORED, + "time_nanos" BIGINT NOT NULL, + "duration" BIGINT NOT NULL, + "float32" REAL NOT NULL, + "float64" DOUBLE PRECISION NOT NULL, + "bech32address" TEXT NOT NULL, + "enum" "test_my_enum" NOT NULL, + "json" JSONB NOT NULL, + PRIMARY KEY ("id", "ts_nanos") +); +GRANT SELECT ON TABLE "test_all_kinds" TO PUBLIC; + +Creating table test_singleton +CREATE TABLE IF NOT EXISTS "test_singleton" ( + _id INTEGER NOT NULL CHECK (_id = 1), + "foo" TEXT NOT NULL, + "bar" INTEGER NULL, + "an_enum" "test_my_enum" NOT NULL, + PRIMARY KEY (_id) +); +GRANT SELECT ON TABLE "test_singleton" TO PUBLIC; + +Creating table test_vote +CREATE TABLE IF NOT EXISTS "test_vote" ( + "proposal" BIGINT NOT NULL, + "address" TEXT NOT NULL, + "vote" "test_vote_type" NOT NULL, + PRIMARY KEY ("proposal", "address") +); +GRANT SELECT ON TABLE "test_vote" TO PUBLIC; + diff --git a/schema/field.go b/schema/field.go index 2839d5240b..19a1b4085d 100644 --- a/schema/field.go +++ b/schema/field.go @@ -33,7 +33,7 @@ func (c Field) Validate() error { // valid kind if err := c.Kind.Validate(); err != nil { - return fmt.Errorf("invalid field kind for %q: %w", c.Name, err) + return fmt.Errorf("invalid field kind for %q: %v", c.Name, err) //nolint:errorlint // false positive due to using go1.12 } // address prefix only valid with Bech32AddressKind @@ -46,7 +46,7 @@ func (c Field) Validate() error { // enum definition only valid with EnumKind if c.Kind == EnumKind { if err := c.EnumDefinition.Validate(); err != nil { - return fmt.Errorf("invalid enum definition for field %q: %w", c.Name, err) + return fmt.Errorf("invalid enum definition for field %q: %v", c.Name, err) //nolint:errorlint // false positive due to using go1.12 } } else if c.Kind != EnumKind && (c.EnumDefinition.Name != "" || c.EnumDefinition.Values != nil) { return fmt.Errorf("enum definition is only valid for field %q with type EnumKind", c.Name) @@ -67,7 +67,7 @@ func (c Field) ValidateValue(value interface{}) error { } err := c.Kind.ValidateValueType(value) if err != nil { - return fmt.Errorf("invalid value for field %q: %w", c.Name, err) + return fmt.Errorf("invalid value for field %q: %v", c.Name, err) //nolint:errorlint // false positive due to using go1.12 } if c.Kind == EnumKind { diff --git a/schema/object_type.go b/schema/object_type.go index 9560c5d4e3..a8fa432d80 100644 --- a/schema/object_type.go +++ b/schema/object_type.go @@ -43,7 +43,7 @@ func (o ObjectType) validate(enumValueMap map[string]map[string]bool) error { for _, field := range o.KeyFields { if err := field.Validate(); err != nil { - return fmt.Errorf("invalid key field %q: %w", field.Name, err) + return fmt.Errorf("invalid key field %q: %v", field.Name, err) //nolint:errorlint // false positive due to using go1.12 } if field.Nullable { @@ -62,7 +62,7 @@ func (o ObjectType) validate(enumValueMap map[string]map[string]bool) error { for _, field := range o.ValueFields { if err := field.Validate(); err != nil { - return fmt.Errorf("invalid value field %q: %w", field.Name, err) + return fmt.Errorf("invalid value field %q: %v", field.Name, err) //nolint:errorlint // false positive due to using go1.12 } if fieldNames[field.Name] { @@ -89,7 +89,7 @@ func (o ObjectType) ValidateObjectUpdate(update ObjectUpdate) error { } if err := ValidateObjectKey(o.KeyFields, update.Key); err != nil { - return fmt.Errorf("invalid key for object type %q: %w", update.TypeName, err) + return fmt.Errorf("invalid key for object type %q: %v", update.TypeName, err) //nolint:errorlint // false positive due to using go1.12 } if update.Delete {