Merge pull request #4 from vulcanize/ian_update

Add indexes to `event.emitter_addr` and `event_entry.key`

CHANGELOG.md

@@ -101,6 +101,7 @@ Lotus-workers can now be built to leverage the SupraSeal C2 sealing optimization
 - fix(client): single-root error message ([filecoin-project/lotus#11214](https://github.com/filecoin-project/lotus/pull/11214))
 - fix: worker: Convert `DC_[SectorSize]_[ResourceRestriction]` if set ([filecoin-project/lotus#11224](https://github.com/filecoin-project/lotus/pull/11224))
 - chore: backport #11338 onto release/v1.25.0 ([filecoin-project/lotus#11350](https://github.com/filecoin-project/lotus/pull/11350))
+- fix: Exclude reverted events in `eth_getLogs` results [filecoin-project/lotus#11318](https://github.com/filecoin-project/lotus/pull/11318)
 
 ## Dependencies
 - deps: update go-libp2p to v0.28.1 ([filecoin-project/lotus#10998](https://github.com/filecoin-project/lotus/pull/10998))

chain/events/filter/event.go

@@ -388,7 +388,7 @@ func (m *EventFilterManager) Install(ctx context.Context, minHeight, maxHeight a
 
 	if m.EventIndex != nil && minHeight != -1 && minHeight < currentHeight {
 		// Filter needs historic events
-		if err := m.EventIndex.PrefillFilter(ctx, f); err != nil {
+		if err := m.EventIndex.PrefillFilter(ctx, f, true); err != nil {
 			return nil, err
 		}
 	}

chain/events/filter/index.go

@@ -46,6 +46,7 @@ var ddls = []string{
 	)`,
 
 	`CREATE INDEX IF NOT EXISTS height_tipset_key ON event (height,tipset_key)`,
+	`CREATE INDEX IF NOT EXISTS event_emitter_addr ON event (emitter_addr)`,
 
 	`CREATE TABLE IF NOT EXISTS event_entry (
 		event_id INTEGER,
@@ -56,6 +57,8 @@ var ddls = []string{
 		value BLOB NOT NULL
 	)`,
 
+	`CREATE INDEX IF NOT EXISTS event_entry_key_index ON event_entry (key)`,
+
 	// metadata containing version of schema
 	`CREATE TABLE IF NOT EXISTS _meta (
     	version UINT64 NOT NULL UNIQUE
@@ -63,6 +66,7 @@ var ddls = []string{
 
 	`INSERT OR IGNORE INTO _meta (version) VALUES (1)`,
 	`INSERT OR IGNORE INTO _meta (version) VALUES (2)`,
+	`INSERT OR IGNORE INTO _meta (version) VALUES (3)`,
 }
 
 var (
@@ -70,7 +74,7 @@ var (
 )
 
 const (
-	schemaVersion = 2
+	schemaVersion = 3
 
 	eventExists          = `SELECT MAX(id) FROM event WHERE height=? AND tipset_key=? AND tipset_key_cid=? AND emitter_addr=? AND event_index=? AND message_cid=? AND message_index=?`
 	insertEvent          = `INSERT OR IGNORE INTO event(height, tipset_key, tipset_key_cid, emitter_addr, event_index, message_cid, message_index, reverted) VALUES(?, ?, ?, ?, ?, ?, ?, ?)`
@@ -321,6 +325,22 @@ func NewEventIndex(ctx context.Context, path string, chainStore *store.ChainStor
 			version = 2
 		}
 
+		if version == 2 {
+			log.Infof("upgrading event index from version 1 to version 2")
+
+			// to upgrade to version 3 we only need to create an index on the event_entry.key column
+			// and on the event.emitter_addr column
+			// which means we can just reapply the schema (it will not have any effect on existing data)
+			for _, ddl := range ddls {
+				if _, err := db.Exec(ddl); err != nil {
+					_ = db.Close()
+					return nil, xerrors.Errorf("could not upgrade index to version 3, exec ddl %q: %w", ddl, err)
+				}
+			}
+
+			version = 3
+		}
+
 		if version != schemaVersion {
 			_ = db.Close()
 			return nil, xerrors.Errorf("invalid database version: got %d, expected %d", version, schemaVersion)
@@ -481,7 +501,7 @@ func (ei *EventIndex) CollectEvents(ctx context.Context, te *TipSetEvents, rever
 }
 
 // PrefillFilter fills a filter's collection of events from the historic index
-func (ei *EventIndex) PrefillFilter(ctx context.Context, f *EventFilter) error {
+func (ei *EventIndex) PrefillFilter(ctx context.Context, f *EventFilter, excludeReverted bool) error {
 	clauses := []string{}
 	values := []any{}
 	joins := []string{}
@@ -500,6 +520,11 @@ func (ei *EventIndex) PrefillFilter(ctx context.Context, f *EventFilter) error {
 		}
 	}
 
+	if excludeReverted {
+		clauses = append(clauses, "event.reverted=?")
+		values = append(values, false)
+	}
+
 	if len(f.addresses) > 0 {
 		subclauses := []string{}
 		for _, addr := range f.addresses {

chain/events/filter/index_test.go

@@ -272,7 +272,7 @@ func TestEventIndexPrefillFilter(t *testing.T) {
 	for _, tc := range testCases {
 		tc := tc // appease lint
 		t.Run(tc.name, func(t *testing.T) {
-			if err := ei.PrefillFilter(context.Background(), tc.filter); err != nil {
+			if err := ei.PrefillFilter(context.Background(), tc.filter, false); err != nil {
 				require.NoError(t, err, "prefill filter events")
 			}
 
@@ -281,3 +281,619 @@ func TestEventIndexPrefillFilter(t *testing.T) {
 		})
 	}
 }
+
+func TestEventIndexPrefillFilterExcludeReverted(t *testing.T) {
+	rng := pseudo.New(pseudo.NewSource(299792458))
+	a1 := randomF4Addr(t, rng)
+	a2 := randomF4Addr(t, rng)
+	a3 := randomF4Addr(t, rng)
+
+	a1ID := abi.ActorID(1)
+	a2ID := abi.ActorID(2)
+
+	addrMap := addressMap{}
+	addrMap.add(a1ID, a1)
+	addrMap.add(a2ID, a2)
+
+	ev1 := fakeEvent(
+		a1ID,
+		[]kv{
+			{k: "type", v: []byte("approval")},
+			{k: "signer", v: []byte("addr1")},
+		},
+		[]kv{
+			{k: "amount", v: []byte("2988181")},
+		},
+	)
+	ev2 := fakeEvent(
+		a2ID,
+		[]kv{
+			{k: "type", v: []byte("approval")},
+			{k: "signer", v: []byte("addr2")},
+		},
+		[]kv{
+			{k: "amount", v: []byte("2988182")},
+		},
+	)
+
+	st := newStore()
+	events := []*types.Event{ev1}
+	revertedEvents := []*types.Event{ev2}
+	em := executedMessage{
+		msg: fakeMessage(randomF4Addr(t, rng), randomF4Addr(t, rng)),
+		rct: fakeReceipt(t, rng, st, events),
+		evs: events,
+	}
+	revertedEm := executedMessage{
+		msg: fakeMessage(randomF4Addr(t, rng), randomF4Addr(t, rng)),
+		rct: fakeReceipt(t, rng, st, revertedEvents),
+		evs: revertedEvents,
+	}
+
+	events14000 := buildTipSetEvents(t, rng, 14000, em)
+	revertedEvents14000 := buildTipSetEvents(t, rng, 14000, revertedEm)
+	cid14000, err := events14000.msgTs.Key().Cid()
+	require.NoError(t, err, "tipset cid")
+	reveredCID14000, err := revertedEvents14000.msgTs.Key().Cid()
+	require.NoError(t, err, "tipset cid")
+
+	noCollectedEvents := []*CollectedEvent{}
+	oneCollectedEvent := []*CollectedEvent{
+		{
+			Entries:     ev1.Entries,
+			EmitterAddr: a1,
+			EventIdx:    0,
+			Reverted:    false,
+			Height:      14000,
+			TipSetKey:   events14000.msgTs.Key(),
+			MsgIdx:      0,
+			MsgCid:      em.msg.Cid(),
+		},
+	}
+	twoCollectedEvent := []*CollectedEvent{
+		{
+			Entries:     ev1.Entries,
+			EmitterAddr: a1,
+			EventIdx:    0,
+			Reverted:    false,
+			Height:      14000,
+			TipSetKey:   events14000.msgTs.Key(),
+			MsgIdx:      0,
+			MsgCid:      em.msg.Cid(),
+		},
+		{
+			Entries:     ev2.Entries,
+			EmitterAddr: a2,
+			EventIdx:    0,
+			Reverted:    true,
+			Height:      14000,
+			TipSetKey:   revertedEvents14000.msgTs.Key(),
+			MsgIdx:      0,
+			MsgCid:      revertedEm.msg.Cid(),
+		},
+	}
+	oneCollectedRevertedEvent := []*CollectedEvent{
+		{
+			Entries:     ev2.Entries,
+			EmitterAddr: a2,
+			EventIdx:    0,
+			Reverted:    true,
+			Height:      14000,
+			TipSetKey:   revertedEvents14000.msgTs.Key(),
+			MsgIdx:      0,
+			MsgCid:      revertedEm.msg.Cid(),
+		},
+	}
+
+	workDir, err := os.MkdirTemp("", "lotusevents")
+	require.NoError(t, err, "create temporary work directory")
+
+	defer func() {
+		_ = os.RemoveAll(workDir)
+	}()
+	t.Logf("using work dir %q", workDir)
+
+	dbPath := filepath.Join(workDir, "actorevents.db")
+
+	ei, err := NewEventIndex(context.Background(), dbPath, nil)
+	require.NoError(t, err, "create event index")
+	if err := ei.CollectEvents(context.Background(), revertedEvents14000, false, addrMap.ResolveAddress); err != nil {
+		require.NoError(t, err, "collect reverted events")
+	}
+	if err := ei.CollectEvents(context.Background(), revertedEvents14000, true, addrMap.ResolveAddress); err != nil {
+		require.NoError(t, err, "revert reverted events")
+	}
+	if err := ei.CollectEvents(context.Background(), events14000, false, addrMap.ResolveAddress); err != nil {
+		require.NoError(t, err, "collect events")
+	}
+
+	inclusiveTestCases := []struct {
+		name   string
+		filter *EventFilter
+		te     *TipSetEvents
+		want   []*CollectedEvent
+	}{
+		{
+			name: "nomatch tipset min height",
+			filter: &EventFilter{
+				minHeight: 14001,
+				maxHeight: -1,
+			},
+			te:   events14000,
+			want: noCollectedEvents,
+		},
+		{
+			name: "nomatch tipset max height",
+			filter: &EventFilter{
+				minHeight: -1,
+				maxHeight: 13999,
+			},
+			te:   events14000,
+			want: noCollectedEvents,
+		},
+		{
+			name: "match tipset min height",
+			filter: &EventFilter{
+				minHeight: 14000,
+				maxHeight: -1,
+			},
+			te:   events14000,
+			want: twoCollectedEvent,
+		},
+		{
+			name: "match tipset cid",
+			filter: &EventFilter{
+				minHeight: -1,
+				maxHeight: -1,
+				tipsetCid: cid14000,
+			},
+			te:   events14000,
+			want: oneCollectedEvent,
+		},
+		{
+			name: "match tipset cid",
+			filter: &EventFilter{
+				minHeight: -1,
+				maxHeight: -1,
+				tipsetCid: reveredCID14000,
+			},
+			te:   revertedEvents14000,
+			want: oneCollectedRevertedEvent,
+		},
+		{
+			name: "nomatch address",
+			filter: &EventFilter{
+				minHeight: -1,
+				maxHeight: -1,
+				addresses: []address.Address{a3},
+			},
+			te:   events14000,
+			want: noCollectedEvents,
+		},
+		{
+			name: "match address 2",
+			filter: &EventFilter{
+				minHeight: -1,
+				maxHeight: -1,
+				addresses: []address.Address{a2},
+			},
+			te:   revertedEvents14000,
+			want: oneCollectedRevertedEvent,
+		},
+		{
+			name: "match address 1",
+			filter: &EventFilter{
+				minHeight: -1,
+				maxHeight: -1,
+				addresses: []address.Address{a1},
+			},
+			te:   events14000,
+			want: oneCollectedEvent,
+		},
+		{
+			name: "match one entry",
+			filter: &EventFilter{
+				minHeight: -1,
+				maxHeight: -1,
+				keys: map[string][][]byte{
+					"type": {
+						[]byte("approval"),
+					},
+				},
+			},
+			te:   events14000,
+			want: twoCollectedEvent,
+		},
+		{
+			name: "match one entry with alternate values",
+			filter: &EventFilter{
+				minHeight: -1,
+				maxHeight: -1,
+				keys: map[string][][]byte{
+					"type": {
+						[]byte("cancel"),
+						[]byte("propose"),
+						[]byte("approval"),
+					},
+				},
+			},
+			te:   events14000,
+			want: twoCollectedEvent,
+		},
+		{
+			name: "nomatch one entry by missing value",
+			filter: &EventFilter{
+				minHeight: -1,
+				maxHeight: -1,
+				keys: map[string][][]byte{
+					"type": {
+						[]byte("cancel"),
+						[]byte("propose"),
+					},
+				},
+			},
+			te:   events14000,
+			want: noCollectedEvents,
+		},
+		{
+			name: "nomatch one entry by missing key",
+			filter: &EventFilter{
+				minHeight: -1,
+				maxHeight: -1,
+				keys: map[string][][]byte{
+					"method": {
+						[]byte("approval"),
+					},
+				},
+			},
+			te:   events14000,
+			want: noCollectedEvents,
+		},
+		{
+			name: "match one entry with multiple keys",
+			filter: &EventFilter{
+				minHeight: -1,
+				maxHeight: -1,
+				keys: map[string][][]byte{
+					"type": {
+						[]byte("approval"),
+					},
+					"signer": {
+						[]byte("addr1"),
+					},
+				},
+			},
+			te:   events14000,
+			want: oneCollectedEvent,
+		},
+		{
+			name: "match one entry with multiple keys",
+			filter: &EventFilter{
+				minHeight: -1,
+				maxHeight: -1,
+				keys: map[string][][]byte{
+					"type": {
+						[]byte("approval"),
+					},
+					"signer": {
+						[]byte("addr2"),
+					},
+				},
+			},
+			te:   revertedEvents14000,
+			want: oneCollectedRevertedEvent,
+		},
+		{
+			name: "nomatch one entry with one mismatching key",
+			filter: &EventFilter{
+				minHeight: -1,
+				maxHeight: -1,
+				keys: map[string][][]byte{
+					"type": {
+						[]byte("approval"),
+					},
+					"approver": {
+						[]byte("addr1"),
+					},
+				},
+			},
+			te:   events14000,
+			want: noCollectedEvents,
+		},
+		{
+			name: "nomatch one entry with one mismatching value",
+			filter: &EventFilter{
+				minHeight: -1,
+				maxHeight: -1,
+				keys: map[string][][]byte{
+					"type": {
+						[]byte("approval"),
+					},
+					"signer": {
+						[]byte("addr3"),
+					},
+				},
+			},
+			te:   events14000,
+			want: noCollectedEvents,
+		},
+		{
+			name: "nomatch one entry with one unindexed key",
+			filter: &EventFilter{
+				minHeight: -1,
+				maxHeight: -1,
+				keys: map[string][][]byte{
+					"amount": {
+						[]byte("2988181"),
+					},
+				},
+			},
+			te:   events14000,
+			want: noCollectedEvents,
+		},
+		{
+			name: "nomatch one entry with one unindexed key",
+			filter: &EventFilter{
+				minHeight: -1,
+				maxHeight: -1,
+				keys: map[string][][]byte{
+					"amount": {
+						[]byte("2988182"),
+					},
+				},
+			},
+			te:   events14000,
+			want: noCollectedEvents,
+		},
+	}
+
+	exclusiveTestCases := []struct {
+		name   string
+		filter *EventFilter
+		te     *TipSetEvents
+		want   []*CollectedEvent
+	}{
+		{
+			name: "nomatch tipset min height",
+			filter: &EventFilter{
+				minHeight: 14001,
+				maxHeight: -1,
+			},
+			te:   events14000,
+			want: noCollectedEvents,
+		},
+		{
+			name: "nomatch tipset max height",
+			filter: &EventFilter{
+				minHeight: -1,
+				maxHeight: 13999,
+			},
+			te:   events14000,
+			want: noCollectedEvents,
+		},
+		{
+			name: "match tipset min height",
+			filter: &EventFilter{
+				minHeight: 14000,
+				maxHeight: -1,
+			},
+			te:   events14000,
+			want: oneCollectedEvent,
+		},
+		{
+			name: "match tipset cid",
+			filter: &EventFilter{
+				minHeight: -1,
+				maxHeight: -1,
+				tipsetCid: cid14000,
+			},
+			te:   events14000,
+			want: oneCollectedEvent,
+		},
+		{
+			name: "match tipset cid but reverted",
+			filter: &EventFilter{
+				minHeight: -1,
+				maxHeight: -1,
+				tipsetCid: reveredCID14000,
+			},
+			te:   revertedEvents14000,
+			want: noCollectedEvents,
+		},
+		{
+			name: "nomatch address",
+			filter: &EventFilter{
+				minHeight: -1,
+				maxHeight: -1,
+				addresses: []address.Address{a3},
+			},
+			te:   events14000,
+			want: noCollectedEvents,
+		},
+		{
+			name: "nomatch address 2 but reverted",
+			filter: &EventFilter{
+				minHeight: -1,
+				maxHeight: -1,
+				addresses: []address.Address{a2},
+			},
+			te:   revertedEvents14000,
+			want: noCollectedEvents,
+		},
+		{
+			name: "match address",
+			filter: &EventFilter{
+				minHeight: -1,
+				maxHeight: -1,
+				addresses: []address.Address{a1},
+			},
+			te:   events14000,
+			want: oneCollectedEvent,
+		},
+		{
+			name: "match one entry",
+			filter: &EventFilter{
+				minHeight: -1,
+				maxHeight: -1,
+				keys: map[string][][]byte{
+					"type": {
+						[]byte("approval"),
+					},
+				},
+			},
+			te:   events14000,
+			want: oneCollectedEvent,
+		},
+		{
+			name: "match one entry with alternate values",
+			filter: &EventFilter{
+				minHeight: -1,
+				maxHeight: -1,
+				keys: map[string][][]byte{
+					"type": {
+						[]byte("cancel"),
+						[]byte("propose"),
+						[]byte("approval"),
+					},
+				},
+			},
+			te:   events14000,
+			want: oneCollectedEvent,
+		},
+		{
+			name: "nomatch one entry by missing value",
+			filter: &EventFilter{
+				minHeight: -1,
+				maxHeight: -1,
+				keys: map[string][][]byte{
+					"type": {
+						[]byte("cancel"),
+						[]byte("propose"),
+					},
+				},
+			},
+			te:   events14000,
+			want: noCollectedEvents,
+		},
+		{
+			name: "nomatch one entry by missing key",
+			filter: &EventFilter{
+				minHeight: -1,
+				maxHeight: -1,
+				keys: map[string][][]byte{
+					"method": {
+						[]byte("approval"),
+					},
+				},
+			},
+			te:   events14000,
+			want: noCollectedEvents,
+		},
+		{
+			name: "match one entry with multiple keys",
+			filter: &EventFilter{
+				minHeight: -1,
+				maxHeight: -1,
+				keys: map[string][][]byte{
+					"type": {
+						[]byte("approval"),
+					},
+					"signer": {
+						[]byte("addr1"),
+					},
+				},
+			},
+			te:   events14000,
+			want: oneCollectedEvent,
+		},
+		{
+			name: "nomatch one entry with one mismatching key",
+			filter: &EventFilter{
+				minHeight: -1,
+				maxHeight: -1,
+				keys: map[string][][]byte{
+					"type": {
+						[]byte("approval"),
+					},
+					"approver": {
+						[]byte("addr1"),
+					},
+				},
+			},
+			te:   events14000,
+			want: noCollectedEvents,
+		},
+		{
+			name: "nomatch one entry with matching reverted value",
+			filter: &EventFilter{
+				minHeight: -1,
+				maxHeight: -1,
+				keys: map[string][][]byte{
+					"type": {
+						[]byte("approval"),
+					},
+					"signer": {
+						[]byte("addr2"),
+					},
+				},
+			},
+			te:   events14000,
+			want: noCollectedEvents,
+		},
+		{
+			name: "nomatch one entry with one mismatching value",
+			filter: &EventFilter{
+				minHeight: -1,
+				maxHeight: -1,
+				keys: map[string][][]byte{
+					"type": {
+						[]byte("approval"),
+					},
+					"signer": {
+						[]byte("addr3"),
+					},
+				},
+			},
+			te:   events14000,
+			want: noCollectedEvents,
+		},
+		{
+			name: "nomatch one entry with one unindexed key",
+			filter: &EventFilter{
+				minHeight: -1,
+				maxHeight: -1,
+				keys: map[string][][]byte{
+					"amount": {
+						[]byte("2988181"),
+					},
+				},
+			},
+			te:   events14000,
+			want: noCollectedEvents,
+		},
+	}
+
+	for _, tc := range inclusiveTestCases {
+		tc := tc // appease lint
+		t.Run(tc.name, func(t *testing.T) {
+			if err := ei.PrefillFilter(context.Background(), tc.filter, false); err != nil {
+				require.NoError(t, err, "prefill filter events")
+			}
+
+			coll := tc.filter.TakeCollectedEvents(context.Background())
+			require.ElementsMatch(t, coll, tc.want, tc.name)
+		})
+	}
+
+	for _, tc := range exclusiveTestCases {
+		tc := tc // appease lint
+		t.Run(tc.name, func(t *testing.T) {
+			if err := ei.PrefillFilter(context.Background(), tc.filter, true); err != nil {
+				require.NoError(t, err, "prefill filter events")
+			}
+
+			coll := tc.filter.TakeCollectedEvents(context.Background())
+			require.ElementsMatch(t, coll, tc.want, tc.name)
+		})
+	}
+}