41f4f1fd83
- When chainwatch is ran it will first start a Syncer that continuously collects blocks from the ChainNotify channel and persists them to the blocks_synced table. Once the Syncer has caught the blocks_synced table up to the lotus daemons current head a Processor is started. The Processor selects a batch of contiguous blocks and extracts and stores their data. It attempts to do as much work as it can in parallel. When the blocks are done being processed their corresponding processed_at and is_processed fields in the blocks_synced table are filled out.
104 lines
1.8 KiB
Go
104 lines
1.8 KiB
Go
package processor
|
|
|
|
import (
|
|
"context"
|
|
"time"
|
|
|
|
"golang.org/x/xerrors"
|
|
|
|
"github.com/ipfs/go-cid"
|
|
|
|
"github.com/filecoin-project/lotus/api"
|
|
"github.com/filecoin-project/lotus/chain/types"
|
|
)
|
|
|
|
func (p *Processor) subMpool(ctx context.Context) {
|
|
sub, err := p.node.MpoolSub(ctx)
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
for {
|
|
var updates []api.MpoolUpdate
|
|
|
|
select {
|
|
case update := <-sub:
|
|
updates = append(updates, update)
|
|
case <-ctx.Done():
|
|
return
|
|
}
|
|
|
|
loop:
|
|
for {
|
|
time.Sleep(10 * time.Millisecond)
|
|
select {
|
|
case update := <-sub:
|
|
updates = append(updates, update)
|
|
default:
|
|
break loop
|
|
}
|
|
}
|
|
|
|
msgs := map[cid.Cid]*types.Message{}
|
|
for _, v := range updates {
|
|
if v.Type != api.MpoolAdd {
|
|
continue
|
|
}
|
|
|
|
msgs[v.Message.Message.Cid()] = &v.Message.Message
|
|
}
|
|
|
|
log.Debugf("Processing %d mpool updates", len(msgs))
|
|
|
|
err := p.storeMessages(msgs)
|
|
if err != nil {
|
|
log.Error(err)
|
|
}
|
|
|
|
if err := p.storeMpoolInclusions(updates); err != nil {
|
|
log.Error(err)
|
|
}
|
|
}
|
|
}
|
|
|
|
func (p *Processor) storeMpoolInclusions(msgs []api.MpoolUpdate) error {
|
|
tx, err := p.db.Begin()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if _, err := tx.Exec(`
|
|
create temp table mi (like mpool_messages excluding constraints) on commit drop;
|
|
`); err != nil {
|
|
return xerrors.Errorf("prep temp: %w", err)
|
|
}
|
|
|
|
stmt, err := tx.Prepare(`copy mi (msg, add_ts) from stdin `)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
for _, msg := range msgs {
|
|
if msg.Type != api.MpoolAdd {
|
|
continue
|
|
}
|
|
|
|
if _, err := stmt.Exec(
|
|
msg.Message.Message.Cid().String(),
|
|
time.Now().Unix(),
|
|
); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
if err := stmt.Close(); err != nil {
|
|
return err
|
|
}
|
|
|
|
if _, err := tx.Exec(`insert into mpool_messages select * from mi on conflict do nothing `); err != nil {
|
|
return xerrors.Errorf("actor put: %w", err)
|
|
}
|
|
|
|
return tx.Commit()
|
|
}
|