Merge pull request #1607 from filecoin-project/feat/chainwatch-updates

chainwatch: Record drand entries

chain/vm/invoker.go

@@ -7,6 +7,7 @@ import (
 	"reflect"
 
 	"github.com/filecoin-project/specs-actors/actors/builtin/account"
+	"github.com/filecoin-project/specs-actors/actors/builtin/verifreg"
 	"github.com/filecoin-project/specs-actors/actors/runtime/exitcode"
 
 	"github.com/ipfs/go-cid"
@@ -55,6 +56,7 @@ func NewInvoker() *invoker {
 	inv.Register(builtin.StorageMinerActorCodeID, miner.Actor{}, miner.State{})
 	inv.Register(builtin.MultisigActorCodeID, multisig.Actor{}, multisig.State{})
 	inv.Register(builtin.PaymentChannelActorCodeID, paych.Actor{}, paych.State{})
+	inv.Register(builtin.VerifiedRegistryActorCodeID, verifreg.Actor{}, verifreg.State{})
 	inv.Register(builtin.AccountActorCodeID, account.Actor{}, account.State{})
 
 	return inv

cmd/lotus-chainwatch/storage.go

@@ -39,11 +39,23 @@ func (st *storage) setup() error {
 		return err
 	}
 	_, err = tx.Exec(`
+create table if not exists block_cids
+(
+	cid text not null
+		constraint block_cids_pk
+			primary key
+);
+
+create unique index if not exists block_cids_cid_uindex
+	on block_cids (cid);
+
 create table if not exists blocks_synced
 (
 	cid text not null
 		constraint blocks_synced_pk
-			primary key,
+			primary key
+	    constraint blocks_block_cids_cid_fk
+			references block_cids (cid),
 	add_ts int not null
 );
 
@@ -52,25 +64,52 @@ create unique index if not exists blocks_synced_cid_uindex
 
 create table if not exists block_parents
 (
-	block text not null,
+	block text not null
+	    constraint blocks_block_cids_cid_fk
+			references block_cids (cid),
 	parent text not null
 );
 
 create unique index if not exists block_parents_block_parent_uindex
 	on block_parents (block, parent);
 
+create table if not exists drand_entries
+(
+    round bigint not null
+    	constraint drand_entries_pk
+			primary key,
+	data bytea not null
+);
+create unique index if not exists drand_entries_round_uindex
+	on drand_entries (round);
+
+create table if not exists block_drand_entries
+(
+    round bigint not null
+    	constraint block_drand_entries_drand_entries_round_fk
+			references drand_entries (round),
+	block text not null
+	    constraint blocks_block_cids_cid_fk
+			references block_cids (cid)
+);
+create unique index if not exists block_drand_entries_round_uindex
+	on block_drand_entries (round, block);
+
 create table if not exists blocks
 (
 	cid text not null
 		constraint blocks_pk
-			primary key,
+			primary key
+	    constraint blocks_block_cids_cid_fk
+			references block_cids (cid),
 	parentWeight numeric not null,
 	parentStateRoot text not null,
 	height bigint not null,
 	miner text not null,
 	timestamp bigint not null,
-	vrfproof bytea,
+	ticket bytea not null,
-	eprof bytea
+	eprof bytea,
+	forksig bigint not null
 );
 
 create unique index if not exists block_cid_uindex
@@ -178,7 +217,9 @@ create index if not exists messages_to_index
 
 create table if not exists block_messages
 (
-	block text not null,
+	block text not null
+	    constraint blocks_block_cids_cid_fk
+			references block_cids (cid),
 	message text not null,
 	constraint block_messages_pk
 		primary key (block, message)
@@ -304,35 +345,6 @@ create index if not exists deal_activations_activation_epoch_index
 create unique index if not exists deal_activations_deal_uindex
 	on deal_activations (deal);
 */
-create table if not exists blocks_challenges
-(
-	block text not null
-		constraint blocks_challenges_pk_2
-			primary key
-		constraint blocks_challenges_blocks_cid_fk
-			references blocks,
-	index bigint not null,
-	sector_id bigint not null,
-	partial bytea not null,
-	candidate bigint not null,
-	constraint blocks_challenges_pk
-		unique (block, index)
-);
-
-create index if not exists blocks_challenges_block_index
-	on blocks_challenges (block);
-
-create index if not exists blocks_challenges_block_candidate_index
-	on blocks_challenges (block,candidate);
-
-create index if not exists blocks_challenges_block_index_index
-	on blocks_challenges (block, index);
-
-create index if not exists blocks_challenges_candidate_index
-	on blocks_challenges (candidate);
-
-create index if not exists blocks_challenges_index_index
-	on blocks_challenges (index);
 
 `)
 	if err != nil {
@@ -505,35 +517,106 @@ func (st *storage) storeHeaders(bhs map[cid.Cid]*types.BlockHeader, sync bool) e
 
 	if _, err := tx.Exec(`
 
+create temp table bc (like block_cids excluding constraints) on commit drop;
+create temp table de (like drand_entries excluding constraints) on commit drop;
+create temp table bde (like block_drand_entries excluding constraints) on commit drop;
 create temp table tbp (like block_parents excluding constraints) on commit drop;
 create temp table bs (like blocks_synced excluding constraints) on commit drop;
 create temp table b (like blocks excluding constraints) on commit drop;
-create temp table c (like blocks_challenges excluding constraints) on commit drop;
 
 
 `); err != nil {
 		return xerrors.Errorf("prep temp: %w", err)
 	}
 
-	stmt, err := tx.Prepare(`copy tbp (block, parent) from STDIN`)
+	{
-	if err != nil {
+		stmt, err := tx.Prepare(`copy bc (cid) from STDIN`)
-		return err
+		if err != nil {
-	}
+			return err
+		}
 
-	for _, bh := range bhs {
+		for _, bh := range bhs {
-		for _, parent := range bh.Parents {
+			if _, err := stmt.Exec(bh.Cid().String()); err != nil {
-			if _, err := stmt.Exec(bh.Cid().String(), parent.String()); err != nil {
 				log.Error(err)
 			}
 		}
+
+		if err := stmt.Close(); err != nil {
+			return err
+		}
+
+		if _, err := tx.Exec(`insert into block_cids select * from bc on conflict do nothing `); err != nil {
+			return xerrors.Errorf("drand entries put: %w", err)
+		}
 	}
 
-	if err := stmt.Close(); err != nil {
+	{
-		return err
+		stmt, err := tx.Prepare(`copy de (round, data) from STDIN`)
+		if err != nil {
+			return err
+		}
+
+		for _, bh := range bhs {
+			for _, ent := range bh.BeaconEntries {
+				if _, err := stmt.Exec(ent.Round, ent.Data); err != nil {
+					log.Error(err)
+				}
+			}
+		}
+
+		if err := stmt.Close(); err != nil {
+			return err
+		}
+
+		if _, err := tx.Exec(`insert into drand_entries select * from de on conflict do nothing `); err != nil {
+			return xerrors.Errorf("drand entries put: %w", err)
+		}
 	}
 
-	if _, err := tx.Exec(`insert into block_parents select * from tbp on conflict do nothing `); err != nil {
+	{
-		return xerrors.Errorf("parent put: %w", err)
+		stmt, err := tx.Prepare(`copy bde (round, block) from STDIN`)
+		if err != nil {
+			return err
+		}
+
+		for _, bh := range bhs {
+			for _, ent := range bh.BeaconEntries {
+				if _, err := stmt.Exec(ent.Round, bh.Cid().String()); err != nil {
+					log.Error(err)
+				}
+			}
+		}
+
+		if err := stmt.Close(); err != nil {
+			return err
+		}
+
+		if _, err := tx.Exec(`insert into block_drand_entries select * from bde on conflict do nothing `); err != nil {
+			return xerrors.Errorf("block drand entries put: %w", err)
+		}
+	}
+
+	{
+		stmt, err := tx.Prepare(`copy tbp (block, parent) from STDIN`)
+		if err != nil {
+			return err
+		}
+
+		for _, bh := range bhs {
+			for _, parent := range bh.Parents {
+				if _, err := stmt.Exec(bh.Cid().String(), parent.String()); err != nil {
+					log.Error(err)
+				}
+			}
+		}
+
+		if err := stmt.Close(); err != nil {
+			return err
+		}
+
+		if _, err := tx.Exec(`insert into block_parents select * from tbp on conflict do nothing `); err != nil {
+			return xerrors.Errorf("parent put: %w", err)
+		}
 	}
 
 	if sync {
@@ -559,12 +642,16 @@ create temp table c (like blocks_challenges excluding constraints) on commit dro
 		}
 	}
 
-	stmt2, err := tx.Prepare(`copy b (cid, parentWeight, parentStateRoot, height, miner, "timestamp", vrfproof) from stdin`)
+	stmt2, err := tx.Prepare(`copy b (cid, parentWeight, parentStateRoot, height, miner, "timestamp", ticket, eprof, forksig) from stdin`)
 	if err != nil {
 		return err
 	}
 
 	for _, bh := range bhs {
+		var eprof interface{}
+		if bh.ElectionProof != nil {
+			eprof = bh.ElectionProof.VRFProof
+		}
 
 		if _, err := stmt2.Exec(
 			bh.Cid().String(),
@@ -573,7 +660,9 @@ create temp table c (like blocks_challenges excluding constraints) on commit dro
 			bh.Height,
 			bh.Miner.String(),
 			bh.Timestamp,
-			bh.Ticket.VRFProof); err != nil {
+			bh.Ticket.VRFProof,
+			eprof,
+			bh.ForkSignaling); err != nil {
 			log.Error(err)
 		}
 	}
@@ -586,7 +675,7 @@ create temp table c (like blocks_challenges excluding constraints) on commit dro
 		return xerrors.Errorf("blk put: %w", err)
 	}
 
-	return nil
+	return tx.Commit()
 }
 
 func (st *storage) storeMessages(msgs map[cid.Cid]*types.Message) error {

go.mod

@@ -25,7 +25,7 @@ require (
 	github.com/filecoin-project/go-paramfetch v0.0.2-0.20200218225740-47c639bab663
 	github.com/filecoin-project/go-statestore v0.1.0
 	github.com/filecoin-project/sector-storage v0.0.0-20200423222053-9eb049a833b9
-	github.com/filecoin-project/specs-actors v1.0.1-0.20200424174946-11410d0bbcaf
+	github.com/filecoin-project/specs-actors v1.0.1-0.20200424220637-349bd6297517
 	github.com/filecoin-project/specs-storage v0.0.0-20200417134612-61b2d91a6102
 	github.com/filecoin-project/storage-fsm v0.0.0-20200423114251-f3bea4aa8bd7
 	github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1

go.sum

@@ -174,8 +174,8 @@ github.com/filecoin-project/sector-storage v0.0.0-20200423222053-9eb049a833b9/go
 github.com/filecoin-project/specs-actors v0.0.0-20200210130641-2d1fbd8672cf/go.mod h1:xtDZUB6pe4Pksa/bAJbJ693OilaC5Wbot9jMhLm3cZA=
 github.com/filecoin-project/specs-actors v0.0.0-20200409043918-e569f4a2f504/go.mod h1:mdJraXq5vMy0+/FqVQIrnNlpQ/Em6zeu06G/ltQ0/lA=
 github.com/filecoin-project/specs-actors v1.0.0/go.mod h1:nQYnFbQ7Y0bHZyq6HDEuVlCPR+U3z5Q3wMOQ+2aiV+Y=
-github.com/filecoin-project/specs-actors v1.0.1-0.20200424174946-11410d0bbcaf h1:sPETeTr35OuGIgL2laDCEzB6wO+trTrw4k9vH/m82yU=
+github.com/filecoin-project/specs-actors v1.0.1-0.20200424220637-349bd6297517 h1:Qi8xaf90j4pdMn8NZZ8FWROsvaTu2iD/C2TT7gDRNQw=
-github.com/filecoin-project/specs-actors v1.0.1-0.20200424174946-11410d0bbcaf/go.mod h1:nQYnFbQ7Y0bHZyq6HDEuVlCPR+U3z5Q3wMOQ+2aiV+Y=
+github.com/filecoin-project/specs-actors v1.0.1-0.20200424220637-349bd6297517/go.mod h1:nQYnFbQ7Y0bHZyq6HDEuVlCPR+U3z5Q3wMOQ+2aiV+Y=
 github.com/filecoin-project/specs-storage v0.0.0-20200410185809-9fbaaa08f275 h1:6OTcpsTQBQM0f/A67oEi4E4YtYd6fzkMqbU8cPIWMMs=
 github.com/filecoin-project/specs-storage v0.0.0-20200410185809-9fbaaa08f275/go.mod h1:xJ1/xl9+8zZeSSSFmDC3Wr6uusCTxyYPI0VeNVSFmPE=
 github.com/filecoin-project/specs-storage v0.0.0-20200417134612-61b2d91a6102 h1:T3f/zkuvgtgqcXrb0NO3BicuveGOxxUAMPa/Yif2kuE=