cerc-io/lotus
16
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

Merge pull request #1340 from filecoin-project/feat/lib-dedupe

Browse Source 
Switch to using extracted libs
This commit is contained in:
Łukasz Magiera 2020-03-06 20:22:21 +01:00 committed by GitHub
commit 3a923e8aae
39 changed files with 90 additions and 1807 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
chain/vm/syscalls.go
View File

@ -3,11 +3,9 @@ package vm
import (
"context"
"fmt"
"math/bits"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-sectorbuilder"
"github.com/filecoin-project/lotus/lib/zerocomm"
"github.com/filecoin-project/specs-actors/actors/abi"
"github.com/filecoin-project/specs-actors/actors/crypto"
"github.com/filecoin-project/specs-actors/actors/runtime"
@ -36,28 +34,6 @@ func (ss *syscallShim) ComputeUnsealedSectorCID(st abi.RegisteredProof, pieces [
sum += p.Size
}
ssize, err := st.SectorSize()
if err != nil {
return cid.Undef, err
}
{
// pad remaining space with 0 CommPs
toFill := uint64(abi.PaddedPieceSize(ssize) - sum)
n := bits.OnesCount64(toFill)
for i := 0; i < n; i++ {
next := bits.TrailingZeros64(toFill)
psize := uint64(1) << next
toFill ^= psize
unpadded := abi.PaddedPieceSize(psize).Unpadded()
pieces = append(pieces, abi.PieceInfo{
Size: unpadded.Padded(),
PieceCID: zerocomm.ForSize(unpadded),
})
}
}
commd, err := sectorbuilder.GenerateUnsealedCID(st, pieces)
if err != nil {
log.Errorf("generate data commitment failed: %s", err)

10
gen/main.go
View File

@ -9,7 +9,6 @@ import (
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/chain/blocksync"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/lib/statemachine"
"github.com/filecoin-project/lotus/node/hello"
"github.com/filecoin-project/lotus/paychmgr"
"github.com/filecoin-project/lotus/storage/sealing"
@ -83,13 +82,4 @@ func main() {
fmt.Println(err)
os.Exit(1)
}
err = gen.WriteMapEncodersToFile("./lib/statemachine/cbor_gen.go", "statemachine",
statemachine.TestState{},
statemachine.TestEvent{},
)
if err != nil {
fmt.Printf("%+v\n", err)
os.Exit(1)
}
}

2
go.mod
View File

@ -22,6 +22,7 @@ require (
github.com/filecoin-project/go-padreader v0.0.0-20200210211231-548257017ca6
github.com/filecoin-project/go-paramfetch v0.0.2-0.20200218225740-47c639bab663
github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200306043753-5cdbe369b47d
github.com/filecoin-project/go-statemachine v0.0.0-20200226041606-2074af6d51d9
github.com/filecoin-project/go-statestore v0.1.0
github.com/filecoin-project/specs-actors v0.0.0-20200306043603-709a3ce21094
github.com/filecoin-project/specs-storage v0.0.0-20200303233430-1a5a408f7513
@ -84,7 +85,6 @@ require (
github.com/multiformats/go-multiaddr-dns v0.2.0
github.com/multiformats/go-multiaddr-net v0.1.2
github.com/multiformats/go-multihash v0.0.13
github.com/multiformats/go-varint v0.0.5
github.com/opentracing/opentracing-go v1.1.0
github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829
github.com/prometheus/common v0.4.0

160
lib/rlepluslazy/bits.go
View File

@ -1,160 +0,0 @@
package rlepluslazy
import (
"sort"
)
type it2b struct {
source RunIterator
curIdx uint64
run Run
}
func (it *it2b) HasNext() bool {
return it.run.Valid()
}
func (it *it2b) Next() (uint64, error) {
it.run.Len--
res := it.curIdx
it.curIdx++
return res, it.prep()
}
func (it *it2b) prep() error {
for !it.run.Valid() && it.source.HasNext() {
var err error
it.run, err = it.source.NextRun()
if err != nil {
return err
}
if !it.run.Val {
it.curIdx += it.run.Len
it.run.Len = 0
}
}
return nil
}
func BitsFromRuns(source RunIterator) (BitIterator, error) {
it := &it2b{source: source}
if err := it.prep(); err != nil {
return nil, err
}
return it, nil
}
type sliceIt struct {
s []uint64
}
func (it sliceIt) HasNext() bool {
return len(it.s) != 0
}
func (it *sliceIt) Next() (uint64, error) {
res := it.s[0]
it.s = it.s[1:]
return res, nil
}
func BitsFromSlice(slice []uint64) BitIterator {
sort.Slice(slice, func(i, j int) bool { return slice[i] < slice[j] })
return &sliceIt{slice}
}
type it2r struct {
source BitIterator
runIdx uint64
run [2]Run
}
func (it *it2r) HasNext() bool {
return it.run[0].Valid()
}
func (it *it2r) NextRun() (Run, error) {
res := it.run[0]
it.runIdx = it.runIdx + res.Len
it.run[0], it.run[1] = it.run[1], Run{}
return res, it.prep()
}
func (it *it2r) prep() error {
if !it.HasNext() {
return nil
}
if it.run[0].Val == false {
it.run[1].Val = true
it.run[1].Len = 1
return nil
}
for it.source.HasNext() && !it.run[1].Valid() {
nB, err := it.source.Next()
if err != nil {
return err
}
//fmt.Printf("runIdx: %d, run[0].Len: %d, nB: %d\n", it.runIdx, it.run[0].Len, nB)
if it.runIdx+it.run[0].Len == nB {
it.run[0].Len++
} else {
it.run[1].Len = nB - it.runIdx - it.run[0].Len
it.run[1].Val = false
}
}
return nil
}
func (it *it2r) init() error {
if it.source.HasNext() {
nB, err := it.source.Next()
if err != nil {
return err
}
it.run[0].Len = nB
it.run[0].Val = false
it.run[1].Len = 1
it.run[1].Val = true
}
if !it.run[0].Valid() {
it.run[0], it.run[1] = it.run[1], Run{}
return it.prep()
}
return nil
}
func SliceFromRuns(source RunIterator) ([]uint64, error) {
rit, err := BitsFromRuns(source)
if err != nil {
return nil, err
}
res := make([]uint64, 0)
for rit.HasNext() {
bit, err := rit.Next()
if err != nil {
return nil, err
}
res = append(res, bit)
}
return res, nil
}
func RunsFromBits(source BitIterator) (RunIterator, error) {
it := &it2r{source: source}
if err := it.init(); err != nil {
return nil, err
}
return it, nil
}
func RunsFromSlice(slice []uint64) (RunIterator, error) {
return RunsFromBits(BitsFromSlice(slice))
}

27
lib/rlepluslazy/bits_test.go
View File

@ -1,27 +0,0 @@
package rlepluslazy
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestRunsFromBits(t *testing.T) {
expected := []Run{Run{Val: false, Len: 0x1},
{Val: true, Len: 0x3},
{Val: false, Len: 0x2},
{Val: true, Len: 0x3},
}
rit, err := RunsFromBits(BitsFromSlice([]uint64{1, 2, 3, 6, 7, 8}))
assert.NoError(t, err)
i := 10
output := make([]Run, 0, 4)
for rit.HasNext() && i > 0 {
run, err := rit.NextRun()
assert.NoError(t, err)
i--
output = append(output, run)
}
assert.NotEqual(t, 0, i, "too many iterations")
assert.Equal(t, expected, output)
}

114
lib/rlepluslazy/bitvec.go
View File

@ -1,114 +0,0 @@
package rlepluslazy
type rbitvec struct {
index int
bits uint16
bitCap byte
vec []byte
}
func readBitvec(vec []byte) *rbitvec {
bv := &rbitvec{
vec: vec,
index: 1,
bitCap: 8,
}
if len(vec) > 0 {
bv.bits = uint16(bv.vec[0])
}
return bv
}
// bitMasks is a mask for selecting N first bits out of a byte
var bitMasks = [9]byte{
0x0,
0x1,
0x3,
0x7,
0xF,
0x1F,
0x3F,
0x7F,
0xFF,
}
func (bv *rbitvec) Get(count byte) byte {
res := byte(bv.bits) & bitMasks[count] // select count bits
bv.bits = bv.bits >> count // remove those bits from storage
bv.bitCap = bv.bitCap - count // decrease nuber of stored bits
if bv.index < len(bv.vec) { // if vector allows
// add bits onto the end of temporary storage
bv.bits = bv.bits | uint16(bv.vec[bv.index])<<bv.bitCap
}
// Here be dragons
// This is equivalent to
// if bv.bitCap < 8 {
// bv.index++
// bv.bitCap = bv.bitCap + 8
// }
// but implemented without branches because the branch here is unpredictable
// Why this is without branches and reading has branch?
// Because branch above is predictable, in 99.99% of cases it will be true
// if bitCap < 8 it underflows, then high bits get set to 1s
// we shift by 7 so the highest bit is in place of the lowest
inc := (bv.bitCap - 8) >> 7 // inc == 1 iff bitcap<8 (+10% perf)
bv.index = bv.index + int(inc) // increase index if we need more bits
bv.bitCap = bv.bitCap + inc*8 // increase bitCap by 8
return res
}
func writeBitvec(buf []byte) *wbitvec {
// reslice to 0 length for consistent input but to keep capacity
return &wbitvec{buf: buf[:0]}
}
type wbitvec struct {
buf []byte // buffer we will be saving to
index int // index of at which the next byte will be saved
bits uint16 // temporary storage for bits
bitCap byte // number of bits stored in temporary storage
}
func (bv *wbitvec) Out() []byte {
if bv.bitCap != 0 {
// if there are some bits in temporary storage we need to save them
bv.buf = append(bv.buf, 0)[:bv.index+1]
bv.buf[bv.index] = byte(bv.bits)
}
if bv.bitCap > 8 {
// if we store some needed bits in second byte, save them also
bv.buf = append(bv.buf, byte(bv.bitCap>>8))
bv.index++
bv.bits = bv.bits - 8
}
return bv.buf
}
func (bv *wbitvec) Put(val byte, count byte) {
// put val into its place in bv.bits
bv.bits = bv.bits | uint16(val)<<bv.bitCap
// increase bitCap by the number of bits
bv.bitCap = bv.bitCap + count
// increase len of the buffer if it is needed
if bv.index+1 > cap(bv.buf) {
bv.buf = append(bv.buf, 0)
}
bv.buf = bv.buf[:bv.index+1]
// save the bits
bv.buf[bv.index] = byte(bv.bits)
// Warning, dragons again
// if bitCap is greater than 7 it underflows, same thing as in Put
inc := (7 - bv.bitCap) >> 7 // inc == 1 iff bitcap>=8
bv.index = bv.index + int(inc) // increase index for the next save
bv.bitCap = bv.bitCap - inc*8 // we store less bits now in temporary buffer
bv.bits = bv.bits >> (inc * 8) // we can discard those bits as they were saved
}

21
lib/rlepluslazy/bitvec_test.go
View File

@ -1,21 +0,0 @@
package rlepluslazy
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestReadBitVec(t *testing.T) {
buf := []byte{0x0, 0xff}
bv := readBitvec(buf)
o := bv.Get(1)
assert.EqualValues(t, 0, o)
o = bv.Get(8)
assert.EqualValues(t, 0x80, o)
o = bv.Get(7)
assert.EqualValues(t, 0x7f, o)
}

24
lib/rlepluslazy/interface.go
View File

@ -1,24 +0,0 @@
package rlepluslazy
type Run struct {
Val bool
Len uint64
}
func (r Run) Valid() bool {
return r.Len != 0
}
type RunIterator interface {
NextRun() (Run, error)
HasNext() bool
}
type RunIterable interface {
RunIterator() (RunIterator, error)
}
type BitIterator interface {
Next() (uint64, error)
HasNext() bool
}

47
lib/rlepluslazy/rleplus.go
View File

@ -1,47 +0,0 @@
package rlepluslazy
import (
"errors"
"fmt"
"golang.org/x/xerrors"
)
const Version = 0
var (
ErrWrongVersion = errors.New("invalid RLE+ version")
ErrDecode = fmt.Errorf("invalid encoding for RLE+ version %d", Version)
)
type RLE struct {
buf []byte
}
func FromBuf(buf []byte) (RLE, error) {
rle := RLE{buf: buf}
if len(buf) > 0 && buf[0]&3 != Version {
return RLE{}, xerrors.Errorf("could not create RLE+ for a buffer: %w", ErrWrongVersion)
}
_, err := rle.Count()
if err != nil {
return RLE{}, err
}
return rle, nil
}
func (rle *RLE) RunIterator() (RunIterator, error) {
source, err := DecodeRLE(rle.buf)
return source, err
}
func (rle *RLE) Count() (uint64, error) {
it, err := rle.RunIterator()
if err != nil {
return 0, err
}
return Count(it)
}

3
lib/rlepluslazy/rleplus_golden_test.go
View File

File diff suppressed because one or more lines are too long

77
lib/rlepluslazy/rleplus_reader.go
View File

@ -1,77 +0,0 @@
package rlepluslazy
import (
"github.com/multiformats/go-varint"
"golang.org/x/xerrors"
)
func DecodeRLE(buf []byte) (RunIterator, error) {
bv := readBitvec(buf)
ver := bv.Get(2) // Read version
if ver != Version {
return nil, ErrWrongVersion
}
it := &rleIterator{bv: bv}
// next run is previous in relation to prep
// so we invert the value
it.nextRun.Val = bv.Get(1) != 1
if err := it.prep(); err != nil {
return nil, err
}
return it, nil
}
type rleIterator struct {
bv *rbitvec
nextRun Run
}
func (it *rleIterator) HasNext() bool {
return it.nextRun.Valid()
}
func (it *rleIterator) NextRun() (Run, error) {
ret := it.nextRun
return ret, it.prep()
}
func (it *rleIterator) prep() error {
x := it.bv.Get(1)
switch x {
case 1:
it.nextRun.Len = 1
case 0:
y := it.bv.Get(1)
switch y {
case 1:
it.nextRun.Len = uint64(it.bv.Get(4))
case 0:
var buf = make([]byte, 0, 10)
for {
b := it.bv.Get(8)
buf = append(buf, b)
if b&0x80 == 0 {
break
}
if len(buf) > 10 {
return xerrors.Errorf("run too long: %w", ErrDecode)
}
}
var err error
it.nextRun.Len, _, err = varint.FromUvarint(buf)
if err != nil {
return err
}
}
}
it.nextRun.Val = !it.nextRun.Val
return nil
}

170
lib/rlepluslazy/rleplus_test.go
View File

@ -1,170 +0,0 @@
package rlepluslazy
import (
"math/rand"
"testing"
"github.com/filecoin-project/lotus/extern/rleplus"
"github.com/stretchr/testify/assert"
)
func TestDecode(t *testing.T) {
// Encoding bitvec![LittleEndian; 1, 0, 1, 0, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
// in the Rust reference implementation gives an encoding of [223, 145, 136, 0] (without version field)
// The bit vector is equivalent to the integer set { 0, 2, 4, 5, 6, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27 }
// This is the above reference output with a version header "00" manually added
referenceEncoding := []byte{124, 71, 34, 2}
expectedNumbers := []uint64{0, 2, 4, 5, 6, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27}
runs, err := RunsFromBits(BitsFromSlice(expectedNumbers))
assert.NoError(t, err)
encoded, err := EncodeRuns(runs, []byte{})
assert.NoError(t, err)
// Our encoded bytes are the same as the ref bytes
assert.Equal(t, len(referenceEncoding), len(encoded))
assert.Equal(t, referenceEncoding, encoded)
rle, err := FromBuf(encoded)
assert.NoError(t, err)
decoded := make([]uint64, 0, len(expectedNumbers))
rit, err := rle.RunIterator()
assert.NoError(t, err)
it, err := BitsFromRuns(rit)
assert.NoError(t, err)
for it.HasNext() {
bit, err := it.Next()
assert.NoError(t, err)
decoded = append(decoded, bit)
}
// Our decoded integers are the same as expected
assert.Equal(t, expectedNumbers, decoded)
}
func TestGoldenGen(t *testing.T) {
t.SkipNow()
N := 10000
mod := uint32(1) << 20
runExProp := float32(0.93)
bits := make([]uint64, N)
for i := 0; i < N; i++ {
x := rand.Uint32() % mod
bits[i] = uint64(x)
for rand.Float32() < runExProp && i+1 < N {
i++
x = (x + 1) % mod
bits[i] = uint64(x)
}
}
out, _, err := rleplus.Encode(bits)
assert.NoError(t, err)
t.Logf("%#v", out)
_, runs := rleplus.RunLengths(bits)
t.Logf("runs: %v", runs)
t.Logf("len: %d", len(out))
}
func TestGolden(t *testing.T) {
expected, _ := rleplus.Decode(goldenRLE)
res := make([]uint64, 0, len(expected))
rle, err := FromBuf(goldenRLE)
assert.NoError(t, err)
rit, err := rle.RunIterator()
assert.NoError(t, err)
it, err := BitsFromRuns(rit)
assert.NoError(t, err)
for it.HasNext() {
bit, err := it.Next()
assert.NoError(t, err)
res = append(res, bit)
}
assert.Equal(t, expected, res)
}
func TestGoldenLoop(t *testing.T) {
rle, err := FromBuf(goldenRLE)
assert.NoError(t, err)
rit, err := rle.RunIterator()
assert.NoError(t, err)
buf, err := EncodeRuns(rit, nil)
assert.NoError(t, err)
assert.Equal(t, goldenRLE, buf)
}
var Res uint64 = 0
func BenchmarkRunIterator(b *testing.B) {
b.ReportAllocs()
var r uint64
for i := 0; i < b.N; i++ {
rle, _ := FromBuf(goldenRLE)
rit, _ := rle.RunIterator()
for rit.HasNext() {
run, _ := rit.NextRun()
if run.Val {
r = r + run.Len
}
}
}
Res = Res + r
}
func BenchmarkRunsToBits(b *testing.B) {
b.ReportAllocs()
var r uint64
for i := 0; i < b.N; i++ {
rle, _ := FromBuf(goldenRLE)
rit, _ := rle.RunIterator()
it, _ := BitsFromRuns(rit)
for it.HasNext() {
bit, _ := it.Next()
if bit < 1<<63 {
r++
}
}
}
Res = Res + r
}
func BenchmarkOldRLE(b *testing.B) {
b.ReportAllocs()
var r uint64
for i := 0; i < b.N; i++ {
rle, _ := rleplus.Decode(goldenRLE)
r = r + uint64(len(rle))
}
Res = Res + r
}
func BenchmarkDecodeEncode(b *testing.B) {
b.ReportAllocs()
var r uint64
out := make([]byte, 0, len(goldenRLE))
for i := 0; i < b.N; i++ {
rle, _ := FromBuf(goldenRLE)
rit, _ := rle.RunIterator()
out, _ = EncodeRuns(rit, out)
r = r + uint64(len(out))
}
/*
for i := 0; i < b.N; i++ {
rle, _ := rleplus.Decode(goldenRLE)
out, _, _ := rleplus.Encode(rle)
r = r + uint64(len(out))
}
*/
Res = Res + r
}

51
lib/rlepluslazy/rleplus_writer.go
View File

@ -1,51 +0,0 @@
package rlepluslazy
import (
"encoding/binary"
)
func EncodeRuns(rit RunIterator, buf []byte) ([]byte, error) {
bv := writeBitvec(buf)
bv.Put(0, 2)
first := true
varBuf := make([]byte, binary.MaxVarintLen64)
for rit.HasNext() {
run, err := rit.NextRun()
if err != nil {
return nil, err
}
if first {
if run.Val {
bv.Put(1, 1)
} else {
bv.Put(0, 1)
}
first = false
}
switch {
case run.Len == 1:
bv.Put(1, 1)
case run.Len < 16:
bv.Put(2, 2)
bv.Put(byte(run.Len), 4)
case run.Len >= 16:
bv.Put(0, 2)
numBytes := binary.PutUvarint(varBuf, run.Len)
for i := 0; i < numBytes; i++ {
bv.Put(varBuf[i], 8)
}
}
}
if first {
bv.Put(0, 1)
}
return bv.Out(), nil
}

120
lib/rlepluslazy/runs.go
View File

@ -1,120 +0,0 @@
package rlepluslazy
import (
"math"
"golang.org/x/xerrors"
)
func Sum(a, b RunIterator) (RunIterator, error) {
it := addIt{a: a, b: b}
it.prep()
return &it, nil
}
type addIt struct {
a RunIterator
b RunIterator
next Run
arun Run
brun Run
}
func (it *addIt) prep() error {
var err error
fetch := func() error {
if !it.arun.Valid() && it.a.HasNext() {
it.arun, err = it.a.NextRun()
if err != nil {
return err
}
}
if !it.brun.Valid() && it.b.HasNext() {
it.brun, err = it.b.NextRun()
if err != nil {
return err
}
}
return nil
}
if err := fetch(); err != nil {
return err
}
// one is not valid
if !it.arun.Valid() {
it.next = it.brun
it.brun.Len = 0
return nil
}
if !it.brun.Valid() {
it.next = it.arun
it.arun.Len = 0
return nil
}
if !(it.arun.Val || it.brun.Val) {
min := it.arun.Len
if it.brun.Len < min {
min = it.brun.Len
}
it.next = Run{Val: it.arun.Val, Len: min}
it.arun.Len -= it.next.Len
it.brun.Len -= it.next.Len
return nil
}
it.next = Run{Val: true}
// different vals, 'true' wins
for (it.arun.Val && it.arun.Valid()) || (it.brun.Val && it.brun.Valid()) {
min := it.arun.Len
if it.brun.Len < min && it.brun.Valid() || !it.arun.Valid() {
min = it.brun.Len
}
it.next.Len += min
if it.arun.Valid() {
it.arun.Len -= min
}
if it.brun.Valid() {
it.brun.Len -= min
}
if err := fetch(); err != nil {
return err
}
}
return nil
}
func (it *addIt) HasNext() bool {
return it.next.Valid()
}
func (it *addIt) NextRun() (Run, error) {
next := it.next
return next, it.prep()
}
func Count(ri RunIterator) (uint64, error) {
var count uint64
for ri.HasNext() {
r, err := ri.NextRun()
if err != nil {
return 0, err
}
if r.Val {
if math.MaxUint64-r.Len < count {
return 0, xerrors.New("RLE+ overflows")
}
count += r.Len
}
}
return count, nil
}

94
lib/rlepluslazy/runs_test.go
View File

@ -1,94 +0,0 @@
package rlepluslazy
import (
"math/rand"
"sort"
"testing"
"github.com/stretchr/testify/assert"
)
func TestSumRuns(t *testing.T) {
{
a, err := RunsFromSlice([]uint64{0, 1, 2, 3, 4, 5, 6, 7, 8, 11, 12, 13, 14})
assert.NoError(t, err)
b, err := RunsFromSlice([]uint64{0, 1, 2, 3, 9, 10, 16, 17, 18, 50, 51, 70})
assert.NoError(t, err)
s, err := Sum(a, b)
assert.NoError(t, err)
bis, err := SliceFromRuns(s)
assert.Equal(t, []uint64{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 17, 18, 50, 51, 70}, bis)
assert.NoError(t, err)
}
{
a, err := RunsFromSlice([]uint64{0, 1, 2, 3, 4, 5, 6, 7, 8, 11, 12, 13, 14})
assert.NoError(t, err)
b, err := RunsFromSlice([]uint64{0, 1, 2, 3, 9, 10, 16, 17, 18, 50, 51, 70})
assert.NoError(t, err)
s, err := Sum(b, a)
assert.NoError(t, err)
bis, err := SliceFromRuns(s)
assert.Equal(t, []uint64{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 17, 18, 50, 51, 70}, bis)
assert.NoError(t, err)
}
}
func randomBits(N int, max uint64) []uint64 {
all := make(map[uint64]struct{})
for len(all) <= N {
x := rand.Uint64() % max
if _, has := all[x]; has {
continue
}
all[x] = struct{}{}
}
res := make([]uint64, 0, N)
for x := range all {
res = append(res, x)
}
sort.Slice(res, func(i, j int) bool { return res[i] < res[j] })
return res
}
func sum(a, b []uint64) []uint64 {
all := make(map[uint64]struct{})
for _, x := range a {
all[x] = struct{}{}
}
for _, x := range b {
all[x] = struct{}{}
}
res := make([]uint64, 0, len(all))
for x := range all {
res = append(res, x)
}
sort.Slice(res, func(i, j int) bool { return res[i] < res[j] })
return res
}
func TestSumRandom(t *testing.T) {
N := 100
for i := 0; i < N; i++ {
abits := randomBits(1000, 1500)
bbits := randomBits(1000, 1500)
sumbits := sum(abits, bbits)
a, err := RunsFromSlice(abits)
assert.NoError(t, err)
b, err := RunsFromSlice(bbits)
assert.NoError(t, err)
s, err := Sum(b, a)
assert.NoError(t, err)
bis, err := SliceFromRuns(s)
assert.NoError(t, err)
assert.Equal(t, sumbits, bis)
}
}

241
lib/statemachine/cbor_gen.go
View File

@ -1,241 +0,0 @@
// Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT.
package statemachine
import (
"fmt"
"io"
cbg "github.com/whyrusleeping/cbor-gen"
xerrors "golang.org/x/xerrors"
)
var _ = xerrors.Errorf
func (t *TestState) MarshalCBOR(w io.Writer) error {
if t == nil {
_, err := w.Write(cbg.CborNull)
return err
}
if _, err := w.Write([]byte{162}); err != nil {
return err
}
// t.A (uint64) (uint64)
if len("A") > cbg.MaxLength {
return xerrors.Errorf("Value in field \"A\" was too long")
}
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("A")))); err != nil {
return err
}
if _, err := w.Write([]byte("A")); err != nil {
return err
}
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.A))); err != nil {
return err
}
// t.B (uint64) (uint64)
if len("B") > cbg.MaxLength {
return xerrors.Errorf("Value in field \"B\" was too long")
}
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("B")))); err != nil {
return err
}
if _, err := w.Write([]byte("B")); err != nil {
return err
}
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.B))); err != nil {
return err
}
return nil
}
func (t *TestState) UnmarshalCBOR(r io.Reader) error {
br := cbg.GetPeeker(r)
maj, extra, err := cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajMap {
return fmt.Errorf("cbor input should be of type map")
}
if extra > cbg.MaxLength {
return fmt.Errorf("TestState: map struct too large (%d)", extra)
}
var name string
n := extra
for i := uint64(0); i < n; i++ {
{
sval, err := cbg.ReadString(br)
if err != nil {
return err
}
name = string(sval)
}
switch name {
// t.A (uint64) (uint64)
case "A":
{
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.A = uint64(extra)
}
// t.B (uint64) (uint64)
case "B":
{
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.B = uint64(extra)
}
default:
return fmt.Errorf("unknown struct field %d: '%s'", i, name)
}
}
return nil
}
func (t *TestEvent) MarshalCBOR(w io.Writer) error {
if t == nil {
_, err := w.Write(cbg.CborNull)
return err
}
if _, err := w.Write([]byte{162}); err != nil {
return err
}
// t.A (string) (string)
if len("A") > cbg.MaxLength {
return xerrors.Errorf("Value in field \"A\" was too long")
}
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("A")))); err != nil {
return err
}
if _, err := w.Write([]byte("A")); err != nil {
return err
}
if len(t.A) > cbg.MaxLength {
return xerrors.Errorf("Value in field t.A was too long")
}
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(t.A)))); err != nil {
return err
}
if _, err := w.Write([]byte(t.A)); err != nil {
return err
}
// t.Val (uint64) (uint64)
if len("Val") > cbg.MaxLength {
return xerrors.Errorf("Value in field \"Val\" was too long")
}
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("Val")))); err != nil {
return err
}
if _, err := w.Write([]byte("Val")); err != nil {
return err
}
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Val))); err != nil {
return err
}
return nil
}
func (t *TestEvent) UnmarshalCBOR(r io.Reader) error {
br := cbg.GetPeeker(r)
maj, extra, err := cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajMap {
return fmt.Errorf("cbor input should be of type map")
}
if extra > cbg.MaxLength {
return fmt.Errorf("TestEvent: map struct too large (%d)", extra)
}
var name string
n := extra
for i := uint64(0); i < n; i++ {
{
sval, err := cbg.ReadString(br)
if err != nil {
return err
}
name = string(sval)
}
switch name {
// t.A (string) (string)
case "A":
{
sval, err := cbg.ReadString(br)
if err != nil {
return err
}
t.A = string(sval)
}
// t.Val (uint64) (uint64)
case "Val":
{
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.Val = uint64(extra)
}
default:
return fmt.Errorf("unknown struct field %d: '%s'", i, name)
}
}
return nil
}

16
lib/statemachine/context.go
View File

@ -1,16 +0,0 @@
package statemachine
import "context"
type Context struct {
ctx context.Context
send func(evt interface{}) error
}
func (ctx *Context) Context() context.Context {
return ctx.ctx
}
func (ctx *Context) Send(evt interface{}) error {
return ctx.send(evt)
}

116
lib/statemachine/group.go
View File

@ -1,116 +0,0 @@
package statemachine
import (
"context"
"reflect"
"sync"
"github.com/filecoin-project/go-statestore"
"github.com/ipfs/go-datastore"
"golang.org/x/xerrors"
)
type StateHandler interface {
// returns
Plan(events []Event, user interface{}) (interface{}, error)
}
// StateGroup manages a group of state machines sharing the same logic
type StateGroup struct {
sts *statestore.StateStore
hnd StateHandler
stateType reflect.Type
lk sync.Mutex
sms map[datastore.Key]*StateMachine
}
// stateType: T - (MyStateStruct{})
func New(ds datastore.Datastore, hnd StateHandler, stateType interface{}) *StateGroup {
return &StateGroup{
sts: statestore.New(ds),
hnd: hnd,
stateType: reflect.TypeOf(stateType),
sms: map[datastore.Key]*StateMachine{},
}
}
// Send sends an event to machine identified by `id`.
// `evt` is going to be passed into StateHandler.Planner, in the events[].User param
//
// If a state machine with the specified id doesn't exits, it's created, and it's
// state is set to zero-value of stateType provided in group constructor
func (s *StateGroup) Send(id interface{}, evt interface{}) (err error) {
s.lk.Lock()
defer s.lk.Unlock()
sm, exist := s.sms[statestore.ToKey(id)]
if !exist {
sm, err = s.loadOrCreate(id)
if err != nil {
return xerrors.Errorf("loadOrCreate state: %w", err)
}
s.sms[statestore.ToKey(id)] = sm
}
return sm.send(Event{User: evt})
}
func (s *StateGroup) loadOrCreate(name interface{}) (*StateMachine, error) {
exists, err := s.sts.Has(name)
if err != nil {
return nil, xerrors.Errorf("failed to check if state for %v exists: %w", name, err)
}
if !exists {
userState := reflect.New(s.stateType).Interface()
err = s.sts.Begin(name, userState)
if err != nil {
return nil, xerrors.Errorf("saving initial state: %w", err)
}
}
res := &StateMachine{
planner: s.hnd.Plan,
eventsIn: make(chan Event),
name: name,
st: s.sts.Get(name),
stateType: s.stateType,
stageDone: make(chan struct{}),
closing: make(chan struct{}),
closed: make(chan struct{}),
}
go res.run()
return res, nil
}
// Stop stops all state machines in this group
func (s *StateGroup) Stop(ctx context.Context) error {
s.lk.Lock()
defer s.lk.Unlock()
for _, sm := range s.sms {
if err := sm.stop(ctx); err != nil {
return err
}
}
return nil
}
// List outputs states of all state machines in this group
// out: *[]StateT
func (s *StateGroup) List(out interface{}) error {
return s.sts.List(out)
}
// Get gets state for a single state machine
func (s *StateGroup) Get(id interface{}) *statestore.StoredState {
return s.sts.Get(id)
}

127
lib/statemachine/machine.go
View File

@ -1,127 +0,0 @@
package statemachine
import (
"context"
"reflect"
"sync/atomic"
"github.com/filecoin-project/go-statestore"
logging "github.com/ipfs/go-log"
)
var log = logging.Logger("evtsm")
type Event struct {
User interface{}
}
// TODO: This probably should be returning an int indicating partial event processing
// (or something like errPartial(nEvents))
// returns func(ctx Context, st <T>) (func(*<T>), error), where <T> is the typeOf(User) param
type Planner func(events []Event, user interface{}) (interface{}, error)
type StateMachine struct {
planner Planner
eventsIn chan Event
name interface{}
st *statestore.StoredState
stateType reflect.Type
stageDone chan struct{}
closing chan struct{}
closed chan struct{}
busy int32
}
func (fsm *StateMachine) run() {
defer close(fsm.closed)
var pendingEvents []Event
for {
// NOTE: This requires at least one event to be sent to trigger a stage
// This means that after restarting the state machine users of this
// code must send a 'restart' event
select {
case evt := <-fsm.eventsIn:
pendingEvents = append(pendingEvents, evt)
case <-fsm.stageDone:
if len(pendingEvents) == 0 {
continue
}
case <-fsm.closing:
return
}
if atomic.CompareAndSwapInt32(&fsm.busy, 0, 1) {
var nextStep interface{}
var ustate interface{}
err := fsm.mutateUser(func(user interface{}) (err error) {
nextStep, err = fsm.planner(pendingEvents, user)
ustate = user
return err
})
if err != nil {
log.Errorf("Executing event planner failed: %+v", err)
return
}
pendingEvents = nil
if nextStep == nil {
continue
}
ctx := Context{
ctx: context.TODO(),
send: func(evt interface{}) error {
return fsm.send(Event{User: evt})
},
}
go func() {
res := reflect.ValueOf(nextStep).Call([]reflect.Value{reflect.ValueOf(ctx), reflect.ValueOf(ustate).Elem()})
if res[0].Interface() != nil {
log.Errorf("executing step: %+v", res[0].Interface().(error)) // TODO: propagate top level
return
}
atomic.StoreInt32(&fsm.busy, 0)
fsm.stageDone <- struct{}{}
}()
}
}
}
func (fsm *StateMachine) mutateUser(cb func(user interface{}) error) error {
mutt := reflect.FuncOf([]reflect.Type{reflect.PtrTo(fsm.stateType)}, []reflect.Type{reflect.TypeOf(new(error)).Elem()}, false)
mutf := reflect.MakeFunc(mutt, func(args []reflect.Value) (results []reflect.Value) {
err := cb(args[0].Interface())
return []reflect.Value{reflect.ValueOf(&err).Elem()}
})
return fsm.st.Mutate(mutf.Interface())
}
func (fsm *StateMachine) send(evt Event) error {
fsm.eventsIn <- evt // TODO: ctx, at least
return nil
}
func (fsm *StateMachine) stop(ctx context.Context) error {
close(fsm.closing)
select {
case <-fsm.closed:
return nil
case <-ctx.Done():
return ctx.Err()
}
}

105
lib/statemachine/machine_test.go
View File

@ -1,105 +0,0 @@
package statemachine
import (
"context"
"testing"
"github.com/ipfs/go-datastore"
logging "github.com/ipfs/go-log"
"gotest.tools/assert"
)
func init() {
logging.SetLogLevel("*", "INFO")
}
type testHandler struct {
t *testing.T
proceed chan struct{}
done chan struct{}
}
func (t *testHandler) Plan(events []Event, state interface{}) (interface{}, error) {
return t.plan(events, state.(*TestState))
}
func (t *testHandler) plan(events []Event, state *TestState) (func(Context, TestState) error, error) {
for _, event := range events {
e := event.User.(*TestEvent)
switch e.A {
case "restart":
case "start":
state.A = 1
case "b":
state.A = 2
state.B = e.Val
}
}
switch state.A {
case 1:
return t.step0, nil
case 2:
return t.step1, nil
default:
t.t.Fatal(state.A)
}
panic("how?")
}
func (t *testHandler) step0(ctx Context, st TestState) error {
ctx.Send(&TestEvent{A: "b", Val: 55})
<-t.proceed
return nil
}
func (t *testHandler) step1(ctx Context, st TestState) error {
assert.Equal(t.t, uint64(2), st.A)
close(t.done)
return nil
}
func TestBasic(t *testing.T) {
for i := 0; i < 1000; i++ { // run a few times to expose any races
ds := datastore.NewMapDatastore()
th := &testHandler{t: t, done: make(chan struct{}), proceed: make(chan struct{})}
close(th.proceed)
smm := New(ds, th, TestState{})
if err := smm.Send(uint64(2), &TestEvent{A: "start"}); err != nil {
t.Fatalf("%+v", err)
}
<-th.done
}
}
func TestPersist(t *testing.T) {
for i := 0; i < 1000; i++ { // run a few times to expose any races
ds := datastore.NewMapDatastore()
th := &testHandler{t: t, done: make(chan struct{}), proceed: make(chan struct{})}
smm := New(ds, th, TestState{})
if err := smm.Send(uint64(2), &TestEvent{A: "start"}); err != nil {
t.Fatalf("%+v", err)
}
if err := smm.Stop(context.Background()); err != nil {
t.Fatal(err)
return
}
smm = New(ds, th, TestState{})
if err := smm.Send(uint64(2), &TestEvent{A: "restart"}); err != nil {
t.Fatalf("%+v", err)
}
close(th.proceed)
<-th.done
}
}
var _ StateHandler = &testHandler{}

11
lib/statemachine/testing.go
View File

@ -1,11 +0,0 @@
package statemachine
type TestState struct {
A uint64
B uint64
}
type TestEvent struct {
A string
Val uint64
}

55
lib/zerocomm/zerocomm.go
View File

@ -1,55 +0,0 @@
package zerocomm
import (
"math/bits"
commcid "github.com/filecoin-project/go-fil-commcid"
"github.com/filecoin-project/specs-actors/actors/abi"
"github.com/ipfs/go-cid"
)
const levels = 37
const skip = 2 // can't generate for 32, 64b
var pieceComms = [levels - skip][32]byte{
{0x37, 0x31, 0xbb, 0x99, 0xac, 0x68, 0x9f, 0x66, 0xee, 0xf5, 0x97, 0x3e, 0x4a, 0x94, 0xda, 0x18, 0x8f, 0x4d, 0xdc, 0xae, 0x58, 0x7, 0x24, 0xfc, 0x6f, 0x3f, 0xd6, 0xd, 0xfd, 0x48, 0x83, 0x33},
{0x64, 0x2a, 0x60, 0x7e, 0xf8, 0x86, 0xb0, 0x4, 0xbf, 0x2c, 0x19, 0x78, 0x46, 0x3a, 0xe1, 0xd4, 0x69, 0x3a, 0xc0, 0xf4, 0x10, 0xeb, 0x2d, 0x1b, 0x7a, 0x47, 0xfe, 0x20, 0x5e, 0x5e, 0x75, 0xf},
{0x57, 0xa2, 0x38, 0x1a, 0x28, 0x65, 0x2b, 0xf4, 0x7f, 0x6b, 0xef, 0x7a, 0xca, 0x67, 0x9b, 0xe4, 0xae, 0xde, 0x58, 0x71, 0xab, 0x5c, 0xf3, 0xeb, 0x2c, 0x8, 0x11, 0x44, 0x88, 0xcb, 0x85, 0x26},
{0x1f, 0x7a, 0xc9, 0x59, 0x55, 0x10, 0xe0, 0x9e, 0xa4, 0x1c, 0x46, 0xb, 0x17, 0x64, 0x30, 0xbb, 0x32, 0x2c, 0xd6, 0xfb, 0x41, 0x2e, 0xc5, 0x7c, 0xb1, 0x7d, 0x98, 0x9a, 0x43, 0x10, 0x37, 0x2f},
{0xfc, 0x7e, 0x92, 0x82, 0x96, 0xe5, 0x16, 0xfa, 0xad, 0xe9, 0x86, 0xb2, 0x8f, 0x92, 0xd4, 0x4a, 0x4f, 0x24, 0xb9, 0x35, 0x48, 0x52, 0x23, 0x37, 0x6a, 0x79, 0x90, 0x27, 0xbc, 0x18, 0xf8, 0x33},
{0x8, 0xc4, 0x7b, 0x38, 0xee, 0x13, 0xbc, 0x43, 0xf4, 0x1b, 0x91, 0x5c, 0xe, 0xed, 0x99, 0x11, 0xa2, 0x60, 0x86, 0xb3, 0xed, 0x62, 0x40, 0x1b, 0xf9, 0xd5, 0x8b, 0x8d, 0x19, 0xdf, 0xf6, 0x24},
{0xb2, 0xe4, 0x7b, 0xfb, 0x11, 0xfa, 0xcd, 0x94, 0x1f, 0x62, 0xaf, 0x5c, 0x75, 0xf, 0x3e, 0xa5, 0xcc, 0x4d, 0xf5, 0x17, 0xd5, 0xc4, 0xf1, 0x6d, 0xb2, 0xb4, 0xd7, 0x7b, 0xae, 0xc1, 0xa3, 0x2f},
{0xf9, 0x22, 0x61, 0x60, 0xc8, 0xf9, 0x27, 0xbf, 0xdc, 0xc4, 0x18, 0xcd, 0xf2, 0x3, 0x49, 0x31, 0x46, 0x0, 0x8e, 0xae, 0xfb, 0x7d, 0x2, 0x19, 0x4d, 0x5e, 0x54, 0x81, 0x89, 0x0, 0x51, 0x8},
{0x2c, 0x1a, 0x96, 0x4b, 0xb9, 0xb, 0x59, 0xeb, 0xfe, 0xf, 0x6d, 0xa2, 0x9a, 0xd6, 0x5a, 0xe3, 0xe4, 0x17, 0x72, 0x4a, 0x8f, 0x7c, 0x11, 0x74, 0x5a, 0x40, 0xca, 0xc1, 0xe5, 0xe7, 0x40, 0x11},
{0xfe, 0xe3, 0x78, 0xce, 0xf1, 0x64, 0x4, 0xb1, 0x99, 0xed, 0xe0, 0xb1, 0x3e, 0x11, 0xb6, 0x24, 0xff, 0x9d, 0x78, 0x4f, 0xbb, 0xed, 0x87, 0x8d, 0x83, 0x29, 0x7e, 0x79, 0x5e, 0x2, 0x4f, 0x2},
{0x8e, 0x9e, 0x24, 0x3, 0xfa, 0x88, 0x4c, 0xf6, 0x23, 0x7f, 0x60, 0xdf, 0x25, 0xf8, 0x3e, 0xe4, 0xd, 0xca, 0x9e, 0xd8, 0x79, 0xeb, 0x6f, 0x63, 0x52, 0xd1, 0x50, 0x84, 0xf5, 0xad, 0xd, 0x3f},
{0x75, 0x2d, 0x96, 0x93, 0xfa, 0x16, 0x75, 0x24, 0x39, 0x54, 0x76, 0xe3, 0x17, 0xa9, 0x85, 0x80, 0xf0, 0x9, 0x47, 0xaf, 0xb7, 0xa3, 0x5, 0x40, 0xd6, 0x25, 0xa9, 0x29, 0x1c, 0xc1, 0x2a, 0x7},
{0x70, 0x22, 0xf6, 0xf, 0x7e, 0xf6, 0xad, 0xfa, 0x17, 0x11, 0x7a, 0x52, 0x61, 0x9e, 0x30, 0xce, 0xa8, 0x2c, 0x68, 0x7, 0x5a, 0xdf, 0x1c, 0x66, 0x77, 0x86, 0xec, 0x50, 0x6e, 0xef, 0x2d, 0x19},
{0xd9, 0x98, 0x87, 0xb9, 0x73, 0x57, 0x3a, 0x96, 0xe1, 0x13, 0x93, 0x64, 0x52, 0x36, 0xc1, 0x7b, 0x1f, 0x4c, 0x70, 0x34, 0xd7, 0x23, 0xc7, 0xa9, 0x9f, 0x70, 0x9b, 0xb4, 0xda, 0x61, 0x16, 0x2b},
{0xd0, 0xb5, 0x30, 0xdb, 0xb0, 0xb4, 0xf2, 0x5c, 0x5d, 0x2f, 0x2a, 0x28, 0xdf, 0xee, 0x80, 0x8b, 0x53, 0x41, 0x2a, 0x2, 0x93, 0x1f, 0x18, 0xc4, 0x99, 0xf5, 0xa2, 0x54, 0x8, 0x6b, 0x13, 0x26},
{0x84, 0xc0, 0x42, 0x1b, 0xa0, 0x68, 0x5a, 0x1, 0xbf, 0x79, 0x5a, 0x23, 0x44, 0x6, 0x4f, 0xe4, 0x24, 0xbd, 0x52, 0xa9, 0xd2, 0x43, 0x77, 0xb3, 0x94, 0xff, 0x4c, 0x4b, 0x45, 0x68, 0xe8, 0x11},
{0x65, 0xf2, 0x9e, 0x5d, 0x98, 0xd2, 0x46, 0xc3, 0x8b, 0x38, 0x8c, 0xfc, 0x6, 0xdb, 0x1f, 0x6b, 0x2, 0x13, 0x3, 0xc5, 0xa2, 0x89, 0x0, 0xb, 0xdc, 0xe8, 0x32, 0xa9, 0xc3, 0xec, 0x42, 0x1c},
{0xa2, 0x24, 0x75, 0x8, 0x28, 0x58, 0x50, 0x96, 0x5b, 0x7e, 0x33, 0x4b, 0x31, 0x27, 0xb0, 0xc0, 0x42, 0xb1, 0xd0, 0x46, 0xdc, 0x54, 0x40, 0x21, 0x37, 0x62, 0x7c, 0xd8, 0x79, 0x9c, 0xe1, 0x3a},
{0xda, 0xfd, 0xab, 0x6d, 0xa9, 0x36, 0x44, 0x53, 0xc2, 0x6d, 0x33, 0x72, 0x6b, 0x9f, 0xef, 0xe3, 0x43, 0xbe, 0x8f, 0x81, 0x64, 0x9e, 0xc0, 0x9, 0xaa, 0xd3, 0xfa, 0xff, 0x50, 0x61, 0x75, 0x8},
{0xd9, 0x41, 0xd5, 0xe0, 0xd6, 0x31, 0x4a, 0x99, 0x5c, 0x33, 0xff, 0xbd, 0x4f, 0xbe, 0x69, 0x11, 0x8d, 0x73, 0xd4, 0xe5, 0xfd, 0x2c, 0xd3, 0x1f, 0xf, 0x7c, 0x86, 0xeb, 0xdd, 0x14, 0xe7, 0x6},
{0x51, 0x4c, 0x43, 0x5c, 0x3d, 0x4, 0xd3, 0x49, 0xa5, 0x36, 0x5f, 0xbd, 0x59, 0xff, 0xc7, 0x13, 0x62, 0x91, 0x11, 0x78, 0x59, 0x91, 0xc1, 0xa3, 0xc5, 0x3a, 0xf2, 0x20, 0x79, 0x74, 0x1a, 0x2f},
{0xad, 0x6, 0x85, 0x39, 0x69, 0xd3, 0x7d, 0x34, 0xff, 0x8, 0xe0, 0x9f, 0x56, 0x93, 0xa, 0x4a, 0xd1, 0x9a, 0x89, 0xde, 0xf6, 0xc, 0xbf, 0xee, 0x7e, 0x1d, 0x33, 0x81, 0xc1, 0xe7, 0x1c, 0x37},
{0x39, 0x56, 0xe, 0x7b, 0x13, 0xa9, 0x3b, 0x7, 0xa2, 0x43, 0xfd, 0x27, 0x20, 0xff, 0xa7, 0xcb, 0x3e, 0x1d, 0x2e, 0x50, 0x5a, 0xb3, 0x62, 0x9e, 0x79, 0xf4, 0x63, 0x13, 0x51, 0x2c, 0xda, 0x6},
{0xcc, 0xc3, 0xc0, 0x12, 0xf5, 0xb0, 0x5e, 0x81, 0x1a, 0x2b, 0xbf, 0xdd, 0xf, 0x68, 0x33, 0xb8, 0x42, 0x75, 0xb4, 0x7b, 0xf2, 0x29, 0xc0, 0x5, 0x2a, 0x82, 0x48, 0x4f, 0x3c, 0x1a, 0x5b, 0x3d},
{0x7d, 0xf2, 0x9b, 0x69, 0x77, 0x31, 0x99, 0xe8, 0xf2, 0xb4, 0xb, 0x77, 0x91, 0x9d, 0x4, 0x85, 0x9, 0xee, 0xd7, 0x68, 0xe2, 0xc7, 0x29, 0x7b, 0x1f, 0x14, 0x37, 0x3, 0x4f, 0xc3, 0xc6, 0x2c},
{0x66, 0xce, 0x5, 0xa3, 0x66, 0x75, 0x52, 0xcf, 0x45, 0xc0, 0x2b, 0xcc, 0x4e, 0x83, 0x92, 0x91, 0x9b, 0xde, 0xac, 0x35, 0xde, 0x2f, 0xf5, 0x62, 0x71, 0x84, 0x8e, 0x9f, 0x7b, 0x67, 0x51, 0x7},
{0xd8, 0x61, 0x2, 0x18, 0x42, 0x5a, 0xb5, 0xe9, 0x5b, 0x1c, 0xa6, 0x23, 0x9d, 0x29, 0xa2, 0xe4, 0x20, 0xd7, 0x6, 0xa9, 0x6f, 0x37, 0x3e, 0x2f, 0x9c, 0x9a, 0x91, 0xd7, 0x59, 0xd1, 0x9b, 0x1},
{0x6d, 0x36, 0x4b, 0x1e, 0xf8, 0x46, 0x44, 0x1a, 0x5a, 0x4a, 0x68, 0x86, 0x23, 0x14, 0xac, 0xc0, 0xa4, 0x6f, 0x1, 0x67, 0x17, 0xe5, 0x34, 0x43, 0xe8, 0x39, 0xee, 0xdf, 0x83, 0xc2, 0x85, 0x3c},
{0x7, 0x7e, 0x5f, 0xde, 0x35, 0xc5, 0xa, 0x93, 0x3, 0xa5, 0x50, 0x9, 0xe3, 0x49, 0x8a, 0x4e, 0xbe, 0xdf, 0xf3, 0x9c, 0x42, 0xb7, 0x10, 0xb7, 0x30, 0xd8, 0xec, 0x7a, 0xc7, 0xaf, 0xa6, 0x3e},
{0xe6, 0x40, 0x5, 0xa6, 0xbf, 0xe3, 0x77, 0x79, 0x53, 0xb8, 0xad, 0x6e, 0xf9, 0x3f, 0xf, 0xca, 0x10, 0x49, 0xb2, 0x4, 0x16, 0x54, 0xf2, 0xa4, 0x11, 0xf7, 0x70, 0x27, 0x99, 0xce, 0xce, 0x2},
{0x25, 0x9d, 0x3d, 0x6b, 0x1f, 0x4d, 0x87, 0x6d, 0x11, 0x85, 0xe1, 0x12, 0x3a, 0xf6, 0xf5, 0x50, 0x1a, 0xf0, 0xf6, 0x7c, 0xf1, 0x5b, 0x52, 0x16, 0x25, 0x5b, 0x7b, 0x17, 0x8d, 0x12, 0x5, 0x1d},
{0x3f, 0x9a, 0x4d, 0x41, 0x1d, 0xa4, 0xef, 0x1b, 0x36, 0xf3, 0x5f, 0xf0, 0xa1, 0x95, 0xae, 0x39, 0x2a, 0xb2, 0x3f, 0xee, 0x79, 0x67, 0xb7, 0xc4, 0x1b, 0x3, 0xd1, 0x61, 0x3f, 0xc2, 0x92, 0x39},
{0xfe, 0x4e, 0xf3, 0x28, 0xc6, 0x1a, 0xa3, 0x9c, 0xfd, 0xb2, 0x48, 0x4e, 0xaa, 0x32, 0xa1, 0x51, 0xb1, 0xfe, 0x3d, 0xfd, 0x1f, 0x96, 0xdd, 0x8c, 0x97, 0x11, 0xfd, 0x86, 0xd6, 0xc5, 0x81, 0x13},
{0xf5, 0x5d, 0x68, 0x90, 0xe, 0x2d, 0x83, 0x81, 0xec, 0xcb, 0x81, 0x64, 0xcb, 0x99, 0x76, 0xf2, 0x4b, 0x2d, 0xe0, 0xdd, 0x61, 0xa3, 0x1b, 0x97, 0xce, 0x6e, 0xb2, 0x38, 0x50, 0xd5, 0xe8, 0x19},
{0xaa, 0xaa, 0x8c, 0x4c, 0xb4, 0xa, 0xac, 0xee, 0x1e, 0x2, 0xdc, 0x65, 0x42, 0x4b, 0x2a, 0x6c, 0x8e, 0x99, 0xf8, 0x3, 0xb7, 0x2f, 0x79, 0x29, 0xc4, 0x10, 0x1d, 0x7f, 0xae, 0x6b, 0xff, 0x32},
}
func ForSize(sz abi.UnpaddedPieceSize) cid.Cid {
level := bits.TrailingZeros64(uint64(sz.Padded())) - skip - 5 // 2^5 = 32
return commcid.DataCommitmentV1ToCID(pieceComms[level][:])
}

106
lib/zerocomm/zerocomm_test.go
View File

@ -1,106 +0,0 @@
package zerocomm
import (
"bytes"
"fmt"
"io"
"testing"
commcid "github.com/filecoin-project/go-fil-commcid"
"github.com/filecoin-project/go-sectorbuilder"
abi "github.com/filecoin-project/specs-actors/actors/abi"
"github.com/ipfs/go-cid"
"github.com/filecoin-project/lotus/lib/nullreader"
)
func TestComms(t *testing.T) {
t.Skip("don't have enough ram") // no, but seriously, currently this needs like 3tb of /tmp
var expPieceComms [levels - skip]cid.Cid
{
l2, err := sectorbuilder.GeneratePieceCIDFromFile(abi.RegisteredProof_StackedDRG2KiBPoSt, bytes.NewReader(make([]byte, 127)), 127)
if err != nil {
t.Fatal(err)
}
expPieceComms[0] = l2
}
for i := 1; i < levels-2; i++ {
var err error
sz := abi.UnpaddedPieceSize(127 << i)
fmt.Println(i, sz)
r := io.LimitReader(&nullreader.Reader{}, int64(sz))
expPieceComms[i], err = sectorbuilder.GeneratePieceCIDFromFile(abi.RegisteredProof_StackedDRG2KiBPoSt, r, sz)
if err != nil {
t.Fatal(err)
}
}
for i, comm := range expPieceComms {
c, err := commcid.CIDToPieceCommitmentV1(comm)
if err != nil {
t.Fatal(err)
}
if string(c) != string(pieceComms[i][:]) {
t.Errorf("zero commitment %d didn't match", i)
}
}
for _, comm := range expPieceComms { // Could do codegen, but this is good enough
fmt.Printf("%#v,\n", comm)
}
}
func TestCommsSmall(t *testing.T) {
var expPieceComms [8]cid.Cid
lvls := len(expPieceComms) + skip
{
l2, err := sectorbuilder.GeneratePieceCIDFromFile(abi.RegisteredProof_StackedDRG2KiBPoSt, bytes.NewReader(make([]byte, 127)), 127)
if err != nil {
t.Fatal(err)
}
expPieceComms[0] = l2
}
for i := 1; i < lvls-2; i++ {
var err error
sz := abi.UnpaddedPieceSize(127 << i)
fmt.Println(i, sz)
r := io.LimitReader(&nullreader.Reader{}, int64(sz))
expPieceComms[i], err = sectorbuilder.GeneratePieceCIDFromFile(abi.RegisteredProof_StackedDRG2KiBPoSt, r, sz)
if err != nil {
t.Fatal(err)
}
}
for i, comm := range expPieceComms {
c, err := commcid.CIDToPieceCommitmentV1(comm)
if err != nil {
t.Fatal(err)
}
if string(c) != string(pieceComms[i][:]) {
t.Errorf("zero commitment %d didn't match", i)
}
}
for _, comm := range expPieceComms { // Could do codegen, but this is good enough
fmt.Printf("%#v,\n", comm)
}
}
func TestForSise(t *testing.T) {
exp, err := sectorbuilder.GeneratePieceCIDFromFile(abi.RegisteredProof_StackedDRG2KiBPoSt, bytes.NewReader(make([]byte, 1016)), 1016)
if err != nil {
return
}
actual := ForSize(1016)
if !exp.Equals(actual) {
t.Errorf("zero commitment didn't match")
}
}

4
storage/sbmock/preseal.go
View File

@ -3,6 +3,7 @@ package sbmock
import (
"github.com/filecoin-project/go-address"
commcid "github.com/filecoin-project/go-fil-commcid"
"github.com/filecoin-project/go-sectorbuilder"
"github.com/filecoin-project/specs-actors/actors/abi"
"github.com/filecoin-project/specs-actors/actors/abi/big"
"github.com/filecoin-project/specs-actors/actors/builtin/market"
@ -12,7 +13,6 @@ import (
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/chain/wallet"
"github.com/filecoin-project/lotus/genesis"
"github.com/filecoin-project/lotus/lib/zerocomm"
)
func PreSeal(ssize abi.SectorSize, maddr address.Address, sectors int) (*genesis.Miner, *types.KeyInfo, error) {
@ -39,7 +39,7 @@ func PreSeal(ssize abi.SectorSize, maddr address.Address, sectors int) (*genesis
preseal := &genesis.PreSeal{}
preseal.ProofType = st
preseal.CommD = zerocomm.ForSize(abi.PaddedPieceSize(ssize).Unpadded())
preseal.CommD = sectorbuilder.ZeroPieceCommitment(abi.PaddedPieceSize(ssize).Unpadded())
d, _ := commcid.CIDToPieceCommitmentV1(preseal.CommD)
r := commDR(d)
preseal.CommR = commcid.ReplicaCommitmentV1ToCID(r[:])

4
storage/sealing/checks.go
View File

@ -4,6 +4,7 @@ import (
"bytes"
"context"
"github.com/filecoin-project/go-sectorbuilder"
"github.com/ipfs/go-cid"
cbg "github.com/whyrusleeping/cbor-gen"
"golang.org/x/xerrors"
@ -15,7 +16,6 @@ import (
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/actors"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/lib/zerocomm"
)
// TODO: For now we handle this by halting state execution, when we get jsonrpc reconnecting
@ -42,7 +42,7 @@ func checkPieces(ctx context.Context, si SectorInfo, api sealingApi) error {
for i, piece := range si.Pieces {
if piece.DealID == nil {
exp := zerocomm.ForSize(piece.Size)
exp := sectorbuilder.ZeroPieceCommitment(piece.Size)
if piece.CommP != exp {
return &ErrInvalidPiece{xerrors.Errorf("deal %d piece %d had non-zero CommP %+v", piece.DealID, i, piece.CommP)}
}

11
storage/sealing/fsm.go
View File

@ -6,17 +6,18 @@ import (
"reflect"
"time"
"github.com/filecoin-project/specs-actors/actors/abi"
"golang.org/x/xerrors"
"github.com/filecoin-project/go-statemachine"
"github.com/filecoin-project/specs-actors/actors/abi"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/lib/statemachine"
)
func (m *Sealing) Plan(events []statemachine.Event, user interface{}) (interface{}, error) {
func (m *Sealing) Plan(events []statemachine.Event, user interface{}) (interface{}, uint64, error) {
next, err := m.plan(events, user.(*SectorInfo))
if err != nil || next == nil {
return nil, err
return nil, 0, err
}
return func(ctx statemachine.Context, si SectorInfo) error {
@ -27,7 +28,7 @@ func (m *Sealing) Plan(events []statemachine.Event, user interface{}) (interface
}
return nil
}, nil
}, uint64(len(events)), nil // TODO: This processed event count is not very correct
}
var fsmPlanners = []func(events []statemachine.Event, state *SectorInfo) error{

3
storage/sealing/fsm_test.go
View File

@ -6,8 +6,9 @@ import (
logging "github.com/ipfs/go-log/v2"
"github.com/stretchr/testify/require"
"github.com/filecoin-project/go-statemachine"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/lib/statemachine"
)
func init() {

2
storage/sealing/sealing.go
View File

@ -12,6 +12,7 @@ import (
"golang.org/x/xerrors"
"github.com/filecoin-project/go-padreader"
"github.com/filecoin-project/go-statemachine"
"github.com/filecoin-project/specs-actors/actors/abi"
"github.com/filecoin-project/specs-actors/actors/builtin/miner"
"github.com/filecoin-project/specs-actors/actors/crypto"
@ -20,7 +21,6 @@ import (
"github.com/filecoin-project/lotus/chain/events"
"github.com/filecoin-project/lotus/chain/store"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/lib/statemachine"
"github.com/filecoin-project/lotus/storage/sealmgr"
)

2
storage/sealing/states.go
View File

@ -5,6 +5,7 @@ import (
"github.com/filecoin-project/specs-actors/actors/crypto"
"github.com/filecoin-project/go-statemachine"
"github.com/filecoin-project/specs-actors/actors/abi"
"github.com/filecoin-project/specs-actors/actors/builtin"
"github.com/filecoin-project/specs-actors/actors/builtin/miner"
@ -14,7 +15,6 @@ import (
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/actors"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/lib/statemachine"
)
func (m *Sealing) handlePacking(ctx statemachine.Context, sector SectorInfo) error {

2
storage/sealing/states_failed.go
View File

@ -4,6 +4,7 @@ import (
"bytes"
"time"
"github.com/filecoin-project/go-statemachine"
"github.com/filecoin-project/specs-actors/actors/builtin/miner"
"github.com/filecoin-project/specs-actors/actors/util/adt"
"golang.org/x/xerrors"
@ -12,7 +13,6 @@ import (
"github.com/filecoin-project/lotus/api/apibstore"
"github.com/filecoin-project/lotus/chain/store"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/lib/statemachine"
)
const minRetryTime = 1 * time.Minute