Improve complex code and comment it

License: MIT
Signed-off-by: Jakub Sztandera <kubuxu@protonmail.ch>

docs/rewards.m

@@ -0,0 +1,29 @@
+
+HalvingPeriod = vpa(6 * 365 * 24 * 60 * 2);
+AdjusmentPeriod = vpa(20160);
+Decay = exp(log(vpa(0.5)) / HalvingPeriod * AdjusmentPeriod);
+Total = vpa(1400e6);
+IV = Total * (1-Decay) / AdjusmentPeriod;
+
+
+syms R(x)
+R(x) = IV * (Decay.^floor(x/AdjusmentPeriod));
+
+Hmax = HalvingPeriod*5;
+heights = linspace(0, Hmax, Hmax/AdjusmentPeriod);
+Rewards = R(heights);
+
+R2 = zeros(size(heights));
+
+coffer = Total;
+for h = 1:size(heights,2)
+  k = IV*(coffer/Total);
+  coffer = coffer - k*AdjusmentPeriod;
+  R2(h) = k;
+end
+hYears = heights/2/60/24/365;
+plot(hYears, Rewards, 'go', hYears, R2, 'r-')
+legend('formula', 'incremental');
+
+for6y = Rewards(1:HalvingPeriod/AdjusmentPeriod)*AdjusmentPeriod;
+inc6t = R2(1:HalvingPeriod/AdjusmentPeriod)*AdjusmentPeriod;

docs/rewards.m~

@@ -0,0 +1,30 @@
+
+HalvingPeriod = vpa(6 * 365 * 24 * 60 * 2);
+AdjusmentPeriod = vpa(20160);
+Decay = exp(log(vpa(0.5)) / HalvingPeriod * AdjusmentPeriod);
+Total = vpa(1400e6);
+IV = Total * (1-Decay) / AdjusmentPeriod;
+
+
+
+syms R(x)
+R(x) = IV * (Decay.^floor(x/AdjusmentPeriod));
+
+Hmax = HalvingPeriod*10;
+heights = linspace(0, Hmax, HalvingPeriod/AdjusmentPeriod);
+Rewards = R(heights);
+
+plot(heights/2/60/24/365, Rewards, 'r-')
+
+R2 = zeros(size(heights));
+
+coffer = Total;
+for h = 1:size(heights,2)
+  
+  k = IV*(coffer/Total);
+  coffer = coffer - k*AdjusmentPeriod;
+  R2(h) = k;
+  
+end
+hYears = heights/2/60/24/365;
+plot(hYears, Rewards, 'r-', hYears, R2, 'g-')

docs/rewards2.m

@@ -0,0 +1,19 @@
+syms height;
+
+
+IV = 153;
+HalvingPeriod = (6 * 365 * 24 * 60 * 2);
+AdjusmentPeriod = vpa(20160);
+Total = vpa(1400e6);
+
+syms coffer(height)
+syms R(height);
+syms h
+%coffer(height) = piecewise(height <= 0, Total, height > 0, Total - symsum(R(h), h, 0, height));
+coffer(height) = piecewise(height < 0, Total, height >= 0, coffer(height-1) - R(height));
+R(height) = coffer(height-1)/Total * IV;
+syms x
+r = R(x);
+
+fplot(r, [0, 5]);
+fibonacci

lib/rlepluslazy/bits.go

@@ -129,6 +129,23 @@ func (it *it2r) init() error {
 	return nil
 }
 
+func SliceFromRuns(source RunIterator) ([]uint64, error) {
+	rit, err := BitsFromRuns(source)
+	if err != nil {
+		return nil, err
+	}
+
+	res := make([]uint64, 0)
+	for rit.HasNext() {
+		bit, err := rit.Next()
+		if err != nil {
+			return nil, err
+		}
+		res = append(res, bit)
+	}
+	return res, nil
+}
+
 func RunsFromBits(source BitIterator) (RunIterator, error) {
 	it := &it2r{source: source}
 
@@ -137,3 +154,7 @@ func RunsFromBits(source BitIterator) (RunIterator, error) {
 	}
 	return it, nil
 }
+
+func RunsFromSlice(slice []uint64) (RunIterator, error) {
+	return RunsFromBits(BitsFromSlice(slice))
+}

lib/rlepluslazy/bitvec.go

@@ -10,18 +10,18 @@ type rbitvec struct {
 }
 
 func readBitvec(vec []byte) *rbitvec {
-	bv := &rbitvec{vec: vec}
+	bv := &rbitvec{
+		vec:    vec,
+		index:  1,
+		bitCap: 8,
+	}
 	if len(vec) > 0 {
 		bv.bits = uint16(bv.vec[0])
 	}
 	return bv
 }
 
-const (
-	minCap = 8
-	maxCap = 16
-)
-
+// bitMasks is a mask for selecting N first bits out of a byte
 var bitMasks = [9]byte{
 	0x0,
 	0x1,
@@ -35,54 +35,80 @@ var bitMasks = [9]byte{
 }
 
 func (bv *rbitvec) Get(count byte) byte {
-	res := byte(bv.bits) & bitMasks[count]
-	bv.bits = bv.bits >> count
-	bv.bitCap = bv.bitCap - count
+	res := byte(bv.bits) & bitMasks[count] // select count bits
+	bv.bits = bv.bits >> count             // remove those bits from storage
+	bv.bitCap = bv.bitCap - count          // decrease nuber of stored bits
 
-	if bv.index < len(bv.vec) {
+	if bv.index < len(bv.vec) { // if vector allows
+		// add bits onto the end of temporary storage
 		bv.bits = bv.bits | uint16(bv.vec[bv.index])<<bv.bitCap
 	}
+
 	// Here be dragons
-	inc := (bv.bitCap - 8) >> 7 // inc == 1 iff bitcap<8 (+10% perf)
-	bv.index = bv.index + int(inc)
-	bv.bitCap = bv.bitCap + inc<<3
+	// This is equivalent to
+	// if bv.bitCap < 8 {
+	//     bv.index++
+	//     bv.bitCap = bv.bitCap + 8
+	// }
+	// but implemented without branches because the branch here is unpredictable
+	// Why this is without branches and reading has branch?
+	//  Because branch above is predictable, in 99.99% of cases it will be true
+
+	// if bitCap < 8 it underflows, then high bits get set to 1s
+	// we shift by 7 so the highest bit is in place of the lowest
+	inc := (bv.bitCap - 8) >> 7    // inc == 1 iff bitcap<8 (+10% perf)
+	bv.index = bv.index + int(inc) // increase index if we need more bits
+	bv.bitCap = bv.bitCap + inc*8  // increase bitCap by 8
 
 	return res
 }
 
 func writeBitvec(buf []byte) *wbitvec {
+	// reslice to 0 length for consistent input but to keep capacity
 	return &wbitvec{buf: buf[:0]}
 }
 
 type wbitvec struct {
-	buf   []byte
-	index int
+	buf   []byte // buffer we will be saving to
+	index int    // index of at which the next byte will be saved
 
-	bits   uint16
-	bitCap byte
+	bits   uint16 // temporary storage for bits
+	bitCap byte   // number of bits stored in temporary storage
 }
 
 func (bv *wbitvec) Out() []byte {
 	if bv.bitCap != 0 {
+		// if there are some bits in temporary storage we need to save them
 		bv.buf = append(bv.buf, 0)[:bv.index+1]
 		bv.buf[bv.index] = byte(bv.bits)
 	}
 	if bv.bitCap > 8 {
+		// if we store some needed bits in second byte, save them also
 		bv.buf = append(bv.buf, byte(bv.bitCap>>8))
+		bv.index++
+		bv.bits = bv.bits - 8
 	}
 	return bv.buf
 }
 
 func (bv *wbitvec) Put(val byte, count byte) {
+	// put val into its place in bv.bits
 	bv.bits = bv.bits | uint16(val)<<bv.bitCap
+	// increase bitCap by the number of bits
 	bv.bitCap = bv.bitCap + count
 
-	bv.buf = append(bv.buf, 0)[:bv.index+1] // increase cap, keep len
+	// increase len of the buffer if it is needed
+	if bv.index+1 > cap(bv.buf) {
+		bv.buf = append(bv.buf, 0)
+	}
+	bv.buf = bv.buf[:bv.index+1]
+	// save the bits
 	bv.buf[bv.index] = byte(bv.bits)
 
 	// Warning, dragons again
-	inc := (^(bv.bitCap - 8)) >> 7 // inc == 1 iff bitcap>=8
-	bv.index = bv.index + int(inc)
-	bv.bitCap = bv.bitCap - inc<<3
-	bv.bits = bv.bits >> (inc << 3)
+	// if bitCap is greater than 7 it underflows, same thing as in Put
+	inc := (7 - bv.bitCap) >> 7    // inc == 1 iff bitcap>=8
+	bv.index = bv.index + int(inc) // increase index for the next save
+	bv.bitCap = bv.bitCap - inc*8  // we store less bits now in temporary buffer
+	bv.bits = bv.bits >> (inc * 8) // we can discard those bits as they were saved
 }

lib/rlepluslazy/bitvec.go.bak

@@ -0,0 +1,63 @@
+package rlepluslazy
+
+type rbitvec struct {
+	index int
+
+	bits   uint64
+	bitCap byte
+
+	vec []byte
+}
+
+func readBitvec(vec []byte) *rbitvec {
+	bv := &rbitvec{vec: vec}
+	for n := 7; n >= 0; n-- {
+		var o uint64
+		if len(bv.vec) > n {
+			o = uint64(bv.vec[n])
+		}
+
+		bv.bits = bv.bits<<8 | o
+		bv.index++
+	}
+	bv.bitCap = 64
+	return bv
+}
+
+const (
+	minCap = 8
+)
+
+var bitMasks = [9]byte{
+	0x0,
+	0x1,
+	0x3,
+	0x7,
+	0xF,
+	0x1F,
+	0x3F,
+	0x7F,
+	0xFF,
+}
+
+func (bv *rbitvec) Get(count byte) byte {
+	res := byte(bv.bits) & bitMasks[count]
+	bv.bits = bv.bits >> count
+	bv.bitCap = bv.bitCap - count
+
+	if bv.bitCap < minCap {
+		var add uint64
+		for n := 6; n >= 0; n-- {
+			var o uint64
+			if len(bv.vec) > bv.index+n {
+				o = uint64(bv.vec[bv.index+n])
+			}
+
+			add = add<<8 | o
+		}
+		bv.index = bv.index + 7
+		bv.bits = bv.bits | add<<(bv.bitCap)
+		bv.bitCap = bv.bitCap + 7*8
+	}
+	return res
+}

lib/rlepluslazy/bitvec_test.go

@@ -0,0 +1,21 @@
+package rlepluslazy
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestReadBitVec(t *testing.T) {
+	buf := []byte{0x0, 0xff}
+	bv := readBitvec(buf)
+
+	o := bv.Get(1)
+	assert.EqualValues(t, 0, o)
+
+	o = bv.Get(8)
+	assert.EqualValues(t, 0x80, o)
+
+	o = bv.Get(7)
+	assert.EqualValues(t, 0x7f, o)
+}

lib/rlepluslazy/rleplus.go

@@ -22,9 +22,14 @@ type RLE struct {
 
 type change struct {
 	set   bool
+	reset bool
 	index uint64
 }
 
+func (c change) valid() bool {
+	return c.reset || c.set
+}
+
 func FromBuf(buf []byte) (*RLE, error) {
 	rle := &RLE{buf: buf}
 
@@ -36,7 +41,131 @@ func FromBuf(buf []byte) (*RLE, error) {
 }
 
 func (rle *RLE) RunIterator() (RunIterator, error) {
-	return DecodeRLE(rle.buf)
+	source, err := DecodeRLE(rle.buf)
+	return source, err
+}
+
+/*
+	if err != nil {
+		return nil, err
+	}
+
+	if len(rle.changes) == 0 {
+		return source, nil
+	}
+
+	ch := rle.changes
+	// using stable sort because we want to preserve change order
+	sort.SliceStable(ch, func(i int, j int) bool {
+		return ch[i].index < ch[j].index
+	})
+	for i := range ch {
+		if i+1 >= len(ch) {
+			break
+		}
+		if ch[i].index == ch[i+1].index {
+			ch[i].set = false
+			ch[i].reset = false
+		}
+	}
+
+	sets := make([]uint64, 0, len(ch)/2)
+	clears := make([]uint64, 0, len(ch)/2)
+	for _, c := range ch {
+		if c.set {
+			sets = append(sets, c.index)
+		} else if c.reset {
+			clears = append(clears, c.index)
+		}
+	}
+
+	setRuns, err := RunsFromSlice(sets)
+	if err != nil {
+		return nil, err
+	}
+	clearRuns, err := RunsFromSlice(clears)
+	if err != nil {
+		return nil, err
+	}
+
+	it := &chit{source: source, sets: setRuns, clears: clearRuns}
+	return nil, nil
+}
+
+type chit struct {
+	source RunIterator
+	sets   RunIterator
+	clears RunIterator
+
+	next Run
+
+	nextSource Run
+	nextSet    Run
+	nextClear  Run
+}
+
+func (it *chit) prep() error {
+	var err error
+	if !it.nextSource.Valid() && it.source.HasNext() {
+		it.nextSource, err = it.source.NextRun()
+		if err != nil {
+			return err
+		}
+	}
+
+	if !it.nextSet.Valid() && it.sets.HasNext() {
+		it.nextSet, err = it.sets.NextRun()
+		if err != nil {
+			return err
+		}
+	}
+
+	if !it.nextClear.Valid() && it.clears.HasNext() {
+		it.nextClear, err = it.clears.NextRun()
+		if err != nil {
+			return err
+		}
+	}
+
+	for len(it.changes) != 0 && !it.changes[0].valid() {
+		it.changes = it.changes[1:]
+	}
+
+	var ch change
+	if len(it.changes) != 0 {
+		ch = it.changes[0]
+	}
+
+	if it.source.HasNext() {
+		var err error
+		it.nextRun, err = it.source.NextRun()
+		if err != nil {
+			return err
+		}
+	}
+
+	if ch.valid() && ch.index < it.runIndex+it.nextRun.Len {
+		if ch.set != it.nextRun.Val {
+			// split run
+			whole := it.nextRun
+			it.nextRun.Len = ch.index - it.runIndex
+
+		}
+
+		// if we are here then change was valid so len(it.changes) != 0
+		it.changes = it.changes[1:]
+	} else {
+		it.runIndex = it.runIndex + it.nextRun.Len
+	}
+	return nil
+}
+
+func (it *chit) HasNext() bool {
+	return it.nextRun.Valid()
+}
+
+func (it *chit) NextRun() (Run, error) {
+	return it.nextRun, it.prep()
 }
 
 func (rle *RLE) Set(index uint64) {
@@ -44,5 +173,10 @@ func (rle *RLE) Set(index uint64) {
 }
 
 func (rle *RLE) Clear(index uint64) {
-	rle.changes = append(rle.changes, change{set: false, index: index})
+	rle.changes = append(rle.changes, change{reset: true, index: index})
 }
+
+func (rle *RLE) Merge(other *RLE) RunIterator {
+	return nil
+}
+*/

lib/rlepluslazy/rleplus_test.go

@@ -4,7 +4,7 @@ import (
 	"math/rand"
 	"testing"
 
-	"github.com/filecoin-project/go-lotus/extern/rleplus"
+	"github.com/filecoin-project/lotus/extern/rleplus"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -151,20 +151,20 @@ func BenchmarkOldRLE(b *testing.B) {
 func BenchmarkDecodeEncode(b *testing.B) {
 	b.ReportAllocs()
 	var r uint64
+	out := make([]byte, 0, len(goldenRLE))
+	for i := 0; i < b.N; i++ {
+		rle, _ := FromBuf(goldenRLE)
+		rit, _ := rle.RunIterator()
+		out, _ = EncodeRuns(rit, out)
+		r = r + uint64(len(out))
+	}
+
 	/*
-		out := make([]byte, 0, len(goldenRLE))
 		for i := 0; i < b.N; i++ {
-			rle, _ := FromBuf(goldenRLE)
-			rit, _ := rle.RunIterator()
-			out, _ = EncodeRuns(rit, out)
+			rle, _ := rleplus.Decode(goldenRLE)
+			out, _, _ := rleplus.Encode(rle)
 			r = r + uint64(len(out))
 		}
 	*/
-
-	for i := 0; i < b.N; i++ {
-		rle, _ := rleplus.Decode(goldenRLE)
-		out, _, _ := rleplus.Encode(rle)
-		r = r + uint64(len(out))
-	}
 	Res = Res + r
 }

lib/rlepluslazy/runs.go

@@ -0,0 +1,96 @@
+package rlepluslazy
+
+func Sum(a, b RunIterator) (RunIterator, error) {
+	it := addIt{a: a, b: b}
+	it.prep()
+	return &it, nil
+}
+
+type addIt struct {
+	a RunIterator
+	b RunIterator
+
+	next Run
+
+	arun Run
+	brun Run
+}
+
+func (it *addIt) prep() error {
+	var err error
+
+	fetch := func() error {
+		if !it.arun.Valid() && it.a.HasNext() {
+			it.arun, err = it.a.NextRun()
+			if err != nil {
+				return err
+			}
+		}
+
+		if !it.brun.Valid() && it.b.HasNext() {
+			it.brun, err = it.b.NextRun()
+			if err != nil {
+				return err
+			}
+		}
+		return nil
+	}
+
+	if err := fetch(); err != nil {
+		return err
+	}
+
+	// one is not valid
+	if !it.arun.Valid() {
+		it.next = it.brun
+		it.brun.Len = 0
+		return nil
+	}
+
+	if !it.brun.Valid() {
+		it.next = it.arun
+		it.arun.Len = 0
+		return nil
+	}
+
+	if !it.arun.Val && !it.brun.Val {
+		min := it.arun.Len
+		if it.brun.Len < min {
+			min = it.brun.Len
+		}
+		it.next = Run{Val: it.arun.Val, Len: min}
+		it.arun.Len -= it.next.Len
+		it.brun.Len -= it.next.Len
+		return nil
+	}
+
+	it.next = Run{Val: true}
+	// different vals, 'true' wins
+	for (it.arun.Val && it.arun.Valid()) || (it.brun.Val && it.brun.Valid()) {
+		min := it.arun.Len
+		if it.brun.Len < min && it.brun.Valid() || !it.arun.Valid() {
+			min = it.brun.Len
+		}
+		it.next.Len += min
+		if it.arun.Valid() {
+			it.arun.Len -= min
+		}
+		if it.brun.Valid() {
+			it.brun.Len -= min
+		}
+		if err := fetch(); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (it *addIt) HasNext() bool {
+	return it.next.Valid()
+}
+
+func (it *addIt) NextRun() (Run, error) {
+	next := it.next
+	return next, it.prep()
+}

lib/rlepluslazy/runs_test.go

@@ -0,0 +1,94 @@
+package rlepluslazy
+
+import (
+	"math/rand"
+	"sort"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestSumRuns(t *testing.T) {
+	{
+		a, err := RunsFromSlice([]uint64{0, 1, 2, 3, 4, 5, 6, 7, 8, 11, 12, 13, 14})
+		assert.NoError(t, err)
+		b, err := RunsFromSlice([]uint64{0, 1, 2, 3, 9, 10, 16, 17, 18, 50, 51, 70})
+		assert.NoError(t, err)
+
+		s, err := Sum(a, b)
+		assert.NoError(t, err)
+		bis, err := SliceFromRuns(s)
+		assert.Equal(t, []uint64{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 17, 18, 50, 51, 70}, bis)
+		assert.NoError(t, err)
+	}
+
+	{
+		a, err := RunsFromSlice([]uint64{0, 1, 2, 3, 4, 5, 6, 7, 8, 11, 12, 13, 14})
+		assert.NoError(t, err)
+		b, err := RunsFromSlice([]uint64{0, 1, 2, 3, 9, 10, 16, 17, 18, 50, 51, 70})
+		assert.NoError(t, err)
+
+		s, err := Sum(b, a)
+		assert.NoError(t, err)
+		bis, err := SliceFromRuns(s)
+		assert.Equal(t, []uint64{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 17, 18, 50, 51, 70}, bis)
+		assert.NoError(t, err)
+	}
+}
+
+func randomBits(N int, max uint64) []uint64 {
+	all := make(map[uint64]struct{})
+	for len(all) <= N {
+		x := rand.Uint64() % max
+		if _, has := all[x]; has {
+			continue
+		}
+		all[x] = struct{}{}
+	}
+
+	res := make([]uint64, 0, N)
+	for x := range all {
+		res = append(res, x)
+	}
+	sort.Slice(res, func(i, j int) bool { return res[i] < res[j] })
+
+	return res
+}
+
+func sum(a, b []uint64) []uint64 {
+	all := make(map[uint64]struct{})
+	for _, x := range a {
+		all[x] = struct{}{}
+	}
+	for _, x := range b {
+		all[x] = struct{}{}
+	}
+	res := make([]uint64, 0, len(all))
+	for x := range all {
+		res = append(res, x)
+	}
+	sort.Slice(res, func(i, j int) bool { return res[i] < res[j] })
+
+	return res
+}
+
+func TestSumRandom(t *testing.T) {
+	N := 100
+
+	for i := 0; i < N; i++ {
+		abits := randomBits(1000, 2000)
+		bbits := randomBits(1000, 2000)
+		sumbits := sum(abits, bbits)
+
+		a, err := RunsFromSlice(abits)
+		assert.NoError(t, err)
+		b, err := RunsFromSlice(bbits)
+		assert.NoError(t, err)
+
+		s, err := Sum(b, a)
+		assert.NoError(t, err)
+		bis, err := SliceFromRuns(s)
+		assert.NoError(t, err)
+		assert.Equal(t, sumbits, bis)
+	}
+}