Add rle encoder

License: MIT
Signed-off-by: Jakub Sztandera <kubuxu@protonmail.ch>
This commit is contained in:
Jakub Sztandera 2019-09-23 17:50:41 +02:00 committed by Jakub Sztandera
parent 3d6071ed3f
commit d3dfd8a73b
No known key found for this signature in database
GPG Key ID: 9A9AF56F8B3879BA
2 changed files with 92 additions and 4 deletions

View File

@ -18,14 +18,14 @@ func TestDecode(t *testing.T) {
expectedNumbers := []uint64{0, 2, 4, 5, 6, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27} expectedNumbers := []uint64{0, 2, 4, 5, 6, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27}
encoded, _, err := rleplus.Encode(expectedNumbers) runs, err := RunsFromBits(BitsFromSlice(expectedNumbers))
assert.NoError(t, err)
encoded, err := EncodeRuns(runs, []byte{})
assert.NoError(t, err) assert.NoError(t, err)
// Our encoded bytes are the same as the ref bytes // Our encoded bytes are the same as the ref bytes
assert.Equal(t, len(referenceEncoding), len(encoded)) assert.Equal(t, len(referenceEncoding), len(encoded))
for idx, expected := range referenceEncoding { assert.Equal(t, referenceEncoding, encoded)
assert.Equal(t, expected, encoded[idx])
}
rle, err := FromBuf(referenceEncoding) rle, err := FromBuf(referenceEncoding)
assert.NoError(t, err) assert.NoError(t, err)
@ -88,7 +88,19 @@ func TestGolden(t *testing.T) {
res = append(res, bit) res = append(res, bit)
} }
assert.Equal(t, expected, res) assert.Equal(t, expected, res)
}
func TestGoldenLoop(t *testing.T) {
rle, err := FromBuf(goldenRLE)
assert.NoError(t, err)
rit, err := rle.RunIterator()
assert.NoError(t, err)
buf, err := EncodeRuns(rit, nil)
assert.NoError(t, err)
assert.Equal(t, goldenRLE, buf)
} }
var Res uint64 = 0 var Res uint64 = 0
@ -151,3 +163,24 @@ func BenchmarkOldRLE(b *testing.B) {
} }
Res = Res + r Res = Res + r
} }
func BenchmarkDecodeEncode(b *testing.B) {
b.ReportAllocs()
var r uint64
out := make([]byte, 0, len(goldenRLE))
for i := 0; i < b.N; i++ {
rle, _ := FromBuf(goldenRLE)
rit, _ := rle.RunIterator()
out, _ = EncodeRuns(rit, out)
r = r + uint64(len(out))
}
/*
for i := 0; i < b.N; i++ {
rle, _ := rleplus.Decode(goldenRLE)
out, _, _ := rleplus.Encode(rle)
r = r + uint64(len(out))
}
*/
Res = Res + r
}

View File

@ -0,0 +1,55 @@
package rlepluslazy
import (
"encoding/binary"
bitvector "github.com/filecoin-project/go-lotus/lib/rlepluslazy/internal"
)
func EncodeRuns(rit RunIterator, buf []byte) ([]byte, error) {
v := bitvector.NewBitVector(buf[:0], bitvector.LSB0)
v.Extend(0, 2, bitvector.LSB0) // Version
first := true
varBuf := make([]byte, binary.MaxVarintLen64)
for rit.HasNext() {
run, err := rit.NextRun()
if err != nil {
return nil, err
}
if first {
if run.Val {
v.Push(1)
} else {
v.Push(0)
}
first = false
}
switch {
case run.Len == 1:
v.Push(1)
case run.Len < 16:
v.Push(0)
v.Push(1)
v.Extend(byte(run.Len), 4, bitvector.LSB0)
case run.Len >= 16:
v.Push(0)
v.Push(0)
numBytes := binary.PutUvarint(varBuf, run.Len)
for i := 0; i < numBytes; i++ {
v.Extend(varBuf[i], 8, bitvector.LSB0)
}
}
}
if first {
v.Push(0)
}
return v.Buf, nil
}