e5b3c4757d
- Add RunIterator and decoder from RLE - Add BitIterator and BitsFromRuns - Add BitsFromSlice - Add RunsFromBits License: MIT Signed-off-by: Jakub Sztandera <kubuxu@protonmail.ch>
119 lines
2.8 KiB
Go
119 lines
2.8 KiB
Go
package rlepluslazy
|
|
|
|
import (
|
|
"math/rand"
|
|
"runtime"
|
|
"testing"
|
|
|
|
"github.com/filecoin-project/go-lotus/extern/rleplus"
|
|
"github.com/stretchr/testify/assert"
|
|
)
|
|
|
|
func TestDecode(t *testing.T) {
|
|
// Encoding bitvec![LittleEndian; 1, 0, 1, 0, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
|
|
// in the Rust reference implementation gives an encoding of [223, 145, 136, 0] (without version field)
|
|
// The bit vector is equivalent to the integer set { 0, 2, 4, 5, 6, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27 }
|
|
|
|
// This is the above reference output with a version header "00" manually added
|
|
referenceEncoding := []byte{124, 71, 34, 2}
|
|
|
|
expectedNumbers := []uint64{0, 2, 4, 5, 6, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27}
|
|
|
|
encoded, _, err := rleplus.Encode(expectedNumbers)
|
|
assert.NoError(t, err)
|
|
|
|
// Our encoded bytes are the same as the ref bytes
|
|
assert.Equal(t, len(referenceEncoding), len(encoded))
|
|
for idx, expected := range referenceEncoding {
|
|
assert.Equal(t, expected, encoded[idx])
|
|
}
|
|
|
|
rle, err := FromBuf(referenceEncoding)
|
|
assert.NoError(t, err)
|
|
decoded := make([]uint64, 0, len(expectedNumbers))
|
|
|
|
rit, err := rle.RunIterator()
|
|
assert.NoError(t, err)
|
|
|
|
it, err := BitsFromRuns(rit)
|
|
assert.NoError(t, err)
|
|
for it.HasNext() {
|
|
bit, err := it.Next()
|
|
assert.NoError(t, err)
|
|
decoded = append(decoded, bit)
|
|
}
|
|
|
|
// Our decoded integers are the same as expected
|
|
assert.Equal(t, expectedNumbers, decoded)
|
|
}
|
|
|
|
func TestGolden(t *testing.T) {
|
|
t.SkipNow()
|
|
N := 1000
|
|
mod := uint32(1) << 16
|
|
runExProp := float32(0.9)
|
|
|
|
bits := make([]uint64, N)
|
|
|
|
for i := 0; i < N; i++ {
|
|
x := rand.Uint32() % mod
|
|
bits[i] = uint64(x)
|
|
for rand.Float32() < runExProp && i+1 < N {
|
|
i++
|
|
x = (x + 1) % mod
|
|
bits[i] = uint64(x)
|
|
}
|
|
}
|
|
|
|
out, _, err := rleplus.Encode(bits)
|
|
assert.NoError(t, err)
|
|
t.Logf("%#v", out)
|
|
_, runs := rleplus.RunLengths(bits)
|
|
t.Logf("runs: %v", runs)
|
|
}
|
|
|
|
func BenchmarkIterator(b *testing.B) {
|
|
decoded := make([]uint64, 0, 1000)
|
|
|
|
for i := 0; i < b.N; i++ {
|
|
decoded = decoded[:0]
|
|
rle, _ := FromBuf(goldenRLE)
|
|
it, _ := rle.Iterator()
|
|
for it.HasNext() {
|
|
bit, _ := it.Next()
|
|
decoded = append(decoded, bit)
|
|
}
|
|
runtime.KeepAlive(decoded)
|
|
}
|
|
}
|
|
|
|
func BenchmarkRunIterator(b *testing.B) {
|
|
runs := make([]Run, 0, 1000)
|
|
for i := 0; i < b.N; i++ {
|
|
runs = runs[:0]
|
|
rle, _ := FromBuf(goldenRLE)
|
|
rit, _ := rle.RunIterator()
|
|
for rit.HasNext() {
|
|
run, _ := rit.NextRun()
|
|
runs = append(runs, run)
|
|
}
|
|
runtime.KeepAlive(runs)
|
|
}
|
|
}
|
|
|
|
func BenchmarkRunsToBits(b *testing.B) {
|
|
decoded := make([]uint64, 0, 1000)
|
|
|
|
for i := 0; i < b.N; i++ {
|
|
decoded = decoded[:0]
|
|
rle, _ := FromBuf(goldenRLE)
|
|
rit, _ := rle.RunIterator()
|
|
it, _ := BitsFromRuns(rit)
|
|
for it.HasNext() {
|
|
bit, _ := it.Next()
|
|
decoded = append(decoded, bit)
|
|
}
|
|
runtime.KeepAlive(decoded)
|
|
}
|
|
}
|