influxdb/tsdb/tsm1/batch_float_test.go

439 lines
9.2 KiB
Go
Raw Normal View History

2018-09-26 17:39:21 +00:00
package tsm1_test
import (
"bytes"
"fmt"
"math"
"math/rand"
"reflect"
"testing"
"testing/quick"
"github.com/google/go-cmp/cmp"
"github.com/influxdata/influxdb/tsdb/tsm1"
2018-09-26 17:39:21 +00:00
)
2018-09-21 01:18:22 +00:00
var fullBlockFloat64Ones []float64
func init() {
for i := 0; i < 1000; i++ {
fullBlockFloat64Ones = append(fullBlockFloat64Ones, 1.0)
}
}
Batch oriented float encoders This commit adds a tsm1 function for encoding a batch of floats into a buffer. Further, it replaces the `bitstream` library used in the existing encoders (and all the current decoders) with inlined bit expressions within the encoder, significantly reducing the function call overhead for larger batches. The following benchmarks compare the performance of the existing iterator based encoders, and the new batch oriented encoders. They look at a sequential input slice and a randomly generated input slice. name old time/op new time/op delta EncodeFloats/10_seq 1.14µs ± 3% 0.24µs ± 3% -78.94% (p=0.000 n=10+10) EncodeFloats/10_ran 1.69µs ± 2% 0.21µs ± 3% -87.43% (p=0.000 n=10+10) EncodeFloats/100_seq 7.07µs ± 1% 1.72µs ± 1% -75.62% (p=0.000 n=7+9) EncodeFloats/100_ran 15.8µs ± 4% 1.8µs ± 1% -88.60% (p=0.000 n=10+9) EncodeFloats/1000_seq 50.2µs ± 3% 16.2µs ± 2% -67.66% (p=0.000 n=10+10) EncodeFloats/1000_ran 174µs ± 2% 16µs ± 2% -90.77% (p=0.000 n=10+10) name old alloc/op new alloc/op delta EncodeFloats/10_seq 0.00B 0.00B ~ (all equal) EncodeFloats/10_ran 0.00B 0.00B ~ (all equal) EncodeFloats/100_seq 0.00B 0.00B ~ (all equal) EncodeFloats/100_ran 0.00B 0.00B ~ (all equal) EncodeFloats/1000_seq 0.00B 0.00B ~ (all equal) EncodeFloats/1000_ran 0.00B 0.00B ~ (all equal) name old allocs/op new allocs/op delta EncodeFloats/10_seq 0.00 0.00 ~ (all equal) EncodeFloats/10_ran 0.00 0.00 ~ (all equal) EncodeFloats/100_seq 0.00 0.00 ~ (all equal) EncodeFloats/100_ran 0.00 0.00 ~ (all equal) EncodeFloats/1000_seq 0.00 0.00 ~ (all equal) EncodeFloats/1000_ran 0.00 0.00 ~ (all equal)
2018-09-11 12:04:52 +00:00
func TestFloatArrayEncodeAll(t *testing.T) {
examples := [][]float64{
{12, 12, 24, 13, 24, 24, 24, 24}, // From example paper.
{-3.8970913068231994e+307, -9.036931257783943e+307, 1.7173073833490201e+308,
-9.312369166661538e+307, -2.2435523083555231e+307, 1.4779121287289644e+307,
1.771273431601434e+308, 8.140360378221364e+307, 4.783405048208089e+307,
-2.8044680049605344e+307, 4.412915337205696e+307, -1.2779380602005046e+308,
1.6235802318921885e+308, -1.3402901846299688e+307, 1.6961015582104055e+308,
-1.067980796435633e+308, -3.02868987458268e+307, 1.7641793640790284e+308,
1.6587191845856813e+307, -1.786073304985983e+308, 1.0694549382051123e+308,
3.5635180996210295e+307}, // Failed during early development
{6.00065e+06, 6.000656e+06, 6.000657e+06, 6.000659e+06, 6.000661e+06}, // Similar values.
twoHoursData,
2018-09-21 01:18:22 +00:00
fullBlockFloat64Ones,
Batch oriented float encoders This commit adds a tsm1 function for encoding a batch of floats into a buffer. Further, it replaces the `bitstream` library used in the existing encoders (and all the current decoders) with inlined bit expressions within the encoder, significantly reducing the function call overhead for larger batches. The following benchmarks compare the performance of the existing iterator based encoders, and the new batch oriented encoders. They look at a sequential input slice and a randomly generated input slice. name old time/op new time/op delta EncodeFloats/10_seq 1.14µs ± 3% 0.24µs ± 3% -78.94% (p=0.000 n=10+10) EncodeFloats/10_ran 1.69µs ± 2% 0.21µs ± 3% -87.43% (p=0.000 n=10+10) EncodeFloats/100_seq 7.07µs ± 1% 1.72µs ± 1% -75.62% (p=0.000 n=7+9) EncodeFloats/100_ran 15.8µs ± 4% 1.8µs ± 1% -88.60% (p=0.000 n=10+9) EncodeFloats/1000_seq 50.2µs ± 3% 16.2µs ± 2% -67.66% (p=0.000 n=10+10) EncodeFloats/1000_ran 174µs ± 2% 16µs ± 2% -90.77% (p=0.000 n=10+10) name old alloc/op new alloc/op delta EncodeFloats/10_seq 0.00B 0.00B ~ (all equal) EncodeFloats/10_ran 0.00B 0.00B ~ (all equal) EncodeFloats/100_seq 0.00B 0.00B ~ (all equal) EncodeFloats/100_ran 0.00B 0.00B ~ (all equal) EncodeFloats/1000_seq 0.00B 0.00B ~ (all equal) EncodeFloats/1000_ran 0.00B 0.00B ~ (all equal) name old allocs/op new allocs/op delta EncodeFloats/10_seq 0.00 0.00 ~ (all equal) EncodeFloats/10_ran 0.00 0.00 ~ (all equal) EncodeFloats/100_seq 0.00 0.00 ~ (all equal) EncodeFloats/100_ran 0.00 0.00 ~ (all equal) EncodeFloats/1000_seq 0.00 0.00 ~ (all equal) EncodeFloats/1000_ran 0.00 0.00 ~ (all equal)
2018-09-11 12:04:52 +00:00
{},
}
for _, example := range examples {
src := example
var buf []byte
buf, err := tsm1.FloatArrayEncodeAll(src, buf)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
result, err := tsm1.FloatArrayDecodeAll(buf, nil)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if got, exp := result, src; !reflect.DeepEqual(got, exp) {
t.Fatalf("got result %v, expected %v", got, exp)
}
}
}
func TestFloatArrayEncode_Compare(t *testing.T) {
// generate random values
input := make([]float64, 1000)
for i := 0; i < len(input); i++ {
input[i] = (rand.Float64() * math.MaxFloat64) - math.MaxFloat32
}
s := tsm1.NewFloatEncoder()
for _, v := range input {
s.Write(v)
}
s.Flush()
buf1, err := s.Bytes()
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
var buf2 []byte
buf2, err = tsm1.FloatArrayEncodeAll(input, buf2)
if err != nil {
t.Fatalf("unexpected error: %v\nbuf: %db %x", err, len(buf2), buf2)
}
result, err := tsm1.FloatArrayDecodeAll(buf2, nil)
if err != nil {
dumpBufs(buf1, buf2)
t.Fatalf("unexpected error: %v\nbuf: %db %x", err, len(buf2), buf2)
}
if got, exp := result, input; !reflect.DeepEqual(got, exp) {
t.Fatalf("got result %v, expected %v", got, exp)
}
// Check that the encoders are byte for byte the same...
if !bytes.Equal(buf1, buf2) {
dumpBufs(buf1, buf2)
t.Fatalf("Raw bytes differ for encoders")
}
}
func dumpBufs(a, b []byte) {
longest := len(a)
if len(b) > longest {
longest = len(b)
}
for i := 0; i < longest; i++ {
var as, bs string
if i < len(a) {
as = fmt.Sprintf("%08b", a[i])
}
if i < len(b) {
bs = fmt.Sprintf("%08b", b[i])
}
same := as == bs
fmt.Printf("%d (%d) %s - %s :: %v\n", i, i*8, as, bs, same)
}
fmt.Println()
}
func TestFloatArrayEncodeAll_NaN(t *testing.T) {
examples := [][]float64{
{1.0, math.NaN(), 2.0},
{1.22, math.NaN()},
{math.NaN(), math.NaN()},
{math.NaN()},
}
for _, example := range examples {
var buf []byte
_, err := tsm1.FloatArrayEncodeAll(example, buf)
if err == nil {
t.Fatalf("expected error. got nil")
}
}
}
func Test_FloatArrayEncodeAll_Quick(t *testing.T) {
quick.Check(func(values []float64) bool {
src := values
if src == nil {
src = []float64{}
}
for i, v := range src {
if math.IsNaN(v) {
src[i] = 1.0 // Remove invalid values
}
}
s := tsm1.NewFloatEncoder()
for _, p := range src {
s.Write(p)
}
s.Flush()
buf1, err := s.Bytes()
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
var buf2 []byte
buf2, err = tsm1.FloatArrayEncodeAll(src, buf2)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
result, err := tsm1.FloatArrayDecodeAll(buf2, nil)
if err != nil {
dumpBufs(buf1, buf2)
fmt.Println(src)
t.Fatalf("unexpected error: %v", err)
}
if got, exp := result, src[:]; !reflect.DeepEqual(got, exp) {
Batch oriented float encoders This commit adds a tsm1 function for encoding a batch of floats into a buffer. Further, it replaces the `bitstream` library used in the existing encoders (and all the current decoders) with inlined bit expressions within the encoder, significantly reducing the function call overhead for larger batches. The following benchmarks compare the performance of the existing iterator based encoders, and the new batch oriented encoders. They look at a sequential input slice and a randomly generated input slice. name old time/op new time/op delta EncodeFloats/10_seq 1.14µs ± 3% 0.24µs ± 3% -78.94% (p=0.000 n=10+10) EncodeFloats/10_ran 1.69µs ± 2% 0.21µs ± 3% -87.43% (p=0.000 n=10+10) EncodeFloats/100_seq 7.07µs ± 1% 1.72µs ± 1% -75.62% (p=0.000 n=7+9) EncodeFloats/100_ran 15.8µs ± 4% 1.8µs ± 1% -88.60% (p=0.000 n=10+9) EncodeFloats/1000_seq 50.2µs ± 3% 16.2µs ± 2% -67.66% (p=0.000 n=10+10) EncodeFloats/1000_ran 174µs ± 2% 16µs ± 2% -90.77% (p=0.000 n=10+10) name old alloc/op new alloc/op delta EncodeFloats/10_seq 0.00B 0.00B ~ (all equal) EncodeFloats/10_ran 0.00B 0.00B ~ (all equal) EncodeFloats/100_seq 0.00B 0.00B ~ (all equal) EncodeFloats/100_ran 0.00B 0.00B ~ (all equal) EncodeFloats/1000_seq 0.00B 0.00B ~ (all equal) EncodeFloats/1000_ran 0.00B 0.00B ~ (all equal) name old allocs/op new allocs/op delta EncodeFloats/10_seq 0.00 0.00 ~ (all equal) EncodeFloats/10_ran 0.00 0.00 ~ (all equal) EncodeFloats/100_seq 0.00 0.00 ~ (all equal) EncodeFloats/100_ran 0.00 0.00 ~ (all equal) EncodeFloats/1000_seq 0.00 0.00 ~ (all equal) EncodeFloats/1000_ran 0.00 0.00 ~ (all equal)
2018-09-11 12:04:52 +00:00
t.Fatalf("got result %v, expected %v", got, exp)
}
return true
}, nil)
}
func TestDecodeFloatArrayAll_Empty(t *testing.T) {
s := tsm1.NewFloatEncoder()
s.Flush()
b, err := s.Bytes()
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
var got []float64
if _, err := tsm1.FloatArrayDecodeAll(b, got); err != nil {
t.Fatal(err)
}
}
2018-09-05 10:13:11 +00:00
func TestFloatArrayDecodeAll_Simple(t *testing.T) {
2018-09-26 17:39:21 +00:00
// Example from the paper
s := tsm1.NewFloatEncoder()
exp := []float64{
12,
12,
24,
// extra tests
// floating point masking/shifting bug
13,
24,
// delta-of-delta sizes
24,
24,
24,
}
for _, f := range exp {
s.Write(f)
}
s.Flush()
b, err := s.Bytes()
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
buf := make([]float64, 8)
2018-09-05 10:13:11 +00:00
got, err := tsm1.FloatArrayDecodeAll(b, buf)
2018-09-26 17:39:21 +00:00
if err != nil {
t.Fatalf("unexpected decode error %q", err)
}
if !cmp.Equal(got, exp) {
t.Fatalf("unexpected values -got/+exp\n%s", cmp.Diff(got, exp))
}
}
Batch oriented float encoders This commit adds a tsm1 function for encoding a batch of floats into a buffer. Further, it replaces the `bitstream` library used in the existing encoders (and all the current decoders) with inlined bit expressions within the encoder, significantly reducing the function call overhead for larger batches. The following benchmarks compare the performance of the existing iterator based encoders, and the new batch oriented encoders. They look at a sequential input slice and a randomly generated input slice. name old time/op new time/op delta EncodeFloats/10_seq 1.14µs ± 3% 0.24µs ± 3% -78.94% (p=0.000 n=10+10) EncodeFloats/10_ran 1.69µs ± 2% 0.21µs ± 3% -87.43% (p=0.000 n=10+10) EncodeFloats/100_seq 7.07µs ± 1% 1.72µs ± 1% -75.62% (p=0.000 n=7+9) EncodeFloats/100_ran 15.8µs ± 4% 1.8µs ± 1% -88.60% (p=0.000 n=10+9) EncodeFloats/1000_seq 50.2µs ± 3% 16.2µs ± 2% -67.66% (p=0.000 n=10+10) EncodeFloats/1000_ran 174µs ± 2% 16µs ± 2% -90.77% (p=0.000 n=10+10) name old alloc/op new alloc/op delta EncodeFloats/10_seq 0.00B 0.00B ~ (all equal) EncodeFloats/10_ran 0.00B 0.00B ~ (all equal) EncodeFloats/100_seq 0.00B 0.00B ~ (all equal) EncodeFloats/100_ran 0.00B 0.00B ~ (all equal) EncodeFloats/1000_seq 0.00B 0.00B ~ (all equal) EncodeFloats/1000_ran 0.00B 0.00B ~ (all equal) name old allocs/op new allocs/op delta EncodeFloats/10_seq 0.00 0.00 ~ (all equal) EncodeFloats/10_ran 0.00 0.00 ~ (all equal) EncodeFloats/100_seq 0.00 0.00 ~ (all equal) EncodeFloats/100_ran 0.00 0.00 ~ (all equal) EncodeFloats/1000_seq 0.00 0.00 ~ (all equal) EncodeFloats/1000_ran 0.00 0.00 ~ (all equal)
2018-09-11 12:04:52 +00:00
func TestFloatArrayDecodeAll_Empty(t *testing.T) {
2018-09-28 12:47:08 +00:00
s := tsm1.NewFloatEncoder()
s.Flush()
b, err := s.Bytes()
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
buf := make([]float64, 8)
Batch oriented float encoders This commit adds a tsm1 function for encoding a batch of floats into a buffer. Further, it replaces the `bitstream` library used in the existing encoders (and all the current decoders) with inlined bit expressions within the encoder, significantly reducing the function call overhead for larger batches. The following benchmarks compare the performance of the existing iterator based encoders, and the new batch oriented encoders. They look at a sequential input slice and a randomly generated input slice. name old time/op new time/op delta EncodeFloats/10_seq 1.14µs ± 3% 0.24µs ± 3% -78.94% (p=0.000 n=10+10) EncodeFloats/10_ran 1.69µs ± 2% 0.21µs ± 3% -87.43% (p=0.000 n=10+10) EncodeFloats/100_seq 7.07µs ± 1% 1.72µs ± 1% -75.62% (p=0.000 n=7+9) EncodeFloats/100_ran 15.8µs ± 4% 1.8µs ± 1% -88.60% (p=0.000 n=10+9) EncodeFloats/1000_seq 50.2µs ± 3% 16.2µs ± 2% -67.66% (p=0.000 n=10+10) EncodeFloats/1000_ran 174µs ± 2% 16µs ± 2% -90.77% (p=0.000 n=10+10) name old alloc/op new alloc/op delta EncodeFloats/10_seq 0.00B 0.00B ~ (all equal) EncodeFloats/10_ran 0.00B 0.00B ~ (all equal) EncodeFloats/100_seq 0.00B 0.00B ~ (all equal) EncodeFloats/100_ran 0.00B 0.00B ~ (all equal) EncodeFloats/1000_seq 0.00B 0.00B ~ (all equal) EncodeFloats/1000_ran 0.00B 0.00B ~ (all equal) name old allocs/op new allocs/op delta EncodeFloats/10_seq 0.00 0.00 ~ (all equal) EncodeFloats/10_ran 0.00 0.00 ~ (all equal) EncodeFloats/100_seq 0.00 0.00 ~ (all equal) EncodeFloats/100_ran 0.00 0.00 ~ (all equal) EncodeFloats/1000_seq 0.00 0.00 ~ (all equal) EncodeFloats/1000_ran 0.00 0.00 ~ (all equal)
2018-09-11 12:04:52 +00:00
got, err := tsm1.FloatArrayDecodeAll(b, buf)
2018-09-28 12:47:08 +00:00
if err != nil {
t.Fatalf("unexpected decode error %q", err)
}
if exp := []float64{}; !cmp.Equal(got, exp) {
t.Fatalf("unexpected values -got/+exp\n%s", cmp.Diff(got, exp))
}
}
Batch oriented float encoders This commit adds a tsm1 function for encoding a batch of floats into a buffer. Further, it replaces the `bitstream` library used in the existing encoders (and all the current decoders) with inlined bit expressions within the encoder, significantly reducing the function call overhead for larger batches. The following benchmarks compare the performance of the existing iterator based encoders, and the new batch oriented encoders. They look at a sequential input slice and a randomly generated input slice. name old time/op new time/op delta EncodeFloats/10_seq 1.14µs ± 3% 0.24µs ± 3% -78.94% (p=0.000 n=10+10) EncodeFloats/10_ran 1.69µs ± 2% 0.21µs ± 3% -87.43% (p=0.000 n=10+10) EncodeFloats/100_seq 7.07µs ± 1% 1.72µs ± 1% -75.62% (p=0.000 n=7+9) EncodeFloats/100_ran 15.8µs ± 4% 1.8µs ± 1% -88.60% (p=0.000 n=10+9) EncodeFloats/1000_seq 50.2µs ± 3% 16.2µs ± 2% -67.66% (p=0.000 n=10+10) EncodeFloats/1000_ran 174µs ± 2% 16µs ± 2% -90.77% (p=0.000 n=10+10) name old alloc/op new alloc/op delta EncodeFloats/10_seq 0.00B 0.00B ~ (all equal) EncodeFloats/10_ran 0.00B 0.00B ~ (all equal) EncodeFloats/100_seq 0.00B 0.00B ~ (all equal) EncodeFloats/100_ran 0.00B 0.00B ~ (all equal) EncodeFloats/1000_seq 0.00B 0.00B ~ (all equal) EncodeFloats/1000_ran 0.00B 0.00B ~ (all equal) name old allocs/op new allocs/op delta EncodeFloats/10_seq 0.00 0.00 ~ (all equal) EncodeFloats/10_ran 0.00 0.00 ~ (all equal) EncodeFloats/100_seq 0.00 0.00 ~ (all equal) EncodeFloats/100_ran 0.00 0.00 ~ (all equal) EncodeFloats/1000_seq 0.00 0.00 ~ (all equal) EncodeFloats/1000_ran 0.00 0.00 ~ (all equal)
2018-09-11 12:04:52 +00:00
var bufResult []byte
func BenchmarkEncodeFloats(b *testing.B) {
var err error
cases := []int{10, 100, 1000}
enc := tsm1.NewFloatEncoder()
for _, n := range cases {
b.Run(fmt.Sprintf("%d_seq", n), func(b *testing.B) {
input := make([]float64, n)
for i := 0; i < n; i++ {
input[i] = float64(i)
}
b.Run("itr", func(b *testing.B) {
b.ReportAllocs()
Batch oriented int encoders This commit adds a tsm1 function for encoding a batch of ints into a provided buffer. The following benchmarks compare the performance of the existing iterator based encoders, and the new batch oriented encoders. They look at a sequential input slice, a randomly generated input slice and a duplicate slice: name old time/op new time/op delta EncodeIntegers/10_seq 144ns ± 2% 41ns ± 1% -71.46% (p=0.000 n=10+10) EncodeIntegers/10_ran 304ns ± 7% 140ns ± 2% -53.99% (p=0.000 n=10+10) EncodeIntegers/10_dup 147ns ± 4% 41ns ± 2% -72.14% (p=0.000 n=10+9) EncodeIntegers/100_seq 483ns ± 7% 208ns ± 1% -56.98% (p=0.000 n=10+9) EncodeIntegers/100_ran 1.64µs ± 7% 1.01µs ± 1% -38.42% (p=0.000 n=9+9) EncodeIntegers/100_dup 484ns ±14% 210ns ± 2% -56.63% (p=0.000 n=10+10) EncodeIntegers/1000_seq 3.11µs ± 2% 1.81µs ± 2% -41.68% (p=0.000 n=10+10) EncodeIntegers/1000_ran 16.9µs ±10% 11.0µs ± 2% -34.58% (p=0.000 n=10+10) EncodeIntegers/1000_dup 3.05µs ± 3% 1.81µs ± 2% -40.71% (p=0.000 n=10+8) name old alloc/op new alloc/op delta EncodeIntegers/10_seq 32.0B ± 0% 0.0B -100.00% (p=0.000 n=10+10) EncodeIntegers/10_ran 32.0B ± 0% 0.0B -100.00% (p=0.000 n=10+10) EncodeIntegers/10_dup 32.0B ± 0% 0.0B -100.00% (p=0.000 n=10+10) EncodeIntegers/100_seq 32.0B ± 0% 0.0B -100.00% (p=0.000 n=10+10) EncodeIntegers/100_ran 128B ± 0% 0B -100.00% (p=0.000 n=10+10) EncodeIntegers/100_dup 32.0B ± 0% 0.0B -100.00% (p=0.000 n=10+10) EncodeIntegers/1000_seq 32.0B ± 0% 0.0B -100.00% (p=0.000 n=10+10) EncodeIntegers/1000_ran 1.15kB ± 0% 0.00kB -100.00% (p=0.000 n=10+10) EncodeIntegers/1000_dup 32.0B ± 0% 0.0B -100.00% (p=0.000 n=10+10) name old allocs/op new allocs/op delta EncodeIntegers/10_seq 1.00 ± 0% 0.00 -100.00% (p=0.000 n=10+10) EncodeIntegers/10_ran 1.00 ± 0% 0.00 -100.00% (p=0.000 n=10+10) EncodeIntegers/10_dup 1.00 ± 0% 0.00 -100.00% (p=0.000 n=10+10) EncodeIntegers/100_seq 1.00 ± 0% 0.00 -100.00% (p=0.000 n=10+10) EncodeIntegers/100_ran 1.00 ± 0% 0.00 -100.00% (p=0.000 n=10+10) EncodeIntegers/100_dup 1.00 ± 0% 0.00 -100.00% (p=0.000 n=10+10) EncodeIntegers/1000_seq 1.00 ± 0% 0.00 -100.00% (p=0.000 n=10+10) EncodeIntegers/1000_ran 1.00 ± 0% 0.00 -100.00% (p=0.000 n=10+10) EncodeIntegers/1000_dup 1.00 ± 0% 0.00 -100.00% (p=0.000 n=10+10)
2018-09-11 17:04:06 +00:00
enc.Reset()
Batch oriented float encoders This commit adds a tsm1 function for encoding a batch of floats into a buffer. Further, it replaces the `bitstream` library used in the existing encoders (and all the current decoders) with inlined bit expressions within the encoder, significantly reducing the function call overhead for larger batches. The following benchmarks compare the performance of the existing iterator based encoders, and the new batch oriented encoders. They look at a sequential input slice and a randomly generated input slice. name old time/op new time/op delta EncodeFloats/10_seq 1.14µs ± 3% 0.24µs ± 3% -78.94% (p=0.000 n=10+10) EncodeFloats/10_ran 1.69µs ± 2% 0.21µs ± 3% -87.43% (p=0.000 n=10+10) EncodeFloats/100_seq 7.07µs ± 1% 1.72µs ± 1% -75.62% (p=0.000 n=7+9) EncodeFloats/100_ran 15.8µs ± 4% 1.8µs ± 1% -88.60% (p=0.000 n=10+9) EncodeFloats/1000_seq 50.2µs ± 3% 16.2µs ± 2% -67.66% (p=0.000 n=10+10) EncodeFloats/1000_ran 174µs ± 2% 16µs ± 2% -90.77% (p=0.000 n=10+10) name old alloc/op new alloc/op delta EncodeFloats/10_seq 0.00B 0.00B ~ (all equal) EncodeFloats/10_ran 0.00B 0.00B ~ (all equal) EncodeFloats/100_seq 0.00B 0.00B ~ (all equal) EncodeFloats/100_ran 0.00B 0.00B ~ (all equal) EncodeFloats/1000_seq 0.00B 0.00B ~ (all equal) EncodeFloats/1000_ran 0.00B 0.00B ~ (all equal) name old allocs/op new allocs/op delta EncodeFloats/10_seq 0.00 0.00 ~ (all equal) EncodeFloats/10_ran 0.00 0.00 ~ (all equal) EncodeFloats/100_seq 0.00 0.00 ~ (all equal) EncodeFloats/100_ran 0.00 0.00 ~ (all equal) EncodeFloats/1000_seq 0.00 0.00 ~ (all equal) EncodeFloats/1000_ran 0.00 0.00 ~ (all equal)
2018-09-11 12:04:52 +00:00
b.ResetTimer()
for n := 0; n < b.N; n++ {
enc.Reset()
for _, x := range input {
enc.Write(x)
}
enc.Flush()
if bufResult, err = enc.Bytes(); err != nil {
b.Fatal(err)
} else {
b.SetBytes(int64(len(bufResult)))
Batch oriented float encoders This commit adds a tsm1 function for encoding a batch of floats into a buffer. Further, it replaces the `bitstream` library used in the existing encoders (and all the current decoders) with inlined bit expressions within the encoder, significantly reducing the function call overhead for larger batches. The following benchmarks compare the performance of the existing iterator based encoders, and the new batch oriented encoders. They look at a sequential input slice and a randomly generated input slice. name old time/op new time/op delta EncodeFloats/10_seq 1.14µs ± 3% 0.24µs ± 3% -78.94% (p=0.000 n=10+10) EncodeFloats/10_ran 1.69µs ± 2% 0.21µs ± 3% -87.43% (p=0.000 n=10+10) EncodeFloats/100_seq 7.07µs ± 1% 1.72µs ± 1% -75.62% (p=0.000 n=7+9) EncodeFloats/100_ran 15.8µs ± 4% 1.8µs ± 1% -88.60% (p=0.000 n=10+9) EncodeFloats/1000_seq 50.2µs ± 3% 16.2µs ± 2% -67.66% (p=0.000 n=10+10) EncodeFloats/1000_ran 174µs ± 2% 16µs ± 2% -90.77% (p=0.000 n=10+10) name old alloc/op new alloc/op delta EncodeFloats/10_seq 0.00B 0.00B ~ (all equal) EncodeFloats/10_ran 0.00B 0.00B ~ (all equal) EncodeFloats/100_seq 0.00B 0.00B ~ (all equal) EncodeFloats/100_ran 0.00B 0.00B ~ (all equal) EncodeFloats/1000_seq 0.00B 0.00B ~ (all equal) EncodeFloats/1000_ran 0.00B 0.00B ~ (all equal) name old allocs/op new allocs/op delta EncodeFloats/10_seq 0.00 0.00 ~ (all equal) EncodeFloats/10_ran 0.00 0.00 ~ (all equal) EncodeFloats/100_seq 0.00 0.00 ~ (all equal) EncodeFloats/100_ran 0.00 0.00 ~ (all equal) EncodeFloats/1000_seq 0.00 0.00 ~ (all equal) EncodeFloats/1000_ran 0.00 0.00 ~ (all equal)
2018-09-11 12:04:52 +00:00
}
}
})
b.Run("batch", func(b *testing.B) {
b.ReportAllocs()
b.ResetTimer()
for n := 0; n < b.N; n++ {
if bufResult, err = tsm1.FloatArrayEncodeAll(input, bufResult); err != nil {
b.Fatal(err)
} else {
b.SetBytes(int64(len(bufResult)))
Batch oriented float encoders This commit adds a tsm1 function for encoding a batch of floats into a buffer. Further, it replaces the `bitstream` library used in the existing encoders (and all the current decoders) with inlined bit expressions within the encoder, significantly reducing the function call overhead for larger batches. The following benchmarks compare the performance of the existing iterator based encoders, and the new batch oriented encoders. They look at a sequential input slice and a randomly generated input slice. name old time/op new time/op delta EncodeFloats/10_seq 1.14µs ± 3% 0.24µs ± 3% -78.94% (p=0.000 n=10+10) EncodeFloats/10_ran 1.69µs ± 2% 0.21µs ± 3% -87.43% (p=0.000 n=10+10) EncodeFloats/100_seq 7.07µs ± 1% 1.72µs ± 1% -75.62% (p=0.000 n=7+9) EncodeFloats/100_ran 15.8µs ± 4% 1.8µs ± 1% -88.60% (p=0.000 n=10+9) EncodeFloats/1000_seq 50.2µs ± 3% 16.2µs ± 2% -67.66% (p=0.000 n=10+10) EncodeFloats/1000_ran 174µs ± 2% 16µs ± 2% -90.77% (p=0.000 n=10+10) name old alloc/op new alloc/op delta EncodeFloats/10_seq 0.00B 0.00B ~ (all equal) EncodeFloats/10_ran 0.00B 0.00B ~ (all equal) EncodeFloats/100_seq 0.00B 0.00B ~ (all equal) EncodeFloats/100_ran 0.00B 0.00B ~ (all equal) EncodeFloats/1000_seq 0.00B 0.00B ~ (all equal) EncodeFloats/1000_ran 0.00B 0.00B ~ (all equal) name old allocs/op new allocs/op delta EncodeFloats/10_seq 0.00 0.00 ~ (all equal) EncodeFloats/10_ran 0.00 0.00 ~ (all equal) EncodeFloats/100_seq 0.00 0.00 ~ (all equal) EncodeFloats/100_ran 0.00 0.00 ~ (all equal) EncodeFloats/1000_seq 0.00 0.00 ~ (all equal) EncodeFloats/1000_ran 0.00 0.00 ~ (all equal)
2018-09-11 12:04:52 +00:00
}
}
})
})
b.Run(fmt.Sprintf("%d_ran", n), func(b *testing.B) {
input := make([]float64, n)
for i := 0; i < n; i++ {
input[i] = rand.Float64() * 100.0
}
b.Run("itr", func(b *testing.B) {
b.ReportAllocs()
b.ResetTimer()
for n := 0; n < b.N; n++ {
enc.Reset()
for _, x := range input {
enc.Write(x)
}
enc.Flush()
if bufResult, err = enc.Bytes(); err != nil {
b.Fatal(err)
} else {
b.SetBytes(int64(len(bufResult)))
Batch oriented float encoders This commit adds a tsm1 function for encoding a batch of floats into a buffer. Further, it replaces the `bitstream` library used in the existing encoders (and all the current decoders) with inlined bit expressions within the encoder, significantly reducing the function call overhead for larger batches. The following benchmarks compare the performance of the existing iterator based encoders, and the new batch oriented encoders. They look at a sequential input slice and a randomly generated input slice. name old time/op new time/op delta EncodeFloats/10_seq 1.14µs ± 3% 0.24µs ± 3% -78.94% (p=0.000 n=10+10) EncodeFloats/10_ran 1.69µs ± 2% 0.21µs ± 3% -87.43% (p=0.000 n=10+10) EncodeFloats/100_seq 7.07µs ± 1% 1.72µs ± 1% -75.62% (p=0.000 n=7+9) EncodeFloats/100_ran 15.8µs ± 4% 1.8µs ± 1% -88.60% (p=0.000 n=10+9) EncodeFloats/1000_seq 50.2µs ± 3% 16.2µs ± 2% -67.66% (p=0.000 n=10+10) EncodeFloats/1000_ran 174µs ± 2% 16µs ± 2% -90.77% (p=0.000 n=10+10) name old alloc/op new alloc/op delta EncodeFloats/10_seq 0.00B 0.00B ~ (all equal) EncodeFloats/10_ran 0.00B 0.00B ~ (all equal) EncodeFloats/100_seq 0.00B 0.00B ~ (all equal) EncodeFloats/100_ran 0.00B 0.00B ~ (all equal) EncodeFloats/1000_seq 0.00B 0.00B ~ (all equal) EncodeFloats/1000_ran 0.00B 0.00B ~ (all equal) name old allocs/op new allocs/op delta EncodeFloats/10_seq 0.00 0.00 ~ (all equal) EncodeFloats/10_ran 0.00 0.00 ~ (all equal) EncodeFloats/100_seq 0.00 0.00 ~ (all equal) EncodeFloats/100_ran 0.00 0.00 ~ (all equal) EncodeFloats/1000_seq 0.00 0.00 ~ (all equal) EncodeFloats/1000_ran 0.00 0.00 ~ (all equal)
2018-09-11 12:04:52 +00:00
}
}
})
b.Run("batch", func(b *testing.B) {
b.ReportAllocs()
b.ResetTimer()
for n := 0; n < b.N; n++ {
if bufResult, err = tsm1.FloatArrayEncodeAll(input, bufResult); err != nil {
b.Fatal(err)
} else {
b.SetBytes(int64(len(bufResult)))
Batch oriented float encoders This commit adds a tsm1 function for encoding a batch of floats into a buffer. Further, it replaces the `bitstream` library used in the existing encoders (and all the current decoders) with inlined bit expressions within the encoder, significantly reducing the function call overhead for larger batches. The following benchmarks compare the performance of the existing iterator based encoders, and the new batch oriented encoders. They look at a sequential input slice and a randomly generated input slice. name old time/op new time/op delta EncodeFloats/10_seq 1.14µs ± 3% 0.24µs ± 3% -78.94% (p=0.000 n=10+10) EncodeFloats/10_ran 1.69µs ± 2% 0.21µs ± 3% -87.43% (p=0.000 n=10+10) EncodeFloats/100_seq 7.07µs ± 1% 1.72µs ± 1% -75.62% (p=0.000 n=7+9) EncodeFloats/100_ran 15.8µs ± 4% 1.8µs ± 1% -88.60% (p=0.000 n=10+9) EncodeFloats/1000_seq 50.2µs ± 3% 16.2µs ± 2% -67.66% (p=0.000 n=10+10) EncodeFloats/1000_ran 174µs ± 2% 16µs ± 2% -90.77% (p=0.000 n=10+10) name old alloc/op new alloc/op delta EncodeFloats/10_seq 0.00B 0.00B ~ (all equal) EncodeFloats/10_ran 0.00B 0.00B ~ (all equal) EncodeFloats/100_seq 0.00B 0.00B ~ (all equal) EncodeFloats/100_ran 0.00B 0.00B ~ (all equal) EncodeFloats/1000_seq 0.00B 0.00B ~ (all equal) EncodeFloats/1000_ran 0.00B 0.00B ~ (all equal) name old allocs/op new allocs/op delta EncodeFloats/10_seq 0.00 0.00 ~ (all equal) EncodeFloats/10_ran 0.00 0.00 ~ (all equal) EncodeFloats/100_seq 0.00 0.00 ~ (all equal) EncodeFloats/100_ran 0.00 0.00 ~ (all equal) EncodeFloats/1000_seq 0.00 0.00 ~ (all equal) EncodeFloats/1000_ran 0.00 0.00 ~ (all equal)
2018-09-11 12:04:52 +00:00
}
}
})
})
}
}
func BenchmarkDecodeFloats(b *testing.B) {
cases := []int{1, 55, 550, 1000}
for _, n := range cases {
b.Run(fmt.Sprintf("%d_seq", n), func(b *testing.B) {
s := tsm1.NewFloatEncoder()
for i := 0; i < n; i++ {
s.Write(float64(i))
}
s.Flush()
data, err := s.Bytes()
if err != nil {
b.Fatalf("unexpected error: %v", err)
}
b.SetBytes(int64(len(data)))
b.ResetTimer()
dst := make([]float64, n)
for i := 0; i < b.N; i++ {
got, err := tsm1.FloatArrayDecodeAll(data, dst)
if err != nil {
b.Fatalf("unexpected error\n%s", err.Error())
}
if len(got) != n {
b.Fatalf("unexpected length -got/+exp\n%s", cmp.Diff(len(got), n))
}
}
})
b.Run(fmt.Sprintf("%d_ran", n), func(b *testing.B) {
s := tsm1.NewFloatEncoder()
for i := 0; i < n; i++ {
s.Write(rand.Float64() * 100.0)
}
s.Flush()
data, err := s.Bytes()
if err != nil {
b.Fatalf("unexpected error: %v", err)
}
b.SetBytes(int64(len(data)))
b.ResetTimer()
dst := make([]float64, n)
for i := 0; i < b.N; i++ {
got, err := tsm1.FloatArrayDecodeAll(data, dst)
if err != nil {
b.Fatalf("unexpected error\n%s", err.Error())
}
if len(got) != n {
b.Fatalf("unexpected length -got/+exp\n%s", cmp.Diff(len(got), n))
}
}
})
}
}
2018-09-05 10:13:11 +00:00
func BenchmarkFloatArrayDecodeAll(b *testing.B) {
2018-09-26 17:39:21 +00:00
benchmarks := []int{
1,
55,
550,
1000,
}
for _, size := range benchmarks {
s := tsm1.NewFloatEncoder()
for c := 0; c < size; c++ {
s.Write(twoHoursData[c%len(twoHoursData)])
}
s.Flush()
bytes, err := s.Bytes()
if err != nil {
b.Fatalf("unexpected error: %v", err)
}
b.Run(fmt.Sprintf("%d", size), func(b *testing.B) {
b.SetBytes(int64(len(bytes)))
b.ResetTimer()
dst := make([]float64, size)
for i := 0; i < b.N; i++ {
2018-09-05 10:13:11 +00:00
got, err := tsm1.FloatArrayDecodeAll(bytes, dst)
2018-09-26 17:39:21 +00:00
if err != nil {
b.Fatalf("unexpected error\n%s", err.Error())
2018-09-26 17:39:21 +00:00
}
if len(got) != size {
b.Fatalf("unexpected length -got/+exp\n%s", cmp.Diff(len(got), size))
2018-09-26 17:39:21 +00:00
}
}
})
}
}