Initial import of models package

models/inline_fnv.go

@@ -0,0 +1,32 @@
+package models // import "github.com/influxdata/platform/models"
+
+// from stdlib hash/fnv/fnv.go
+const (
+	prime64  = 1099511628211
+	offset64 = 14695981039346656037
+)
+
+// InlineFNV64a is an alloc-free port of the standard library's fnv64a.
+// See https://en.wikipedia.org/wiki/Fowler%E2%80%93Noll%E2%80%93Vo_hash_function.
+type InlineFNV64a uint64
+
+// NewInlineFNV64a returns a new instance of InlineFNV64a.
+func NewInlineFNV64a() InlineFNV64a {
+	return offset64
+}
+
+// Write adds data to the running hash.
+func (s *InlineFNV64a) Write(data []byte) (int, error) {
+	hash := uint64(*s)
+	for _, c := range data {
+		hash ^= uint64(c)
+		hash *= prime64
+	}
+	*s = InlineFNV64a(hash)
+	return len(data), nil
+}
+
+// Sum64 returns the uint64 of the current resulting hash.
+func (s *InlineFNV64a) Sum64() uint64 {
+	return uint64(*s)
+}

models/inline_fnv_test.go

@@ -0,0 +1,29 @@
+package models_test
+
+import (
+	"hash/fnv"
+	"testing"
+	"testing/quick"
+
+	"github.com/influxdata/platform/models"
+)
+
+func TestInlineFNV64aEquivalenceFuzz(t *testing.T) {
+	f := func(data []byte) bool {
+		stdlibFNV := fnv.New64a()
+		stdlibFNV.Write(data)
+		want := stdlibFNV.Sum64()
+
+		inlineFNV := models.NewInlineFNV64a()
+		inlineFNV.Write(data)
+		got := inlineFNV.Sum64()
+
+		return want == got
+	}
+	cfg := &quick.Config{
+		MaxCount: 10000,
+	}
+	if err := quick.Check(f, cfg); err != nil {
+		t.Fatal(err)
+	}
+}

models/inline_strconv_parse.go

@@ -0,0 +1,44 @@
+package models // import "github.com/influxdata/platform/models"
+
+import (
+	"reflect"
+	"strconv"
+	"unsafe"
+)
+
+// parseIntBytes is a zero-alloc wrapper around strconv.ParseInt.
+func parseIntBytes(b []byte, base int, bitSize int) (i int64, err error) {
+	s := unsafeBytesToString(b)
+	return strconv.ParseInt(s, base, bitSize)
+}
+
+// parseUintBytes is a zero-alloc wrapper around strconv.ParseUint.
+func parseUintBytes(b []byte, base int, bitSize int) (i uint64, err error) {
+	s := unsafeBytesToString(b)
+	return strconv.ParseUint(s, base, bitSize)
+}
+
+// parseFloatBytes is a zero-alloc wrapper around strconv.ParseFloat.
+func parseFloatBytes(b []byte, bitSize int) (float64, error) {
+	s := unsafeBytesToString(b)
+	return strconv.ParseFloat(s, bitSize)
+}
+
+// parseBoolBytes is a zero-alloc wrapper around strconv.ParseBool.
+func parseBoolBytes(b []byte) (bool, error) {
+	return strconv.ParseBool(unsafeBytesToString(b))
+}
+
+// unsafeBytesToString converts a []byte to a string without a heap allocation.
+//
+// It is unsafe, and is intended to prepare input to short-lived functions
+// that require strings.
+func unsafeBytesToString(in []byte) string {
+	src := *(*reflect.SliceHeader)(unsafe.Pointer(&in))
+	dst := reflect.StringHeader{
+		Data: src.Data,
+		Len:  src.Len,
+	}
+	s := *(*string)(unsafe.Pointer(&dst))
+	return s
+}

models/inline_strconv_parse_test.go

@@ -0,0 +1,103 @@
+package models
+
+import (
+	"strconv"
+	"testing"
+	"testing/quick"
+)
+
+func TestParseIntBytesEquivalenceFuzz(t *testing.T) {
+	f := func(b []byte, base int, bitSize int) bool {
+		exp, expErr := strconv.ParseInt(string(b), base, bitSize)
+		got, gotErr := parseIntBytes(b, base, bitSize)
+
+		return exp == got && checkErrs(expErr, gotErr)
+	}
+
+	cfg := &quick.Config{
+		MaxCount: 10000,
+	}
+
+	if err := quick.Check(f, cfg); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestParseIntBytesValid64bitBase10EquivalenceFuzz(t *testing.T) {
+	buf := []byte{}
+	f := func(n int64) bool {
+		buf = strconv.AppendInt(buf[:0], n, 10)
+
+		exp, expErr := strconv.ParseInt(string(buf), 10, 64)
+		got, gotErr := parseIntBytes(buf, 10, 64)
+
+		return exp == got && checkErrs(expErr, gotErr)
+	}
+
+	cfg := &quick.Config{
+		MaxCount: 10000,
+	}
+
+	if err := quick.Check(f, cfg); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestParseFloatBytesEquivalenceFuzz(t *testing.T) {
+	f := func(b []byte, bitSize int) bool {
+		exp, expErr := strconv.ParseFloat(string(b), bitSize)
+		got, gotErr := parseFloatBytes(b, bitSize)
+
+		return exp == got && checkErrs(expErr, gotErr)
+	}
+
+	cfg := &quick.Config{
+		MaxCount: 10000,
+	}
+
+	if err := quick.Check(f, cfg); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestParseFloatBytesValid64bitEquivalenceFuzz(t *testing.T) {
+	buf := []byte{}
+	f := func(n float64) bool {
+		buf = strconv.AppendFloat(buf[:0], n, 'f', -1, 64)
+
+		exp, expErr := strconv.ParseFloat(string(buf), 64)
+		got, gotErr := parseFloatBytes(buf, 64)
+
+		return exp == got && checkErrs(expErr, gotErr)
+	}
+
+	cfg := &quick.Config{
+		MaxCount: 10000,
+	}
+
+	if err := quick.Check(f, cfg); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestParseBoolBytesEquivalence(t *testing.T) {
+	var buf []byte
+	for _, s := range []string{"1", "t", "T", "TRUE", "true", "True", "0", "f", "F", "FALSE", "false", "False", "fail", "TrUe", "FAlSE", "numbers", ""} {
+		buf = append(buf[:0], s...)
+
+		exp, expErr := strconv.ParseBool(s)
+		got, gotErr := parseBoolBytes(buf)
+
+		if got != exp || !checkErrs(expErr, gotErr) {
+			t.Errorf("Failed to parse boolean value %q correctly: wanted (%t, %v), got (%t, %v)", s, exp, expErr, got, gotErr)
+		}
+	}
+}
+
+func checkErrs(a, b error) bool {
+	if (a == nil) != (b == nil) {
+		return false
+	}
+
+	return a == nil || a.Error() == b.Error()
+}

models/points_internal_test.go

@@ -0,0 +1,17 @@
+package models
+
+import "testing"
+
+func TestMarshalPointNoFields(t *testing.T) {
+	points, err := ParsePointsString("m,k=v f=0i")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// It's unclear how this can ever happen, but we've observed points that were marshalled without any fields.
+	points[0].(*point).fields = []byte{}
+
+	if _, err := points[0].MarshalBinary(); err != ErrPointMustHaveAField {
+		t.Fatalf("got error %v, exp %v", err, ErrPointMustHaveAField)
+	}
+}

models/rows.go

@@ -0,0 +1,62 @@
+package models
+
+import (
+	"sort"
+)
+
+// Row represents a single row returned from the execution of a statement.
+type Row struct {
+	Name    string            `json:"name,omitempty"`
+	Tags    map[string]string `json:"tags,omitempty"`
+	Columns []string          `json:"columns,omitempty"`
+	Values  [][]interface{}   `json:"values,omitempty"`
+	Partial bool              `json:"partial,omitempty"`
+}
+
+// SameSeries returns true if r contains values for the same series as o.
+func (r *Row) SameSeries(o *Row) bool {
+	return r.tagsHash() == o.tagsHash() && r.Name == o.Name
+}
+
+// tagsHash returns a hash of tag key/value pairs.
+func (r *Row) tagsHash() uint64 {
+	h := NewInlineFNV64a()
+	keys := r.tagsKeys()
+	for _, k := range keys {
+		h.Write([]byte(k))
+		h.Write([]byte(r.Tags[k]))
+	}
+	return h.Sum64()
+}
+
+// tagKeys returns a sorted list of tag keys.
+func (r *Row) tagsKeys() []string {
+	a := make([]string, 0, len(r.Tags))
+	for k := range r.Tags {
+		a = append(a, k)
+	}
+	sort.Strings(a)
+	return a
+}
+
+// Rows represents a collection of rows. Rows implements sort.Interface.
+type Rows []*Row
+
+// Len implements sort.Interface.
+func (p Rows) Len() int { return len(p) }
+
+// Less implements sort.Interface.
+func (p Rows) Less(i, j int) bool {
+	// Sort by name first.
+	if p[i].Name != p[j].Name {
+		return p[i].Name < p[j].Name
+	}
+
+	// Sort by tag set hash. Tags don't have a meaningful sort order so we
+	// just compute a hash and sort by that instead. This allows the tests
+	// to receive rows in a predictable order every time.
+	return p[i].tagsHash() < p[j].tagsHash()
+}
+
+// Swap implements sort.Interface.
+func (p Rows) Swap(i, j int) { p[i], p[j] = p[j], p[i] }

models/statistic.go

@@ -0,0 +1,42 @@
+package models
+
+// Statistic is the representation of a statistic used by the monitoring service.
+type Statistic struct {
+	Name   string                 `json:"name"`
+	Tags   map[string]string      `json:"tags"`
+	Values map[string]interface{} `json:"values"`
+}
+
+// NewStatistic returns an initialized Statistic.
+func NewStatistic(name string) Statistic {
+	return Statistic{
+		Name:   name,
+		Tags:   make(map[string]string),
+		Values: make(map[string]interface{}),
+	}
+}
+
+// StatisticTags is a map that can be merged with others without causing
+// mutations to either map.
+type StatisticTags map[string]string
+
+// Merge creates a new map containing the merged contents of tags and t.
+// If both tags and the receiver map contain the same key, the value in tags
+// is used in the resulting map.
+//
+// Merge always returns a usable map.
+func (t StatisticTags) Merge(tags map[string]string) map[string]string {
+	// Add everything in tags to the result.
+	out := make(map[string]string, len(tags))
+	for k, v := range tags {
+		out[k] = v
+	}
+
+	// Only add values from t that don't appear in tags.
+	for k, v := range t {
+		if _, ok := tags[k]; !ok {
+			out[k] = v
+		}
+	}
+	return out
+}

models/statistic_test.go

@@ -0,0 +1,55 @@
+package models_test
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/influxdata/platform/models"
+)
+
+func TestTags_Merge(t *testing.T) {
+	examples := []struct {
+		Base   map[string]string
+		Arg    map[string]string
+		Result map[string]string
+	}{
+		{
+			Base:   nil,
+			Arg:    nil,
+			Result: map[string]string{},
+		},
+		{
+			Base:   nil,
+			Arg:    map[string]string{"foo": "foo"},
+			Result: map[string]string{"foo": "foo"},
+		},
+		{
+			Base:   map[string]string{"foo": "foo"},
+			Arg:    nil,
+			Result: map[string]string{"foo": "foo"},
+		},
+		{
+			Base:   map[string]string{"foo": "foo"},
+			Arg:    map[string]string{"bar": "bar"},
+			Result: map[string]string{"foo": "foo", "bar": "bar"},
+		},
+		{
+			Base:   map[string]string{"foo": "foo", "bar": "bar"},
+			Arg:    map[string]string{"zoo": "zoo"},
+			Result: map[string]string{"foo": "foo", "bar": "bar", "zoo": "zoo"},
+		},
+		{
+			Base:   map[string]string{"foo": "foo", "bar": "bar"},
+			Arg:    map[string]string{"bar": "newbar"},
+			Result: map[string]string{"foo": "foo", "bar": "newbar"},
+		},
+	}
+
+	for i, example := range examples {
+		i++
+		result := models.StatisticTags(example.Base).Merge(example.Arg)
+		if got, exp := result, example.Result; !reflect.DeepEqual(got, exp) {
+			t.Errorf("[Example %d] got %#v, expected %#v", i, got, exp)
+		}
+	}
+}

models/time.go

@@ -0,0 +1,74 @@
+package models
+
+// Helper time methods since parsing time can easily overflow and we only support a
+// specific time range.
+
+import (
+	"fmt"
+	"math"
+	"time"
+)
+
+const (
+	// MinNanoTime is the minumum time that can be represented.
+	//
+	// 1677-09-21 00:12:43.145224194 +0000 UTC
+	//
+	// The two lowest minimum integers are used as sentinel values.  The
+	// minimum value needs to be used as a value lower than any other value for
+	// comparisons and another separate value is needed to act as a sentinel
+	// default value that is unusable by the user, but usable internally.
+	// Because these two values need to be used for a special purpose, we do
+	// not allow users to write points at these two times.
+	MinNanoTime = int64(math.MinInt64) + 2
+
+	// MaxNanoTime is the maximum time that can be represented.
+	//
+	// 2262-04-11 23:47:16.854775806 +0000 UTC
+	//
+	// The highest time represented by a nanosecond needs to be used for an
+	// exclusive range in the shard group, so the maximum time needs to be one
+	// less than the possible maximum number of nanoseconds representable by an
+	// int64 so that we don't lose a point at that one time.
+	MaxNanoTime = int64(math.MaxInt64) - 1
+)
+
+var (
+	minNanoTime = time.Unix(0, MinNanoTime).UTC()
+	maxNanoTime = time.Unix(0, MaxNanoTime).UTC()
+
+	// ErrTimeOutOfRange gets returned when time is out of the representable range using int64 nanoseconds since the epoch.
+	ErrTimeOutOfRange = fmt.Errorf("time outside range %d - %d", MinNanoTime, MaxNanoTime)
+)
+
+// SafeCalcTime safely calculates the time given. Will return error if the time is outside the
+// supported range.
+func SafeCalcTime(timestamp int64, precision string) (time.Time, error) {
+	mult := GetPrecisionMultiplier(precision)
+	if t, ok := safeSignedMult(timestamp, mult); ok {
+		tme := time.Unix(0, t).UTC()
+		return tme, CheckTime(tme)
+	}
+
+	return time.Time{}, ErrTimeOutOfRange
+}
+
+// CheckTime checks that a time is within the safe range.
+func CheckTime(t time.Time) error {
+	if t.Before(minNanoTime) || t.After(maxNanoTime) {
+		return ErrTimeOutOfRange
+	}
+	return nil
+}
+
+// Perform the multiplication and check to make sure it didn't overflow.
+func safeSignedMult(a, b int64) (int64, bool) {
+	if a == 0 || b == 0 || a == 1 || b == 1 {
+		return a * b, true
+	}
+	if a == MinNanoTime || b == MaxNanoTime {
+		return 0, false
+	}
+	c := a * b
+	return c, c/b == a
+}

models/uint_support.go

@@ -0,0 +1,7 @@
+// +build uint uint64
+
+package models
+
+func init() {
+	EnableUintSupport()
+}

tsdb/cursor.go

@@ -3,8 +3,8 @@ package tsdb
 import (
 	"context"
 
-	"github.com/influxdata/influxdb/models"
 	"github.com/influxdata/influxdb/query"
+	"github.com/influxdata/platform/models"
 )
 
 // EOF represents a "not found" key returned by a Cursor.

tsdb/engine.go

@@ -11,11 +11,11 @@ import (
 	"sort"
 	"time"
 
-	"github.com/influxdata/influxdb/models"
 	"github.com/influxdata/influxdb/pkg/estimator"
 	"github.com/influxdata/influxdb/pkg/limiter"
 	"github.com/influxdata/influxdb/query"
 	"github.com/influxdata/influxql"
+	"github.com/influxdata/platform/models"
 	"go.uber.org/zap"
 )
 

tsdb/field_validator.go

@@ -4,8 +4,8 @@ import (
 	"bytes"
 	"fmt"
 
-	"github.com/influxdata/influxdb/models"
 	"github.com/influxdata/influxql"
+	"github.com/influxdata/platform/models"
 )
 
 // FieldValidator should return a PartialWriteError if the point should not be written.

tsdb/index.go

@@ -7,10 +7,10 @@ import (
 	"regexp"
 	"sort"
 
-	"github.com/influxdata/influxdb/models"
 	"github.com/influxdata/influxdb/pkg/estimator"
 	"github.com/influxdata/influxdb/query"
 	"github.com/influxdata/influxql"
+	"github.com/influxdata/platform/models"
 	"go.uber.org/zap"
 )
 

tsdb/meta.go

@@ -3,7 +3,7 @@ package tsdb
 //go:generate protoc --gogo_out=. internal/meta.proto
 
 import (
-	"github.com/influxdata/influxdb/models"
+	"github.com/influxdata/platform/models"
 )
 
 // MakeTagsKey converts a tag set to bytes for use as a lookup key.

tsdb/series_collection.go

@@ -5,8 +5,8 @@ import (
 	"sync/atomic"
 	"unsafe"
 
-	"github.com/influxdata/influxdb/models"
 	"github.com/influxdata/influxdb/pkg/bytesutil"
+	"github.com/influxdata/platform/models"
 )
 
 // SeriesCollection is a struct of arrays representation of a collection of series that allows

tsdb/series_cursor.go

@@ -5,8 +5,8 @@ import (
 	"sort"
 	"sync"
 
-	"github.com/influxdata/influxdb/models"
 	"github.com/influxdata/influxql"
+	"github.com/influxdata/platform/models"
 )
 
 type SeriesCursor interface {

tsdb/series_file.go

@@ -10,8 +10,8 @@ import (
 	"sync"
 
 	"github.com/cespare/xxhash"
-	"github.com/influxdata/influxdb/models"
 	"github.com/influxdata/influxdb/pkg/binaryutil"
+	"github.com/influxdata/platform/models"
 	"go.uber.org/zap"
 	"golang.org/x/sync/errgroup"
 )

tsdb/series_id.go

@@ -3,7 +3,7 @@ package tsdb
 import (
 	"unsafe"
 
-	"github.com/influxdata/influxdb/models"
+	"github.com/influxdata/platform/models"
 )
 
 const (

tsdb/series_partition.go

@@ -10,8 +10,8 @@ import (
 	"sync"
 
 	"github.com/influxdata/influxdb/logger"
-	"github.com/influxdata/influxdb/models"
 	"github.com/influxdata/influxdb/pkg/rhh"
+	"github.com/influxdata/platform/models"
 	"go.uber.org/zap"
 )
 

tsdb/shard.go

@@ -6,8 +6,8 @@ import (
 	"path/filepath"
 	"sync"
 
-	"github.com/influxdata/influxdb/models"
 	"github.com/influxdata/influxql"
+	"github.com/influxdata/platform/models"
 	"go.uber.org/zap"
 )