influxdb/storage/flux/table.gen.go

3785 lines
88 KiB
Go
Raw Normal View History

// Generated by tmpl
// https://github.com/benbjohnson/tmpl
//
// DO NOT EDIT!
// Source: table.gen.go.tmpl
package storageflux
import (
"math"
"sync"
"github.com/apache/arrow/go/arrow/array"
"github.com/influxdata/flux"
"github.com/influxdata/flux/arrow"
"github.com/influxdata/flux/execute"
"github.com/influxdata/flux/memory"
"github.com/influxdata/influxdb/v2"
"github.com/influxdata/influxdb/v2/models"
storage "github.com/influxdata/influxdb/v2/storage/reads"
"github.com/influxdata/influxdb/v2/tsdb/cursors"
)
//
// *********** Float ***********
//
type floatTable struct {
table
mu sync.Mutex
cur cursors.FloatArrayCursor
alloc *memory.Allocator
}
func newFloatTable(
done chan struct{},
cur cursors.FloatArrayCursor,
bounds execute.Bounds,
key flux.GroupKey,
cols []flux.ColMeta,
tags models.Tags,
defs [][]byte,
cache *tagsCache,
alloc *memory.Allocator,
) *floatTable {
t := &floatTable{
table: newTable(done, bounds, key, cols, defs, cache, alloc),
cur: cur,
}
t.readTags(tags)
t.init(t.advance)
return t
}
func (t *floatTable) Close() {
t.mu.Lock()
if t.cur != nil {
t.cur.Close()
t.cur = nil
}
t.mu.Unlock()
}
func (t *floatTable) Statistics() cursors.CursorStats {
t.mu.Lock()
defer t.mu.Unlock()
cur := t.cur
if cur == nil {
return cursors.CursorStats{}
}
cs := cur.Stats()
return cursors.CursorStats{
ScannedValues: cs.ScannedValues,
ScannedBytes: cs.ScannedBytes,
}
}
func (t *floatTable) Do(f func(flux.ColReader) error) error {
return t.do(f, t.advance)
}
func (t *floatTable) advance() bool {
a := t.cur.Next()
l := a.Len()
if l == 0 {
return false
}
// Retrieve the buffer for the data to avoid allocating
// additional slices. If the buffer is still being used
// because the references were retained, then we will
// allocate a new buffer.
cr := t.allocateBuffer(l)
cr.cols[timeColIdx] = arrow.NewInt(a.Timestamps, t.alloc)
cr.cols[valueColIdx] = t.toArrowBuffer(a.Values)
t.appendTags(cr)
t.appendBounds(cr)
return true
}
// window table
type floatWindowTable struct {
floatTable
windowEvery int64
offset int64
arr *cursors.FloatArray
nextTS int64
idxInArr int
createEmpty bool
timeColumn string
}
func newFloatWindowTable(
done chan struct{},
cur cursors.FloatArrayCursor,
bounds execute.Bounds,
every int64,
offset int64,
createEmpty bool,
timeColumn string,
key flux.GroupKey,
cols []flux.ColMeta,
tags models.Tags,
defs [][]byte,
cache *tagsCache,
alloc *memory.Allocator,
) *floatWindowTable {
t := &floatWindowTable{
floatTable: floatTable{
table: newTable(done, bounds, key, cols, defs, cache, alloc),
cur: cur,
},
windowEvery: every,
offset: offset,
createEmpty: createEmpty,
timeColumn: timeColumn,
}
if t.createEmpty {
start := int64(bounds.Start)
t.nextTS = storage.WindowStop(start, every, offset)
}
t.readTags(tags)
t.init(t.advance)
return t
}
func (t *floatWindowTable) Do(f func(flux.ColReader) error) error {
return t.do(f, t.advance)
}
// createNextBufferTimes will read the timestamps from the array
// cursor and construct the values for the next buffer.
func (t *floatWindowTable) createNextBufferTimes() (start, stop *array.Int64, ok bool) {
startB := arrow.NewIntBuilder(t.alloc)
stopB := arrow.NewIntBuilder(t.alloc)
if t.createEmpty {
// There are no more windows when the start time is greater
// than or equal to the stop time.
if startT := t.nextTS - t.windowEvery; startT >= int64(t.bounds.Stop) {
return nil, nil, false
}
// Create a buffer with the buffer size.
// TODO(jsternberg): Calculate the exact size with max points as the maximum.
startB.Resize(storage.MaxPointsPerBlock)
stopB.Resize(storage.MaxPointsPerBlock)
for ; ; t.nextTS += t.windowEvery {
startT, stopT := t.getWindowBoundsFor(t.nextTS)
if startT >= int64(t.bounds.Stop) {
break
}
startB.Append(startT)
stopB.Append(stopT)
}
start = startB.NewInt64Array()
stop = stopB.NewInt64Array()
return start, stop, true
}
// Retrieve the next buffer so we can copy the timestamps.
if !t.nextBuffer() {
return nil, nil, false
}
// Copy over the timestamps from the next buffer and adjust
// times for the boundaries.
startB.Resize(len(t.arr.Timestamps))
stopB.Resize(len(t.arr.Timestamps))
for _, stopT := range t.arr.Timestamps {
startT, stopT := t.getWindowBoundsFor(stopT)
startB.Append(startT)
stopB.Append(stopT)
}
start = startB.NewInt64Array()
stop = stopB.NewInt64Array()
return start, stop, true
}
func (t *floatWindowTable) getWindowBoundsFor(ts int64) (startT, stopT int64) {
startT, stopT = ts-t.windowEvery, ts
if startT < int64(t.bounds.Start) {
startT = int64(t.bounds.Start)
}
if stopT > int64(t.bounds.Stop) {
stopT = int64(t.bounds.Stop)
}
return startT, stopT
}
// nextAt will retrieve the next value that can be used with
// the given stop timestamp. If no values can be used with the timestamp,
// it will return the default value and false.
func (t *floatWindowTable) nextAt(ts int64) (v float64, ok bool) {
if !t.nextBuffer() {
return
} else if !t.isInWindow(ts, t.arr.Timestamps[t.idxInArr]) {
return
}
v, ok = t.arr.Values[t.idxInArr], true
t.idxInArr++
return v, ok
}
// isInWindow will check if the given time at stop can be used within
// the window stop time for ts. The ts may be a truncated stop time
// because of a restricted boundary while stop will be the true
// stop time returned by storage.
func (t *floatWindowTable) isInWindow(ts int64, stop int64) bool {
// This method checks if the stop time is a valid stop time for
// that interval. This calculation is different from the calculation
// of the window itself. For example, for a 10 second window that
// starts at 20 seconds, we would include points between [20, 30).
// The stop time for this interval would be 30, but because the stop
// time can be truncated, valid stop times range from anywhere between
// (20, 30]. The storage engine will always produce 30 as the end time
// but we may have truncated the stop time because of the boundary
// and this is why we are checking for this range instead of checking
// if the two values are equal.
start := stop - t.windowEvery
return start < ts && ts <= stop
}
// nextBuffer will ensure the array cursor is filled
// and will return true if there is at least one value
// that can be read from it.
func (t *floatWindowTable) nextBuffer() bool {
// Discard the current array cursor if we have
// exceeded it.
if t.arr != nil && t.idxInArr >= t.arr.Len() {
t.arr = nil
}
// Retrieve the next array cursor if needed.
if t.arr == nil {
arr := t.cur.Next()
if arr.Len() == 0 {
return false
}
t.arr, t.idxInArr = arr, 0
}
return true
}
// appendValues will scan the timestamps and append values
// that match those timestamps from the buffer.
func (t *floatWindowTable) appendValues(intervals []int64, appendValue func(v float64), appendNull func()) {
for i := 0; i < len(intervals); i++ {
if v, ok := t.nextAt(intervals[i]); ok {
appendValue(v)
continue
}
appendNull()
}
}
func (t *floatWindowTable) advance() bool {
if !t.nextBuffer() {
return false
}
// Create the timestamps for the next window.
start, stop, ok := t.createNextBufferTimes()
if !ok {
return false
}
values := t.mergeValues(stop.Int64Values())
// Retrieve the buffer for the data to avoid allocating
// additional slices. If the buffer is still being used
// because the references were retained, then we will
// allocate a new buffer.
cr := t.allocateBuffer(stop.Len())
if t.timeColumn != "" {
switch t.timeColumn {
case execute.DefaultStopColLabel:
cr.cols[timeColIdx] = stop
start.Release()
case execute.DefaultStartColLabel:
cr.cols[timeColIdx] = start
stop.Release()
}
cr.cols[valueColIdx] = values
t.appendBounds(cr)
} else {
cr.cols[startColIdx] = start
cr.cols[stopColIdx] = stop
cr.cols[windowedValueColIdx] = values
}
t.appendTags(cr)
return true
}
// This table implementation will not have any empty windows.
type floatWindowSelectorTable struct {
floatTable
windowEvery int64
offset int64
timeColumn string
}
func newFloatWindowSelectorTable(
done chan struct{},
cur cursors.FloatArrayCursor,
bounds execute.Bounds,
every int64,
offset int64,
timeColumn string,
key flux.GroupKey,
cols []flux.ColMeta,
tags models.Tags,
defs [][]byte,
cache *tagsCache,
alloc *memory.Allocator,
) *floatWindowSelectorTable {
t := &floatWindowSelectorTable{
floatTable: floatTable{
table: newTable(done, bounds, key, cols, defs, cache, alloc),
cur: cur,
},
windowEvery: every,
offset: offset,
timeColumn: timeColumn,
}
t.readTags(tags)
t.init(t.advance)
return t
}
func (t *floatWindowSelectorTable) Do(f func(flux.ColReader) error) error {
return t.do(f, t.advance)
}
func (t *floatWindowSelectorTable) advance() bool {
arr := t.cur.Next()
if arr.Len() == 0 {
return false
}
cr := t.allocateBuffer(arr.Len())
switch t.timeColumn {
case execute.DefaultStartColLabel:
cr.cols[timeColIdx] = t.startTimes(arr)
t.appendBounds(cr)
case execute.DefaultStopColLabel:
cr.cols[timeColIdx] = t.stopTimes(arr)
t.appendBounds(cr)
default:
cr.cols[startColIdx] = t.startTimes(arr)
cr.cols[stopColIdx] = t.stopTimes(arr)
cr.cols[timeColIdx] = arrow.NewInt(arr.Timestamps, t.alloc)
}
cr.cols[valueColIdx] = t.toArrowBuffer(arr.Values)
t.appendTags(cr)
return true
}
func (t *floatWindowSelectorTable) startTimes(arr *cursors.FloatArray) *array.Int64 {
start := arrow.NewIntBuilder(t.alloc)
start.Resize(arr.Len())
rangeStart := int64(t.bounds.Start)
for _, v := range arr.Timestamps {
if windowStart := storage.WindowStart(v, t.windowEvery, t.offset); windowStart < rangeStart {
start.Append(rangeStart)
} else {
start.Append(windowStart)
}
}
return start.NewInt64Array()
}
func (t *floatWindowSelectorTable) stopTimes(arr *cursors.FloatArray) *array.Int64 {
stop := arrow.NewIntBuilder(t.alloc)
stop.Resize(arr.Len())
rangeStop := int64(t.bounds.Stop)
for _, v := range arr.Timestamps {
if windowStop := storage.WindowStop(v, t.windowEvery, t.offset); windowStop > rangeStop {
stop.Append(rangeStop)
} else {
stop.Append(windowStop)
}
}
return stop.NewInt64Array()
}
// This table implementation may contain empty windows
// in addition to non-empty windows.
type floatEmptyWindowSelectorTable struct {
floatTable
arr *cursors.FloatArray
idx int
rangeStart int64
rangeStop int64
windowStart int64
windowStop int64
windowEvery int64
timeColumn string
}
func newFloatEmptyWindowSelectorTable(
done chan struct{},
cur cursors.FloatArrayCursor,
bounds execute.Bounds,
windowEvery int64,
offset int64,
timeColumn string,
key flux.GroupKey,
cols []flux.ColMeta,
tags models.Tags,
defs [][]byte,
cache *tagsCache,
alloc *memory.Allocator,
) *floatEmptyWindowSelectorTable {
rangeStart := int64(bounds.Start)
rangeStop := int64(bounds.Stop)
t := &floatEmptyWindowSelectorTable{
floatTable: floatTable{
table: newTable(done, bounds, key, cols, defs, cache, alloc),
cur: cur,
},
arr: cur.Next(),
rangeStart: rangeStart,
rangeStop: rangeStop,
windowStart: storage.WindowStart(rangeStart, windowEvery, offset),
windowStop: storage.WindowStop(rangeStart, windowEvery, offset),
windowEvery: windowEvery,
timeColumn: timeColumn,
}
t.readTags(tags)
t.init(t.advance)
return t
}
func (t *floatEmptyWindowSelectorTable) Do(f func(flux.ColReader) error) error {
return t.do(f, t.advance)
}
func (t *floatEmptyWindowSelectorTable) advance() bool {
if t.arr.Len() == 0 {
return false
}
values := t.arrowBuilder()
values.Resize(storage.MaxPointsPerBlock)
var cr *colReader
switch t.timeColumn {
case execute.DefaultStartColLabel:
start := t.startTimes(values)
cr = t.allocateBuffer(start.Len())
cr.cols[timeColIdx] = start
t.appendBounds(cr)
case execute.DefaultStopColLabel:
stop := t.stopTimes(values)
cr = t.allocateBuffer(stop.Len())
cr.cols[timeColIdx] = stop
t.appendBounds(cr)
default:
start, stop, time := t.startStopTimes(values)
cr = t.allocateBuffer(time.Len())
cr.cols[startColIdx] = start
cr.cols[stopColIdx] = stop
cr.cols[timeColIdx] = time
}
cr.cols[valueColIdx] = values.NewFloat64Array()
t.appendTags(cr)
return true
}
func (t *floatEmptyWindowSelectorTable) startTimes(builder *array.Float64Builder) *array.Int64 {
start := arrow.NewIntBuilder(t.alloc)
start.Resize(storage.MaxPointsPerBlock)
for t.windowStart < t.rangeStop {
// The first window should start at the
// beginning of the time range.
if t.windowStart < t.rangeStart {
start.Append(t.rangeStart)
} else {
start.Append(t.windowStart)
}
var v int64
if t.arr.Len() == 0 {
v = math.MaxInt64
} else {
v = t.arr.Timestamps[t.idx]
}
// If the current timestamp falls within the
// current window, append the value to the
// builder, otherwise append a null value.
if t.windowStart <= v && v < t.windowStop {
t.append(builder, t.arr.Values[t.idx])
t.idx++
} else {
builder.AppendNull()
}
t.windowStart += t.windowEvery
t.windowStop += t.windowEvery
// If the current array is non-empty and has
// been read in its entirety, call Next().
if t.arr.Len() > 0 && t.idx == t.arr.Len() {
t.arr = t.cur.Next()
t.idx = 0
}
if start.Len() == storage.MaxPointsPerBlock {
break
}
}
return start.NewInt64Array()
}
func (t *floatEmptyWindowSelectorTable) stopTimes(builder *array.Float64Builder) *array.Int64 {
stop := arrow.NewIntBuilder(t.alloc)
stop.Resize(storage.MaxPointsPerBlock)
for t.windowStart < t.rangeStop {
// The last window should stop at the end of
// the time range.
if t.windowStop > t.rangeStop {
stop.Append(t.rangeStop)
} else {
stop.Append(t.windowStop)
}
var v int64
if t.arr.Len() == 0 {
v = math.MaxInt64
} else {
v = t.arr.Timestamps[t.idx]
}
// If the current timestamp falls within the
// current window, append the value to the
// builder, otherwise append a null value.
if t.windowStart <= v && v < t.windowStop {
t.append(builder, t.arr.Values[t.idx])
t.idx++
} else {
builder.AppendNull()
}
t.windowStart += t.windowEvery
t.windowStop += t.windowEvery
// If the current array is non-empty and has
// been read in its entirety, call Next().
if t.arr.Len() > 0 && t.idx == t.arr.Len() {
t.arr = t.cur.Next()
t.idx = 0
}
if stop.Len() == storage.MaxPointsPerBlock {
break
}
}
return stop.NewInt64Array()
}
func (t *floatEmptyWindowSelectorTable) startStopTimes(builder *array.Float64Builder) (*array.Int64, *array.Int64, *array.Int64) {
start := arrow.NewIntBuilder(t.alloc)
start.Resize(storage.MaxPointsPerBlock)
stop := arrow.NewIntBuilder(t.alloc)
stop.Resize(storage.MaxPointsPerBlock)
time := arrow.NewIntBuilder(t.alloc)
time.Resize(storage.MaxPointsPerBlock)
for t.windowStart < t.rangeStop {
// The first window should start at the
// beginning of the time range.
if t.windowStart < t.rangeStart {
start.Append(t.rangeStart)
} else {
start.Append(t.windowStart)
}
// The last window should stop at the end of
// the time range.
if t.windowStop > t.rangeStop {
stop.Append(t.rangeStop)
} else {
stop.Append(t.windowStop)
}
var v int64
if t.arr.Len() == 0 {
v = math.MaxInt64
} else {
v = t.arr.Timestamps[t.idx]
}
// If the current timestamp falls within the
// current window, append the value to the
// builder, otherwise append a null value.
if t.windowStart <= v && v < t.windowStop {
time.Append(v)
t.append(builder, t.arr.Values[t.idx])
t.idx++
} else {
time.AppendNull()
builder.AppendNull()
}
t.windowStart += t.windowEvery
t.windowStop += t.windowEvery
// If the current array is non-empty and has
// been read in its entirety, call Next().
if t.arr.Len() > 0 && t.idx == t.arr.Len() {
t.arr = t.cur.Next()
t.idx = 0
}
if time.Len() == storage.MaxPointsPerBlock {
break
}
}
return start.NewInt64Array(), stop.NewInt64Array(), time.NewInt64Array()
}
// group table
type floatGroupTable struct {
table
mu sync.Mutex
gc storage.GroupCursor
cur cursors.FloatArrayCursor
}
func newFloatGroupTable(
done chan struct{},
gc storage.GroupCursor,
cur cursors.FloatArrayCursor,
bounds execute.Bounds,
key flux.GroupKey,
cols []flux.ColMeta,
tags models.Tags,
defs [][]byte,
cache *tagsCache,
alloc *memory.Allocator,
) *floatGroupTable {
t := &floatGroupTable{
table: newTable(done, bounds, key, cols, defs, cache, alloc),
gc: gc,
cur: cur,
}
t.readTags(tags)
t.init(t.advance)
return t
}
func (t *floatGroupTable) Close() {
t.mu.Lock()
if t.cur != nil {
t.cur.Close()
t.cur = nil
}
if t.gc != nil {
t.gc.Close()
t.gc = nil
}
t.mu.Unlock()
}
func (t *floatGroupTable) Do(f func(flux.ColReader) error) error {
return t.do(f, t.advance)
}
func (t *floatGroupTable) advance() bool {
RETRY:
a := t.cur.Next()
l := a.Len()
if l == 0 {
if t.advanceCursor() {
goto RETRY
}
return false
}
// Retrieve the buffer for the data to avoid allocating
// additional slices. If the buffer is still being used
// because the references were retained, then we will
// allocate a new buffer.
cr := t.allocateBuffer(l)
cr.cols[timeColIdx] = arrow.NewInt(a.Timestamps, t.alloc)
cr.cols[valueColIdx] = t.toArrowBuffer(a.Values)
t.appendTags(cr)
t.appendBounds(cr)
return true
}
func (t *floatGroupTable) advanceCursor() bool {
t.cur.Close()
t.cur = nil
for t.gc.Next() {
cur := t.gc.Cursor()
if cur == nil {
continue
}
if typedCur, ok := cur.(cursors.FloatArrayCursor); !ok {
// TODO(sgc): error or skip?
cur.Close()
t.err = &influxdb.Error{
Code: influxdb.EInvalid,
Err: &GroupCursorError{
typ: "float",
cursor: cur,
},
}
return false
} else {
t.readTags(t.gc.Tags())
t.cur = typedCur
return true
}
}
return false
}
func (t *floatGroupTable) Statistics() cursors.CursorStats {
if t.cur == nil {
return cursors.CursorStats{}
}
cs := t.cur.Stats()
return cursors.CursorStats{
ScannedValues: cs.ScannedValues,
ScannedBytes: cs.ScannedBytes,
}
}
//
// *********** Integer ***********
//
type integerTable struct {
table
mu sync.Mutex
cur cursors.IntegerArrayCursor
alloc *memory.Allocator
}
func newIntegerTable(
done chan struct{},
cur cursors.IntegerArrayCursor,
bounds execute.Bounds,
key flux.GroupKey,
cols []flux.ColMeta,
tags models.Tags,
defs [][]byte,
cache *tagsCache,
alloc *memory.Allocator,
) *integerTable {
t := &integerTable{
table: newTable(done, bounds, key, cols, defs, cache, alloc),
cur: cur,
}
t.readTags(tags)
t.init(t.advance)
return t
}
func (t *integerTable) Close() {
t.mu.Lock()
if t.cur != nil {
t.cur.Close()
t.cur = nil
}
t.mu.Unlock()
}
func (t *integerTable) Statistics() cursors.CursorStats {
t.mu.Lock()
defer t.mu.Unlock()
cur := t.cur
if cur == nil {
return cursors.CursorStats{}
}
cs := cur.Stats()
return cursors.CursorStats{
ScannedValues: cs.ScannedValues,
ScannedBytes: cs.ScannedBytes,
}
}
func (t *integerTable) Do(f func(flux.ColReader) error) error {
return t.do(f, t.advance)
}
func (t *integerTable) advance() bool {
a := t.cur.Next()
l := a.Len()
if l == 0 {
return false
}
// Retrieve the buffer for the data to avoid allocating
// additional slices. If the buffer is still being used
// because the references were retained, then we will
// allocate a new buffer.
cr := t.allocateBuffer(l)
cr.cols[timeColIdx] = arrow.NewInt(a.Timestamps, t.alloc)
cr.cols[valueColIdx] = t.toArrowBuffer(a.Values)
t.appendTags(cr)
t.appendBounds(cr)
return true
}
// window table
type integerWindowTable struct {
integerTable
windowEvery int64
offset int64
arr *cursors.IntegerArray
nextTS int64
idxInArr int
createEmpty bool
timeColumn string
fillValue *int64
}
func newIntegerWindowTable(
done chan struct{},
cur cursors.IntegerArrayCursor,
bounds execute.Bounds,
every int64,
offset int64,
createEmpty bool,
timeColumn string,
fillValue *int64,
key flux.GroupKey,
cols []flux.ColMeta,
tags models.Tags,
defs [][]byte,
cache *tagsCache,
alloc *memory.Allocator,
) *integerWindowTable {
t := &integerWindowTable{
integerTable: integerTable{
table: newTable(done, bounds, key, cols, defs, cache, alloc),
cur: cur,
},
windowEvery: every,
offset: offset,
createEmpty: createEmpty,
timeColumn: timeColumn,
fillValue: fillValue,
}
if t.createEmpty {
start := int64(bounds.Start)
t.nextTS = storage.WindowStop(start, every, offset)
}
t.readTags(tags)
t.init(t.advance)
return t
}
func (t *integerWindowTable) Do(f func(flux.ColReader) error) error {
return t.do(f, t.advance)
}
// createNextBufferTimes will read the timestamps from the array
// cursor and construct the values for the next buffer.
func (t *integerWindowTable) createNextBufferTimes() (start, stop *array.Int64, ok bool) {
startB := arrow.NewIntBuilder(t.alloc)
stopB := arrow.NewIntBuilder(t.alloc)
if t.createEmpty {
// There are no more windows when the start time is greater
// than or equal to the stop time.
if startT := t.nextTS - t.windowEvery; startT >= int64(t.bounds.Stop) {
return nil, nil, false
}
// Create a buffer with the buffer size.
// TODO(jsternberg): Calculate the exact size with max points as the maximum.
startB.Resize(storage.MaxPointsPerBlock)
stopB.Resize(storage.MaxPointsPerBlock)
for ; ; t.nextTS += t.windowEvery {
startT, stopT := t.getWindowBoundsFor(t.nextTS)
if startT >= int64(t.bounds.Stop) {
break
}
startB.Append(startT)
stopB.Append(stopT)
}
start = startB.NewInt64Array()
stop = stopB.NewInt64Array()
return start, stop, true
}
// Retrieve the next buffer so we can copy the timestamps.
if !t.nextBuffer() {
return nil, nil, false
}
// Copy over the timestamps from the next buffer and adjust
// times for the boundaries.
startB.Resize(len(t.arr.Timestamps))
stopB.Resize(len(t.arr.Timestamps))
for _, stopT := range t.arr.Timestamps {
startT, stopT := t.getWindowBoundsFor(stopT)
startB.Append(startT)
stopB.Append(stopT)
}
start = startB.NewInt64Array()
stop = stopB.NewInt64Array()
return start, stop, true
}
func (t *integerWindowTable) getWindowBoundsFor(ts int64) (startT, stopT int64) {
startT, stopT = ts-t.windowEvery, ts
if startT < int64(t.bounds.Start) {
startT = int64(t.bounds.Start)
}
if stopT > int64(t.bounds.Stop) {
stopT = int64(t.bounds.Stop)
}
return startT, stopT
}
// nextAt will retrieve the next value that can be used with
// the given stop timestamp. If no values can be used with the timestamp,
// it will return the default value and false.
func (t *integerWindowTable) nextAt(ts int64) (v int64, ok bool) {
if !t.nextBuffer() {
return
} else if !t.isInWindow(ts, t.arr.Timestamps[t.idxInArr]) {
return
}
v, ok = t.arr.Values[t.idxInArr], true
t.idxInArr++
return v, ok
}
// isInWindow will check if the given time at stop can be used within
// the window stop time for ts. The ts may be a truncated stop time
// because of a restricted boundary while stop will be the true
// stop time returned by storage.
func (t *integerWindowTable) isInWindow(ts int64, stop int64) bool {
// This method checks if the stop time is a valid stop time for
// that interval. This calculation is different from the calculation
// of the window itself. For example, for a 10 second window that
// starts at 20 seconds, we would include points between [20, 30).
// The stop time for this interval would be 30, but because the stop
// time can be truncated, valid stop times range from anywhere between
// (20, 30]. The storage engine will always produce 30 as the end time
// but we may have truncated the stop time because of the boundary
// and this is why we are checking for this range instead of checking
// if the two values are equal.
start := stop - t.windowEvery
return start < ts && ts <= stop
}
// nextBuffer will ensure the array cursor is filled
// and will return true if there is at least one value
// that can be read from it.
func (t *integerWindowTable) nextBuffer() bool {
// Discard the current array cursor if we have
// exceeded it.
if t.arr != nil && t.idxInArr >= t.arr.Len() {
t.arr = nil
}
// Retrieve the next array cursor if needed.
if t.arr == nil {
arr := t.cur.Next()
if arr.Len() == 0 {
return false
}
t.arr, t.idxInArr = arr, 0
}
return true
}
// appendValues will scan the timestamps and append values
// that match those timestamps from the buffer.
func (t *integerWindowTable) appendValues(intervals []int64, appendValue func(v int64), appendNull func()) {
for i := 0; i < len(intervals); i++ {
if v, ok := t.nextAt(intervals[i]); ok {
appendValue(v)
continue
}
appendNull()
}
}
func (t *integerWindowTable) advance() bool {
if !t.nextBuffer() {
return false
}
// Create the timestamps for the next window.
start, stop, ok := t.createNextBufferTimes()
if !ok {
return false
}
values := t.mergeValues(stop.Int64Values())
// Retrieve the buffer for the data to avoid allocating
// additional slices. If the buffer is still being used
// because the references were retained, then we will
// allocate a new buffer.
cr := t.allocateBuffer(stop.Len())
if t.timeColumn != "" {
switch t.timeColumn {
case execute.DefaultStopColLabel:
cr.cols[timeColIdx] = stop
start.Release()
case execute.DefaultStartColLabel:
cr.cols[timeColIdx] = start
stop.Release()
}
cr.cols[valueColIdx] = values
t.appendBounds(cr)
} else {
cr.cols[startColIdx] = start
cr.cols[stopColIdx] = stop
cr.cols[windowedValueColIdx] = values
}
t.appendTags(cr)
return true
}
// This table implementation will not have any empty windows.
type integerWindowSelectorTable struct {
integerTable
windowEvery int64
offset int64
timeColumn string
}
func newIntegerWindowSelectorTable(
done chan struct{},
cur cursors.IntegerArrayCursor,
bounds execute.Bounds,
every int64,
offset int64,
timeColumn string,
key flux.GroupKey,
cols []flux.ColMeta,
tags models.Tags,
defs [][]byte,
cache *tagsCache,
alloc *memory.Allocator,
) *integerWindowSelectorTable {
t := &integerWindowSelectorTable{
integerTable: integerTable{
table: newTable(done, bounds, key, cols, defs, cache, alloc),
cur: cur,
},
windowEvery: every,
offset: offset,
timeColumn: timeColumn,
}
t.readTags(tags)
t.init(t.advance)
return t
}
func (t *integerWindowSelectorTable) Do(f func(flux.ColReader) error) error {
return t.do(f, t.advance)
}
func (t *integerWindowSelectorTable) advance() bool {
arr := t.cur.Next()
if arr.Len() == 0 {
return false
}
cr := t.allocateBuffer(arr.Len())
switch t.timeColumn {
case execute.DefaultStartColLabel:
cr.cols[timeColIdx] = t.startTimes(arr)
t.appendBounds(cr)
case execute.DefaultStopColLabel:
cr.cols[timeColIdx] = t.stopTimes(arr)
t.appendBounds(cr)
default:
cr.cols[startColIdx] = t.startTimes(arr)
cr.cols[stopColIdx] = t.stopTimes(arr)
cr.cols[timeColIdx] = arrow.NewInt(arr.Timestamps, t.alloc)
}
cr.cols[valueColIdx] = t.toArrowBuffer(arr.Values)
t.appendTags(cr)
return true
}
func (t *integerWindowSelectorTable) startTimes(arr *cursors.IntegerArray) *array.Int64 {
start := arrow.NewIntBuilder(t.alloc)
start.Resize(arr.Len())
rangeStart := int64(t.bounds.Start)
for _, v := range arr.Timestamps {
if windowStart := storage.WindowStart(v, t.windowEvery, t.offset); windowStart < rangeStart {
start.Append(rangeStart)
} else {
start.Append(windowStart)
}
}
return start.NewInt64Array()
}
func (t *integerWindowSelectorTable) stopTimes(arr *cursors.IntegerArray) *array.Int64 {
stop := arrow.NewIntBuilder(t.alloc)
stop.Resize(arr.Len())
rangeStop := int64(t.bounds.Stop)
for _, v := range arr.Timestamps {
if windowStop := storage.WindowStop(v, t.windowEvery, t.offset); windowStop > rangeStop {
stop.Append(rangeStop)
} else {
stop.Append(windowStop)
}
}
return stop.NewInt64Array()
}
// This table implementation may contain empty windows
// in addition to non-empty windows.
type integerEmptyWindowSelectorTable struct {
integerTable
arr *cursors.IntegerArray
idx int
rangeStart int64
rangeStop int64
windowStart int64
windowStop int64
windowEvery int64
timeColumn string
}
func newIntegerEmptyWindowSelectorTable(
done chan struct{},
cur cursors.IntegerArrayCursor,
bounds execute.Bounds,
windowEvery int64,
offset int64,
timeColumn string,
key flux.GroupKey,
cols []flux.ColMeta,
tags models.Tags,
defs [][]byte,
cache *tagsCache,
alloc *memory.Allocator,
) *integerEmptyWindowSelectorTable {
rangeStart := int64(bounds.Start)
rangeStop := int64(bounds.Stop)
t := &integerEmptyWindowSelectorTable{
integerTable: integerTable{
table: newTable(done, bounds, key, cols, defs, cache, alloc),
cur: cur,
},
arr: cur.Next(),
rangeStart: rangeStart,
rangeStop: rangeStop,
windowStart: storage.WindowStart(rangeStart, windowEvery, offset),
windowStop: storage.WindowStop(rangeStart, windowEvery, offset),
windowEvery: windowEvery,
timeColumn: timeColumn,
}
t.readTags(tags)
t.init(t.advance)
return t
}
func (t *integerEmptyWindowSelectorTable) Do(f func(flux.ColReader) error) error {
return t.do(f, t.advance)
}
func (t *integerEmptyWindowSelectorTable) advance() bool {
if t.arr.Len() == 0 {
return false
}
values := t.arrowBuilder()
values.Resize(storage.MaxPointsPerBlock)
var cr *colReader
switch t.timeColumn {
case execute.DefaultStartColLabel:
start := t.startTimes(values)
cr = t.allocateBuffer(start.Len())
cr.cols[timeColIdx] = start
t.appendBounds(cr)
case execute.DefaultStopColLabel:
stop := t.stopTimes(values)
cr = t.allocateBuffer(stop.Len())
cr.cols[timeColIdx] = stop
t.appendBounds(cr)
default:
start, stop, time := t.startStopTimes(values)
cr = t.allocateBuffer(time.Len())
cr.cols[startColIdx] = start
cr.cols[stopColIdx] = stop
cr.cols[timeColIdx] = time
}
cr.cols[valueColIdx] = values.NewInt64Array()
t.appendTags(cr)
return true
}
func (t *integerEmptyWindowSelectorTable) startTimes(builder *array.Int64Builder) *array.Int64 {
start := arrow.NewIntBuilder(t.alloc)
start.Resize(storage.MaxPointsPerBlock)
for t.windowStart < t.rangeStop {
// The first window should start at the
// beginning of the time range.
if t.windowStart < t.rangeStart {
start.Append(t.rangeStart)
} else {
start.Append(t.windowStart)
}
var v int64
if t.arr.Len() == 0 {
v = math.MaxInt64
} else {
v = t.arr.Timestamps[t.idx]
}
// If the current timestamp falls within the
// current window, append the value to the
// builder, otherwise append a null value.
if t.windowStart <= v && v < t.windowStop {
t.append(builder, t.arr.Values[t.idx])
t.idx++
} else {
builder.AppendNull()
}
t.windowStart += t.windowEvery
t.windowStop += t.windowEvery
// If the current array is non-empty and has
// been read in its entirety, call Next().
if t.arr.Len() > 0 && t.idx == t.arr.Len() {
t.arr = t.cur.Next()
t.idx = 0
}
if start.Len() == storage.MaxPointsPerBlock {
break
}
}
return start.NewInt64Array()
}
func (t *integerEmptyWindowSelectorTable) stopTimes(builder *array.Int64Builder) *array.Int64 {
stop := arrow.NewIntBuilder(t.alloc)
stop.Resize(storage.MaxPointsPerBlock)
for t.windowStart < t.rangeStop {
// The last window should stop at the end of
// the time range.
if t.windowStop > t.rangeStop {
stop.Append(t.rangeStop)
} else {
stop.Append(t.windowStop)
}
var v int64
if t.arr.Len() == 0 {
v = math.MaxInt64
} else {
v = t.arr.Timestamps[t.idx]
}
// If the current timestamp falls within the
// current window, append the value to the
// builder, otherwise append a null value.
if t.windowStart <= v && v < t.windowStop {
t.append(builder, t.arr.Values[t.idx])
t.idx++
} else {
builder.AppendNull()
}
t.windowStart += t.windowEvery
t.windowStop += t.windowEvery
// If the current array is non-empty and has
// been read in its entirety, call Next().
if t.arr.Len() > 0 && t.idx == t.arr.Len() {
t.arr = t.cur.Next()
t.idx = 0
}
if stop.Len() == storage.MaxPointsPerBlock {
break
}
}
return stop.NewInt64Array()
}
func (t *integerEmptyWindowSelectorTable) startStopTimes(builder *array.Int64Builder) (*array.Int64, *array.Int64, *array.Int64) {
start := arrow.NewIntBuilder(t.alloc)
start.Resize(storage.MaxPointsPerBlock)
stop := arrow.NewIntBuilder(t.alloc)
stop.Resize(storage.MaxPointsPerBlock)
time := arrow.NewIntBuilder(t.alloc)
time.Resize(storage.MaxPointsPerBlock)
for t.windowStart < t.rangeStop {
// The first window should start at the
// beginning of the time range.
if t.windowStart < t.rangeStart {
start.Append(t.rangeStart)
} else {
start.Append(t.windowStart)
}
// The last window should stop at the end of
// the time range.
if t.windowStop > t.rangeStop {
stop.Append(t.rangeStop)
} else {
stop.Append(t.windowStop)
}
var v int64
if t.arr.Len() == 0 {
v = math.MaxInt64
} else {
v = t.arr.Timestamps[t.idx]
}
// If the current timestamp falls within the
// current window, append the value to the
// builder, otherwise append a null value.
if t.windowStart <= v && v < t.windowStop {
time.Append(v)
t.append(builder, t.arr.Values[t.idx])
t.idx++
} else {
time.AppendNull()
builder.AppendNull()
}
t.windowStart += t.windowEvery
t.windowStop += t.windowEvery
// If the current array is non-empty and has
// been read in its entirety, call Next().
if t.arr.Len() > 0 && t.idx == t.arr.Len() {
t.arr = t.cur.Next()
t.idx = 0
}
if time.Len() == storage.MaxPointsPerBlock {
break
}
}
return start.NewInt64Array(), stop.NewInt64Array(), time.NewInt64Array()
}
// group table
type integerGroupTable struct {
table
mu sync.Mutex
gc storage.GroupCursor
cur cursors.IntegerArrayCursor
}
func newIntegerGroupTable(
done chan struct{},
gc storage.GroupCursor,
cur cursors.IntegerArrayCursor,
bounds execute.Bounds,
key flux.GroupKey,
cols []flux.ColMeta,
tags models.Tags,
defs [][]byte,
cache *tagsCache,
alloc *memory.Allocator,
) *integerGroupTable {
t := &integerGroupTable{
table: newTable(done, bounds, key, cols, defs, cache, alloc),
gc: gc,
cur: cur,
}
t.readTags(tags)
t.init(t.advance)
return t
}
func (t *integerGroupTable) Close() {
t.mu.Lock()
if t.cur != nil {
t.cur.Close()
t.cur = nil
}
if t.gc != nil {
t.gc.Close()
t.gc = nil
}
t.mu.Unlock()
}
func (t *integerGroupTable) Do(f func(flux.ColReader) error) error {
return t.do(f, t.advance)
}
func (t *integerGroupTable) advance() bool {
RETRY:
a := t.cur.Next()
l := a.Len()
if l == 0 {
if t.advanceCursor() {
goto RETRY
}
return false
}
// Retrieve the buffer for the data to avoid allocating
// additional slices. If the buffer is still being used
// because the references were retained, then we will
// allocate a new buffer.
cr := t.allocateBuffer(l)
cr.cols[timeColIdx] = arrow.NewInt(a.Timestamps, t.alloc)
cr.cols[valueColIdx] = t.toArrowBuffer(a.Values)
t.appendTags(cr)
t.appendBounds(cr)
return true
}
func (t *integerGroupTable) advanceCursor() bool {
t.cur.Close()
t.cur = nil
for t.gc.Next() {
cur := t.gc.Cursor()
if cur == nil {
continue
}
if typedCur, ok := cur.(cursors.IntegerArrayCursor); !ok {
// TODO(sgc): error or skip?
cur.Close()
t.err = &influxdb.Error{
Code: influxdb.EInvalid,
Err: &GroupCursorError{
typ: "integer",
cursor: cur,
},
}
return false
} else {
t.readTags(t.gc.Tags())
t.cur = typedCur
return true
}
}
return false
}
func (t *integerGroupTable) Statistics() cursors.CursorStats {
if t.cur == nil {
return cursors.CursorStats{}
}
cs := t.cur.Stats()
return cursors.CursorStats{
ScannedValues: cs.ScannedValues,
ScannedBytes: cs.ScannedBytes,
}
}
//
// *********** Unsigned ***********
//
type unsignedTable struct {
table
mu sync.Mutex
cur cursors.UnsignedArrayCursor
alloc *memory.Allocator
}
func newUnsignedTable(
done chan struct{},
cur cursors.UnsignedArrayCursor,
bounds execute.Bounds,
key flux.GroupKey,
cols []flux.ColMeta,
tags models.Tags,
defs [][]byte,
cache *tagsCache,
alloc *memory.Allocator,
) *unsignedTable {
t := &unsignedTable{
table: newTable(done, bounds, key, cols, defs, cache, alloc),
cur: cur,
}
t.readTags(tags)
t.init(t.advance)
return t
}
func (t *unsignedTable) Close() {
t.mu.Lock()
if t.cur != nil {
t.cur.Close()
t.cur = nil
}
t.mu.Unlock()
}
func (t *unsignedTable) Statistics() cursors.CursorStats {
t.mu.Lock()
defer t.mu.Unlock()
cur := t.cur
if cur == nil {
return cursors.CursorStats{}
}
cs := cur.Stats()
return cursors.CursorStats{
ScannedValues: cs.ScannedValues,
ScannedBytes: cs.ScannedBytes,
}
}
func (t *unsignedTable) Do(f func(flux.ColReader) error) error {
return t.do(f, t.advance)
}
func (t *unsignedTable) advance() bool {
a := t.cur.Next()
l := a.Len()
if l == 0 {
return false
}
// Retrieve the buffer for the data to avoid allocating
// additional slices. If the buffer is still being used
// because the references were retained, then we will
// allocate a new buffer.
cr := t.allocateBuffer(l)
cr.cols[timeColIdx] = arrow.NewInt(a.Timestamps, t.alloc)
cr.cols[valueColIdx] = t.toArrowBuffer(a.Values)
t.appendTags(cr)
t.appendBounds(cr)
return true
}
// window table
type unsignedWindowTable struct {
unsignedTable
windowEvery int64
offset int64
arr *cursors.UnsignedArray
nextTS int64
idxInArr int
createEmpty bool
timeColumn string
}
func newUnsignedWindowTable(
done chan struct{},
cur cursors.UnsignedArrayCursor,
bounds execute.Bounds,
every int64,
offset int64,
createEmpty bool,
timeColumn string,
key flux.GroupKey,
cols []flux.ColMeta,
tags models.Tags,
defs [][]byte,
cache *tagsCache,
alloc *memory.Allocator,
) *unsignedWindowTable {
t := &unsignedWindowTable{
unsignedTable: unsignedTable{
table: newTable(done, bounds, key, cols, defs, cache, alloc),
cur: cur,
},
windowEvery: every,
offset: offset,
createEmpty: createEmpty,
timeColumn: timeColumn,
}
if t.createEmpty {
start := int64(bounds.Start)
t.nextTS = storage.WindowStop(start, every, offset)
}
t.readTags(tags)
t.init(t.advance)
return t
}
func (t *unsignedWindowTable) Do(f func(flux.ColReader) error) error {
return t.do(f, t.advance)
}
// createNextBufferTimes will read the timestamps from the array
// cursor and construct the values for the next buffer.
func (t *unsignedWindowTable) createNextBufferTimes() (start, stop *array.Int64, ok bool) {
startB := arrow.NewIntBuilder(t.alloc)
stopB := arrow.NewIntBuilder(t.alloc)
if t.createEmpty {
// There are no more windows when the start time is greater
// than or equal to the stop time.
if startT := t.nextTS - t.windowEvery; startT >= int64(t.bounds.Stop) {
return nil, nil, false
}
// Create a buffer with the buffer size.
// TODO(jsternberg): Calculate the exact size with max points as the maximum.
startB.Resize(storage.MaxPointsPerBlock)
stopB.Resize(storage.MaxPointsPerBlock)
for ; ; t.nextTS += t.windowEvery {
startT, stopT := t.getWindowBoundsFor(t.nextTS)
if startT >= int64(t.bounds.Stop) {
break
}
startB.Append(startT)
stopB.Append(stopT)
}
start = startB.NewInt64Array()
stop = stopB.NewInt64Array()
return start, stop, true
}
// Retrieve the next buffer so we can copy the timestamps.
if !t.nextBuffer() {
return nil, nil, false
}
// Copy over the timestamps from the next buffer and adjust
// times for the boundaries.
startB.Resize(len(t.arr.Timestamps))
stopB.Resize(len(t.arr.Timestamps))
for _, stopT := range t.arr.Timestamps {
startT, stopT := t.getWindowBoundsFor(stopT)
startB.Append(startT)
stopB.Append(stopT)
}
start = startB.NewInt64Array()
stop = stopB.NewInt64Array()
return start, stop, true
}
func (t *unsignedWindowTable) getWindowBoundsFor(ts int64) (startT, stopT int64) {
startT, stopT = ts-t.windowEvery, ts
if startT < int64(t.bounds.Start) {
startT = int64(t.bounds.Start)
}
if stopT > int64(t.bounds.Stop) {
stopT = int64(t.bounds.Stop)
}
return startT, stopT
}
// nextAt will retrieve the next value that can be used with
// the given stop timestamp. If no values can be used with the timestamp,
// it will return the default value and false.
func (t *unsignedWindowTable) nextAt(ts int64) (v uint64, ok bool) {
if !t.nextBuffer() {
return
} else if !t.isInWindow(ts, t.arr.Timestamps[t.idxInArr]) {
return
}
v, ok = t.arr.Values[t.idxInArr], true
t.idxInArr++
return v, ok
}
// isInWindow will check if the given time at stop can be used within
// the window stop time for ts. The ts may be a truncated stop time
// because of a restricted boundary while stop will be the true
// stop time returned by storage.
func (t *unsignedWindowTable) isInWindow(ts int64, stop int64) bool {
// This method checks if the stop time is a valid stop time for
// that interval. This calculation is different from the calculation
// of the window itself. For example, for a 10 second window that
// starts at 20 seconds, we would include points between [20, 30).
// The stop time for this interval would be 30, but because the stop
// time can be truncated, valid stop times range from anywhere between
// (20, 30]. The storage engine will always produce 30 as the end time
// but we may have truncated the stop time because of the boundary
// and this is why we are checking for this range instead of checking
// if the two values are equal.
start := stop - t.windowEvery
return start < ts && ts <= stop
}
// nextBuffer will ensure the array cursor is filled
// and will return true if there is at least one value
// that can be read from it.
func (t *unsignedWindowTable) nextBuffer() bool {
// Discard the current array cursor if we have
// exceeded it.
if t.arr != nil && t.idxInArr >= t.arr.Len() {
t.arr = nil
}
// Retrieve the next array cursor if needed.
if t.arr == nil {
arr := t.cur.Next()
if arr.Len() == 0 {
return false
}
t.arr, t.idxInArr = arr, 0
}
return true
}
// appendValues will scan the timestamps and append values
// that match those timestamps from the buffer.
func (t *unsignedWindowTable) appendValues(intervals []int64, appendValue func(v uint64), appendNull func()) {
for i := 0; i < len(intervals); i++ {
if v, ok := t.nextAt(intervals[i]); ok {
appendValue(v)
continue
}
appendNull()
}
}
func (t *unsignedWindowTable) advance() bool {
if !t.nextBuffer() {
return false
}
// Create the timestamps for the next window.
start, stop, ok := t.createNextBufferTimes()
if !ok {
return false
}
values := t.mergeValues(stop.Int64Values())
// Retrieve the buffer for the data to avoid allocating
// additional slices. If the buffer is still being used
// because the references were retained, then we will
// allocate a new buffer.
cr := t.allocateBuffer(stop.Len())
if t.timeColumn != "" {
switch t.timeColumn {
case execute.DefaultStopColLabel:
cr.cols[timeColIdx] = stop
start.Release()
case execute.DefaultStartColLabel:
cr.cols[timeColIdx] = start
stop.Release()
}
cr.cols[valueColIdx] = values
t.appendBounds(cr)
} else {
cr.cols[startColIdx] = start
cr.cols[stopColIdx] = stop
cr.cols[windowedValueColIdx] = values
}
t.appendTags(cr)
return true
}
// This table implementation will not have any empty windows.
type unsignedWindowSelectorTable struct {
unsignedTable
windowEvery int64
offset int64
timeColumn string
}
func newUnsignedWindowSelectorTable(
done chan struct{},
cur cursors.UnsignedArrayCursor,
bounds execute.Bounds,
every int64,
offset int64,
timeColumn string,
key flux.GroupKey,
cols []flux.ColMeta,
tags models.Tags,
defs [][]byte,
cache *tagsCache,
alloc *memory.Allocator,
) *unsignedWindowSelectorTable {
t := &unsignedWindowSelectorTable{
unsignedTable: unsignedTable{
table: newTable(done, bounds, key, cols, defs, cache, alloc),
cur: cur,
},
windowEvery: every,
offset: offset,
timeColumn: timeColumn,
}
t.readTags(tags)
t.init(t.advance)
return t
}
func (t *unsignedWindowSelectorTable) Do(f func(flux.ColReader) error) error {
return t.do(f, t.advance)
}
func (t *unsignedWindowSelectorTable) advance() bool {
arr := t.cur.Next()
if arr.Len() == 0 {
return false
}
cr := t.allocateBuffer(arr.Len())
switch t.timeColumn {
case execute.DefaultStartColLabel:
cr.cols[timeColIdx] = t.startTimes(arr)
t.appendBounds(cr)
case execute.DefaultStopColLabel:
cr.cols[timeColIdx] = t.stopTimes(arr)
t.appendBounds(cr)
default:
cr.cols[startColIdx] = t.startTimes(arr)
cr.cols[stopColIdx] = t.stopTimes(arr)
cr.cols[timeColIdx] = arrow.NewInt(arr.Timestamps, t.alloc)
}
cr.cols[valueColIdx] = t.toArrowBuffer(arr.Values)
t.appendTags(cr)
return true
}
func (t *unsignedWindowSelectorTable) startTimes(arr *cursors.UnsignedArray) *array.Int64 {
start := arrow.NewIntBuilder(t.alloc)
start.Resize(arr.Len())
rangeStart := int64(t.bounds.Start)
for _, v := range arr.Timestamps {
if windowStart := storage.WindowStart(v, t.windowEvery, t.offset); windowStart < rangeStart {
start.Append(rangeStart)
} else {
start.Append(windowStart)
}
}
return start.NewInt64Array()
}
func (t *unsignedWindowSelectorTable) stopTimes(arr *cursors.UnsignedArray) *array.Int64 {
stop := arrow.NewIntBuilder(t.alloc)
stop.Resize(arr.Len())
rangeStop := int64(t.bounds.Stop)
for _, v := range arr.Timestamps {
if windowStop := storage.WindowStop(v, t.windowEvery, t.offset); windowStop > rangeStop {
stop.Append(rangeStop)
} else {
stop.Append(windowStop)
}
}
return stop.NewInt64Array()
}
// This table implementation may contain empty windows
// in addition to non-empty windows.
type unsignedEmptyWindowSelectorTable struct {
unsignedTable
arr *cursors.UnsignedArray
idx int
rangeStart int64
rangeStop int64
windowStart int64
windowStop int64
windowEvery int64
timeColumn string
}
func newUnsignedEmptyWindowSelectorTable(
done chan struct{},
cur cursors.UnsignedArrayCursor,
bounds execute.Bounds,
windowEvery int64,
offset int64,
timeColumn string,
key flux.GroupKey,
cols []flux.ColMeta,
tags models.Tags,
defs [][]byte,
cache *tagsCache,
alloc *memory.Allocator,
) *unsignedEmptyWindowSelectorTable {
rangeStart := int64(bounds.Start)
rangeStop := int64(bounds.Stop)
t := &unsignedEmptyWindowSelectorTable{
unsignedTable: unsignedTable{
table: newTable(done, bounds, key, cols, defs, cache, alloc),
cur: cur,
},
arr: cur.Next(),
rangeStart: rangeStart,
rangeStop: rangeStop,
windowStart: storage.WindowStart(rangeStart, windowEvery, offset),
windowStop: storage.WindowStop(rangeStart, windowEvery, offset),
windowEvery: windowEvery,
timeColumn: timeColumn,
}
t.readTags(tags)
t.init(t.advance)
return t
}
func (t *unsignedEmptyWindowSelectorTable) Do(f func(flux.ColReader) error) error {
return t.do(f, t.advance)
}
func (t *unsignedEmptyWindowSelectorTable) advance() bool {
if t.arr.Len() == 0 {
return false
}
values := t.arrowBuilder()
values.Resize(storage.MaxPointsPerBlock)
var cr *colReader
switch t.timeColumn {
case execute.DefaultStartColLabel:
start := t.startTimes(values)
cr = t.allocateBuffer(start.Len())
cr.cols[timeColIdx] = start
t.appendBounds(cr)
case execute.DefaultStopColLabel:
stop := t.stopTimes(values)
cr = t.allocateBuffer(stop.Len())
cr.cols[timeColIdx] = stop
t.appendBounds(cr)
default:
start, stop, time := t.startStopTimes(values)
cr = t.allocateBuffer(time.Len())
cr.cols[startColIdx] = start
cr.cols[stopColIdx] = stop
cr.cols[timeColIdx] = time
}
cr.cols[valueColIdx] = values.NewUint64Array()
t.appendTags(cr)
return true
}
func (t *unsignedEmptyWindowSelectorTable) startTimes(builder *array.Uint64Builder) *array.Int64 {
start := arrow.NewIntBuilder(t.alloc)
start.Resize(storage.MaxPointsPerBlock)
for t.windowStart < t.rangeStop {
// The first window should start at the
// beginning of the time range.
if t.windowStart < t.rangeStart {
start.Append(t.rangeStart)
} else {
start.Append(t.windowStart)
}
var v int64
if t.arr.Len() == 0 {
v = math.MaxInt64
} else {
v = t.arr.Timestamps[t.idx]
}
// If the current timestamp falls within the
// current window, append the value to the
// builder, otherwise append a null value.
if t.windowStart <= v && v < t.windowStop {
t.append(builder, t.arr.Values[t.idx])
t.idx++
} else {
builder.AppendNull()
}
t.windowStart += t.windowEvery
t.windowStop += t.windowEvery
// If the current array is non-empty and has
// been read in its entirety, call Next().
if t.arr.Len() > 0 && t.idx == t.arr.Len() {
t.arr = t.cur.Next()
t.idx = 0
}
if start.Len() == storage.MaxPointsPerBlock {
break
}
}
return start.NewInt64Array()
}
func (t *unsignedEmptyWindowSelectorTable) stopTimes(builder *array.Uint64Builder) *array.Int64 {
stop := arrow.NewIntBuilder(t.alloc)
stop.Resize(storage.MaxPointsPerBlock)
for t.windowStart < t.rangeStop {
// The last window should stop at the end of
// the time range.
if t.windowStop > t.rangeStop {
stop.Append(t.rangeStop)
} else {
stop.Append(t.windowStop)
}
var v int64
if t.arr.Len() == 0 {
v = math.MaxInt64
} else {
v = t.arr.Timestamps[t.idx]
}
// If the current timestamp falls within the
// current window, append the value to the
// builder, otherwise append a null value.
if t.windowStart <= v && v < t.windowStop {
t.append(builder, t.arr.Values[t.idx])
t.idx++
} else {
builder.AppendNull()
}
t.windowStart += t.windowEvery
t.windowStop += t.windowEvery
// If the current array is non-empty and has
// been read in its entirety, call Next().
if t.arr.Len() > 0 && t.idx == t.arr.Len() {
t.arr = t.cur.Next()
t.idx = 0
}
if stop.Len() == storage.MaxPointsPerBlock {
break
}
}
return stop.NewInt64Array()
}
func (t *unsignedEmptyWindowSelectorTable) startStopTimes(builder *array.Uint64Builder) (*array.Int64, *array.Int64, *array.Int64) {
start := arrow.NewIntBuilder(t.alloc)
start.Resize(storage.MaxPointsPerBlock)
stop := arrow.NewIntBuilder(t.alloc)
stop.Resize(storage.MaxPointsPerBlock)
time := arrow.NewIntBuilder(t.alloc)
time.Resize(storage.MaxPointsPerBlock)
for t.windowStart < t.rangeStop {
// The first window should start at the
// beginning of the time range.
if t.windowStart < t.rangeStart {
start.Append(t.rangeStart)
} else {
start.Append(t.windowStart)
}
// The last window should stop at the end of
// the time range.
if t.windowStop > t.rangeStop {
stop.Append(t.rangeStop)
} else {
stop.Append(t.windowStop)
}
var v int64
if t.arr.Len() == 0 {
v = math.MaxInt64
} else {
v = t.arr.Timestamps[t.idx]
}
// If the current timestamp falls within the
// current window, append the value to the
// builder, otherwise append a null value.
if t.windowStart <= v && v < t.windowStop {
time.Append(v)
t.append(builder, t.arr.Values[t.idx])
t.idx++
} else {
time.AppendNull()
builder.AppendNull()
}
t.windowStart += t.windowEvery
t.windowStop += t.windowEvery
// If the current array is non-empty and has
// been read in its entirety, call Next().
if t.arr.Len() > 0 && t.idx == t.arr.Len() {
t.arr = t.cur.Next()
t.idx = 0
}
if time.Len() == storage.MaxPointsPerBlock {
break
}
}
return start.NewInt64Array(), stop.NewInt64Array(), time.NewInt64Array()
}
// group table
type unsignedGroupTable struct {
table
mu sync.Mutex
gc storage.GroupCursor
cur cursors.UnsignedArrayCursor
}
func newUnsignedGroupTable(
done chan struct{},
gc storage.GroupCursor,
cur cursors.UnsignedArrayCursor,
bounds execute.Bounds,
key flux.GroupKey,
cols []flux.ColMeta,
tags models.Tags,
defs [][]byte,
cache *tagsCache,
alloc *memory.Allocator,
) *unsignedGroupTable {
t := &unsignedGroupTable{
table: newTable(done, bounds, key, cols, defs, cache, alloc),
gc: gc,
cur: cur,
}
t.readTags(tags)
t.init(t.advance)
return t
}
func (t *unsignedGroupTable) Close() {
t.mu.Lock()
if t.cur != nil {
t.cur.Close()
t.cur = nil
}
if t.gc != nil {
t.gc.Close()
t.gc = nil
}
t.mu.Unlock()
}
func (t *unsignedGroupTable) Do(f func(flux.ColReader) error) error {
return t.do(f, t.advance)
}
func (t *unsignedGroupTable) advance() bool {
RETRY:
a := t.cur.Next()
l := a.Len()
if l == 0 {
if t.advanceCursor() {
goto RETRY
}
return false
}
// Retrieve the buffer for the data to avoid allocating
// additional slices. If the buffer is still being used
// because the references were retained, then we will
// allocate a new buffer.
cr := t.allocateBuffer(l)
cr.cols[timeColIdx] = arrow.NewInt(a.Timestamps, t.alloc)
cr.cols[valueColIdx] = t.toArrowBuffer(a.Values)
t.appendTags(cr)
t.appendBounds(cr)
return true
}
func (t *unsignedGroupTable) advanceCursor() bool {
t.cur.Close()
t.cur = nil
for t.gc.Next() {
cur := t.gc.Cursor()
if cur == nil {
continue
}
if typedCur, ok := cur.(cursors.UnsignedArrayCursor); !ok {
// TODO(sgc): error or skip?
cur.Close()
t.err = &influxdb.Error{
Code: influxdb.EInvalid,
Err: &GroupCursorError{
typ: "unsigned",
cursor: cur,
},
}
return false
} else {
t.readTags(t.gc.Tags())
t.cur = typedCur
return true
}
}
return false
}
func (t *unsignedGroupTable) Statistics() cursors.CursorStats {
if t.cur == nil {
return cursors.CursorStats{}
}
cs := t.cur.Stats()
return cursors.CursorStats{
ScannedValues: cs.ScannedValues,
ScannedBytes: cs.ScannedBytes,
}
}
//
// *********** String ***********
//
type stringTable struct {
table
mu sync.Mutex
cur cursors.StringArrayCursor
alloc *memory.Allocator
}
func newStringTable(
done chan struct{},
cur cursors.StringArrayCursor,
bounds execute.Bounds,
key flux.GroupKey,
cols []flux.ColMeta,
tags models.Tags,
defs [][]byte,
cache *tagsCache,
alloc *memory.Allocator,
) *stringTable {
t := &stringTable{
table: newTable(done, bounds, key, cols, defs, cache, alloc),
cur: cur,
}
t.readTags(tags)
t.init(t.advance)
return t
}
func (t *stringTable) Close() {
t.mu.Lock()
if t.cur != nil {
t.cur.Close()
t.cur = nil
}
t.mu.Unlock()
}
func (t *stringTable) Statistics() cursors.CursorStats {
t.mu.Lock()
defer t.mu.Unlock()
cur := t.cur
if cur == nil {
return cursors.CursorStats{}
}
cs := cur.Stats()
return cursors.CursorStats{
ScannedValues: cs.ScannedValues,
ScannedBytes: cs.ScannedBytes,
}
}
func (t *stringTable) Do(f func(flux.ColReader) error) error {
return t.do(f, t.advance)
}
func (t *stringTable) advance() bool {
a := t.cur.Next()
l := a.Len()
if l == 0 {
return false
}
// Retrieve the buffer for the data to avoid allocating
// additional slices. If the buffer is still being used
// because the references were retained, then we will
// allocate a new buffer.
cr := t.allocateBuffer(l)
cr.cols[timeColIdx] = arrow.NewInt(a.Timestamps, t.alloc)
cr.cols[valueColIdx] = t.toArrowBuffer(a.Values)
t.appendTags(cr)
t.appendBounds(cr)
return true
}
// window table
type stringWindowTable struct {
stringTable
windowEvery int64
offset int64
arr *cursors.StringArray
nextTS int64
idxInArr int
createEmpty bool
timeColumn string
}
func newStringWindowTable(
done chan struct{},
cur cursors.StringArrayCursor,
bounds execute.Bounds,
every int64,
offset int64,
createEmpty bool,
timeColumn string,
key flux.GroupKey,
cols []flux.ColMeta,
tags models.Tags,
defs [][]byte,
cache *tagsCache,
alloc *memory.Allocator,
) *stringWindowTable {
t := &stringWindowTable{
stringTable: stringTable{
table: newTable(done, bounds, key, cols, defs, cache, alloc),
cur: cur,
},
windowEvery: every,
offset: offset,
createEmpty: createEmpty,
timeColumn: timeColumn,
}
if t.createEmpty {
start := int64(bounds.Start)
t.nextTS = storage.WindowStop(start, every, offset)
}
t.readTags(tags)
t.init(t.advance)
return t
}
func (t *stringWindowTable) Do(f func(flux.ColReader) error) error {
return t.do(f, t.advance)
}
// createNextBufferTimes will read the timestamps from the array
// cursor and construct the values for the next buffer.
func (t *stringWindowTable) createNextBufferTimes() (start, stop *array.Int64, ok bool) {
startB := arrow.NewIntBuilder(t.alloc)
stopB := arrow.NewIntBuilder(t.alloc)
if t.createEmpty {
// There are no more windows when the start time is greater
// than or equal to the stop time.
if startT := t.nextTS - t.windowEvery; startT >= int64(t.bounds.Stop) {
return nil, nil, false
}
// Create a buffer with the buffer size.
// TODO(jsternberg): Calculate the exact size with max points as the maximum.
startB.Resize(storage.MaxPointsPerBlock)
stopB.Resize(storage.MaxPointsPerBlock)
for ; ; t.nextTS += t.windowEvery {
startT, stopT := t.getWindowBoundsFor(t.nextTS)
if startT >= int64(t.bounds.Stop) {
break
}
startB.Append(startT)
stopB.Append(stopT)
}
start = startB.NewInt64Array()
stop = stopB.NewInt64Array()
return start, stop, true
}
// Retrieve the next buffer so we can copy the timestamps.
if !t.nextBuffer() {
return nil, nil, false
}
// Copy over the timestamps from the next buffer and adjust
// times for the boundaries.
startB.Resize(len(t.arr.Timestamps))
stopB.Resize(len(t.arr.Timestamps))
for _, stopT := range t.arr.Timestamps {
startT, stopT := t.getWindowBoundsFor(stopT)
startB.Append(startT)
stopB.Append(stopT)
}
start = startB.NewInt64Array()
stop = stopB.NewInt64Array()
return start, stop, true
}
func (t *stringWindowTable) getWindowBoundsFor(ts int64) (startT, stopT int64) {
startT, stopT = ts-t.windowEvery, ts
if startT < int64(t.bounds.Start) {
startT = int64(t.bounds.Start)
}
if stopT > int64(t.bounds.Stop) {
stopT = int64(t.bounds.Stop)
}
return startT, stopT
}
// nextAt will retrieve the next value that can be used with
// the given stop timestamp. If no values can be used with the timestamp,
// it will return the default value and false.
func (t *stringWindowTable) nextAt(ts int64) (v string, ok bool) {
if !t.nextBuffer() {
return
} else if !t.isInWindow(ts, t.arr.Timestamps[t.idxInArr]) {
return
}
v, ok = t.arr.Values[t.idxInArr], true
t.idxInArr++
return v, ok
}
// isInWindow will check if the given time at stop can be used within
// the window stop time for ts. The ts may be a truncated stop time
// because of a restricted boundary while stop will be the true
// stop time returned by storage.
func (t *stringWindowTable) isInWindow(ts int64, stop int64) bool {
// This method checks if the stop time is a valid stop time for
// that interval. This calculation is different from the calculation
// of the window itself. For example, for a 10 second window that
// starts at 20 seconds, we would include points between [20, 30).
// The stop time for this interval would be 30, but because the stop
// time can be truncated, valid stop times range from anywhere between
// (20, 30]. The storage engine will always produce 30 as the end time
// but we may have truncated the stop time because of the boundary
// and this is why we are checking for this range instead of checking
// if the two values are equal.
start := stop - t.windowEvery
return start < ts && ts <= stop
}
// nextBuffer will ensure the array cursor is filled
// and will return true if there is at least one value
// that can be read from it.
func (t *stringWindowTable) nextBuffer() bool {
// Discard the current array cursor if we have
// exceeded it.
if t.arr != nil && t.idxInArr >= t.arr.Len() {
t.arr = nil
}
// Retrieve the next array cursor if needed.
if t.arr == nil {
arr := t.cur.Next()
if arr.Len() == 0 {
return false
}
t.arr, t.idxInArr = arr, 0
}
return true
}
// appendValues will scan the timestamps and append values
// that match those timestamps from the buffer.
func (t *stringWindowTable) appendValues(intervals []int64, appendValue func(v string), appendNull func()) {
for i := 0; i < len(intervals); i++ {
if v, ok := t.nextAt(intervals[i]); ok {
appendValue(v)
continue
}
appendNull()
}
}
func (t *stringWindowTable) advance() bool {
if !t.nextBuffer() {
return false
}
// Create the timestamps for the next window.
start, stop, ok := t.createNextBufferTimes()
if !ok {
return false
}
values := t.mergeValues(stop.Int64Values())
// Retrieve the buffer for the data to avoid allocating
// additional slices. If the buffer is still being used
// because the references were retained, then we will
// allocate a new buffer.
cr := t.allocateBuffer(stop.Len())
if t.timeColumn != "" {
switch t.timeColumn {
case execute.DefaultStopColLabel:
cr.cols[timeColIdx] = stop
start.Release()
case execute.DefaultStartColLabel:
cr.cols[timeColIdx] = start
stop.Release()
}
cr.cols[valueColIdx] = values
t.appendBounds(cr)
} else {
cr.cols[startColIdx] = start
cr.cols[stopColIdx] = stop
cr.cols[windowedValueColIdx] = values
}
t.appendTags(cr)
return true
}
// This table implementation will not have any empty windows.
type stringWindowSelectorTable struct {
stringTable
windowEvery int64
offset int64
timeColumn string
}
func newStringWindowSelectorTable(
done chan struct{},
cur cursors.StringArrayCursor,
bounds execute.Bounds,
every int64,
offset int64,
timeColumn string,
key flux.GroupKey,
cols []flux.ColMeta,
tags models.Tags,
defs [][]byte,
cache *tagsCache,
alloc *memory.Allocator,
) *stringWindowSelectorTable {
t := &stringWindowSelectorTable{
stringTable: stringTable{
table: newTable(done, bounds, key, cols, defs, cache, alloc),
cur: cur,
},
windowEvery: every,
offset: offset,
timeColumn: timeColumn,
}
t.readTags(tags)
t.init(t.advance)
return t
}
func (t *stringWindowSelectorTable) Do(f func(flux.ColReader) error) error {
return t.do(f, t.advance)
}
func (t *stringWindowSelectorTable) advance() bool {
arr := t.cur.Next()
if arr.Len() == 0 {
return false
}
cr := t.allocateBuffer(arr.Len())
switch t.timeColumn {
case execute.DefaultStartColLabel:
cr.cols[timeColIdx] = t.startTimes(arr)
t.appendBounds(cr)
case execute.DefaultStopColLabel:
cr.cols[timeColIdx] = t.stopTimes(arr)
t.appendBounds(cr)
default:
cr.cols[startColIdx] = t.startTimes(arr)
cr.cols[stopColIdx] = t.stopTimes(arr)
cr.cols[timeColIdx] = arrow.NewInt(arr.Timestamps, t.alloc)
}
cr.cols[valueColIdx] = t.toArrowBuffer(arr.Values)
t.appendTags(cr)
return true
}
func (t *stringWindowSelectorTable) startTimes(arr *cursors.StringArray) *array.Int64 {
start := arrow.NewIntBuilder(t.alloc)
start.Resize(arr.Len())
rangeStart := int64(t.bounds.Start)
for _, v := range arr.Timestamps {
if windowStart := storage.WindowStart(v, t.windowEvery, t.offset); windowStart < rangeStart {
start.Append(rangeStart)
} else {
start.Append(windowStart)
}
}
return start.NewInt64Array()
}
func (t *stringWindowSelectorTable) stopTimes(arr *cursors.StringArray) *array.Int64 {
stop := arrow.NewIntBuilder(t.alloc)
stop.Resize(arr.Len())
rangeStop := int64(t.bounds.Stop)
for _, v := range arr.Timestamps {
if windowStop := storage.WindowStop(v, t.windowEvery, t.offset); windowStop > rangeStop {
stop.Append(rangeStop)
} else {
stop.Append(windowStop)
}
}
return stop.NewInt64Array()
}
// This table implementation may contain empty windows
// in addition to non-empty windows.
type stringEmptyWindowSelectorTable struct {
stringTable
arr *cursors.StringArray
idx int
rangeStart int64
rangeStop int64
windowStart int64
windowStop int64
windowEvery int64
timeColumn string
}
func newStringEmptyWindowSelectorTable(
done chan struct{},
cur cursors.StringArrayCursor,
bounds execute.Bounds,
windowEvery int64,
offset int64,
timeColumn string,
key flux.GroupKey,
cols []flux.ColMeta,
tags models.Tags,
defs [][]byte,
cache *tagsCache,
alloc *memory.Allocator,
) *stringEmptyWindowSelectorTable {
rangeStart := int64(bounds.Start)
rangeStop := int64(bounds.Stop)
t := &stringEmptyWindowSelectorTable{
stringTable: stringTable{
table: newTable(done, bounds, key, cols, defs, cache, alloc),
cur: cur,
},
arr: cur.Next(),
rangeStart: rangeStart,
rangeStop: rangeStop,
windowStart: storage.WindowStart(rangeStart, windowEvery, offset),
windowStop: storage.WindowStop(rangeStart, windowEvery, offset),
windowEvery: windowEvery,
timeColumn: timeColumn,
}
t.readTags(tags)
t.init(t.advance)
return t
}
func (t *stringEmptyWindowSelectorTable) Do(f func(flux.ColReader) error) error {
return t.do(f, t.advance)
}
func (t *stringEmptyWindowSelectorTable) advance() bool {
if t.arr.Len() == 0 {
return false
}
values := t.arrowBuilder()
values.Resize(storage.MaxPointsPerBlock)
var cr *colReader
switch t.timeColumn {
case execute.DefaultStartColLabel:
start := t.startTimes(values)
cr = t.allocateBuffer(start.Len())
cr.cols[timeColIdx] = start
t.appendBounds(cr)
case execute.DefaultStopColLabel:
stop := t.stopTimes(values)
cr = t.allocateBuffer(stop.Len())
cr.cols[timeColIdx] = stop
t.appendBounds(cr)
default:
start, stop, time := t.startStopTimes(values)
cr = t.allocateBuffer(time.Len())
cr.cols[startColIdx] = start
cr.cols[stopColIdx] = stop
cr.cols[timeColIdx] = time
}
cr.cols[valueColIdx] = values.NewBinaryArray()
t.appendTags(cr)
return true
}
func (t *stringEmptyWindowSelectorTable) startTimes(builder *array.BinaryBuilder) *array.Int64 {
start := arrow.NewIntBuilder(t.alloc)
start.Resize(storage.MaxPointsPerBlock)
for t.windowStart < t.rangeStop {
// The first window should start at the
// beginning of the time range.
if t.windowStart < t.rangeStart {
start.Append(t.rangeStart)
} else {
start.Append(t.windowStart)
}
var v int64
if t.arr.Len() == 0 {
v = math.MaxInt64
} else {
v = t.arr.Timestamps[t.idx]
}
// If the current timestamp falls within the
// current window, append the value to the
// builder, otherwise append a null value.
if t.windowStart <= v && v < t.windowStop {
t.append(builder, t.arr.Values[t.idx])
t.idx++
} else {
builder.AppendNull()
}
t.windowStart += t.windowEvery
t.windowStop += t.windowEvery
// If the current array is non-empty and has
// been read in its entirety, call Next().
if t.arr.Len() > 0 && t.idx == t.arr.Len() {
t.arr = t.cur.Next()
t.idx = 0
}
if start.Len() == storage.MaxPointsPerBlock {
break
}
}
return start.NewInt64Array()
}
func (t *stringEmptyWindowSelectorTable) stopTimes(builder *array.BinaryBuilder) *array.Int64 {
stop := arrow.NewIntBuilder(t.alloc)
stop.Resize(storage.MaxPointsPerBlock)
for t.windowStart < t.rangeStop {
// The last window should stop at the end of
// the time range.
if t.windowStop > t.rangeStop {
stop.Append(t.rangeStop)
} else {
stop.Append(t.windowStop)
}
var v int64
if t.arr.Len() == 0 {
v = math.MaxInt64
} else {
v = t.arr.Timestamps[t.idx]
}
// If the current timestamp falls within the
// current window, append the value to the
// builder, otherwise append a null value.
if t.windowStart <= v && v < t.windowStop {
t.append(builder, t.arr.Values[t.idx])
t.idx++
} else {
builder.AppendNull()
}
t.windowStart += t.windowEvery
t.windowStop += t.windowEvery
// If the current array is non-empty and has
// been read in its entirety, call Next().
if t.arr.Len() > 0 && t.idx == t.arr.Len() {
t.arr = t.cur.Next()
t.idx = 0
}
if stop.Len() == storage.MaxPointsPerBlock {
break
}
}
return stop.NewInt64Array()
}
func (t *stringEmptyWindowSelectorTable) startStopTimes(builder *array.BinaryBuilder) (*array.Int64, *array.Int64, *array.Int64) {
start := arrow.NewIntBuilder(t.alloc)
start.Resize(storage.MaxPointsPerBlock)
stop := arrow.NewIntBuilder(t.alloc)
stop.Resize(storage.MaxPointsPerBlock)
time := arrow.NewIntBuilder(t.alloc)
time.Resize(storage.MaxPointsPerBlock)
for t.windowStart < t.rangeStop {
// The first window should start at the
// beginning of the time range.
if t.windowStart < t.rangeStart {
start.Append(t.rangeStart)
} else {
start.Append(t.windowStart)
}
// The last window should stop at the end of
// the time range.
if t.windowStop > t.rangeStop {
stop.Append(t.rangeStop)
} else {
stop.Append(t.windowStop)
}
var v int64
if t.arr.Len() == 0 {
v = math.MaxInt64
} else {
v = t.arr.Timestamps[t.idx]
}
// If the current timestamp falls within the
// current window, append the value to the
// builder, otherwise append a null value.
if t.windowStart <= v && v < t.windowStop {
time.Append(v)
t.append(builder, t.arr.Values[t.idx])
t.idx++
} else {
time.AppendNull()
builder.AppendNull()
}
t.windowStart += t.windowEvery
t.windowStop += t.windowEvery
// If the current array is non-empty and has
// been read in its entirety, call Next().
if t.arr.Len() > 0 && t.idx == t.arr.Len() {
t.arr = t.cur.Next()
t.idx = 0
}
if time.Len() == storage.MaxPointsPerBlock {
break
}
}
return start.NewInt64Array(), stop.NewInt64Array(), time.NewInt64Array()
}
// group table
type stringGroupTable struct {
table
mu sync.Mutex
gc storage.GroupCursor
cur cursors.StringArrayCursor
}
func newStringGroupTable(
done chan struct{},
gc storage.GroupCursor,
cur cursors.StringArrayCursor,
bounds execute.Bounds,
key flux.GroupKey,
cols []flux.ColMeta,
tags models.Tags,
defs [][]byte,
cache *tagsCache,
alloc *memory.Allocator,
) *stringGroupTable {
t := &stringGroupTable{
table: newTable(done, bounds, key, cols, defs, cache, alloc),
gc: gc,
cur: cur,
}
t.readTags(tags)
t.init(t.advance)
return t
}
func (t *stringGroupTable) Close() {
t.mu.Lock()
if t.cur != nil {
t.cur.Close()
t.cur = nil
}
if t.gc != nil {
t.gc.Close()
t.gc = nil
}
t.mu.Unlock()
}
func (t *stringGroupTable) Do(f func(flux.ColReader) error) error {
return t.do(f, t.advance)
}
func (t *stringGroupTable) advance() bool {
RETRY:
a := t.cur.Next()
l := a.Len()
if l == 0 {
if t.advanceCursor() {
goto RETRY
}
return false
}
// Retrieve the buffer for the data to avoid allocating
// additional slices. If the buffer is still being used
// because the references were retained, then we will
// allocate a new buffer.
cr := t.allocateBuffer(l)
cr.cols[timeColIdx] = arrow.NewInt(a.Timestamps, t.alloc)
cr.cols[valueColIdx] = t.toArrowBuffer(a.Values)
t.appendTags(cr)
t.appendBounds(cr)
return true
}
func (t *stringGroupTable) advanceCursor() bool {
t.cur.Close()
t.cur = nil
for t.gc.Next() {
cur := t.gc.Cursor()
if cur == nil {
continue
}
if typedCur, ok := cur.(cursors.StringArrayCursor); !ok {
// TODO(sgc): error or skip?
cur.Close()
t.err = &influxdb.Error{
Code: influxdb.EInvalid,
Err: &GroupCursorError{
typ: "string",
cursor: cur,
},
}
return false
} else {
t.readTags(t.gc.Tags())
t.cur = typedCur
return true
}
}
return false
}
func (t *stringGroupTable) Statistics() cursors.CursorStats {
if t.cur == nil {
return cursors.CursorStats{}
}
cs := t.cur.Stats()
return cursors.CursorStats{
ScannedValues: cs.ScannedValues,
ScannedBytes: cs.ScannedBytes,
}
}
//
// *********** Boolean ***********
//
type booleanTable struct {
table
mu sync.Mutex
cur cursors.BooleanArrayCursor
alloc *memory.Allocator
}
func newBooleanTable(
done chan struct{},
cur cursors.BooleanArrayCursor,
bounds execute.Bounds,
key flux.GroupKey,
cols []flux.ColMeta,
tags models.Tags,
defs [][]byte,
cache *tagsCache,
alloc *memory.Allocator,
) *booleanTable {
t := &booleanTable{
table: newTable(done, bounds, key, cols, defs, cache, alloc),
cur: cur,
}
t.readTags(tags)
t.init(t.advance)
return t
}
func (t *booleanTable) Close() {
t.mu.Lock()
if t.cur != nil {
t.cur.Close()
t.cur = nil
}
t.mu.Unlock()
}
func (t *booleanTable) Statistics() cursors.CursorStats {
t.mu.Lock()
defer t.mu.Unlock()
cur := t.cur
if cur == nil {
return cursors.CursorStats{}
}
cs := cur.Stats()
return cursors.CursorStats{
ScannedValues: cs.ScannedValues,
ScannedBytes: cs.ScannedBytes,
}
}
func (t *booleanTable) Do(f func(flux.ColReader) error) error {
return t.do(f, t.advance)
}
func (t *booleanTable) advance() bool {
a := t.cur.Next()
l := a.Len()
if l == 0 {
return false
}
// Retrieve the buffer for the data to avoid allocating
// additional slices. If the buffer is still being used
// because the references were retained, then we will
// allocate a new buffer.
cr := t.allocateBuffer(l)
cr.cols[timeColIdx] = arrow.NewInt(a.Timestamps, t.alloc)
cr.cols[valueColIdx] = t.toArrowBuffer(a.Values)
t.appendTags(cr)
t.appendBounds(cr)
return true
}
// window table
type booleanWindowTable struct {
booleanTable
windowEvery int64
offset int64
arr *cursors.BooleanArray
nextTS int64
idxInArr int
createEmpty bool
timeColumn string
}
func newBooleanWindowTable(
done chan struct{},
cur cursors.BooleanArrayCursor,
bounds execute.Bounds,
every int64,
offset int64,
createEmpty bool,
timeColumn string,
key flux.GroupKey,
cols []flux.ColMeta,
tags models.Tags,
defs [][]byte,
cache *tagsCache,
alloc *memory.Allocator,
) *booleanWindowTable {
t := &booleanWindowTable{
booleanTable: booleanTable{
table: newTable(done, bounds, key, cols, defs, cache, alloc),
cur: cur,
},
windowEvery: every,
offset: offset,
createEmpty: createEmpty,
timeColumn: timeColumn,
}
if t.createEmpty {
start := int64(bounds.Start)
t.nextTS = storage.WindowStop(start, every, offset)
}
t.readTags(tags)
t.init(t.advance)
return t
}
func (t *booleanWindowTable) Do(f func(flux.ColReader) error) error {
return t.do(f, t.advance)
}
// createNextBufferTimes will read the timestamps from the array
// cursor and construct the values for the next buffer.
func (t *booleanWindowTable) createNextBufferTimes() (start, stop *array.Int64, ok bool) {
startB := arrow.NewIntBuilder(t.alloc)
stopB := arrow.NewIntBuilder(t.alloc)
if t.createEmpty {
// There are no more windows when the start time is greater
// than or equal to the stop time.
if startT := t.nextTS - t.windowEvery; startT >= int64(t.bounds.Stop) {
return nil, nil, false
}
// Create a buffer with the buffer size.
// TODO(jsternberg): Calculate the exact size with max points as the maximum.
startB.Resize(storage.MaxPointsPerBlock)
stopB.Resize(storage.MaxPointsPerBlock)
for ; ; t.nextTS += t.windowEvery {
startT, stopT := t.getWindowBoundsFor(t.nextTS)
if startT >= int64(t.bounds.Stop) {
break
}
startB.Append(startT)
stopB.Append(stopT)
}
start = startB.NewInt64Array()
stop = stopB.NewInt64Array()
return start, stop, true
}
// Retrieve the next buffer so we can copy the timestamps.
if !t.nextBuffer() {
return nil, nil, false
}
// Copy over the timestamps from the next buffer and adjust
// times for the boundaries.
startB.Resize(len(t.arr.Timestamps))
stopB.Resize(len(t.arr.Timestamps))
for _, stopT := range t.arr.Timestamps {
startT, stopT := t.getWindowBoundsFor(stopT)
startB.Append(startT)
stopB.Append(stopT)
}
start = startB.NewInt64Array()
stop = stopB.NewInt64Array()
return start, stop, true
}
func (t *booleanWindowTable) getWindowBoundsFor(ts int64) (startT, stopT int64) {
startT, stopT = ts-t.windowEvery, ts
if startT < int64(t.bounds.Start) {
startT = int64(t.bounds.Start)
}
if stopT > int64(t.bounds.Stop) {
stopT = int64(t.bounds.Stop)
}
return startT, stopT
}
// nextAt will retrieve the next value that can be used with
// the given stop timestamp. If no values can be used with the timestamp,
// it will return the default value and false.
func (t *booleanWindowTable) nextAt(ts int64) (v bool, ok bool) {
if !t.nextBuffer() {
return
} else if !t.isInWindow(ts, t.arr.Timestamps[t.idxInArr]) {
return
}
v, ok = t.arr.Values[t.idxInArr], true
t.idxInArr++
return v, ok
}
// isInWindow will check if the given time at stop can be used within
// the window stop time for ts. The ts may be a truncated stop time
// because of a restricted boundary while stop will be the true
// stop time returned by storage.
func (t *booleanWindowTable) isInWindow(ts int64, stop int64) bool {
// This method checks if the stop time is a valid stop time for
// that interval. This calculation is different from the calculation
// of the window itself. For example, for a 10 second window that
// starts at 20 seconds, we would include points between [20, 30).
// The stop time for this interval would be 30, but because the stop
// time can be truncated, valid stop times range from anywhere between
// (20, 30]. The storage engine will always produce 30 as the end time
// but we may have truncated the stop time because of the boundary
// and this is why we are checking for this range instead of checking
// if the two values are equal.
start := stop - t.windowEvery
return start < ts && ts <= stop
}
// nextBuffer will ensure the array cursor is filled
// and will return true if there is at least one value
// that can be read from it.
func (t *booleanWindowTable) nextBuffer() bool {
// Discard the current array cursor if we have
// exceeded it.
if t.arr != nil && t.idxInArr >= t.arr.Len() {
t.arr = nil
}
// Retrieve the next array cursor if needed.
if t.arr == nil {
arr := t.cur.Next()
if arr.Len() == 0 {
return false
}
t.arr, t.idxInArr = arr, 0
}
return true
}
// appendValues will scan the timestamps and append values
// that match those timestamps from the buffer.
func (t *booleanWindowTable) appendValues(intervals []int64, appendValue func(v bool), appendNull func()) {
for i := 0; i < len(intervals); i++ {
if v, ok := t.nextAt(intervals[i]); ok {
appendValue(v)
continue
}
appendNull()
}
}
func (t *booleanWindowTable) advance() bool {
if !t.nextBuffer() {
return false
}
// Create the timestamps for the next window.
start, stop, ok := t.createNextBufferTimes()
if !ok {
return false
}
values := t.mergeValues(stop.Int64Values())
// Retrieve the buffer for the data to avoid allocating
// additional slices. If the buffer is still being used
// because the references were retained, then we will
// allocate a new buffer.
cr := t.allocateBuffer(stop.Len())
if t.timeColumn != "" {
switch t.timeColumn {
case execute.DefaultStopColLabel:
cr.cols[timeColIdx] = stop
start.Release()
case execute.DefaultStartColLabel:
cr.cols[timeColIdx] = start
stop.Release()
}
cr.cols[valueColIdx] = values
t.appendBounds(cr)
} else {
cr.cols[startColIdx] = start
cr.cols[stopColIdx] = stop
cr.cols[windowedValueColIdx] = values
}
t.appendTags(cr)
return true
}
// This table implementation will not have any empty windows.
type booleanWindowSelectorTable struct {
booleanTable
windowEvery int64
offset int64
timeColumn string
}
func newBooleanWindowSelectorTable(
done chan struct{},
cur cursors.BooleanArrayCursor,
bounds execute.Bounds,
every int64,
offset int64,
timeColumn string,
key flux.GroupKey,
cols []flux.ColMeta,
tags models.Tags,
defs [][]byte,
cache *tagsCache,
alloc *memory.Allocator,
) *booleanWindowSelectorTable {
t := &booleanWindowSelectorTable{
booleanTable: booleanTable{
table: newTable(done, bounds, key, cols, defs, cache, alloc),
cur: cur,
},
windowEvery: every,
offset: offset,
timeColumn: timeColumn,
}
t.readTags(tags)
t.init(t.advance)
return t
}
func (t *booleanWindowSelectorTable) Do(f func(flux.ColReader) error) error {
return t.do(f, t.advance)
}
func (t *booleanWindowSelectorTable) advance() bool {
arr := t.cur.Next()
if arr.Len() == 0 {
return false
}
cr := t.allocateBuffer(arr.Len())
switch t.timeColumn {
case execute.DefaultStartColLabel:
cr.cols[timeColIdx] = t.startTimes(arr)
t.appendBounds(cr)
case execute.DefaultStopColLabel:
cr.cols[timeColIdx] = t.stopTimes(arr)
t.appendBounds(cr)
default:
cr.cols[startColIdx] = t.startTimes(arr)
cr.cols[stopColIdx] = t.stopTimes(arr)
cr.cols[timeColIdx] = arrow.NewInt(arr.Timestamps, t.alloc)
}
cr.cols[valueColIdx] = t.toArrowBuffer(arr.Values)
t.appendTags(cr)
return true
}
func (t *booleanWindowSelectorTable) startTimes(arr *cursors.BooleanArray) *array.Int64 {
start := arrow.NewIntBuilder(t.alloc)
start.Resize(arr.Len())
rangeStart := int64(t.bounds.Start)
for _, v := range arr.Timestamps {
if windowStart := storage.WindowStart(v, t.windowEvery, t.offset); windowStart < rangeStart {
start.Append(rangeStart)
} else {
start.Append(windowStart)
}
}
return start.NewInt64Array()
}
func (t *booleanWindowSelectorTable) stopTimes(arr *cursors.BooleanArray) *array.Int64 {
stop := arrow.NewIntBuilder(t.alloc)
stop.Resize(arr.Len())
rangeStop := int64(t.bounds.Stop)
for _, v := range arr.Timestamps {
if windowStop := storage.WindowStop(v, t.windowEvery, t.offset); windowStop > rangeStop {
stop.Append(rangeStop)
} else {
stop.Append(windowStop)
}
}
return stop.NewInt64Array()
}
// This table implementation may contain empty windows
// in addition to non-empty windows.
type booleanEmptyWindowSelectorTable struct {
booleanTable
arr *cursors.BooleanArray
idx int
rangeStart int64
rangeStop int64
windowStart int64
windowStop int64
windowEvery int64
timeColumn string
}
func newBooleanEmptyWindowSelectorTable(
done chan struct{},
cur cursors.BooleanArrayCursor,
bounds execute.Bounds,
windowEvery int64,
offset int64,
timeColumn string,
key flux.GroupKey,
cols []flux.ColMeta,
tags models.Tags,
defs [][]byte,
cache *tagsCache,
alloc *memory.Allocator,
) *booleanEmptyWindowSelectorTable {
rangeStart := int64(bounds.Start)
rangeStop := int64(bounds.Stop)
t := &booleanEmptyWindowSelectorTable{
booleanTable: booleanTable{
table: newTable(done, bounds, key, cols, defs, cache, alloc),
cur: cur,
},
arr: cur.Next(),
rangeStart: rangeStart,
rangeStop: rangeStop,
windowStart: storage.WindowStart(rangeStart, windowEvery, offset),
windowStop: storage.WindowStop(rangeStart, windowEvery, offset),
windowEvery: windowEvery,
timeColumn: timeColumn,
}
t.readTags(tags)
t.init(t.advance)
return t
}
func (t *booleanEmptyWindowSelectorTable) Do(f func(flux.ColReader) error) error {
return t.do(f, t.advance)
}
func (t *booleanEmptyWindowSelectorTable) advance() bool {
if t.arr.Len() == 0 {
return false
}
values := t.arrowBuilder()
values.Resize(storage.MaxPointsPerBlock)
var cr *colReader
switch t.timeColumn {
case execute.DefaultStartColLabel:
start := t.startTimes(values)
cr = t.allocateBuffer(start.Len())
cr.cols[timeColIdx] = start
t.appendBounds(cr)
case execute.DefaultStopColLabel:
stop := t.stopTimes(values)
cr = t.allocateBuffer(stop.Len())
cr.cols[timeColIdx] = stop
t.appendBounds(cr)
default:
start, stop, time := t.startStopTimes(values)
cr = t.allocateBuffer(time.Len())
cr.cols[startColIdx] = start
cr.cols[stopColIdx] = stop
cr.cols[timeColIdx] = time
}
cr.cols[valueColIdx] = values.NewBooleanArray()
t.appendTags(cr)
return true
}
func (t *booleanEmptyWindowSelectorTable) startTimes(builder *array.BooleanBuilder) *array.Int64 {
start := arrow.NewIntBuilder(t.alloc)
start.Resize(storage.MaxPointsPerBlock)
for t.windowStart < t.rangeStop {
// The first window should start at the
// beginning of the time range.
if t.windowStart < t.rangeStart {
start.Append(t.rangeStart)
} else {
start.Append(t.windowStart)
}
var v int64
if t.arr.Len() == 0 {
v = math.MaxInt64
} else {
v = t.arr.Timestamps[t.idx]
}
// If the current timestamp falls within the
// current window, append the value to the
// builder, otherwise append a null value.
if t.windowStart <= v && v < t.windowStop {
t.append(builder, t.arr.Values[t.idx])
t.idx++
} else {
builder.AppendNull()
}
t.windowStart += t.windowEvery
t.windowStop += t.windowEvery
// If the current array is non-empty and has
// been read in its entirety, call Next().
if t.arr.Len() > 0 && t.idx == t.arr.Len() {
t.arr = t.cur.Next()
t.idx = 0
}
if start.Len() == storage.MaxPointsPerBlock {
break
}
}
return start.NewInt64Array()
}
func (t *booleanEmptyWindowSelectorTable) stopTimes(builder *array.BooleanBuilder) *array.Int64 {
stop := arrow.NewIntBuilder(t.alloc)
stop.Resize(storage.MaxPointsPerBlock)
for t.windowStart < t.rangeStop {
// The last window should stop at the end of
// the time range.
if t.windowStop > t.rangeStop {
stop.Append(t.rangeStop)
} else {
stop.Append(t.windowStop)
}
var v int64
if t.arr.Len() == 0 {
v = math.MaxInt64
} else {
v = t.arr.Timestamps[t.idx]
}
// If the current timestamp falls within the
// current window, append the value to the
// builder, otherwise append a null value.
if t.windowStart <= v && v < t.windowStop {
t.append(builder, t.arr.Values[t.idx])
t.idx++
} else {
builder.AppendNull()
}
t.windowStart += t.windowEvery
t.windowStop += t.windowEvery
// If the current array is non-empty and has
// been read in its entirety, call Next().
if t.arr.Len() > 0 && t.idx == t.arr.Len() {
t.arr = t.cur.Next()
t.idx = 0
}
if stop.Len() == storage.MaxPointsPerBlock {
break
}
}
return stop.NewInt64Array()
}
func (t *booleanEmptyWindowSelectorTable) startStopTimes(builder *array.BooleanBuilder) (*array.Int64, *array.Int64, *array.Int64) {
start := arrow.NewIntBuilder(t.alloc)
start.Resize(storage.MaxPointsPerBlock)
stop := arrow.NewIntBuilder(t.alloc)
stop.Resize(storage.MaxPointsPerBlock)
time := arrow.NewIntBuilder(t.alloc)
time.Resize(storage.MaxPointsPerBlock)
for t.windowStart < t.rangeStop {
// The first window should start at the
// beginning of the time range.
if t.windowStart < t.rangeStart {
start.Append(t.rangeStart)
} else {
start.Append(t.windowStart)
}
// The last window should stop at the end of
// the time range.
if t.windowStop > t.rangeStop {
stop.Append(t.rangeStop)
} else {
stop.Append(t.windowStop)
}
var v int64
if t.arr.Len() == 0 {
v = math.MaxInt64
} else {
v = t.arr.Timestamps[t.idx]
}
// If the current timestamp falls within the
// current window, append the value to the
// builder, otherwise append a null value.
if t.windowStart <= v && v < t.windowStop {
time.Append(v)
t.append(builder, t.arr.Values[t.idx])
t.idx++
} else {
time.AppendNull()
builder.AppendNull()
}
t.windowStart += t.windowEvery
t.windowStop += t.windowEvery
// If the current array is non-empty and has
// been read in its entirety, call Next().
if t.arr.Len() > 0 && t.idx == t.arr.Len() {
t.arr = t.cur.Next()
t.idx = 0
}
if time.Len() == storage.MaxPointsPerBlock {
break
}
}
return start.NewInt64Array(), stop.NewInt64Array(), time.NewInt64Array()
}
// group table
type booleanGroupTable struct {
table
mu sync.Mutex
gc storage.GroupCursor
cur cursors.BooleanArrayCursor
}
func newBooleanGroupTable(
done chan struct{},
gc storage.GroupCursor,
cur cursors.BooleanArrayCursor,
bounds execute.Bounds,
key flux.GroupKey,
cols []flux.ColMeta,
tags models.Tags,
defs [][]byte,
cache *tagsCache,
alloc *memory.Allocator,
) *booleanGroupTable {
t := &booleanGroupTable{
table: newTable(done, bounds, key, cols, defs, cache, alloc),
gc: gc,
cur: cur,
}
t.readTags(tags)
t.init(t.advance)
return t
}
func (t *booleanGroupTable) Close() {
t.mu.Lock()
if t.cur != nil {
t.cur.Close()
t.cur = nil
}
if t.gc != nil {
t.gc.Close()
t.gc = nil
}
t.mu.Unlock()
}
func (t *booleanGroupTable) Do(f func(flux.ColReader) error) error {
return t.do(f, t.advance)
}
func (t *booleanGroupTable) advance() bool {
RETRY:
a := t.cur.Next()
l := a.Len()
if l == 0 {
if t.advanceCursor() {
goto RETRY
}
return false
}
// Retrieve the buffer for the data to avoid allocating
// additional slices. If the buffer is still being used
// because the references were retained, then we will
// allocate a new buffer.
cr := t.allocateBuffer(l)
cr.cols[timeColIdx] = arrow.NewInt(a.Timestamps, t.alloc)
cr.cols[valueColIdx] = t.toArrowBuffer(a.Values)
t.appendTags(cr)
t.appendBounds(cr)
return true
}
func (t *booleanGroupTable) advanceCursor() bool {
t.cur.Close()
t.cur = nil
for t.gc.Next() {
cur := t.gc.Cursor()
if cur == nil {
continue
}
if typedCur, ok := cur.(cursors.BooleanArrayCursor); !ok {
// TODO(sgc): error or skip?
cur.Close()
t.err = &influxdb.Error{
Code: influxdb.EInvalid,
Err: &GroupCursorError{
typ: "boolean",
cursor: cur,
},
}
return false
} else {
t.readTags(t.gc.Tags())
t.cur = typedCur
return true
}
}
return false
}
func (t *booleanGroupTable) Statistics() cursors.CursorStats {
if t.cur == nil {
return cursors.CursorStats{}
}
cs := t.cur.Stats()
return cursors.CursorStats{
ScannedValues: cs.ScannedValues,
ScannedBytes: cs.ScannedBytes,
}
}