393 lines
9.7 KiB
Go
393 lines
9.7 KiB
Go
package functions
|
|
|
|
import (
|
|
"fmt"
|
|
"math"
|
|
|
|
"github.com/influxdata/platform/query"
|
|
"github.com/influxdata/platform/query/execute"
|
|
"github.com/influxdata/platform/query/plan"
|
|
"github.com/influxdata/platform/query/semantic"
|
|
"github.com/influxdata/platform/query/values"
|
|
"github.com/pkg/errors"
|
|
)
|
|
|
|
const WindowKind = "window"
|
|
|
|
type WindowOpSpec struct {
|
|
Every query.Duration `json:"every"`
|
|
Period query.Duration `json:"period"`
|
|
Start query.Time `json:"start"`
|
|
Round query.Duration `json:"round"`
|
|
Triggering query.TriggerSpec `json:"triggering"`
|
|
TimeCol string `json:"time_col"`
|
|
StopColLabel string `json:"stop_col_label"`
|
|
StartColLabel string `json:"start_col_label"`
|
|
}
|
|
|
|
var infinityVar = values.NewDurationValue(math.MaxInt64)
|
|
|
|
var windowSignature = query.DefaultFunctionSignature()
|
|
|
|
func init() {
|
|
windowSignature.Params["every"] = semantic.Duration
|
|
windowSignature.Params["period"] = semantic.Duration
|
|
windowSignature.Params["round"] = semantic.Duration
|
|
windowSignature.Params["start"] = semantic.Time
|
|
|
|
query.RegisterFunction(WindowKind, createWindowOpSpec, windowSignature)
|
|
query.RegisterOpSpec(WindowKind, newWindowOp)
|
|
query.RegisterBuiltInValue("inf", infinityVar)
|
|
plan.RegisterProcedureSpec(WindowKind, newWindowProcedure, WindowKind)
|
|
execute.RegisterTransformation(WindowKind, createWindowTransformation)
|
|
}
|
|
|
|
func createWindowOpSpec(args query.Arguments, a *query.Administration) (query.OperationSpec, error) {
|
|
if err := a.AddParentFromArgs(args); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
spec := new(WindowOpSpec)
|
|
every, everySet, err := args.GetDuration("every")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if everySet {
|
|
spec.Every = query.Duration(every)
|
|
}
|
|
period, periodSet, err := args.GetDuration("period")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if periodSet {
|
|
spec.Period = period
|
|
}
|
|
if round, ok, err := args.GetDuration("round"); err != nil {
|
|
return nil, err
|
|
} else if ok {
|
|
spec.Round = round
|
|
}
|
|
if start, ok, err := args.GetTime("start"); err != nil {
|
|
return nil, err
|
|
} else if ok {
|
|
spec.Start = start
|
|
}
|
|
|
|
if !everySet && !periodSet {
|
|
return nil, errors.New(`window function requires at least one of "every" or "period" to be set`)
|
|
}
|
|
|
|
if label, ok, err := args.GetString("timeCol"); err != nil {
|
|
return nil, err
|
|
} else if ok {
|
|
spec.TimeCol = label
|
|
} else {
|
|
spec.TimeCol = execute.DefaultTimeColLabel
|
|
}
|
|
if label, ok, err := args.GetString("startColLabel"); err != nil {
|
|
return nil, err
|
|
} else if ok {
|
|
spec.StartColLabel = label
|
|
} else {
|
|
spec.StartColLabel = execute.DefaultStartColLabel
|
|
}
|
|
if label, ok, err := args.GetString("stopColLabel"); err != nil {
|
|
return nil, err
|
|
} else if ok {
|
|
spec.StopColLabel = label
|
|
} else {
|
|
spec.StopColLabel = execute.DefaultStopColLabel
|
|
}
|
|
|
|
// Apply defaults
|
|
if !everySet {
|
|
spec.Every = spec.Period
|
|
}
|
|
if !periodSet {
|
|
spec.Period = spec.Every
|
|
}
|
|
return spec, nil
|
|
}
|
|
|
|
func newWindowOp() query.OperationSpec {
|
|
return new(WindowOpSpec)
|
|
}
|
|
|
|
func (s *WindowOpSpec) Kind() query.OperationKind {
|
|
return WindowKind
|
|
}
|
|
|
|
type WindowProcedureSpec struct {
|
|
Window plan.WindowSpec
|
|
Triggering query.TriggerSpec
|
|
TimeCol,
|
|
StartColLabel,
|
|
StopColLabel string
|
|
}
|
|
|
|
func newWindowProcedure(qs query.OperationSpec, pa plan.Administration) (plan.ProcedureSpec, error) {
|
|
s, ok := qs.(*WindowOpSpec)
|
|
if !ok {
|
|
return nil, fmt.Errorf("invalid spec type %T", qs)
|
|
}
|
|
p := &WindowProcedureSpec{
|
|
Window: plan.WindowSpec{
|
|
Every: s.Every,
|
|
Period: s.Period,
|
|
Round: s.Round,
|
|
Start: s.Start,
|
|
},
|
|
Triggering: s.Triggering,
|
|
TimeCol: s.TimeCol,
|
|
StartColLabel: s.StartColLabel,
|
|
StopColLabel: s.StopColLabel,
|
|
}
|
|
if p.Triggering == nil {
|
|
p.Triggering = query.DefaultTrigger
|
|
}
|
|
return p, nil
|
|
}
|
|
|
|
func (s *WindowProcedureSpec) Kind() plan.ProcedureKind {
|
|
return WindowKind
|
|
}
|
|
func (s *WindowProcedureSpec) Copy() plan.ProcedureSpec {
|
|
ns := new(WindowProcedureSpec)
|
|
ns.Window = s.Window
|
|
ns.Triggering = s.Triggering
|
|
return ns
|
|
}
|
|
|
|
func (s *WindowProcedureSpec) TriggerSpec() query.TriggerSpec {
|
|
return s.Triggering
|
|
}
|
|
|
|
func createWindowTransformation(id execute.DatasetID, mode execute.AccumulationMode, spec plan.ProcedureSpec, a execute.Administration) (execute.Transformation, execute.Dataset, error) {
|
|
s, ok := spec.(*WindowProcedureSpec)
|
|
if !ok {
|
|
return nil, nil, fmt.Errorf("invalid spec type %T", spec)
|
|
}
|
|
cache := execute.NewBlockBuilderCache(a.Allocator())
|
|
d := execute.NewDataset(id, mode, cache)
|
|
var start execute.Time
|
|
if s.Window.Start.IsZero() {
|
|
start = a.ResolveTime(query.Now).Truncate(execute.Duration(s.Window.Every))
|
|
} else {
|
|
start = a.ResolveTime(s.Window.Start)
|
|
}
|
|
t := NewFixedWindowTransformation(
|
|
d,
|
|
cache,
|
|
a.Bounds(),
|
|
execute.Window{
|
|
Every: execute.Duration(s.Window.Every),
|
|
Period: execute.Duration(s.Window.Period),
|
|
Round: execute.Duration(s.Window.Round),
|
|
Start: start,
|
|
},
|
|
s.TimeCol,
|
|
s.StartColLabel,
|
|
s.StopColLabel,
|
|
)
|
|
return t, d, nil
|
|
}
|
|
|
|
type fixedWindowTransformation struct {
|
|
d execute.Dataset
|
|
cache execute.BlockBuilderCache
|
|
w execute.Window
|
|
bounds execute.Bounds
|
|
|
|
offset execute.Duration
|
|
|
|
timeCol,
|
|
startColLabel,
|
|
stopColLabel string
|
|
}
|
|
|
|
func NewFixedWindowTransformation(
|
|
d execute.Dataset,
|
|
cache execute.BlockBuilderCache,
|
|
bounds execute.Bounds,
|
|
w execute.Window,
|
|
timeCol,
|
|
startColLabel,
|
|
stopColLabel string,
|
|
) execute.Transformation {
|
|
offset := execute.Duration(w.Start - w.Start.Truncate(w.Every))
|
|
return &fixedWindowTransformation{
|
|
d: d,
|
|
cache: cache,
|
|
w: w,
|
|
bounds: bounds,
|
|
offset: offset,
|
|
timeCol: timeCol,
|
|
startColLabel: startColLabel,
|
|
stopColLabel: stopColLabel,
|
|
}
|
|
}
|
|
|
|
func (t *fixedWindowTransformation) RetractBlock(id execute.DatasetID, key query.PartitionKey) (err error) {
|
|
panic("not implemented")
|
|
//tagKey := meta.Tags().Key()
|
|
//t.cache.ForEachBuilder(func(bk execute.BlockKey, bld execute.BlockBuilder) {
|
|
// if err != nil {
|
|
// return
|
|
// }
|
|
// if bld.Bounds().Overlaps(meta.Bounds()) && tagKey == bld.Tags().Key() {
|
|
// err = t.d.RetractBlock(bk)
|
|
// }
|
|
//})
|
|
//return
|
|
}
|
|
|
|
func (t *fixedWindowTransformation) Process(id execute.DatasetID, b query.Block) error {
|
|
timeIdx := execute.ColIdx(t.timeCol, b.Cols())
|
|
|
|
newCols := make([]query.ColMeta, 0, len(b.Cols())+2)
|
|
keyCols := make([]query.ColMeta, 0, len(b.Cols())+2)
|
|
keyColMap := make([]int, 0, len(b.Cols())+2)
|
|
startColIdx := -1
|
|
stopColIdx := -1
|
|
for j, c := range b.Cols() {
|
|
keyed := b.Key().HasCol(c.Label)
|
|
if c.Label == t.startColLabel {
|
|
startColIdx = j
|
|
keyed = true
|
|
}
|
|
if c.Label == t.stopColLabel {
|
|
stopColIdx = j
|
|
keyed = true
|
|
}
|
|
newCols = append(newCols, c)
|
|
if keyed {
|
|
keyCols = append(keyCols, c)
|
|
keyColMap = append(keyColMap, j)
|
|
}
|
|
}
|
|
if startColIdx == -1 {
|
|
startColIdx = len(newCols)
|
|
c := query.ColMeta{
|
|
Label: t.startColLabel,
|
|
Type: query.TTime,
|
|
}
|
|
newCols = append(newCols, c)
|
|
keyCols = append(keyCols, c)
|
|
keyColMap = append(keyColMap, startColIdx)
|
|
}
|
|
if stopColIdx == -1 {
|
|
stopColIdx = len(newCols)
|
|
c := query.ColMeta{
|
|
Label: t.stopColLabel,
|
|
Type: query.TTime,
|
|
}
|
|
newCols = append(newCols, c)
|
|
keyCols = append(keyCols, c)
|
|
keyColMap = append(keyColMap, stopColIdx)
|
|
}
|
|
|
|
return b.Do(func(cr query.ColReader) error {
|
|
l := cr.Len()
|
|
for i := 0; i < l; i++ {
|
|
tm := cr.Times(timeIdx)[i]
|
|
bounds := t.getWindowBounds(tm)
|
|
for _, bnds := range bounds {
|
|
// Update key
|
|
cols := make([]query.ColMeta, len(keyCols))
|
|
values := make([]interface{}, len(keyCols))
|
|
for j, c := range keyCols {
|
|
cols[j] = c
|
|
switch c.Label {
|
|
case t.startColLabel:
|
|
values[j] = bnds.Start
|
|
case t.stopColLabel:
|
|
values[j] = bnds.Stop
|
|
default:
|
|
values[j] = b.Key().Value(keyColMap[j])
|
|
}
|
|
}
|
|
key := execute.NewPartitionKey(cols, values)
|
|
builder, created := t.cache.BlockBuilder(key)
|
|
if created {
|
|
for _, c := range newCols {
|
|
builder.AddCol(c)
|
|
}
|
|
}
|
|
for j, c := range builder.Cols() {
|
|
switch c.Label {
|
|
case t.startColLabel:
|
|
builder.AppendTime(startColIdx, bnds.Start)
|
|
case t.stopColLabel:
|
|
builder.AppendTime(stopColIdx, bnds.Stop)
|
|
default:
|
|
switch c.Type {
|
|
case query.TBool:
|
|
builder.AppendBool(j, cr.Bools(j)[i])
|
|
case query.TInt:
|
|
builder.AppendInt(j, cr.Ints(j)[i])
|
|
case query.TUInt:
|
|
builder.AppendUInt(j, cr.UInts(j)[i])
|
|
case query.TFloat:
|
|
builder.AppendFloat(j, cr.Floats(j)[i])
|
|
case query.TString:
|
|
builder.AppendString(j, cr.Strings(j)[i])
|
|
case query.TTime:
|
|
builder.AppendTime(j, cr.Times(j)[i])
|
|
default:
|
|
execute.PanicUnknownType(c.Type)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return nil
|
|
})
|
|
}
|
|
|
|
func (t *fixedWindowTransformation) getWindowBounds(now execute.Time) []execute.Bounds {
|
|
stop := now.Truncate(t.w.Every) + execute.Time(t.offset)
|
|
if now >= stop {
|
|
stop += execute.Time(t.w.Every)
|
|
}
|
|
start := stop - execute.Time(t.w.Period)
|
|
|
|
var bounds []execute.Bounds
|
|
|
|
for now >= start {
|
|
bnds := execute.Bounds{
|
|
Start: start,
|
|
Stop: stop,
|
|
}
|
|
|
|
// Check global bounds
|
|
if bnds.Stop > t.bounds.Stop {
|
|
bnds.Stop = t.bounds.Stop
|
|
}
|
|
|
|
if bnds.Start < t.bounds.Start {
|
|
bnds.Start = t.bounds.Start
|
|
}
|
|
|
|
// Check bounds again since we just clamped them.
|
|
if bnds.Contains(now) {
|
|
bounds = append(bounds, bnds)
|
|
}
|
|
|
|
// Shift up to next bounds
|
|
stop += execute.Time(t.w.Every)
|
|
start += execute.Time(t.w.Every)
|
|
}
|
|
|
|
return bounds
|
|
}
|
|
|
|
func (t *fixedWindowTransformation) UpdateWatermark(id execute.DatasetID, mark execute.Time) error {
|
|
return t.d.UpdateWatermark(mark)
|
|
}
|
|
func (t *fixedWindowTransformation) UpdateProcessingTime(id execute.DatasetID, pt execute.Time) error {
|
|
return t.d.UpdateProcessingTime(pt)
|
|
}
|
|
func (t *fixedWindowTransformation) Finish(id execute.DatasetID, err error) {
|
|
t.d.Finish(err)
|
|
}
|