fix: GROUP BY queries with offset that crosses a DST boundary fail. (#25082) (#25087)

This is actually the second fix for https://github.com/influxdata/influxdb/issues/20238 for when the time zone falls back in autumn. closes https://github.com/influxdata/influxdb/issues/25078 (cherry picked from commit d60741b506) closes https://github.com/influxdata/influxdb/issues/25080
2024-06-24 13:40:04 -07:00 · 2024-06-24 13:40:04 -07:00 · 0c77b4cbd2
parent cb8cfe3510
commit 0c77b4cbd2
2 changed files with 169 additions and 1 deletions
--- a/influxql/query/iterator.go
+++ b/influxql/query/iterator.go
@ -838,6 +838,10 @@ func (opt IteratorOptions) Window(t int64) (start, end int64) {
 		end = t + dt
 	}

+	// As above, the offset has to happen before the time zone calculation.
+	// This is another fix for https://github.com/influxdata/influxdb/issues/20238
+	// that was missed the first time.
+	end += int64(opt.Interval.Offset)
 	// Retrieve the zone offset for the end time.
 	if opt.Location != nil {
 		_, endOffset := opt.Zone(end)
@ -863,7 +867,6 @@ func (opt IteratorOptions) Window(t int64) (start, end int64) {
 			}
 		}
 	}
-	end += int64(opt.Interval.Offset)
 	return
 }

--- a/influxql/query/iterator_test.go
+++ b/influxql/query/iterator_test.go
@ -12,6 +12,7 @@ import (
 	"github.com/davecgh/go-spew/spew"
 	"github.com/influxdata/influxdb/v2/influxql/query"
 	"github.com/influxdata/influxdb/v2/pkg/deep"
+	"github.com/influxdata/influxdb/v2/pkg/testing/assert"
 	"github.com/influxdata/influxql"
 )

@ -880,6 +881,170 @@ func TestGroupByIterator_DST(t *testing.T) {
 	}
 }

+// A count() GROUP BY query with an offset that caused an interval
+// to cross a daylight savings change back to standard time dropped
+// rows in a grouped count() expression.
+
+func TestGroupByIterator_DST_End(t *testing.T) {
+	// const RFC822 string = "02 Jan 06 15:04 MST"
+	inputIter := &IntegerIterator{
+		Points: []query.IntegerPoint{
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-02T00:00:01Z").UnixNano(), Value: 1},
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-02T01:00:01Z").UnixNano(), Value: 1},
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-02T02:00:01Z").UnixNano(), Value: 1},
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-02T03:00:01Z").UnixNano(), Value: 1},
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-02T04:00:01Z").UnixNano(), Value: 1},
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-02T05:00:01Z").UnixNano(), Value: 1},
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-02T06:00:01Z").UnixNano(), Value: 1},
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-02T07:00:01Z").UnixNano(), Value: 1},
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-02T08:00:01Z").UnixNano(), Value: 1},
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-02T09:00:01Z").UnixNano(), Value: 1},
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-02T10:00:01Z").UnixNano(), Value: 1},
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-02T11:00:01Z").UnixNano(), Value: 1},
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-02T12:00:01Z").UnixNano(), Value: 1},
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-02T13:00:01Z").UnixNano(), Value: 1},
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-02T14:00:01Z").UnixNano(), Value: 1},
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-02T15:00:01Z").UnixNano(), Value: 1},
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-02T16:00:01Z").UnixNano(), Value: 1},
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-02T17:00:01Z").UnixNano(), Value: 1},
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-02T18:00:01Z").UnixNano(), Value: 1},
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-02T19:00:01Z").UnixNano(), Value: 1},
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-02T20:00:01Z").UnixNano(), Value: 1},
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-02T21:00:01Z").UnixNano(), Value: 1},
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-02T22:00:01Z").UnixNano(), Value: 1},
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-02T23:00:01Z").UnixNano(), Value: 1},
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-03T00:00:01Z").UnixNano(), Value: 1},
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-03T01:00:01Z").UnixNano(), Value: 1},
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-03T02:00:01Z").UnixNano(), Value: 1},
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-03T03:00:01Z").UnixNano(), Value: 1},
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-03T04:00:01Z").UnixNano(), Value: 1},
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-03T05:00:01Z").UnixNano(), Value: 1},
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-03T06:00:01Z").UnixNano(), Value: 1},
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-03T07:00:01Z").UnixNano(), Value: 1},
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-03T08:00:01Z").UnixNano(), Value: 1},
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-03T09:00:01Z").UnixNano(), Value: 1},
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-03T10:00:01Z").UnixNano(), Value: 1},
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-03T11:00:01Z").UnixNano(), Value: 1},
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-03T12:00:01Z").UnixNano(), Value: 1},
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-03T13:00:01Z").UnixNano(), Value: 1},
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-03T14:00:01Z").UnixNano(), Value: 1},
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-03T15:00:01Z").UnixNano(), Value: 1},
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-03T16:00:01Z").UnixNano(), Value: 1},
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-03T17:00:01Z").UnixNano(), Value: 1},
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-03T18:00:01Z").UnixNano(), Value: 1},
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-03T19:00:01Z").UnixNano(), Value: 1},
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-03T20:00:01Z").UnixNano(), Value: 1},
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-03T21:00:01Z").UnixNano(), Value: 1},
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-03T22:00:01Z").UnixNano(), Value: 1},
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-03T23:00:01Z").UnixNano(), Value: 1},
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-04T00:00:01Z").UnixNano(), Value: 1},
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-04T01:00:01Z").UnixNano(), Value: 1},
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-04T02:00:01Z").UnixNano(), Value: 1},
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-04T03:00:01Z").UnixNano(), Value: 1},
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-04T04:00:01Z").UnixNano(), Value: 1},
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-04T05:00:01Z").UnixNano(), Value: 1},
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-04T06:00:01Z").UnixNano(), Value: 1},
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-04T07:00:01Z").UnixNano(), Value: 1},
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-04T08:00:01Z").UnixNano(), Value: 1},
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-04T09:00:01Z").UnixNano(), Value: 1},
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-04T10:00:01Z").UnixNano(), Value: 1},
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-04T11:00:01Z").UnixNano(), Value: 1},
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-04T12:00:01Z").UnixNano(), Value: 1},
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-04T13:00:01Z").UnixNano(), Value: 1},
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-04T14:00:01Z").UnixNano(), Value: 1},
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-04T15:00:01Z").UnixNano(), Value: 1},
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-04T16:00:01Z").UnixNano(), Value: 1},
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-04T17:00:01Z").UnixNano(), Value: 1},
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-04T18:00:01Z").UnixNano(), Value: 1},
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-04T19:00:01Z").UnixNano(), Value: 1},
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-04T20:00:01Z").UnixNano(), Value: 1},
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-04T21:00:01Z").UnixNano(), Value: 1},
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-04T22:00:01Z").UnixNano(), Value: 1},
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-04T23:00:01Z").UnixNano(), Value: 1},
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-05T00:00:01Z").UnixNano(), Value: 1},
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-05T01:00:01Z").UnixNano(), Value: 1},
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-05T02:00:01Z").UnixNano(), Value: 1},
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-05T03:00:01Z").UnixNano(), Value: 1},
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-05T04:00:01Z").UnixNano(), Value: 1},
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-05T05:00:01Z").UnixNano(), Value: 1},
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-05T06:00:01Z").UnixNano(), Value: 1},
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-05T07:00:01Z").UnixNano(), Value: 1},
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-05T08:00:01Z").UnixNano(), Value: 1},
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-05T09:00:01Z").UnixNano(), Value: 1},
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-05T10:00:01Z").UnixNano(), Value: 1},
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-05T11:00:01Z").UnixNano(), Value: 1},
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-05T12:00:01Z").UnixNano(), Value: 1},
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-05T13:00:01Z").UnixNano(), Value: 1},
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-05T14:00:01Z").UnixNano(), Value: 1},
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-05T15:00:01Z").UnixNano(), Value: 1},
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-05T16:00:01Z").UnixNano(), Value: 1},
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-05T17:00:01Z").UnixNano(), Value: 1},
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-05T18:00:01Z").UnixNano(), Value: 1},
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-05T19:00:01Z").UnixNano(), Value: 1},
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-05T20:00:01Z").UnixNano(), Value: 1},
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-05T21:00:01Z").UnixNano(), Value: 1},
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-05T22:00:01Z").UnixNano(), Value: 1},
+			{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-05T23:00:01Z").UnixNano(), Value: 1},
+		},
+	}
+	const location = "America/Los_Angeles"
+	loc, err := time.LoadLocation(location)
+	if err != nil {
+		t.Fatalf("Cannot find timezone for %s: %s", location, err)
+	}
+	opt := query.IteratorOptions{
+		StartTime: mustParseTime("2023-11-02T00:00:00Z").UnixNano(),
+		EndTime:   mustParseTime("2023-11-06T00:00:00Z").UnixNano(),
+		Ascending: true,
+
+		Ordered:   true,
+		StripName: false,
+		Fill:      influxql.NoFill,
+		FillValue: nil,
+		Dedupe:    false,
+		Interval: query.Interval{
+			Duration: 24 * time.Hour,
+			Offset:   12 * time.Hour,
+		},
+		Expr:     MustParseExpr("count(Value)"),
+		Location: loc,
+	}
+
+	groupByIter, err := query.NewCallIterator(inputIter, opt)
+	if err != nil {
+		t.Fatalf("Cannot create Count and Group By iterator: %s", err)
+	} else {
+		groupByIter = query.NewFillIterator(groupByIter, MustParseExpr("count(Value)"), opt)
+	}
+
+	if all, err := (Iterators{groupByIter}).ReadAll(); err != nil {
+		t.Fatalf("unexpected error: %s", err)
+	} else {
+		results := [][]query.IntegerPoint{
+			{query.IntegerPoint{Name: "a", Aggregated: 19, Time: mustParseTime("2023-11-01T12:00:00-07:00").UnixNano(), Value: 19}},
+			{query.IntegerPoint{Name: "a", Aggregated: 24, Time: mustParseTime("2023-11-02T12:00:00-07:00").UnixNano(), Value: 24}},
+			{query.IntegerPoint{Name: "a", Aggregated: 24, Time: mustParseTime("2023-11-03T12:00:00-07:00").UnixNano(), Value: 24}},
+			// The extra time from falling back means more than 24 hours counted
+			{query.IntegerPoint{Name: "a", Aggregated: 25, Time: mustParseTime("2023-11-04T12:00:00-07:00").UnixNano(), Value: 25}},
+			{query.IntegerPoint{Name: "a", Aggregated: 04, Time: mustParseTime("2023-11-05T12:00:00-08:00").UnixNano(), Value: 4}},
+		}
+
+		for i, a := range all {
+			for j, p := range a {
+				switch ip := p.(type) {
+				case *query.IntegerPoint:
+					assert.Equal(t, results[i][j].Time, ip.Time,
+						"Time mismatch at i=%d j=%d: expect %v, got %v", i, j, time.Unix(0, results[i][j].Time), time.Unix(0, ip.Time))
+					assert.Equal(t, results[i][j].Value, ip.Value,
+						"Value mismatch at i=%d j=%d: expect %d, got %d", i, j, results[i][j].Value, ip.Value)
+					assert.Equal(t, results[i][j].Aggregated, ip.Aggregated,
+						"Aggregated mismatch at i=%d j=%d: expect %d, got %d", i, j, results[i][j].Aggregated, ip.Aggregated)
+				}
+			}
+		}
+	}
+}
+
 func TestFillIterator_DST(t *testing.T) {
 	for _, tt := range []struct {
 		name       string