From 7fb7faaacaf005d61e6310d10ff07da5dd2b4bd0 Mon Sep 17 00:00:00 2001 From: Jason Wilder Date: Wed, 18 May 2016 17:21:10 -0600 Subject: [PATCH] Fix points already read from being returned more than once If there were duplicate points in multiple blocks, we would correctly dedup the points and mark the regions of the blocks we've read. Unfortunately, we were not excluding the already points as the cursor moved to points in the later blocks which could cause points to be return twice incorrectly. Fixes #6611 --- CHANGELOG.md | 3 +-- tsdb/engine/tsm1/file_store.go | 24 ++++++++++++++++++++++++ 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9aa2d0fd97..7c8d3fed1c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,6 @@ - [#6623](https://github.com/influxdata/influxdb/pull/6623): Speed up drop database - [#6519](https://github.com/influxdata/influxdb/issues/6519): Support cast syntax for selecting a specific type. - [#6654](https://github.com/influxdata/influxdb/pull/6654): Add new HTTP statistics to monitoring - - [#6664](https://github.com/influxdata/influxdb/pull/6664): Adds monitoring statistic for on-disk shard size. ### Bugfixes @@ -27,10 +26,10 @@ - [#6235](https://github.com/influxdata/influxdb/issues/6235): Fix measurement field panic in tsm1 engine. - [#6663](https://github.com/influxdata/influxdb/issues/6663): Fixing panic in SHOW FIELD KEYS. - [#6624](https://github.com/influxdata/influxdb/issues/6624): Ensure clients requesting gzip encoded bodies don't receive empty body - - [#6652](https://github.com/influxdata/influxdb/issues/6652): Fix panic: interface conversion: tsm1.Value is *tsm1.StringValue, not *tsm1.FloatValue - [#6406](https://github.com/influxdata/influxdb/issues/6406): Max index entries exceeded - [#6557](https://github.com/influxdata/influxdb/issues/6557): Overwriting points on large series can cause memory spikes during compactions +- [#6611](https://github.com/influxdata/influxdb/issues/6611): Queries slow down hundreds times after overwriting points ## v0.13.0 [2016-05-12] diff --git a/tsdb/engine/tsm1/file_store.go b/tsdb/engine/tsm1/file_store.go index 9282b735b7..f30c6b18a6 100644 --- a/tsdb/engine/tsm1/file_store.go +++ b/tsdb/engine/tsm1/file_store.go @@ -966,6 +966,9 @@ func (c *KeyCursor) ReadFloatBlock(tdec *TimeDecoder, vdec *FloatDecoder, buf *[ // Remove any tombstoned values v = c.filterFloatValues(tombstones, v) + // Remove values we already read + v = FloatValues(v).Exclude(cur.readMin, cur.readMax) + if len(v) > 0 { v = FloatValues(v).Include(first.entry.MinTime, first.entry.MaxTime) @@ -986,6 +989,9 @@ func (c *KeyCursor) ReadFloatBlock(tdec *TimeDecoder, vdec *FloatDecoder, buf *[ // Remove any tombstoned values v = c.filterFloatValues(tombstones, v) + // Remove values we already read + v = FloatValues(v).Exclude(cur.readMin, cur.readMax) + // If the block we decoded should have all of it's values included, mark it as read so we // don't use it again. if len(v) > 0 { @@ -1050,6 +1056,9 @@ func (c *KeyCursor) ReadIntegerBlock(tdec *TimeDecoder, vdec *IntegerDecoder, bu // Remove any tombstoned values v = c.filterIntegerValues(tombstones, v) + // Remove values we already read + v = IntegerValues(v).Exclude(cur.readMin, cur.readMax) + if len(v) > 0 { v = IntegerValues(v).Include(first.entry.MinTime, first.entry.MaxTime) @@ -1070,6 +1079,9 @@ func (c *KeyCursor) ReadIntegerBlock(tdec *TimeDecoder, vdec *IntegerDecoder, bu // Remove any tombstoned values v = c.filterIntegerValues(tombstones, v) + // Remove values we already read + v = IntegerValues(v).Exclude(cur.readMin, cur.readMax) + // If the block we decoded should have all of it's values included, mark it as read so we // don't use it again. if len(v) > 0 { @@ -1134,6 +1146,9 @@ func (c *KeyCursor) ReadStringBlock(tdec *TimeDecoder, vdec *StringDecoder, buf // Remove any tombstoned values v = c.filterStringValues(tombstones, v) + // Remove values we already read + v = StringValues(v).Exclude(cur.readMin, cur.readMax) + if len(v) > 0 { v = StringValues(v).Include(first.entry.MinTime, first.entry.MaxTime) @@ -1154,6 +1169,9 @@ func (c *KeyCursor) ReadStringBlock(tdec *TimeDecoder, vdec *StringDecoder, buf // Remove any tombstoned values v = c.filterStringValues(tombstones, v) + // Remove values we already read + v = StringValues(v).Exclude(cur.readMin, cur.readMax) + // If the block we decoded should have all of it's values included, mark it as read so we // don't use it again. if len(v) > 0 { @@ -1218,6 +1236,9 @@ func (c *KeyCursor) ReadBooleanBlock(tdec *TimeDecoder, vdec *BooleanDecoder, bu // Remove any tombstoned values v = c.filterBooleanValues(tombstones, v) + // Remove values we already read + v = BooleanValues(v).Exclude(cur.readMin, cur.readMax) + if len(v) > 0 { v = BooleanValues(v).Include(first.entry.MinTime, first.entry.MaxTime) @@ -1238,6 +1259,9 @@ func (c *KeyCursor) ReadBooleanBlock(tdec *TimeDecoder, vdec *BooleanDecoder, bu // Remove any tombstoned values v = c.filterBooleanValues(tombstones, v) + // Remove values we already read + v = BooleanValues(v).Exclude(cur.readMin, cur.readMax) + // If the block we decoded should have all of it's values included, mark it as read so we // don't use it again. if len(v) > 0 {