Merge pull request #7705 from influxdata/jw-dedup

Optimize Values.Deduplicate

tsdb/engine/tsm1/encoding.gen.go

@@ -56,18 +56,20 @@ func (a Values) assertOrdered() {
 // Deduplicate returns a new slice with any values that have the same timestamp removed.
 // The Value that appears last in the slice is the one that is kept.
 func (a Values) Deduplicate() Values {
-	m := make(map[int64]Value)
-	for _, val := range a {
-		m[val.UnixNano()] = val
+	if len(a) == 0 {
+		return a
 	}
+	sort.Stable(a)
+	var i int
+	for j := 1; j < len(a); j++ {
+		v := a[j]
+		if v.UnixNano() != a[i].UnixNano() {
+			i++
+		}
+		a[i] = v
 
-	other := make(Values, 0, len(m))
-	for _, val := range m {
-		other = append(other, val)
 	}
-
-	sort.Sort(other)
-	return other
+	return a[:i+1]
 }
 
 //  Exclude returns the subset of values not in [min, max]
@@ -227,18 +229,20 @@ func (a FloatValues) assertOrdered() {
 // Deduplicate returns a new slice with any values that have the same timestamp removed.
 // The Value that appears last in the slice is the one that is kept.
 func (a FloatValues) Deduplicate() FloatValues {
-	m := make(map[int64]FloatValue)
-	for _, val := range a {
-		m[val.UnixNano()] = val
+	if len(a) == 0 {
+		return a
 	}
+	sort.Stable(a)
+	var i int
+	for j := 1; j < len(a); j++ {
+		v := a[j]
+		if v.UnixNano() != a[i].UnixNano() {
+			i++
+		}
+		a[i] = v
 
-	other := make(FloatValues, 0, len(m))
-	for _, val := range m {
-		other = append(other, val)
 	}
-
-	sort.Sort(other)
-	return other
+	return a[:i+1]
 }
 
 //  Exclude returns the subset of values not in [min, max]
@@ -398,18 +402,20 @@ func (a IntegerValues) assertOrdered() {
 // Deduplicate returns a new slice with any values that have the same timestamp removed.
 // The Value that appears last in the slice is the one that is kept.
 func (a IntegerValues) Deduplicate() IntegerValues {
-	m := make(map[int64]IntegerValue)
-	for _, val := range a {
-		m[val.UnixNano()] = val
+	if len(a) == 0 {
+		return a
 	}
+	sort.Stable(a)
+	var i int
+	for j := 1; j < len(a); j++ {
+		v := a[j]
+		if v.UnixNano() != a[i].UnixNano() {
+			i++
+		}
+		a[i] = v
 
-	other := make(IntegerValues, 0, len(m))
-	for _, val := range m {
-		other = append(other, val)
 	}
-
-	sort.Sort(other)
-	return other
+	return a[:i+1]
 }
 
 //  Exclude returns the subset of values not in [min, max]
@@ -569,18 +575,20 @@ func (a StringValues) assertOrdered() {
 // Deduplicate returns a new slice with any values that have the same timestamp removed.
 // The Value that appears last in the slice is the one that is kept.
 func (a StringValues) Deduplicate() StringValues {
-	m := make(map[int64]StringValue)
-	for _, val := range a {
-		m[val.UnixNano()] = val
+	if len(a) == 0 {
+		return a
 	}
+	sort.Stable(a)
+	var i int
+	for j := 1; j < len(a); j++ {
+		v := a[j]
+		if v.UnixNano() != a[i].UnixNano() {
+			i++
+		}
+		a[i] = v
 
-	other := make(StringValues, 0, len(m))
-	for _, val := range m {
-		other = append(other, val)
 	}
-
-	sort.Sort(other)
-	return other
+	return a[:i+1]
 }
 
 //  Exclude returns the subset of values not in [min, max]
@@ -740,18 +748,20 @@ func (a BooleanValues) assertOrdered() {
 // Deduplicate returns a new slice with any values that have the same timestamp removed.
 // The Value that appears last in the slice is the one that is kept.
 func (a BooleanValues) Deduplicate() BooleanValues {
-	m := make(map[int64]BooleanValue)
-	for _, val := range a {
-		m[val.UnixNano()] = val
+	if len(a) == 0 {
+		return a
 	}
+	sort.Stable(a)
+	var i int
+	for j := 1; j < len(a); j++ {
+		v := a[j]
+		if v.UnixNano() != a[i].UnixNano() {
+			i++
+		}
+		a[i] = v
 
-	other := make(BooleanValues, 0, len(m))
-	for _, val := range m {
-		other = append(other, val)
 	}
-
-	sort.Sort(other)
-	return other
+	return a[:i+1]
 }
 
 //  Exclude returns the subset of values not in [min, max]

tsdb/engine/tsm1/encoding.gen.go.tmpl

@@ -53,18 +53,20 @@ func (a {{.Name}}Values) assertOrdered() {
 // Deduplicate returns a new slice with any values that have the same timestamp removed.
 // The Value that appears last in the slice is the one that is kept.
 func (a {{.Name}}Values) Deduplicate() {{.Name}}Values {
-	m := make(map[int64]{{.Name}}Value)
-	for _, val := range a {
-		m[val.UnixNano()] = val
+	if len(a) == 0 {
+		return a
 	}
+	sort.Stable(a)
+	var i int
+	for j := 1; j < len(a); j++ {
+		v := a[j]
+		if v.UnixNano() != a[i].UnixNano() {
+			i++
+		}
+		a[i] = v
 
-	other := make({{.Name}}Values, 0, len(m))
-	for _, val := range m {
-		other = append(other, val)
 	}
-
-	sort.Sort(other)
-	return other
+	return a[:i+1]
 }
 
 //  Exclude returns the subset of values not in [min, max]

tsdb/engine/tsm1/encoding_test.go

@@ -493,10 +493,16 @@ func TestValues_MergeFloat(t *testing.T) {
 			t.Fatalf("test(%d): value length mismatch: exp %v, got %v", i, exp, got)
 		}
 
+		dedup := tsm1.Values(append(test.a, test.b...)).Deduplicate()
+
 		for i := range test.exp {
 			if exp, got := test.exp[i].String(), got[i].String(); exp != got {
 				t.Fatalf("value mismatch:\n exp %v\n got %v", exp, got)
 			}
+
+			if exp, got := test.exp[i].String(), dedup[i].String(); exp != got {
+				t.Fatalf("value mismatch:\n exp %v\n got %v", exp, got)
+			}
 		}
 	}
 }
@@ -626,10 +632,16 @@ func TestIntegerValues_Merge(t *testing.T) {
 			t.Fatalf("test(%d): value length mismatch: exp %v, got %v", i, exp, got)
 		}
 
+		dedup := tsm1.IntegerValues(append(test.a, test.b...)).Deduplicate()
+
 		for i := range test.exp {
 			if exp, got := test.exp[i].String(), got[i].String(); exp != got {
 				t.Fatalf("value mismatch:\n exp %v\n got %v", exp, got)
 			}
+
+			if exp, got := test.exp[i].String(), dedup[i].String(); exp != got {
+				t.Fatalf("value mismatch:\n exp %v\n got %v", exp, got)
+			}
 		}
 	}
 }
@@ -755,10 +767,16 @@ func TestFloatValues_Merge(t *testing.T) {
 			t.Fatalf("test(%d): value length mismatch: exp %v, got %v", i, exp, got)
 		}
 
+		dedup := tsm1.FloatValues(append(test.a, test.b...)).Deduplicate()
+
 		for i := range test.exp {
 			if exp, got := test.exp[i].String(), got[i].String(); exp != got {
 				t.Fatalf("value mismatch:\n exp %v\n got %v", exp, got)
 			}
+
+			if exp, got := test.exp[i].String(), dedup[i].String(); exp != got {
+				t.Fatalf("value mismatch:\n exp %v\n got %v", exp, got)
+			}
 		}
 	}
 }
@@ -884,10 +902,16 @@ func TestBooleanValues_Merge(t *testing.T) {
 			t.Fatalf("test(%d): value length mismatch: exp %v, got %v", i, exp, got)
 		}
 
+		dedup := tsm1.BooleanValues(append(test.a, test.b...)).Deduplicate()
+
 		for i := range test.exp {
 			if exp, got := test.exp[i].String(), got[i].String(); exp != got {
 				t.Fatalf("value mismatch:\n exp %v\n got %v", exp, got)
 			}
+
+			if exp, got := test.exp[i].String(), dedup[i].String(); exp != got {
+				t.Fatalf("value mismatch:\n exp %v\n got %v", exp, got)
+			}
 		}
 	}
 }
@@ -1017,10 +1041,16 @@ func TestStringValues_Merge(t *testing.T) {
 			t.Fatalf("test(%d): value length mismatch: exp %v, got %v", i, exp, got)
 		}
 
+		dedup := tsm1.StringValues(append(test.a, test.b...)).Deduplicate()
+
 		for i := range test.exp {
 			if exp, got := test.exp[i].String(), got[i].String(); exp != got {
 				t.Fatalf("value mismatch:\n exp %v\n got %v", exp, got)
 			}
+
+			if exp, got := test.exp[i].String(), dedup[i].String(); exp != got {
+				t.Fatalf("value mismatch:\n exp %v\n got %v", exp, got)
+			}
 		}
 	}
 }