From 2e3ef6c9d73d7fec2b7a94965eb528db3e03a10d Mon Sep 17 00:00:00 2001 From: Peng Liu Date: Wed, 16 May 2018 11:19:57 -0700 Subject: [PATCH 1/3] Parallel compression Drop-in of pgzip for gzip --- Godeps | 1 + LICENSE_OF_DEPENDENCIES.md | 1 + cmd/influxd/backup/backup.go | 2 +- cmd/influxd/restore/restore.go | 2 +- 4 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Godeps b/Godeps index 804dc49f25..b655975e06 100644 --- a/Godeps +++ b/Godeps @@ -19,6 +19,7 @@ github.com/influxdata/yamux 1f58ded512de5feabbe30b60c7d33a7a896c5f16 github.com/influxdata/yarpc f0da2db138cad2fb425541938fc28dd5a5bc6918 github.com/jsternberg/zap-logfmt ac4bd917e18a4548ce6e0e765b29a4e7f397b0b6 github.com/jwilder/encoding b4e1701a28efcc637d9afcca7d38e495fe909a09 +github.com/klauspost/pgzip 0bf5dcad4ada2814c3c00f996a982270bb81a506 github.com/mattn/go-isatty 6ca4dbf54d38eea1a992b3c722a76a5d1c4cb25c github.com/matttproud/golang_protobuf_extensions 3247c84500bff8d9fb6d579d800f20b3e091582c github.com/mschoch/smat 90eadee771aeab36e8bf796039b8c261bebebe4f diff --git a/LICENSE_OF_DEPENDENCIES.md b/LICENSE_OF_DEPENDENCIES.md index ea6fc69f30..b5b0df85b6 100644 --- a/LICENSE_OF_DEPENDENCIES.md +++ b/LICENSE_OF_DEPENDENCIES.md @@ -23,6 +23,7 @@ - github.com/influxdata/yarpc [MIT LICENSE](https://github.com/influxdata/yarpc/blob/master/LICENSE) - github.com/jsternberg/zap-logfmt [MIT LICENSE](https://github.com/jsternberg/zap-logfmt/blob/master/LICENSE) - github.com/jwilder/encoding [MIT LICENSE](https://github.com/jwilder/encoding/blob/master/LICENSE) +- github.com/klauspost/pgzip [MIT LICENSE](https://github.com/klauspost/pgzip/blob/master/LICENSE) - github.com/mattn/go-isatty [MIT LICENSE](https://github.com/mattn/go-isatty/blob/master/LICENSE) - github.com/matttproud/golang_protobuf_extensions [APACHE LICENSE](https://github.com/matttproud/golang_protobuf_extensions/blob/master/LICENSE) - github.com/opentracing/opentracing-go [MIT LICENSE](https://github.com/opentracing/opentracing-go/blob/master/LICENSE) diff --git a/cmd/influxd/backup/backup.go b/cmd/influxd/backup/backup.go index 486daafd29..dcc1198823 100644 --- a/cmd/influxd/backup/backup.go +++ b/cmd/influxd/backup/backup.go @@ -2,7 +2,6 @@ package backup import ( - "compress/gzip" "encoding/binary" "encoding/json" "errors" @@ -20,6 +19,7 @@ import ( "github.com/influxdata/influxdb/cmd/influxd/backup_util" "github.com/influxdata/influxdb/services/snapshotter" "github.com/influxdata/influxdb/tcp" + gzip "github.com/klauspost/pgzip" ) const ( diff --git a/cmd/influxd/restore/restore.go b/cmd/influxd/restore/restore.go index 25848d5fb8..eadd5f0b75 100644 --- a/cmd/influxd/restore/restore.go +++ b/cmd/influxd/restore/restore.go @@ -16,7 +16,7 @@ import ( "strconv" "strings" - "compress/gzip" + gzip "github.com/klauspost/pgzip" "github.com/influxdata/influxdb/cmd/influxd/backup_util" tarstream "github.com/influxdata/influxdb/pkg/tar" From 403559154fa977972718d811fbb618aab512f152 Mon Sep 17 00:00:00 2001 From: Peng Liu Date: Tue, 12 Jun 2018 00:12:53 -0700 Subject: [PATCH 2/3] Add dep for pgzip --- Gopkg.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Gopkg.toml b/Gopkg.toml index 95d2b4ffea..5d0e64d772 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -66,6 +66,10 @@ name = "github.com/prometheus/client_golang" revision = "661e31bf844dfca9aeba15f27ea8aa0d485ad212" +[[constraint]] + name = "github.com/klauspost/pgzip" + revision = "0bf5dcad4ada2814c3c00f996a982270bb81a506" + [prune] go-tests = true unused-packages = true From fe425d4bdad508fe28c4bef525111e69edf42003 Mon Sep 17 00:00:00 2001 From: Edd Robinson Date: Wed, 13 Jun 2018 11:25:13 +0100 Subject: [PATCH 3/3] Use correct version of pgzip --- Gopkg.lock | 26 +++++++++++++++++++++++++- Gopkg.toml | 2 +- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index 2d66f57bed..e72afbd01b 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -150,6 +150,30 @@ packages = ["simple8b"] revision = "b4e1701a28efcc637d9afcca7d38e495fe909a09" +[[projects]] + name = "github.com/klauspost/compress" + packages = ["flate"] + revision = "6c8db69c4b49dd4df1fff66996cf556176d0b9bf" + version = "v1.2.1" + +[[projects]] + name = "github.com/klauspost/cpuid" + packages = ["."] + revision = "ae7887de9fa5d2db4eaa8174a7eff2c1ac00f2da" + version = "v1.1" + +[[projects]] + name = "github.com/klauspost/crc32" + packages = ["."] + revision = "cb6bfca970f6908083f26f39a79009d608efd5cd" + version = "v1.1" + +[[projects]] + name = "github.com/klauspost/pgzip" + packages = ["."] + revision = "0bf5dcad4ada2814c3c00f996a982270bb81a506" + version = "v1.1" + [[projects]] branch = "master" name = "github.com/mattn/go-isatty" @@ -337,6 +361,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "9bb6fd90291e987c895b33e55cd20d968e141b4583d47e28aa6dfb733b7a320b" + inputs-digest = "635dd31293fe8e0706f3b721b1d05ea2d12771c2dcaf86d17517347472755b31" solver-name = "gps-cdcl" solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index 5d0e64d772..c7d0817796 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -68,7 +68,7 @@ [[constraint]] name = "github.com/klauspost/pgzip" - revision = "0bf5dcad4ada2814c3c00f996a982270bb81a506" + version = "1.1.0" [prune] go-tests = true