Merge pull request #9856 from pliu/backup_restore_parallel_compression

Parallelize compression/decompression during backup and restore

Godeps

@@ -19,6 +19,7 @@ github.com/influxdata/yamux 1f58ded512de5feabbe30b60c7d33a7a896c5f16
 github.com/influxdata/yarpc f0da2db138cad2fb425541938fc28dd5a5bc6918
 github.com/jsternberg/zap-logfmt ac4bd917e18a4548ce6e0e765b29a4e7f397b0b6
 github.com/jwilder/encoding b4e1701a28efcc637d9afcca7d38e495fe909a09
+github.com/klauspost/pgzip 0bf5dcad4ada2814c3c00f996a982270bb81a506
 github.com/mattn/go-isatty 6ca4dbf54d38eea1a992b3c722a76a5d1c4cb25c
 github.com/matttproud/golang_protobuf_extensions 3247c84500bff8d9fb6d579d800f20b3e091582c
 github.com/mschoch/smat 90eadee771aeab36e8bf796039b8c261bebebe4f

Gopkg.lock

@@ -150,6 +150,30 @@
   packages = ["simple8b"]
   revision = "b4e1701a28efcc637d9afcca7d38e495fe909a09"
 
+[[projects]]
+  name = "github.com/klauspost/compress"
+  packages = ["flate"]
+  revision = "6c8db69c4b49dd4df1fff66996cf556176d0b9bf"
+  version = "v1.2.1"
+
+[[projects]]
+  name = "github.com/klauspost/cpuid"
+  packages = ["."]
+  revision = "ae7887de9fa5d2db4eaa8174a7eff2c1ac00f2da"
+  version = "v1.1"
+
+[[projects]]
+  name = "github.com/klauspost/crc32"
+  packages = ["."]
+  revision = "cb6bfca970f6908083f26f39a79009d608efd5cd"
+  version = "v1.1"
+
+[[projects]]
+  name = "github.com/klauspost/pgzip"
+  packages = ["."]
+  revision = "0bf5dcad4ada2814c3c00f996a982270bb81a506"
+  version = "v1.1"
+
 [[projects]]
   branch = "master"
   name = "github.com/mattn/go-isatty"
@@ -337,6 +361,6 @@
 [solve-meta]
   analyzer-name = "dep"
   analyzer-version = 1
-  inputs-digest = "9bb6fd90291e987c895b33e55cd20d968e141b4583d47e28aa6dfb733b7a320b"
+  inputs-digest = "635dd31293fe8e0706f3b721b1d05ea2d12771c2dcaf86d17517347472755b31"
   solver-name = "gps-cdcl"
   solver-version = 1

Gopkg.toml

@@ -66,6 +66,10 @@
   name = "github.com/prometheus/client_golang"
   revision = "661e31bf844dfca9aeba15f27ea8aa0d485ad212"
 
+[[constraint]]
+  name = "github.com/klauspost/pgzip"
+  version = "1.1.0"
+
 [prune]
   go-tests = true
   unused-packages = true

LICENSE_OF_DEPENDENCIES.md

@@ -23,6 +23,7 @@
 - github.com/influxdata/yarpc [MIT LICENSE](https://github.com/influxdata/yarpc/blob/master/LICENSE)
 - github.com/jsternberg/zap-logfmt [MIT LICENSE](https://github.com/jsternberg/zap-logfmt/blob/master/LICENSE)
 - github.com/jwilder/encoding [MIT LICENSE](https://github.com/jwilder/encoding/blob/master/LICENSE)
+- github.com/klauspost/pgzip [MIT LICENSE](https://github.com/klauspost/pgzip/blob/master/LICENSE)
 - github.com/mattn/go-isatty [MIT LICENSE](https://github.com/mattn/go-isatty/blob/master/LICENSE)
 - github.com/matttproud/golang_protobuf_extensions [APACHE LICENSE](https://github.com/matttproud/golang_protobuf_extensions/blob/master/LICENSE)
 - github.com/opentracing/opentracing-go [MIT LICENSE](https://github.com/opentracing/opentracing-go/blob/master/LICENSE)

cmd/influxd/backup/backup.go

@@ -2,7 +2,6 @@
 package backup
 
 import (
-	"compress/gzip"
 	"encoding/binary"
 	"encoding/json"
 	"errors"
@@ -20,6 +19,7 @@ import (
 	"github.com/influxdata/influxdb/cmd/influxd/backup_util"
 	"github.com/influxdata/influxdb/services/snapshotter"
 	"github.com/influxdata/influxdb/tcp"
+        gzip "github.com/klauspost/pgzip"
 )
 
 const (

cmd/influxd/restore/restore.go

@@ -16,7 +16,7 @@ import (
 	"strconv"
 	"strings"
 
-	"compress/gzip"
+	gzip "github.com/klauspost/pgzip"
 
 	"github.com/influxdata/influxdb/cmd/influxd/backup_util"
 	tarstream "github.com/influxdata/influxdb/pkg/tar"