velero/pkg/restore/backup_extractor.go

108 lines
2.5 KiB
Go
Raw Normal View History

/*
Copyright 2019 the Velero contributors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package restore
import (
"archive/tar"
"compress/gzip"
"io"
"path/filepath"
"github.com/sirupsen/logrus"
"github.com/heptio/velero/pkg/util/filesystem"
)
// backupExtractor unzips/extracts a backup tarball to a local
// temp directory.
type backupExtractor struct {
log logrus.FieldLogger
fileSystem filesystem.Interface
}
// unzipAndExtractBackup extracts a reader on a gzipped tarball to a local temp directory
func (e *backupExtractor) unzipAndExtractBackup(src io.Reader) (string, error) {
gzr, err := gzip.NewReader(src)
if err != nil {
e.log.Infof("error creating gzip reader: %v", err)
return "", err
}
defer gzr.Close()
return e.readBackup(tar.NewReader(gzr))
}
Update ark restore to not open every single file open during extraction of the data Original error was: ``` ark -n <redacted> restore logs <redacted> time="2019-03-06T18:31:06Z" level=info msg="Not including resource" groupResource=nodes logSource="pkg/restore/restore.go:124" time="2019-03-06T18:31:06Z" level=info msg="Not including resource" groupResource=events logSource="pkg/restore/restore.go:124" time="2019-03-06T18:31:06Z" level=info msg="Not including resource" groupResource=events.events.k8s.io logSource="pkg/restore/restore.go:124" time="2019-03-06T18:31:06Z" level=info msg="Not including resource" groupResource=backups.ark.heptio.com logSource="pkg/restore/restore.go:124" time="2019-03-06T18:31:06Z" level=info msg="Not including resource" groupResource=restores.ark.heptio.com logSource="pkg/restore/restore.go:124" time="2019-03-06T18:31:06Z" level=info msg="Starting restore of backup backup/<redacted>" logSource="pkg/restore/restore.go:342" time="2019-03-06T18:31:06Z" level=info msg="error unzipping and extracting: open /tmp/604421455/resources/rolebindings.rbac.authorization.k8s.io/namespaces/<redacted>/<redacted>: too many open files" logSource="pkg/restore/restore.go:346" ``` Downloading the directory from s3 and untarring it I found 1036 files. The ulimit -n output says 1024. This is our team's best guess at a root cause. But the code fixed in the PR definitely is holding all the files open until the method closes: https://blog.learngoprogramming.com/gotchas-of-defer-in-go-1-8d070894cb01 Please note my go code abilities are not great and I did not test this. I just edited the file in github. All I did was remove the defer and put fileClose after the copy is done. Theoretically this should only hold one file open at a time now. Let me know if you want me to do any further steps. Thank you, -Asaf Signed-off-by: Asaf Erlich <aerlich@groupon.com>
2019-03-06 19:16:47 +00:00
func (e *backupExtractor) writeFile(target string, tarRdr *tar.Reader) error {
file, err := e.fileSystem.Create(target)
if err != nil {
return err
}
defer file.Close()
if _, err := io.Copy(file, tarRdr); err != nil {
return err
}
return nil
}
func (e *backupExtractor) readBackup(tarRdr *tar.Reader) (string, error) {
dir, err := e.fileSystem.TempDir("", "")
if err != nil {
e.log.Infof("error creating temp dir: %v", err)
return "", err
}
for {
header, err := tarRdr.Next()
if err == io.EOF {
break
}
if err != nil {
e.log.Infof("error reading tar: %v", err)
return "", err
}
target := filepath.Join(dir, header.Name)
switch header.Typeflag {
case tar.TypeDir:
err := e.fileSystem.MkdirAll(target, header.FileInfo().Mode())
if err != nil {
e.log.Infof("mkdirall error: %v", err)
return "", err
}
case tar.TypeReg:
// make sure we have the directory created
err := e.fileSystem.MkdirAll(filepath.Dir(target), header.FileInfo().Mode())
if err != nil {
e.log.Infof("mkdirall error: %v", err)
return "", err
}
// create the file
Update ark restore to not open every single file open during extraction of the data Original error was: ``` ark -n <redacted> restore logs <redacted> time="2019-03-06T18:31:06Z" level=info msg="Not including resource" groupResource=nodes logSource="pkg/restore/restore.go:124" time="2019-03-06T18:31:06Z" level=info msg="Not including resource" groupResource=events logSource="pkg/restore/restore.go:124" time="2019-03-06T18:31:06Z" level=info msg="Not including resource" groupResource=events.events.k8s.io logSource="pkg/restore/restore.go:124" time="2019-03-06T18:31:06Z" level=info msg="Not including resource" groupResource=backups.ark.heptio.com logSource="pkg/restore/restore.go:124" time="2019-03-06T18:31:06Z" level=info msg="Not including resource" groupResource=restores.ark.heptio.com logSource="pkg/restore/restore.go:124" time="2019-03-06T18:31:06Z" level=info msg="Starting restore of backup backup/<redacted>" logSource="pkg/restore/restore.go:342" time="2019-03-06T18:31:06Z" level=info msg="error unzipping and extracting: open /tmp/604421455/resources/rolebindings.rbac.authorization.k8s.io/namespaces/<redacted>/<redacted>: too many open files" logSource="pkg/restore/restore.go:346" ``` Downloading the directory from s3 and untarring it I found 1036 files. The ulimit -n output says 1024. This is our team's best guess at a root cause. But the code fixed in the PR definitely is holding all the files open until the method closes: https://blog.learngoprogramming.com/gotchas-of-defer-in-go-1-8d070894cb01 Please note my go code abilities are not great and I did not test this. I just edited the file in github. All I did was remove the defer and put fileClose after the copy is done. Theoretically this should only hold one file open at a time now. Let me know if you want me to do any further steps. Thank you, -Asaf Signed-off-by: Asaf Erlich <aerlich@groupon.com>
2019-03-06 19:16:47 +00:00
if err := e.writeFile(target, tarRdr); err != nil {
e.log.Infof("error copying: %v", err)
return "", err
}
}
}
return dir, nil
}