182 lines
7.6 KiB
Go
182 lines
7.6 KiB
Go
/*
|
|
Copyright the Velero contributors.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
package e2e
|
|
|
|
import (
|
|
"fmt"
|
|
"os"
|
|
"os/exec"
|
|
"strconv"
|
|
"time"
|
|
|
|
"github.com/pkg/errors"
|
|
"golang.org/x/net/context"
|
|
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
|
|
veleroexec "github.com/vmware-tanzu/velero/pkg/util/exec"
|
|
)
|
|
|
|
const (
|
|
kibishiiNamespace = "kibishii-workload"
|
|
jumpPadPod = "jump-pad"
|
|
)
|
|
|
|
func installKibishii(ctx context.Context, namespace string, cloudPlatform string) error {
|
|
// We use kustomize to generate YAML for Kibishii from the checked-in yaml directories
|
|
kibishiiInstallCmd := exec.CommandContext(ctx, "kubectl", "apply", "-n", namespace, "-k",
|
|
"github.com/vmware-tanzu-experiments/distributed-data-generator/kubernetes/yaml/"+cloudPlatform)
|
|
|
|
_, _, err := veleroexec.RunCommand(kibishiiInstallCmd)
|
|
if err != nil {
|
|
return errors.Wrap(err, "failed to install kibishii")
|
|
}
|
|
|
|
kibishiiSetWaitCmd := exec.CommandContext(ctx, "kubectl", "rollout", "status", "statefulset.apps/kibishii-deployment",
|
|
"-n", namespace, "-w", "--timeout=30m")
|
|
kibishiiSetWaitCmd.Stdout = os.Stdout
|
|
kibishiiSetWaitCmd.Stderr = os.Stderr
|
|
_, _, err = veleroexec.RunCommand(kibishiiSetWaitCmd)
|
|
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
fmt.Printf("Waiting for kibishii jump-pad pod to be ready\n")
|
|
jumpPadWaitCmd := exec.CommandContext(ctx, "kubectl", "wait", "--for=condition=ready", "-n", namespace, "pod/jump-pad")
|
|
jumpPadWaitCmd.Stdout = os.Stdout
|
|
jumpPadWaitCmd.Stderr = os.Stderr
|
|
_, _, err = veleroexec.RunCommand(jumpPadWaitCmd)
|
|
if err != nil {
|
|
return errors.Wrapf(err, "Failed to wait for ready status of pod %s/%s", namespace, jumpPadPod)
|
|
}
|
|
|
|
return err
|
|
}
|
|
|
|
func generateData(ctx context.Context, namespace string, levels int, filesPerLevel int, dirsPerLevel int, fileSize int,
|
|
blockSize int, passNum int, expectedNodes int) error {
|
|
kibishiiGenerateCmd := exec.CommandContext(ctx, "kubectl", "exec", "-n", namespace, "jump-pad", "--",
|
|
"/usr/local/bin/generate.sh", strconv.Itoa(levels), strconv.Itoa(filesPerLevel), strconv.Itoa(dirsPerLevel), strconv.Itoa(fileSize),
|
|
strconv.Itoa(blockSize), strconv.Itoa(passNum), strconv.Itoa(expectedNodes))
|
|
fmt.Printf("kibishiiGenerateCmd cmd =%v\n", kibishiiGenerateCmd)
|
|
|
|
_, _, err := veleroexec.RunCommand(kibishiiGenerateCmd)
|
|
if err != nil {
|
|
return errors.Wrap(err, "failed to generate")
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func verifyData(ctx context.Context, namespace string, levels int, filesPerLevel int, dirsPerLevel int, fileSize int,
|
|
blockSize int, passNum int, expectedNodes int) error {
|
|
kibishiiVerifyCmd := exec.CommandContext(ctx, "kubectl", "exec", "-n", namespace, "jump-pad", "--",
|
|
"/usr/local/bin/verify.sh", strconv.Itoa(levels), strconv.Itoa(filesPerLevel), strconv.Itoa(dirsPerLevel), strconv.Itoa(fileSize),
|
|
strconv.Itoa(blockSize), strconv.Itoa(passNum), strconv.Itoa(expectedNodes))
|
|
fmt.Printf("kibishiiVerifyCmd cmd =%v\n", kibishiiVerifyCmd)
|
|
|
|
_, _, err := veleroexec.RunCommand(kibishiiVerifyCmd)
|
|
if err != nil {
|
|
return errors.Wrap(err, "failed to verify")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// runKibishiiTests runs kibishii tests on the provider.
|
|
func runKibishiiTests(client testClient, providerName, veleroCLI, veleroNamespace, backupName, restoreName, backupLocation string,
|
|
useVolumeSnapshots bool) error {
|
|
fiveMinTimeout, _ := context.WithTimeout(context.Background(), 5*time.Minute)
|
|
oneHourTimeout, _ := context.WithTimeout(context.Background(), time.Minute*60)
|
|
timeout := 10 * time.Minute
|
|
interval := 5 * time.Second
|
|
|
|
if err := createNamespace(fiveMinTimeout, client, kibishiiNamespace); err != nil {
|
|
return errors.Wrapf(err, "Failed to create namespace %s to install Kibishii workload", kibishiiNamespace)
|
|
}
|
|
|
|
if err := installKibishii(fiveMinTimeout, kibishiiNamespace, providerName); err != nil {
|
|
return errors.Wrap(err, "Failed to install Kibishii workload")
|
|
}
|
|
|
|
// wait for kibishii pod startup
|
|
// TODO - Fix kibishii so we can check that it is ready to go
|
|
fmt.Printf("Waiting for kibishii pods to be ready\n")
|
|
if err := waitForKibishiiPods(oneHourTimeout, client, kibishiiNamespace); err != nil {
|
|
return errors.Wrapf(err, "Failed to wait for ready status of kibishii pods in %s", kibishiiNamespace)
|
|
}
|
|
|
|
if err := generateData(oneHourTimeout, kibishiiNamespace, 2, 10, 10, 1024, 1024, 0, 2); err != nil {
|
|
return errors.Wrap(err, "Failed to generate data")
|
|
}
|
|
|
|
if err := veleroBackupNamespace(oneHourTimeout, veleroCLI, veleroNamespace, backupName, kibishiiNamespace, backupLocation, useVolumeSnapshots); err != nil {
|
|
veleroBackupLogs(fiveMinTimeout, veleroCLI, veleroNamespace, backupName)
|
|
return errors.Wrapf(err, "Failed to backup kibishii namespace %s", kibishiiNamespace)
|
|
}
|
|
|
|
if providerName == "vsphere" && useVolumeSnapshots {
|
|
// Wait for uploads started by the Velero Plug-in for vSphere to complete
|
|
// TODO - remove after upload progress monitoring is implemented
|
|
fmt.Println("Waiting for vSphere uploads to complete")
|
|
if err := waitForVSphereUploadCompletion(oneHourTimeout, time.Hour, kibishiiNamespace); err != nil {
|
|
return errors.Wrapf(err, "Error waiting for uploads to complete")
|
|
}
|
|
}
|
|
fmt.Printf("Simulating a disaster by removing namespace %s\n", kibishiiNamespace)
|
|
if err := client.clientGo.CoreV1().Namespaces().Delete(oneHourTimeout, kibishiiNamespace, metav1.DeleteOptions{}); err != nil {
|
|
return errors.Wrap(err, "Failed to simulate a disaster")
|
|
}
|
|
// wait for ns delete
|
|
err := waitForNamespaceDeletion(interval, timeout, client, kibishiiNamespace)
|
|
if err != nil {
|
|
return errors.Wrapf(err, fmt.Sprintf("Failed to wait for deletion of namespace %s", kibishiiNamespace))
|
|
}
|
|
|
|
if err := veleroRestore(oneHourTimeout, veleroCLI, veleroNamespace, restoreName, backupName); err != nil {
|
|
veleroRestoreLogs(fiveMinTimeout, veleroCLI, veleroNamespace, restoreName)
|
|
return errors.Wrapf(err, "Restore %s failed from backup %s", restoreName, backupName)
|
|
}
|
|
|
|
// wait for kibishii pod startup
|
|
// TODO - Fix kibishii so we can check that it is ready to go
|
|
fmt.Printf("Waiting for kibishii pods to be ready\n")
|
|
if err := waitForKibishiiPods(oneHourTimeout, client, kibishiiNamespace); err != nil {
|
|
return errors.Wrapf(err, "Failed to wait for ready status of kibishii pods in %s", kibishiiNamespace)
|
|
}
|
|
|
|
// TODO - check that namespace exists
|
|
fmt.Printf("running kibishii verify\n")
|
|
if err := verifyData(oneHourTimeout, kibishiiNamespace, 2, 10, 10, 1024, 1024, 0, 2); err != nil {
|
|
return errors.Wrap(err, "Failed to verify data generated by kibishii")
|
|
}
|
|
|
|
if err := client.clientGo.CoreV1().Namespaces().Delete(oneHourTimeout, kibishiiNamespace, metav1.DeleteOptions{}); err != nil {
|
|
return errors.Wrapf(err, "Failed to cleanup %s workload namespace", kibishiiNamespace)
|
|
}
|
|
// wait for ns delete
|
|
if err = waitForNamespaceDeletion(interval, timeout, client, kibishiiNamespace); err != nil {
|
|
return errors.Wrapf(err, fmt.Sprintf("Failed to wait for deletion of namespace %s", kibishiiNamespace))
|
|
}
|
|
fmt.Printf("kibishii test completed successfully\n")
|
|
return nil
|
|
}
|
|
|
|
func waitForKibishiiPods(ctx context.Context, client testClient, kibishiiNamespace string) error {
|
|
return waitForPods(ctx, client, kibishiiNamespace, []string{"jump-pad", "etcd0", "etcd1", "etcd2", "kibishii-deployment-0", "kibishii-deployment-1"})
|
|
}
|