Migrate Ginkgo testing framework to V2, consolidate integration tests (#5097)

* Upgrade and convert ginkgo from v1 to v2
* Move all integration tests into integration folder
* Update TESTING.md

Signed-off-by: Derek Nola <derek.nola@suse.com>
pull/5106/head
Derek Nola 2022-02-09 08:22:53 -08:00 committed by GitHub
parent 13728058a4
commit e28be2912c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
18 changed files with 66 additions and 65 deletions

View File

@ -56,7 +56,7 @@ jobs:
- name: Run Integration Tests
run: |
chmod +x ./dist/artifacts/k3s
sudo -E env "PATH=$PATH" go test ./pkg/... ./tests/integration/... -run Integration
sudo -E env "PATH=$PATH" go test ./tests/integration/... -run Integration
- name: On Failure, Launch Debug Session
if: ${{ failure() }}
uses: mxschmitt/action-tmate@v3

2
go.mod
View File

@ -88,7 +88,7 @@ require (
github.com/mattn/go-sqlite3 v1.14.8
github.com/minio/minio-go/v7 v7.0.7
github.com/natefinch/lumberjack v2.0.0+incompatible
github.com/onsi/ginkgo v1.16.4
github.com/onsi/ginkgo/v2 v2.1.1
github.com/onsi/gomega v1.17.0
github.com/opencontainers/runc v1.0.3
github.com/opencontainers/selinux v1.8.3

3
go.sum
View File

@ -509,6 +509,7 @@ github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLe
github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
@ -884,6 +885,8 @@ github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108
github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc=
github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0=
github.com/onsi/ginkgo/v2 v2.1.1 h1:LCnPB85AvFNr91s0B2aDzEiiIg6MUwLYbryC1NSlWi8=
github.com/onsi/ginkgo/v2 v2.1.1/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c=
github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
github.com/onsi/gomega v1.3.0/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=

View File

@ -3,8 +3,7 @@ package flock_test
import (
"testing"
. "github.com/onsi/ginkgo"
"github.com/onsi/ginkgo/reporters"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/rancher/k3s/pkg/flock"
)
@ -31,7 +30,5 @@ var _ = Describe("file locks", func() {
func TestFlock(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecsWithDefaultAndCustomReporters(t, "Flock Suite", []Reporter{
reporters.NewJUnitReporter("/tmp/results/junit-flock.xml"),
})
RunSpecs(t, "Flock Suite")
}

View File

@ -35,7 +35,7 @@ To facilitate unit test creation, see `tests/util/runtime.go` helper functions.
All unit tests should be placed within the package of the file they test.
All unit test files should be named: `<FILE_UNDER_TEST>_test.go`.
All unit test functions should be named: `Test_Unit<FUNCTION_TO_TEST>` or `Test_Unit<RECEIVER>_<METHOD_TO_TEST>`.
See the [etcd unit test](https://github.com/k3s-io/k3s/blob/master/pkg/etcd/etcd_test.go) as an example.
See the [etcd unit test](../pkg/etcd/etcd_test.go) as an example.
### Running
@ -62,29 +62,27 @@ To facilitate K3s CLI testing, see `tests/util/cmd.go` helper functions.
### Format
Integration tests can be placed in two areas:
1. Next to the go package they intend to test.
2. In `tests/integration/<TEST_NAME>` for package agnostic testing.
Package specific integration tests should use the `<PACKAGE_UNDER_TEST>_test` package.
Package agnostic integration tests should use the `integration` package.
All integration tests should be placed under `tests/integration/<TEST_NAME>`.
All integration test files should be named: `<TEST_NAME>_int_test.go`.
All integration test functions should be named: `Test_Integration<TEST_NAME>`.
See the [etcd snapshot test](https://github.com/k3s-io/k3s/blob/master/pkg/etcd/etcd_int_test.go) as a package specific example.
See the [local storage test](https://github.com/k3s-io/k3s/blob/master/tests/integration/localstorage/localstorage_int_test.go) as a package agnostic example.
See the [local storage test](../tests/integration/localstorage/localstorage_int_test.go) as an example.
### Running
Integration tests can be run with no k3s cluster present, each test will spin up and kill the appropriate k3s server it needs.
Note: Integration tests must be run as root, prefix the commands below with `sudo -E env "PATH=$PATH"` if a sudo user.
```bash
go test ./pkg/... ./tests/integration/... -run Integration
go test ./tests/integration/... -run Integration
```
Additionally, to generate JUnit reporting for the tests, the Ginkgo CLI is used
```
ginkgo --junit-report=result.xml ./tests/integration/...
```
Integration tests can be run on an existing single-node cluster via compile time flag, tests will skip if the server is not configured correctly.
```bash
go test -ldflags "-X 'github.com/rancher/k3s/tests/util.existingServer=True'" ./pkg/... ./tests/integration/... -run Integration
go test -ldflags "-X 'github.com/rancher/k3s/tests/util.existingServer=True'" ./tests/integration/... -run Integration
```
Integration tests can also be run via a [Sonobuoy](https://sonobuoy.io/docs/v0.53.2/) plugin on an existing single-node cluster.
@ -201,10 +199,25 @@ See the [validate cluster test](../tests/e2e/validatecluster/validatecluster_tes
Generally, E2E tests are run as a nightly Jenkins job for QA. They can still be run locally but additional setup may be required. By default, all E2E tests are designed with `libvirt` as the underlying VM provider. Instructions for installing libvirt and its associated vagrant plugin, `vagrant-libvirt` can be found [here.](https://github.com/vagrant-libvirt/vagrant-libvirt#installation) `VirtualBox` is also supported as a backup VM provider.
Once setup is complete, E2E tests can be run with:
Once setup is complete, all E2E tests can be run with:
```bash
go test ./tests/e2e/... -run E2E
go test -timeout=15m ./tests/e2e/... -run E2E
```
Tests can be run individually with:
```bash
go test -timeout=15m ./tests/e2e/validatecluster/... -run E2E
#or
go test -timeout=15m ./tests/e2e/... -run E2EClusterValidation
```
Additionally, to generate junit reporting for the tests, the Ginkgo CLI is used. Installation instructions can be found [here.](https://onsi.github.io/ginkgo/#getting-started)
To run the all E2E tests and generate JUnit testing reports:
```
ginkgo --junit-report=result.xml ./tests/e2e/...
```
Note: The `go test` default timeout is 10 minutes, thus the `-timeout` flag should be used. The `ginkgo` default timeout is 1 hour, no timeout flag is needed.
## Contributing New Or Updated Tests

View File

@ -4,7 +4,7 @@ go 1.15
require (
github.com/kr/pretty v0.2.0 // indirect
github.com/onsi/ginkgo v1.16.4
github.com/onsi/ginkgo/v2 v2.1.1
github.com/onsi/gomega v1.17.0
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
)

View File

@ -1,3 +1,6 @@
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
@ -19,7 +22,9 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs=
github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
@ -32,6 +37,8 @@ github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W
github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc=
github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0=
github.com/onsi/ginkgo/v2 v2.1.1 h1:LCnPB85AvFNr91s0B2aDzEiiIg6MUwLYbryC1NSlWi8=
github.com/onsi/ginkgo/v2 v2.1.1/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c=
github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
github.com/onsi/gomega v1.17.0 h1:9Luw4uT5HTjHTN8+aNcSThgH1vdXnmdJ8xIfZ4wyTRE=
@ -60,6 +67,7 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

View File

@ -7,8 +7,7 @@ import (
"strings"
"testing"
. "github.com/onsi/ginkgo"
"github.com/onsi/ginkgo/reporters"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/rancher/k3s/tests/e2e"
)
@ -24,9 +23,7 @@ var installType = flag.String("installType", "", "version or nil to use latest c
func Test_E2EUpgradeValidation(t *testing.T) {
RegisterFailHandler(Fail)
flag.Parse()
RunSpecsWithDefaultAndCustomReporters(t, "Create Cluster Test Suite", []Reporter{
reporters.NewJUnitReporter("/tmp/results/junit-cc.xml"),
})
RunSpecs(t, "Create Cluster Test Suite")
}
var (

View File

@ -7,8 +7,7 @@ import (
"strings"
"testing"
. "github.com/onsi/ginkgo"
"github.com/onsi/ginkgo/reporters"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/rancher/k3s/tests/e2e"
)
@ -24,9 +23,7 @@ var installType = flag.String("installType", "", "version or nil to use latest c
func Test_E2EClusterValidation(t *testing.T) {
RegisterFailHandler(Fail)
flag.Parse()
RunSpecsWithDefaultAndCustomReporters(t, "Create Cluster Test Suite", []Reporter{
reporters.NewJUnitReporter("/tmp/results/junit-cc.xml"),
})
RunSpecs(t, "Create Cluster Test Suite")
}
var (

View File

@ -5,8 +5,7 @@ import (
"strings"
"testing"
. "github.com/onsi/ginkgo"
"github.com/onsi/ginkgo/reporters"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
testutil "github.com/rancher/k3s/tests/util"
)
@ -51,13 +50,12 @@ var _ = Describe("custom etcd args", func() {
var _ = AfterSuite(func() {
if !testutil.IsExistingServer() {
Expect(testutil.K3sKillServer(customEtcdArgsServer, true)).To(Succeed())
Expect(testutil.K3sKillServer(customEtcdArgsServer, false)).To(Succeed())
Expect(testutil.K3sCleanup(customEtcdArgsServer, true, "")).To(Succeed())
}
})
func Test_IntegrationCustomEtcdArgs(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecsWithDefaultAndCustomReporters(t, "Custom etcd Arguments", []Reporter{
reporters.NewJUnitReporter("/tmp/results/junit-ls.xml"),
})
RunSpecs(t, "Custom etcd Arguments")
}

View File

@ -5,8 +5,7 @@ import (
"strings"
"testing"
. "github.com/onsi/ginkgo"
"github.com/onsi/ginkgo/reporters"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
testutil "github.com/rancher/k3s/tests/util"
)
@ -59,7 +58,5 @@ var _ = AfterSuite(func() {
func Test_IntegrationDualStack(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecsWithDefaultAndCustomReporters(t, "Dual-Stack Suite", []Reporter{
reporters.NewJUnitReporter("/tmp/results/junit-ls.xml"),
})
RunSpecs(t, "Dual-Stack Suite")
}

View File

@ -4,8 +4,7 @@ import (
"strings"
"testing"
. "github.com/onsi/ginkgo"
"github.com/onsi/ginkgo/reporters"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
testutil "github.com/rancher/k3s/tests/util"
)
@ -113,9 +112,7 @@ var _ = AfterSuite(func() {
}
})
func Test_RestoreSnapshotIntegrationEtcd(t *testing.T) {
func Test_IntegrationEtcdRestoreSnapshot(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecsWithDefaultAndCustomReporters(t, "Etcd Suite", []Reporter{
reporters.NewJUnitReporter("/tmp/results/junit-etcd-restore.xml"),
})
RunSpecs(t, "Etcd Restore Suite")
}

View File

@ -6,8 +6,7 @@ import (
"testing"
"time"
. "github.com/onsi/ginkgo"
"github.com/onsi/ginkgo/reporters"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
testutil "github.com/rancher/k3s/tests/util"
)
@ -117,9 +116,7 @@ var _ = AfterSuite(func() {
}
})
func Test_IntegrationEtcd(t *testing.T) {
func Test_IntegrationEtcdSnapshot(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecsWithDefaultAndCustomReporters(t, "Etcd Suite", []Reporter{
reporters.NewJUnitReporter("/tmp/results/junit-etcd.xml"),
})
RunSpecs(t, "Etcd Snapshot Suite")
}

View File

@ -7,8 +7,7 @@ import (
"strings"
"testing"
. "github.com/onsi/ginkgo"
"github.com/onsi/ginkgo/reporters"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
testutil "github.com/rancher/k3s/tests/util"
)
@ -88,7 +87,5 @@ var _ = AfterSuite(func() {
func Test_IntegrationLocalStorage(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecsWithDefaultAndCustomReporters(t, "Local Storage Suite", []Reporter{
reporters.NewJUnitReporter("/tmp/results/junit-ls.xml"),
})
RunSpecs(t, "Local Storage Suite")
}

View File

@ -7,8 +7,7 @@ import (
"testing"
"time"
. "github.com/onsi/ginkgo"
"github.com/onsi/ginkgo/reporters"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
testutil "github.com/rancher/k3s/tests/util"
)
@ -149,7 +148,5 @@ var _ = AfterSuite(func() {
func Test_IntegrationSecretsEncryption(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecsWithDefaultAndCustomReporters(t, "Secrets Encryption Suite", []Reporter{
reporters.NewJUnitReporter("/tmp/results/junit-se.xml"),
})
RunSpecs(t, "Secrets Encryption Suite")
}

View File

@ -169,6 +169,9 @@ func K3sKillServer(server *K3sServer, releaseLock bool) error {
if err := syscall.Kill(-pgid, syscall.SIGKILL); err != nil {
return err
}
if err := server.cmd.Process.Kill(); err != nil {
return err
}
if releaseLock {
return flock.Release(server.lock)
}