Merge pull request #10798 from govargo/csi-hostpath-v1.6.0

Upgrade csi-hostpath-driver addon to v1.6.0

deploy/addons/csi-hostpath-driver/deploy/csi-hostpath-attacher.yaml.tmpl

@@ -19,6 +19,7 @@ metadata:
   namespace: kube-system
   labels:
     app: csi-hostpath-attacher
+    addonmanager.kubernetes.io/mode: Reconcile
 spec:
   selector:
     app: csi-hostpath-attacher
@@ -42,6 +43,7 @@ spec:
     metadata:
       labels:
         app: csi-hostpath-attacher
+        addonmanager.kubernetes.io/mode: Reconcile
         kubernetes.io/minikube-addons: csi-hostpath-driver
     spec:
       affinity:

deploy/addons/csi-hostpath-driver/deploy/csi-hostpath-driverinfo.yaml.tmpl

@@ -17,6 +17,8 @@ kind: CSIDriver
 metadata:
   name: hostpath.csi.k8s.io
   namespace: kube-system
+  labels:
+    addonmanager.kubernetes.io/mode: Reconcile
 spec:
   # Supports persistent and ephemeral inline volumes.
   volumeLifecycleModes:

deploy/addons/csi-hostpath-driver/deploy/csi-hostpath-plugin.yaml.tmpl

@@ -23,6 +23,7 @@ metadata:
   namespace: kube-system
   labels:
     app: csi-hostpathplugin
+    addonmanager.kubernetes.io/mode: Reconcile
 spec:
   selector:
     app: csi-hostpathplugin
@@ -35,6 +36,8 @@ apiVersion: apps/v1
 metadata:
   name: csi-hostpathplugin
   namespace: kube-system
+  labels:
+    addonmanager.kubernetes.io/mode: Reconcile
 spec:
   serviceName: "csi-hostpathplugin"
   # One replica only:
@@ -45,13 +48,45 @@ spec:
   selector:
     matchLabels:
       app: csi-hostpathplugin
+      addonmanager.kubernetes.io/mode: Reconcile
   template:
     metadata:
       labels:
         app: csi-hostpathplugin
+        addonmanager.kubernetes.io/mode: Reconcile
         kubernetes.io/minikube-addons: csi-hostpath-driver
     spec:
+      serviceAccount: csi-external-health-monitor-controller
       containers:
+        - name: csi-external-health-monitor-agent
+          image: {{.CustomRegistries.HostMonitorAgent  | default .ImageRepository | default .Registries.HostMonitorAgent }}{{.Images.HostMonitorAgent}}
+          args:
+            - "--v=5"
+            - "--csi-address=$(ADDRESS)"
+          env:
+            - name: NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+            - name: ADDRESS
+              value: /csi/csi.sock
+          imagePullPolicy: "IfNotPresent"
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /csi
+        - name: csi-external-health-monitor-controller
+          image: {{.CustomRegistries.HostMonitorController  | default .ImageRepository | default .Registries.HostMonitorController }}{{.Images.HostMonitorController}}
+          args:
+            - "--v=5"
+            - "--csi-address=$(ADDRESS)"
+            - "--leader-election"
+          env:
+            - name: ADDRESS
+              value: /csi/csi.sock
+          imagePullPolicy: "IfNotPresent"
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /csi
         - name: node-driver-registrar
           image: {{.CustomRegistries.NodeDriverRegistrar  | default .ImageRepository | default .Registries.NodeDriverRegistrar }}{{.Images.NodeDriverRegistrar}}
           args:

deploy/addons/csi-hostpath-driver/deploy/csi-hostpath-provisioner.yaml.tmpl

@@ -19,6 +19,7 @@ metadata:
   namespace: kube-system
   labels:
     app: csi-hostpath-provisioner
+    addonmanager.kubernetes.io/mode: Reconcile
 spec:
   selector:
     app: csi-hostpath-provisioner 
@@ -32,6 +33,8 @@ apiVersion: apps/v1
 metadata:
   name: csi-hostpath-provisioner
   namespace: kube-system
+  labels:
+    addonmanager.kubernetes.io/mode: Reconcile
 spec:
   serviceName: "csi-hostpath-provisioner"
   replicas: 1
@@ -42,6 +45,7 @@ spec:
     metadata:
       labels:
         app: csi-hostpath-provisioner
+        addonmanager.kubernetes.io/mode: Reconcile
         kubernetes.io/minikube-addons: csi-hostpath-driver
     spec:
       affinity:

deploy/addons/csi-hostpath-driver/deploy/csi-hostpath-resizer.yaml.tmpl

@@ -19,6 +19,7 @@ metadata:
   namespace: kube-system
   labels:
     app: csi-hostpath-resizer
+    addonmanager.kubernetes.io/mode: Reconcile
 spec:
   selector:
     app: csi-hostpath-resizer
@@ -32,6 +33,8 @@ apiVersion: apps/v1
 metadata:
   name: csi-hostpath-resizer
   namespace: kube-system
+  labels:
+    addonmanager.kubernetes.io/mode: Reconcile
 spec:
   serviceName: "csi-hostpath-resizer"
   replicas: 1
@@ -42,6 +45,7 @@ spec:
     metadata:
       labels:
         app: csi-hostpath-resizer
+        addonmanager.kubernetes.io/mode: Reconcile
         kubernetes.io/minikube-addons: csi-hostpath-driver
     spec:
       affinity:

deploy/addons/csi-hostpath-driver/deploy/csi-hostpath-snapshotter.yaml.tmpl

@@ -19,6 +19,7 @@ metadata:
   namespace: kube-system
   labels:
     app: csi-hostpath-snapshotter
+    addonmanager.kubernetes.io/mode: Reconcile
 spec:
   selector:
     app: csi-hostpath-snapshotter
@@ -32,16 +33,20 @@ apiVersion: apps/v1
 metadata:
   name: csi-hostpath-snapshotter
   namespace: kube-system
+  labels:
+    addonmanager.kubernetes.io/mode: Reconcile
 spec:
   serviceName: "csi-hostpath-snapshotter"
   replicas: 1
   selector:
     matchLabels:
       app: csi-hostpath-snapshotter
+      addonmanager.kubernetes.io/mode: Reconcile
   template:
     metadata:
       labels:
         app: csi-hostpath-snapshotter
+        addonmanager.kubernetes.io/mode: Reconcile
         kubernetes.io/minikube-addons: csi-hostpath-driver
     spec:
       affinity:

deploy/addons/csi-hostpath-driver/deploy/csi-hostpath-storageclass.yaml.tmpl

@@ -16,6 +16,8 @@ apiVersion: storage.k8s.io/v1
 kind: StorageClass
 metadata:
   name: csi-hostpath-sc
+  labels:
+    addonmanager.kubernetes.io/mode: Reconcile
 provisioner: hostpath.csi.k8s.io #csi-hostpath
 reclaimPolicy: Delete
 volumeBindingMode: Immediate

deploy/addons/csi-hostpath-driver/rbac/rbac-external-health-monitor-agent.yaml.tmpl

@@ -0,0 +1,66 @@
+# Copyright 2018 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This YAML file contains all RBAC objects that are necessary to run external
+# CSI health monitor agent.
+#
+# In production, each CSI driver deployment has to be customized:
+# - to avoid conflicts, use non-default namespace and different names
+#   for non-namespaced entities like the ClusterRole
+# - decide whether the deployment replicates the external CSI
+#   health monitor agent, in which case leadership election must be enabled;
+#   this influences the RBAC setup, see below
+
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: csi-external-health-monitor-agent
+  namespace: kube-system
+
+---
+# Health monitor agent must be able to work with PVs, PVCs, Nodes and Pods
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: external-health-monitor-agent-runner
+rules:
+  - apiGroups: [""]
+    resources: ["persistentvolumes"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["nodes"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["pods"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["events"]
+    verbs: ["get", "list", "watch", "create", "patch"]
+
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: csi-external-health-monitor-agent-role
+subjects:
+  - kind: ServiceAccount
+    name: csi-external-health-monitor-agent
+    namespace: kube-system
+roleRef:
+  kind: ClusterRole
+  name: external-health-monitor-agent-runner
+  apiGroup: rbac.authorization.k8s.io

deploy/addons/csi-hostpath-driver/rbac/rbac-external-health-monitor-controller.yaml.tmpl

@@ -0,0 +1,94 @@
+# Copyright 2018 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This YAML file contains all RBAC objects that are necessary to run external
+# CSI health monitor controller.
+#
+# In production, each CSI driver deployment has to be customized:
+# - to avoid conflicts, use non-default namespace and different names
+#   for non-namespaced entities like the ClusterRole
+# - decide whether the deployment replicates the external CSI
+#   health monitor controller, in which case leadership election must be enabled;
+#   this influences the RBAC setup, see below
+
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: csi-external-health-monitor-controller
+  namespace: kube-system
+
+---
+# Health monitor controller must be able to work with PVs, PVCs, Nodes and Pods
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: external-health-monitor-controller-runner
+rules:
+  - apiGroups: [""]
+    resources: ["persistentvolumes"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["nodes"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["pods"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["events"]
+    verbs: ["get", "list", "watch", "create", "patch"]
+
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: csi-external-health-monitor-controller-role
+subjects:
+  - kind: ServiceAccount
+    name: csi-external-health-monitor-controller
+    namespace: kube-system
+roleRef:
+  kind: ClusterRole
+  name: external-health-monitor-controller-runner
+  apiGroup: rbac.authorization.k8s.io
+
+---
+# Health monitor controller must be able to work with configmaps or leases in the current namespace
+# if (and only if) leadership election is enabled
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  namespace: kube-system
+  name: external-health-monitor-controller-cfg
+rules:
+- apiGroups: ["coordination.k8s.io"]
+  resources: ["leases"]
+  verbs: ["get", "watch", "list", "delete", "update", "create"]
+
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: csi-external-health-monitor-controller-role-cfg
+  namespace: kube-system
+subjects:
+  - kind: ServiceAccount
+    name: csi-external-health-monitor-controller
+    namespace: kube-system
+roleRef:
+  kind: Role
+  name: external-health-monitor-controller-cfg
+  apiGroup: rbac.authorization.k8s.io

deploy/addons/volumesnapshots/csi-hostpath-snapshotclass.yaml.tmpl

@@ -0,0 +1,24 @@
+# Copyright 2018 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Usage of the v1 API implies that the cluster must have
+# external-snapshotter v4.x installed.
+apiVersion: snapshot.storage.k8s.io/v1
+kind: VolumeSnapshotClass
+metadata:
+  name: csi-hostpath-snapclass
+  labels:
+    addonmanager.kubernetes.io/mode: EnsureExists
+driver: hostpath.csi.k8s.io #csi-hostpath
+deletionPolicy: Delete

pkg/minikube/assets/addons.go

@@ -518,6 +518,13 @@ var Addons = map[string]*Addon{
 		"GCPAuthWebhook": "gcr.io",
 	}),
 	"volumesnapshots": NewAddon([]*BinAsset{
+		// make sure the order of apply. `csi-hostpath-snapshotclass` must be the first position, because it depends on `snapshot.storage.k8s.io_volumesnapshotclasses`
+		// if user disable volumesnapshots addon and delete `csi-hostpath-snapshotclass` after `snapshot.storage.k8s.io_volumesnapshotclasses`, kubernetes will return the error
+		MustBinAsset(
+			"deploy/addons/volumesnapshots/csi-hostpath-snapshotclass.yaml.tmpl",
+			vmpath.GuestAddonsDir,
+			"csi-hostpath-snapshotclass.yaml",
+			"0640"),
 		MustBinAsset(
 			"deploy/addons/volumesnapshots/snapshot.storage.k8s.io_volumesnapshotclasses.yaml.tmpl",
 			vmpath.GuestAddonsDir,
@@ -544,7 +551,7 @@ var Addons = map[string]*Addon{
 			"volume-snapshot-controller-deployment.yaml",
 			"0640"),
 	}, false, "volumesnapshots", map[string]string{
-		"SnapshotController": "sig-storage/snapshot-controller:v4.0.0",
+		"SnapshotController": "sig-storage/snapshot-controller:v4.0.0@sha256:00fcc441ea9f72899c25eed61d602272a2a58c5f0014332bdcb5ac24acef08e4",
 	}, map[string]string{
 		"SnapshotController": "k8s.gcr.io",
 	}),
@@ -554,6 +561,16 @@ var Addons = map[string]*Addon{
 			vmpath.GuestAddonsDir,
 			"rbac-external-attacher.yaml",
 			"0640"),
+		MustBinAsset(
+			"deploy/addons/csi-hostpath-driver/rbac/rbac-external-health-monitor-agent.yaml.tmpl",
+			vmpath.GuestAddonsDir,
+			"rbac-external-health-monitor-agent.yaml",
+			"0640"),
+		MustBinAsset(
+			"deploy/addons/csi-hostpath-driver/rbac/rbac-external-health-monitor-controller.yaml.tmpl",
+			vmpath.GuestAddonsDir,
+			"rbac-external-health-monitor-controller.yaml",
+			"0640"),
 		MustBinAsset(
 			"deploy/addons/csi-hostpath-driver/rbac/rbac-external-provisioner.yaml.tmpl",
 			vmpath.GuestAddonsDir,
@@ -605,21 +622,25 @@ var Addons = map[string]*Addon{
 			"csi-hostpath-storageclass.yaml",
 			"0640"),
 	}, false, "csi-hostpath-driver", map[string]string{
-		"Attacher":            "k8scsi/csi-attacher:v3.0.0-rc1@sha256:8fcb9472310dd424c4da8ee06ff200b5e6f091dff39a079e470599e4d0dcf328",
+		"Attacher":              "sig-storage/csi-attacher:v3.1.0@sha256:50c3cfd458fc8e0bf3c8c521eac39172009382fc66dc5044a330d137c6ed0b09",
-		"NodeDriverRegistrar": "k8scsi/csi-node-driver-registrar:v1.3.0@sha256:9622c6a6dac7499a055a382930f4de82905a3c5735c0753f7094115c9c871309",
+		"HostMonitorAgent":      "sig-storage/csi-external-health-monitor-agent:v0.2.0@sha256:c20d4a4772599e68944452edfcecc944a1df28c19e94b942d526ca25a522ea02",
-		"HostPathPlugin":      "k8scsi/hostpathplugin:v1.4.0-rc2@sha256:aa223f9df8c1d477a9f2a4a2a7d104561e6d365e54671aacbc770dffcc0683ad",
+		"HostMonitorController": "sig-storage/csi-external-health-monitor-controller:v0.2.0@sha256:14988b598a180cc0282f3f4bc982371baf9a9c9b80878fb385f8ae8bd04ecf16",
-		"LivenessProbe":       "k8scsi/livenessprobe:v1.1.0@sha256:dde617756e0f602adc566ab71fd885f1dad451ad3fb063ac991c95a2ff47aea5",
+		"NodeDriverRegistrar":   "sig-storage/csi-node-driver-registrar:v2.0.1@sha256:e07f914c32f0505e4c470a62a40ee43f84cbf8dc46ff861f31b14457ccbad108",
-		"Resizer":             "k8scsi/csi-resizer:v0.6.0-rc1@sha256:75ad39004ac49267981c9cb3323a7f73f0b203e1c181117363bf215e10144e8a",
+		"HostPathPlugin":        "sig-storage/hostpathplugin:v1.6.0@sha256:b526bd29630261eceecf2d38c84d4f340a424d57e1e2661111e2649a4663b659",
-		"Snapshotter":         "k8scsi/csi-snapshotter:v2.1.0@sha256:35ead85dd09aa8cc612fdb598d4e0e2f048bef816f1b74df5eeab67cd21b10aa",
+		"LivenessProbe":         "sig-storage/livenessprobe:v2.2.0@sha256:48da0e4ed7238ad461ea05f68c25921783c37b315f21a5c5a2780157a6460994",
-		"Provisioner":         "k8s-staging-sig-storage/csi-provisioner:v2.0.0-rc2@sha256:8f36191970a82677ffe222007b08395dd7af0a5bb5b93db0e82523b43de2bfb2",
+		"Resizer":               "sig-storage/csi-resizer:v1.1.0@sha256:7a5ba58a44e0d749e0767e4e37315bcf6a61f33ce3185c1991848af4db0fb70a",
+		"Snapshotter":           "sig-storage/csi-snapshotter:v4.0.0@sha256:51f2dfde5bccac7854b3704689506aeecfb793328427b91115ba253a93e60782",
+		"Provisioner":           "sig-storage/csi-provisioner:v2.1.0@sha256:20c828075d1e36f679d6a91e905b0927141eef5e15be0c9a1ca4a6a0ed9313d2",
 	}, map[string]string{
-		"Attacher":            "quay.io",
+		"Attacher":              "k8s.gcr.io",
-		"NodeDriverRegistrar": "quay.io",
+		"HostMonitorAgent":      "k8s.gcr.io",
-		"HostPathPlugin":      "quay.io",
+		"HostMonitorController": "k8s.gcr.io",
-		"LivenessProbe":       "quay.io",
+		"NodeDriverRegistrar":   "k8s.gcr.io",
-		"Resizer":             "quay.io",
+		"HostPathPlugin":        "k8s.gcr.io",
-		"Snapshotter":         "quay.io",
+		"LivenessProbe":         "k8s.gcr.io",
-		"Provisioner":         "gcr.io",
+		"Resizer":               "k8s.gcr.io",
+		"Snapshotter":           "k8s.gcr.io",
+		"Provisioner":           "k8s.gcr.io",
 	}),
 }
 

site/content/en/docs/tutorials/volume_snapshots_and_csi.md

@@ -65,23 +65,16 @@ minikube addons enable volumesnapshots
 minikube addons enable csi-hostpath-driver
 ```
 
-<h2 class="step"><span class="fa-stack fa-1x"><i class="fa fa-circle fa-stack-2x"></i><strong class="fa-stack-1x text-primary">3</strong></span>Register volume snapshot class</h2>
+<h2 class="step"><span class="fa-stack fa-1x"><i class="fa fa-circle fa-stack-2x"></i><strong class="fa-stack-1x text-primary">3</strong></span>Check volume snapshot class</h2>
 
-Before creating volume snapshot, you have to register [Volume Snapshot Classes](https://kubernetes.io/docs/concepts/storage/volume-snapshot-classes/) to your cluster.
+When you create the volume snapshot, you have to register [Volume Snapshot Classes](https://kubernetes.io/docs/concepts/storage/volume-snapshot-classes/) to your cluster.
-Here is an example `VolumeSnapshotClass` config to register:
+The default `VolumeSnapshotClass` called `csi-hostpath-snapclass` is already registered by `csi-hostpath-driver` addon.
-
+You can check the `VolumeSnapshotClass` by the following command:
-```yaml
-# snapshotclass.yaml
-apiVersion: snapshot.storage.k8s.io/v1
-kind: VolumeSnapshotClass
-metadata:
-  name: csi-hostpath-snapclass
-driver: hostpath.csi.k8s.io #csi-hostpath
-deletionPolicy: Delete
-```
 
 ```shell
-kubectl apply -f snapshotclass.yaml
+kubectl get volumesnapshotclasses
+NAME                     DRIVER                DELETIONPOLICY   AGE
+csi-hostpath-snapclass   hostpath.csi.k8s.io   Delete           10s
 ```
 
 <h2 class="step"><span class="fa-stack fa-1x"><i class="fa fa-circle fa-stack-2x"></i><strong class="fa-stack-1x text-primary">4</strong></span>Prepare persistent volume</h2>
@@ -138,6 +131,7 @@ kubectl apply -f example-csi-snapshot.yaml
 You could get volume snapshot. You can confirm your volume snapshot by the following command:
 
 ```shell
+kubectl get volumesnapshot
 NAME            READYTOUSE   SOURCEPVC   SOURCESNAPSHOTCONTENT   RESTORESIZE   SNAPSHOTCLASS            SNAPSHOTCONTENT                                    CREATIONTIME   AGE
 snapshot-demo   true         csi-pvc                             1Gi           csi-hostpath-snapclass   snapcontent-19730fcb-c34a-4f1a-abf2-6c5a9808076b   5s             5s
 ```

test/integration/addons_test.go

@@ -491,12 +491,6 @@ func validateCSIDriverAndSnapshots(ctx context.Context, t *testing.T, profile st
 		t.Fatalf("failed waiting for pod task-pv-pod: %v", err)
 	}
 
-	// create sample snapshotclass
-	rr, err = Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "create", "-f", filepath.Join(*testdataDir, "csi-hostpath-driver", "snapshotclass.yaml")))
-	if err != nil {
-		t.Logf("creating snapshostclass with %s failed: %v", rr.Command(), err)
-	}
-
 	// create volume snapshot
 	rr, err = Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "create", "-f", filepath.Join(*testdataDir, "csi-hostpath-driver", "snapshot.yaml")))
 	if err != nil {

test/integration/testdata/csi-hostpath-driver/snapshotclass.yaml

@@ -1,6 +0,0 @@
-apiVersion: snapshot.storage.k8s.io/v1
-kind: VolumeSnapshotClass
-metadata:
-  name: csi-hostpath-snapclass
-driver: hostpath.csi.k8s.io #csi-hostpath
-deletionPolicy: Delete