diff --git a/deploy/addons/volcano/volcano-development.yaml.tmpl b/deploy/addons/volcano/volcano-development.yaml.tmpl index e6889550db..ce450f4241 100644 --- a/deploy/addons/volcano/volcano-development.yaml.tmpl +++ b/deploy/addons/volcano/volcano-development.yaml.tmpl @@ -140,7 +140,7 @@ spec: priorityClassName: system-cluster-critical containers: - args: - - --enabled-admission=/jobs/mutate,/jobs/validate,/podgroups/mutate,/pods/validate,/pods/mutate,/queues/mutate,/queues/validate + - --enabled-admission=/jobs/mutate,/jobs/validate,/podgroups/validate,/queues/mutate,/queues/validate,/hypernodes/validate - --tls-cert-file=/admission.local.config/certificates/tls.crt - --tls-private-key-file=/admission.local.config/certificates/tls.key - --ca-cert-file=/admission.local.config/certificates/ca.crt @@ -269,7 +269,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.16.4 + controller-gen.kubebuilder.io/version: v0.17.0 name: jobs.batch.volcano.sh spec: group: batch.volcano.sh @@ -323,6 +323,18 @@ spec: format: int32 minimum: 1 type: integer + networkTopology: + properties: + highestTierAllowed: + default: 1 + type: integer + mode: + default: hard + enum: + - hard + - soft + type: string + type: object plugins: additionalProperties: items: @@ -3083,6 +3095,39 @@ spec: x-kubernetes-list-map-keys: - name x-kubernetes-list-type: map + resources: + properties: + claims: + items: + properties: + name: + type: string + request: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object restartPolicy: type: string runtimeClassName: @@ -3125,6 +3170,8 @@ spec: runAsUser: format: int64 type: integer + seLinuxChangePolicy: + type: string seLinuxOptions: properties: level: @@ -4264,7 +4311,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.16.4 + controller-gen.kubebuilder.io/version: v0.17.0 name: commands.bus.volcano.sh spec: group: bus.volcano.sh @@ -4361,6 +4408,15 @@ metadata: namespace: volcano-system --- # Source: volcano/templates/controllers.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: volcano-controller-configmap + namespace: volcano-system +data: + volcano-controller.conf: | +--- +# Source: volcano/templates/controllers.yaml kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: @@ -4415,10 +4471,10 @@ rules: resources: ["networkpolicies"] verbs: ["get", "create", "delete"] - apiGroups: ["apps"] - resources: ["daemonsets", "statefulsets"] + resources: ["daemonsets"] verbs: ["get"] - apiGroups: ["apps"] - resources: ["replicasets"] + resources: ["replicasets", "statefulsets"] verbs: ["get", "list", "watch"] - apiGroups: ["batch"] resources: ["jobs"] @@ -4426,6 +4482,9 @@ rules: - apiGroups: ["coordination.k8s.io"] resources: ["leases"] verbs: ["get", "create", "update", "watch"] + - apiGroups: ["topology.volcano.sh"] + resources: ["hypernodes", "hypernodes/status"] + verbs: ["list", "watch", "get", "create", "delete", "update", "patch"] --- # Source: volcano/templates/controllers.yaml kind: ClusterRoleBinding @@ -4513,6 +4572,11 @@ spec: - ALL runAsNonRoot: true runAsUser: 1000 + env: + - name: KUBE_POD_NAMESPACE + value: volcano-system + - name: HELM_RELEASE_NAME + value: volcano --- # Source: volcano/templates/scheduler.yaml apiVersion: v1 @@ -4601,13 +4665,16 @@ rules: verbs: ["get", "list", "watch", "create", "delete", "update"] - apiGroups: ["scheduling.incubator.k8s.io", "scheduling.volcano.sh"] resources: ["queues/status"] - verbs: ["patch"] + verbs: ["update"] - apiGroups: ["scheduling.incubator.k8s.io", "scheduling.volcano.sh"] resources: ["podgroups"] verbs: ["list", "watch", "update"] - apiGroups: ["nodeinfo.volcano.sh"] resources: ["numatopologies"] verbs: ["get", "list", "watch", "delete"] + - apiGroups: ["topology.volcano.sh"] + resources: ["hypernodes", "hypernodes/status"] + verbs: ["get", "list", "watch"] - apiGroups: [""] resources: ["configmaps"] verbs: ["get", "create", "delete", "update"] @@ -4617,6 +4684,15 @@ rules: - apiGroups: ["coordination.k8s.io"] resources: ["leases"] verbs: ["get", "create", "update", "watch"] + - apiGroups: ["resource.k8s.io"] + resources: ["resourceclaims"] + verbs: ["get", "list", "watch", "create", "update", "patch"] + - apiGroups: ["resource.k8s.io"] + resources: ["resourceclaims/status"] + verbs: ["update"] + - apiGroups: ["resource.k8s.io"] + resources: ["deviceclasses","resourceslices"] + verbs: ["get", "list", "watch", "create"] --- # Source: volcano/templates/scheduler.yaml kind: ClusterRoleBinding @@ -4717,15 +4793,14 @@ spec: configMap: name: volcano-scheduler-configmap - name: klog-sock - hostPath: - path: /tmp/klog-socks + emptyDir: {} --- # Source: volcano/templates/scheduling_v1beta1_podgroup.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.16.4 + controller-gen.kubebuilder.io/version: v0.17.0 name: podgroups.scheduling.volcano.sh spec: group: scheduling.volcano.sh @@ -4811,6 +4886,24 @@ spec: if there's not enough resources to start each task, the scheduler will not start anyone. type: object + networkTopology: + description: NetworkTopology defines the NetworkTopology config, this + field works in conjunction with network topology feature and hyperNode + CRD. + properties: + highestTierAllowed: + default: 1 + description: HighestTierAllowed specifies the highest tier that + a job allowed to cross when scheduling. + type: integer + mode: + default: hard + description: Mode specifies the mode of the network topology constrain. + enum: + - hard + - soft + type: string + type: object priorityClassName: description: |- If specified, indicates the PodGroup's priority. "system-node-critical" and @@ -4821,6 +4914,7 @@ spec: default. type: string queue: + default: default description: |- Queue defines the queue to allocate resource for PodGroup; if queue does not exist, the PodGroup will not be scheduled. Defaults to `default` Queue with the lowest weight. @@ -4887,7 +4981,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.16.4 + controller-gen.kubebuilder.io/version: v0.17.0 name: queues.scheduling.volcano.sh spec: group: scheduling.volcano.sh @@ -4901,7 +4995,11 @@ spec: singular: queue scope: Cluster versions: - - name: v1beta1 + - additionalPrinterColumns: + - jsonPath: .spec.parent + name: PARENT + type: string + name: v1beta1 schema: openAPIV3Schema: description: Queue is a queue of PodGroup. @@ -5033,7 +5131,10 @@ spec: description: Type define the type of queue type: string weight: + default: 1 format: int32 + maximum: 65535 + minimum: 1 type: integer type: object status: @@ -5103,7 +5204,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.16.4 + controller-gen.kubebuilder.io/version: v0.17.0 name: numatopologies.nodeinfo.volcano.sh spec: group: nodeinfo.volcano.sh @@ -5190,44 +5291,231 @@ spec: served: true storage: true --- -# Source: volcano/templates/webhooks.yaml -apiVersion: admissionregistration.k8s.io/v1 -kind: MutatingWebhookConfiguration +# Source: volcano/templates/topology_v1alpha1_hypernodes.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition metadata: - name: volcano-admission-service-pods-mutate -webhooks: - - admissionReviewVersions: - - v1 - clientConfig: - service: - name: volcano-admission-service - namespace: volcano-system - path: /pods/mutate - port: 443 - failurePolicy: Fail - matchPolicy: Equivalent - name: mutatepod.volcano.sh - namespaceSelector: - matchExpressions: - - key: kubernetes.io/metadata.name - operator: NotIn - values: - - volcano-system - - kube-system - objectSelector: {} - reinvocationPolicy: Never - rules: - - apiGroups: - - "" - apiVersions: - - v1 - operations: - - CREATE - resources: - - pods - scope: '*' - sideEffects: NoneOnDryRun - timeoutSeconds: 10 + annotations: + controller-gen.kubebuilder.io/version: v0.17.0 + name: hypernodes.topology.volcano.sh +spec: + group: topology.volcano.sh + names: + kind: HyperNode + listKind: HyperNodeList + plural: hypernodes + shortNames: + - hn + singular: hypernode + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .spec.tier + name: Tier + type: string + - jsonPath: .status.nodeCount + name: NodeCount + type: integer + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: HyperNode represents a collection of nodes sharing similar network + topology or performance characteristics. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: Spec defines the desired configuration of the HyperNode. + properties: + members: + description: Members defines a list of node groups or individual nodes + included in the HyperNode. + items: + description: MemberSpec represents a specific node or a hyperNodes + in the hyperNode. + properties: + selector: + description: Selector defines the selection rules for this member. + properties: + exactMatch: + description: ExactMatch defines the exact match criteria. + properties: + name: + description: Name specifies the exact name of the node + to match. + type: string + type: object + labelMatch: + description: LabelMatch defines the labels match criteria + (only take effect when Member Type is "Node"). + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + regexMatch: + description: RegexMatch defines the regex match criteria. + properties: + pattern: + description: Pattern defines the regex pattern to match + node names. + type: string + type: object + type: object + x-kubernetes-validations: + - message: Either ExactMatch or RegexMatch or LabelMatch must + be specified + rule: has(self.exactMatch) || has(self.regexMatch) || has(self.labelMatch) + - message: Only one of ExactMatch, RegexMatch, or LabelMatch + can be specified + rule: '(has(self.exactMatch) ? 1 : 0) + (has(self.regexMatch) + ? 1 : 0) + (has(self.labelMatch) ? 1 : 0) <= 1' + type: + description: Type specifies the member type. + enum: + - Node + - HyperNode + type: string + required: + - type + type: object + type: array + tier: + description: Tier categorizes the performance level of the HyperNode. + type: integer + required: + - tier + type: object + status: + description: Status provides the current state of the HyperNode. + properties: + conditions: + description: Conditions provide details about the current state of + the HyperNode. + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + nodeCount: + description: NodeCount is the total number of nodes currently in the + HyperNode. + format: int64 + minimum: 0 + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} --- # Source: volcano/templates/webhooks.yaml apiVersion: admissionregistration.k8s.io/v1 @@ -5271,45 +5559,6 @@ webhooks: # Source: volcano/templates/webhooks.yaml apiVersion: admissionregistration.k8s.io/v1 kind: MutatingWebhookConfiguration -metadata: - name: volcano-admission-service-podgroups-mutate -webhooks: - - admissionReviewVersions: - - v1 - clientConfig: - service: - name: volcano-admission-service - namespace: volcano-system - path: /podgroups/mutate - port: 443 - failurePolicy: Fail - matchPolicy: Equivalent - name: mutatepodgroup.volcano.sh - namespaceSelector: - matchExpressions: - - key: kubernetes.io/metadata.name - operator: NotIn - values: - - volcano-system - - kube-system - objectSelector: {} - reinvocationPolicy: Never - rules: - - apiGroups: - - scheduling.volcano.sh - apiVersions: - - v1beta1 - operations: - - CREATE - resources: - - podgroups - scope: '*' - sideEffects: NoneOnDryRun - timeoutSeconds: 10 ---- -# Source: volcano/templates/webhooks.yaml -apiVersion: admissionregistration.k8s.io/v1 -kind: MutatingWebhookConfiguration metadata: name: volcano-admission-service-jobs-mutate webhooks: @@ -5388,44 +5637,6 @@ webhooks: # Source: volcano/templates/webhooks.yaml apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingWebhookConfiguration -metadata: - name: volcano-admission-service-pods-validate -webhooks: - - admissionReviewVersions: - - v1 - clientConfig: - service: - name: volcano-admission-service - namespace: volcano-system - path: /pods/validate - port: 443 - failurePolicy: Fail - matchPolicy: Equivalent - name: validatepod.volcano.sh - namespaceSelector: - matchExpressions: - - key: kubernetes.io/metadata.name - operator: NotIn - values: - - volcano-system - - kube-system - objectSelector: {} - rules: - - apiGroups: - - "" - apiVersions: - - v1 - operations: - - CREATE - resources: - - pods - scope: '*' - sideEffects: NoneOnDryRun - timeoutSeconds: 10 ---- -# Source: volcano/templates/webhooks.yaml -apiVersion: admissionregistration.k8s.io/v1 -kind: ValidatingWebhookConfiguration metadata: name: volcano-admission-service-queues-validate webhooks: @@ -5463,12 +5674,80 @@ webhooks: sideEffects: NoneOnDryRun timeoutSeconds: 10 --- +# Source: volcano/templates/webhooks.yaml +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: volcano-admission-service-podgroups-validate +webhooks: + - admissionReviewVersions: + - v1 + clientConfig: + service: + name: volcano-admission-service + namespace: volcano-system + path: /podgroups/validate + port: 443 + failurePolicy: Fail + matchPolicy: Equivalent + name: validatepodgroup.volcano.sh + namespaceSelector: + matchExpressions: + - key: kubernetes.io/metadata.name + operator: NotIn + values: + - volcano-system + - kube-system + objectSelector: {} + rules: + - apiGroups: + - scheduling.volcano.sh + apiVersions: + - v1beta1 + operations: + - CREATE + resources: + - podgroups + scope: '*' + sideEffects: NoneOnDryRun + timeoutSeconds: 10 +--- +# Source: volcano/templates/webhooks.yaml +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: volcano-admission-service-hypernodes-validate +webhooks: + - admissionReviewVersions: + - v1 + clientConfig: + service: + name: volcano-admission-service + namespace: volcano-system + path: /hypernodes/validate + port: 443 + failurePolicy: Fail + matchPolicy: Equivalent + name: validatehypernodes.volcano.sh + rules: + - apiGroups: + - topology.volcano.sh + apiVersions: + - v1alpha1 + operations: + - CREATE + - UPDATE + resources: + - hypernodes + sideEffects: None + timeoutSeconds: 10 +--- # Source: jobflow/templates/flow_v1alpha1_jobflows.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.16.4 + controller-gen.kubebuilder.io/version: v0.17.0 name: jobtemplates.flow.volcano.sh spec: group: flow.volcano.sh @@ -5504,6 +5783,18 @@ spec: format: int32 minimum: 1 type: integer + networkTopology: + properties: + highestTierAllowed: + default: 1 + type: integer + mode: + default: hard + enum: + - hard + - soft + type: string + type: object plugins: additionalProperties: items: @@ -8264,6 +8555,39 @@ spec: x-kubernetes-list-map-keys: - name x-kubernetes-list-type: map + resources: + properties: + claims: + items: + properties: + name: + type: string + request: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object restartPolicy: type: string runtimeClassName: @@ -8306,6 +8630,8 @@ spec: runAsUser: format: int64 type: integer + seLinuxChangePolicy: + type: string seLinuxOptions: properties: level: @@ -9382,7 +9708,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.16.4 + controller-gen.kubebuilder.io/version: v0.17.0 name: jobflows.flow.volcano.sh spec: group: flow.volcano.sh diff --git a/pkg/minikube/assets/addons.go b/pkg/minikube/assets/addons.go index 54400b4d71..ab49f26874 100644 --- a/pkg/minikube/assets/addons.go +++ b/pkg/minikube/assets/addons.go @@ -610,9 +610,9 @@ var Addons = map[string]*Addon{ "volcano-deployment.yaml", "0640"), }, false, "volcano", "third-party (volcano)", "hwdef", "", map[string]string{ - "vc_webhook_manager": "volcanosh/vc-webhook-manager:v1.11.2@sha256:e3dd5fc9c8af79bfa2182ccd4f48ba0a87c2047d3bdd59dd415288bf19c80ddc", - "vc_controller_manager": "volcanosh/vc-controller-manager:v1.11.2@sha256:ef164e8b3061838a315442ad9ffeb9699a636a73123c5269665fd7aeab06757c", - "vc_scheduler": "volcanosh/vc-scheduler:v1.11.2@sha256:5b77f6b38127db41afe4a38bbf585fa2ea5555998459d34f4f1691233f506121", + "vc_webhook_manager": "volcanosh/vc-webhook-manager:v1.12.1@sha256:f8b50088a7329220cbdcc624067943a76a005bb18bda77647e618aab26cf759d", + "vc_controller_manager": "volcanosh/vc-controller-manager:v1.12.1@sha256:3815883c32f62c3a60b8208ba834f304d91d8f05cddfabd440aa15f7f8bef296", + "vc_scheduler": "volcanosh/vc-scheduler:v1.12.1@sha256:b24ea8af2d167a3525e8fc603b32eca6c9b46ef509fa7e87f09e1fadb992faf2", }, map[string]string{ "vc_webhook_manager": "docker.io", "vc_controller_manager": "docker.io",