ceph helm

ceph/ceph-csi-cephfs/templates/NOTES.txt

@@ -0,0 +1,2 @@
+Examples on how to configure a storage class and start using the driver are here:
+https://github.com/ceph/ceph-csi/tree/devel/examples/cephfs

ceph/ceph-csi-cephfs/templates/_helpers.tpl

@@ -0,0 +1,90 @@
+{{/* vim: set filetype=mustache: */}}
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "ceph-csi-cephfs.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "ceph-csi-cephfs.fullname" -}}
+{{- if .Values.fullnameOverride -}}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- $name := default .Chart.Name .Values.nameOverride -}}
+{{- if contains $name .Release.Name -}}
+{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "ceph-csi-cephfs.nodeplugin.fullname" -}}
+{{- if .Values.nodeplugin.fullnameOverride -}}
+{{- .Values.nodeplugin.fullnameOverride | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- $name := default .Chart.Name .Values.nameOverride -}}
+{{- if contains $name .Release.Name -}}
+{{- printf "%s-%s" .Release.Name .Values.nodeplugin.name | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- printf "%s-%s-%s" .Release.Name $name .Values.nodeplugin.name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "ceph-csi-cephfs.provisioner.fullname" -}}
+{{- if .Values.provisioner.fullnameOverride -}}
+{{- .Values.provisioner.fullnameOverride | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- $name := default .Chart.Name .Values.nameOverride -}}
+{{- if contains $name .Release.Name -}}
+{{- printf "%s-%s" .Release.Name .Values.provisioner.name | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- printf "%s-%s-%s" .Release.Name $name .Values.provisioner.name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "ceph-csi-cephfs.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Create the name of the service account to use
+*/}}
+{{- define "ceph-csi-cephfs.serviceAccountName.nodeplugin" -}}
+{{- if .Values.serviceAccounts.nodeplugin.create -}}
+    {{ default (include "ceph-csi-cephfs.nodeplugin.fullname" .) .Values.serviceAccounts.nodeplugin.name }}
+{{- else -}}
+    {{ default "default" .Values.serviceAccounts.nodeplugin.name }}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Create the name of the service account to use
+*/}}
+{{- define "ceph-csi-cephfs.serviceAccountName.provisioner" -}}
+{{- if .Values.serviceAccounts.provisioner.create -}}
+    {{ default (include "ceph-csi-cephfs.provisioner.fullname" .) .Values.serviceAccounts.provisioner.name }}
+{{- else -}}
+    {{ default "default" .Values.serviceAccounts.provisioner.name }}
+{{- end -}}
+{{- end -}}

ceph/ceph-csi-cephfs/templates/ceph-conf.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: ceph-config
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app: {{ include "ceph-csi-cephfs.name" . }}
+    chart: {{ include "ceph-csi-cephfs.chart" . }}
+    component: {{ .Values.nodeplugin.name }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+data:
+  ceph.conf: |
+{{ tpl .Values.cephconf . | indent 4 }}
+  keyring: ""

ceph/ceph-csi-cephfs/templates/csidriver-crd.yaml

@@ -0,0 +1,11 @@
+{{ if semverCompare ">=1.18.0-beta.1" .Capabilities.KubeVersion.Version }}
+apiVersion: storage.k8s.io/v1
+{{ else }}
+apiVersion: storage.k8s.io/v1beta1
+{{ end }}
+kind: CSIDriver
+metadata:
+  name: {{ .Values.driverName }}
+spec:
+  attachRequired: true
+  podInfoOnMount: false

ceph/ceph-csi-cephfs/templates/csiplugin-configmap.yaml

@@ -0,0 +1,16 @@
+{{- if not .Values.externallyManagedConfigmap }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ .Values.configMapName | quote }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app: {{ include "ceph-csi-cephfs.name" . }}
+    chart: {{ include "ceph-csi-cephfs.chart" . }}
+    component: {{ .Values.provisioner.name }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+data:
+  config.json: |-
+{{ toJson .Values.csiConfig | indent 4 -}}
+{{- end }}

ceph/ceph-csi-cephfs/templates/nodeplugin-clusterrole.yaml

@@ -0,0 +1,18 @@
+{{- if .Values.rbac.create -}}
+{{- if .Values.topology.enabled }}
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: {{ include "ceph-csi-cephfs.nodeplugin.fullname" . }}
+  labels:
+    app: {{ include "ceph-csi-cephfs.name" . }}
+    chart: {{ include "ceph-csi-cephfs.chart" . }}
+    component: {{ .Values.nodeplugin.name }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+rules:
+  - apiGroups: [""]
+    resources: ["nodes"]
+    verbs: ["get"]
+{{- end }}
+{{- end -}}

ceph/ceph-csi-cephfs/templates/nodeplugin-clusterrolebinding.yaml

@@ -0,0 +1,22 @@
+{{- if .Values.rbac.create -}}
+{{- if .Values.topology.enabled }}
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: {{ include "ceph-csi-cephfs.nodeplugin.fullname" . }}
+  labels:
+    app: {{ include "ceph-csi-cephfs.name" . }}
+    chart: {{ include "ceph-csi-cephfs.chart" . }}
+    component: {{ .Values.nodeplugin.name }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ include "ceph-csi-cephfs.serviceAccountName.nodeplugin" . }}
+    namespace: {{ .Release.Namespace }}
+roleRef:
+  kind: ClusterRole
+  name: {{ include "ceph-csi-cephfs.nodeplugin.fullname" . }}
+  apiGroup: rbac.authorization.k8s.io
+{{- end }}
+{{- end -}}

ceph/ceph-csi-cephfs/templates/nodeplugin-daemonset.yaml

@@ -0,0 +1,213 @@
+kind: DaemonSet
+apiVersion: apps/v1
+metadata:
+  name: {{ include "ceph-csi-cephfs.nodeplugin.fullname" . }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app: {{ include "ceph-csi-cephfs.name" . }}
+    chart: {{ include "ceph-csi-cephfs.chart" . }}
+    component: {{ .Values.nodeplugin.name }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+spec:
+  selector:
+    matchLabels:
+      app: {{ include "ceph-csi-cephfs.name" . }}
+      component: {{ .Values.nodeplugin.name }}
+      release: {{ .Release.Name }}
+  updateStrategy:
+    type: {{ .Values.nodeplugin.updateStrategy }}
+  template:
+    metadata:
+      labels:
+        app: {{ include "ceph-csi-cephfs.name" . }}
+        chart: {{ include "ceph-csi-cephfs.chart" . }}
+        component: {{ .Values.nodeplugin.name }}
+        release: {{ .Release.Name }}
+        heritage: {{ .Release.Service }}
+    spec:
+      serviceAccountName: {{ include "ceph-csi-cephfs.serviceAccountName.nodeplugin" . }}
+{{- if .Values.nodeplugin.priorityClassName }}
+      priorityClassName: {{ .Values.nodeplugin.priorityClassName }}
+{{- end }}
+      hostNetwork: true
+      # to use e.g. Rook orchestrated cluster, and mons' FQDN is
+      # resolved through k8s service, set dns policy to cluster first
+      dnsPolicy: ClusterFirstWithHostNet
+      containers:
+        - name: driver-registrar
+          # This is necessary only for systems with SELinux, where
+          # non-privileged sidecar containers cannot access unix domain socket
+          # created by privileged CSI driver container.
+          securityContext:
+            privileged: true
+          image: "{{ .Values.nodeplugin.registrar.image.repository }}:{{ .Values.nodeplugin.registrar.image.tag }}"
+          imagePullPolicy: {{ .Values.nodeplugin.registrar.image.pullPolicy }}
+          args:
+            - "--v={{ .Values.logLevel }}"
+            - "--csi-address=/csi/{{ .Values.pluginSocketFile }}"
+            - "--kubelet-registration-path={{ .Values.kubeletDir }}/plugins/{{ .Values.driverName }}/{{ .Values.pluginSocketFile }}"
+          env:
+            - name: KUBE_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /csi
+            - name: registration-dir
+              mountPath: /registration
+          resources:
+{{ toYaml .Values.nodeplugin.registrar.resources | indent 12 }}
+        - name: csi-cephfsplugin
+          image: "{{ .Values.nodeplugin.plugin.image.repository }}:{{ .Values.nodeplugin.plugin.image.tag }}"
+          imagePullPolicy: {{ .Values.nodeplugin.plugin.image.pullPolicy }}
+          args:
+            - "--nodeid=$(NODE_ID)"
+            - "--type=cephfs"
+            - "--nodeserver=true"
+            - "--pidlimit=-1"
+{{- if .Values.nodeplugin.forcecephkernelclient }}
+            - "--forcecephkernelclient={{ .Values.nodeplugin.forcecephkernelclient }}"
+{{- end }}
+            - "--endpoint=$(CSI_ENDPOINT)"
+            - "--v={{ .Values.logLevel }}"
+            - "--drivername=$(DRIVER_NAME)"
+{{- if .Values.topology.enabled }}
+            - "--domainlabels={{ .Values.topology.domainLabels | join "," }}"
+{{- end }}
+{{- if .Values.nodeplugin.profiling.enabled }}
+            - "--enableprofiling={{ .Values.nodeplugin.profiling.enabled }}"
+{{- end }}
+          env:
+            - name: POD_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
+            - name: DRIVER_NAME
+              value: {{ .Values.driverName }}
+            - name: NODE_ID
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+            - name: CSI_ENDPOINT
+              value: "unix:///csi/{{ .Values.pluginSocketFile }}"
+          securityContext:
+            privileged: true
+            capabilities:
+              add: ["SYS_ADMIN"]
+            allowPrivilegeEscalation: true
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /csi
+            - name: mountpoint-dir
+              mountPath: {{ .Values.kubeletDir }}/pods
+              mountPropagation: Bidirectional
+            - name: plugin-dir
+              mountPath: {{ .Values.kubeletDir }}/plugins
+              mountPropagation: "Bidirectional"
+            - mountPath: /dev
+              name: host-dev
+            - mountPath: /run/mount
+              name: host-mount
+            - mountPath: /sys
+              name: host-sys
+            - mountPath: /etc/selinux
+              name: etc-selinux
+              readOnly: true
+            - mountPath: /lib/modules
+              name: lib-modules
+              readOnly: true
+            - name: ceph-config
+              mountPath: /etc/ceph/
+            - name: ceph-csi-config
+              mountPath: /etc/ceph-csi-config/
+            - name: keys-tmp-dir
+              mountPath: /tmp/csi/keys
+          resources:
+{{ toYaml .Values.nodeplugin.plugin.resources | indent 12 }}
+{{- if .Values.nodeplugin.httpMetrics.enabled }}
+        - name: liveness-prometheus
+          securityContext:
+            privileged: true
+          image: "{{ .Values.nodeplugin.plugin.image.repository }}:{{ .Values.nodeplugin.plugin.image.tag }}"
+          imagePullPolicy: {{ .Values.nodeplugin.plugin.image.pullPolicy }}
+          args:
+            - "--type=liveness"
+            - "--endpoint=$(CSI_ENDPOINT)"
+            - "--metricsport={{ .Values.nodeplugin.httpMetrics.containerPort }}"
+            - "--metricspath=/metrics"
+            - "--polltime=60s"
+            - "--timeout=3s"
+          env:
+            - name: CSI_ENDPOINT
+              value: "unix:///csi/{{ .Values.pluginSocketFile }}"
+            - name: POD_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /csi
+          resources:
+{{ toYaml .Values.nodeplugin.plugin.resources | indent 12 }}
+{{- end }}
+      volumes:
+        - name: socket-dir
+          hostPath:
+            path: "{{ .Values.kubeletDir }}/plugins/{{ .Values.driverName }}"
+            type: DirectoryOrCreate
+        - name: registration-dir
+          hostPath:
+            path: {{ .Values.kubeletDir }}/plugins_registry
+            type: Directory
+        - name: mountpoint-dir
+          hostPath:
+            path: {{ .Values.kubeletDir }}/pods
+            type: DirectoryOrCreate
+        - name: plugin-dir
+          hostPath:
+            path: {{ .Values.kubeletDir }}/plugins
+            type: Directory
+        - name: host-sys
+          hostPath:
+            path: /sys
+        - name: etc-selinux
+          hostPath:
+            path: /etc/selinux
+        - name: host-mount
+          hostPath:
+            path: /run/mount
+        - name: lib-modules
+          hostPath:
+            path: /lib/modules
+        - name: host-dev
+          hostPath:
+            path: /dev
+        - name: ceph-config
+          configMap:
+            name: ceph-config
+        - name: ceph-csi-config
+          configMap:
+            name: {{ .Values.configMapName | quote }}
+{{- if .Values.configMapKey }}
+            items:
+              - key: {{ .Values.configMapKey | quote }}
+                path: config.json
+{{- end }}
+        - name: keys-tmp-dir
+          emptyDir: {
+            medium: "Memory"
+          }
+{{- if .Values.nodeplugin.affinity }}
+      affinity:
+{{ toYaml .Values.nodeplugin.affinity | indent 8 -}}
+{{- end -}}
+{{- if .Values.nodeplugin.nodeSelector }}
+      nodeSelector:
+{{ toYaml .Values.nodeplugin.nodeSelector | indent 8 -}}
+{{- end -}}
+{{- if .Values.nodeplugin.tolerations }}
+      tolerations:
+{{ toYaml .Values.nodeplugin.tolerations | indent 8 -}}
+{{- end -}}

ceph/ceph-csi-cephfs/templates/nodeplugin-http-service.yaml

@@ -0,0 +1,41 @@
+{{- if .Values.nodeplugin.httpMetrics.service.enabled -}}
+apiVersion: v1
+kind: Service
+metadata:
+{{- if .Values.nodeplugin.httpMetrics.service.annotations }}
+  annotations:
+{{ toYaml .Values.nodeplugin.httpMetrics.service.annotations | indent 4 }}
+{{- end }}
+  name: {{ include "ceph-csi-cephfs.nodeplugin.fullname" . }}-http-metrics
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app: {{ include "ceph-csi-cephfs.fullname" . }}
+    chart: {{ include "ceph-csi-cephfs.chart" . }}
+    component: {{ .Values.nodeplugin.name }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+spec:
+{{- if .Values.nodeplugin.httpMetrics.service.clusterIP }}
+  clusterIP: "{{ .Values.nodeplugin.httpMetrics.service.clusterIP }}"
+{{- end }}
+{{- if .Values.nodeplugin.httpMetrics.service.externalIPs }}
+  externalIPs:
+{{ toYaml .Values.nodeplugin.httpMetrics.service.externalIPs | indent 4 }}
+{{- end }}
+{{- if .Values.nodeplugin.httpMetrics.service.loadBalancerIP }}
+  loadBalancerIP: "{{ .Values.nodeplugin.httpMetrics.service.loadBalancerIP }}"
+{{- end }}
+{{- if .Values.nodeplugin.httpMetrics.service.loadBalancerSourceRanges }}
+  loadBalancerSourceRanges:
+{{ toYaml .Values.nodeplugin.httpMetrics.service.loadBalancerSourceRanges | indent 4 }}
+{{- end }}
+  ports:
+    - name: http-metrics
+      port: {{ .Values.nodeplugin.httpMetrics.service.servicePort }}
+      targetPort: {{ .Values.nodeplugin.httpMetrics.containerPort }}
+  selector:
+    app: {{ include "ceph-csi-cephfs.name" . }}
+    component: {{ .Values.nodeplugin.name }}
+    release: {{ .Release.Name }}
+  type: "{{ .Values.nodeplugin.httpMetrics.service.type }}"
+{{- end -}}

ceph/ceph-csi-cephfs/templates/nodeplugin-psp.yaml

@@ -0,0 +1,46 @@
+{{- if .Values.nodeplugin.podSecurityPolicy.enabled -}}
+apiVersion: policy/v1beta1
+kind: PodSecurityPolicy
+metadata:
+  name: {{ include "ceph-csi-cephfs.nodeplugin.fullname" . }}
+  labels:
+    app: {{ include "ceph-csi-cephfs.fullname" . }}
+    chart: {{ include "ceph-csi-cephfs.chart" . }}
+    component: {{ .Values.nodeplugin.name }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+spec:
+  allowPrivilegeEscalation: true
+  allowedCapabilities:
+    - 'SYS_ADMIN'
+  fsGroup:
+    rule: RunAsAny
+  privileged: true
+  hostNetwork: true
+  hostPID: true
+  runAsUser:
+    rule: RunAsAny
+  seLinux:
+    rule: RunAsAny
+  supplementalGroups:
+    rule: RunAsAny
+  volumes:
+    - 'configMap'
+    - 'emptyDir'
+    - 'projected'
+    - 'secret'
+    - 'hostPath'
+  allowedHostPaths:
+    - pathPrefix: '/dev'
+      readOnly: false
+    - pathPrefix: '/run/mount'
+      readOnly: false
+    - pathPrefix: '/sys'
+      readOnly: false
+    - pathPrefix: '/etc/selinux'
+      readOnly: true
+    - pathPrefix: '/lib/modules'
+      readOnly: true
+    - pathPrefix: '{{ .Values.kubeletDir }}'
+      readOnly: false
+{{- end }}

ceph/ceph-csi-cephfs/templates/nodeplugin-role.yaml

@@ -0,0 +1,18 @@
+{{- if and .Values.rbac.create .Values.nodeplugin.podSecurityPolicy.enabled -}}
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: {{ include "ceph-csi-cephfs.nodeplugin.fullname" . }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app: {{ include "ceph-csi-cephfs.fullname" . }}
+    chart: {{ include "ceph-csi-cephfs.chart" . }}
+    component: {{ .Values.nodeplugin.name }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+rules:
+  - apiGroups: ['policy']
+    resources: ['podsecuritypolicies']
+    verbs: ['use']
+    resourceNames: ['{{ include "ceph-csi-cephfs.nodeplugin.fullname" . }}']
+{{- end -}}

ceph/ceph-csi-cephfs/templates/nodeplugin-rolebinding.yaml

@@ -0,0 +1,21 @@
+{{- if and .Values.rbac.create .Values.nodeplugin.podSecurityPolicy.enabled -}}
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: {{ include "ceph-csi-cephfs.nodeplugin.fullname" . }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app: {{ include "ceph-csi-cephfs.fullname" . }}
+    chart: {{ include "ceph-csi-cephfs.chart" . }}
+    component: {{ .Values.nodeplugin.name }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ include "ceph-csi-cephfs.serviceAccountName.nodeplugin" . }}
+    namespace: {{ .Release.Namespace }}
+roleRef:
+  kind: Role
+  name: {{ include "ceph-csi-cephfs.nodeplugin.fullname" . }}
+  apiGroup: rbac.authorization.k8s.io
+{{- end -}}

ceph/ceph-csi-cephfs/templates/nodeplugin-serviceaccount.yaml

@@ -0,0 +1,13 @@
+{{- if .Values.serviceAccounts.nodeplugin.create -}}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ include "ceph-csi-cephfs.serviceAccountName.nodeplugin" . }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app: {{ include "ceph-csi-cephfs.name" . }}
+    chart: {{ include "ceph-csi-cephfs.chart" . }}
+    component: {{ .Values.nodeplugin.name }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+{{- end -}}

ceph/ceph-csi-cephfs/templates/provisioner-clusterrole.yaml

@@ -0,0 +1,63 @@
+{{- if .Values.rbac.create -}}
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: {{ include "ceph-csi-cephfs.provisioner.fullname" . }}
+  labels:
+    app: {{ include "ceph-csi-cephfs.name" . }}
+    chart: {{ include "ceph-csi-cephfs.chart" . }}
+    component: {{ .Values.provisioner.name }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+rules:
+  - apiGroups: [""]
+    resources: ["secrets"]
+    verbs: ["get", "list"]
+  - apiGroups: [""]
+    resources: ["persistentvolumes"]
+    verbs: ["get", "list", "watch", "create", "delete","patch"]
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims"]
+    verbs: ["get", "list", "watch", "update"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["storageclasses"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["events"]
+    verbs: ["list", "watch", "create", "update", "patch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshots"]
+    verbs: ["get", "list"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotcontents"]
+    verbs: ["create", "get", "list", "watch", "update", "delete"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotclasses"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotcontents/status"]
+    verbs: ["update"]
+{{- if .Values.provisioner.attacher.enabled }}
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["volumeattachments"]
+    verbs: ["get", "list", "watch", "update", "patch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["volumeattachments/status"]
+    verbs: ["patch"]
+{{- end -}}
+{{- if semverCompare ">=1.15" .Capabilities.KubeVersion.Version -}}
+{{- if .Values.provisioner.resizer.enabled }}
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims/status"]
+    verbs: ["update", "patch"]
+{{- end -}}
+{{- end -}}
+{{- if .Values.topology.enabled }}
+  - apiGroups: [""]
+    resources: ["nodes"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["csinodes"]
+    verbs: ["get", "list", "watch"]
+{{- end }}
+{{- end -}}

ceph/ceph-csi-cephfs/templates/provisioner-clusterrolebinding.yaml

@@ -0,0 +1,20 @@
+{{- if .Values.rbac.create -}}
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: {{ include "ceph-csi-cephfs.provisioner.fullname" . }}
+  labels:
+    app: {{ include "ceph-csi-cephfs.name" . }}
+    chart: {{ include "ceph-csi-cephfs.chart" . }}
+    component: {{ .Values.provisioner.name }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ include "ceph-csi-cephfs.serviceAccountName.provisioner" . }}
+    namespace: {{ .Release.Namespace }}
+roleRef:
+  kind: ClusterRole
+  name: {{ include "ceph-csi-cephfs.provisioner.fullname" . }}
+  apiGroup: rbac.authorization.k8s.io
+{{- end -}}

ceph/ceph-csi-cephfs/templates/provisioner-deployment.yaml

@@ -0,0 +1,241 @@
+kind: Deployment
+apiVersion: apps/v1
+metadata:
+  name: {{ include "ceph-csi-cephfs.provisioner.fullname" . }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app: {{ include "ceph-csi-cephfs.name" . }}
+    chart: {{ include "ceph-csi-cephfs.chart" . }}
+    component: {{ .Values.provisioner.name }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+spec:
+  replicas: {{ .Values.provisioner.replicaCount }}
+  strategy:
+    type: {{ .Values.provisioner.strategy.type }}
+{{- if eq .Values.provisioner.strategy.type "RollingUpdate" }}
+    rollingUpdate:
+      maxUnavailable: {{ .Values.provisioner.strategy.rollingUpdate.maxUnavailable }}
+{{- end }}
+  selector:
+    matchLabels:
+      app: {{ include "ceph-csi-cephfs.name" . }}
+      component: {{ .Values.provisioner.name }}
+      release: {{ .Release.Name }}
+  template:
+    metadata:
+      labels:
+        app: {{ include "ceph-csi-cephfs.name" . }}
+        chart: {{ include "ceph-csi-cephfs.chart" . }}
+        component: {{ .Values.provisioner.name }}
+        release: {{ .Release.Name }}
+        heritage: {{ .Release.Service }}
+    spec:
+{{- if gt (int .Values.provisioner.replicaCount) 1 }}
+      affinity:
+        podAntiAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            - labelSelector:
+                matchExpressions:
+                  - key: app
+                    operator: In
+                    values:
+                      - {{ include "ceph-csi-cephfs.name" . }}
+                  - key: component
+                    operator: In
+                    values:
+                      - {{ .Values.provisioner.name }}
+              topologyKey: "kubernetes.io/hostname"
+{{- end }}
+      serviceAccountName: {{ include "ceph-csi-cephfs.serviceAccountName.provisioner" . }}
+{{- if .Values.provisioner.priorityClassName }}
+      priorityClassName: {{ .Values.provisioner.priorityClassName }}
+{{- end }}
+      containers:
+        - name: csi-provisioner
+          image: "{{ .Values.provisioner.provisioner.image.repository }}:{{ .Values.provisioner.provisioner.image.tag }}"
+          imagePullPolicy: {{ .Values.provisioner.provisioner.image.pullPolicy }}
+          args:
+            - "--csi-address=$(ADDRESS)"
+            - "--v={{ .Values.logLevel }}"
+            - "--timeout={{ .Values.provisioner.timeout }}"
+            - "--leader-election=true"
+            - "--retry-interval-start=500ms"
+            - "--extra-create-metadata=true"
+{{- if .Values.topology.enabled }}
+            - "--feature-gates=Topology=true"
+{{- end }}
+          env:
+            - name: ADDRESS
+              value: "unix:///csi/{{ .Values.provisionerSocketFile }}"
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /csi
+          resources:
+{{ toYaml .Values.provisioner.provisioner.resources | indent 12 }}
+        - name: csi-snapshotter
+          image: {{ .Values.provisioner.snapshotter.image.repository }}:{{ .Values.provisioner.snapshotter.image.tag }}
+          imagePullPolicy: {{ .Values.provisioner.snapshotter.image.pullPolicy }}
+          args:
+            - "--csi-address=$(ADDRESS)"
+            - "--v={{ .Values.logLevel }}"
+            - "--timeout={{ .Values.provisioner.timeout }}"
+            - "--leader-election=true"
+          env:
+            - name: ADDRESS
+              value: "unix:///csi/{{ .Values.provisionerSocketFile }}"
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /csi
+          resources:
+{{ toYaml .Values.provisioner.snapshotter.resources | indent 12 }}
+{{- if .Values.provisioner.attacher.enabled }}
+        - name: csi-attacher
+          image: "{{ .Values.provisioner.attacher.image.repository }}:{{ .Values.provisioner.attacher.image.tag }}"
+          imagePullPolicy: {{ .Values.provisioner.attacher.image.pullPolicy }}
+          args:
+            - "--v={{ .Values.logLevel }}"
+            - "--csi-address=$(ADDRESS)"
+            - "--leader-election=true"
+            - "--retry-interval-start=500ms"
+          env:
+            - name: ADDRESS
+              value: "unix:///csi/{{ .Values.provisionerSocketFile }}"
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /csi
+          resources:
+{{ toYaml .Values.nodeplugin.plugin.resources | indent 12 }}
+{{- end }}
+{{- if semverCompare ">=1.15" .Capabilities.KubeVersion.Version -}}
+{{- if .Values.provisioner.resizer.enabled }}
+        - name: csi-resizer
+          image: "{{ .Values.provisioner.resizer.image.repository }}:{{ .Values.provisioner.resizer.image.tag }}"
+          imagePullPolicy: {{ .Values.provisioner.resizer.image.pullPolicy }}
+          args:
+            - "--v={{ .Values.logLevel }}"
+            - "--csi-address=$(ADDRESS)"
+            - "--timeout={{ .Values.provisioner.timeout }}"
+            - "--leader-election"
+            - "--retry-interval-start=500ms"
+            - "--handle-volume-inuse-error=false"
+          env:
+            - name: ADDRESS
+              value: "unix:///csi/{{ .Values.provisionerSocketFile }}"
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /csi
+          resources:
+{{ toYaml .Values.provisioner.resizer.resources | indent 12 }}
+{{- end }}
+{{- end }}
+        - name: csi-cephfsplugin
+          image: "{{ .Values.nodeplugin.plugin.image.repository }}:{{ .Values.nodeplugin.plugin.image.tag }}"
+          imagePullPolicy: {{ .Values.nodeplugin.plugin.image.pullPolicy }}
+          args:
+            - "--nodeid=$(NODE_ID)"
+            - "--type=cephfs"
+            - "--controllerserver=true"
+            - "--pidlimit=-1"
+            - "--endpoint=$(CSI_ENDPOINT)"
+            - "--v={{ .Values.logLevel }}"
+            - "--drivername=$(DRIVER_NAME)"
+{{- if .Values.provisioner.profiling.enabled }}
+            - "--enableprofiling={{ .Values.provisioner.profiling.enabled }}"
+{{- end }}
+          env:
+            - name: POD_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
+            - name: DRIVER_NAME
+              value: {{ .Values.driverName }}
+            - name: NODE_ID
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+            - name: CSI_ENDPOINT
+              value: "unix:///csi/{{ .Values.provisionerSocketFile }}"
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /csi
+            - name: host-sys
+              mountPath: /sys
+            - name: lib-modules
+              mountPath: /lib/modules
+              readOnly: true
+            - name: host-dev
+              mountPath: /dev
+            - name: ceph-config
+              mountPath: /etc/ceph/
+            - name: ceph-csi-config
+              mountPath: /etc/ceph-csi-config/
+            - name: keys-tmp-dir
+              mountPath: /tmp/csi/keys
+          resources:
+{{ toYaml .Values.nodeplugin.plugin.resources | indent 12 }}
+{{- if .Values.provisioner.httpMetrics.enabled }}
+        - name: liveness-prometheus
+          image: "{{ .Values.nodeplugin.plugin.image.repository }}:{{ .Values.nodeplugin.plugin.image.tag }}"
+          imagePullPolicy: {{ .Values.nodeplugin.plugin.image.pullPolicy }}
+          args:
+            - "--type=liveness"
+            - "--endpoint=$(CSI_ENDPOINT)"
+            - "--metricsport={{ .Values.provisioner.httpMetrics.containerPort }}"
+            - "--metricspath=/metrics"
+            - "--polltime=60s"
+            - "--timeout=3s"
+          env:
+            - name: CSI_ENDPOINT
+              value: "unix:///csi/{{ .Values.provisionerSocketFile }}"
+            - name: POD_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /csi
+          resources:
+{{ toYaml .Values.nodeplugin.plugin.resources | indent 12 }}
+{{- end }}
+      volumes:
+        - name: socket-dir
+          emptyDir: {
+            medium: "Memory"
+          }
+        - name: host-sys
+          hostPath:
+            path: /sys
+        - name: lib-modules
+          hostPath:
+            path: /lib/modules
+        - name: host-dev
+          hostPath:
+            path: /dev
+        - name: ceph-config
+          configMap:
+            name: ceph-config
+        - name: ceph-csi-config
+          configMap:
+            name: {{ .Values.configMapName | quote }}
+{{- if .Values.configMapKey }}
+            items:
+              - key: {{ .Values.configMapKey | quote }}
+                path: config.json
+{{- end }}
+        - name: keys-tmp-dir
+          emptyDir: {
+            medium: "Memory"
+          }
+{{- if .Values.provisioner.affinity }}
+      affinity:
+{{ toYaml .Values.provisioner.affinity | indent 8 -}}
+{{- end -}}
+{{- if .Values.provisioner.nodeSelector }}
+      nodeSelector:
+{{ toYaml .Values.provisioner.nodeSelector | indent 8 -}}
+{{- end -}}
+{{- if .Values.provisioner.tolerations }}
+      tolerations:
+{{ toYaml .Values.provisioner.tolerations | indent 8 -}}
+{{- end -}}

ceph/ceph-csi-cephfs/templates/provisioner-http-service.yaml

@@ -0,0 +1,41 @@
+{{- if .Values.provisioner.httpMetrics.service.enabled -}}
+apiVersion: v1
+kind: Service
+metadata:
+{{- if .Values.provisioner.httpMetrics.service.annotations }}
+  annotations:
+{{ toYaml .Values.provisioner.httpMetrics.service.annotations | indent 4 }}
+{{- end }}
+  name: {{ include "ceph-csi-cephfs.provisioner.fullname" . }}-http-metrics
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app: {{ include "ceph-csi-cephfs.fullname" . }}
+    chart: {{ include "ceph-csi-cephfs.chart" . }}
+    component: {{ .Values.provisioner.name }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+spec:
+{{- if .Values.provisioner.httpMetrics.service.clusterIP }}
+  clusterIP: "{{ .Values.provisioner.httpMetrics.service.clusterIP }}"
+{{- end }}
+{{- if .Values.provisioner.httpMetrics.service.externalIPs }}
+  externalIPs:
+{{ toYaml .Values.provisioner.httpMetrics.service.externalIPs | indent 4 }}
+{{- end }}
+{{- if .Values.provisioner.httpMetrics.service.loadBalancerIP }}
+  loadBalancerIP: "{{ .Values.provisioner.httpMetrics.service.loadBalancerIP }}"
+{{- end }}
+{{- if .Values.provisioner.httpMetrics.service.loadBalancerSourceRanges }}
+  loadBalancerSourceRanges:
+{{ toYaml .Values.provisioner.httpMetrics.service.loadBalancerSourceRanges | indent 4 }}
+{{- end }}
+  ports:
+    - name: http-metrics
+      port: {{ .Values.provisioner.httpMetrics.service.servicePort }}
+      targetPort: {{ .Values.provisioner.httpMetrics.containerPort }}
+  selector:
+    app: {{ include "ceph-csi-cephfs.name" . }}
+    component: {{ .Values.provisioner.name }}
+    release: {{ .Release.Name }}
+  type: "{{ .Values.provisioner.httpMetrics.service.type }}"
+{{- end -}}

ceph/ceph-csi-cephfs/templates/provisioner-psp.yaml

@@ -0,0 +1,34 @@
+{{- if .Values.provisioner.podSecurityPolicy.enabled -}}
+apiVersion: policy/v1beta1
+kind: PodSecurityPolicy
+metadata:
+  name: {{ include "ceph-csi-cephfs.provisioner.fullname" . }}
+  labels:
+    app: {{ include "ceph-csi-cephfs.name" . }}
+    chart: {{ include "ceph-csi-cephfs.chart" . }}
+    component: {{ .Values.provisioner.name }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+spec:
+  fsGroup:
+    rule: RunAsAny
+  runAsUser:
+    rule: RunAsAny
+  seLinux:
+    rule: RunAsAny
+  supplementalGroups:
+    rule: RunAsAny
+  volumes:
+    - 'configMap'
+    - 'emptyDir'
+    - 'projected'
+    - 'secret'
+    - 'hostPath'
+  allowedHostPaths:
+    - pathPrefix: '/dev'
+      readOnly: false
+    - pathPrefix: '/sys'
+      readOnly: false
+    - pathPrefix: '/lib/modules'
+      readOnly: true
+{{- end }}

ceph/ceph-csi-cephfs/templates/provisioner-role.yaml

@@ -0,0 +1,26 @@
+{{- if .Values.rbac.create -}}
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: {{ include "ceph-csi-cephfs.provisioner.fullname" . }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app: {{ include "ceph-csi-cephfs.name" . }}
+    chart: {{ include "ceph-csi-cephfs.chart" . }}
+    component: {{ .Values.provisioner.name }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+rules:
+  - apiGroups: [""]
+    resources: ["configmaps"]
+    verbs: ["get", "list", "watch", "create", "delete"]
+  - apiGroups: ["coordination.k8s.io"]
+    resources: ["leases"]
+    verbs: ["get", "watch", "list", "delete", "update", "create"]
+{{- if .Values.provisioner.podSecurityPolicy.enabled }}
+  - apiGroups: ['policy']
+    resources: ['podsecuritypolicies']
+    verbs: ['use']
+    resourceNames: ['{{ include "ceph-csi-cephfs.provisioner.fullname" . }}']
+{{- end -}}
+{{- end -}}

ceph/ceph-csi-cephfs/templates/provisioner-rolebinding.yaml

@@ -0,0 +1,21 @@
+{{- if .Values.rbac.create -}}
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: {{ include "ceph-csi-cephfs.provisioner.fullname" . }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app: {{ include "ceph-csi-cephfs.name" . }}
+    chart: {{ include "ceph-csi-cephfs.chart" . }}
+    component: {{ .Values.provisioner.name }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ include "ceph-csi-cephfs.serviceAccountName.provisioner" . }}
+    namespace: {{ .Release.Namespace }}
+roleRef:
+  kind: Role
+  name: {{ include "ceph-csi-cephfs.provisioner.fullname" . }}
+  apiGroup: rbac.authorization.k8s.io
+{{- end -}}

ceph/ceph-csi-cephfs/templates/provisioner-serviceaccount.yaml

@@ -0,0 +1,13 @@
+{{- if .Values.serviceAccounts.provisioner.create -}}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ include "ceph-csi-cephfs.serviceAccountName.provisioner" . }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app: {{ include "ceph-csi-cephfs.name" . }}
+    chart: {{ include "ceph-csi-cephfs.chart" . }}
+    component: {{ .Values.provisioner.name }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+{{- end -}}

ceph/ceph-csi-cephfs/templates/secret.yaml

@@ -0,0 +1,15 @@
+{{- if .Values.secret.create -}}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ .Values.secret.name }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app: {{ include "ceph-csi-cephfs.name" . }}
+    chart: {{ include "ceph-csi-cephfs.chart" . }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+stringData:
+  adminID: {{ .Values.secret.adminID }}
+  adminKey: {{ .Values.secret.adminKey }}
+{{- end -}}

ceph/ceph-csi-cephfs/templates/storageclass.yaml

@@ -0,0 +1,61 @@
+{{- if .Values.storageClass.create -}}
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: {{ .Values.storageClass.name }}
+  namespace: {{ .Release.Namespace }}
+{{- if .Values.storageClass.annotations }}
+  annotations:
+{{ toYaml .Values.storageClass.annotations | indent 4 }}
+{{- end }}
+  labels:
+    app: {{ include "ceph-csi-cephfs.name" . }}
+    chart: {{ include "ceph-csi-cephfs.chart" . }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+provisioner: {{ .Values.driverName }}
+parameters:
+  clusterID: {{ .Values.storageClass.clusterID }}
+  fsName: {{ .Values.storageClass.fsName }}
+{{- if .Values.storageClass.pool }}
+  pool: {{ .Values.storageClass.pool }}
+{{- end }}
+{{- if .Values.storageClass.fuseMountOptions }}
+  fuseMountOptions: "{{ .Values.storageClass.fuseMountOptions }}"
+{{- end }}
+{{- if .Values.storageClass.kernelMountOptions }}
+  kernelMountOptions: "{{ .Values.storageClass.kernelMountOptions }}"
+{{- end }}
+{{- if .Values.storageClass.mounter }}
+  mounter: "{{ .Values.storageClass.mounter }}"
+{{- end }}
+{{- if .Values.storageClass.volumeNamePrefix }}
+  volumeNamePrefix: "{{ .Values.storageClass.volumeNamePrefix }}"
+{{- end }}
+  csi.storage.k8s.io/provisioner-secret-name: {{ .Values.storageClass.provisionerSecret }}
+{{- if .Values.storageClass.provisionerSecretNamespace }}
+  csi.storage.k8s.io/provisioner-secret-namespace: {{ .Values.storageClass.provisionerSecretNamespace }}
+{{ else }}
+  csi.storage.k8s.io/provisioner-secret-namespace: {{ .Release.Namespace }}
+{{- end }}
+  csi.storage.k8s.io/controller-expand-secret-name: {{ .Values.storageClass.controllerExpandSecret }}
+{{- if .Values.storageClass.controllerExpandSecretNamespace }}
+  csi.storage.k8s.io/controller-expand-secret-namespace: {{ .Values.storageClass.controllerExpandSecretNamespace }}
+{{ else }}
+  csi.storage.k8s.io/controller-expand-secret-namespace: {{ .Release.Namespace }}
+{{- end }}
+  csi.storage.k8s.io/node-stage-secret-name: {{ .Values.storageClass.nodeStageSecret }}
+{{- if .Values.storageClass.nodeStageSecretNamespace }}
+  csi.storage.k8s.io/node-stage-secret-namespace: {{ .Values.storageClass.nodeStageSecretNamespace }}
+{{ else }}
+  csi.storage.k8s.io/node-stage-secret-namespace: {{ .Release.Namespace }}
+{{- end }}
+reclaimPolicy: {{ .Values.storageClass.reclaimPolicy }}
+allowVolumeExpansion: {{ .Values.storageClass.allowVolumeExpansion }}
+{{- if .Values.storageClass.mountOptions }}
+mountOptions:
+  {{- range .Values.storageClass.mountOptions }}
+  - {{ . }}
+  {{- end }}
+{{- end }}
+{{- end -}}