diff --git a/ceph/ceph-csi-cephfs/.helmignore b/ceph/ceph-csi-cephfs/.helmignore new file mode 100644 index 0000000..f0c1319 --- /dev/null +++ b/ceph/ceph-csi-cephfs/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/ceph/ceph-csi-cephfs/Chart.yaml b/ceph/ceph-csi-cephfs/Chart.yaml new file mode 100644 index 0000000..9238c26 --- /dev/null +++ b/ceph/ceph-csi-cephfs/Chart.yaml @@ -0,0 +1,15 @@ +--- +apiVersion: v1 +appVersion: canary +description: "Container Storage Interface (CSI) driver, +provisioner, snapshotter and attacher for Ceph cephfs" +name: ceph-csi-cephfs +version: 3-canary +keywords: + - ceph + - cephfs + - ceph-csi +home: https://github.com/ceph/ceph-csi +sources: + - https://github.com/ceph/ceph-csi/tree/devel/charts/ceph-csi-cephfs +icon: https://raw.githubusercontent.com/ceph/ceph-csi/devel/assets/ceph-logo.png diff --git a/ceph/ceph-csi-cephfs/README.md b/ceph/ceph-csi-cephfs/README.md new file mode 100644 index 0000000..302bec8 --- /dev/null +++ b/ceph/ceph-csi-cephfs/README.md @@ -0,0 +1,168 @@ +# ceph-csi-cephfs + +The ceph-csi-cephfs chart adds cephFS volume support to your cluster. + +## Install from release repo + +Add chart repository to install helm charts from it + +```console +helm repo add ceph-csi https://ceph.github.io/csi-charts +``` + +## Install from local Chart + +we need to enter into the directory where all charts are present + +```console +cd charts +``` + +**Note:** charts directory is present in root of the ceph-csi project + +### Install Chart + +To install the Chart into your Kubernetes cluster + +- For helm 2.x + + ```bash + helm install --namespace "ceph-csi-cephfs" --name "ceph-csi-cephfs" ceph-csi/ceph-csi-cephfs + ``` + +- For helm 3.x + + Create the namespace where Helm should install the components with + + ```bash + kubectl create namespace ceph-csi-cephfs + ``` + + Run the installation + + ```bash + helm install --namespace "ceph-csi-cephfs" "ceph-csi-cephfs" ceph-csi/ceph-csi-cephfs + ``` + +After installation succeeds, you can get a status of Chart + +```bash +helm status "ceph-csi-cephfs" +``` + +### Delete Chart + +If you want to delete your Chart, use this command + +- For helm 2.x + + ```bash + helm delete --purge "ceph-csi-cephfs" + ``` + +- For helm 3.x + + ```bash + helm uninstall "ceph-csi-cephfs" --namespace "ceph-csi-cephfs" + ``` + +If you want to delete the namespace, use this command + +```bash +kubectl delete namespace ceph-csi-cephfs +``` + +### Configuration + +The following table lists the configurable parameters of the ceph-csi-cephfs +charts and their default values. + +| Parameter | Description | Default | +| ---------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------- | +| `rbac.create` | Specifies whether RBAC resources should be created | `true` | +| `serviceAccounts.nodeplugin.create` | Specifies whether a nodeplugin ServiceAccount should be created | `true` | +| `serviceAccounts.nodeplugin.name` | The name of the nodeplugin ServiceAccount to use. If not set and create is true, a name is generated using the fullname | "" | +| `serviceAccounts.provisioner.create` | Specifies whether a provisioner ServiceAccount should be created | `true` | +| `serviceAccounts.provisioner.name` | The name of the provisioner ServiceAccount of provisioner to use. If not set and create is true, a name is generated using the fullname | "" | +| `csiConfig` | Configuration for the CSI to connect to the cluster | [] | +| `logLevel` | Set logging level for csi containers. Supported values from 0 to 5. 0 for general useful logs, 5 for trace level verbosity. | `5` | +| `nodeplugin.name` | Specifies the nodeplugin name | `nodeplugin` | +| `nodeplugin.updateStrategy` | Specifies the update Strategy. If you are using ceph-fuse client set this value to OnDelete | `RollingUpdate` | +| `nodeplugin.priorityClassName` | Set user created priorityclassName for csi plugin pods. default is system-node-critical which is highest priority | `system-node-critical` | +| `nodeplugin.profiling.enabled` | Specifies whether profiling should be enabled | `false` | +| `nodeplugin.registrar.image.repository` | Node-Registrar image repository URL | `k8s.gcr.io/sig-storage/csi-node-driver-registrar` | +| `nodeplugin.registrar.image.tag` | Image tag | `v2.2.0` | +| `nodeplugin.registrar.image.pullPolicy` | Image pull policy | `IfNotPresent` | +| `nodeplugin.plugin.image.repository` | Nodeplugin image repository URL | `quay.io/cephcsi/cephcsi` | +| `nodeplugin.plugin.image.tag` | Image tag | `canary` | +| `nodeplugin.plugin.image.pullPolicy` | Image pull policy | `IfNotPresent` | +| `nodeplugin.nodeSelector` | Kubernetes `nodeSelector` to add to the Daemonset | `{}` | +| `nodeplugin.tolerations` | List of Kubernetes `tolerations` to add to the Daemonset | `{}` | +| `nodeplugin.forcecephkernelclient` | Set to true to enable Ceph Kernel clients on kernel < 4.17 which support quotas | `true` | +| `nodeplugin.podSecurityPolicy.enabled` | If true, create & use [Pod Security Policy resources](https://kubernetes.io/docs/concepts/policy/pod-security-policy/). | `false` | +| `provisioner.name` | Specifies the name of provisioner | `provisioner` | +| `provisioner.replicaCount` | Specifies the replicaCount | `3` | +| `provisioner.timeout` | GRPC timeout for waiting for creation or deletion of a volume | `60s` | +| `provisioner.priorityClassName` | Set user created priorityclassName for csi provisioner pods. Default is `system-cluster-critical` which is less priority than `system-node-critical` | `system-cluster-critical` | +| `provisioner.profiling.enabled` | Specifies whether profiling should be enabled | `false` | +| `provisioner.provisioner.image.repository` | Specifies the csi-provisioner image repository URL | `k8s.gcr.io/sig-storage/csi-provisioner` | +| `provisioner.provisioner.image.tag` | Specifies image tag | `v2.2.2` | +| `provisioner.provisioner.image.pullPolicy` | Specifies pull policy | `IfNotPresent` | +| `provisioner.attacher.image.repository` | Specifies the csi-attacher image repository URL | `k8s.gcr.io/sig-storage/csi-attacher` | +| `provisioner.attacher.image.tag` | Specifies image tag | `v3.2.1` | +| `provisioner.attacher.image.pullPolicy` | Specifies pull policy | `IfNotPresent` | +| `provisioner.attacher.name` | Specifies the name of csi-attacher sidecar | `attacher` | +| `provisioner.attacher.enabled` | Specifies whether attacher sidecar is enabled | `true` | +| `provisioner.resizer.image.repository` | Specifies the csi-resizer image repository URL | `k8s.gcr.io/sig-storage/csi-resizer` | +| `provisioner.resizer.image.tag` | Specifies image tag | `v1.2.0` | +| `provisioner.resizer.image.pullPolicy` | Specifies pull policy | `IfNotPresent` | +| `provisioner.resizer.name` | Specifies the name of csi-resizer sidecar | `resizer` | +| `provisioner.resizer.enabled` | Specifies whether resizer sidecar is enabled | `true` | +| `provisioner.snapshotter.image.repository` | Specifies the csi-snapshotter image repository URL | `k8s.gcr.io/sig-storage/csi-snapshotter` | +| `provisioner.snapshotter.image.tag` | Specifies image tag | `v4.1.1` | +| `provisioner.snapshotter.image.pullPolicy` | Specifies pull policy | `IfNotPresent` | +| `provisioner.nodeSelector` | Specifies the node selector for provisioner deployment | `{}` | +| `provisioner.tolerations` | Specifies the tolerations for provisioner deployment | `{}` | +| `provisioner.affinity` | Specifies the affinity for provisioner deployment | `{}` | +| `provisioner.podSecurityPolicy.enabled` | Specifies whether podSecurityPolicy is enabled | `false` | +| `topology.enabled` | Specifies whether topology based provisioning support should be exposed by CSI | `false` | +| `topology.domainLabels` | DomainLabels define which node labels to use as domains for CSI nodeplugins to advertise their domains | `{}` | +| `provisionerSocketFile` | The filename of the provisioner socket | `csi-provisioner.sock` | +| `pluginSocketFile` | The filename of the plugin socket | `csi.sock` | +| `kubeletDir` | Kubelet working directory | `/var/lib/kubelet` | +| `driverName` | Name of the csi-driver | `cephfs.csi.ceph.com` | +| `configMapName` | Name of the configmap which contains cluster configuration | `ceph-csi-config` | +| `externallyManagedConfigmap` | Specifies the use of an externally provided configmap | `false` | +| `storageClass.create` | Specifies whether the StorageClass should be created | `false` | +| `storageClass.name` | Specifies the cephFS StorageClass name | `csi-cephfs-sc` | +| `storageClass.annotations` | Specifies the annotations for the cephFS storageClass | `[]` | +| `storageClass.clusterID` | String representing a Ceph cluster to provision storage from | `` | +| `storageClass.fsName` | CephFS filesystem name into which the volume shall be created | `myfs` | +| `storageClass.pool` | Ceph pool into which volume data shall be stored | `""` | +| `storageClass.fuseMountOptions` | Comma separated string of Ceph-fuse mount options | `""` | +| `storageclass.kernelMountOptions` | Comma separated string of CephFS kernel mount options | `""` | +| `storageClass.mounter` | The driver can use either ceph-fuse (fuse) or ceph kernelclient (kernel) | `""` | +| `storageClass.volumeNamePrefix` | Prefix to use for naming subvolumes | `""` | +| `storageClass.provisionerSecret` | The secrets have to contain user and/or Ceph admin credentials. | `csi-cephfs-secret` | +| `storageClass.provisionerSecretNamespace` | Specifies the provisioner secret namespace | `""` | +| `storageClass.controllerExpandSecret` | Specifies the controller expand secret name | `csi-cephfs-secret` | +| `storageClass.controllerExpandSecretNamespace` | Specifies the controller expand secret namespace | `""` | +| `storageClass.nodeStageSecret` | Specifies the node stage secret name | `csi-cephfs-secret` | +| `storageClass.nodeStageSecretNamespace` | Specifies the node stage secret namespace | `""` | +| `storageClass.reclaimPolicy` | Specifies the reclaim policy of the StorageClass | `Delete` | +| `storageClass.allowVolumeExpansion` | Specifies whether volume expansion should be allowed | `true` | +| `storageClass.mountOptions` | Specifies the mount options | `[]` | +| `secret.create` | Specifies whether the secret should be created | `false` | +| `secret.name` | Specifies the cephFS secret name | `csi-cephfs-secret` | +| `secret.adminID` | Specifies the admin ID of the cephFS secret | `` | +| `secret.adminKey` | Specifies the key that corresponds to the adminID | `<Ceph auth key corresponding to ID above>` | + +### Command Line + +You can pass the settings with helm command line parameters. +Specify each parameter using the --set key=value argument to helm install. +For Example: + +```bash +helm install --set configMapName=ceph-csi-config --set provisioner.podSecurityPolicy.enabled=true +``` diff --git a/ceph/ceph-csi-cephfs/templates/NOTES.txt b/ceph/ceph-csi-cephfs/templates/NOTES.txt new file mode 100644 index 0000000..c0c8c55 --- /dev/null +++ b/ceph/ceph-csi-cephfs/templates/NOTES.txt @@ -0,0 +1,2 @@ +Examples on how to configure a storage class and start using the driver are here: +https://github.com/ceph/ceph-csi/tree/devel/examples/cephfs diff --git a/ceph/ceph-csi-cephfs/templates/_helpers.tpl b/ceph/ceph-csi-cephfs/templates/_helpers.tpl new file mode 100644 index 0000000..635cca6 --- /dev/null +++ b/ceph/ceph-csi-cephfs/templates/_helpers.tpl @@ -0,0 +1,90 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "ceph-csi-cephfs.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "ceph-csi-cephfs.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "ceph-csi-cephfs.nodeplugin.fullname" -}} +{{- if .Values.nodeplugin.fullnameOverride -}} +{{- .Values.nodeplugin.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- printf "%s-%s" .Release.Name .Values.nodeplugin.name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s-%s" .Release.Name $name .Values.nodeplugin.name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "ceph-csi-cephfs.provisioner.fullname" -}} +{{- if .Values.provisioner.fullnameOverride -}} +{{- .Values.provisioner.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- printf "%s-%s" .Release.Name .Values.provisioner.name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s-%s" .Release.Name $name .Values.provisioner.name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "ceph-csi-cephfs.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create the name of the service account to use +*/}} +{{- define "ceph-csi-cephfs.serviceAccountName.nodeplugin" -}} +{{- if .Values.serviceAccounts.nodeplugin.create -}} + {{ default (include "ceph-csi-cephfs.nodeplugin.fullname" .) .Values.serviceAccounts.nodeplugin.name }} +{{- else -}} + {{ default "default" .Values.serviceAccounts.nodeplugin.name }} +{{- end -}} +{{- end -}} + +{{/* +Create the name of the service account to use +*/}} +{{- define "ceph-csi-cephfs.serviceAccountName.provisioner" -}} +{{- if .Values.serviceAccounts.provisioner.create -}} + {{ default (include "ceph-csi-cephfs.provisioner.fullname" .) .Values.serviceAccounts.provisioner.name }} +{{- else -}} + {{ default "default" .Values.serviceAccounts.provisioner.name }} +{{- end -}} +{{- end -}} diff --git a/ceph/ceph-csi-cephfs/templates/ceph-conf.yaml b/ceph/ceph-csi-cephfs/templates/ceph-conf.yaml new file mode 100644 index 0000000..25c773b --- /dev/null +++ b/ceph/ceph-csi-cephfs/templates/ceph-conf.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: ceph-config + namespace: {{ .Release.Namespace }} + labels: + app: {{ include "ceph-csi-cephfs.name" . }} + chart: {{ include "ceph-csi-cephfs.chart" . }} + component: {{ .Values.nodeplugin.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +data: + ceph.conf: | +{{ tpl .Values.cephconf . | indent 4 }} + keyring: "" diff --git a/ceph/ceph-csi-cephfs/templates/csidriver-crd.yaml b/ceph/ceph-csi-cephfs/templates/csidriver-crd.yaml new file mode 100644 index 0000000..92c3d91 --- /dev/null +++ b/ceph/ceph-csi-cephfs/templates/csidriver-crd.yaml @@ -0,0 +1,11 @@ +{{ if semverCompare ">=1.18.0-beta.1" .Capabilities.KubeVersion.Version }} +apiVersion: storage.k8s.io/v1 +{{ else }} +apiVersion: storage.k8s.io/v1beta1 +{{ end }} +kind: CSIDriver +metadata: + name: {{ .Values.driverName }} +spec: + attachRequired: true + podInfoOnMount: false diff --git a/ceph/ceph-csi-cephfs/templates/csiplugin-configmap.yaml b/ceph/ceph-csi-cephfs/templates/csiplugin-configmap.yaml new file mode 100644 index 0000000..1f59b66 --- /dev/null +++ b/ceph/ceph-csi-cephfs/templates/csiplugin-configmap.yaml @@ -0,0 +1,16 @@ +{{- if not .Values.externallyManagedConfigmap }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Values.configMapName | quote }} + namespace: {{ .Release.Namespace }} + labels: + app: {{ include "ceph-csi-cephfs.name" . }} + chart: {{ include "ceph-csi-cephfs.chart" . }} + component: {{ .Values.provisioner.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +data: + config.json: |- +{{ toJson .Values.csiConfig | indent 4 -}} +{{- end }} diff --git a/ceph/ceph-csi-cephfs/templates/nodeplugin-clusterrole.yaml b/ceph/ceph-csi-cephfs/templates/nodeplugin-clusterrole.yaml new file mode 100644 index 0000000..f5bb71e --- /dev/null +++ b/ceph/ceph-csi-cephfs/templates/nodeplugin-clusterrole.yaml @@ -0,0 +1,18 @@ +{{- if .Values.rbac.create -}} +{{- if .Values.topology.enabled }} +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "ceph-csi-cephfs.nodeplugin.fullname" . }} + labels: + app: {{ include "ceph-csi-cephfs.name" . }} + chart: {{ include "ceph-csi-cephfs.chart" . }} + component: {{ .Values.nodeplugin.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get"] +{{- end }} +{{- end -}} diff --git a/ceph/ceph-csi-cephfs/templates/nodeplugin-clusterrolebinding.yaml b/ceph/ceph-csi-cephfs/templates/nodeplugin-clusterrolebinding.yaml new file mode 100644 index 0000000..f394bab --- /dev/null +++ b/ceph/ceph-csi-cephfs/templates/nodeplugin-clusterrolebinding.yaml @@ -0,0 +1,22 @@ +{{- if .Values.rbac.create -}} +{{- if .Values.topology.enabled }} +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "ceph-csi-cephfs.nodeplugin.fullname" . }} + labels: + app: {{ include "ceph-csi-cephfs.name" . }} + chart: {{ include "ceph-csi-cephfs.chart" . }} + component: {{ .Values.nodeplugin.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +subjects: + - kind: ServiceAccount + name: {{ include "ceph-csi-cephfs.serviceAccountName.nodeplugin" . }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ include "ceph-csi-cephfs.nodeplugin.fullname" . }} + apiGroup: rbac.authorization.k8s.io +{{- end }} +{{- end -}} diff --git a/ceph/ceph-csi-cephfs/templates/nodeplugin-daemonset.yaml b/ceph/ceph-csi-cephfs/templates/nodeplugin-daemonset.yaml new file mode 100644 index 0000000..8a4c43c --- /dev/null +++ b/ceph/ceph-csi-cephfs/templates/nodeplugin-daemonset.yaml @@ -0,0 +1,213 @@ +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: {{ include "ceph-csi-cephfs.nodeplugin.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + app: {{ include "ceph-csi-cephfs.name" . }} + chart: {{ include "ceph-csi-cephfs.chart" . }} + component: {{ .Values.nodeplugin.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + selector: + matchLabels: + app: {{ include "ceph-csi-cephfs.name" . }} + component: {{ .Values.nodeplugin.name }} + release: {{ .Release.Name }} + updateStrategy: + type: {{ .Values.nodeplugin.updateStrategy }} + template: + metadata: + labels: + app: {{ include "ceph-csi-cephfs.name" . }} + chart: {{ include "ceph-csi-cephfs.chart" . }} + component: {{ .Values.nodeplugin.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + spec: + serviceAccountName: {{ include "ceph-csi-cephfs.serviceAccountName.nodeplugin" . }} +{{- if .Values.nodeplugin.priorityClassName }} + priorityClassName: {{ .Values.nodeplugin.priorityClassName }} +{{- end }} + hostNetwork: true + # to use e.g. Rook orchestrated cluster, and mons' FQDN is + # resolved through k8s service, set dns policy to cluster first + dnsPolicy: ClusterFirstWithHostNet + containers: + - name: driver-registrar + # This is necessary only for systems with SELinux, where + # non-privileged sidecar containers cannot access unix domain socket + # created by privileged CSI driver container. + securityContext: + privileged: true + image: "{{ .Values.nodeplugin.registrar.image.repository }}:{{ .Values.nodeplugin.registrar.image.tag }}" + imagePullPolicy: {{ .Values.nodeplugin.registrar.image.pullPolicy }} + args: + - "--v={{ .Values.logLevel }}" + - "--csi-address=/csi/{{ .Values.pluginSocketFile }}" + - "--kubelet-registration-path={{ .Values.kubeletDir }}/plugins/{{ .Values.driverName }}/{{ .Values.pluginSocketFile }}" + env: + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: registration-dir + mountPath: /registration + resources: +{{ toYaml .Values.nodeplugin.registrar.resources | indent 12 }} + - name: csi-cephfsplugin + image: "{{ .Values.nodeplugin.plugin.image.repository }}:{{ .Values.nodeplugin.plugin.image.tag }}" + imagePullPolicy: {{ .Values.nodeplugin.plugin.image.pullPolicy }} + args: + - "--nodeid=$(NODE_ID)" + - "--type=cephfs" + - "--nodeserver=true" + - "--pidlimit=-1" +{{- if .Values.nodeplugin.forcecephkernelclient }} + - "--forcecephkernelclient={{ .Values.nodeplugin.forcecephkernelclient }}" +{{- end }} + - "--endpoint=$(CSI_ENDPOINT)" + - "--v={{ .Values.logLevel }}" + - "--drivername=$(DRIVER_NAME)" +{{- if .Values.topology.enabled }} + - "--domainlabels={{ .Values.topology.domainLabels | join "," }}" +{{- end }} +{{- if .Values.nodeplugin.profiling.enabled }} + - "--enableprofiling={{ .Values.nodeplugin.profiling.enabled }}" +{{- end }} + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: DRIVER_NAME + value: {{ .Values.driverName }} + - name: NODE_ID + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: CSI_ENDPOINT + value: "unix:///csi/{{ .Values.pluginSocketFile }}" + securityContext: + privileged: true + capabilities: + add: ["SYS_ADMIN"] + allowPrivilegeEscalation: true + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: mountpoint-dir + mountPath: {{ .Values.kubeletDir }}/pods + mountPropagation: Bidirectional + - name: plugin-dir + mountPath: {{ .Values.kubeletDir }}/plugins + mountPropagation: "Bidirectional" + - mountPath: /dev + name: host-dev + - mountPath: /run/mount + name: host-mount + - mountPath: /sys + name: host-sys + - mountPath: /etc/selinux + name: etc-selinux + readOnly: true + - mountPath: /lib/modules + name: lib-modules + readOnly: true + - name: ceph-config + mountPath: /etc/ceph/ + - name: ceph-csi-config + mountPath: /etc/ceph-csi-config/ + - name: keys-tmp-dir + mountPath: /tmp/csi/keys + resources: +{{ toYaml .Values.nodeplugin.plugin.resources | indent 12 }} +{{- if .Values.nodeplugin.httpMetrics.enabled }} + - name: liveness-prometheus + securityContext: + privileged: true + image: "{{ .Values.nodeplugin.plugin.image.repository }}:{{ .Values.nodeplugin.plugin.image.tag }}" + imagePullPolicy: {{ .Values.nodeplugin.plugin.image.pullPolicy }} + args: + - "--type=liveness" + - "--endpoint=$(CSI_ENDPOINT)" + - "--metricsport={{ .Values.nodeplugin.httpMetrics.containerPort }}" + - "--metricspath=/metrics" + - "--polltime=60s" + - "--timeout=3s" + env: + - name: CSI_ENDPOINT + value: "unix:///csi/{{ .Values.pluginSocketFile }}" + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + volumeMounts: + - name: socket-dir + mountPath: /csi + resources: +{{ toYaml .Values.nodeplugin.plugin.resources | indent 12 }} +{{- end }} + volumes: + - name: socket-dir + hostPath: + path: "{{ .Values.kubeletDir }}/plugins/{{ .Values.driverName }}" + type: DirectoryOrCreate + - name: registration-dir + hostPath: + path: {{ .Values.kubeletDir }}/plugins_registry + type: Directory + - name: mountpoint-dir + hostPath: + path: {{ .Values.kubeletDir }}/pods + type: DirectoryOrCreate + - name: plugin-dir + hostPath: + path: {{ .Values.kubeletDir }}/plugins + type: Directory + - name: host-sys + hostPath: + path: /sys + - name: etc-selinux + hostPath: + path: /etc/selinux + - name: host-mount + hostPath: + path: /run/mount + - name: lib-modules + hostPath: + path: /lib/modules + - name: host-dev + hostPath: + path: /dev + - name: ceph-config + configMap: + name: ceph-config + - name: ceph-csi-config + configMap: + name: {{ .Values.configMapName | quote }} +{{- if .Values.configMapKey }} + items: + - key: {{ .Values.configMapKey | quote }} + path: config.json +{{- end }} + - name: keys-tmp-dir + emptyDir: { + medium: "Memory" + } +{{- if .Values.nodeplugin.affinity }} + affinity: +{{ toYaml .Values.nodeplugin.affinity | indent 8 -}} +{{- end -}} +{{- if .Values.nodeplugin.nodeSelector }} + nodeSelector: +{{ toYaml .Values.nodeplugin.nodeSelector | indent 8 -}} +{{- end -}} +{{- if .Values.nodeplugin.tolerations }} + tolerations: +{{ toYaml .Values.nodeplugin.tolerations | indent 8 -}} +{{- end -}} diff --git a/ceph/ceph-csi-cephfs/templates/nodeplugin-http-service.yaml b/ceph/ceph-csi-cephfs/templates/nodeplugin-http-service.yaml new file mode 100644 index 0000000..95fee84 --- /dev/null +++ b/ceph/ceph-csi-cephfs/templates/nodeplugin-http-service.yaml @@ -0,0 +1,41 @@ +{{- if .Values.nodeplugin.httpMetrics.service.enabled -}} +apiVersion: v1 +kind: Service +metadata: +{{- if .Values.nodeplugin.httpMetrics.service.annotations }} + annotations: +{{ toYaml .Values.nodeplugin.httpMetrics.service.annotations | indent 4 }} +{{- end }} + name: {{ include "ceph-csi-cephfs.nodeplugin.fullname" . }}-http-metrics + namespace: {{ .Release.Namespace }} + labels: + app: {{ include "ceph-csi-cephfs.fullname" . }} + chart: {{ include "ceph-csi-cephfs.chart" . }} + component: {{ .Values.nodeplugin.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: +{{- if .Values.nodeplugin.httpMetrics.service.clusterIP }} + clusterIP: "{{ .Values.nodeplugin.httpMetrics.service.clusterIP }}" +{{- end }} +{{- if .Values.nodeplugin.httpMetrics.service.externalIPs }} + externalIPs: +{{ toYaml .Values.nodeplugin.httpMetrics.service.externalIPs | indent 4 }} +{{- end }} +{{- if .Values.nodeplugin.httpMetrics.service.loadBalancerIP }} + loadBalancerIP: "{{ .Values.nodeplugin.httpMetrics.service.loadBalancerIP }}" +{{- end }} +{{- if .Values.nodeplugin.httpMetrics.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: +{{ toYaml .Values.nodeplugin.httpMetrics.service.loadBalancerSourceRanges | indent 4 }} +{{- end }} + ports: + - name: http-metrics + port: {{ .Values.nodeplugin.httpMetrics.service.servicePort }} + targetPort: {{ .Values.nodeplugin.httpMetrics.containerPort }} + selector: + app: {{ include "ceph-csi-cephfs.name" . }} + component: {{ .Values.nodeplugin.name }} + release: {{ .Release.Name }} + type: "{{ .Values.nodeplugin.httpMetrics.service.type }}" +{{- end -}} diff --git a/ceph/ceph-csi-cephfs/templates/nodeplugin-psp.yaml b/ceph/ceph-csi-cephfs/templates/nodeplugin-psp.yaml new file mode 100644 index 0000000..d4ed417 --- /dev/null +++ b/ceph/ceph-csi-cephfs/templates/nodeplugin-psp.yaml @@ -0,0 +1,46 @@ +{{- if .Values.nodeplugin.podSecurityPolicy.enabled -}} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ include "ceph-csi-cephfs.nodeplugin.fullname" . }} + labels: + app: {{ include "ceph-csi-cephfs.fullname" . }} + chart: {{ include "ceph-csi-cephfs.chart" . }} + component: {{ .Values.nodeplugin.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + allowPrivilegeEscalation: true + allowedCapabilities: + - 'SYS_ADMIN' + fsGroup: + rule: RunAsAny + privileged: true + hostNetwork: true + hostPID: true + runAsUser: + rule: RunAsAny + seLinux: + rule: RunAsAny + supplementalGroups: + rule: RunAsAny + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'hostPath' + allowedHostPaths: + - pathPrefix: '/dev' + readOnly: false + - pathPrefix: '/run/mount' + readOnly: false + - pathPrefix: '/sys' + readOnly: false + - pathPrefix: '/etc/selinux' + readOnly: true + - pathPrefix: '/lib/modules' + readOnly: true + - pathPrefix: '{{ .Values.kubeletDir }}' + readOnly: false +{{- end }} diff --git a/ceph/ceph-csi-cephfs/templates/nodeplugin-role.yaml b/ceph/ceph-csi-cephfs/templates/nodeplugin-role.yaml new file mode 100644 index 0000000..4b211d7 --- /dev/null +++ b/ceph/ceph-csi-cephfs/templates/nodeplugin-role.yaml @@ -0,0 +1,18 @@ +{{- if and .Values.rbac.create .Values.nodeplugin.podSecurityPolicy.enabled -}} +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "ceph-csi-cephfs.nodeplugin.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + app: {{ include "ceph-csi-cephfs.fullname" . }} + chart: {{ include "ceph-csi-cephfs.chart" . }} + component: {{ .Values.nodeplugin.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +rules: + - apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: ['{{ include "ceph-csi-cephfs.nodeplugin.fullname" . }}'] +{{- end -}} diff --git a/ceph/ceph-csi-cephfs/templates/nodeplugin-rolebinding.yaml b/ceph/ceph-csi-cephfs/templates/nodeplugin-rolebinding.yaml new file mode 100644 index 0000000..19b3b6d --- /dev/null +++ b/ceph/ceph-csi-cephfs/templates/nodeplugin-rolebinding.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.rbac.create .Values.nodeplugin.podSecurityPolicy.enabled -}} +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "ceph-csi-cephfs.nodeplugin.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + app: {{ include "ceph-csi-cephfs.fullname" . }} + chart: {{ include "ceph-csi-cephfs.chart" . }} + component: {{ .Values.nodeplugin.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +subjects: + - kind: ServiceAccount + name: {{ include "ceph-csi-cephfs.serviceAccountName.nodeplugin" . }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: Role + name: {{ include "ceph-csi-cephfs.nodeplugin.fullname" . }} + apiGroup: rbac.authorization.k8s.io +{{- end -}} diff --git a/ceph/ceph-csi-cephfs/templates/nodeplugin-serviceaccount.yaml b/ceph/ceph-csi-cephfs/templates/nodeplugin-serviceaccount.yaml new file mode 100644 index 0000000..5dedaf4 --- /dev/null +++ b/ceph/ceph-csi-cephfs/templates/nodeplugin-serviceaccount.yaml @@ -0,0 +1,13 @@ +{{- if .Values.serviceAccounts.nodeplugin.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "ceph-csi-cephfs.serviceAccountName.nodeplugin" . }} + namespace: {{ .Release.Namespace }} + labels: + app: {{ include "ceph-csi-cephfs.name" . }} + chart: {{ include "ceph-csi-cephfs.chart" . }} + component: {{ .Values.nodeplugin.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +{{- end -}} diff --git a/ceph/ceph-csi-cephfs/templates/provisioner-clusterrole.yaml b/ceph/ceph-csi-cephfs/templates/provisioner-clusterrole.yaml new file mode 100644 index 0000000..c970766 --- /dev/null +++ b/ceph/ceph-csi-cephfs/templates/provisioner-clusterrole.yaml @@ -0,0 +1,63 @@ +{{- if .Values.rbac.create -}} +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "ceph-csi-cephfs.provisioner.fullname" . }} + labels: + app: {{ include "ceph-csi-cephfs.name" . }} + chart: {{ include "ceph-csi-cephfs.chart" . }} + component: {{ .Values.provisioner.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "delete","patch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "delete"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents/status"] + verbs: ["update"] +{{- if .Values.provisioner.attacher.enabled }} + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments/status"] + verbs: ["patch"] +{{- end -}} +{{- if semverCompare ">=1.15" .Capabilities.KubeVersion.Version -}} +{{- if .Values.provisioner.resizer.enabled }} + - apiGroups: [""] + resources: ["persistentvolumeclaims/status"] + verbs: ["update", "patch"] +{{- end -}} +{{- end -}} +{{- if .Values.topology.enabled }} + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] + verbs: ["get", "list", "watch"] +{{- end }} +{{- end -}} diff --git a/ceph/ceph-csi-cephfs/templates/provisioner-clusterrolebinding.yaml b/ceph/ceph-csi-cephfs/templates/provisioner-clusterrolebinding.yaml new file mode 100644 index 0000000..82d5d13 --- /dev/null +++ b/ceph/ceph-csi-cephfs/templates/provisioner-clusterrolebinding.yaml @@ -0,0 +1,20 @@ +{{- if .Values.rbac.create -}} +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "ceph-csi-cephfs.provisioner.fullname" . }} + labels: + app: {{ include "ceph-csi-cephfs.name" . }} + chart: {{ include "ceph-csi-cephfs.chart" . }} + component: {{ .Values.provisioner.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +subjects: + - kind: ServiceAccount + name: {{ include "ceph-csi-cephfs.serviceAccountName.provisioner" . }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ include "ceph-csi-cephfs.provisioner.fullname" . }} + apiGroup: rbac.authorization.k8s.io +{{- end -}} diff --git a/ceph/ceph-csi-cephfs/templates/provisioner-deployment.yaml b/ceph/ceph-csi-cephfs/templates/provisioner-deployment.yaml new file mode 100644 index 0000000..cf1ab09 --- /dev/null +++ b/ceph/ceph-csi-cephfs/templates/provisioner-deployment.yaml @@ -0,0 +1,241 @@ +kind: Deployment +apiVersion: apps/v1 +metadata: + name: {{ include "ceph-csi-cephfs.provisioner.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + app: {{ include "ceph-csi-cephfs.name" . }} + chart: {{ include "ceph-csi-cephfs.chart" . }} + component: {{ .Values.provisioner.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + replicas: {{ .Values.provisioner.replicaCount }} + strategy: + type: {{ .Values.provisioner.strategy.type }} +{{- if eq .Values.provisioner.strategy.type "RollingUpdate" }} + rollingUpdate: + maxUnavailable: {{ .Values.provisioner.strategy.rollingUpdate.maxUnavailable }} +{{- end }} + selector: + matchLabels: + app: {{ include "ceph-csi-cephfs.name" . }} + component: {{ .Values.provisioner.name }} + release: {{ .Release.Name }} + template: + metadata: + labels: + app: {{ include "ceph-csi-cephfs.name" . }} + chart: {{ include "ceph-csi-cephfs.chart" . }} + component: {{ .Values.provisioner.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + spec: +{{- if gt (int .Values.provisioner.replicaCount) 1 }} + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app + operator: In + values: + - {{ include "ceph-csi-cephfs.name" . }} + - key: component + operator: In + values: + - {{ .Values.provisioner.name }} + topologyKey: "kubernetes.io/hostname" +{{- end }} + serviceAccountName: {{ include "ceph-csi-cephfs.serviceAccountName.provisioner" . }} +{{- if .Values.provisioner.priorityClassName }} + priorityClassName: {{ .Values.provisioner.priorityClassName }} +{{- end }} + containers: + - name: csi-provisioner + image: "{{ .Values.provisioner.provisioner.image.repository }}:{{ .Values.provisioner.provisioner.image.tag }}" + imagePullPolicy: {{ .Values.provisioner.provisioner.image.pullPolicy }} + args: + - "--csi-address=$(ADDRESS)" + - "--v={{ .Values.logLevel }}" + - "--timeout={{ .Values.provisioner.timeout }}" + - "--leader-election=true" + - "--retry-interval-start=500ms" + - "--extra-create-metadata=true" +{{- if .Values.topology.enabled }} + - "--feature-gates=Topology=true" +{{- end }} + env: + - name: ADDRESS + value: "unix:///csi/{{ .Values.provisionerSocketFile }}" + volumeMounts: + - name: socket-dir + mountPath: /csi + resources: +{{ toYaml .Values.provisioner.provisioner.resources | indent 12 }} + - name: csi-snapshotter + image: {{ .Values.provisioner.snapshotter.image.repository }}:{{ .Values.provisioner.snapshotter.image.tag }} + imagePullPolicy: {{ .Values.provisioner.snapshotter.image.pullPolicy }} + args: + - "--csi-address=$(ADDRESS)" + - "--v={{ .Values.logLevel }}" + - "--timeout={{ .Values.provisioner.timeout }}" + - "--leader-election=true" + env: + - name: ADDRESS + value: "unix:///csi/{{ .Values.provisionerSocketFile }}" + volumeMounts: + - name: socket-dir + mountPath: /csi + resources: +{{ toYaml .Values.provisioner.snapshotter.resources | indent 12 }} +{{- if .Values.provisioner.attacher.enabled }} + - name: csi-attacher + image: "{{ .Values.provisioner.attacher.image.repository }}:{{ .Values.provisioner.attacher.image.tag }}" + imagePullPolicy: {{ .Values.provisioner.attacher.image.pullPolicy }} + args: + - "--v={{ .Values.logLevel }}" + - "--csi-address=$(ADDRESS)" + - "--leader-election=true" + - "--retry-interval-start=500ms" + env: + - name: ADDRESS + value: "unix:///csi/{{ .Values.provisionerSocketFile }}" + volumeMounts: + - name: socket-dir + mountPath: /csi + resources: +{{ toYaml .Values.nodeplugin.plugin.resources | indent 12 }} +{{- end }} +{{- if semverCompare ">=1.15" .Capabilities.KubeVersion.Version -}} +{{- if .Values.provisioner.resizer.enabled }} + - name: csi-resizer + image: "{{ .Values.provisioner.resizer.image.repository }}:{{ .Values.provisioner.resizer.image.tag }}" + imagePullPolicy: {{ .Values.provisioner.resizer.image.pullPolicy }} + args: + - "--v={{ .Values.logLevel }}" + - "--csi-address=$(ADDRESS)" + - "--timeout={{ .Values.provisioner.timeout }}" + - "--leader-election" + - "--retry-interval-start=500ms" + - "--handle-volume-inuse-error=false" + env: + - name: ADDRESS + value: "unix:///csi/{{ .Values.provisionerSocketFile }}" + volumeMounts: + - name: socket-dir + mountPath: /csi + resources: +{{ toYaml .Values.provisioner.resizer.resources | indent 12 }} +{{- end }} +{{- end }} + - name: csi-cephfsplugin + image: "{{ .Values.nodeplugin.plugin.image.repository }}:{{ .Values.nodeplugin.plugin.image.tag }}" + imagePullPolicy: {{ .Values.nodeplugin.plugin.image.pullPolicy }} + args: + - "--nodeid=$(NODE_ID)" + - "--type=cephfs" + - "--controllerserver=true" + - "--pidlimit=-1" + - "--endpoint=$(CSI_ENDPOINT)" + - "--v={{ .Values.logLevel }}" + - "--drivername=$(DRIVER_NAME)" +{{- if .Values.provisioner.profiling.enabled }} + - "--enableprofiling={{ .Values.provisioner.profiling.enabled }}" +{{- end }} + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: DRIVER_NAME + value: {{ .Values.driverName }} + - name: NODE_ID + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: CSI_ENDPOINT + value: "unix:///csi/{{ .Values.provisionerSocketFile }}" + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: host-sys + mountPath: /sys + - name: lib-modules + mountPath: /lib/modules + readOnly: true + - name: host-dev + mountPath: /dev + - name: ceph-config + mountPath: /etc/ceph/ + - name: ceph-csi-config + mountPath: /etc/ceph-csi-config/ + - name: keys-tmp-dir + mountPath: /tmp/csi/keys + resources: +{{ toYaml .Values.nodeplugin.plugin.resources | indent 12 }} +{{- if .Values.provisioner.httpMetrics.enabled }} + - name: liveness-prometheus + image: "{{ .Values.nodeplugin.plugin.image.repository }}:{{ .Values.nodeplugin.plugin.image.tag }}" + imagePullPolicy: {{ .Values.nodeplugin.plugin.image.pullPolicy }} + args: + - "--type=liveness" + - "--endpoint=$(CSI_ENDPOINT)" + - "--metricsport={{ .Values.provisioner.httpMetrics.containerPort }}" + - "--metricspath=/metrics" + - "--polltime=60s" + - "--timeout=3s" + env: + - name: CSI_ENDPOINT + value: "unix:///csi/{{ .Values.provisionerSocketFile }}" + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + volumeMounts: + - name: socket-dir + mountPath: /csi + resources: +{{ toYaml .Values.nodeplugin.plugin.resources | indent 12 }} +{{- end }} + volumes: + - name: socket-dir + emptyDir: { + medium: "Memory" + } + - name: host-sys + hostPath: + path: /sys + - name: lib-modules + hostPath: + path: /lib/modules + - name: host-dev + hostPath: + path: /dev + - name: ceph-config + configMap: + name: ceph-config + - name: ceph-csi-config + configMap: + name: {{ .Values.configMapName | quote }} +{{- if .Values.configMapKey }} + items: + - key: {{ .Values.configMapKey | quote }} + path: config.json +{{- end }} + - name: keys-tmp-dir + emptyDir: { + medium: "Memory" + } +{{- if .Values.provisioner.affinity }} + affinity: +{{ toYaml .Values.provisioner.affinity | indent 8 -}} +{{- end -}} +{{- if .Values.provisioner.nodeSelector }} + nodeSelector: +{{ toYaml .Values.provisioner.nodeSelector | indent 8 -}} +{{- end -}} +{{- if .Values.provisioner.tolerations }} + tolerations: +{{ toYaml .Values.provisioner.tolerations | indent 8 -}} +{{- end -}} diff --git a/ceph/ceph-csi-cephfs/templates/provisioner-http-service.yaml b/ceph/ceph-csi-cephfs/templates/provisioner-http-service.yaml new file mode 100644 index 0000000..5483242 --- /dev/null +++ b/ceph/ceph-csi-cephfs/templates/provisioner-http-service.yaml @@ -0,0 +1,41 @@ +{{- if .Values.provisioner.httpMetrics.service.enabled -}} +apiVersion: v1 +kind: Service +metadata: +{{- if .Values.provisioner.httpMetrics.service.annotations }} + annotations: +{{ toYaml .Values.provisioner.httpMetrics.service.annotations | indent 4 }} +{{- end }} + name: {{ include "ceph-csi-cephfs.provisioner.fullname" . }}-http-metrics + namespace: {{ .Release.Namespace }} + labels: + app: {{ include "ceph-csi-cephfs.fullname" . }} + chart: {{ include "ceph-csi-cephfs.chart" . }} + component: {{ .Values.provisioner.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: +{{- if .Values.provisioner.httpMetrics.service.clusterIP }} + clusterIP: "{{ .Values.provisioner.httpMetrics.service.clusterIP }}" +{{- end }} +{{- if .Values.provisioner.httpMetrics.service.externalIPs }} + externalIPs: +{{ toYaml .Values.provisioner.httpMetrics.service.externalIPs | indent 4 }} +{{- end }} +{{- if .Values.provisioner.httpMetrics.service.loadBalancerIP }} + loadBalancerIP: "{{ .Values.provisioner.httpMetrics.service.loadBalancerIP }}" +{{- end }} +{{- if .Values.provisioner.httpMetrics.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: +{{ toYaml .Values.provisioner.httpMetrics.service.loadBalancerSourceRanges | indent 4 }} +{{- end }} + ports: + - name: http-metrics + port: {{ .Values.provisioner.httpMetrics.service.servicePort }} + targetPort: {{ .Values.provisioner.httpMetrics.containerPort }} + selector: + app: {{ include "ceph-csi-cephfs.name" . }} + component: {{ .Values.provisioner.name }} + release: {{ .Release.Name }} + type: "{{ .Values.provisioner.httpMetrics.service.type }}" +{{- end -}} diff --git a/ceph/ceph-csi-cephfs/templates/provisioner-psp.yaml b/ceph/ceph-csi-cephfs/templates/provisioner-psp.yaml new file mode 100644 index 0000000..fceecd0 --- /dev/null +++ b/ceph/ceph-csi-cephfs/templates/provisioner-psp.yaml @@ -0,0 +1,34 @@ +{{- if .Values.provisioner.podSecurityPolicy.enabled -}} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ include "ceph-csi-cephfs.provisioner.fullname" . }} + labels: + app: {{ include "ceph-csi-cephfs.name" . }} + chart: {{ include "ceph-csi-cephfs.chart" . }} + component: {{ .Values.provisioner.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + fsGroup: + rule: RunAsAny + runAsUser: + rule: RunAsAny + seLinux: + rule: RunAsAny + supplementalGroups: + rule: RunAsAny + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'hostPath' + allowedHostPaths: + - pathPrefix: '/dev' + readOnly: false + - pathPrefix: '/sys' + readOnly: false + - pathPrefix: '/lib/modules' + readOnly: true +{{- end }} diff --git a/ceph/ceph-csi-cephfs/templates/provisioner-role.yaml b/ceph/ceph-csi-cephfs/templates/provisioner-role.yaml new file mode 100644 index 0000000..983bc60 --- /dev/null +++ b/ceph/ceph-csi-cephfs/templates/provisioner-role.yaml @@ -0,0 +1,26 @@ +{{- if .Values.rbac.create -}} +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "ceph-csi-cephfs.provisioner.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + app: {{ include "ceph-csi-cephfs.name" . }} + chart: {{ include "ceph-csi-cephfs.chart" . }} + component: {{ .Values.provisioner.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +rules: + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] +{{- if .Values.provisioner.podSecurityPolicy.enabled }} + - apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: ['{{ include "ceph-csi-cephfs.provisioner.fullname" . }}'] +{{- end -}} +{{- end -}} diff --git a/ceph/ceph-csi-cephfs/templates/provisioner-rolebinding.yaml b/ceph/ceph-csi-cephfs/templates/provisioner-rolebinding.yaml new file mode 100644 index 0000000..66fb725 --- /dev/null +++ b/ceph/ceph-csi-cephfs/templates/provisioner-rolebinding.yaml @@ -0,0 +1,21 @@ +{{- if .Values.rbac.create -}} +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "ceph-csi-cephfs.provisioner.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + app: {{ include "ceph-csi-cephfs.name" . }} + chart: {{ include "ceph-csi-cephfs.chart" . }} + component: {{ .Values.provisioner.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +subjects: + - kind: ServiceAccount + name: {{ include "ceph-csi-cephfs.serviceAccountName.provisioner" . }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: Role + name: {{ include "ceph-csi-cephfs.provisioner.fullname" . }} + apiGroup: rbac.authorization.k8s.io +{{- end -}} diff --git a/ceph/ceph-csi-cephfs/templates/provisioner-serviceaccount.yaml b/ceph/ceph-csi-cephfs/templates/provisioner-serviceaccount.yaml new file mode 100644 index 0000000..c4ba5c1 --- /dev/null +++ b/ceph/ceph-csi-cephfs/templates/provisioner-serviceaccount.yaml @@ -0,0 +1,13 @@ +{{- if .Values.serviceAccounts.provisioner.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "ceph-csi-cephfs.serviceAccountName.provisioner" . }} + namespace: {{ .Release.Namespace }} + labels: + app: {{ include "ceph-csi-cephfs.name" . }} + chart: {{ include "ceph-csi-cephfs.chart" . }} + component: {{ .Values.provisioner.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +{{- end -}} diff --git a/ceph/ceph-csi-cephfs/templates/secret.yaml b/ceph/ceph-csi-cephfs/templates/secret.yaml new file mode 100644 index 0000000..7063543 --- /dev/null +++ b/ceph/ceph-csi-cephfs/templates/secret.yaml @@ -0,0 +1,15 @@ +{{- if .Values.secret.create -}} +apiVersion: v1 +kind: Secret +metadata: + name: {{ .Values.secret.name }} + namespace: {{ .Release.Namespace }} + labels: + app: {{ include "ceph-csi-cephfs.name" . }} + chart: {{ include "ceph-csi-cephfs.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +stringData: + adminID: {{ .Values.secret.adminID }} + adminKey: {{ .Values.secret.adminKey }} +{{- end -}} diff --git a/ceph/ceph-csi-cephfs/templates/storageclass.yaml b/ceph/ceph-csi-cephfs/templates/storageclass.yaml new file mode 100644 index 0000000..62709ad --- /dev/null +++ b/ceph/ceph-csi-cephfs/templates/storageclass.yaml @@ -0,0 +1,61 @@ +{{- if .Values.storageClass.create -}} +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: {{ .Values.storageClass.name }} + namespace: {{ .Release.Namespace }} +{{- if .Values.storageClass.annotations }} + annotations: +{{ toYaml .Values.storageClass.annotations | indent 4 }} +{{- end }} + labels: + app: {{ include "ceph-csi-cephfs.name" . }} + chart: {{ include "ceph-csi-cephfs.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +provisioner: {{ .Values.driverName }} +parameters: + clusterID: {{ .Values.storageClass.clusterID }} + fsName: {{ .Values.storageClass.fsName }} +{{- if .Values.storageClass.pool }} + pool: {{ .Values.storageClass.pool }} +{{- end }} +{{- if .Values.storageClass.fuseMountOptions }} + fuseMountOptions: "{{ .Values.storageClass.fuseMountOptions }}" +{{- end }} +{{- if .Values.storageClass.kernelMountOptions }} + kernelMountOptions: "{{ .Values.storageClass.kernelMountOptions }}" +{{- end }} +{{- if .Values.storageClass.mounter }} + mounter: "{{ .Values.storageClass.mounter }}" +{{- end }} +{{- if .Values.storageClass.volumeNamePrefix }} + volumeNamePrefix: "{{ .Values.storageClass.volumeNamePrefix }}" +{{- end }} + csi.storage.k8s.io/provisioner-secret-name: {{ .Values.storageClass.provisionerSecret }} +{{- if .Values.storageClass.provisionerSecretNamespace }} + csi.storage.k8s.io/provisioner-secret-namespace: {{ .Values.storageClass.provisionerSecretNamespace }} +{{ else }} + csi.storage.k8s.io/provisioner-secret-namespace: {{ .Release.Namespace }} +{{- end }} + csi.storage.k8s.io/controller-expand-secret-name: {{ .Values.storageClass.controllerExpandSecret }} +{{- if .Values.storageClass.controllerExpandSecretNamespace }} + csi.storage.k8s.io/controller-expand-secret-namespace: {{ .Values.storageClass.controllerExpandSecretNamespace }} +{{ else }} + csi.storage.k8s.io/controller-expand-secret-namespace: {{ .Release.Namespace }} +{{- end }} + csi.storage.k8s.io/node-stage-secret-name: {{ .Values.storageClass.nodeStageSecret }} +{{- if .Values.storageClass.nodeStageSecretNamespace }} + csi.storage.k8s.io/node-stage-secret-namespace: {{ .Values.storageClass.nodeStageSecretNamespace }} +{{ else }} + csi.storage.k8s.io/node-stage-secret-namespace: {{ .Release.Namespace }} +{{- end }} +reclaimPolicy: {{ .Values.storageClass.reclaimPolicy }} +allowVolumeExpansion: {{ .Values.storageClass.allowVolumeExpansion }} +{{- if .Values.storageClass.mountOptions }} +mountOptions: + {{- range .Values.storageClass.mountOptions }} + - {{ . }} + {{- end }} +{{- end }} +{{- end -}} diff --git a/ceph/ceph-csi-cephfs/values.yaml b/ceph/ceph-csi-cephfs/values.yaml new file mode 100644 index 0000000..1eef954 --- /dev/null +++ b/ceph/ceph-csi-cephfs/values.yaml @@ -0,0 +1,321 @@ +--- +rbac: + # Specifies whether RBAC resources should be created + create: true + +serviceAccounts: + nodeplugin: + # Specifies whether a ServiceAccount should be created + create: true + # The name of the ServiceAccount to use. + # If not set and create is true, a name is generated using the fullname + name: + provisioner: + # Specifies whether a ServiceAccount should be created + create: true + # The name of the ServiceAccount to use. + # If not set and create is true, a name is generated using the fullname + name: + +# Configuration for the CSI to connect to the cluster +# Ref: https://github.com/ceph/ceph-csi/blob/devel/examples/README.md +# Example: +# csiConfig: +# - clusterID: "<cluster-id>" +# monitors: +# - "<MONValue1>" +# - "<MONValue2>" +# cephFS: +# subvolumeGroup: "csi" +csiConfig: [] + +# Set logging level for csi containers. +# Supported values from 0 to 5. 0 for general useful logs, +# 5 for trace level verbosity. +logLevel: 5 + +nodeplugin: + name: nodeplugin + # if you are using ceph-fuse client set this value to OnDelete + updateStrategy: RollingUpdate + + # set user created priorityclassName for csi plugin pods. default is + # system-node-critical which is highest priority + priorityClassName: system-node-critical + + httpMetrics: + # Metrics only available for cephcsi/cephcsi => 1.2.0 + # Specifies whether http metrics should be exposed + enabled: true + # The port of the container to expose the metrics + containerPort: 8081 + + service: + # Specifies whether a service should be created for the metrics + enabled: true + # The port to use for the service + servicePort: 8080 + type: ClusterIP + + # Annotations for the service + # Example: + # annotations: + # prometheus.io/scrape: "true" + # prometheus.io/port: "9080" + annotations: {} + + clusterIP: "" + + ## List of IP addresses at which the stats-exporter service is available + ## Ref: https://kubernetes.io/docs/user-guide/services/#external-ips + ## + externalIPs: [] + + loadBalancerIP: "" + loadBalancerSourceRanges: [] + + profiling: + enabled: false + + registrar: + image: + repository: k8s.gcr.io/sig-storage/csi-node-driver-registrar + tag: v2.3.0 + pullPolicy: IfNotPresent + resources: {} + + plugin: + image: + repository: quay.io/cephcsi/cephcsi + tag: canary + pullPolicy: IfNotPresent + resources: {} + + nodeSelector: {} + + tolerations: [] + + affinity: {} + + # Set to true to enable Ceph Kernel clients + # on kernel < 4.17 which support quotas + # forcecephkernelclient: true + + # If true, create & use Pod Security Policy resources + # https://kubernetes.io/docs/concepts/policy/pod-security-policy/ + podSecurityPolicy: + enabled: false + +provisioner: + name: provisioner + replicaCount: 3 + strategy: + # RollingUpdate strategy replaces old pods with new ones gradually, + # without incurring downtime. + type: RollingUpdate + rollingUpdate: + # maxUnavailable is the maximum number of pods that can be + # unavailable during the update process. + maxUnavailable: 50% + # Timeout for waiting for creation or deletion of a volume + timeout: 60s + + # set user created priorityclassName for csi provisioner pods. default is + # system-cluster-critical which is less priority than system-node-critical + priorityClassName: system-cluster-critical + + httpMetrics: + # Metrics only available for cephcsi/cephcsi => 1.2.0 + # Specifies whether http metrics should be exposed + enabled: true + # The port of the container to expose the metrics + containerPort: 8081 + + service: + # Specifies whether a service should be created for the metrics + enabled: true + # The port to use for the service + servicePort: 8080 + type: ClusterIP + + # Annotations for the service + # Example: + # annotations: + # prometheus.io/scrape: "true" + # prometheus.io/port: "9080" + annotations: {} + + clusterIP: "" + + ## List of IP addresses at which the stats-exporter service is available + ## Ref: https://kubernetes.io/docs/user-guide/services/#external-ips + ## + externalIPs: [] + + loadBalancerIP: "" + loadBalancerSourceRanges: [] + + profiling: + enabled: false + + provisioner: + image: + repository: k8s.gcr.io/sig-storage/csi-provisioner + tag: v3.0.0 + pullPolicy: IfNotPresent + resources: {} + + attacher: + name: attacher + enabled: true + image: + repository: k8s.gcr.io/sig-storage/csi-attacher + tag: v3.3.0 + pullPolicy: IfNotPresent + resources: {} + + resizer: + name: resizer + enabled: true + image: + repository: k8s.gcr.io/sig-storage/csi-resizer + tag: v1.3.0 + pullPolicy: IfNotPresent + resources: {} + + snapshotter: + image: + repository: k8s.gcr.io/sig-storage/csi-snapshotter + tag: v4.2.0 + pullPolicy: IfNotPresent + resources: {} + + nodeSelector: {} + + tolerations: [] + + affinity: {} + + # If true, create & use Pod Security Policy resources + # https://kubernetes.io/docs/concepts/policy/pod-security-policy/ + podSecurityPolicy: + enabled: false + +topology: + # Specifies whether topology based provisioning support should + # be exposed by CSI + enabled: false + # domainLabels define which node labels to use as domains + # for CSI nodeplugins to advertise their domains + # NOTE: the value here serves as an example and needs to be + # updated with node labels that define domains of interest + domainLabels: + - failure-domain/region + - failure-domain/zone + +storageClass: + # Specifies whether the Storage class should be created + create: false + name: csi-cephfs-sc + # Annotations for the storage class + # Example: + # annotations: + # storageclass.kubernetes.io/is-default-class: "true" + annotations: {} + + # String representing a Ceph cluster to provision storage from. + # Should be unique across all Ceph clusters in use for provisioning, + # cannot be greater than 36 bytes in length, and should remain immutable for + # the lifetime of the StorageClass in use. + clusterID: <cluster-ID> + # (required) CephFS filesystem name into which the volume shall be created + # eg: fsName: myfs + fsName: myfs + # (optional) Ceph pool into which volume data shall be stored + # pool: <cephfs-data-pool> + # For eg: + # pool: "replicapool" + pool: "" + # (optional) Comma separated string of Ceph-fuse mount options. + # For eg: + # fuseMountOptions: debug + fuseMountOptions: "" + # (optional) Comma separated string of Cephfs kernel mount options. + # Check man mount.ceph for mount options. For eg: + # kernelMountOptions: readdir_max_bytes=1048576,norbytes + kernelMountOptions: "" + # (optional) The driver can use either ceph-fuse (fuse) or + # ceph kernelclient (kernel). + # If omitted, default volume mounter will be used - this is + # determined by probing for ceph-fuse and mount.ceph + # mounter: kernel + mounter: "" + # (optional) Prefix to use for naming subvolumes. + # If omitted, defaults to "csi-vol-". + # volumeNamePrefix: "foo-bar-" + volumeNamePrefix: "" + # The secrets have to contain user and/or Ceph admin credentials. + provisionerSecret: csi-cephfs-secret + # If the Namespaces are not specified, the secrets are assumed to + # be in the Release namespace. + provisionerSecretNamespace: "" + controllerExpandSecret: csi-cephfs-secret + controllerExpandSecretNamespace: "" + nodeStageSecret: csi-cephfs-secret + nodeStageSecretNamespace: "" + reclaimPolicy: Delete + allowVolumeExpansion: true + mountOptions: [] + # Mount Options + # Example: + # mountOptions: + # - discard + +secret: + # Specifies whether the secret should be created + create: false + name: csi-cephfs-secret + # Key values correspond to a user name and its key, as defined in the + # ceph cluster. User ID should have required access to the 'pool' + # specified in the storage class + adminID: <plaintext ID> + adminKey: <Ceph auth key corresponding to ID above> + +# This is a sample configmap that helps define a Ceph configuration as required +# by the CSI plugins. +# Sample ceph.conf available at +# https://github.com/ceph/ceph/blob/master/src/sample.ceph.conf Detailed +# documentation is available at +# https://docs.ceph.com/en/latest/rados/configuration/ceph-conf/ +cephconf: | + [global] + auth_cluster_required = cephx + auth_service_required = cephx + auth_client_required = cephx + + # Workaround for http://tracker.ceph.com/issues/23446 + fuse_set_user_groups = false + + # ceph-fuse which uses libfuse2 by default has write buffer size of 2KiB + # adding 'fuse_big_writes = true' option by default to override this limit + # see https://github.com/ceph/ceph-csi/issues/1928 + fuse_big_writes = true + +######################################################### +# Variables for 'internal' use please use with caution! # +######################################################### + +# The filename of the provisioner socket +provisionerSocketFile: csi-provisioner.sock +# The filename of the plugin socket +pluginSocketFile: csi.sock +# kubelet working directory,can be set using `--root-dir` when starting kubelet. +kubeletDir: /var/lib/kubelet +# Name of the csi-driver +driverName: cephfs.csi.ceph.com +# Name of the configmap used for state +configMapName: ceph-csi-config +# Key to use in the Configmap if not config.json +# configMapKey: +# Use an externally provided configmap +externallyManagedConfigmap: false diff --git a/ceph/ceph-csi-rbd/.helmignore b/ceph/ceph-csi-rbd/.helmignore new file mode 100644 index 0000000..f0c1319 --- /dev/null +++ b/ceph/ceph-csi-rbd/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/ceph/ceph-csi-rbd/Chart.yaml b/ceph/ceph-csi-rbd/Chart.yaml new file mode 100644 index 0000000..107647b --- /dev/null +++ b/ceph/ceph-csi-rbd/Chart.yaml @@ -0,0 +1,15 @@ +--- +apiVersion: v1 +appVersion: canary +description: "Container Storage Interface (CSI) driver, +provisioner, snapshotter, and attacher for Ceph RBD" +name: ceph-csi-rbd +version: 3-canary +keywords: + - ceph + - rbd + - ceph-csi +home: https://github.com/ceph/ceph-csi +sources: + - https://github.com/ceph/ceph-csi/tree/devel/charts/ceph-csi-rbd +icon: https://raw.githubusercontent.com/ceph/ceph-csi/devel/assets/ceph-logo.png diff --git a/ceph/ceph-csi-rbd/README.md b/ceph/ceph-csi-rbd/README.md new file mode 100644 index 0000000..117facb --- /dev/null +++ b/ceph/ceph-csi-rbd/README.md @@ -0,0 +1,188 @@ +# ceph-csi-rbd + +The ceph-csi-rbd chart adds rbd volume support to your cluster. + +## Install from release repo + +Add chart repository to install helm charts from it + +```console +helm repo add ceph-csi https://ceph.github.io/csi-charts +``` + +## Install from local Chart + +we need to enter into the directory where all charts are present + +```console +cd charts +``` + +**Note:** charts directory is present in root of the ceph-csi project + +### Install chart + +To install the Chart into your Kubernetes cluster + +- For helm 2.x + + ```bash + helm install --namespace "ceph-csi-rbd" --name "ceph-csi-rbd" ceph-csi/ceph-csi-rbd + ``` + +- For helm 3.x + + Create the namespace where Helm should install the components with + + ```bash + kubectl create namespace "ceph-csi-rbd" + ``` + + Run the installation + + ```bash + helm install --namespace "ceph-csi-rbd" "ceph-csi-rbd" ceph-csi/ceph-csi-rbd + ``` + +After installation succeeds, you can get a status of Chart + +```bash +helm status "ceph-csi-rbd" +``` + +### Delete Chart + +If you want to delete your Chart, use this command + +- For helm 2.x + + ```bash + helm delete --purge "ceph-csi-rbd" + ``` + +- For helm 3.x + + ```bash + helm uninstall "ceph-csi-rbd" --namespace "ceph-csi-rbd" + ``` + +If you want to delete the namespace, use this command + +```bash +kubectl delete namespace ceph-csi-rbd +``` + +### Configuration + +The following table lists the configurable parameters of the ceph-csi-cephfs +charts and their default values. + +| Parameter | Description | Default | +| ---------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------- | +| `rbac.create` | Specifies whether RBAC resources should be created | `true` | +| `serviceAccounts.nodeplugin.create` | Specifies whether a nodeplugin ServiceAccount should be created | `true` | +| `serviceAccounts.nodeplugin.name` | The name of the nodeplugin ServiceAccount to use. If not set and create is true, a name is generated using the fullname | "" | +| `serviceAccounts.provisioner.create` | Specifies whether a provisioner ServiceAccount should be created | `true` | +| `serviceAccounts.provisioner.name` | The name of the provisioner ServiceAccount to use. If not set and create is true, a name is generated using the fullname | "" | +| `csiConfig` | Configuration for the CSI to connect to the cluster | [] | +| `csiMapping` | Configuration details of clusterID,PoolID,FscID mapping | [] | +| `encryptionKMSConfig` | Configuration for the encryption KMS | `{}` | +| `logLevel` | Set logging level for csi containers. Supported values from 0 to 5. 0 for general useful logs, 5 for trace level verbosity. | `5` | +| `nodeplugin.name` | Specifies the nodeplugins name | `nodeplugin` | +| `nodeplugin.updateStrategy` | Specifies the update Strategy. If you are using ceph-fuse client set this value to OnDelete | `RollingUpdate` | +| `nodeplugin.priorityClassName` | Set user created priorityclassName for csi plugin pods. default is system-node-critical which is highest priority | `system-node-critical` | +| `nodeplugin.profiling.enabled` | Specifies whether profiling should be enabled | `false` | +| `nodeplugin.registrar.image.repository` | Node Registrar image repository URL | `k8s.gcr.io/sig-storage/csi-node-driver-registrar` | +| `nodeplugin.registrar.image.tag` | Image tag | `v2.2.0` | +| `nodeplugin.registrar.image.pullPolicy` | Image pull policy | `IfNotPresent` | +| `nodeplugin.plugin.image.repository` | Nodeplugin image repository URL | `quay.io/cephcsi/cephcsi` | +| `nodeplugin.plugin.image.tag` | Image tag | `canary` | +| `nodeplugin.plugin.image.pullPolicy` | Image pull policy | `IfNotPresent` | +| `nodeplugin.nodeSelector` | Kubernetes `nodeSelector` to add to the Daemonset | `{}` | +| `nodeplugin.tolerations` | List of Kubernetes `tolerations` to add to the Daemonset | `{}` | +| `nodeplugin.podSecurityPolicy.enabled` | If true, create & use [Pod Security Policy resources](https://kubernetes.io/docs/concepts/policy/pod-security-policy/). | `false` | +| `provisioner.name` | Specifies the name of provisioner | `provisioner` | +| `provisioner.replicaCount` | Specifies the replicaCount | `3` | +| `provisioner.defaultFSType` | Specifies the default Fstype | `ext4` | +| `provisioner.deployController` | It enables or disables the deployment of controller which generates the OMAP data if it is not present | `true` | +| `provisioner.hardMaxCloneDepth` | Hard limit for maximum number of nested volume clones that are taken before a flatten occurs | `8` | +| `provisioner.softMaxCloneDepth` | Soft limit for maximum number of nested volume clones that are taken before a flatten occurs | `4` | +| `provisioner.maxSnapshotsOnImage` | Maximum number of snapshots allowed on rbd image without flattening | `450` | +| `provisioner.minSnapshotsOnImage` | Minimum number of snapshots allowed on rbd image to trigger flattening | `250` | +| `provisioner.skipForceFlatten` | Skip image flattening if kernel support mapping of rbd images which has the deep-flatten feature | `false` | +| `provisioner.timeout` | GRPC timeout for waiting for creation or deletion of a volume | `60s` | +| `provisioner.priorityClassName` | Set user created priorityclassName for csi provisioner pods. Default is `system-cluster-critical` which is less priority than `system-node-critical` | `system-cluster-critical` | +| `provisioner.profiling.enabled` | Specifies whether profiling should be enabled | `false` | +| `provisioner.provisioner.image.repository` | Specifies the csi-provisioner image repository URL | `k8s.gcr.io/sig-storage/csi-provisioner` | +| `provisioner.provisioner.image.tag` | Specifies image tag | `v2.2.2` | +| `provisioner.provisioner.image.pullPolicy` | Specifies pull policy | `IfNotPresent` | +| `provisioner.attacher.image.repository` | Specifies the csi-attacher image repository URL | `k8s.gcr.io/sig-storage/csi-attacher` | +| `provisioner.attacher.image.tag` | Specifies image tag | `v3.2.1` | +| `provisioner.attacher.image.pullPolicy` | Specifies pull policy | `IfNotPresent` | +| `provisioner.attacher.name` | Specifies the name of csi-attacher sidecar | `attacher` | +| `provisioner.attacher.enabled` | Specifies whether attacher sidecar is enabled | `true` | +| `provisioner.resizer.image.repository` | Specifies the csi-resizer image repository URL | `k8s.gcr.io/sig-storage/csi-resizer` | +| `provisioner.resizer.image.tag` | Specifies image tag | `v1.2.0` | +| `provisioner.resizer.image.pullPolicy` | Specifies pull policy | `IfNotPresent` | +| `provisioner.resizer.name` | Specifies the name of csi-resizer sidecar | `resizer` | +| `provisioner.resizer.enabled` | Specifies whether resizer sidecar is enabled | `true` | +| `provisioner.snapshotter.image.repository` | Specifies the csi-snapshotter image repository URL | `k8s.gcr.io/sig-storage/csi-snapshotter` | +| `provisioner.snapshotter.image.tag` | Specifies image tag | `v4.1.1` | +| `provisioner.snapshotter.image.pullPolicy` | Specifies pull policy | `IfNotPresent` | +| `provisioner.nodeSelector` | Specifies the node selector for provisioner deployment | `{}` | +| `provisioner.tolerations` | Specifies the tolerations for provisioner deployment | `{}` | +| `provisioner.affinity` | Specifies the affinity for provisioner deployment | `{}` | +| `provisioner.podSecurityPolicy.enabled` | Specifies whether podSecurityPolicy is enabled | `false` | +| `topology.enabled` | Specifies whether topology based provisioning support should be exposed by CSI | `false` | +| `topology.domainLabels` | DomainLabels define which node labels to use as domains for CSI nodeplugins to advertise their domains | `{}` | +| `provisionerSocketFile` | The filename of the provisioner socket | `csi-provisioner.sock` | +| `pluginSocketFile` | The filename of the plugin socket | `csi.sock` | +| `kubeletDir` | kubelet working directory | `/var/lib/kubelet` | +| `cephLogDirHostPath` | Host path location for ceph client processes logging, ex: rbd-nbd | `/var/log/ceph` | +| `driverName` | Name of the csi-driver | `rbd.csi.ceph.com` | +| `configMapName` | Name of the configmap which contains cluster configuration | `ceph-csi-config` | +| `externallyManagedConfigmap` | Specifies the use of an externally provided configmap | `false` | +| `kmsConfigMapName` | Name of the configmap used for encryption kms configuration | `ceph-csi-encryption-kms-config` | +| `storageClass.create` | Specifies whether the StorageClass should be created | `false` | +| `storageClass.name` | Specifies the rbd StorageClass name | `csi-rbd-sc` | +| `storageClass.annotations` | Specifies the annotations for the rbd StorageClass | `[]` | +| `storageClass.clusterID` | String representing a Ceph cluster to provision storage from | `<cluster-ID>` | +| `storageClass.dataPool` | Specifies the erasure coded pool | `""` | +| `storageClass.pool` | Ceph pool into which the RBD image shall be created | `replicapool` | +| `storageClass.thickProvision` | Specifies whether thick provision should be enabled | `false` | +| `storageclass.imageFeatures` | Specifies RBD image features | `layering` | +| `storageclass.tryOtherMounters` | Specifies whether to try other mounters in case if the current mounter fails to mount the rbd image for any reason | `false` | +| `storageClass.mounter` | Specifies RBD mounter | `""` | +| `storageClass.cephLogDir` | ceph client log location, it is the target bindmount path used inside container | `"/var/log/ceph"` | +| `storageClass.cephLogStrategy` | ceph client log strategy, available options `remove` or `compress` or `preserve` | `"remove"` | +| `storageClass.volumeNamePrefix` | Prefix to use for naming RBD images | `""` | +| `storageClass.encrypted` | Specifies whether volume should be encrypted. Set it to true if you want to enable encryption | `""` | +| `storageClass.encryptionKMSID` | Specifies the encryption kms id | `""` | +| `storageClass.topologyConstrainedPools` | Add topology constrained pools configuration, if topology based pools are setup, and topology constrained provisioning is required | `[]` | +| `storageClass.mapOptions` | Specifies comma-separated list of map options | `""` | +| `storageClass.unmapOtpions` | Specifies comma-separated list of unmap options | `""` | +| `storageClass.provisionerSecret` | The secrets have to contain user and/or Ceph admin credentials. | `csi-rbd-secret` | +| `storageClass.provisionerSecretNamespace` | Specifies the provisioner secret namespace | `""` | +| `storageClass.controllerExpandSecret` | Specifies the controller expand secret name | `csi-rbd-secret` | +| `storageClass.controllerExpandSecretNamespace` | Specifies the controller expand secret namespace | `""` | +| `storageClass.nodeStageSecret` | Specifies the node stage secret name | `csi-rbd-secret` | +| `storageClass.nodeStageSecretNamespace` | Specifies the node stage secret namespace | `""` | +| `storageClass.fstype` | Specify the filesystem type of the volume | `ext4` | +| `storageClass.reclaimPolicy` | Specifies the reclaim policy of the StorageClass | `Delete` | +| `storageClass.allowVolumeExpansion` | Specifies whether volume expansion should be allowed | `true` | +| `storageClass.mountOptions` | Specifies the mount options for storageClass | `[]` | +| `secret.create` | Specifies whether the secret should be created | `false` | +| `secret.name` | Specifies the rbd secret name | `csi-rbd-secret` | +| `secret.userID` | Specifies the user ID of the rbd secret | `<plaintext ID>` | +| `secret.userKey` | Specifies the key that corresponds to the userID | `<Ceph auth key corresponding to ID above>` | +| `secret.encryptionPassphrase` | Specifies the encryption passphrase of the secret | `test_passphrase` | + +### Command Line + +You can pass the settings with helm command line parameters. +Specify each parameter using the --set key=value argument to helm install. +For Example: + +```bash +helm install --set configMapName=ceph-csi-config --set provisioner.podSecurityPolicy.enabled=true +``` diff --git a/ceph/ceph-csi-rbd/templates/NOTES.txt b/ceph/ceph-csi-rbd/templates/NOTES.txt new file mode 100644 index 0000000..47e90f7 --- /dev/null +++ b/ceph/ceph-csi-rbd/templates/NOTES.txt @@ -0,0 +1,2 @@ +Examples on how to configure a storage class and start using the driver are here: +https://github.com/ceph/ceph-csi/tree/devel/examples/rbd diff --git a/ceph/ceph-csi-rbd/templates/_helpers.tpl b/ceph/ceph-csi-rbd/templates/_helpers.tpl new file mode 100644 index 0000000..0a2613d --- /dev/null +++ b/ceph/ceph-csi-rbd/templates/_helpers.tpl @@ -0,0 +1,90 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "ceph-csi-rbd.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "ceph-csi-rbd.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "ceph-csi-rbd.nodeplugin.fullname" -}} +{{- if .Values.nodeplugin.fullnameOverride -}} +{{- .Values.nodeplugin.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- printf "%s-%s" .Release.Name .Values.nodeplugin.name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s-%s" .Release.Name $name .Values.nodeplugin.name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "ceph-csi-rbd.provisioner.fullname" -}} +{{- if .Values.provisioner.fullnameOverride -}} +{{- .Values.provisioner.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- printf "%s-%s" .Release.Name .Values.provisioner.name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s-%s" .Release.Name $name .Values.provisioner.name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "ceph-csi-rbd.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create the name of the service account to use +*/}} +{{- define "ceph-csi-rbd.serviceAccountName.nodeplugin" -}} +{{- if .Values.serviceAccounts.nodeplugin.create -}} + {{ default (include "ceph-csi-rbd.nodeplugin.fullname" .) .Values.serviceAccounts.nodeplugin.name }} +{{- else -}} + {{ default "default" .Values.serviceAccounts.nodeplugin.name }} +{{- end -}} +{{- end -}} + +{{/* +Create the name of the service account to use +*/}} +{{- define "ceph-csi-rbd.serviceAccountName.provisioner" -}} +{{- if .Values.serviceAccounts.provisioner.create -}} + {{ default (include "ceph-csi-rbd.provisioner.fullname" .) .Values.serviceAccounts.provisioner.name }} +{{- else -}} + {{ default "default" .Values.serviceAccounts.provisioner.name }} +{{- end -}} +{{- end -}} diff --git a/ceph/ceph-csi-rbd/templates/ceph-conf.yaml b/ceph/ceph-csi-rbd/templates/ceph-conf.yaml new file mode 100644 index 0000000..a9a6f00 --- /dev/null +++ b/ceph/ceph-csi-rbd/templates/ceph-conf.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: ceph-config + namespace: {{ .Release.Namespace }} + labels: + app: {{ include "ceph-csi-rbd.name" . }} + chart: {{ include "ceph-csi-rbd.chart" . }} + component: {{ .Values.nodeplugin.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +data: + ceph.conf: | +{{ tpl .Values.cephconf . | indent 4 }} + keyring: "" diff --git a/ceph/ceph-csi-rbd/templates/csidriver-crd.yaml b/ceph/ceph-csi-rbd/templates/csidriver-crd.yaml new file mode 100644 index 0000000..92c3d91 --- /dev/null +++ b/ceph/ceph-csi-rbd/templates/csidriver-crd.yaml @@ -0,0 +1,11 @@ +{{ if semverCompare ">=1.18.0-beta.1" .Capabilities.KubeVersion.Version }} +apiVersion: storage.k8s.io/v1 +{{ else }} +apiVersion: storage.k8s.io/v1beta1 +{{ end }} +kind: CSIDriver +metadata: + name: {{ .Values.driverName }} +spec: + attachRequired: true + podInfoOnMount: false diff --git a/ceph/ceph-csi-rbd/templates/csiplugin-configmap.yaml b/ceph/ceph-csi-rbd/templates/csiplugin-configmap.yaml new file mode 100644 index 0000000..f094402 --- /dev/null +++ b/ceph/ceph-csi-rbd/templates/csiplugin-configmap.yaml @@ -0,0 +1,18 @@ +{{- if not .Values.externallyManagedConfigmap }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Values.configMapName | quote }} + namespace: {{ .Release.Namespace }} + labels: + app: {{ include "ceph-csi-rbd.name" . }} + chart: {{ include "ceph-csi-rbd.chart" . }} + component: {{ .Values.nodeplugin.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +data: + config.json: |- +{{ toJson .Values.csiConfig | indent 4 }} + cluster-mapping.json: |- +{{ toJson .Values.csiMapping | indent 4 }} +{{- end }} diff --git a/ceph/ceph-csi-rbd/templates/encryptionkms-configmap.yaml b/ceph/ceph-csi-rbd/templates/encryptionkms-configmap.yaml new file mode 100644 index 0000000..47b7d09 --- /dev/null +++ b/ceph/ceph-csi-rbd/templates/encryptionkms-configmap.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Values.kmsConfigMapName | quote }} + namespace: {{ .Release.Namespace }} + labels: + app: {{ include "ceph-csi-rbd.name" . }} + chart: {{ include "ceph-csi-rbd.chart" . }} + component: {{ .Values.nodeplugin.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +data: + config.json: |- +{{ toJson .Values.encryptionKMSConfig | indent 4 -}} diff --git a/ceph/ceph-csi-rbd/templates/nodeplugin-clusterrole.yaml b/ceph/ceph-csi-rbd/templates/nodeplugin-clusterrole.yaml new file mode 100644 index 0000000..93ec30e --- /dev/null +++ b/ceph/ceph-csi-rbd/templates/nodeplugin-clusterrole.yaml @@ -0,0 +1,34 @@ +{{- if .Values.rbac.create -}} +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "ceph-csi-rbd.nodeplugin.fullname" . }} + labels: + app: {{ include "ceph-csi-rbd.name" . }} + chart: {{ include "ceph-csi-rbd.chart" . }} + component: {{ .Values.nodeplugin.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +rules: +{{- if .Values.topology.enabled }} + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get"] +{{- end }} + # allow to read Vault Token and connection options from the Tenants namespace + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get"] + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get"] + - apiGroups: [""] + resources: ["serviceaccounts"] + verbs: ["get"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["list", "get"] +{{- end -}} diff --git a/ceph/ceph-csi-rbd/templates/nodeplugin-clusterrolebinding.yaml b/ceph/ceph-csi-rbd/templates/nodeplugin-clusterrolebinding.yaml new file mode 100644 index 0000000..bf52865 --- /dev/null +++ b/ceph/ceph-csi-rbd/templates/nodeplugin-clusterrolebinding.yaml @@ -0,0 +1,20 @@ +{{- if .Values.rbac.create -}} +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "ceph-csi-rbd.nodeplugin.fullname" . }} + labels: + app: {{ include "ceph-csi-rbd.name" . }} + chart: {{ include "ceph-csi-rbd.chart" . }} + component: {{ .Values.nodeplugin.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +subjects: + - kind: ServiceAccount + name: {{ include "ceph-csi-rbd.serviceAccountName.nodeplugin" . }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ include "ceph-csi-rbd.nodeplugin.fullname" . }} + apiGroup: rbac.authorization.k8s.io +{{- end -}} diff --git a/ceph/ceph-csi-rbd/templates/nodeplugin-daemonset.yaml b/ceph/ceph-csi-rbd/templates/nodeplugin-daemonset.yaml new file mode 100644 index 0000000..422ad08 --- /dev/null +++ b/ceph/ceph-csi-rbd/templates/nodeplugin-daemonset.yaml @@ -0,0 +1,227 @@ +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: {{ include "ceph-csi-rbd.nodeplugin.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + app: {{ include "ceph-csi-rbd.name" . }} + chart: {{ include "ceph-csi-rbd.chart" . }} + component: {{ .Values.nodeplugin.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + selector: + matchLabels: + app: {{ include "ceph-csi-rbd.name" . }} + component: {{ .Values.nodeplugin.name }} + release: {{ .Release.Name }} + updateStrategy: + type: {{ .Values.nodeplugin.updateStrategy }} + template: + metadata: + labels: + app: {{ include "ceph-csi-rbd.name" . }} + chart: {{ include "ceph-csi-rbd.chart" . }} + component: {{ .Values.nodeplugin.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + spec: + serviceAccountName: {{ include "ceph-csi-rbd.serviceAccountName.nodeplugin" . }} + hostNetwork: true + hostPID: true +{{- if .Values.nodeplugin.priorityClassName }} + priorityClassName: {{ .Values.nodeplugin.priorityClassName }} +{{- end }} + # to use e.g. Rook orchestrated cluster, and mons' FQDN is + # resolved through k8s service, set dns policy to cluster first + dnsPolicy: ClusterFirstWithHostNet + containers: + - name: driver-registrar + # This is necessary only for systems with SELinux, where + # non-privileged sidecar containers cannot access unix domain socket + # created by privileged CSI driver container. + securityContext: + privileged: true + image: "{{ .Values.nodeplugin.registrar.image.repository }}:{{ .Values.nodeplugin.registrar.image.tag }}" + imagePullPolicy: {{ .Values.nodeplugin.registrar.image.pullPolicy }} + args: + - "--v={{ .Values.logLevel }}" + - "--csi-address=/csi/{{ .Values.pluginSocketFile }}" + - "--kubelet-registration-path={{ .Values.kubeletDir }}/plugins/{{ .Values.driverName }}/{{ .Values.pluginSocketFile }}" + env: + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: registration-dir + mountPath: /registration + resources: +{{ toYaml .Values.nodeplugin.registrar.resources | indent 12 }} + - name: csi-rbdplugin + image: "{{ .Values.nodeplugin.plugin.image.repository }}:{{ .Values.nodeplugin.plugin.image.tag }}" + imagePullPolicy: {{ .Values.nodeplugin.plugin.image.pullPolicy }} + args: + - "--nodeid=$(NODE_ID)" + - "--pluginpath={{ .Values.kubeletDir }}/plugins" + - "--stagingpath={{ .Values.kubeletDir }}/plugins/kubernetes.io/csi/pv/" + - "--type=rbd" + - "--nodeserver=true" + - "--pidlimit=-1" + - "--endpoint=$(CSI_ENDPOINT)" + - "--csi-addons-endpoint=$(CSI_ADDONS_ENDPOINT)" + - "--v={{ .Values.logLevel }}" + - "--drivername=$(DRIVER_NAME)" +{{- if .Values.topology.enabled }} + - "--domainlabels={{ .Values.topology.domainLabels | join "," }}" +{{- end }} +{{- if .Values.nodeplugin.profiling.enabled }} + - "--enableprofiling={{ .Values.nodeplugin.profiling.enabled }}" +{{- end }} + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: DRIVER_NAME + value: {{ .Values.driverName }} + - name: NODE_ID + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: CSI_ENDPOINT + value: "unix:///csi/{{ .Values.pluginSocketFile }}" + - name: CSI_ADDONS_ENDPOINT + value: "unix:///csi/csi-addons.sock" + securityContext: + privileged: true + capabilities: + add: ["SYS_ADMIN"] + allowPrivilegeEscalation: true + volumeMounts: + - name: socket-dir + mountPath: /csi + - mountPath: /dev + name: host-dev + - mountPath: /run/mount + name: host-mount + - mountPath: /sys + name: host-sys + - mountPath: /etc/selinux + name: etc-selinux + readOnly: true + - mountPath: /lib/modules + name: lib-modules + readOnly: true + - name: ceph-csi-config + mountPath: /etc/ceph-csi-config/ + - name: ceph-config + mountPath: /etc/ceph/ + - name: ceph-csi-encryption-kms-config + mountPath: /etc/ceph-csi-encryption-kms-config/ + - name: plugin-dir + mountPath: {{ .Values.kubeletDir }}/plugins + mountPropagation: "Bidirectional" + - name: mountpoint-dir + mountPath: {{ .Values.kubeletDir }}/pods + mountPropagation: "Bidirectional" + - name: keys-tmp-dir + mountPath: /tmp/csi/keys + - name: ceph-logdir + mountPath: /var/log/ceph + resources: +{{ toYaml .Values.nodeplugin.plugin.resources | indent 12 }} +{{- if .Values.nodeplugin.httpMetrics.enabled }} + - name: liveness-prometheus + securityContext: + privileged: true + image: "{{ .Values.nodeplugin.plugin.image.repository }}:{{ .Values.nodeplugin.plugin.image.tag }}" + imagePullPolicy: {{ .Values.nodeplugin.plugin.image.pullPolicy }} + args: + - "--type=liveness" + - "--endpoint=$(CSI_ENDPOINT)" + - "--metricsport={{ .Values.nodeplugin.httpMetrics.containerPort }}" + - "--metricspath=/metrics" + - "--polltime=60s" + - "--timeout=3s" + env: + - name: CSI_ENDPOINT + value: "unix:///csi/{{ .Values.pluginSocketFile }}" + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + volumeMounts: + - name: socket-dir + mountPath: /csi + resources: +{{ toYaml .Values.nodeplugin.plugin.resources | indent 12 }} +{{- end }} + volumes: + - name: socket-dir + hostPath: + path: "{{ .Values.kubeletDir }}/plugins/{{ .Values.driverName }}" + type: DirectoryOrCreate + - name: registration-dir + hostPath: + path: {{ .Values.kubeletDir }}/plugins_registry + type: Directory + - name: plugin-dir + hostPath: + path: {{ .Values.kubeletDir }}/plugins + type: Directory + - name: mountpoint-dir + hostPath: + path: {{ .Values.kubeletDir }}/pods + type: DirectoryOrCreate + - name: ceph-logdir + hostPath: + path: {{ .Values.cephLogDirHostPath }} + type: DirectoryOrCreate + - name: host-dev + hostPath: + path: /dev + - name: host-mount + hostPath: + path: /run/mount + - name: host-sys + hostPath: + path: /sys + - name: etc-selinux + hostPath: + path: /etc/selinux + - name: lib-modules + hostPath: + path: /lib/modules + - name: ceph-config + configMap: + name: ceph-config + - name: ceph-csi-config + configMap: + name: {{ .Values.configMapName | quote }} +{{- if .Values.configMapKey }} + items: + - key: {{ .Values.configMapKey | quote }} + path: config.json +{{- end }} + - name: ceph-csi-encryption-kms-config + configMap: + name: {{ .Values.kmsConfigMapName | quote }} + - name: keys-tmp-dir + emptyDir: { + medium: "Memory" + } +{{- if .Values.nodeplugin.affinity }} + affinity: +{{ toYaml .Values.nodeplugin.affinity | indent 8 -}} +{{- end -}} +{{- if .Values.nodeplugin.nodeSelector }} + nodeSelector: +{{ toYaml .Values.nodeplugin.nodeSelector | indent 8 -}} +{{- end -}} +{{- if .Values.nodeplugin.tolerations }} + tolerations: +{{ toYaml .Values.nodeplugin.tolerations | indent 8 -}} +{{- end -}} diff --git a/ceph/ceph-csi-rbd/templates/nodeplugin-http-service.yaml b/ceph/ceph-csi-rbd/templates/nodeplugin-http-service.yaml new file mode 100644 index 0000000..68a3942 --- /dev/null +++ b/ceph/ceph-csi-rbd/templates/nodeplugin-http-service.yaml @@ -0,0 +1,41 @@ +{{- if .Values.nodeplugin.httpMetrics.service.enabled -}} +apiVersion: v1 +kind: Service +metadata: +{{- if .Values.nodeplugin.httpMetrics.service.annotations }} + annotations: +{{ toYaml .Values.nodeplugin.httpMetrics.service.annotations | indent 4 }} +{{- end }} + name: {{ include "ceph-csi-rbd.nodeplugin.fullname" . }}-http-metrics + namespace: {{ .Release.Namespace }} + labels: + app: {{ include "ceph-csi-rbd.fullname" . }} + chart: {{ include "ceph-csi-rbd.chart" . }} + component: {{ .Values.nodeplugin.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: +{{- if .Values.nodeplugin.httpMetrics.service.clusterIP }} + clusterIP: "{{ .Values.nodeplugin.httpMetrics.service.clusterIP }}" +{{- end }} +{{- if .Values.nodeplugin.httpMetrics.service.externalIPs }} + externalIPs: +{{ toYaml .Values.nodeplugin.httpMetrics.service.externalIPs | indent 4 }} +{{- end }} +{{- if .Values.nodeplugin.httpMetrics.service.loadBalancerIP }} + loadBalancerIP: "{{ .Values.nodeplugin.httpMetrics.service.loadBalancerIP }}" +{{- end }} +{{- if .Values.nodeplugin.httpMetrics.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: +{{ toYaml .Values.nodeplugin.httpMetrics.service.loadBalancerSourceRanges | indent 4 }} +{{- end }} + ports: + - name: http-metrics + port: {{ .Values.nodeplugin.httpMetrics.service.servicePort }} + targetPort: {{ .Values.nodeplugin.httpMetrics.containerPort }} + selector: + app: {{ include "ceph-csi-rbd.name" . }} + component: {{ .Values.nodeplugin.name }} + release: {{ .Release.Name }} + type: "{{ .Values.nodeplugin.httpMetrics.service.type }}" +{{- end -}} diff --git a/ceph/ceph-csi-rbd/templates/nodeplugin-psp.yaml b/ceph/ceph-csi-rbd/templates/nodeplugin-psp.yaml new file mode 100644 index 0000000..e4c01cd --- /dev/null +++ b/ceph/ceph-csi-rbd/templates/nodeplugin-psp.yaml @@ -0,0 +1,48 @@ +{{- if .Values.nodeplugin.podSecurityPolicy.enabled -}} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ include "ceph-csi-rbd.nodeplugin.fullname" . }} + labels: + app: {{ include "ceph-csi-rbd.name" . }} + chart: {{ include "ceph-csi-rbd.chart" . }} + component: {{ .Values.nodeplugin.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + allowPrivilegeEscalation: true + allowedCapabilities: + - 'SYS_ADMIN' + fsGroup: + rule: RunAsAny + privileged: true + hostNetwork: true + hostPID: true + runAsUser: + rule: RunAsAny + seLinux: + rule: RunAsAny + supplementalGroups: + rule: RunAsAny + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'hostPath' + allowedHostPaths: + - pathPrefix: '/dev' + readOnly: false + - pathPrefix: '/run/mount' + readOnly: false + - pathPrefix: '/sys' + readOnly: false + - pathPrefix: '/etc/selinux' + readOnly: true + - pathPrefix: '/lib/modules' + readOnly: true + - pathPrefix: '{{ .Values.cephLogDirHostPath }}' + readOnly: false + - pathPrefix: '{{ .Values.kubeletDir }}' + readOnly: false +{{- end }} diff --git a/ceph/ceph-csi-rbd/templates/nodeplugin-role.yaml b/ceph/ceph-csi-rbd/templates/nodeplugin-role.yaml new file mode 100644 index 0000000..d9d5a0e --- /dev/null +++ b/ceph/ceph-csi-rbd/templates/nodeplugin-role.yaml @@ -0,0 +1,18 @@ +{{- if and .Values.rbac.create .Values.nodeplugin.podSecurityPolicy.enabled -}} +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "ceph-csi-rbd.nodeplugin.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + app: {{ include "ceph-csi-rbd.name" . }} + chart: {{ include "ceph-csi-rbd.chart" . }} + component: {{ .Values.nodeplugin.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +rules: + - apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: ['{{ include "ceph-csi-rbd.nodeplugin.fullname" . }}'] +{{- end -}} diff --git a/ceph/ceph-csi-rbd/templates/nodeplugin-rolebinding.yaml b/ceph/ceph-csi-rbd/templates/nodeplugin-rolebinding.yaml new file mode 100644 index 0000000..f4cce98 --- /dev/null +++ b/ceph/ceph-csi-rbd/templates/nodeplugin-rolebinding.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.rbac.create .Values.nodeplugin.podSecurityPolicy.enabled -}} +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "ceph-csi-rbd.nodeplugin.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + app: {{ include "ceph-csi-rbd.name" . }} + chart: {{ include "ceph-csi-rbd.chart" . }} + component: {{ .Values.nodeplugin.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +subjects: + - kind: ServiceAccount + name: {{ include "ceph-csi-rbd.serviceAccountName.nodeplugin" . }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: Role + name: {{ include "ceph-csi-rbd.nodeplugin.fullname" . }} + apiGroup: rbac.authorization.k8s.io +{{- end -}} diff --git a/ceph/ceph-csi-rbd/templates/nodeplugin-serviceaccount.yaml b/ceph/ceph-csi-rbd/templates/nodeplugin-serviceaccount.yaml new file mode 100644 index 0000000..36e1ee7 --- /dev/null +++ b/ceph/ceph-csi-rbd/templates/nodeplugin-serviceaccount.yaml @@ -0,0 +1,13 @@ +{{- if .Values.serviceAccounts.nodeplugin.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "ceph-csi-rbd.serviceAccountName.nodeplugin" . }} + namespace: {{ .Release.Namespace }} + labels: + app: {{ include "ceph-csi-rbd.name" . }} + chart: {{ include "ceph-csi-rbd.chart" . }} + component: {{ .Values.nodeplugin.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +{{- end -}} diff --git a/ceph/ceph-csi-rbd/templates/provisioner-clusterrole.yaml b/ceph/ceph-csi-rbd/templates/provisioner-clusterrole.yaml new file mode 100644 index 0000000..0327057 --- /dev/null +++ b/ceph/ceph-csi-rbd/templates/provisioner-clusterrole.yaml @@ -0,0 +1,71 @@ +{{- if .Values.rbac.create -}} +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "ceph-csi-rbd.provisioner.fullname" . }} + labels: + app: {{ include "ceph-csi-rbd.name" . }} + chart: {{ include "ceph-csi-rbd.chart" . }} + component: {{ .Values.provisioner.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "update", "delete", "patch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: [""] + resources: ["endpoints"] + verbs: ["get", "create", "update"] +{{- if .Values.provisioner.attacher.enabled }} + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments/status"] + verbs: ["patch"] +{{- end }} + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "delete"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents/status"] + verbs: ["update"] + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get"] + - apiGroups: [""] + resources: ["serviceaccounts"] + verbs: ["get"] +{{- if .Values.provisioner.resizer.enabled }} + - apiGroups: [""] + resources: ["persistentvolumeclaims/status"] + verbs: ["update", "patch"] +{{- end }} +{{- if .Values.topology.enabled }} + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list","watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] + verbs: ["get", "list", "watch"] +{{- end }} + +{{- end -}} diff --git a/ceph/ceph-csi-rbd/templates/provisioner-clusterrolebinding.yaml b/ceph/ceph-csi-rbd/templates/provisioner-clusterrolebinding.yaml new file mode 100644 index 0000000..5a08610 --- /dev/null +++ b/ceph/ceph-csi-rbd/templates/provisioner-clusterrolebinding.yaml @@ -0,0 +1,20 @@ +{{- if .Values.rbac.create -}} +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "ceph-csi-rbd.provisioner.fullname" . }} + labels: + app: {{ include "ceph-csi-rbd.name" . }} + chart: {{ include "ceph-csi-rbd.chart" . }} + component: {{ .Values.provisioner.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +subjects: + - kind: ServiceAccount + name: {{ include "ceph-csi-rbd.serviceAccountName.provisioner" . }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ include "ceph-csi-rbd.provisioner.fullname" . }} + apiGroup: rbac.authorization.k8s.io +{{- end -}} diff --git a/ceph/ceph-csi-rbd/templates/provisioner-deployment.yaml b/ceph/ceph-csi-rbd/templates/provisioner-deployment.yaml new file mode 100644 index 0000000..86a04db --- /dev/null +++ b/ceph/ceph-csi-rbd/templates/provisioner-deployment.yaml @@ -0,0 +1,281 @@ +kind: Deployment +apiVersion: apps/v1 +metadata: + name: {{ include "ceph-csi-rbd.provisioner.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + app: {{ include "ceph-csi-rbd.name" . }} + chart: {{ include "ceph-csi-rbd.chart" . }} + component: {{ .Values.provisioner.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + replicas: {{ .Values.provisioner.replicaCount }} + strategy: + type: {{ .Values.provisioner.strategy.type }} +{{- if eq .Values.provisioner.strategy.type "RollingUpdate" }} + rollingUpdate: + maxUnavailable: {{ .Values.provisioner.strategy.rollingUpdate.maxUnavailable }} +{{- end }} + selector: + matchLabels: + app: {{ include "ceph-csi-rbd.name" . }} + component: {{ .Values.provisioner.name }} + release: {{ .Release.Name }} + template: + metadata: + labels: + app: {{ include "ceph-csi-rbd.name" . }} + chart: {{ include "ceph-csi-rbd.chart" . }} + component: {{ .Values.provisioner.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + spec: +{{- if gt (int .Values.provisioner.replicaCount) 1 }} + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app + operator: In + values: + - {{ include "ceph-csi-rbd.name" . }} + - key: component + operator: In + values: + - {{ .Values.provisioner.name }} + topologyKey: "kubernetes.io/hostname" +{{- end }} + serviceAccountName: {{ include "ceph-csi-rbd.serviceAccountName.provisioner" . }} +{{- if .Values.provisioner.priorityClassName }} + priorityClassName: {{ .Values.provisioner.priorityClassName }} +{{- end }} + containers: + - name: csi-provisioner + image: "{{ .Values.provisioner.provisioner.image.repository }}:{{ .Values.provisioner.provisioner.image.tag }}" + imagePullPolicy: {{ .Values.provisioner.provisioner.image.pullPolicy }} + args: + - "--csi-address=$(ADDRESS)" + - "--v={{ .Values.logLevel }}" + - "--timeout={{ .Values.provisioner.timeout }}" + - "--leader-election=true" + - "--retry-interval-start=500ms" + - "--default-fstype={{ .Values.provisioner.defaultFSType }}" + - "--extra-create-metadata=true" +{{- if .Values.topology.enabled }} + - "--feature-gates=Topology=true" +{{- end }} + env: + - name: ADDRESS + value: "unix:///csi/{{ .Values.provisionerSocketFile }}" + volumeMounts: + - name: socket-dir + mountPath: /csi + resources: +{{ toYaml .Values.provisioner.provisioner.resources | indent 12 }} +{{- if .Values.provisioner.resizer.enabled }} + - name: csi-resizer + image: "{{ .Values.provisioner.resizer.image.repository }}:{{ .Values.provisioner.resizer.image.tag }}" + imagePullPolicy: {{ .Values.provisioner.resizer.image.pullPolicy }} + args: + - "--v={{ .Values.logLevel }}" + - "--csi-address=$(ADDRESS)" + - "--timeout={{ .Values.provisioner.timeout }}" + - "--leader-election" + - "--retry-interval-start=500ms" + - "--handle-volume-inuse-error=false" + env: + - name: ADDRESS + value: "unix:///csi/{{ .Values.provisionerSocketFile }}" + volumeMounts: + - name: socket-dir + mountPath: /csi + resources: +{{ toYaml .Values.provisioner.resizer.resources | indent 12 }} +{{- end }} + - name: csi-snapshotter + image: {{ .Values.provisioner.snapshotter.image.repository }}:{{ .Values.provisioner.snapshotter.image.tag }} + imagePullPolicy: {{ .Values.provisioner.snapshotter.image.pullPolicy }} + args: + - "--csi-address=$(ADDRESS)" + - "--v={{ .Values.logLevel }}" + - "--timeout={{ .Values.provisioner.timeout }}" + - "--leader-election=true" + env: + - name: ADDRESS + value: "unix:///csi/{{ .Values.provisionerSocketFile }}" + volumeMounts: + - name: socket-dir + mountPath: /csi + resources: +{{ toYaml .Values.provisioner.snapshotter.resources | indent 12 }} +{{- if .Values.provisioner.attacher.enabled }} + - name: csi-attacher + image: "{{ .Values.provisioner.attacher.image.repository }}:{{ .Values.provisioner.attacher.image.tag }}" + imagePullPolicy: {{ .Values.provisioner.attacher.image.pullPolicy }} + args: + - "--v={{ .Values.logLevel }}" + - "--csi-address=$(ADDRESS)" + - "--leader-election=true" + - "--retry-interval-start=500ms" + env: + - name: ADDRESS + value: "unix:///csi/{{ .Values.provisionerSocketFile }}" + volumeMounts: + - name: socket-dir + mountPath: /csi + resources: +{{ toYaml .Values.provisioner.attacher.resources | indent 12 }} +{{- end }} + - name: csi-rbdplugin + image: "{{ .Values.nodeplugin.plugin.image.repository }}:{{ .Values.nodeplugin.plugin.image.tag }}" + imagePullPolicy: {{ .Values.nodeplugin.plugin.image.pullPolicy }} + args: + - "--nodeid=$(NODE_ID)" + - "--type=rbd" + - "--controllerserver=true" + - "--pidlimit=-1" + - "--endpoint=$(CSI_ENDPOINT)" + - "--csi-addons-endpoint=$(CSI_ADDONS_ENDPOINT)" + - "--v={{ .Values.logLevel }}" + - "--drivername=$(DRIVER_NAME)" + - "--rbdhardmaxclonedepth={{ .Values.provisioner.hardMaxCloneDepth }}" + - "--rbdsoftmaxclonedepth={{ .Values.provisioner.softMaxCloneDepth }}" + - "--maxsnapshotsonimage={{ .Values.provisioner.maxSnapshotsOnImage }}" + - "--minsnapshotsonimage={{ .Values.provisioner.minSnapshotsOnImage }}" + {{- if .Values.provisioner.skipForceFlatten }} + - "--skipforceflatten={{ .Values.provisioner.skipForceFlatten }}" + {{- end }} + {{- if .Values.provisioner.profiling.enabled }} + - "--enableprofiling={{ .Values.provisioner.profiling.enabled }}" + {{- end }} + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: DRIVER_NAME + value: {{ .Values.driverName }} + - name: NODE_ID + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: CSI_ENDPOINT + value: "unix:///csi/{{ .Values.provisionerSocketFile }}" + - name: CSI_ADDONS_ENDPOINT + value: "unix:///csi/csi-addons.sock" + volumeMounts: + - name: socket-dir + mountPath: /csi + - mountPath: /dev + name: host-dev + - mountPath: /sys + name: host-sys + - mountPath: /lib/modules + name: lib-modules + readOnly: true + - name: ceph-csi-config + mountPath: /etc/ceph-csi-config/ + - name: ceph-config + mountPath: /etc/ceph/ + - name: ceph-csi-encryption-kms-config + mountPath: /etc/ceph-csi-encryption-kms-config/ + - name: keys-tmp-dir + mountPath: /tmp/csi/keys + resources: +{{ toYaml .Values.nodeplugin.plugin.resources | indent 12 }} +{{- if .Values.provisioner.deployController }} + - name: csi-rbdplugin-controller + image: "{{ .Values.nodeplugin.plugin.image.repository }}:{{ .Values.nodeplugin.plugin.image.tag }}" + imagePullPolicy: {{ .Values.nodeplugin.plugin.image.pullPolicy }} + args: + - "--type=controller" + - "--v={{ .Values.logLevel }}" + - "--drivername=$(DRIVER_NAME)" + - "--drivernamespace=$(DRIVER_NAMESPACE)" + env: + - name: DRIVER_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: DRIVER_NAME + value: {{ .Values.driverName }} + volumeMounts: + - name: ceph-csi-config + mountPath: /etc/ceph-csi-config/ + - name: keys-tmp-dir + mountPath: /tmp/csi/keys + - name: ceph-config + mountPath: /etc/ceph/ + resources: +{{ toYaml .Values.nodeplugin.plugin.resources | indent 12 }} +{{- end }} +{{- if .Values.provisioner.httpMetrics.enabled }} + - name: liveness-prometheus + image: "{{ .Values.nodeplugin.plugin.image.repository }}:{{ .Values.nodeplugin.plugin.image.tag }}" + imagePullPolicy: {{ .Values.nodeplugin.plugin.image.pullPolicy }} + args: + - "--type=liveness" + - "--endpoint=$(CSI_ENDPOINT)" + - "--metricsport={{ .Values.provisioner.httpMetrics.containerPort }}" + - "--metricspath=/metrics" + - "--polltime=60s" + - "--timeout=3s" + env: + - name: CSI_ENDPOINT + value: "unix:///csi/{{ .Values.provisionerSocketFile }}" + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + volumeMounts: + - name: socket-dir + mountPath: /csi + resources: +{{ toYaml .Values.nodeplugin.plugin.resources | indent 12 }} +{{- end }} + volumes: + - name: socket-dir + emptyDir: { + medium: "Memory" + } + - name: host-dev + hostPath: + path: /dev + - name: host-sys + hostPath: + path: /sys + - name: lib-modules + hostPath: + path: /lib/modules + - name: ceph-config + configMap: + name: ceph-config + - name: ceph-csi-config + configMap: + name: {{ .Values.configMapName | quote }} +{{- if .Values.configMapKey }} + items: + - key: {{ .Values.configMapKey | quote }} + path: config.json +{{- end }} + - name: ceph-csi-encryption-kms-config + configMap: + name: {{ .Values.kmsConfigMapName | quote }} + - name: keys-tmp-dir + emptyDir: { + medium: "Memory" + } +{{- if .Values.provisioner.affinity }} + affinity: +{{ toYaml .Values.provisioner.affinity | indent 8 -}} +{{- end -}} +{{- if .Values.provisioner.nodeSelector }} + nodeSelector: +{{ toYaml .Values.provisioner.nodeSelector | indent 8 -}} +{{- end -}} +{{- if .Values.provisioner.tolerations }} + tolerations: +{{ toYaml .Values.provisioner.tolerations | indent 8 -}} +{{- end -}} diff --git a/ceph/ceph-csi-rbd/templates/provisioner-http-service.yaml b/ceph/ceph-csi-rbd/templates/provisioner-http-service.yaml new file mode 100644 index 0000000..0d4c925 --- /dev/null +++ b/ceph/ceph-csi-rbd/templates/provisioner-http-service.yaml @@ -0,0 +1,41 @@ +{{- if .Values.provisioner.httpMetrics.service.enabled -}} +apiVersion: v1 +kind: Service +metadata: +{{- if .Values.provisioner.httpMetrics.service.annotations }} + annotations: +{{ toYaml .Values.provisioner.httpMetrics.service.annotations | indent 4 }} +{{- end }} + name: {{ include "ceph-csi-rbd.provisioner.fullname" . }}-http-metrics + namespace: {{ .Release.Namespace }} + labels: + app: {{ include "ceph-csi-rbd.fullname" . }} + chart: {{ include "ceph-csi-rbd.chart" . }} + component: {{ .Values.provisioner.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: +{{- if .Values.provisioner.httpMetrics.service.clusterIP }} + clusterIP: "{{ .Values.provisioner.httpMetrics.service.clusterIP }}" +{{- end }} +{{- if .Values.provisioner.httpMetrics.service.externalIPs }} + externalIPs: +{{ toYaml .Values.provisioner.httpMetrics.service.externalIPs | indent 4 }} +{{- end }} +{{- if .Values.provisioner.httpMetrics.service.loadBalancerIP }} + loadBalancerIP: "{{ .Values.provisioner.httpMetrics.service.loadBalancerIP }}" +{{- end }} +{{- if .Values.provisioner.httpMetrics.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: +{{ toYaml .Values.provisioner.httpMetrics.service.loadBalancerSourceRanges | indent 4 }} +{{- end }} + ports: + - name: http-metrics + port: {{ .Values.provisioner.httpMetrics.service.servicePort }} + targetPort: {{ .Values.provisioner.httpMetrics.containerPort }} + selector: + app: {{ include "ceph-csi-rbd.name" . }} + component: {{ .Values.provisioner.name }} + release: {{ .Release.Name }} + type: "{{ .Values.provisioner.httpMetrics.service.type }}" +{{- end -}} diff --git a/ceph/ceph-csi-rbd/templates/provisioner-psp.yaml b/ceph/ceph-csi-rbd/templates/provisioner-psp.yaml new file mode 100644 index 0000000..111226e --- /dev/null +++ b/ceph/ceph-csi-rbd/templates/provisioner-psp.yaml @@ -0,0 +1,34 @@ +{{- if .Values.provisioner.podSecurityPolicy.enabled -}} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ include "ceph-csi-rbd.provisioner.fullname" . }} + labels: + app: {{ include "ceph-csi-rbd.name" . }} + chart: {{ include "ceph-csi-rbd.chart" . }} + component: {{ .Values.provisioner.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + fsGroup: + rule: RunAsAny + runAsUser: + rule: RunAsAny + seLinux: + rule: RunAsAny + supplementalGroups: + rule: RunAsAny + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'hostPath' + allowedHostPaths: + - pathPrefix: '/dev' + readOnly: false + - pathPrefix: '/sys' + readOnly: false + - pathPrefix: '/lib/modules' + readOnly: true +{{- end }} diff --git a/ceph/ceph-csi-rbd/templates/provisioner-role.yaml b/ceph/ceph-csi-rbd/templates/provisioner-role.yaml new file mode 100644 index 0000000..ae16fde --- /dev/null +++ b/ceph/ceph-csi-rbd/templates/provisioner-role.yaml @@ -0,0 +1,26 @@ +{{- if .Values.rbac.create -}} +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "ceph-csi-rbd.provisioner.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + app: {{ include "ceph-csi-rbd.name" . }} + chart: {{ include "ceph-csi-rbd.chart" . }} + component: {{ .Values.provisioner.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +rules: + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "watch", "create","update", "delete"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] +{{- if .Values.provisioner.podSecurityPolicy.enabled }} + - apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: ['{{ include "ceph-csi-rbd.provisioner.fullname" . }}'] +{{- end -}} +{{- end -}} diff --git a/ceph/ceph-csi-rbd/templates/provisioner-rolebinding.yaml b/ceph/ceph-csi-rbd/templates/provisioner-rolebinding.yaml new file mode 100644 index 0000000..23fa39f --- /dev/null +++ b/ceph/ceph-csi-rbd/templates/provisioner-rolebinding.yaml @@ -0,0 +1,21 @@ +{{- if .Values.rbac.create -}} +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "ceph-csi-rbd.provisioner.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + app: {{ include "ceph-csi-rbd.name" . }} + chart: {{ include "ceph-csi-rbd.chart" . }} + component: {{ .Values.provisioner.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +subjects: + - kind: ServiceAccount + name: {{ include "ceph-csi-rbd.serviceAccountName.provisioner" . }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: Role + name: {{ include "ceph-csi-rbd.provisioner.fullname" . }} + apiGroup: rbac.authorization.k8s.io +{{- end -}} diff --git a/ceph/ceph-csi-rbd/templates/provisioner-serviceaccount.yaml b/ceph/ceph-csi-rbd/templates/provisioner-serviceaccount.yaml new file mode 100644 index 0000000..893b43a --- /dev/null +++ b/ceph/ceph-csi-rbd/templates/provisioner-serviceaccount.yaml @@ -0,0 +1,13 @@ +{{- if .Values.serviceAccounts.provisioner.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "ceph-csi-rbd.serviceAccountName.provisioner" . }} + namespace: {{ .Release.Namespace }} + labels: + app: {{ include "ceph-csi-rbd.name" . }} + chart: {{ include "ceph-csi-rbd.chart" . }} + component: {{ .Values.provisioner.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +{{- end -}} diff --git a/ceph/ceph-csi-rbd/templates/secret.yaml b/ceph/ceph-csi-rbd/templates/secret.yaml new file mode 100644 index 0000000..1553cee --- /dev/null +++ b/ceph/ceph-csi-rbd/templates/secret.yaml @@ -0,0 +1,17 @@ +{{- if .Values.secret.create -}} +apiVersion: v1 +kind: Secret +metadata: + name: {{ .Values.secret.name }} + namespace: {{ .Release.Namespace }} + labels: + app: {{ include "ceph-csi-rbd.name" . }} + chart: {{ include "ceph-csi-rbd.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +stringData: + userID: {{ .Values.secret.userID }} + userKey: {{ .Values.secret.userKey }} + + encryptionPassphrase: {{ .Values.secret.encryptionPassphrase }} +{{- end -}} diff --git a/ceph/ceph-csi-rbd/templates/storageclass.yaml b/ceph/ceph-csi-rbd/templates/storageclass.yaml new file mode 100644 index 0000000..e4da516 --- /dev/null +++ b/ceph/ceph-csi-rbd/templates/storageclass.yaml @@ -0,0 +1,83 @@ +{{- if .Values.storageClass.create -}} +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: {{ .Values.storageClass.name }} + namespace: {{ .Release.Namespace }} +{{- if .Values.storageClass.annotations }} + annotations: +{{ toYaml .Values.storageClass.annotations | indent 4 }} +{{- end }} + labels: + app: {{ include "ceph-csi-rbd.name" . }} + chart: {{ include "ceph-csi-rbd.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +provisioner: {{ .Values.driverName }} +parameters: + clusterID: {{ .Values.storageClass.clusterID }} + pool: {{ .Values.storageClass.pool }} + imageFeatures: {{ .Values.storageClass.imageFeatures }} + thickProvision: {{ .Values.storageClass.thickProvision | quote}} +{{- if .Values.storageClass.tryOtherMounters }} + tryOtherMounters: {{ .Values.storageClass.tryOtherMounters | quote}} +{{- end }} +{{- if .Values.storageClass.mounter }} + mounter: {{ .Values.storageClass.mounter }} +{{- end }} +{{- if .Values.storageClass.cephLogDir }} + cephLogDir: {{ .Values.storageClass.cephLogDir }} +{{- end }} +{{- if .Values.storageClass.cephLogStrategy }} + cephLogStrategy: {{ .Values.storageClass.cephLogStrategy }} +{{- end }} +{{- if .Values.storageClass.dataPool }} + dataPool: {{ .Values.storageClass.dataPool }} +{{- end }} +{{- if .Values.storageClass.volumeNamePrefix }} + volumeNamePrefix: "{{ .Values.storageClass.volumeNamePrefix }}" +{{- end }} +{{- if .Values.storageClass.encrypted }} + encrypted: "{{ .Values.storageClass.encrypted }}" +{{- end }} +{{- if .Values.storageClass.encryptionKMSID }} + encryptionKMSID: {{ .Values.storageClass.encryptionKMSID }} +{{- end }} +{{- if .Values.storageClass.topologyConstrainedPools }} + topologyConstrainedPools: + {{ toYaml .Values.storageClass.topologyConstrainedPools | indent 4 -}} +{{- end }} +{{- if .Values.storageClass.mapOptions }} + mapOptions: {{ .Values.storageClass.mapOptions }} +{{- end }} +{{- if .Values.storageClass.unmapOptions }} + unmapOptions: {{ .Values.storageClass.unmapOptions }} +{{- end }} + csi.storage.k8s.io/provisioner-secret-name: {{ .Values.storageClass.provisionerSecret }} +{{- if .Values.storageClass.provisionerSecretNamespace }} + csi.storage.k8s.io/provisioner-secret-namespace: {{ .Values.storageClass.provisionerSecretNamespace }} +{{ else }} + csi.storage.k8s.io/provisioner-secret-namespace: {{ .Release.Namespace }} +{{- end }} + csi.storage.k8s.io/controller-expand-secret-name: {{ .Values.storageClass.controllerExpandSecret }} +{{- if .Values.storageClass.controllerExpandSecretNamespace }} + csi.storage.k8s.io/controller-expand-secret-namespace: {{ .Values.storageClass.controllerExpandSecretNamespace }} +{{ else }} + csi.storage.k8s.io/controller-expand-secret-namespace: {{ .Release.Namespace }} +{{- end }} + csi.storage.k8s.io/node-stage-secret-name: {{ .Values.storageClass.nodeStageSecret }} +{{- if .Values.storageClass.nodeStageSecretNamespace }} + csi.storage.k8s.io/node-stage-secret-namespace: {{ .Values.storageClass.nodeStageSecretNamespace }} +{{ else }} + csi.storage.k8s.io/node-stage-secret-namespace: {{ .Release.Namespace }} +{{- end }} + csi.storage.k8s.io/fstype: {{ .Values.storageClass.fstype }} +reclaimPolicy: {{ .Values.storageClass.reclaimPolicy }} +allowVolumeExpansion: {{ .Values.storageClass.allowVolumeExpansion }} +{{- if .Values.storageClass.mountOptions }} +mountOptions: + {{- range .Values.storageClass.mountOptions }} + - {{ . }} + {{- end }} +{{- end }} +{{- end -}} diff --git a/ceph/ceph-csi-rbd/values.yaml b/ceph/ceph-csi-rbd/values.yaml new file mode 100644 index 0000000..e37e17d --- /dev/null +++ b/ceph/ceph-csi-rbd/values.yaml @@ -0,0 +1,459 @@ +--- +rbac: + # Specifies whether RBAC resources should be created + create: true + +serviceAccounts: + nodeplugin: + # Specifies whether a ServiceAccount should be created + create: true + # The name of the ServiceAccount to use. + # If not set and create is true, a name is generated using the fullname + name: + provisioner: + # Specifies whether a ServiceAccount should be created + create: true + # The name of the ServiceAccount to use. + # If not set and create is true, a name is generated using the fullname + name: + +# Configuration for the CSI to connect to the cluster +# Ref: https://github.com/ceph/ceph-csi/blob/devel/examples/README.md +# Example: +# csiConfig: +# - clusterID: "<cluster-id>" +# monitors: +# - "<MONValue1>" +# - "<MONValue2>" +csiConfig: [] + +# Configuration details of clusterID,PoolID and FscID mapping +# csiMapping: +# - clusterIDMapping: +# clusterID on site1: clusterID on site2 +# RBDPoolIDMapping: +# - poolID on site1: poolID on site2 +# CephFSFscIDMapping: +# - CephFS FscID on site1: CephFS FscID on site2 +csiMapping: [] + +# Configuration for the encryption KMS +# Ref: https://github.com/ceph/ceph-csi/blob/devel/docs/deploy-rbd.md +# Example: +# encryptionKMSConfig: +# vault-unique-id-1: +# encryptionKMSType: vault +# vaultAddress: https://vault.example.com +# vaultAuthPath: /v1/auth/kubernetes/login +# vaultRole: csi-kubernetes +# vaultPassphraseRoot: /v1/secret +# vaultPassphrasePath: ceph-csi/ +# vaultCAVerify: "false" +encryptionKMSConfig: {} + +# Set logging level for csi containers. +# Supported values from 0 to 5. 0 for general useful logs, +# 5 for trace level verbosity. +logLevel: 5 + +nodeplugin: + name: nodeplugin + # set user created priorityclassName for csi plugin pods. default is + # system-node-critical which is high priority + priorityClassName: system-node-critical + # if you are using rbd-nbd client set this value to OnDelete + updateStrategy: RollingUpdate + + httpMetrics: + # Metrics only available for cephcsi/cephcsi => 1.2.0 + # Specifies whether http metrics should be exposed + enabled: true + # The port of the container to expose the metrics + containerPort: 8080 + + service: + # Specifies whether a service should be created for the metrics + enabled: true + # The port to use for the service + servicePort: 8080 + type: ClusterIP + + # Annotations for the service + # Example: + # annotations: + # prometheus.io/scrape: "true" + # prometheus.io/port: "8080" + annotations: {} + + clusterIP: "" + + ## List of IP addresses at which the stats-exporter service is available + ## Ref: https://kubernetes.io/docs/user-guide/services/#external-ips + ## + externalIPs: [] + + loadBalancerIP: "" + loadBalancerSourceRanges: [] + + profiling: + # enable profiling to check for memory leaks + enabled: false + + registrar: + image: + repository: k8s.gcr.io/sig-storage/csi-node-driver-registrar + tag: v2.3.0 + pullPolicy: IfNotPresent + resources: {} + + plugin: + image: + repository: quay.io/cephcsi/cephcsi + tag: canary + pullPolicy: IfNotPresent + resources: {} + + nodeSelector: {} + + tolerations: [] + + affinity: {} + + # If true, create & use Pod Security Policy resources + # https://kubernetes.io/docs/concepts/policy/pod-security-policy/ + podSecurityPolicy: + enabled: false + +provisioner: + name: provisioner + replicaCount: 3 + strategy: + # RollingUpdate strategy replaces old pods with new ones gradually, + # without incurring downtime. + type: RollingUpdate + rollingUpdate: + # maxUnavailable is the maximum number of pods that can be + # unavailable during the update process. + maxUnavailable: 50% + # if fstype is not specified in storageclass, ext4 is default + defaultFSType: ext4 + # deployController to enable or disable the deployment of controller which + # generates the OMAP data if its not Present. + deployController: true + # Timeout for waiting for creation or deletion of a volume + timeout: 60s + # Hard limit for maximum number of nested volume clones that are taken before + # a flatten occurs + hardMaxCloneDepth: 8 + # Soft limit for maximum number of nested volume clones that are taken before + # a flatten occurs + softMaxCloneDepth: 4 + # Maximum number of snapshots allowed on rbd image without flattening + maxSnapshotsOnImage: 450 + # Minimum number of snapshots allowed on rbd image to trigger flattening + minSnapshotsOnImage: 250 + # skip image flattening if kernel support mapping of rbd images + # which has the deep-flatten feature + # skipForceFlatten: false + + # set user created priorityclassName for csi provisioner pods. default is + # system-cluster-critical which is less priority than system-node-critical + priorityClassName: system-cluster-critical + + httpMetrics: + # Metrics only available for cephcsi/cephcsi => 1.2.0 + # Specifies whether http metrics should be exposed + enabled: true + # The port of the container to expose the metrics + containerPort: 8080 + + service: + # Specifies whether a service should be created for the metrics + enabled: true + # The port to use for the service + servicePort: 8080 + type: ClusterIP + + # Annotations for the service + # Example: + # annotations: + # prometheus.io/scrape: "true" + # prometheus.io/port: "8080" + annotations: {} + + clusterIP: "" + + ## List of IP addresses at which the stats-exporter service is available + ## Ref: https://kubernetes.io/docs/user-guide/services/#external-ips + ## + externalIPs: [] + + loadBalancerIP: "" + loadBalancerSourceRanges: [] + + profiling: + # enable profiling to check for memory leaks + enabled: false + + provisioner: + image: + repository: k8s.gcr.io/sig-storage/csi-provisioner + tag: v3.0.0 + pullPolicy: IfNotPresent + resources: {} + + attacher: + name: attacher + enabled: true + image: + repository: k8s.gcr.io/sig-storage/csi-attacher + tag: v3.3.0 + pullPolicy: IfNotPresent + resources: {} + + resizer: + name: resizer + enabled: true + image: + repository: k8s.gcr.io/sig-storage/csi-resizer + tag: v1.3.0 + pullPolicy: IfNotPresent + resources: {} + + snapshotter: + image: + repository: k8s.gcr.io/sig-storage/csi-snapshotter + tag: v4.2.0 + pullPolicy: IfNotPresent + resources: {} + + nodeSelector: {} + + tolerations: [] + + affinity: {} + + # If true, create & use Pod Security Policy resources + # https://kubernetes.io/docs/concepts/policy/pod-security-policy/ + podSecurityPolicy: + enabled: false + +topology: + # Specifies whether topology based provisioning support should + # be exposed by CSI + enabled: false + # domainLabels define which node labels to use as domains + # for CSI nodeplugins to advertise their domains + # NOTE: the value here serves as an example and needs to be + # updated with node labels that define domains of interest + domainLabels: + - failure-domain/region + - failure-domain/zone + +storageClass: + # Specifies whether the storageclass should be created + create: false + name: csi-rbd-sc + + # Annotations for the storage class + # Example: + # annotations: + # storageclass.kubernetes.io/is-default-class: "true" + annotations: {} + + # (required) String representing a Ceph cluster to provision storage from. + # Should be unique across all Ceph clusters in use for provisioning, + # cannot be greater than 36 bytes in length, and should remain immutable for + # the lifetime of the StorageClass in use. + clusterID: <cluster-ID> + + # (optional) If you want to use erasure coded pool with RBD, you need to + # create two pools. one erasure coded and one replicated. + # You need to specify the replicated pool here in the `pool` parameter, it is + # used for the metadata of the images. + # The erasure coded pool must be set as the `dataPool` parameter below. + # dataPool: <ec-data-pool> + dataPool: "" + + # (required) Ceph pool into which the RBD image shall be created + # eg: pool: replicapool + pool: replicapool + + # Set thickProvision to true if you want RBD images to be fully allocated on + # creation (thin provisioning is the default). + thickProvision: false + + # (required) RBD image features, CSI creates image with image-format 2 + # CSI RBD currently supports `layering`, `journaling`, `exclusive-lock`, + # `object-map`, `fast-diff` features. If `journaling` is enabled, must + # enable `exclusive-lock` too. + # imageFeatures: layering,journaling,exclusive-lock,object-map,fast-diff + imageFeatures: "layering" + + # (optional) Specifies whether to try other mounters in case if the current + # mounter fails to mount the rbd image for any reason. True means fallback + # to next mounter, default is set to false. + # Note: tryOtherMounters is currently useful to fallback from krbd to rbd-nbd + # in case if any of the specified imageFeatures is not supported by krbd + # driver on node scheduled for application pod launch, but in the future this + # should work with any mounter type. + # tryOtherMounters: false + + # (optional) uncomment the following to use rbd-nbd as mounter + # on supported nodes + # mounter: rbd-nbd + mounter: "" + + # (optional) ceph client log location, eg: rbd-nbd + # By default host-path /var/log/ceph of node is bind-mounted into + # csi-rbdplugin pod at /var/log/ceph mount path. This is to configure + # target bindmount path used inside container for ceph clients logging. + # See docs/rbd-nbd.md for available configuration options. + # cephLogDir: /var/log/ceph + cephLogDir: "" + + # (optional) ceph client log strategy + # By default, log file belonging to a particular volume will be deleted + # on unmap, but you can choose to just compress instead of deleting it + # or even preserve the log file in text format as it is. + # Available options `remove` or `compress` or `preserve` + # cephLogStrategy: remove + cephLogStrategy: "" + + # (optional) Prefix to use for naming RBD images. + # If omitted, defaults to "csi-vol-". + # volumeNamePrefix: "foo-bar-" + volumeNamePrefix: "" + + # (optional) Instruct the plugin it has to encrypt the volume + # By default it is disabled. Valid values are "true" or "false". + # A string is expected here, i.e. "true", not true. + # encrypted: "true" + encrypted: "" + + # (optional) Use external key management system for encryption passphrases by + # specifying a unique ID matching KMS ConfigMap. The ID is only used for + # correlation to configmap entry. + encryptionKMSID: "" + + # Add topology constrained pools configuration, if topology based pools + # are setup, and topology constrained provisioning is required. + # For further information read TODO<doc> + # topologyConstrainedPools: | + # [{"poolName":"pool0", + # "dataPool":"ec-pool0" # optional, erasure-coded pool for data + # "domainSegments":[ + # {"domainLabel":"region","value":"east"}, + # {"domainLabel":"zone","value":"zone1"}]}, + # {"poolName":"pool1", + # "dataPool":"ec-pool1" # optional, erasure-coded pool for data + # "domainSegments":[ + # {"domainLabel":"region","value":"east"}, + # {"domainLabel":"zone","value":"zone2"}]}, + # {"poolName":"pool2", + # "dataPool":"ec-pool2" # optional, erasure-coded pool for data + # "domainSegments":[ + # {"domainLabel":"region","value":"west"}, + # {"domainLabel":"zone","value":"zone1"}]} + # ] + topologyConstrainedPools: [] + + # (optional) mapOptions is a comma-separated list of map options. + # For krbd options refer + # https://docs.ceph.com/docs/master/man/8/rbd/#kernel-rbd-krbd-options + # For nbd options refer + # https://docs.ceph.com/docs/master/man/8/rbd-nbd/#options + # Format: + # mapOptions: "<mounter>:op1,op2;<mounter>:op1,op2" + # An empty mounter field is treated as krbd type for compatibility. + # eg: + # mapOptions: "krbd:lock_on_read,queue_depth=1024;nbd:try-netlink" + mapOptions: "" + + # (optional) unmapOptions is a comma-separated list of unmap options. + # For krbd options refer + # https://docs.ceph.com/docs/master/man/8/rbd/#kernel-rbd-krbd-options + # For nbd options refer + # https://docs.ceph.com/docs/master/man/8/rbd-nbd/#options + # Format: + # unmapOptions: "<mounter>:op1,op2;<mounter>:op1,op2" + # An empty mounter field is treated as krbd type for compatibility. + # eg: + # unmapOptions: "krbd:force;nbd:force" + unmapOptions: "" + + # The secrets have to contain Ceph credentials with required access + # to the 'pool'. + provisionerSecret: csi-rbd-secret + # If Namespaces are left empty, the secrets are assumed to be in the + # Release namespace. + provisionerSecretNamespace: "" + controllerExpandSecret: csi-rbd-secret + controllerExpandSecretNamespace: "" + nodeStageSecret: csi-rbd-secret + nodeStageSecretNamespace: "" + # Specify the filesystem type of the volume. If not specified, + # csi-provisioner will set default as `ext4`. + fstype: ext4 + reclaimPolicy: Delete + allowVolumeExpansion: true + mountOptions: [] + # Mount Options + # Example: + # mountOptions: + # - discard + +secret: + # Specifies whether the secret should be created + create: false + name: csi-rbd-secret + # Key values correspond to a user name and its key, as defined in the + # ceph cluster. User ID should have required access to the 'pool' + # specified in the storage class + userID: <plaintext ID> + userKey: <Ceph auth key corresponding to userID above> + # Encryption passphrase + encryptionPassphrase: test_passphrase + +# This is a sample configmap that helps define a Ceph configuration as required +# by the CSI plugins. +# Sample ceph.conf available at +# https://github.com/ceph/ceph/blob/master/src/sample.ceph.conf Detailed +# documentation is available at +# https://docs.ceph.com/en/latest/rados/configuration/ceph-conf/ +cephconf: | + [global] + auth_cluster_required = cephx + auth_service_required = cephx + auth_client_required = cephx + + # Workaround for http://tracker.ceph.com/issues/23446 + fuse_set_user_groups = false + + # ceph-fuse which uses libfuse2 by default has write buffer size of 2KiB + # adding 'fuse_big_writes = true' option by default to override this limit + # see https://github.com/ceph/ceph-csi/issues/1928 + fuse_big_writes = true + +######################################################### +# Variables for 'internal' use please use with caution! # +######################################################### + +# The filename of the provisioner socket +provisionerSocketFile: csi-provisioner.sock +# The filename of the plugin socket +pluginSocketFile: csi.sock +# kubelet working directory,can be set using `--root-dir` when starting kubelet. +kubeletDir: /var/lib/kubelet +# Host path location for ceph client processes logging, ex: rbd-nbd +cephLogDirHostPath: /var/log/ceph +# Name of the csi-driver +driverName: rbd.csi.ceph.com +# Name of the configmap used for state +configMapName: ceph-csi-config +# Key to use in the Configmap if not config.json +# configMapKey: +# Use an externally provided configmap +externallyManagedConfigmap: false +# Name of the configmap used for encryption kms configuration +kmsConfigMapName: ceph-csi-encryption-kms-config diff --git a/local-values/ceph/rdb.yaml b/local-values/ceph/rdb.yaml new file mode 100644 index 0000000..df976d5 --- /dev/null +++ b/local-values/ceph/rdb.yaml @@ -0,0 +1,465 @@ +--- +rbac: + # Specifies whether RBAC resources should be created + create: true + +serviceAccounts: + nodeplugin: + # Specifies whether a ServiceAccount should be created + create: true + # The name of the ServiceAccount to use. + # If not set and create is true, a name is generated using the fullname + name: + provisioner: + # Specifies whether a ServiceAccount should be created + create: true + # The name of the ServiceAccount to use. + # If not set and create is true, a name is generated using the fullname + name: + +# Configuration for the CSI to connect to the cluster +# Ref: https://github.com/ceph/ceph-csi/blob/devel/examples/README.md +# Example: +# csiConfig: +# - clusterID: "<cluster-id>" +# monitors: +# - "<MONValue1>" +# - "<MONValue2>" +csiConfig: + - clusterID: "837817cc-7148-11ec-8c46-c81f66de6d53" + monitors: + - "192.168.1.207:6789" + - "192.168.1.208:6789" + - "192.168.1.209:6789" +# csiConfig: [] + +# Configuration details of clusterID,PoolID and FscID mapping +# csiMapping: +# - clusterIDMapping: +# clusterID on site1: clusterID on site2 +# RBDPoolIDMapping: +# - poolID on site1: poolID on site2 +# CephFSFscIDMapping: +# - CephFS FscID on site1: CephFS FscID on site2 +csiMapping: [] + +# Configuration for the encryption KMS +# Ref: https://github.com/ceph/ceph-csi/blob/devel/docs/deploy-rbd.md +# Example: +# encryptionKMSConfig: +# vault-unique-id-1: +# encryptionKMSType: vault +# vaultAddress: https://vault.example.com +# vaultAuthPath: /v1/auth/kubernetes/login +# vaultRole: csi-kubernetes +# vaultPassphraseRoot: /v1/secret +# vaultPassphrasePath: ceph-csi/ +# vaultCAVerify: "false" +encryptionKMSConfig: {} + +# Set logging level for csi containers. +# Supported values from 0 to 5. 0 for general useful logs, +# 5 for trace level verbosity. +logLevel: 5 + +nodeplugin: + name: nodeplugin + # set user created priorityclassName for csi plugin pods. default is + # system-node-critical which is high priority + priorityClassName: system-node-critical + # if you are using rbd-nbd client set this value to OnDelete + updateStrategy: RollingUpdate + + httpMetrics: + # Metrics only available for cephcsi/cephcsi => 1.2.0 + # Specifies whether http metrics should be exposed + enabled: true + # The port of the container to expose the metrics + containerPort: 8080 + + service: + # Specifies whether a service should be created for the metrics + enabled: true + # The port to use for the service + servicePort: 8080 + type: ClusterIP + + # Annotations for the service + # Example: + # annotations: + # prometheus.io/scrape: "true" + # prometheus.io/port: "8080" + annotations: {} + + clusterIP: "" + + ## List of IP addresses at which the stats-exporter service is available + ## Ref: https://kubernetes.io/docs/user-guide/services/#external-ips + ## + externalIPs: [] + + loadBalancerIP: "" + loadBalancerSourceRanges: [] + + profiling: + # enable profiling to check for memory leaks + enabled: false + + registrar: + image: + repository: opsdockerimage/sig-storage-csi-node-driver-registrar + tag: v2.3.0 + pullPolicy: IfNotPresent + resources: {} + + plugin: + image: + repository: quay.io/cephcsi/cephcsi + tag: canary + pullPolicy: IfNotPresent + resources: {} + + nodeSelector: {} + + tolerations: [] + + affinity: {} + + # If true, create & use Pod Security Policy resources + # https://kubernetes.io/docs/concepts/policy/pod-security-policy/ + podSecurityPolicy: + enabled: false + +provisioner: + name: provisioner + replicaCount: 3 + strategy: + # RollingUpdate strategy replaces old pods with new ones gradually, + # without incurring downtime. + type: RollingUpdate + rollingUpdate: + # maxUnavailable is the maximum number of pods that can be + # unavailable during the update process. + maxUnavailable: 50% + # if fstype is not specified in storageclass, ext4 is default + defaultFSType: ext4 + # deployController to enable or disable the deployment of controller which + # generates the OMAP data if its not Present. + deployController: true + # Timeout for waiting for creation or deletion of a volume + timeout: 60s + # Hard limit for maximum number of nested volume clones that are taken before + # a flatten occurs + hardMaxCloneDepth: 8 + # Soft limit for maximum number of nested volume clones that are taken before + # a flatten occurs + softMaxCloneDepth: 4 + # Maximum number of snapshots allowed on rbd image without flattening + maxSnapshotsOnImage: 450 + # Minimum number of snapshots allowed on rbd image to trigger flattening + minSnapshotsOnImage: 250 + # skip image flattening if kernel support mapping of rbd images + # which has the deep-flatten feature + # skipForceFlatten: false + + # set user created priorityclassName for csi provisioner pods. default is + # system-cluster-critical which is less priority than system-node-critical + priorityClassName: system-cluster-critical + + httpMetrics: + # Metrics only available for cephcsi/cephcsi => 1.2.0 + # Specifies whether http metrics should be exposed + enabled: true + # The port of the container to expose the metrics + containerPort: 8080 + + service: + # Specifies whether a service should be created for the metrics + enabled: true + # The port to use for the service + servicePort: 8080 + type: ClusterIP + + # Annotations for the service + # Example: + # annotations: + # prometheus.io/scrape: "true" + # prometheus.io/port: "8080" + annotations: {} + + clusterIP: "" + + ## List of IP addresses at which the stats-exporter service is available + ## Ref: https://kubernetes.io/docs/user-guide/services/#external-ips + ## + externalIPs: [] + + loadBalancerIP: "" + loadBalancerSourceRanges: [] + + profiling: + # enable profiling to check for memory leaks + enabled: false + + provisioner: + image: + repository: opsdockerimage/sig-storage-csi-provisioner + tag: v3.0.0 + pullPolicy: IfNotPresent + resources: {} + + attacher: + name: attacher + enabled: true + image: + repository: opsdockerimage/sig-storage-csi-attacher + tag: v3.3.0 + pullPolicy: IfNotPresent + resources: {} + + resizer: + name: resizer + enabled: true + image: + repository: opsdockerimage/sig-storage-csi-resizer + tag: v1.3.0 + pullPolicy: IfNotPresent + resources: {} + + snapshotter: + image: + repository: opsdockerimage/sig-storage-csi-snapshotter + tag: v4.2.0 + pullPolicy: IfNotPresent + resources: {} + + nodeSelector: {} + + tolerations: [] + + affinity: {} + + # If true, create & use Pod Security Policy resources + # https://kubernetes.io/docs/concepts/policy/pod-security-policy/ + podSecurityPolicy: + enabled: false + +topology: + # Specifies whether topology based provisioning support should + # be exposed by CSI + enabled: false + # domainLabels define which node labels to use as domains + # for CSI nodeplugins to advertise their domains + # NOTE: the value here serves as an example and needs to be + # updated with node labels that define domains of interest + domainLabels: + - failure-domain/region + - failure-domain/zone + +storageClass: + # Specifies whether the storageclass should be created + create: true + name: csi-rbd-sc + + # Annotations for the storage class + # Example: + # annotations: + # storageclass.kubernetes.io/is-default-class: "true" + annotations: {} + + # (required) String representing a Ceph cluster to provision storage from. + # Should be unique across all Ceph clusters in use for provisioning, + # cannot be greater than 36 bytes in length, and should remain immutable for + # the lifetime of the StorageClass in use. + clusterID: "837817cc-7148-11ec-8c46-c81f66de6d53" + + # (optional) If you want to use erasure coded pool with RBD, you need to + # create two pools. one erasure coded and one replicated. + # You need to specify the replicated pool here in the `pool` parameter, it is + # used for the metadata of the images. + # The erasure coded pool must be set as the `dataPool` parameter below. + # dataPool: <ec-data-pool> + dataPool: "k8s" + + # (required) Ceph pool into which the RBD image shall be created + # eg: pool: replicapool + pool: replicapool + + # Set thickProvision to true if you want RBD images to be fully allocated on + # creation (thin provisioning is the default). + thickProvision: false + + # (required) RBD image features, CSI creates image with image-format 2 + # CSI RBD currently supports `layering`, `journaling`, `exclusive-lock`, + # `object-map`, `fast-diff` features. If `journaling` is enabled, must + # enable `exclusive-lock` too. + # imageFeatures: layering,journaling,exclusive-lock,object-map,fast-diff + imageFeatures: "layering" + + # (optional) Specifies whether to try other mounters in case if the current + # mounter fails to mount the rbd image for any reason. True means fallback + # to next mounter, default is set to false. + # Note: tryOtherMounters is currently useful to fallback from krbd to rbd-nbd + # in case if any of the specified imageFeatures is not supported by krbd + # driver on node scheduled for application pod launch, but in the future this + # should work with any mounter type. + # tryOtherMounters: false + + # (optional) uncomment the following to use rbd-nbd as mounter + # on supported nodes + # mounter: rbd-nbd + mounter: "" + + # (optional) ceph client log location, eg: rbd-nbd + # By default host-path /var/log/ceph of node is bind-mounted into + # csi-rbdplugin pod at /var/log/ceph mount path. This is to configure + # target bindmount path used inside container for ceph clients logging. + # See docs/rbd-nbd.md for available configuration options. + # cephLogDir: /var/log/ceph + cephLogDir: "" + + # (optional) ceph client log strategy + # By default, log file belonging to a particular volume will be deleted + # on unmap, but you can choose to just compress instead of deleting it + # or even preserve the log file in text format as it is. + # Available options `remove` or `compress` or `preserve` + # cephLogStrategy: remove + cephLogStrategy: "" + + # (optional) Prefix to use for naming RBD images. + # If omitted, defaults to "csi-vol-". + # volumeNamePrefix: "foo-bar-" + volumeNamePrefix: "" + + # (optional) Instruct the plugin it has to encrypt the volume + # By default it is disabled. Valid values are "true" or "false". + # A string is expected here, i.e. "true", not true. + # encrypted: "true" + encrypted: "" + + # (optional) Use external key management system for encryption passphrases by + # specifying a unique ID matching KMS ConfigMap. The ID is only used for + # correlation to configmap entry. + encryptionKMSID: "" + + # Add topology constrained pools configuration, if topology based pools + # are setup, and topology constrained provisioning is required. + # For further information read TODO<doc> + # topologyConstrainedPools: | + # [{"poolName":"pool0", + # "dataPool":"ec-pool0" # optional, erasure-coded pool for data + # "domainSegments":[ + # {"domainLabel":"region","value":"east"}, + # {"domainLabel":"zone","value":"zone1"}]}, + # {"poolName":"pool1", + # "dataPool":"ec-pool1" # optional, erasure-coded pool for data + # "domainSegments":[ + # {"domainLabel":"region","value":"east"}, + # {"domainLabel":"zone","value":"zone2"}]}, + # {"poolName":"pool2", + # "dataPool":"ec-pool2" # optional, erasure-coded pool for data + # "domainSegments":[ + # {"domainLabel":"region","value":"west"}, + # {"domainLabel":"zone","value":"zone1"}]} + # ] + topologyConstrainedPools: [] + + # (optional) mapOptions is a comma-separated list of map options. + # For krbd options refer + # https://docs.ceph.com/docs/master/man/8/rbd/#kernel-rbd-krbd-options + # For nbd options refer + # https://docs.ceph.com/docs/master/man/8/rbd-nbd/#options + # Format: + # mapOptions: "<mounter>:op1,op2;<mounter>:op1,op2" + # An empty mounter field is treated as krbd type for compatibility. + # eg: + # mapOptions: "krbd:lock_on_read,queue_depth=1024;nbd:try-netlink" + mapOptions: "" + + # (optional) unmapOptions is a comma-separated list of unmap options. + # For krbd options refer + # https://docs.ceph.com/docs/master/man/8/rbd/#kernel-rbd-krbd-options + # For nbd options refer + # https://docs.ceph.com/docs/master/man/8/rbd-nbd/#options + # Format: + # unmapOptions: "<mounter>:op1,op2;<mounter>:op1,op2" + # An empty mounter field is treated as krbd type for compatibility. + # eg: + # unmapOptions: "krbd:force;nbd:force" + unmapOptions: "" + + # The secrets have to contain Ceph credentials with required access + # to the 'pool'. + provisionerSecret: csi-rbd-secret + # If Namespaces are left empty, the secrets are assumed to be in the + # Release namespace. + provisionerSecretNamespace: "" + controllerExpandSecret: csi-rbd-secret + controllerExpandSecretNamespace: "" + nodeStageSecret: csi-rbd-secret + nodeStageSecretNamespace: "" + # Specify the filesystem type of the volume. If not specified, + # csi-provisioner will set default as `ext4`. + fstype: ext4 + reclaimPolicy: Delete + allowVolumeExpansion: true + mountOptions: [] + # Mount Options + # Example: + # mountOptions: + # - discard + +secret: + # Specifies whether the secret should be created + create: true + name: csi-rbd-secret + # Key values correspond to a user name and its key, as defined in the + # ceph cluster. User ID should have required access to the 'pool' + # specified in the storage class + userID: admin + userKey: AQBg4llf+9CAGdsAds4tQzS+0O7dscB5ZTiTEQ== + # Encryption passphrase + encryptionPassphrase: test_passphrase + +# This is a sample configmap that helps define a Ceph configuration as required +# by the CSI plugins. +# Sample ceph.conf available at +# https://github.com/ceph/ceph/blob/master/src/sample.ceph.conf Detailed +# documentation is available at +# https://docs.ceph.com/en/latest/rados/configuration/ceph-conf/ +cephconf: | + [global] + auth_cluster_required = cephx + auth_service_required = cephx + auth_client_required = cephx + + # Workaround for http://tracker.ceph.com/issues/23446 + fuse_set_user_groups = false + + # ceph-fuse which uses libfuse2 by default has write buffer size of 2KiB + # adding 'fuse_big_writes = true' option by default to override this limit + # see https://github.com/ceph/ceph-csi/issues/1928 + fuse_big_writes = true + +######################################################### +# Variables for 'internal' use please use with caution! # +######################################################### + +# The filename of the provisioner socket +provisionerSocketFile: csi-provisioner.sock +# The filename of the plugin socket +pluginSocketFile: csi.sock +# kubelet working directory,can be set using `--root-dir` when starting kubelet. +kubeletDir: /var/lib/kubelet +# Host path location for ceph client processes logging, ex: rbd-nbd +cephLogDirHostPath: /var/log/ceph +# Name of the csi-driver +driverName: rbd.csi.ceph.com +# Name of the configmap used for state +configMapName: ceph-csi-config +# Key to use in the Configmap if not config.json +# configMapKey: +# Use an externally provided configmap +externallyManagedConfigmap: false +# Name of the configmap used for encryption kms configuration +kmsConfigMapName: ceph-csi-encryption-kms-config