Kubernetes Deployment
July 4, 2023
I wanted to document the approach taken to to various bits of the Helm charts that I've authored. This is the Helm chart for deployment of a pod which includes Istio, Apache and PHP-FPM in separate docker containers.
{{ $env := .Values.global.env }}
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ .Values.global.env }}-{{ .Chart.Name }}
labels:
{{- include "labels.standard" . | nindent 8 }}
spec:
selector:
matchLabels:
app: {{ .Values.global.env }}-{{ .Chart.Name }}
template:
metadata:
labels:
{{- include "labels.standard" . | nindent 8 }}
networking/allow-ingress-access: "true"
annotations:
"traffic.sidecar.istio.io/excludeInboundPorts": "{{ .Values.global.app.NEW_RELIC_MONITORING_PORT }}"
proxy.istio.io/config: '{ "holdApplicationUntilProxyStarts": true }'
checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
spec:
topologySpreadConstraints:
- maxSkew: 1
topologyKey: topology.kubernetes.io/zone
whenUnsatisfiable: ScheduleAnyway
labelSelector:
matchLabels:
app: {{ .Values.global.env }}-{{ .Chart.Name }}
{{- with .Values.global.nodeSelector }}
nodeSelector:
{{- toYaml . | nindent 8 }}
{{- end }}
imagePullSecrets:
- name: regcred
serviceAccountName: {{ .Values.global.env }}-{{ .Chart.Name }}
containers:
- name: apache
image: "{{ .Values.global.imageRegistry}}/{{ .Values.global.imageRepositoryPrefix}}/{{ .Chart.Name }}/apache:{{ .Values.version }}"
imagePullPolicy: IfNotPresent
lifecycle:
preStop:
exec:
command: ["/bin/sh","-c","sleep 5 && apachectl -k graceful-stop"]
ports:
- name: http
containerPort: 80
protocol: TCP
livenessProbe:
httpGet:
path: /health/live.php
port: http
readinessProbe:
httpGet:
path: /health/live.php
port: http
periodSeconds: 10
timeoutSeconds: 1
startupProbe:
httpGet:
path: /health/ready
port: http
failureThreshold: 30
periodSeconds: 10
timeoutSeconds: 3
resources: {{- toYaml .Values.apache.resources | nindent 12 }}
volumeMounts:
- name: "{{ $env }}-{{ .Chart.Name }}-apache-config"
mountPath: /etc/apache2/sites-enabled/my-app-apache.conf
subPath: my-app-apache.conf
readOnly: true
- name: php
image: "{{ .Values.global.imageRegistry}}/{{ .Values.global.imageRepositoryPrefix}}/{{ .Chart.Name }}/php-fpm:{{ .Values.version }}"
imagePullPolicy: IfNotPresent
envFrom:
- configMapRef:
name: {{ .Values.global.env }}-{{ .Chart.Name }}
env:
# php max children
- name: PHP_MAX_CHILDREN
{{- if hasSuffix "M" (trimSuffix "i" .Values.php.resources.requests.memory) }}
# MBs
value: "{{ div (sub (trimSuffix "i" .Values.php.resources.requests.memory | trimSuffix "M") .Values.php.osMemoryUsageMb) .Values.php.processMemoryUsageMb }}"
{{- else }}
# GBs
value: "{{ div (sub (trimSuffix "i" .Values.php.resources.requests.memory | trimSuffix "G" | mul 1024) .Values.php.osMemoryUsageMb) Values.php.processMemoryUsageMb }}"
{{- end }}
# secrets from values
{{- range $key, $val := .Values.php.secrets }}
- name: {{ $key }}
valueFrom:
secretKeyRef:
key: {{ $val }}
name: {{ $env }}-{{ $val }}
{{- end }}
# env vars from values
{{- range $key, $val := .Values.php.envVars }}
- name: {{ $key }}
value: "{{ tpl $val $ }}"
{{- end }}
ports:
- name: php-fpm
containerPort: 9080
protocol: TCP
livenessProbe:
exec:
command:
- sh
- -c
- FCGI_CONNECT=localhost:9080 php-fpm-healthcheck
initialDelaySeconds: 0
periodSeconds: 10
readinessProbe:
exec:
command:
- sh
- -c
- FCGI_CONNECT=localhost:9080 php-fpm-healthcheck
initialDelaySeconds: 1
periodSeconds: 5
resources: {{- toYaml .Values.php.resources | nindent 12 }}
volumeMounts:
- name: "{{ $env }}-{{ .Chart.Name }}-php-fpm-config"
mountPath: /usr/local/etc/php-fpm/pool.d/my-app-fpm.conf
subPath: my-app-fpm.conf
readOnly: true
volumes:
- name: "{{ $env }}-{{ .Chart.Name }}-apache-config"
configMap:
name: "{{ $env }}-{{ .Chart.Name }}-apache-config"
- name: "{{ $env }}-{{ .Chart.Name }}-php-fpm-config"
configMap:
name: "{{ $env }}-{{ .Chart.Name }}-php-fpm-config"
This matches up with a values.yaml file in this format:
version: v1.0.1300
minReplicas: 1
maxReplicas: 2
apache:
resources:
requests:
memory: 500Mi
cpu: 250m
limits:
memory: 500Mi
php:
resources:
requests:
memory: 2Gi
cpu: 2
limits:
memory: 2Gi
# max children is configured based on (amount of memory set in requests - OS memory) / php-fpm memory usage e.g.1GB - 256 / 40 = 19 max children
# use below command to get average memory use
# ps --no-headers -o "rss,cmd" -C php-fpm | grep $DCG_COMPONENT | awk '{ sum+=$1 } END { printf ("%d%s\n", sum/NR/1024,"Mb") }'
osMemoryUsageMb: 150
processMemoryUsageMb: 55
maxRequests: 400
secrets:
SECRET_APPS: secret-apps
ANOTHER_SECRET_APPS: another-secret-apps
envVars:
APP_ENVIRONMENT_NAME: "prod"
ANOTHER_ENV_VAR: "value"
Secrets need a SecretProviderClass, and this is also generated from the same values.yaml
{{ $env := .Values.global.env }}
{{- if .Values.php.secrets }}
apiVersion: secrets-store.csi.x-k8s.io/v1
kind: SecretProviderClass
metadata:
name: {{ .Values.global.env }}-secrets-{{ .Chart.Name }}
labels:
{{- include "labels.standard" . | nindent 8 }}
spec:
provider: azure
secretObjects:
{{- range $key, $val := .Values.php.secrets }}
- secretName: {{ $env }}-{{ $val }}
labels:
{{- include "labels.standard" $ | nindent 8 }}
type: Opaque
data:
- objectName: {{ $key }}
key: {{ $val }}
{{- end }}
parameters:
usePodIdentity: "false"
keyvaultName: "{{ .Values.global.keyVaultName }}"
cloudName: ""
objects: |
array:
{{- range $key, $val := .Values.php.secrets }}
- |
objectName: {{ $val | quote }}
objectAlias: {{ $key | quote }}
objectType: secret
objectVersion: ""
{{- end }}
tenantId: {{ .Values.global.tenantId }}
{{- end }}
