![23
affinity:
podAntiAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
labelSelector:
matchExpressions:
key: app
operator: In
values:
{{ template "redisenterprise.name" . }}
key: release
operator: In
values:
{{ .Release.Name }}
key: redis.io/role
operator: In
values:
node
topologyKey: kubernetes.io/hostname
terminationGracePeriodSeconds: 31536000
serviceAccountName: {{ template "redisenterprise.serviceAccountName" . }}
{{ with .Values.imagePullSecrets }}
imagePullSecrets:
{{ toYaml . | indent 8 }}
{{ end }}
containers:
name: redis
image: {{ .Values.redisImage.repository }}:{{ .Values.redisImage.tag }}
imagePullPolicy: {{ .Values.redisImage.pullPolicy }}
readinessProbe:
exec:
command:
# check that the node is bootstrapped and that its connected and synced.
bash
c
curl silent localhost:8080/v1/bootstrap && /opt/redislabs/bin/rladmin status
nodes | grep node:$(cat /etc/opt/redislabs/node.id) | grep OK
initialDelaySeconds: 20
timeoutSeconds: 5
lifecycle:
preStop:
exec:
command:
# enslave the node, if this current node is master, change the master to
the first slave node.
bash
c
/opt/redislabs/bin/rladmin node $(cat /etc/opt/redislabs/node.id) enslave
&& ((/opt/redislabs/bin/rladmin status nodes | grep node:$(cat
/etc/opt/redislabs/node.id) | grep q master) && /opt/redislabs/bin/rlutil
change_master master=$(/opt/redislabs/bin/rladmin status nodes | grep slave |
head 1 | cut d " " f 1| cut d ":" f2) && sleep 10) || /bin/true
resources:
{{ toYaml .Values.redisResources | indent 10 }}
ports:
containerPort: 8001
containerPort: 8443
containerPort: 9443
securityContext:
capabilities:
add:
SYS_RESOURCE
{{ if .Values.persistentVolume.enabled }}
volumeMounts:
mountPath: "/opt/persistent"
name: redisstorage
{{ end }}
env:
name: K8S_ORCHASTRATED_DEPLOYMENT
value: "yes"
name: JOIN_HOSTNAME
value: {{ template "redisenterprise.fullname" . }}
{{ if .Values.persistentVolume.enabled }}
name: PERSISTANCE_PATH
value: /opt/persistent
{{ end }}
name: K8S_SERVICE_NAME
value: {{ template "redisenterprise.fullname" . }}
name: BOOTSTRAP_HANDLE_REDIRECTS
value: "enabled"
name: BOOTSTRAP_CLUSTER_FQDN
value: {{ template "redisenterprise.clusterDNS" . }}
name: BOOTSTRAP_DMC_THREADS
value: "10"
name: BOOTSTRAP_USERNAME
valueFrom:
secretKeyRef:
name: {{ template "redisenterprise.fullname" . }}
key: username
name: BOOTSTRAP_PASSWORD
valueFrom:
secretKeyRef:
name: {{ template "redisenterprise.fullname" . }}
key: password
name: BOOTSTRAP_LICENSE
valueFrom:
secretKeyRef:
name: {{ template "redisenterprise.fullname" . }}
key: license
apiVersion: apps/v1beta1
kind: StatefulSet
metadata:
name: {{ template "redisenterprise.statefulsetName" . }}
labels:
app: {{ template "redisenterprise.name" . }}
chart: {{ template "redisenterprise.chart" . }}
release: {{ .Release.Name }}
heritage: {{ .Release.Service }}
spec:
{{ if .Values.persistentVolume.enabled }}
volumeClaimTemplates:
metadata:
name: redisstorage
labels:
app: {{ template "redisenterprise.name" . }}
chart: {{ template "redisenterprise.chart" . }}
release: {{ .Release.Name }}
heritage: {{ .Release.Service }}
spec:
accessModes: [ "ReadWriteOnce" ]
resources:
requests:
storage: {{ .Values.persistentVolume.size | quote }}
{{ if .Values.persistentVolume.storageClass }}
{{ if (eq "" .Values.persistentVolume.storageClass) }}
storageClassName: ""
{{ else }}
storageClassName: "{{ .Values.persistentVolume.storageClass }}"
{{ end }}
{{ end }}
{{ end }}
serviceName: {{ template "redisenterprise.fullname" . }}
replicas: {{ .Values.replicas }}
updateStrategy:
type: "RollingUpdate"
template:
metadata:
labels:
redis.io/role: node
app: {{ template "redisenterprise.name" . }}
chart: {{ template "redisenterprise.chart" . }}
release: {{ .Release.Name }}
heritage: {{ .Release.Service }}
spec:
{{ with .Values.nodeSelector }}
nodeSelector:
{{ toYaml . | indent 8 }}
{{ end }}](https://image.slidesharecdn.com/copyofredislabsmeetup-k8s282f102f2018-181028191829/85/Redis-Meetup-TLV-K8s-Session-28-10-2018-23-320.jpg)
Redis Meetup TLV - K8s Session 28/10/2018
- 2. Kubernetes
- 3. Redise - Open Source & Proprietary Technology Redise Node Cluster Manager Redise Cluster • Shared nothing, symmetrical cluster architecture • Fully compatible with open source commands & data structures Enterprise Layer Open Source Layer REST API Zero latency proxy
- 4. • Faster time to market with continuity between dev/test and production environments that use Redise Pack • Highly available, easier to scale, simpler to manage Redis technology, integrated with orchestration tools such as PCF, Kubernetes, Mesosphere... • Node in a container approach — All Redise services inside each container. Run Redise clusters on single or multiple nodes Redise in Containers
- 5. Node in a Pod Approach Node 1 Vs Node 2 Node 3 Node 1 Node 2 Node 3 One pod, multiple services per nodeMultiple pods, multiple services per node
- 6. Double Orchestration For fun and profit! 6
- 7. What’s Double Orchestration ? Kubernetes PKS External Redis Cluster Orchestration Node 1 Redis Shards Node 2 Node N Internal
- 8. Why like this? • Resource management - Orchestration platforms are designed to be generic. • Again - Performance is king. • Last but not least, it allows us to maintain a common architecture - regardless of running environment, be it bare metal, VM, K8s, Pivotal Cloud Foundry. (.… Surprisingly enough, not everybody in the world uses containers…)
- 9. Who Does What • Node auto-healing • Node scaling • Failover & scaling • Configuration & monitoring • Service discovery • Upgrade +
- 10. • Provide a solid primary db solution for end-users • Stateful application – Some changes cannot be performed – Some changes need to mutate the state before applying the actual change – Data-loss is unacceptable • Support multiple k8s deployments – Cloud: GKE, AWS, etc – Openshift – PKS – Vanilla – On-prem hardware vendor • Ingress • Packaging Redis Labs Challenges
- 12. Building our deployment puzzle Redis Enterprise Nodes Persistence High Availability DBs proxies Cluster Login pod-1 pod-2 Anti Affinity Stateful Set pod-0 PV-1PV-0 PV-2 k8s Node-A k8s Node-B k8s Node-C Secrets Resource db-1.ns.svc Deployment Services management db-2.ns.svc
- 13. Started out with 9 static yaml files – Hard to deploy – Hard to maintain – Hard to distribute – No control over the deployment life-cycle Our journey .yaml .yaml Helm - A package manager – Customized deployment – Easier to maintain (templating) – Not fully supported everywhere – No control over the deployment life-cycle • Operator – Simple deployment (2 yaml files) – Full control over life-cycle – K8s compatible - no plugin needed
- 14. Operator
- 15. 15 Kubernetes Controllers Stateful Set controller Deployment controllerNode controller Cronjob controller GC controller Endpoints controller PV controller . . . API Resource X Resource X Controller watch act
- 17. 17 kubectl K8s API StatefulSet Controller Watch(StatefulSet) pod-0 pod-1 pod-2 my-sts kubectl create -f my-sts.yaml PV PV PV
- 18. 18 kubectl StatefulSet Controller Watch(StatefulSet) pod-0 pod-1 pod-2 my-sts kubectl scale statefulset my-sts --replicas=5 pod-3 pod-4 K8s API
- 19. 19 kubectl K8s API RedisCluster Controller Watch(RedisCluster) my-redis-cluster kubectl create -f my-redis-cluster.yaml Stateful Set UI service Service Account ...
- 20. 20 kubectl K8s API RedisCluster Controller Watch(RedisCluster) my-redis-cluster kubectl apply -f my-redis-cluster.yaml Stateful Set UI service Service Account ... Redis Enterprise Cluster get-status() example: downscale
- 21. Building our deployment puzzle Nodes Persistence High Availability DBs and UI endpoints Secrets pod-1 pod-2 Pod Disruption Budget Anti Affinity stateful set pod-0 PVPV PV k8s Node-A k8s Node-B k8s Node-C Secrets resource db-1.svc deployment Services manager db-2.svc Management Layer Redis Enterprise Cluster Resource Operator
- 22. 22 my-cluster.yaml kubectl apply -f my-cluster.yaml
- 23. 23 affinity: podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: labelSelector: matchExpressions: key: app operator: In values: {{ template "redisenterprise.name" . }} key: release operator: In values: {{ .Release.Name }} key: redis.io/role operator: In values: node topologyKey: kubernetes.io/hostname terminationGracePeriodSeconds: 31536000 serviceAccountName: {{ template "redisenterprise.serviceAccountName" . }} {{ with .Values.imagePullSecrets }} imagePullSecrets: {{ toYaml . | indent 8 }} {{ end }} containers: name: redis image: {{ .Values.redisImage.repository }}:{{ .Values.redisImage.tag }} imagePullPolicy: {{ .Values.redisImage.pullPolicy }} readinessProbe: exec: command: # check that the node is bootstrapped and that its connected and synced. bash c curl silent localhost:8080/v1/bootstrap && /opt/redislabs/bin/rladmin status nodes | grep node:$(cat /etc/opt/redislabs/node.id) | grep OK initialDelaySeconds: 20 timeoutSeconds: 5 lifecycle: preStop: exec: command: # enslave the node, if this current node is master, change the master to the first slave node. bash c /opt/redislabs/bin/rladmin node $(cat /etc/opt/redislabs/node.id) enslave && ((/opt/redislabs/bin/rladmin status nodes | grep node:$(cat /etc/opt/redislabs/node.id) | grep q master) && /opt/redislabs/bin/rlutil change_master master=$(/opt/redislabs/bin/rladmin status nodes | grep slave | head 1 | cut d " " f 1| cut d ":" f2) && sleep 10) || /bin/true resources: {{ toYaml .Values.redisResources | indent 10 }} ports: containerPort: 8001 containerPort: 8443 containerPort: 9443 securityContext: capabilities: add: SYS_RESOURCE {{ if .Values.persistentVolume.enabled }} volumeMounts: mountPath: "/opt/persistent" name: redisstorage {{ end }} env: name: K8S_ORCHASTRATED_DEPLOYMENT value: "yes" name: JOIN_HOSTNAME value: {{ template "redisenterprise.fullname" . }} {{ if .Values.persistentVolume.enabled }} name: PERSISTANCE_PATH value: /opt/persistent {{ end }} name: K8S_SERVICE_NAME value: {{ template "redisenterprise.fullname" . }} name: BOOTSTRAP_HANDLE_REDIRECTS value: "enabled" name: BOOTSTRAP_CLUSTER_FQDN value: {{ template "redisenterprise.clusterDNS" . }} name: BOOTSTRAP_DMC_THREADS value: "10" name: BOOTSTRAP_USERNAME valueFrom: secretKeyRef: name: {{ template "redisenterprise.fullname" . }} key: username name: BOOTSTRAP_PASSWORD valueFrom: secretKeyRef: name: {{ template "redisenterprise.fullname" . }} key: password name: BOOTSTRAP_LICENSE valueFrom: secretKeyRef: name: {{ template "redisenterprise.fullname" . }} key: license apiVersion: apps/v1beta1 kind: StatefulSet metadata: name: {{ template "redisenterprise.statefulsetName" . }} labels: app: {{ template "redisenterprise.name" . }} chart: {{ template "redisenterprise.chart" . }} release: {{ .Release.Name }} heritage: {{ .Release.Service }} spec: {{ if .Values.persistentVolume.enabled }} volumeClaimTemplates: metadata: name: redisstorage labels: app: {{ template "redisenterprise.name" . }} chart: {{ template "redisenterprise.chart" . }} release: {{ .Release.Name }} heritage: {{ .Release.Service }} spec: accessModes: [ "ReadWriteOnce" ] resources: requests: storage: {{ .Values.persistentVolume.size | quote }} {{ if .Values.persistentVolume.storageClass }} {{ if (eq "" .Values.persistentVolume.storageClass) }} storageClassName: "" {{ else }} storageClassName: "{{ .Values.persistentVolume.storageClass }}" {{ end }} {{ end }} {{ end }} serviceName: {{ template "redisenterprise.fullname" . }} replicas: {{ .Values.replicas }} updateStrategy: type: "RollingUpdate" template: metadata: labels: redis.io/role: node app: {{ template "redisenterprise.name" . }} chart: {{ template "redisenterprise.chart" . }} release: {{ .Release.Name }} heritage: {{ .Release.Service }} spec: {{ with .Values.nodeSelector }} nodeSelector: {{ toYaml . | indent 8 }} {{ end }}
- 24. Demo