From af63c11cc4b8503596a9ba2825d090bd1111b6f9 Mon Sep 17 00:00:00 2001 From: Tim Anderegg Date: Tue, 3 Mar 2026 13:27:12 -0500 Subject: [PATCH 1/3] Vend MongoDB Helm chart. --- .../modules/k8s_data_layer/mongodb_apps.tf | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/terraform-k8s-infrastructure/modules/k8s_data_layer/mongodb_apps.tf b/terraform-k8s-infrastructure/modules/k8s_data_layer/mongodb_apps.tf index 3e15847..57bbc64 100644 --- a/terraform-k8s-infrastructure/modules/k8s_data_layer/mongodb_apps.tf +++ b/terraform-k8s-infrastructure/modules/k8s_data_layer/mongodb_apps.tf @@ -1,12 +1,11 @@ resource "helm_release" "mongodb_apps" { name = "mongodb-apps" - chart = "stable/mongodb-replicaset" + #chart = "stable/mongodb-replicaset" + chart = "${path.module}/charts/mongodb-replicaset" namespace = "core" - version = "3.15.0" + #version = "3.15.0" values = [ file("${path.module}/mongodb_apps_values/mongodb-apps-values.yaml") ] -} - - +} \ No newline at end of file From c1af78ae481620b026741bff583c426ea3dddd5c Mon Sep 17 00:00:00 2001 From: Tim Anderegg Date: Tue, 3 Mar 2026 13:33:41 -0500 Subject: [PATCH 2/3] Adds new chart files. --- .../charts/mongodb-replicaset/Chart.yaml | 16 + .../charts/mongodb-replicaset/README.md | 438 ++++++++++++++++++ .../charts/mongodb-replicaset/test.sh | 48 ++ .../charts/mongodb-replicaset/values.yaml | 198 ++++++++ 4 files changed, 700 insertions(+) create mode 100644 terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/Chart.yaml create mode 100644 terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/README.md create mode 100755 terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/test.sh create mode 100644 terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/values.yaml diff --git a/terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/Chart.yaml b/terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/Chart.yaml new file mode 100644 index 0000000..e78e410 --- /dev/null +++ b/terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/Chart.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +name: mongodb-replicaset +home: https://github.com/mongodb/mongo +version: 3.15.1 +appVersion: 3.6 +description: NoSQL document-oriented database that stores JSON-like documents with + dynamic schemas, simplifying the integration of data in content-driven applications. +icon: https://webassets.mongodb.com/_com_assets/cms/mongodb-logo-rgb-j6w271g1xn.jpg +sources: + - https://github.com/mongodb/mongo + - https://github.com/percona/mongodb_exporter +maintainers: + - name: unguiculus + email: unguiculus@gmail.com + - name: steven-sheehy + email: ssheehy@firescope.com diff --git a/terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/README.md b/terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/README.md new file mode 100644 index 0000000..222df76 --- /dev/null +++ b/terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/README.md @@ -0,0 +1,438 @@ +# MongoDB Helm Chart + +## Prerequisites Details + +* Kubernetes 1.9+ +* Kubernetes beta APIs enabled only if `podDisruptionBudget` is enabled +* PV support on the underlying infrastructure + +## StatefulSet Details + +* https://kubernetes.io/docs/concepts/abstractions/controllers/statefulsets/ + +## StatefulSet Caveats + +* https://kubernetes.io/docs/concepts/abstractions/controllers/statefulsets/#limitations + +## Chart Details + +This chart implements a dynamically scalable [MongoDB replica set](https://docs.mongodb.com/manual/tutorial/deploy-replica-set/) +using Kubernetes StatefulSets and Init Containers. + +## Installing the Chart + +To install the chart with the release name `my-release`: + +``` console +helm repo add stable https://kubernetes-charts.storage.googleapis.com/ +helm install --name my-release stable/mongodb-replicaset +``` + +## Configuration + +The following table lists the configurable parameters of the mongodb chart and their default values. + +| Parameter | Description | Default | +| ----------------------------------- | ------------------------------------------------------------------------- | --------------------------------------------------- | +| `replicas` | Number of replicas in the replica set | `3` | +| `replicaSetName` | The name of the replica set | `rs0` | +| `skipInitialization` | If `true` skip replica set initialization during bootstrapping | `false` +| `podDisruptionBudget` | Pod disruption budget | `{}` | +| `updateStrategy` | Update strategy | `nil` | +| `port` | MongoDB port | `27017` | +| `imagePullSecrets` | Image pull secrets | `[]` | +| `installImage.repository` | Image name for the install container | `unguiculus/mongodb-install` | +| `installImage.tag` | Image tag for the install container | `0.7` | +| `installImage.pullPolicy` | Image pull policy for the init container that establishes the replica set | `IfNotPresent` | +| `copyConfigImage.repository` | Image name for the copy config init container | `busybox` | +| `copyConfigImage.tag` | Image tag for the copy config init container | `1.29.3` | +| `copyConfigImage.pullPolicy` | Image pull policy for the copy config init container | `IfNotPresent` | +| `image.repository` | MongoDB image name | `mongo` | +| `image.tag` | MongoDB image tag | `3.6` | +| `image.pullPolicy` | MongoDB image pull policy | `IfNotPresent` | +| `podAnnotations` | Annotations to be added to MongoDB pods | `{}` | +| `statefulSetAnnotations` | Annotations to be added to MongoDB statefulSet | `{}` | +| `securityContext.enabled` | Enable security context | `true` | +| `securityContext.fsGroup` | Group ID for the container | `999` | +| `securityContext.runAsUser` | User ID for the container | `999` | +| `securityContext.runAsNonRoot` | | `true` | +| `resources` | Pod resource requests and limits | `{}` | +| `persistentVolume.enabled` | If `true`, persistent volume claims are created | `true` | +| `persistentVolume.storageClass` | Persistent volume storage class | `` | +| `persistentVolume.accessModes` | Persistent volume access modes | `[ReadWriteOnce]` | +| `persistentVolume.size` | Persistent volume size | `10Gi` | +| `persistentVolume.annotations` | Persistent volume annotations | `{}` | +| `terminationGracePeriodSeconds` | Duration in seconds the pod needs to terminate gracefully | `30` | +| `tls.enabled` | Enable MongoDB TLS support including authentication | `false` | +| `tls.mode` | Set the SSL operation mode (disabled, allowSSL, preferSSL, requireSSL) | `requireSSL` | +| `tls.cacert` | The CA certificate used for the members | Our self signed CA certificate | +| `tls.cakey` | The CA key used for the members | Our key for the self signed CA certificate | +| `init.resources` | Pod resource requests and limits (for init containers) | `{}` | +| `init.timeout` | The amount of time in seconds to wait for bootstrap to finish | `900` | +| `metrics.enabled` | Enable Prometheus compatible metrics for pods and replicasets | `false` | +| `metrics.image.repository` | Image name for metrics exporter | `bitnami/mongodb-exporter` | +| `metrics.image.tag` | Image tag for metrics exporter | `0.9.0-debian-9-r2` | +| `metrics.image.pullPolicy` | Image pull policy for metrics exporter | `IfNotPresent` | +| `metrics.port` | Port for metrics exporter | `9216` | +| `metrics.path` | URL Path to expose metics | `/metrics` | +| `metrics.resources` | Metrics pod resource requests and limits | `{}` | +| `metrics.securityContext.enabled` | Enable security context | `true` | +| `metrics.securityContext.fsGroup` | Group ID for the metrics container | `1001` | +| `metrics.securityContext.runAsUser` | User ID for the metrics container | `1001` | +| `metrics.prometheusServiceDiscovery`| Adds annotations for Prometheus ServiceDiscovery | `true` | +| `auth.enabled` | If `true`, keyfile access control is enabled | `false` | +| `auth.key` | Key for internal authentication | `` | +| `auth.existingKeySecret` | If set, an existing secret with this name for the key is used | `` | +| `auth.adminUser` | MongoDB admin user | `` | +| `auth.adminPassword` | MongoDB admin password | `` | +| `auth.metricsUser` | MongoDB clusterMonitor user | `` | +| `auth.metricsPassword` | MongoDB clusterMonitor password | `` | +| `auth.existingMetricsSecret` | If set, and existing secret with this name is used for the metrics user | `` | +| `auth.existingAdminSecret` | If set, and existing secret with this name is used for the admin user | `` | +| `secretAnnotations` | Annotations to be added to the secret if auth is enabled | `{}` | +| `serviceAnnotations` | Annotations to be added to the service | `{}` | +| `configmap` | Content of the MongoDB config file | `` | +| `initMongodStandalone` | If set, initContainer executes script in standalone mode | `` | +| `nodeSelector` | Node labels for pod assignment | `{}` | +| `affinity` | Node/pod affinities | `{}` | +| `tolerations` | List of node taints to tolerate | `[]` | +| `priorityClassName` | Pod priority class name | `` | +| `livenessProbe.failureThreshold` | Liveness probe failure threshold | `3` | +| `livenessProbe.initialDelaySeconds` | Liveness probe initial delay seconds | `30` | +| `livenessProbe.periodSeconds` | Liveness probe period seconds | `10` | +| `livenessProbe.successThreshold` | Liveness probe success threshold | `1` | +| `livenessProbe.timeoutSeconds` | Liveness probe timeout seconds | `5` | +| `readinessProbe.failureThreshold` | Readiness probe failure threshold | `3` | +| `readinessProbe.initialDelaySeconds`| Readiness probe initial delay seconds | `5` | +| `readinessProbe.periodSeconds` | Readiness probe period seconds | `10` | +| `readinessProbe.successThreshold` | Readiness probe success threshold | `1` | +| `readinessProbe.timeoutSeconds` | Readiness probe timeout seconds | `1` | +| `extraContainers` | Additional containers to add to the StatefulSet | `[]` | +| `extraVars` | Set environment variables for the main container | `{}` | +| `extraLabels` | Additional labels to add to resources | `{}` | +| `extraVolumes` | Additional volumes to add to the resources | `[]` | +| `global.namespaceOverride` | Override the deployment namespace | Not set (`Release.Namespace`) | + +*MongoDB config file* + +All options that depended on the chart configuration are supplied as command-line arguments to `mongod`. By default, the chart creates an empty config file. Entries may be added via the `configmap` configuration value. + +Specify each parameter using the `--set key=value[,key=value]` argument to `helm install`. + +Alternatively, a YAML file that specifies the values for the parameters can be provided while installing the chart. For example, + +``` console +helm install --name my-release -f values.yaml stable/mongodb-replicaset +``` + +> **Tip**: You can use the default [values.yaml](values.yaml) + +Once you have all 3 nodes in running, you can run the "test.sh" script in this directory, which will insert a key into the primary and check the secondaries for output. This script requires that the `$RELEASE_NAME` environment variable be set, in order to access the pods. + +## Authentication + +By default, this chart creates a MongoDB replica set without authentication. Authentication can be +enabled using the parameter `auth.enabled`. Once enabled, keyfile access control is set up and an +admin user with root privileges is created. User credentials and keyfile may be specified directly. +Alternatively, existing secrets may be provided. The secret for the admin user must contain the +keys `user` and `password`, that for the key file must contain `key.txt`. The user is created with +full `root` permissions but is restricted to the `admin` database for security purposes. It can be +used to create additional users with more specific permissions. + +To connect to the mongo shell with authentication enabled, use a command similar to the following (substituting values as appropriate): + +```shell +kubectl exec -it mongodb-replicaset-0 -- mongo mydb -u admin -p password --authenticationDatabase admin +``` + +## TLS support + +To enable full TLS encryption set `tls.enabled` to `true`. It is recommended to create your own CA by executing: + +```console +openssl genrsa -out ca.key 2048 +openssl req -x509 -new -nodes -key ca.key -days 10000 -out ca.crt -subj "/CN=mydomain.com" +``` + +After that paste the base64 encoded (`cat ca.key | base64 -w0`) cert and key into the fields `tls.cacert` and +`tls.cakey`. Adapt the configmap for the replicaset as follows: + +```yml +configmap: + storage: + dbPath: /data/db + net: + port: 27017 + ssl: + mode: requireSSL + CAFile: /data/configdb/tls.crt + PEMKeyFile: /work-dir/mongo.pem + # Set to false to require mutual TLS encryption + allowConnectionsWithoutCertificates: true + replication: + replSetName: rs0 + security: + authorization: enabled + # # Uncomment to enable mutual TLS encryption + # clusterAuthMode: x509 + keyFile: /keydir/key.txt +``` + +To access the cluster you need one of the certificates generated during cluster setup in `/work-dir/mongo.pem` of the +certain container or you generate your own one via: + +```console +$ cat >openssl.cnf < mongo.pem +$ rm mongo.key mongo.crt +``` + +Please ensure that you exchange the `$HOSTNAME` with your actual hostname and the `$HOSTNAME1`, `$HOSTNAME2`, etc. with +alternative hostnames you want to allow access to the MongoDB replicaset. You should now be able to authenticate to the +mongodb with your `mongo.pem` certificate: + +```console +mongo --ssl --sslCAFile=ca.crt --sslPEMKeyFile=mongo.pem --eval "db.adminCommand('ping')" +``` + +## Promethus metrics + +Enabling the metrics as follows will allow for each replicaset pod to export Prometheus compatible metrics +on server status, individual replicaset information, replication oplogs, and storage engine. + +```yaml +metrics: + enabled: true + image: + repository: ssalaues/mongodb-exporter + tag: 0.6.1 + pullPolicy: IfNotPresent + port: 9216 + path: "/metrics" + socketTimeout: 3s + syncTimeout: 1m + prometheusServiceDiscovery: true + resources: {} +``` + +More information on [MongoDB Exporter](https://github.com/percona/mongodb_exporter) metrics available. + +## Deep dive + +Because the pod names are dependent on the name chosen for it, the following examples use the +environment variable `RELEASENAME`. For example, if the helm release name is `messy-hydra`, one would need to set the following before proceeding. The example scripts below assume 3 pods only. + +```console +export RELEASE_NAME=messy-hydra +``` + +### Cluster Health + +```console +for i in 0 1 2; do kubectl exec $RELEASE_NAME-mongodb-replicaset-$i -- sh -c 'mongo --eval="printjson(db.serverStatus())"'; done +``` + +### Failover + +One can check the roles being played by each node by using the following: + +```console +$ for i in 0 1 2; do kubectl exec $RELEASE_NAME-mongodb-replicaset-$i -- sh -c 'mongo --eval="printjson(rs.isMaster())"'; done + +MongoDB shell version: 3.6.3 +connecting to: mongodb://127.0.0.1:27017 +MongoDB server version: 3.6.3 +{ + "hosts" : [ + "messy-hydra-mongodb-0.messy-hydra-mongodb.default.svc.cluster.local:27017", + "messy-hydra-mongodb-1.messy-hydra-mongodb.default.svc.cluster.local:27017", + "messy-hydra-mongodb-2.messy-hydra-mongodb.default.svc.cluster.local:27017" + ], + "setName" : "rs0", + "setVersion" : 3, + "ismaster" : true, + "secondary" : false, + "primary" : "messy-hydra-mongodb-0.messy-hydra-mongodb.default.svc.cluster.local:27017", + "me" : "messy-hydra-mongodb-0.messy-hydra-mongodb.default.svc.cluster.local:27017", + "electionId" : ObjectId("7fffffff0000000000000001"), + "maxBsonObjectSize" : 16777216, + "maxMessageSizeBytes" : 48000000, + "maxWriteBatchSize" : 1000, + "localTime" : ISODate("2016-09-13T01:10:12.680Z"), + "maxWireVersion" : 4, + "minWireVersion" : 0, + "ok" : 1 +} +``` + +This lets us see which member is primary. + +Let us now test persistence and failover. First, we insert a key (in the below example, we assume pod 0 is the master): + +```console +$ kubectl exec $RELEASE_NAME-mongodb-replicaset-0 -- mongo --eval="printjson(db.test.insert({key1: 'value1'}))" + +MongoDB shell version: 3.6.3 +connecting to: mongodb://127.0.0.1:27017 +{ "nInserted" : 1 } +``` + +Watch existing members: + +```console +$ kubectl run --attach bbox --image=mongo:3.6 --restart=Never --env="RELEASE_NAME=$RELEASE_NAME" -- sh -c 'while true; do for i in 0 1 2; do echo $RELEASE_NAME-mongodb-replicaset-$i $(mongo --host=$RELEASE_NAME-mongodb-replicaset-$i.$RELEASE_NAME-mongodb-replicaset --eval="printjson(rs.isMaster())" | grep primary); sleep 1; done; done'; + +Waiting for pod default/bbox2 to be running, status is Pending, pod ready: false +If you don't see a command prompt, try pressing enter. +messy-hydra-mongodb-2 "primary" : "messy-hydra-mongodb-0.messy-hydra-mongodb.default.svc.cluster.local:27017", +messy-hydra-mongodb-0 "primary" : "messy-hydra-mongodb-0.messy-hydra-mongodb.default.svc.cluster.local:27017", +messy-hydra-mongodb-1 "primary" : "messy-hydra-mongodb-0.messy-hydra-mongodb.default.svc.cluster.local:27017", +messy-hydra-mongodb-2 "primary" : "messy-hydra-mongodb-0.messy-hydra-mongodb.default.svc.cluster.local:27017", +messy-hydra-mongodb-0 "primary" : "messy-hydra-mongodb-0.messy-hydra-mongodb.default.svc.cluster.local:27017", + +``` + +Kill the primary and watch as a new master getting elected. + +```console +$ kubectl delete pod $RELEASE_NAME-mongodb-replicaset-0 + +pod "messy-hydra-mongodb-0" deleted +``` + +Delete all pods and let the statefulset controller bring it up. + +```console +$ kubectl delete po -l "app=mongodb-replicaset,release=$RELEASE_NAME" +$ kubectl get po --watch-only +NAME READY STATUS RESTARTS AGE +messy-hydra-mongodb-0 0/1 Pending 0 0s +messy-hydra-mongodb-0 0/1 Pending 0 0s +messy-hydra-mongodb-0 0/1 Pending 0 7s +messy-hydra-mongodb-0 0/1 Init:0/2 0 7s +messy-hydra-mongodb-0 0/1 Init:1/2 0 27s +messy-hydra-mongodb-0 0/1 Init:1/2 0 28s +messy-hydra-mongodb-0 0/1 PodInitializing 0 31s +messy-hydra-mongodb-0 0/1 Running 0 32s +messy-hydra-mongodb-0 1/1 Running 0 37s +messy-hydra-mongodb-1 0/1 Pending 0 0s +messy-hydra-mongodb-1 0/1 Pending 0 0s +messy-hydra-mongodb-1 0/1 Init:0/2 0 0s +messy-hydra-mongodb-1 0/1 Init:1/2 0 20s +messy-hydra-mongodb-1 0/1 Init:1/2 0 21s +messy-hydra-mongodb-1 0/1 PodInitializing 0 24s +messy-hydra-mongodb-1 0/1 Running 0 25s +messy-hydra-mongodb-1 1/1 Running 0 30s +messy-hydra-mongodb-2 0/1 Pending 0 0s +messy-hydra-mongodb-2 0/1 Pending 0 0s +messy-hydra-mongodb-2 0/1 Init:0/2 0 0s +messy-hydra-mongodb-2 0/1 Init:1/2 0 21s +messy-hydra-mongodb-2 0/1 Init:1/2 0 22s +messy-hydra-mongodb-2 0/1 PodInitializing 0 25s +messy-hydra-mongodb-2 0/1 Running 0 26s +messy-hydra-mongodb-2 1/1 Running 0 30s + + +... +messy-hydra-mongodb-0 "primary" : "messy-hydra-mongodb-0.messy-hydra-mongodb.default.svc.cluster.local:27017", +messy-hydra-mongodb-1 "primary" : "messy-hydra-mongodb-0.messy-hydra-mongodb.default.svc.cluster.local:27017", +messy-hydra-mongodb-2 "primary" : "messy-hydra-mongodb-0.messy-hydra-mongodb.default.svc.cluster.local:27017", +``` + +Check the previously inserted key: + +```console +$ kubectl exec $RELEASE_NAME-mongodb-replicaset-1 -- mongo --eval="rs.slaveOk(); db.test.find({key1:{\$exists:true}}).forEach(printjson)" + +MongoDB shell version: 3.6.3 +connecting to: mongodb://127.0.0.1:27017 +{ "_id" : ObjectId("57b180b1a7311d08f2bfb617"), "key1" : "value1" } +``` + +### Scaling + +Scaling should be managed by `helm upgrade`, which is the recommended way. + +### Indexes and Maintenance + +You can run Mongo in standalone mode and execute Javascript code on each replica at initContainer time using `initMongodStandalone`. +This allows you to create indexes on replicasets following [best practices](https://docs.mongodb.com/manual/tutorial/build-indexes-on-replica-sets/). + +#### Example: Creating Indexes + +```js +initMongodStandalone: |+ + db = db.getSiblingDB("mydb") + db.my_users.createIndex({email: 1}) +``` + +Tail the logs to debug running indexes or to follow their progress + +```sh +kubectl exec -it $RELEASE-mongodb-replicaset-0 -c bootstrap -- tail -f /work-dir/log.txt +``` + +### Migrate existing ReplicaSets into Kubernetes +If you have an existing ReplicaSet that currently is deployed outside of Kubernetes and want to move it into a cluster you can do so by using the `skipInitialization` flag. + +First set the `skipInitialization` variable to `true` in values.yaml and install the Helm chart. That way you end up with uninitialized MongoDB pods that can be added to the existing ReplicaSet. + +Now take care of realizing the DNS correct resolution of all ReplicaSet members. In Kubernetes you can for example use an `ExternalName`. + +``` +apiVersion: v1 +kind: Service +metadata: + name: mongodb01 + namespace: mongo +spec: + type: ExternalName + externalName: mongodb01.mydomain.com +``` + +If you also put each StatefulSet member behind a loadbalancer the ReplicaSet members outside of the cluster will also be able to reach the pods inside the cluster. + +``` +apiVersion: v1 +kind: Service +metadata: + name: mongodb-0 + namespace: mongo +spec: + selector: + statefulset.kubernetes.io/pod-name: mongodb-0 + ports: + - port: 27017 + targetPort: 27017 + type: LoadBalancer +``` + +Now all that is left to do is to put the LoadBalancer IP into the `/etc/hosts` file (or realize the DNS resolution through another way) +``` +1.2.3.4 mongodb-0 +5.6.7.8 mongodb-1 +``` + +With a setup like this each replicaset member can resolve the DNS entry of each other and you can just add the new pods to your existing MongoDB cluster as if they where just normal nodes. + +Of course you need to make sure to get your security settings right. Enforced TLS is a good idea in a setup like this. Also make sure that you activate auth and get the firewall settings right. + +Once you fully migrated remove the old nodes from the replicaset. diff --git a/terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/test.sh b/terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/test.sh new file mode 100755 index 0000000..0b7fd76 --- /dev/null +++ b/terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/test.sh @@ -0,0 +1,48 @@ +#! /bin/bash + +# Copyright 2016 The Kubernetes Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +NS="${RELEASE_NAMESPACE:-default}" +POD_NAME="${RELEASE_NAME:-mongo}-mongodb-replicaset" + +MONGOCACRT=/ca/tls.crt +MONGOPEM=/work-dir/mongo.pem +if [ -f $MONGOPEM ]; then + MONGOARGS="--ssl --sslCAFile $MONGOCACRT --sslPEMKeyFile $MONGOPEM" +fi + +for i in $(seq 0 2); do + pod="${POD_NAME}-$i" + kubectl exec --namespace $NS $pod -- sh -c 'mongo '"$MONGOARGS"' --eval="printjson(rs.isMaster())"' | grep '"ismaster" : true' + + if [ $? -eq 0 ]; then + echo "Found master: $pod" + MASTER=$pod + break + fi +done + +kubectl exec --namespace $NS $MASTER -- mongo "$MONGOARGS" --eval='printjson(db.test.insert({"status": "success"}))' + +# TODO: find maximum duration to wait for slaves to be up-to-date with master. +sleep 2 + +for i in $(seq 0 2); do + pod="${POD_NAME}-$i" + if [[ $pod != $MASTER ]]; then + echo "Reading from slave: $pod" + kubectl exec --namespace $NS $pod -- mongo "$MONGOARGS" --eval='rs.slaveOk(); db.test.find().forEach(printjson)' + fi +done diff --git a/terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/values.yaml b/terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/values.yaml new file mode 100644 index 0000000..c9b8799 --- /dev/null +++ b/terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/values.yaml @@ -0,0 +1,198 @@ +# Override the name of the chart, which in turn changes the name of the containers, services etc. +nameOverride: "" +fullnameOverride: "" + +# Override the deployment namespace +global: +# namespaceOverride: "" + +replicas: 3 +port: 27017 + +## Setting this will skip the replicaset and user creation process during bootstrapping +skipInitialization: false + +replicaSetName: rs0 + +podDisruptionBudget: {} + # maxUnavailable: 1 + # minAvailable: 2 + +# updateStrategy: + # type: RollingUpdate + +auth: + enabled: false + existingKeySecret: "" + existingAdminSecret: "" + existingMetricsSecret: "" + # adminUser: username + # adminPassword: password + # metricsUser: metrics + # metricsPassword: password + # key: keycontent + +## Optionally specify an array of imagePullSecrets. +## Secrets must be manually created in the namespace. +## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ +## +imagePullSecrets: [] +# - myRegistrKeySecretName + +# Specs for the Docker image for the init container that establishes the replica set +installImage: + repository: unguiculus/mongodb-install + tag: 0.7 + pullPolicy: IfNotPresent + +# Specs for the Docker image for the copyConfig init container +copyConfigImage: + repository: busybox + tag: 1.29.3 + pullPolicy: IfNotPresent + +# Specs for the MongoDB image +image: + repository: mongo + tag: 3.6 + pullPolicy: IfNotPresent + +# Additional environment variables to be set in the container +extraVars: {} +# - name: TCMALLOC_AGGRESSIVE_DECOMMIT +# value: "true" + +# Prometheus Metrics Exporter +metrics: + enabled: false + image: + repository: bitnami/mongodb-exporter + tag: 0.11.0 + pullPolicy: IfNotPresent + port: 9216 + path: "/metrics" + prometheusServiceDiscovery: true + resources: {} + securityContext: + enabled: true + runAsUser: 1001 + +# Annotations to be added to MongoDB pods +podAnnotations: {} + +securityContext: + enabled: true + runAsUser: 999 + fsGroup: 999 + runAsNonRoot: true + +init: + resources: {} + timeout: 900 + +resources: {} +# limits: +# cpu: 500m +# memory: 512Mi +# requests: +# cpu: 100m +# memory: 256Mi + +## Node selector +## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector +nodeSelector: {} + +affinity: {} + +tolerations: [] + +# Additional containers to add to the StatefulSet +extraContainers: [] +# - name: mongo-sidecar +# image: stefanprodan/mgob:latest +# ports: +# - name: sidecar-http +# containerPort: 8090 +# volumeMounts: +# - mountPath: /config/test1.yml +# name: mgob-config +# subPath: test1.yml +# - name: mgob-storage +# mountPath: /storage + +extraLabels: {} + +## Additional volumes which can be used by sidecar containers +extraVolumes: [] +# - name: mgob-config +# configMap: +# name: mgob-config + +priorityClassName: "" + +persistentVolume: + enabled: true + ## mongodb-replicaset data Persistent Volume Storage Class + ## If defined, storageClassName: + ## If set to "-", storageClassName: "", which disables dynamic provisioning + ## If undefined (the default) or set to null, no storageClassName spec is + ## set, choosing the default provisioner. (gp2 on AWS, standard on + ## GKE, AWS & OpenStack) + ## + storageClass: "" + accessModes: + - ReadWriteOnce + size: 10Gi + annotations: {} + +# Annotations to be added to the secrets +secretAnnotations: {} + +# Annotations to be added to the service +serviceAnnotations: {} + +# Annotations to be added to the statefulSet +statefulSetAnnotations: {} + +terminationGracePeriodSeconds: 30 + +tls: + # Enable or disable MongoDB TLS support + enabled: false + # Set the SSL operation mode (disabled|allowSSL|preferSSL|requireSSL) + mode: requireSSL + # Please generate your own TLS CA by generating it via: + # $ openssl genrsa -out ca.key 2048 + # $ openssl req -x509 -new -nodes -key ca.key -days 10000 -out ca.crt -subj "/CN=mydomain.com" + # After that you can base64 encode it and paste it here: + # $ cat ca.key | base64 -w0 + # cacert: + # cakey: + +# Entries for the MongoDB config file +configmap: {} + +# Javascript code to execute on each replica at initContainer time +# This is the recommended way to create indexes on replicasets. +# Below is an example that creates indexes in foreground on each replica in standalone mode. +# ref: https://docs.mongodb.com/manual/tutorial/build-indexes-on-replica-sets/ +# initMongodStandalone: |+ +# db = db.getSiblingDB("mydb") +# db.my_users.createIndex({email: 1}) +initMongodStandalone: "" + +# Readiness probe +readinessProbe: + initialDelaySeconds: 5 + timeoutSeconds: 1 + failureThreshold: 3 + periodSeconds: 10 + successThreshold: 1 + +# Liveness probe +livenessProbe: + initialDelaySeconds: 30 + timeoutSeconds: 5 + failureThreshold: 3 + periodSeconds: 10 + successThreshold: 1 From dc595e0b74051e15cfcdd6d16e791ff6678b9c18 Mon Sep 17 00:00:00 2001 From: Tim Anderegg Date: Tue, 3 Mar 2026 13:38:02 -0500 Subject: [PATCH 3/3] Add new chart files. --- .../charts/mongodb-replicaset/OWNERS | 6 + .../mongodb-replicaset/ci/default-values.yaml | 1 + .../mongodb-replicaset/ci/metrics-values.yaml | 10 + .../mongodb-replicaset/ci/tls-values.yaml | 10 + .../mongodb-replicaset/init/on-start.sh | 226 +++++++++++ .../mongodb-replicaset/install/.dockerignore | 1 + .../mongodb-replicaset/install/Dockerfile | 25 ++ .../mongodb-replicaset/install/Makefile | 27 ++ .../mongodb-replicaset/install/install.sh | 36 ++ .../mongodb-replicaset/templates/NOTES.txt | 14 + .../mongodb-replicaset/templates/_helpers.tpl | 93 +++++ .../templates/mongodb-admin-secret.yaml | 23 ++ .../templates/mongodb-ca-secret.yaml | 19 + .../templates/mongodb-init-configmap.yaml | 21 + .../templates/mongodb-keyfile-secret.yaml | 22 ++ .../templates/mongodb-metrics-secret.yaml | 23 ++ .../templates/mongodb-mongodb-configmap.yaml | 16 + .../mongodb-poddisruptionbudget.yaml | 21 + .../templates/mongodb-service-client.yaml | 33 ++ .../templates/mongodb-service.yaml | 26 ++ .../templates/mongodb-statefulset.yaml | 367 ++++++++++++++++++ .../tests/mongodb-up-test-configmap.yaml | 13 + .../templates/tests/mongodb-up-test-pod.yaml | 80 ++++ .../tests/mongodb-up-test.sh | 120 ++++++ 24 files changed, 1233 insertions(+) create mode 100644 terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/OWNERS create mode 100644 terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/ci/default-values.yaml create mode 100644 terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/ci/metrics-values.yaml create mode 100644 terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/ci/tls-values.yaml create mode 100644 terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/init/on-start.sh create mode 100644 terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/install/.dockerignore create mode 100644 terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/install/Dockerfile create mode 100644 terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/install/Makefile create mode 100644 terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/install/install.sh create mode 100644 terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/templates/NOTES.txt create mode 100644 terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/templates/_helpers.tpl create mode 100644 terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/templates/mongodb-admin-secret.yaml create mode 100644 terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/templates/mongodb-ca-secret.yaml create mode 100644 terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/templates/mongodb-init-configmap.yaml create mode 100644 terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/templates/mongodb-keyfile-secret.yaml create mode 100644 terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/templates/mongodb-metrics-secret.yaml create mode 100644 terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/templates/mongodb-mongodb-configmap.yaml create mode 100644 terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/templates/mongodb-poddisruptionbudget.yaml create mode 100644 terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/templates/mongodb-service-client.yaml create mode 100644 terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/templates/mongodb-service.yaml create mode 100644 terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/templates/mongodb-statefulset.yaml create mode 100644 terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/templates/tests/mongodb-up-test-configmap.yaml create mode 100644 terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/templates/tests/mongodb-up-test-pod.yaml create mode 100644 terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/tests/mongodb-up-test.sh diff --git a/terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/OWNERS b/terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/OWNERS new file mode 100644 index 0000000..1e6a850 --- /dev/null +++ b/terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/OWNERS @@ -0,0 +1,6 @@ +approvers: + - unguiculus + - steven-sheehy +reviewers: + - unguiculus + - steven-sheehy diff --git a/terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/ci/default-values.yaml b/terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/ci/default-values.yaml new file mode 100644 index 0000000..a8bad27 --- /dev/null +++ b/terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/ci/default-values.yaml @@ -0,0 +1 @@ +# No config change. Just use defaults. diff --git a/terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/ci/metrics-values.yaml b/terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/ci/metrics-values.yaml new file mode 100644 index 0000000..df64aca --- /dev/null +++ b/terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/ci/metrics-values.yaml @@ -0,0 +1,10 @@ +auth: + enabled: true + adminUser: username + adminPassword: password + metricsUser: metrics + metricsPassword: password + key: keycontent + +metrics: + enabled: true diff --git a/terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/ci/tls-values.yaml b/terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/ci/tls-values.yaml new file mode 100644 index 0000000..043d7ac --- /dev/null +++ b/terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/ci/tls-values.yaml @@ -0,0 +1,10 @@ +tls: + # Enable or disable MongoDB TLS support + enabled: true + # Please generate your own TLS CA by generating it via: + # $ openssl genrsa -out ca.key 2048 + # $ openssl req -x509 -new -nodes -key ca.key -days 10000 -out ca.crt -subj "/CN=mydomain.com" + # After that you can base64 encode it and paste it here: + # $ cat ca.key | base64 -w0 + cacert: "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUNxakNDQVpJQ0NRQ1I1aXNNQlRmQzdUQU5CZ2txaGtpRzl3MEJBUXNGQURBWE1SVXdFd1lEVlFRRERBeHQKZVdSdmJXRnBiaTVqYjIwd0hoY05NVGt4TVRFeU1EZ3hOakUwV2hjTk5EY3dNek13TURneE5qRTBXakFYTVJVdwpFd1lEVlFRRERBeHRlV1J2YldGcGJpNWpiMjB3Z2dFaU1BMEdDU3FHU0liM0RRRUJBUVVBQTRJQkR3QXdnZ0VLCkFvSUJBUUNwM0UrdVpWanhaY3BYNUFCbEtRa2crZjFmSnJzR1JJNVQrMzcyMkIvYnRyTVo4M3FyRTg2RFdEYXEKN0k1YTdlOGFVTGt2ZVpsaW02aWxsUW5CTHJPVUtVZ3R1OWZINlZydlBuMTl3UDFibEMvU0NWZHoxemNSUWlJWQpOMVVWN2VGaWUzdjhiNXVRM2RFcVBPV2FMM0w2N0Q1T0lDb043Z21QL2QwVVBaWjNHdDJLNTZsNXBzY1h4OGYwCkd3ZWdSRGpiVnZmc2dUSW50dEJ6SGh6c0JENUxON054aDd5RWVacW5admtuTDg5S2JZUEFPUk82N3NKUlBhWHMKUDhuVDhqalFJaGlRSUZDNTVXN3JrZ1hid1Znajdwb0kyby9XSDM4WXZ6TG1OVnMyOThYUDZmUXhCQ0NwMmFjRgpkOTVQRjZmbFVJeW9RNGRuOUF5UlpRa0owdlpMQWdNQkFBRXdEUVlKS29aSWh2Y05BUUVMQlFBRGdnRUJBS21XCjY2SlB4V0E4MVlYTEZtclFrdmM2NE9ycFJXTHJtNHFqaFREREtvVzY1V291MzNyOEVVQktzQ2FQOHNWWXhobFQKaWhGcDhIemNqTXpWRjFqU3ZiT0c5UnhrSG16ZEIvL3FwMDdTVFp0S2R1cThad2RVdnh1Z1FXSFNzOHA4YVNHUAowNDlkSDBqUnpEZklyVGp4Z3ZNOUJlWmorTkdqT1FyUGRvQVBKeTl2THowZmYya1gzVjJadTFDWnNnbDNWUXFsCjRsNzB3azFuVk5tTXY4Nnl5cUZXaWFRTWhuSXFjKzBwYUJaRjJFSGNpSExuZWcweVVTZVN4REsrUkk4SE9mT3oKNVFpUHpqSGs1b3czd252NDhQWVJMODdLTWJtRzF0eThyRHMxUlVGWkZueGxHd0t4UmRmckt3aHJJbVRBT2N4Vwo5bVhCU3ZzY3RjM2tIZTRIVFdRPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg==" + cakey: "LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFb3dJQkFBS0NBUUVBcWR4UHJtVlk4V1hLVitRQVpTa0pJUG45WHlhN0JrU09VL3QrOXRnZjI3YXpHZk42CnF4UE9nMWcycXV5T1d1M3ZHbEM1TDNtWllwdW9wWlVKd1M2emxDbElMYnZYeCtsYTd6NTlmY0Q5VzVRdjBnbFgKYzljM0VVSWlHRGRWRmUzaFludDcvRytia04zUktqemxtaTl5K3V3K1RpQXFEZTRKai8zZEZEMldkeHJkaXVlcAplYWJIRjhmSDlCc0hvRVE0MjFiMzdJRXlKN2JRY3g0YzdBUStTemV6Y1llOGhIbWFwMmI1SnkvUFNtMkR3RGtUCnV1N0NVVDJsN0QvSjAvSTQwQ0lZa0NCUXVlVnU2NUlGMjhGWUkrNmFDTnFQMWg5L0dMOHk1alZiTnZmRnorbjAKTVFRZ3FkbW5CWGZlVHhlbjVWQ01xRU9IWi9RTWtXVUpDZEwyU3dJREFRQUJBb0lCQVFDVWM3eWNBVzFMaEpmawpXcHRSemh4eFdxcnJSeEU3ZUIwZ0h2UW16bHFCanRwVyt1bWhyT3pXOC9qTFIzVmUyUVlZYktaOGJIejJwbTR0ClVPVTJsaGRTalFYTkdwZUsyMUtqTjIwN3c3aHFHa2YwL0Q4WE9lZWh5TGU5akZacmxQeGZNdWI0aDU1aGJNdUsKYTdDTElaOE8xL3ZZRWRwUFZGTzlLYlRYSk1CbEZJUERUaFJvR2RCTEFkREZNbzcrUnZYSFRUcXdyWmxDbWRDbgp5eld3WkhIQUZhdEdGWU9ybXcxdlZZY3h0OXk5c0FVZDBrVTQza05jVHVHR0MwMGh1QlZMcW9JZU9mMG12TDB0Ckg0S0d6LzBicGp4NFpoWlNKazd3ZkFsQ0xGL1N5YzVJOEJXWWNCb05Jc0RSbDdDUmpDVUoxYVNBNVNYNzZ2SVoKSlhnRWEyV3hBb0dCQU50M0pDRGtycjNXRmJ3cW1SZ2ZhUVV0UE1FVnZlVnJvQmRqZTBZVFFNbENlNTV2QU1uNQpadEFKQTVKTmxKN2VZRkVEa0JrVURJSDFDM3hlZHVWbEREWXpESzRpR0V1Wk8wVDNERFN3aks2cExsZ3JBN0QyCmZnS29ubVdGck5JdTI4UW1MNHhmcjUrWW9SNUo0L05QdFdWOWwwZk1NOHEwSTd5SVRNODlsZWlqQW9HQkFNWWoKTHk3VER1MWVJVWkrQXJFNTJFMEkwTVJPNWNLS2hxdGxJMnpCZkZlWm5LYWdwbnhCMTMxbi9wcGg0Wm1IYnMzZQpxOXBSb0RJT0h4cm5NOWFCa1JBTHJHNjBMeXF3eU5NNW1JemkvQytJK2RVOG55ZXIvZVNNRTNtdlFzbmpVcEhtClRtTjRrM0l4RWtqRnhCazVndFNlNlA5U0UyOFd6eVZoOGlkZHRjNDVBb0dBYzcwWFBvbWJaZDM3UkdxcXBrQWEKWUhLRThjY0hpSEFEMDVIUk54bDhOeWRxamhrNEwwdnAzcGlDVzZ1eVR6NHpTVVk1dmlBR29KcWNYaEJyWDNxMAp2L2lZSFZVNXZ0U21ueTR5TDY5VDRlQ3k0aWg5SDl3K2hDUnN0Rm1VMUp1RnBxSUV2V0RRKzdmQWNIckRUbE9nCjlFOFJjdm5MN29DbHdBMlpoRW1VUDBVQ2dZQWFhdUtGbWJwcHg1MGtkOEVnSkJoRTNTSUlxb1JUMWVoeXZiOWwKWnI3UFp6bk50YW04ODRKcHhBM2NRNlN5dGEzK1lPd0U1ZEU0RzAzbVptRXcvb0Y2NURPUFp4TEszRnRLWG1tSwpqMUVVZld6aUUzMGM2ditsRTFBZGIxSzJYRXJNRFNyeWRFY2tlSXA1alhUQjhEc1RZa1NxbGlUbE1PTlpscCtVCnhCZlRjUUtCZ0RoZHo4VjU1TzdNc0dyRVlQeGhoK0U0bklLb3BRc0RlNi9QdWRRVlJMRlNwVGpLNWlKcTF2RnIKajFyNDFCNFp0cjBYNGd6MzhrSUpwZGNvNUFxU25zVENreHhnYXh3RTNzVmlqNGZZRWlteDc3TS84VkZVbDZwLwphNmdBbFh2WHFaYmFvTGU3ekM2RXVZWjFtUzJGMVd4UE9KRzZpakFiMVNIQjVPOGFWdFR3Ci0tLS0tRU5EIFJTQSBQUklWQVRFIEtFWS0tLS0tCg==" diff --git a/terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/init/on-start.sh b/terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/init/on-start.sh new file mode 100644 index 0000000..4b14c7e --- /dev/null +++ b/terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/init/on-start.sh @@ -0,0 +1,226 @@ +#!/usr/bin/env bash + +# Copyright 2018 The Kubernetes Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e pipefail + +port=27017 +replica_set="$REPLICA_SET" +script_name=${0##*/} +SECONDS=0 +timeout="${TIMEOUT:-900}" +tls_mode="${TLS_MODE}" + +if [[ "$AUTH" == "true" ]]; then + admin_user="$ADMIN_USER" + admin_password="$ADMIN_PASSWORD" + admin_creds=(-u "$admin_user" -p "$admin_password") + if [[ "$METRICS" == "true" ]]; then + metrics_user="$METRICS_USER" + metrics_password="$METRICS_PASSWORD" + fi + auth_args=("--auth" "--keyFile=/data/configdb/key.txt") +fi + +log() { + local msg="$1" + local timestamp + timestamp=$(date --iso-8601=ns) + echo "[$timestamp] [$script_name] $msg" 2>&1 | tee -a /work-dir/log.txt 1>&2 +} + +retry_until() { + local host="${1}" + local command="${2}" + local expected="${3}" + local creds=("${admin_creds[@]}") + + # Don't need credentials for admin user creation and pings that run on localhost + if [[ "${host}" =~ ^localhost ]]; then + creds=() + fi + + until [[ $(mongo admin --host "${host}" "${creds[@]}" "${ssl_args[@]}" --quiet --eval "${command}" | tail -n1) == "${expected}" ]]; do + sleep 1 + + if (! ps "${pid}" &>/dev/null); then + log "mongod shutdown unexpectedly" + exit 1 + fi + if [[ "${SECONDS}" -ge "${timeout}" ]]; then + log "Timed out after ${timeout}s attempting to bootstrap mongod" + exit 1 + fi + + log "Retrying ${command} on ${host}" + done +} + +shutdown_mongo() { + local host="${1:-localhost}" + local args='force: true' + log "Shutting down MongoDB ($args)..." + if (! mongo admin --host "${host}" "${admin_creds[@]}" "${ssl_args[@]}" --eval "db.shutdownServer({$args})"); then + log "db.shutdownServer() failed, sending the terminate signal" + kill -TERM "${pid}" + fi +} + +init_mongod_standalone() { + if [[ ! -f /init/initMongodStandalone.js ]]; then + log "Skipping init mongod standalone script" + return 0 + elif [[ -z "$(ls -1A /data/db)" ]]; then + log "mongod standalone script currently not supported on initial install" + return 0 + fi + + local port="27018" + log "Starting a MongoDB instance as standalone..." + mongod --config /data/configdb/mongod.conf --dbpath=/data/db "${auth_args[@]}" "${ssl_server_args[@]}" --port "${port}" --bind_ip=0.0.0.0 2>&1 | tee -a /work-dir/log.txt 1>&2 & + export pid=$! + trap shutdown_mongo EXIT + log "Waiting for MongoDB to be ready..." + retry_until "localhost:${port}" "db.adminCommand('ping').ok" "1" + log "Running init js script on standalone mongod" + mongo admin --port "${port}" "${admin_creds[@]}" "${ssl_args[@]}" /init/initMongodStandalone.js + shutdown_mongo "localhost:${port}" +} + +my_hostname=$(hostname) +log "Bootstrapping MongoDB replica set member: $my_hostname" + +log "Reading standard input..." +while read -ra line; do + if [[ "${line}" == *"${my_hostname}"* ]]; then + service_name="$line" + fi + peers=("${peers[@]}" "$line") +done + +# Generate the ca cert +ca_crt=/data/configdb/tls.crt +if [ -f "$ca_crt" ]; then + log "Generating certificate" + ca_key=/data/configdb/tls.key + pem=/work-dir/mongo.pem + ssl_args=(--ssl --sslCAFile "$ca_crt" --sslPEMKeyFile "$pem") + ssl_server_args=(--sslMode "$tls_mode" --sslCAFile "$ca_crt" --sslPEMKeyFile "$pem") + +# Move into /work-dir +pushd /work-dir + +cat >openssl.cnf < $pem + rm mongo.key mongo.crt +fi + +init_mongod_standalone + +if [[ "${SKIP_INIT}" == "true" ]]; then + log "Skipping initialization" + exit 0 +fi + +log "Peers: ${peers[*]}" +log "Starting a MongoDB replica" +mongod --config /data/configdb/mongod.conf --dbpath=/data/db --replSet="$replica_set" --port="${port}" "${auth_args[@]}" "${ssl_server_args[@]}" --bind_ip=0.0.0.0 2>&1 | tee -a /work-dir/log.txt 1>&2 & +pid=$! +trap shutdown_mongo EXIT + +log "Waiting for MongoDB to be ready..." +retry_until "localhost" "db.adminCommand('ping').ok" "1" +log "Initialized." + +# try to find a master +for peer in "${peers[@]}"; do + log "Checking if ${peer} is primary" + # Check rs.status() first since it could be in primary catch up mode which db.isMaster() doesn't show + if [[ $(mongo admin --host "${peer}" "${admin_creds[@]}" "${ssl_args[@]}" --quiet --eval "rs.status().myState" | tail -n1) == "1" ]]; then + retry_until "${peer}" "db.isMaster().ismaster" "true" + log "Found primary: ${peer}" + primary="${peer}" + break + fi +done + +if [[ "${primary}" = "${service_name}" ]]; then + log "This replica is already PRIMARY" +elif [[ -n "${primary}" ]]; then + if [[ $(mongo admin --host "${primary}" "${admin_creds[@]}" "${ssl_args[@]}" --quiet --eval "rs.conf().members.findIndex(m => m.host == '${service_name}:${port}')" | tail -n1) == "-1" ]]; then + log "Adding myself (${service_name}) to replica set..." + if (mongo admin --host "${primary}" "${admin_creds[@]}" "${ssl_args[@]}" --eval "rs.add('${service_name}')" | grep 'Quorum check failed'); then + log 'Quorum check failed, unable to join replicaset. Exiting prematurely.' + exit 1 + fi + fi + + sleep 3 + log 'Waiting for replica to reach SECONDARY state...' + retry_until "${service_name}" "rs.status().myState" "2" + log '✓ Replica reached SECONDARY state.' + +elif (mongo "${ssl_args[@]}" --eval "rs.status()" | grep "no replset config has been received"); then + log "Initiating a new replica set with myself ($service_name)..." + mongo "${ssl_args[@]}" --eval "rs.initiate({'_id': '$replica_set', 'members': [{'_id': 0, 'host': '$service_name'}]})" + + sleep 3 + log 'Waiting for replica to reach PRIMARY state...' + retry_until "localhost" "db.isMaster().ismaster" "true" + primary="${service_name}" + log '✓ Replica reached PRIMARY state.' + + if [[ "${AUTH}" == "true" ]]; then + log "Creating admin user..." + mongo admin "${ssl_args[@]}" --eval "db.createUser({user: '${admin_user}', pwd: '${admin_password}', roles: [{role: 'root', db: 'admin'}]})" + fi +fi + +# User creation +if [[ -n "${primary}" && "$AUTH" == "true" && "$METRICS" == "true" ]]; then + metric_user_count=$(mongo admin --host "${primary}" "${admin_creds[@]}" "${ssl_args[@]}" --eval "db.system.users.find({user: '${metrics_user}'}).count()" --quiet | tail -n1) + if [[ "${metric_user_count}" == "0" ]]; then + log "Creating clusterMonitor user..." + mongo admin --host "${primary}" "${admin_creds[@]}" "${ssl_args[@]}" --eval "db.createUser({user: '${metrics_user}', pwd: '${metrics_password}', roles: [{role: 'clusterMonitor', db: 'admin'}, {role: 'read', db: 'local'}]})" + fi +fi + +log "MongoDB bootstrap complete" +exit 0 + diff --git a/terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/install/.dockerignore b/terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/install/.dockerignore new file mode 100644 index 0000000..33ceb8f --- /dev/null +++ b/terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/install/.dockerignore @@ -0,0 +1 @@ +Makefile \ No newline at end of file diff --git a/terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/install/Dockerfile b/terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/install/Dockerfile new file mode 100644 index 0000000..9084f96 --- /dev/null +++ b/terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/install/Dockerfile @@ -0,0 +1,25 @@ +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM alpine:3.8 + +LABEL maintainer="Reinhard Nägele " + +RUN apk update && apk add bash openssl && wget -qO /peer-finder http://storage.googleapis.com/kubernetes-release/pets/peer-finder + +ENTRYPOINT ["/install.sh"] + +COPY install.sh / + +RUN chmod -c 755 /install.sh /peer-finder diff --git a/terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/install/Makefile b/terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/install/Makefile new file mode 100644 index 0000000..1a97085 --- /dev/null +++ b/terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/install/Makefile @@ -0,0 +1,27 @@ +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +all: push + +TAG = 0.7 +PREFIX = unguiculus/mongodb-install + +build: + docker build -t $(PREFIX):$(TAG) . + +push: build + docker push $(PREFIX):$(TAG) + +clean: + docker rmi $(PREFIX):$(TAG) diff --git a/terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/install/install.sh b/terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/install/install.sh new file mode 100644 index 0000000..3ca80c1 --- /dev/null +++ b/terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/install/install.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash + +# Copyright 2016 The Kubernetes Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This volume is assumed to exist and is shared with the peer-finder +# init container. It contains on-start/change configuration scripts. +WORKDIR_VOLUME="/work-dir" + +for i in "$@"; do + case "$i" in + -w=*|--work-dir=*) + WORKDIR_VOLUME="${i#*=}" + shift + ;; + *) + # unknown option + ;; + esac +done + +echo Installing config scripts into "${WORKDIR_VOLUME}" + +mkdir -p "${WORKDIR_VOLUME}" +cp /peer-finder "${WORKDIR_VOLUME}"/ diff --git a/terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/templates/NOTES.txt b/terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/templates/NOTES.txt new file mode 100644 index 0000000..f32be74 --- /dev/null +++ b/terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/templates/NOTES.txt @@ -0,0 +1,14 @@ +1. After the statefulset is created completely, one can check which instance is primary by running: + + $ for ((i = 0; i < {{ .Values.replicas }}; ++i)); do kubectl exec --namespace {{ template "mongodb-replicaset.namespace" . }} {{ template "mongodb-replicaset.fullname" . }}-$i -- sh -c 'mongo --eval="printjson(rs.isMaster())"'; done + +2. One can insert a key into the primary instance of the mongodb replica set by running the following: + MASTER_POD_NAME must be replaced with the name of the master found from the previous step. + + $ kubectl exec --namespace {{ template "mongodb-replicaset.namespace" . }} MASTER_POD_NAME -- mongo --eval="printjson(db.test.insert({key1: 'value1'}))" + +3. One can fetch the keys stored in the primary or any of the slave nodes in the following manner. + POD_NAME must be replaced by the name of the pod being queried. + + $ kubectl exec --namespace {{ template "mongodb-replicaset.namespace" . }} POD_NAME -- mongo --eval="rs.slaveOk(); db.test.find().forEach(printjson)" + diff --git a/terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/templates/_helpers.tpl b/terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/templates/_helpers.tpl new file mode 100644 index 0000000..d9204a7 --- /dev/null +++ b/terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/templates/_helpers.tpl @@ -0,0 +1,93 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "mongodb-replicaset.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "mongodb-replicaset.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "mongodb-replicaset.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create the name for the admin secret. +*/}} +{{- define "mongodb-replicaset.adminSecret" -}} + {{- if .Values.auth.existingAdminSecret -}} + {{- .Values.auth.existingAdminSecret -}} + {{- else -}} + {{- template "mongodb-replicaset.fullname" . -}}-admin + {{- end -}} +{{- end -}} + +{{- define "mongodb-replicaset.metricsSecret" -}} + {{- if .Values.auth.existingMetricsSecret -}} + {{- .Values.auth.existingMetricsSecret -}} + {{- else -}} + {{- template "mongodb-replicaset.fullname" . -}}-metrics + {{- end -}} +{{- end -}} + + +{{/* +Create the name for the key secret. +*/}} +{{- define "mongodb-replicaset.keySecret" -}} + {{- if .Values.auth.existingKeySecret -}} + {{- .Values.auth.existingKeySecret -}} + {{- else -}} + {{- template "mongodb-replicaset.fullname" . -}}-keyfile + {{- end -}} +{{- end -}} + +{{- define "mongodb-replicaset.connection-string" -}} + {{- $string := "" -}} + {{- if .Values.auth.enabled }} + {{- $string = printf "mongodb://$METRICS_USER:$METRICS_PASSWORD@localhost:%s" (.Values.port|toString) -}} + {{- else -}} + {{- $string = printf "mongodb://localhost:%s" (.Values.port|toString) -}} + {{- end -}} + + {{- if .Values.tls.enabled }} + {{- printf "%s/?ssl=true&tlsCertificateKeyFile=/work-dir/mongo.pem&tlsCAFile=/ca/tls.crt" $string | quote -}} + {{- else -}} + {{- printf $string | quote -}} + {{- end -}} +{{- end -}} + +{{/* +Allow the release namespace to be overridden for multi-namespace deployments in combined charts. +*/}} +{{- define "mongodb-replicaset.namespace" -}} + {{- if .Values.global -}} + {{- if .Values.global.namespaceOverride -}} + {{- .Values.global.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} diff --git a/terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/templates/mongodb-admin-secret.yaml b/terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/templates/mongodb-admin-secret.yaml new file mode 100644 index 0000000..a5dcbe9 --- /dev/null +++ b/terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/templates/mongodb-admin-secret.yaml @@ -0,0 +1,23 @@ +{{- if and (.Values.auth.enabled) (not .Values.auth.existingAdminSecret) -}} +apiVersion: v1 +kind: Secret +metadata: + labels: + app: {{ template "mongodb-replicaset.name" . }} + chart: {{ template "mongodb-replicaset.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +{{- if .Values.extraLabels }} +{{ toYaml .Values.extraLabels | indent 4 }} +{{- end }} +{{- if .Values.secretAnnotations }} + annotations: +{{ toYaml .Values.secretAnnotations | indent 4 }} +{{- end }} + name: {{ template "mongodb-replicaset.adminSecret" . }} + namespace: {{ template "mongodb-replicaset.namespace" . }} +type: Opaque +data: + user: {{ .Values.auth.adminUser | b64enc }} + password: {{ .Values.auth.adminPassword | b64enc }} +{{- end -}} diff --git a/terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/templates/mongodb-ca-secret.yaml b/terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/templates/mongodb-ca-secret.yaml new file mode 100644 index 0000000..5f2e44a --- /dev/null +++ b/terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/templates/mongodb-ca-secret.yaml @@ -0,0 +1,19 @@ +{{- if .Values.tls.enabled -}} +apiVersion: v1 +kind: Secret +type: kubernetes.io/tls +metadata: + labels: + app: {{ template "mongodb-replicaset.name" . }} + chart: {{ template "mongodb-replicaset.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +{{- if .Values.extraLabels }} +{{ toYaml .Values.extraLabels | indent 4 }} +{{- end }} + name: {{ template "mongodb-replicaset.fullname" . }}-ca + namespace: {{ template "mongodb-replicaset.namespace" . }} +data: + tls.key: {{ .Values.tls.cakey }} + tls.crt: {{ .Values.tls.cacert }} +{{- end -}} diff --git a/terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/templates/mongodb-init-configmap.yaml b/terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/templates/mongodb-init-configmap.yaml new file mode 100644 index 0000000..fc03c68 --- /dev/null +++ b/terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/templates/mongodb-init-configmap.yaml @@ -0,0 +1,21 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + app: {{ template "mongodb-replicaset.name" . }} + chart: {{ template "mongodb-replicaset.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +{{- if .Values.extraLabels }} +{{ toYaml .Values.extraLabels | indent 4 }} +{{- end }} + name: {{ template "mongodb-replicaset.fullname" . }}-init + namespace: {{ template "mongodb-replicaset.namespace" . }} +data: + on-start.sh: | +{{ .Files.Get "init/on-start.sh" | indent 4 }} +{{- if .Values.initMongodStandalone }} + initMongodStandalone.js: | +{{ .Values.initMongodStandalone | indent 4 }} +{{- end }} + diff --git a/terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/templates/mongodb-keyfile-secret.yaml b/terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/templates/mongodb-keyfile-secret.yaml new file mode 100644 index 0000000..f6a8f7b --- /dev/null +++ b/terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/templates/mongodb-keyfile-secret.yaml @@ -0,0 +1,22 @@ +{{- if and (.Values.auth.enabled) (not .Values.auth.existingKeySecret) -}} +apiVersion: v1 +kind: Secret +metadata: + labels: + app: {{ template "mongodb-replicaset.name" . }} + chart: {{ template "mongodb-replicaset.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +{{- if .Values.extraLabels }} +{{ toYaml .Values.extraLabels | indent 4 }} +{{- end }} +{{- if .Values.secretAnnotations }} + annotations: +{{ toYaml .Values.secretAnnotations | indent 4 }} +{{- end }} + name: {{ template "mongodb-replicaset.keySecret" . }} + namespace: {{ template "mongodb-replicaset.namespace" . }} +type: Opaque +data: + key.txt: {{ .Values.auth.key | b64enc }} +{{- end -}} diff --git a/terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/templates/mongodb-metrics-secret.yaml b/terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/templates/mongodb-metrics-secret.yaml new file mode 100644 index 0000000..0a4cfec --- /dev/null +++ b/terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/templates/mongodb-metrics-secret.yaml @@ -0,0 +1,23 @@ +{{- if and (.Values.auth.enabled) (not .Values.auth.existingMetricsSecret) (.Values.metrics.enabled) -}} +apiVersion: v1 +kind: Secret +metadata: + labels: + app: {{ template "mongodb-replicaset.name" . }} + chart: {{ template "mongodb-replicaset.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +{{- if .Values.extraLabels }} +{{ toYaml .Values.extraLabels | indent 4 }} +{{- end }} +{{- if .Values.secretAnnotations }} + annotations: +{{ toYaml .Values.secretAnnotations | indent 4 }} +{{- end }} + name: {{ template "mongodb-replicaset.metricsSecret" . }} + namespace: {{ template "mongodb-replicaset.namespace" . }} +type: Opaque +data: + user: {{ .Values.auth.metricsUser | b64enc }} + password: {{ .Values.auth.metricsPassword | b64enc }} +{{- end -}} diff --git a/terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/templates/mongodb-mongodb-configmap.yaml b/terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/templates/mongodb-mongodb-configmap.yaml new file mode 100644 index 0000000..08d2b5a --- /dev/null +++ b/terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/templates/mongodb-mongodb-configmap.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + app: {{ template "mongodb-replicaset.name" . }} + chart: {{ template "mongodb-replicaset.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +{{- if .Values.extraLabels }} +{{ toYaml .Values.extraLabels | indent 4 }} +{{- end }} + name: {{ template "mongodb-replicaset.fullname" . }}-mongodb + namespace: {{ template "mongodb-replicaset.namespace" . }} +data: + mongod.conf: | +{{ toYaml .Values.configmap | indent 4 }} diff --git a/terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/templates/mongodb-poddisruptionbudget.yaml b/terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/templates/mongodb-poddisruptionbudget.yaml new file mode 100644 index 0000000..52f08e3 --- /dev/null +++ b/terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/templates/mongodb-poddisruptionbudget.yaml @@ -0,0 +1,21 @@ +{{- if .Values.podDisruptionBudget -}} +apiVersion: policy/v1beta1 +kind: PodDisruptionBudget +metadata: + labels: + app: {{ template "mongodb-replicaset.name" . }} + chart: {{ template "mongodb-replicaset.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +{{- if .Values.extraLabels }} +{{ toYaml .Values.extraLabels | indent 4 }} +{{- end }} + name: {{ template "mongodb-replicaset.fullname" . }} + namespace: {{ template "mongodb-replicaset.namespace" . }} +spec: + selector: + matchLabels: + app: {{ template "mongodb-replicaset.name" . }} + release: {{ .Release.Name }} +{{ toYaml .Values.podDisruptionBudget | indent 2 }} +{{- end -}} diff --git a/terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/templates/mongodb-service-client.yaml b/terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/templates/mongodb-service-client.yaml new file mode 100644 index 0000000..af1dfdd --- /dev/null +++ b/terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/templates/mongodb-service-client.yaml @@ -0,0 +1,33 @@ +# A headless service for client applications to use +apiVersion: v1 +kind: Service +metadata: + annotations: + {{- if .Values.serviceAnnotations }} +{{ toYaml .Values.serviceAnnotations | indent 4 }} + {{- end }} + labels: + app: {{ template "mongodb-replicaset.name" . }} + chart: {{ template "mongodb-replicaset.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +{{- if .Values.extraLabels }} +{{ toYaml .Values.extraLabels | indent 4 }} +{{- end }} + name: {{ template "mongodb-replicaset.fullname" . }}-client + namespace: {{ template "mongodb-replicaset.namespace" . }} +spec: + type: ClusterIP + clusterIP: None + ports: + - name: mongodb + port: {{ .Values.port }} +{{- if .Values.metrics.enabled }} + - name: metrics + port: {{ .Values.metrics.port }} + targetPort: metrics +{{- end }} + selector: + app: {{ template "mongodb-replicaset.name" . }} + release: {{ .Release.Name }} + diff --git a/terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/templates/mongodb-service.yaml b/terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/templates/mongodb-service.yaml new file mode 100644 index 0000000..0bb2d18 --- /dev/null +++ b/terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/templates/mongodb-service.yaml @@ -0,0 +1,26 @@ +# A headless service to create DNS records for discovery purposes. Use the -client service to connect applications +apiVersion: v1 +kind: Service +metadata: + annotations: + service.alpha.kubernetes.io/tolerate-unready-endpoints: "true" + labels: + app: {{ template "mongodb-replicaset.name" . }} + chart: {{ template "mongodb-replicaset.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +{{- if .Values.extraLabels }} +{{ toYaml .Values.extraLabels | indent 4 }} +{{- end }} + name: {{ template "mongodb-replicaset.fullname" . }} + namespace: {{ template "mongodb-replicaset.namespace" . }} +spec: + type: ClusterIP + clusterIP: None + ports: + - name: mongodb + port: {{ .Values.port }} + publishNotReadyAddresses: true + selector: + app: {{ template "mongodb-replicaset.name" . }} + release: {{ .Release.Name }} diff --git a/terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/templates/mongodb-statefulset.yaml b/terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/templates/mongodb-statefulset.yaml new file mode 100644 index 0000000..9afeda0 --- /dev/null +++ b/terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/templates/mongodb-statefulset.yaml @@ -0,0 +1,367 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + labels: + app: {{ template "mongodb-replicaset.name" . }} + chart: {{ template "mongodb-replicaset.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +{{- if .Values.extraLabels }} +{{ toYaml .Values.extraLabels | indent 4 }} +{{- end }} +{{- if .Values.statefulSetAnnotations }} + annotations: +{{ toYaml .Values.statefulSetAnnotations | indent 4 }} +{{- end }} + name: {{ template "mongodb-replicaset.fullname" . }} + namespace: {{ template "mongodb-replicaset.namespace" . }} +spec: +{{- if .Values.updateStrategy }} + updateStrategy: +{{ toYaml .Values.updateStrategy | indent 4 }} +{{- end }} + selector: + matchLabels: + app: {{ template "mongodb-replicaset.name" . }} + release: {{ .Release.Name }} + serviceName: {{ template "mongodb-replicaset.fullname" . }} + replicas: {{ .Values.replicas }} + template: + metadata: + labels: + app: {{ template "mongodb-replicaset.name" . }} + release: {{ .Release.Name }} +{{- if .Values.extraLabels }} +{{ toYaml .Values.extraLabels | indent 8 }} +{{- end }} + annotations: + checksum/config: {{ include (print $.Template.BasePath "/mongodb-mongodb-configmap.yaml") . | sha256sum }} + {{- if and (.Values.metrics.prometheusServiceDiscovery) (.Values.metrics.enabled) }} + prometheus.io/scrape: "true" + prometheus.io/port: {{ .Values.metrics.port | quote }} + prometheus.io/path: {{ .Values.metrics.path | quote }} + {{- end }} + {{- if .Values.podAnnotations }} +{{ toYaml .Values.podAnnotations | indent 8 }} + {{- end }} + spec: + {{- if .Values.priorityClassName }} + priorityClassName: {{ .Values.priorityClassName }} + {{- end }} + {{- if .Values.imagePullSecrets }} + imagePullSecrets: + {{- range .Values.imagePullSecrets }} + - name: {{ . }} + {{- end}} + {{- end }} + {{- if .Values.securityContext.enabled }} + securityContext: + runAsUser: {{ .Values.securityContext.runAsUser }} + fsGroup: {{ .Values.securityContext.fsGroup }} + runAsNonRoot: {{ .Values.securityContext.runAsNonRoot }} + {{- end }} + terminationGracePeriodSeconds: {{ .Values.terminationGracePeriodSeconds }} + initContainers: + - name: copy-config + image: "{{ .Values.copyConfigImage.repository }}:{{ .Values.copyConfigImage.tag }}" + imagePullPolicy: {{ .Values.copyConfigImage.pullPolicy | quote }} + command: + - "sh" + args: + - "-c" + - | + set -e + set -x + + cp /configdb-readonly/mongod.conf /data/configdb/mongod.conf + + {{- if .Values.tls.enabled }} + cp /ca-readonly/tls.key /data/configdb/tls.key + cp /ca-readonly/tls.crt /data/configdb/tls.crt + {{- end }} + + {{- if .Values.auth.enabled }} + cp /keydir-readonly/key.txt /data/configdb/key.txt + chmod 600 /data/configdb/key.txt + {{- end }} + volumeMounts: + - name: workdir + mountPath: /work-dir + - name: config + mountPath: /configdb-readonly + - name: configdir + mountPath: /data/configdb + {{- if .Values.tls.enabled }} + - name: ca + mountPath: /ca-readonly + {{- end }} + {{- if .Values.auth.enabled }} + - name: keydir + mountPath: /keydir-readonly + {{- end }} + resources: +{{ toYaml .Values.init.resources | indent 12 }} + - name: install + image: "{{ .Values.installImage.repository }}:{{ .Values.installImage.tag }}" + args: + - --work-dir=/work-dir + imagePullPolicy: "{{ .Values.installImage.pullPolicy }}" + volumeMounts: + - name: workdir + mountPath: /work-dir + resources: +{{ toYaml .Values.init.resources | indent 12 }} + - name: bootstrap + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + command: + - /work-dir/peer-finder + args: + - -on-start=/init/on-start.sh + - "-service={{ template "mongodb-replicaset.fullname" . }}" + imagePullPolicy: "{{ .Values.image.pullPolicy }}" + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: REPLICA_SET + value: {{ .Values.replicaSetName }} + - name: TIMEOUT + value: "{{ .Values.init.timeout }}" + - name: SKIP_INIT + value: "{{ .Values.skipInitialization }}" + - name: TLS_MODE + value: {{ .Values.tls.mode }} + {{- if .Values.auth.enabled }} + - name: AUTH + value: "true" + - name: ADMIN_USER + valueFrom: + secretKeyRef: + name: "{{ template "mongodb-replicaset.adminSecret" . }}" + key: user + - name: ADMIN_PASSWORD + valueFrom: + secretKeyRef: + name: "{{ template "mongodb-replicaset.adminSecret" . }}" + key: password + {{- if .Values.metrics.enabled }} + - name: METRICS + value: "true" + - name: METRICS_USER + valueFrom: + secretKeyRef: + name: "{{ template "mongodb-replicaset.metricsSecret" . }}" + key: user + - name: METRICS_PASSWORD + valueFrom: + secretKeyRef: + name: "{{ template "mongodb-replicaset.metricsSecret" . }}" + key: password + {{- end }} + {{- end }} + volumeMounts: + - name: workdir + mountPath: /work-dir + - name: init + mountPath: /init + - name: configdir + mountPath: /data/configdb + - name: datadir + mountPath: /data/db + resources: +{{ toYaml .Values.init.resources | indent 12 }} + containers: + - name: {{ template "mongodb-replicaset.name" . }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: "{{ .Values.image.pullPolicy }}" + {{- if .Values.extraVars }} + env: +{{ toYaml .Values.extraVars | indent 12 }} + {{- end }} + ports: + - name: mongodb + containerPort: 27017 + resources: +{{ toYaml .Values.resources | indent 12 }} + command: + - mongod + args: + - --config=/data/configdb/mongod.conf + - --dbpath=/data/db + - --replSet={{ .Values.replicaSetName }} + - --port=27017 + - --bind_ip=0.0.0.0 + {{- if .Values.auth.enabled }} + - --auth + - --keyFile=/data/configdb/key.txt + {{- end }} + {{- if .Values.tls.enabled }} + - --sslMode={{ .Values.tls.mode }} + - --sslCAFile=/data/configdb/tls.crt + - --sslPEMKeyFile=/work-dir/mongo.pem + {{- end }} + livenessProbe: + exec: + command: + - mongo + {{- if .Values.tls.enabled }} + - --ssl + - --sslCAFile=/data/configdb/tls.crt + - --sslPEMKeyFile=/work-dir/mongo.pem + {{- end }} + - --eval + - "db.adminCommand('ping')" + initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} + timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }} + failureThreshold: {{ .Values.livenessProbe.failureThreshold }} + periodSeconds: {{ .Values.livenessProbe.periodSeconds }} + successThreshold: {{ .Values.livenessProbe.successThreshold }} + readinessProbe: + exec: + command: + - mongo + {{- if .Values.tls.enabled }} + - --ssl + - --sslCAFile=/data/configdb/tls.crt + - --sslPEMKeyFile=/work-dir/mongo.pem + {{- end }} + - --eval + - "db.adminCommand('ping')" + initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} + timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }} + failureThreshold: {{ .Values.readinessProbe.failureThreshold }} + periodSeconds: {{ .Values.readinessProbe.periodSeconds }} + successThreshold: {{ .Values.readinessProbe.successThreshold }} + volumeMounts: + - name: datadir + mountPath: /data/db + - name: configdir + mountPath: /data/configdb + - name: workdir + mountPath: /work-dir +{{ if .Values.metrics.enabled }} + - name: metrics + image: "{{ .Values.metrics.image.repository }}:{{ .Values.metrics.image.tag }}" + imagePullPolicy: {{ .Values.metrics.image.pullPolicy | quote }} + command: + - sh + - -c + - >- + /bin/mongodb_exporter + --mongodb.uri {{ template "mongodb-replicaset.connection-string" . }} + --web.telemetry-path={{ .Values.metrics.path }} + --web.listen-address=:{{ .Values.metrics.port }} + volumeMounts: + {{- if and (.Values.tls.enabled) }} + - name: ca + mountPath: /ca + readOnly: true + {{- end }} + - name: workdir + mountPath: /work-dir + readOnly: true + env: + {{- if .Values.auth.enabled }} + - name: METRICS_USER + valueFrom: + secretKeyRef: + name: "{{ template "mongodb-replicaset.metricsSecret" . }}" + key: user + - name: METRICS_PASSWORD + valueFrom: + secretKeyRef: + name: "{{ template "mongodb-replicaset.metricsSecret" . }}" + key: password + {{- end }} + ports: + - name: metrics + containerPort: {{ .Values.metrics.port }} + resources: +{{ toYaml .Values.metrics.resources | indent 12 }} + {{- if .Values.metrics.securityContext.enabled }} + securityContext: + runAsUser: {{ .Values.metrics.securityContext.runAsUser }} + {{- end }} + livenessProbe: + exec: + command: + - sh + - -c + - >- + /bin/mongodb_exporter + --mongodb.uri {{ template "mongodb-replicaset.connection-string" . }} + --test + initialDelaySeconds: 30 + periodSeconds: 10 +{{ end }} + {{- if .Values.extraContainers }} + {{- tpl (toYaml .Values.extraContainers) . | nindent 8 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: +{{ toYaml . | indent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: +{{ toYaml . | indent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: +{{ toYaml . | indent 8 }} + {{- end }} + volumes: + - name: config + configMap: + name: {{ template "mongodb-replicaset.fullname" . }}-mongodb + - name: init + configMap: + defaultMode: 0755 + name: {{ template "mongodb-replicaset.fullname" . }}-init + {{- if .Values.tls.enabled }} + - name: ca + secret: + defaultMode: 0400 + secretName: {{ template "mongodb-replicaset.fullname" . }}-ca + {{- end }} + {{- if .Values.auth.enabled }} + - name: keydir + secret: + defaultMode: 0400 + secretName: {{ template "mongodb-replicaset.keySecret" . }} + {{- end }} + - name: workdir + emptyDir: {} + - name: configdir + emptyDir: {} +{{- if .Values.extraVolumes }} +{{- tpl (toYaml .Values.extraVolumes) . | nindent 8 }} +{{- end }} +{{- if .Values.persistentVolume.enabled }} + volumeClaimTemplates: + - metadata: + name: datadir + annotations: + {{- range $key, $value := .Values.persistentVolume.annotations }} + {{ $key }}: "{{ $value }}" + {{- end }} + spec: + accessModes: + {{- range .Values.persistentVolume.accessModes }} + - {{ . | quote }} + {{- end }} + resources: + requests: + storage: {{ .Values.persistentVolume.size | quote }} + {{- if .Values.persistentVolume.storageClass }} + {{- if (eq "-" .Values.persistentVolume.storageClass) }} + storageClassName: "" + {{- else }} + storageClassName: "{{ .Values.persistentVolume.storageClass }}" + {{- end }} + {{- end }} +{{- else }} + - name: datadir + emptyDir: {} +{{- end }} diff --git a/terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/templates/tests/mongodb-up-test-configmap.yaml b/terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/templates/tests/mongodb-up-test-configmap.yaml new file mode 100644 index 0000000..0280f5d --- /dev/null +++ b/terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/templates/tests/mongodb-up-test-configmap.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + app: {{ template "mongodb-replicaset.name" . }} + chart: {{ .Chart.Name }}-{{ .Chart.Version }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} + name: {{ template "mongodb-replicaset.fullname" . }}-tests + namespace: {{ template "mongodb-replicaset.namespace" . }} +data: + mongodb-up-test.sh: | +{{ .Files.Get "tests/mongodb-up-test.sh" | indent 4 }} diff --git a/terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/templates/tests/mongodb-up-test-pod.yaml b/terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/templates/tests/mongodb-up-test-pod.yaml new file mode 100644 index 0000000..bd3dfe9 --- /dev/null +++ b/terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/templates/tests/mongodb-up-test-pod.yaml @@ -0,0 +1,80 @@ +apiVersion: v1 +kind: Pod +metadata: + labels: + app: {{ template "mongodb-replicaset.name" . }} + chart: {{ .Chart.Name }}-{{ .Chart.Version }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} + name: {{ template "mongodb-replicaset.fullname" . }}-test + namespace: {{ template "mongodb-replicaset.namespace" . }} + annotations: + "helm.sh/hook": test-success +spec: + initContainers: + - name: test-framework + image: dduportal/bats:0.4.0 + command: + - bash + - -c + - | + set -ex + # copy bats to tools dir + cp -R /usr/local/libexec/ /tools/bats/ + volumeMounts: + - name: tools + mountPath: /tools + containers: + - name: mongo + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + command: + - /tools/bats/bats + - -t + - /tests/mongodb-up-test.sh + env: + - name: FULL_NAME + value: {{ template "mongodb-replicaset.fullname" . }} + - name: NAMESPACE + value: {{ .Release.Namespace }} + - name: REPLICAS + value: "{{ .Values.replicas }}" + {{- if .Values.auth.enabled }} + - name: AUTH + value: "true" + - name: ADMIN_USER + valueFrom: + secretKeyRef: + name: "{{ template "mongodb-replicaset.adminSecret" . }}" + key: user + - name: ADMIN_PASSWORD + valueFrom: + secretKeyRef: + name: "{{ template "mongodb-replicaset.adminSecret" . }}" + key: password + {{- end }} + volumeMounts: + - name: tools + mountPath: /tools + - name: tests + mountPath: /tests + {{- if .Values.tls.enabled }} + - name: tls + mountPath: /tls + {{- end }} + volumes: + - name: tools + emptyDir: {} + - name: tests + configMap: + name: {{ template "mongodb-replicaset.fullname" . }}-tests + {{- if .Values.tls.enabled }} + - name: tls + secret: + secretName: {{ template "mongodb-replicaset.fullname" . }}-ca + items: + - key: tls.crt + path: tls.crt + - key: tls.key + path: tls.key + {{- end }} + restartPolicy: Never diff --git a/terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/tests/mongodb-up-test.sh b/terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/tests/mongodb-up-test.sh new file mode 100644 index 0000000..ebe9c53 --- /dev/null +++ b/terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/tests/mongodb-up-test.sh @@ -0,0 +1,120 @@ +#!/usr/bin/env bash + +set -ex + +CACRT_FILE=/work-dir/tls.crt +CAKEY_FILE=/work-dir/tls.key +MONGOPEM=/work-dir/mongo.pem + +MONGOARGS="--quiet" + +if [ -e "/tls/tls.crt" ]; then + # log "Generating certificate" + mkdir -p /work-dir + cp /tls/tls.crt /work-dir/tls.crt + cp /tls/tls.key /work-dir/tls.key + + # Move into /work-dir + pushd /work-dir + +cat >openssl.cnf < $MONGOPEM + MONGOARGS="$MONGOARGS --ssl --sslCAFile $CACRT_FILE --sslPEMKeyFile $MONGOPEM" +fi + +if [[ "${AUTH}" == "true" ]]; then + MONGOARGS="$MONGOARGS --username $ADMIN_USER --password $ADMIN_PASSWORD --authenticationDatabase admin" +fi + +pod_name() { + local full_name="${FULL_NAME?Environment variable FULL_NAME not set}" + local namespace="${NAMESPACE?Environment variable NAMESPACE not set}" + local index="$1" + echo "$full_name-$index.$full_name.$namespace.svc.cluster.local" +} + +replicas() { + echo "${REPLICAS?Environment variable REPLICAS not set}" +} + +master_pod() { + for ((i = 0; i < $(replicas); ++i)); do + response=$(mongo $MONGOARGS "--host=$(pod_name "$i")" "--eval=rs.isMaster().ismaster") + if [[ "$response" == "true" ]]; then + pod_name "$i" + break + fi + done +} + +setup() { + local ready=0 + until [[ "$ready" -eq $(replicas) ]]; do + echo "Waiting for application to become ready" >&2 + sleep 1 + ready=0 + for ((i = 0; i < $(replicas); ++i)); do + response=$(mongo $MONGOARGS "--host=$(pod_name "$i")" "--eval=rs.status().ok" || true) + if [[ "$response" -eq 1 ]]; then + ready=$((ready + 1)) + fi + done + done +} + +@test "Testing mongodb client is executable" { + mongo -h + [ "$?" -eq 0 ] +} + +@test "Connect mongodb client to mongodb pods" { + for ((i = 0; i < $(replicas); ++i)); do + response=$(mongo $MONGOARGS "--host=$(pod_name "$i")" "--eval=rs.status().ok") + if [[ ! "$response" -eq 1 ]]; then + exit 1 + fi + done +} + +@test "Write key to primary" { + response=$(mongo $MONGOARGS --host=$(master_pod) "--eval=db.test.insert({\"abc\": \"def\"}).nInserted") + if [[ ! "$response" -eq 1 ]]; then + exit 1 + fi +} + +@test "Read key from slaves" { + # wait for slaves to catch up + sleep 10 + + for ((i = 0; i < $(replicas); ++i)); do + response=$(mongo $MONGOARGS --host=$(pod_name "$i") "--eval=rs.slaveOk(); db.test.find({\"abc\":\"def\"})") + if [[ ! "$response" =~ .*def.* ]]; then + exit 1 + fi + done + + # Clean up a document after test + mongo $MONGOARGS --host=$(master_pod) "--eval=db.test.deleteMany({\"abc\": \"def\"})" +}