Vend MongoDB Helm chart.

terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/Chart.yaml

@@ -0,0 +1,16 @@
+apiVersion: v1
+name: mongodb-replicaset
+home: https://github.com/mongodb/mongo
+version: 3.15.1
+appVersion: 3.6
+description: NoSQL document-oriented database that stores JSON-like documents with
+  dynamic schemas, simplifying the integration of data in content-driven applications.
+icon: https://webassets.mongodb.com/_com_assets/cms/mongodb-logo-rgb-j6w271g1xn.jpg
+sources:
+  - https://github.com/mongodb/mongo
+  - https://github.com/percona/mongodb_exporter
+maintainers:
+  - name: unguiculus
+    email: unguiculus@gmail.com
+  - name: steven-sheehy
+    email: ssheehy@firescope.com

terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/OWNERS

@@ -0,0 +1,6 @@
+approvers:
+  - unguiculus
+  - steven-sheehy
+reviewers:
+  - unguiculus
+  - steven-sheehy

terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/ci/default-values.yaml

@@ -0,0 +1 @@
+# No config change. Just use defaults.

terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/ci/metrics-values.yaml

@@ -0,0 +1,10 @@
+auth:
+  enabled: true
+  adminUser: username
+  adminPassword: password
+  metricsUser: metrics
+  metricsPassword: password
+  key: keycontent
+
+metrics:
+  enabled: true

terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/ci/tls-values.yaml

@@ -0,0 +1,10 @@
+tls:
+  # Enable or disable MongoDB TLS support
+  enabled: true
+  # Please generate your own TLS CA by generating it via:
+  # $ openssl genrsa -out ca.key 2048
+  # $ openssl req -x509 -new -nodes -key ca.key -days 10000 -out ca.crt -subj "/CN=mydomain.com"
+  # After that you can base64 encode it and paste it here:
+  # $ cat ca.key | base64 -w0
+  cacert: "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUNxakNDQVpJQ0NRQ1I1aXNNQlRmQzdUQU5CZ2txaGtpRzl3MEJBUXNGQURBWE1SVXdFd1lEVlFRRERBeHQKZVdSdmJXRnBiaTVqYjIwd0hoY05NVGt4TVRFeU1EZ3hOakUwV2hjTk5EY3dNek13TURneE5qRTBXakFYTVJVdwpFd1lEVlFRRERBeHRlV1J2YldGcGJpNWpiMjB3Z2dFaU1BMEdDU3FHU0liM0RRRUJBUVVBQTRJQkR3QXdnZ0VLCkFvSUJBUUNwM0UrdVpWanhaY3BYNUFCbEtRa2crZjFmSnJzR1JJNVQrMzcyMkIvYnRyTVo4M3FyRTg2RFdEYXEKN0k1YTdlOGFVTGt2ZVpsaW02aWxsUW5CTHJPVUtVZ3R1OWZINlZydlBuMTl3UDFibEMvU0NWZHoxemNSUWlJWQpOMVVWN2VGaWUzdjhiNXVRM2RFcVBPV2FMM0w2N0Q1T0lDb043Z21QL2QwVVBaWjNHdDJLNTZsNXBzY1h4OGYwCkd3ZWdSRGpiVnZmc2dUSW50dEJ6SGh6c0JENUxON054aDd5RWVacW5admtuTDg5S2JZUEFPUk82N3NKUlBhWHMKUDhuVDhqalFJaGlRSUZDNTVXN3JrZ1hid1Znajdwb0kyby9XSDM4WXZ6TG1OVnMyOThYUDZmUXhCQ0NwMmFjRgpkOTVQRjZmbFVJeW9RNGRuOUF5UlpRa0owdlpMQWdNQkFBRXdEUVlKS29aSWh2Y05BUUVMQlFBRGdnRUJBS21XCjY2SlB4V0E4MVlYTEZtclFrdmM2NE9ycFJXTHJtNHFqaFREREtvVzY1V291MzNyOEVVQktzQ2FQOHNWWXhobFQKaWhGcDhIemNqTXpWRjFqU3ZiT0c5UnhrSG16ZEIvL3FwMDdTVFp0S2R1cThad2RVdnh1Z1FXSFNzOHA4YVNHUAowNDlkSDBqUnpEZklyVGp4Z3ZNOUJlWmorTkdqT1FyUGRvQVBKeTl2THowZmYya1gzVjJadTFDWnNnbDNWUXFsCjRsNzB3azFuVk5tTXY4Nnl5cUZXaWFRTWhuSXFjKzBwYUJaRjJFSGNpSExuZWcweVVTZVN4REsrUkk4SE9mT3oKNVFpUHpqSGs1b3czd252NDhQWVJMODdLTWJtRzF0eThyRHMxUlVGWkZueGxHd0t4UmRmckt3aHJJbVRBT2N4Vwo5bVhCU3ZzY3RjM2tIZTRIVFdRPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg=="
+  cakey: "LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFb3dJQkFBS0NBUUVBcWR4UHJtVlk4V1hLVitRQVpTa0pJUG45WHlhN0JrU09VL3QrOXRnZjI3YXpHZk42CnF4UE9nMWcycXV5T1d1M3ZHbEM1TDNtWllwdW9wWlVKd1M2emxDbElMYnZYeCtsYTd6NTlmY0Q5VzVRdjBnbFgKYzljM0VVSWlHRGRWRmUzaFludDcvRytia04zUktqemxtaTl5K3V3K1RpQXFEZTRKai8zZEZEMldkeHJkaXVlcAplYWJIRjhmSDlCc0hvRVE0MjFiMzdJRXlKN2JRY3g0YzdBUStTemV6Y1llOGhIbWFwMmI1SnkvUFNtMkR3RGtUCnV1N0NVVDJsN0QvSjAvSTQwQ0lZa0NCUXVlVnU2NUlGMjhGWUkrNmFDTnFQMWg5L0dMOHk1alZiTnZmRnorbjAKTVFRZ3FkbW5CWGZlVHhlbjVWQ01xRU9IWi9RTWtXVUpDZEwyU3dJREFRQUJBb0lCQVFDVWM3eWNBVzFMaEpmawpXcHRSemh4eFdxcnJSeEU3ZUIwZ0h2UW16bHFCanRwVyt1bWhyT3pXOC9qTFIzVmUyUVlZYktaOGJIejJwbTR0ClVPVTJsaGRTalFYTkdwZUsyMUtqTjIwN3c3aHFHa2YwL0Q4WE9lZWh5TGU5akZacmxQeGZNdWI0aDU1aGJNdUsKYTdDTElaOE8xL3ZZRWRwUFZGTzlLYlRYSk1CbEZJUERUaFJvR2RCTEFkREZNbzcrUnZYSFRUcXdyWmxDbWRDbgp5eld3WkhIQUZhdEdGWU9ybXcxdlZZY3h0OXk5c0FVZDBrVTQza05jVHVHR0MwMGh1QlZMcW9JZU9mMG12TDB0Ckg0S0d6LzBicGp4NFpoWlNKazd3ZkFsQ0xGL1N5YzVJOEJXWWNCb05Jc0RSbDdDUmpDVUoxYVNBNVNYNzZ2SVoKSlhnRWEyV3hBb0dCQU50M0pDRGtycjNXRmJ3cW1SZ2ZhUVV0UE1FVnZlVnJvQmRqZTBZVFFNbENlNTV2QU1uNQpadEFKQTVKTmxKN2VZRkVEa0JrVURJSDFDM3hlZHVWbEREWXpESzRpR0V1Wk8wVDNERFN3aks2cExsZ3JBN0QyCmZnS29ubVdGck5JdTI4UW1MNHhmcjUrWW9SNUo0L05QdFdWOWwwZk1NOHEwSTd5SVRNODlsZWlqQW9HQkFNWWoKTHk3VER1MWVJVWkrQXJFNTJFMEkwTVJPNWNLS2hxdGxJMnpCZkZlWm5LYWdwbnhCMTMxbi9wcGg0Wm1IYnMzZQpxOXBSb0RJT0h4cm5NOWFCa1JBTHJHNjBMeXF3eU5NNW1JemkvQytJK2RVOG55ZXIvZVNNRTNtdlFzbmpVcEhtClRtTjRrM0l4RWtqRnhCazVndFNlNlA5U0UyOFd6eVZoOGlkZHRjNDVBb0dBYzcwWFBvbWJaZDM3UkdxcXBrQWEKWUhLRThjY0hpSEFEMDVIUk54bDhOeWRxamhrNEwwdnAzcGlDVzZ1eVR6NHpTVVk1dmlBR29KcWNYaEJyWDNxMAp2L2lZSFZVNXZ0U21ueTR5TDY5VDRlQ3k0aWg5SDl3K2hDUnN0Rm1VMUp1RnBxSUV2V0RRKzdmQWNIckRUbE9nCjlFOFJjdm5MN29DbHdBMlpoRW1VUDBVQ2dZQWFhdUtGbWJwcHg1MGtkOEVnSkJoRTNTSUlxb1JUMWVoeXZiOWwKWnI3UFp6bk50YW04ODRKcHhBM2NRNlN5dGEzK1lPd0U1ZEU0RzAzbVptRXcvb0Y2NURPUFp4TEszRnRLWG1tSwpqMUVVZld6aUUzMGM2ditsRTFBZGIxSzJYRXJNRFNyeWRFY2tlSXA1alhUQjhEc1RZa1NxbGlUbE1PTlpscCtVCnhCZlRjUUtCZ0RoZHo4VjU1TzdNc0dyRVlQeGhoK0U0bklLb3BRc0RlNi9QdWRRVlJMRlNwVGpLNWlKcTF2RnIKajFyNDFCNFp0cjBYNGd6MzhrSUpwZGNvNUFxU25zVENreHhnYXh3RTNzVmlqNGZZRWlteDc3TS84VkZVbDZwLwphNmdBbFh2WHFaYmFvTGU3ekM2RXVZWjFtUzJGMVd4UE9KRzZpakFiMVNIQjVPOGFWdFR3Ci0tLS0tRU5EIFJTQSBQUklWQVRFIEtFWS0tLS0tCg=="

terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/init/on-start.sh

@@ -0,0 +1,226 @@
+#!/usr/bin/env bash
+
+# Copyright 2018 The Kubernetes Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e pipefail
+
+port=27017
+replica_set="$REPLICA_SET"
+script_name=${0##*/}
+SECONDS=0
+timeout="${TIMEOUT:-900}"
+tls_mode="${TLS_MODE}"
+
+if [[ "$AUTH" == "true" ]]; then
+    admin_user="$ADMIN_USER"
+    admin_password="$ADMIN_PASSWORD"
+    admin_creds=(-u "$admin_user" -p "$admin_password")
+    if [[ "$METRICS" == "true" ]]; then
+        metrics_user="$METRICS_USER"
+        metrics_password="$METRICS_PASSWORD"
+    fi
+    auth_args=("--auth" "--keyFile=/data/configdb/key.txt")
+fi
+
+log() {
+    local msg="$1"
+    local timestamp
+    timestamp=$(date --iso-8601=ns)
+    echo "[$timestamp] [$script_name] $msg" 2>&1 | tee -a /work-dir/log.txt 1>&2
+}
+
+retry_until() {
+    local host="${1}"
+    local command="${2}"
+    local expected="${3}"
+    local creds=("${admin_creds[@]}")
+
+    # Don't need credentials for admin user creation and pings that run on localhost
+    if [[ "${host}" =~ ^localhost ]]; then
+        creds=()
+    fi
+
+    until [[ $(mongo admin --host "${host}" "${creds[@]}" "${ssl_args[@]}" --quiet --eval "${command}" | tail -n1) == "${expected}" ]]; do
+        sleep 1
+
+        if (! ps "${pid}" &>/dev/null); then
+            log "mongod shutdown unexpectedly"
+            exit 1
+        fi
+        if [[ "${SECONDS}" -ge "${timeout}" ]]; then
+            log "Timed out after ${timeout}s attempting to bootstrap mongod"
+            exit 1
+        fi
+
+        log "Retrying ${command} on ${host}"
+    done
+}
+
+shutdown_mongo() {
+    local host="${1:-localhost}"
+    local args='force: true'
+    log "Shutting down MongoDB ($args)..."
+    if (! mongo admin --host "${host}" "${admin_creds[@]}" "${ssl_args[@]}" --eval "db.shutdownServer({$args})"); then
+      log "db.shutdownServer() failed, sending the terminate signal"
+      kill -TERM "${pid}"
+    fi
+}
+
+init_mongod_standalone() {
+    if [[ ! -f /init/initMongodStandalone.js ]]; then
+        log "Skipping init mongod standalone script"
+        return 0
+    elif [[ -z "$(ls -1A /data/db)" ]]; then
+        log "mongod standalone script currently not supported on initial install"
+        return 0
+    fi
+
+    local port="27018"
+    log "Starting a MongoDB instance as standalone..."
+    mongod --config /data/configdb/mongod.conf --dbpath=/data/db "${auth_args[@]}" "${ssl_server_args[@]}" --port "${port}" --bind_ip=0.0.0.0 2>&1 | tee -a /work-dir/log.txt 1>&2 &
+    export pid=$!
+    trap shutdown_mongo EXIT
+    log "Waiting for MongoDB to be ready..."
+    retry_until "localhost:${port}" "db.adminCommand('ping').ok" "1"
+    log "Running init js script on standalone mongod"
+    mongo admin --port "${port}" "${admin_creds[@]}" "${ssl_args[@]}" /init/initMongodStandalone.js
+    shutdown_mongo "localhost:${port}"
+}
+
+my_hostname=$(hostname)
+log "Bootstrapping MongoDB replica set member: $my_hostname"
+
+log "Reading standard input..."
+while read -ra line; do
+    if [[ "${line}" == *"${my_hostname}"* ]]; then
+        service_name="$line"
+    fi
+    peers=("${peers[@]}" "$line")
+done
+
+# Generate the ca cert
+ca_crt=/data/configdb/tls.crt
+if [ -f "$ca_crt"  ]; then
+    log "Generating certificate"
+    ca_key=/data/configdb/tls.key
+    pem=/work-dir/mongo.pem
+    ssl_args=(--ssl --sslCAFile "$ca_crt" --sslPEMKeyFile "$pem")
+    ssl_server_args=(--sslMode "$tls_mode" --sslCAFile "$ca_crt" --sslPEMKeyFile "$pem")
+
+# Move into /work-dir
+pushd /work-dir
+
+cat >openssl.cnf <<EOL
+[req]
+req_extensions = v3_req
+distinguished_name = req_distinguished_name
+[req_distinguished_name]
+[ v3_req ]
+basicConstraints = CA:FALSE
+keyUsage = nonRepudiation, digitalSignature, keyEncipherment
+subjectAltName = @alt_names
+[alt_names]
+DNS.1 = $(echo -n "$my_hostname" | sed s/-[0-9]*$//)
+DNS.2 = $my_hostname
+DNS.3 = $service_name
+DNS.4 = localhost
+DNS.5 = 127.0.0.1
+EOL
+
+    # Generate the certs
+    openssl genrsa -out mongo.key 2048
+    openssl req -new -key mongo.key -out mongo.csr -subj "/OU=MongoDB/CN=$my_hostname" -config openssl.cnf
+    openssl x509 -req -in mongo.csr \
+        -CA "$ca_crt" -CAkey "$ca_key" -CAcreateserial \
+        -out mongo.crt -days 3650 -extensions v3_req -extfile openssl.cnf
+
+    rm mongo.csr
+    cat mongo.crt mongo.key > $pem
+    rm mongo.key mongo.crt
+fi
+
+init_mongod_standalone
+
+if [[ "${SKIP_INIT}" == "true" ]]; then
+    log "Skipping initialization"
+    exit 0
+fi
+
+log "Peers: ${peers[*]}"
+log "Starting a MongoDB replica"
+mongod --config /data/configdb/mongod.conf --dbpath=/data/db --replSet="$replica_set" --port="${port}" "${auth_args[@]}" "${ssl_server_args[@]}" --bind_ip=0.0.0.0 2>&1 | tee -a /work-dir/log.txt 1>&2 &
+pid=$!
+trap shutdown_mongo EXIT
+
+log "Waiting for MongoDB to be ready..."
+retry_until "localhost" "db.adminCommand('ping').ok" "1"
+log "Initialized."
+
+# try to find a master
+for peer in "${peers[@]}"; do
+    log "Checking if ${peer} is primary"
+    # Check rs.status() first since it could be in primary catch up mode which db.isMaster() doesn't show
+    if [[ $(mongo admin --host "${peer}" "${admin_creds[@]}" "${ssl_args[@]}" --quiet --eval "rs.status().myState" | tail -n1) == "1" ]]; then
+        retry_until "${peer}" "db.isMaster().ismaster" "true"
+        log "Found primary: ${peer}"
+        primary="${peer}"
+        break
+    fi
+done
+
+if [[ "${primary}" = "${service_name}" ]]; then
+    log "This replica is already PRIMARY"
+elif [[ -n "${primary}" ]]; then
+    if [[ $(mongo admin --host "${primary}" "${admin_creds[@]}" "${ssl_args[@]}" --quiet --eval "rs.conf().members.findIndex(m => m.host == '${service_name}:${port}')" | tail -n1) == "-1" ]]; then
+      log "Adding myself (${service_name}) to replica set..."
+      if (mongo admin --host "${primary}" "${admin_creds[@]}" "${ssl_args[@]}" --eval "rs.add('${service_name}')" | grep 'Quorum check failed'); then
+          log 'Quorum check failed, unable to join replicaset. Exiting prematurely.'
+          exit 1
+      fi
+    fi
+
+    sleep 3
+    log 'Waiting for replica to reach SECONDARY state...'
+    retry_until "${service_name}" "rs.status().myState" "2"
+    log '✓ Replica reached SECONDARY state.'
+
+elif (mongo "${ssl_args[@]}" --eval "rs.status()" | grep "no replset config has been received"); then
+    log "Initiating a new replica set with myself ($service_name)..."
+    mongo "${ssl_args[@]}" --eval "rs.initiate({'_id': '$replica_set', 'members': [{'_id': 0, 'host': '$service_name'}]})"
+
+    sleep 3
+    log 'Waiting for replica to reach PRIMARY state...'
+    retry_until "localhost" "db.isMaster().ismaster" "true"
+    primary="${service_name}"
+    log '✓ Replica reached PRIMARY state.'
+
+    if [[ "${AUTH}" == "true" ]]; then
+        log "Creating admin user..."
+        mongo admin "${ssl_args[@]}" --eval "db.createUser({user: '${admin_user}', pwd: '${admin_password}', roles: [{role: 'root', db: 'admin'}]})"
+    fi
+fi
+
+# User creation
+if [[ -n "${primary}" && "$AUTH" == "true" && "$METRICS" == "true" ]]; then
+    metric_user_count=$(mongo admin --host "${primary}" "${admin_creds[@]}" "${ssl_args[@]}" --eval "db.system.users.find({user: '${metrics_user}'}).count()" --quiet | tail -n1)
+    if [[ "${metric_user_count}" == "0" ]]; then
+        log "Creating clusterMonitor user..."
+        mongo admin --host "${primary}" "${admin_creds[@]}" "${ssl_args[@]}" --eval "db.createUser({user: '${metrics_user}', pwd: '${metrics_password}', roles: [{role: 'clusterMonitor', db: 'admin'}, {role: 'read', db: 'local'}]})"
+    fi
+fi
+
+log "MongoDB bootstrap complete"
+exit 0
+

terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/install/.dockerignore

@@ -0,0 +1 @@
+Makefile

terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/install/Dockerfile

@@ -0,0 +1,25 @@
+# Copyright 2016 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM alpine:3.8
+
+LABEL maintainer="Reinhard Nägele <unguiculus@gmail.com>"
+
+RUN apk update && apk add bash openssl && wget -qO /peer-finder http://storage.googleapis.com/kubernetes-release/pets/peer-finder
+
+ENTRYPOINT ["/install.sh"]
+
+COPY install.sh /
+
+RUN chmod -c 755 /install.sh /peer-finder

terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/install/Makefile

@@ -0,0 +1,27 @@
+# Copyright 2016 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+all: push
+
+TAG = 0.7
+PREFIX = unguiculus/mongodb-install
+
+build:
+	docker build -t $(PREFIX):$(TAG) .
+
+push: build
+	docker push $(PREFIX):$(TAG)
+
+clean:
+	docker rmi $(PREFIX):$(TAG)

terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/install/install.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+
+# Copyright 2016 The Kubernetes Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This volume is assumed to exist and is shared with the peer-finder
+# init container. It contains on-start/change configuration scripts.
+WORKDIR_VOLUME="/work-dir"
+
+for i in "$@"; do
+    case "$i" in
+        -w=*|--work-dir=*)
+            WORKDIR_VOLUME="${i#*=}"
+            shift
+            ;;
+        *)
+            # unknown option
+            ;;
+    esac
+done
+
+echo Installing config scripts into "${WORKDIR_VOLUME}"
+
+mkdir -p "${WORKDIR_VOLUME}"
+cp /peer-finder "${WORKDIR_VOLUME}"/

terraform-k8s-infrastructure/modules/k8s_data_layer/charts/mongodb-replicaset/templates/NOTES.txt

@@ -0,0 +1,14 @@
+1. After the statefulset is created completely, one can check which instance is primary by running:
+
+    $ for ((i = 0; i < {{ .Values.replicas }}; ++i)); do kubectl exec --namespace {{ template "mongodb-replicaset.namespace" . }} {{ template "mongodb-replicaset.fullname" . }}-$i -- sh -c 'mongo --eval="printjson(rs.isMaster())"'; done
+
+2. One can insert a key into the primary instance of the mongodb replica set by running the following:
+    MASTER_POD_NAME must be replaced with the name of the master found from the previous step.
+
+    $ kubectl exec --namespace {{ template "mongodb-replicaset.namespace" . }} MASTER_POD_NAME -- mongo --eval="printjson(db.test.insert({key1: 'value1'}))"
+
+3. One can fetch the keys stored in the primary or any of the slave nodes in the following manner.
+    POD_NAME must be replaced by the name of the pod being queried.
+
+    $ kubectl exec --namespace {{ template "mongodb-replicaset.namespace" . }} POD_NAME -- mongo --eval="rs.slaveOk(); db.test.find().forEach(printjson)"
+