dotfiles/dot_local/bin/executable_kubectl-watch

#!/usr/bin/env bash

if [[ "$1" == "scheduler" ]]
then
    viddy_command()
    {
        cat <<EOF
$ kubectl get pods --field-selector status.phase=Pending --all-namespaces -o wide
$(kubectl get pods --field-selector status.phase=Pending --all-namespaces -o wide)

$ kubectl get pods --field-selector status.phase=Failed --all-namespaces -o wide
$(kubectl get pods --field-selector status.phase=Failed --all-namespaces -o wide)

EOF
        if [[ $(tput cols) -gt 200 ]]
        then
            cat <<EOF
$ kubectl get pod -A -o wide | awk '/NAME/ || ($3 ~ "0/.*" && $4 != "Completed") { print $0 }'
$(kubectl get pod -A -o wide | awk '/NAME/ || ($3 ~ "0/.*" && $4 != "Completed") { print $0 }')

$ kubectl get pod -o wide -A | grep 'NAME\|(.*m.* ago)'
$(kubectl get pod -o wide -A | grep 'NAME\|(.*[ms].* ago)')

EOF
        else
            cat <<EOF
$ kubectl get pod -A | awk '/NAME/ || ($3 ~ "0/.*" && $4 != "Completed") { print $0 }'
$(kubectl get pod -A | awk '/NAME/ || ($3 ~ "0/.*" && $4 != "Completed") { print $0 }')

$ kubectl get pod -A | grep 'NAME\|(.*m.* ago)'
$(kubectl get pod -A | grep 'NAME\|(.*[ms].* ago)')

EOF
        fi
    }
    export -f viddy_command
    viddy_args=(
        --no-title
        --interval "${KUBECTL_WATCH_INTERVAL:-10}"
    )
    # Upstream viddy does not have max-history flag.
    if [[ $(viddy --help) == *max-history* ]]
    then
        viddy_args+=("--max-history=${KUBECTL_WATCH_HISTORY:-10}")
    fi
    viddy "${viddy_args[@]}" viddy_command
elif [[ -z "$1" ]]
then
    viddy -n "${KUBECTL_WATCH_INTERVAL:-10}" "kubectl get nodes -o wide; echo; kubectl get all -A -o wide; echo; kubectl get ingress -o wide; echo; kubectl get secrets -o wide; echo; kubectl get all -n kube-system -A -o wide"
else
    namespace="$1"
    export namespace
    opensearch_query()
    {
            cat <<EOF
{
  "version": true,
  "size": 1000,
  "sort": [
    {
      "@timestamp": {
        "order": "desc",
        "unmapped_type": "boolean"
      }
    }
  ],
  "aggs": {
    "2": {
      "date_histogram": {
        "field": "@timestamp",
        "calendar_interval": "1m",
        "time_zone": "Europe/Moscow",
        "min_doc_count": 1
      }
    }
  },
  "stored_fields": [
    "*"
  ],
  "script_fields": {},
  "docvalue_fields": [
    {
      "field": "@timestamp",
      "format": "date_time"
    },
    {
      "field": "time",
      "format": "date_time"
    }
  ],
  "_source": {
    "excludes": []
  },
  "query": {
    "bool": {
      "must": [],
      "filter": [
        {
          "match_all": {}
        },
        {
          "match_phrase": {
            "kubernetes.namespace_name.keyword": "$namespace"
          }
        },
        {
          "range": {
            "@timestamp": {
              "gte": "$(date -u -d "-60 minutes" +"%Y-%m-%dT%H:%M:%S.%3NZ")",
              "lte": "$(date -u +"%Y-%m-%dT%H:%M:%S.%3NZ")",
              "format": "strict_date_optional_time"
            }
          }
        }
      ],
      "should": [],
      "must_not": []
    }
  },
  "highlight": {
    "pre_tags": [
      "@opensearch-dashboards-highlighted-field@"
    ],
    "post_tags": [
      "@/opensearch-dashboards-highlighted-field@"
    ],
    "fields": {
      "*": {}
    },
    "fragment_size": 2147483647
  }
}
EOF
    }
    export -f opensearch_query
    viddy_command()
    {
        case "$namespace" in
            pdns)
                kustomization="powerdns"
                ;;
            kube-system)
                kustomization="cilium"
                ;;
            *)
                kustomization="$namespace"
                ;;
        esac
        if [[ $(kubectl get -n "$namespace" scaledobjects -o json | jq '.items | length') -gt 0 ]]
        then
            cat <<EOF
$ kubectl get -n "$namespace" scaledobjects
$(kubectl get -n "$namespace" scaledobjects)

EOF
        fi
        if [[ $(kubectl get -n "$namespace" scaledjobs -o json | jq '.items | length') -gt 0 ]]
        then
            cat <<EOF
$ kubectl get -n "$namespace" scaledjobs
$(kubectl get -n "$namespace" scaledjobs)

EOF
        fi
        case "$KUBECONFIG" in
            *config-mjru-cluster1)
                PROMETHEUS_URL=https://prometheus.corp1.majordomo.ru
                OPENSEARCH_ENDPOINT="https://opensearch.corp1.majordomo.ru"
                OPENSEARCH_PASSWORD="$(pass show majordomo/public/opensearch-dashboards/admin)"
                ALERTMANAGER_URL="https://alertmanager.corp1.majordomo.ru"
                VAULT_TOKEN="$(pass show majordomo/public/vault/root)"
                VAULT_ADDR="https://vault.intr"
                ;;
            *config-mjru-cluster2)
                PROMETHEUS_URL=https://prometheus.corp2.majordomo.ru
                OPENSEARCH_ENDPOINT="https://opensearch.corp2.majordomo.ru:9200"
                OPENSEARCH_PASSWORD="$(pass show majordomo/public/opensearch-dashboards/admin)"
                ALERTMANAGER_URL="https://alertmanager.corp2.majordomo.ru"
                ;;
            *config-home-k8s)
                PROMETHEUS_URL=https://prometheus.home.wugi.info
                OPENSEARCH_ENDPOINT="https://node-0.example.com:9200"
                OPENSEARCH_PASSWORD="$(pass show localhost/opensearch-dashboards/admin)"
                ALERTMANAGER_URL="https://alertmanager.home.wugi.info"
                ;;
        esac
        case "$KUBECONFIG" in
            *config-mjru-*)
                cat <<EOF
$ git ls-remote https://gitlab.corp1.majordomo.ru/cd/fluxcd.git master
$(git ls-remote https://gitlab.corp1.majordomo.ru/cd/fluxcd.git master)

EOF
                ;;
            *config-home-*)
                cat <<EOF
$ git ls-remote https://gitlab.com/wigust/dotfiles.git master
$(git ls-remote https://gitlab.com/wigust/dotfiles.git master)

EOF
                ;;
        esac
        if kubectl get -n "$namespace" scaledobjects &> /dev/null
        then
            mapfile -t queries < <(kubectl get -n "$namespace" scaledobjects "${namespace}-scaledobject" -o json | jq --raw-output '.spec.triggers[] | .metadata.query')
            for query in "${queries[@]}"
            do
                cat <<EOF
query: ${query}
$(echo "$query" | curl --max-time 2 --silent --get "${PROMETHEUS_URL}/api/v1/query" --data-urlencode query@-)

EOF
            done
        fi
        if kubectl get -n "$namespace" scaledjobs &> /dev/null
        then
            mapfile -t queries < <(kubectl get -n "$namespace" scaledjobs "${namespace}-scaledobject" -o json | jq --raw-output '.spec.triggers[] | .metadata.query')
            for query in "${queries[@]}"
            do
                cat <<EOF
query: ${query}
$(echo "$query" | curl --max-time 2 --silent --get "${PROMETHEUS_URL}/api/v1/query" --data-urlencode query@-)

EOF
            done
        fi
        cat <<EOF
$ kubectl get -n flux-system gitrepositories.source.toolkit.fluxcd.io flux-system
$(kubectl get -n flux-system gitrepositories.source.toolkit.fluxcd.io flux-system)

EOF
        if kubectl get -n flux-system kustomizations.kustomize.toolkit.fluxcd.io "$kustomization" &> /dev/null
        then
            cat <<EOF
$ kubectl get -n flux-system kustomizations.kustomize.toolkit.fluxcd.io "$kustomization"
$(kubectl get -n flux-system kustomizations.kustomize.toolkit.fluxcd.io "$kustomization")

EOF
        fi
        mapfile -t helmreleases < <(kubectl -n "$namespace" get helmreleases.helm.toolkit.fluxcd.io --no-headers=true --output=custom-columns='NAME:metadata.name')
        for helmrelease in "${helmreleases[@]}"
        do
            if kubectl -n "$namespace" get helmreleases.helm.toolkit.fluxcd.io "$helmrelease" &> /dev/null
            then
                cat <<EOF
$ kubectl -n "$namespace" get helmreleases.helm.toolkit.fluxcd.io "$helmrelease"
$(kubectl -n "$namespace" get helmreleases.helm.toolkit.fluxcd.io "$helmrelease")

$ kubectl -n "$namespace" get -o json helmreleases.helm.toolkit.fluxcd.io "$helmrelease" | yq -y .status
$(kubectl -n "$namespace" get -o json helmreleases.helm.toolkit.fluxcd.io "$helmrelease" | yq -y .status)

EOF
            fi
        done
        if [[ $namespace == "kube-system" ]]
        then
            mapfile -t nodes_with_noschedule_taint < <(kubectl get nodes -o go-template='{{range $item := .items}}{{with $nodename := $item.metadata.name}}{{range $taint := $item.spec.taints}}{{if and (eq $taint.effect "NoSchedule")}}{{printf "%s\n" $nodename}}{{end}}{{end}}{{end}}{{end}}' | sort --version-sort)
            nodes_with_noschedule()
            {
                for node in "${nodes_with_noschedule_taint[@]}"
                do
                    echo -n "${node}: "
                    kubectl get node -o json "$node" \
                        | jq --monochrome-output --compact-output '.metadata.labels | with_entries(select(.key | test("^.*cluster.local.*$"))) | with_entries(select(.value == "false")) | keys'
                done
            }
            cat <<EOF
$ kubectl get node -o wide | awk 'NR<3{print $0;next}{print $0| "sort --version-sort"}'
$(kubectl get node -o wide | awk 'NR<3{print $0;next}{print $0| "sort --version-sort"}')

$ kubectl get nodes ... # with NoSchedule taint
$(nodes_with_noschedule)

$ kubectl top node | awk 'NR<3{print $0;next}{print $0| "sort --version-sort"}'
$(kubectl top node | awk 'NR<3{print $0;next}{print $0| "sort --version-sort"}')

$ kubectl cluster-info
$(kubectl cluster-info)

$ kubectl get --raw='/readyz?verbose'
$(kubectl get --raw='/readyz?verbose')

EOF
            if ihs --help &> /dev/null
            then
                echo "hypervisor hosts:"
                (
                    mapfile -t masters < <(kubectl master)
                    for master in "${masters[@]}"
                    do
                        mapfile -t ip_addresses < <(command dig "${master}.intr" \
                                                        | jc --dig \
                                                        | jq --raw-output .[].answer[].data)
                        for ip_address in "${ip_addresses[@]}"
                        do
                            master_hypervisor()
                            {
                                command dig -x "$ip_address" \
                                    | jc --dig \
                                    | jq --raw-output '[.[].answer[].data][] | select(. | startswith("vm"))'
                            }
                            master_hypervisor_name="$(master_hypervisor)"
                            printf "%s %s %s %s.intr\n" \
                                   "$master" \
                                   "$ip_address" \
                                   "${master_hypervisor_name/%.intr./}" \
                                   "$(ihs vm dump "$master_hypervisor_name" | recsel -Pname | grep ^kvm)"
                        done
                    done
                )
                printf "\n"
            fi
        fi
        if [[ $namespace == "flux-system" ]]
        then
            cat <<EOF
$ kubectl get --namespace flux-system kustomizations ... # suspended
$(kubectl get --namespace flux-system kustomizations.kustomize.toolkit.fluxcd.io --output=json | jq --raw-output '.items[] | select(.spec.suspend == true) | .metadata.name')

$ kubectl get --all-namespaces helmreleases ... # suspended
$(kubectl get --all-namespaces helmreleases.helm.toolkit.fluxcd.io --output=json | jq --raw-output '.items[] | select(.spec.suspend == true) | .metadata.name'
tf-controller)

$ kubectl get --all-namespaces kustomizations.kustomize.toolkit.fluxcd.io
$(kubectl get --all-namespaces kustomizations.kustomize.toolkit.fluxcd.io)

$ kubectl get --all-namespaces helmreleases.helm.toolkit.fluxcd.io
$(kubectl get --all-namespaces helmreleases.helm.toolkit.fluxcd.io)

EOF
        fi

        for namespace_suffix in mda kvm
        do
            if [[ $namespace == "backup-${namespace_suffix}" ]]
            then
                for metrics in performed errors
                do
                    query="backup_${namespace_suffix}_${metrics}_total{namespace=\"$namespace\"}"
                    cat <<EOF
query: ${query}
$(echo "$query" | curl --max-time 2 --silent --get "${PROMETHEUS_URL}/api/v1/query" --data-urlencode query@- | jq --raw-output '.data.result[] | .value[1]' | xargs echo)

EOF
                done
            fi
        done

        cat <<EOF
$ kubectl top --namespace "$namespace" pod
$(kubectl top --namespace "$namespace" pod)

$ kubectl get all --namespace "$namespace" -o wide
$(kubectl get all --namespace "$namespace" -o wide)

$ kubectl get endpoints --namespace "$namespace" -o wide
$(kubectl get endpoints --namespace "$namespace" -o wide)

$ kubectl get ingress --namespace "$namespace" -o wide
$(kubectl get ingress --namespace "$namespace" -o wide)

EOF
        if kubectl get -n "$namespace" ingress &> /dev/null
        then
            mapfile -t hosts < <(kubectl -n "$namespace" get ingress -o json | jq --raw-output '.items[] | .spec.rules[] | .host' | sort -u)
            for host in "${hosts[@]}"
            do
                query="probe_http_status_code{instance=\"https://${host}/\"}"
                cat <<EOF
query: ${query}
$(echo "$query" | curl --max-time 2 --silent --get "${PROMETHEUS_URL}/api/v1/query" --data-urlencode query@-)

EOF
            done
        fi
        cat <<EOF
$ amtool --alertmanager.url="$ALERTMANAGER_URL" alert query --output=simple namespace="$namespace"
$(amtool --alertmanager.url="$ALERTMANAGER_URL" alert query --output=simple namespace="$namespace")

$ kubectl get --namespace "$namespace" certificates -o wide
$(kubectl get --namespace "$namespace" certificates -o wide)

$ kubectl get secrets --namespace "$namespace" -o wide
$(kubectl get secrets --namespace "$namespace" -o wide)

$ kubectl get --namespace "$namespace" serviceaccounts
$(kubectl get --namespace "$namespace" serviceaccounts)

$ kubectl get pvc --namespace "$namespace" -o wide
$(kubectl get pvc --namespace "$namespace" -o wide)

EOF

        query="kubelet_volume_stats_available_bytes{job=\"kubelet\", metrics_path=\"/metrics\", namespace=\"$namespace\"}"
        cat <<EOF
query: ${query}
$(echo "$query" | curl --max-time 2 --silent --get "${PROMETHEUS_URL}/api/v1/query" --data-urlencode query@- | jq --raw-output '.data.result[] | .value[1]' | numfmt --to=iec-i --suffix=B | xargs echo)

$ kubectl get networkpolicies --namespace "$namespace" -o wide
$(kubectl get networkpolicies --namespace "$namespace" -o wide)

$ kubectl get events --namespace "$namespace" -o wide
$(kubectl get events --namespace "$namespace" -o wide)

EOF

        if [[ $namespace == "elasticsearch" ]]
        then
            OPENSEARCH_ENDPOINT="http://es.intr:9200"
            cat <<EOF

# logs
ssh -q fluentd.intr sudo tail -f /home/jenkins/es-curator/curator.log
ssh -q kvm15.intr journalctl -u elasticsearch.service -f | grep -vF '[max_concurrent_shard_requests] is not supported in the metadata section and will be rejected in 7.x'
ssh -q fluentd.intr sudo docker logs --tail 10 -f elk_elasticsearch_1 | grep -vF '[max_concurrent_shard_requests] is not supported in the metadata section and will be rejected in 7.x'
ssh -q staff.intr journalctl -u elasticsearch.service -f | grep -vF '[max_concurrent_shard_requests] is not supported in the metadata section and will be rejected in 7.x'
EOF
        fi

        OPENSEARCH_ARGS=(
            --user "admin:${OPENSEARCH_PASSWORD}"
        )
        if [[ $namespace == "elasticsearch" ]] || [[ $namespace == "opensearch" ]]
        then
            cat <<EOF

$ curl --max-time 2 --insecure --silent -XGET "${OPENSEARCH_ENDPOINT}/_cluster/state/nodes?pretty"
$(curl --max-time 2 --insecure --silent "${OPENSEARCH_ARGS[@]}" -XGET "${OPENSEARCH_ENDPOINT}/_cluster/state/nodes?pretty")

$ curl --max-time 2 --insecure --silent -XGET "${OPENSEARCH_ENDPOINT}/_cat/nodes?h=name,ram.percent,cpu,load_1m,disk.used_percent&v"
$(curl --max-time 2 --insecure --silent "${OPENSEARCH_ARGS[@]}" -XGET "${OPENSEARCH_ENDPOINT}/_cat/nodes?h=name,ram.percent,cpu,load_1m,disk.used_percent&v" | sort --version-sort)

$ curl --max-time 2 --insecure --silent -XGET "${OPENSEARCH_ENDPOINT}/_cat/allocation?v&pretty"
$(curl --max-time 2 --insecure --silent "${OPENSEARCH_ARGS[@]}" -XGET "${OPENSEARCH_ENDPOINT}/_cat/allocation?v&pretty" | awk '{ print $9, $0 | "column --table" }' | awk '{ print $10="", $0 | "column --table" }' | awk 'NR<2{print $0;next}{print $0| "sort --version-sort"}')

$ curl --max-time 2 --insecure --silent -XGET "${OPENSEARCH_ENDPOINT}/_cluster/health?pretty"
$(curl --max-time 2 --insecure --silent "${OPENSEARCH_ARGS[@]}" -XGET "${OPENSEARCH_ENDPOINT}/_cluster/health?pretty")

$ curl --max-time 2 --insecure --silent "${OPENSEARCH_ENDPOINT}/_cat/indices?v&pretty"
$(command curl --max-time 2 --insecure --silent "${OPENSEARCH_ARGS[@]}" "${OPENSEARCH_ENDPOINT}/_cat/indices?v&pretty" | awk '{ print $3, $0 | "column --table" }' | awk '{ print $4="", $0 | "column --table" }' | awk 'NR<2{print $0;next}{print $0| "sort --version-sort"}')

$ curl --max-time 2 --insecure --silent "${OPENSEARCH_ENDPOINT}/_cat/shards?v&pretty"
$(command curl --max-time 2 --insecure --silent "${OPENSEARCH_ARGS[@]}" "${OPENSEARCH_ENDPOINT}/_cat/shards?v&pretty" | awk '{ print $1 "@" $NF, $0 | "column --table" }' | awk '{ print $2="", $NF="", $0 | "column --table" }' | awk 'NR<2{print $0;next}{print $0| "sort --version-sort"}')

$ curl --max-time 2 --header 'Content-Type: application/json' --insecure --silent --data '{"query": {"match_all": {}}}' "${OPENSEARCH_ENDPOINT}/logstash-*/_search" | jq --raw-output '.hits.hits[] | ._source.log'
$(command curl --max-time 2 --header 'Content-Type: application/json' --insecure --silent "${OPENSEARCH_ARGS[@]}" --data '{"query": {"match_all": {}}}' "${OPENSEARCH_ENDPOINT}/logstash-*/_search" | jq --raw-output '.hits.hits[] | ._source.log')

# Show recovery operations in OpenSearch.
$ curl --max-time 2 --header 'Content-Type: application/json' --insecure --silent "${OPENSEARCH_ENDPOINT}/_cat/recovery?active_only=true&v=true"
$(command curl --max-time 2 --header 'Content-Type: application/json' --insecure --silent "${OPENSEARCH_ARGS[@]}" "${OPENSEARCH_ENDPOINT}/_cat/recovery?active_only=true&v=true")
EOF
        fi

        if [[ $namespace == "harbor" ]]
        then
            cat <<EOF
$ git ls-remote https://gitlab.corp1.majordomo.ru/cd/tf-harbor.git master
$(git ls-remote https://gitlab.corp1.majordomo.ru/cd/tf-harbor.git master)

$ kubectl get -n flux-system gitrepositories.source.toolkit.fluxcd.io tf-harbor
$(kubectl get -n flux-system gitrepositories.source.toolkit.fluxcd.io tf-harbor)

$ kubectl describe -n flux-system terraforms.infra.contrib.fluxcd.io tf-harbor
$(kubectl describe -n flux-system terraforms.infra.contrib.fluxcd.io tf-harbor)
EOF
        fi

        if [[ $namespace == "mda" ]]
        then
            query='
node_filesystem_free_bytes{instance="172.16.103.230:9100", fstype!="tmpfs", fstype!="ramfs", mountpoint!="/", mountpoint!="/nix/store", mountpoint!="/boot"}
or node_filesystem_free_bytes{instance="172.16.103.47:9100", fstype!="tmpfs", fstype!="ramfs", mountpoint!="/", mountpoint!="/nix/store", mountpoint!="/boot"}
'
            cat <<EOF
query: ${query}
$(echo "$query" | curl --max-time 2 --silent --get "${PROMETHEUS_URL}/api/v1/query" --data-urlencode query@- | jq --raw-output '.data.result[] | .value[1]' | numfmt --to=iec-i --suffix=B | xargs echo)
EOF
        fi

        if [[ $namespace == "smtp" ]]
        then
            mapfile -t smtp_hosts < <(getent hosts smtpout1.majordomo.ru smtpout2.majordomo.ru | cut -f 1 -d ' ')
            smtp_hosts+=(78.108.86.12) # on smtp-staff.intr server.
            cat <<EOF

mxtoolbox.com:
EOF
            for host in "${smtp_hosts[@]}"
            do
                echo "https://mxtoolbox.com/SuperTool.aspx?action=blacklist%3a${host}&run=toolpage" 
            done
        fi

        if [[ $namespace == "redis" ]]
        then
            cat <<EOF
$ dig redis.intr
$(bash -ic "dig redis.intr 2>&1")

Listing All Databases

In the first place, the number of databases in Redis is fixed. Therefore, we
can extract this information from the configuration file with a simple grep
command:

$ redis-cli -h redis.intr CONFIG GET databases
$(redis-cli -h redis.intr CONFIG GET databases)

Listing All Databases With Entries

Sometimes we'll want to get more information about the databases that contain
keys. In order to do that, we can take advantage of the Redis INFO command,
used to get information and statistics about the server. Here, we specifically
want to focus our attention in the keyspace section, which contains
database-related data:

$ redis-cli -h redis.intr INFO keyspace
$(redis-cli -h redis.intr INFO keyspace)

The output lists the databases containing at least one key, along with a few
statistics:

- number of keys contained
- number of keys with expiration
- keys' average time-to-live

$ redis-cli -h redis.intr INFO
$(redis-cli -h redis.intr INFO)

$ redis-cli -h redis.intr CLIENT LIST
$(redis-cli -h redis.intr CLIENT LIST)
EOF
        fi

        if [[ $namespace == "nfs" ]]
        then
            cat <<EOF
$ dig nfs.intr
$(bash -ic "dig nfs.intr 2>&1")

$ ssh root@kube6.intr showmount -e
$(ssh root@kube6.intr showmount -e)

$ ssh root@kube6.intr showmount
$(ssh root@kube6.intr showmount)

$ ssh root@kube6.intr rpcinfo
$(ssh root@kube6.intr rpcinfo)

$ ssh root@kube6.intr rpcinfo -p
$(ssh root@kube6.intr rpcinfo -p)

$ ssh root@kube6.intr nfsstat
$(ssh root@kube6.intr nfsstat)

$ ssh root@kube6.intr nfsstat -m
$(ssh root@kube6.intr nfsstat -m)

EOF
        fi

        if [[ $namespace == "keda" ]]
        then
            cat <<EOF
$ kubectl get --raw /apis/external.metrics.k8s.io
$(kubectl get --raw /apis/external.metrics.k8s.io)

EOF
        fi

        if [[ $namespace == "vault" ]]
        then
            vault_addrs=(
                "$VAULT_ADDR"
                "http://dh1-mr.intr:8210"
                "http://dh2-mr.intr:8220"
                "http://dh3-mr.intr:8230"
            )
            cat <<EOF
An explanation of each field in the status output follows.
- Seal Type: The type of seal in use. This value should match across cluster
  members.
- Initialized: Whether the underlying storage has been initialized. This
  should always appear with a value of true in any case except that of a new
  and uninitialized server.
- Sealed: Whether the server is in a sealed or unsealed state. A sealed server
  cannot participate in cluster membership or otherwise be used until it is
  unsealed. All members of a healthy cluster should report a value of false.
- Total Shares: The number of key shares made from splitting the root key
  (previously known as master key); this value can only defined during
  initialization.
- Threshold: The number of key shares required to compose the root key; this
  value can only defined during initialization.
- Version: The version of Vault in use on the server.
- Storage Type: The type of storage in use.
- Cluster Name: The cluster name string; this value should match on all
  members of a healthy cluster.
- Cluster ID: The cluster identification string; this value is dynamically
  generated by default, and should match on all members of a healthy cluster.
- HA Enabled: Whether this cluster is using high availability (HA)
  coordination functionality.
- HA Cluster: The cluster address used in client redirects.
- HA Mode: The HA mode. Expected values are Active and Standby. There should
  be one active leader in every healthy cluster. In the example output, the
  pod vault-0 is the Active cluster leader.
- Active Node Address: The address of the active HA cluster leader, used in
  request forwarding.
- Raft Committed Index: The index value for storage items which are committed
  to the log. This value should closely follow or be equal to the value of
  Raft Applied Index in a healthy Vault cluster.
- Raft Applied Index: The index value for storage items which are applied, but
  not yet committed to the log.

EOF
            for vault_addr in "${vault_addrs[@]}"
            do
            cat <<EOF

$ VAULT_ADDR="$vault_addr" vault status
$(VAULT_ADDR="$vault_addr" "${HOME}/.nix-profile/bin/vault" status)
EOF
            done
            if [[ $(tput cols) -gt 255 ]]
            then
                cat <<EOF

$ vault secrets list -detailed
$(vault secrets list -detailed)
EOF
            else
                cat <<EOF

$ vault secrets list
$(vault secrets list)
EOF
            fi
            cat <<EOF

$ vault kv list secret/vaultPass/majordomo
$(vault kv list secret/vaultPass/majordomo)

To add a new key value secret run:
$ vault kv put secret/vaultPass/majordomo/HOSTNAME password=SECRET
EOF
        fi

        echo -e "\n$ curl --max-time 5 --insecure --silent ${OPENSEARCH_ENDPOINT}/logstash-*/_search"
        echo "$(opensearch_query)" \
            | curl --max-time 5 --insecure --silent "${OPENSEARCH_ARGS[@]}" --header 'Content-Type: application/json' --data @- "${OPENSEARCH_ENDPOINT}/logstash-*/_search" \
            | jq --raw-output '.hits.hits | group_by(._source.kubernetes.pod_name)[][] | [._source.kubernetes.pod_name, ._source.log] | @tsv' \
            | sort --version-sort
    }
    export -f viddy_command
    viddy_args=(
        --no-title
        --interval "${KUBECTL_WATCH_INTERVAL:-10}"
    )
    # Upstream viddy does not have max-history flag.
    if [[ $(viddy --help) == *max-history* ]]
    then
        viddy_args+=("--max-history=${KUBECTL_WATCH_HISTORY:-10}")
    fi
    if [[ -n $TMUX ]]
    then
        tmux_window_name="$(tmux display-message -p '#W')"
    fi
    if [[ $tmux_window_name == bash ]]
    then
        tmux rename-window "$namespace"
    fi
    if [[ -e $KUBECTL_WATCH_CGROUP ]]
    then
        if [[ -e /sys/fs/cgroup/viddy ]]
        then
            :
        else
            GID=users
            sudo /run/current-system/profile/bin/cgcreate -a "${USER}:${GID}" -t "${USER}:${GID}" -g memory,cpu:viddy
            echo 134217728 > /sys/fs/cgroup/viddy/memory.max
        fi
        sudo cgclassify -g cpu,memory:viddy $$
        CGROUP_LOGLEVEL=INFO /run/current-system/profile/bin/cgexec -g memory,cpu:viddy viddy "${viddy_args[@]}" viddy_command
    else
        viddy "${viddy_args[@]}" viddy_command
    fi
    # XXX: Dirty hack and has issue that requires to run twice because of
    # racing with tmuxifier.
    if [[ -n $TMUX ]]
    then
        if [[ $(tmux display-message -p '#W') == main ]]
        then
            tmux rename-window "bash"
        else
            tmux rename-window "$tmux_window_name"
        fi
    fi
fi