dotfiles/dot_local/bin/executable_connect

#!/usr/bin/env bash

set -e
set -o pipefail

help_main()
{
    echo "\
Usage: connect COMMANDS ARGS...
Run COMMANDS with ARGS

COMMAND must be one of the sub-commands listed below:

  general commands
    uptime        show uptime on SERVER
    php           show PHP status on SERVER

  disk commands
    io            show input output stats on SERVER
    stat          show iostat on SERVER
    sg            show disk performance on SERVER

  docker commands
    containers    list running containers on SERVER
    deploy        deploy containers to SERVER
    images        list container images on SERVER

  MySQL commands
    clean         kill MySQL connections on SERVER
    mysql         connect to MySQL on server

  networking commands
    ip            invoke ip address and route on SERVER
    firewall      print iptables rules on SERVER
    interfaces    list network interfaces on SERVER
    ip-filter     control ip-filter on server
    net           show network interfaces stats on SERVER
    ping          ping a SERVER
    protected     add DOMAIN to protected on SERVER
    route         show network route to SERVER
    sniff         run tcpdump on SERVER
    traceroute    show traceroute to SERVER

  logs commands
    apache        show USER's Apache logs
    te            show taskexecutor logs

  shell commands
    shell         connect to CISCO
    ssh           connect to server via SSH
    sshrc         connect to server via SSH with configuration

  NGINX commands
    filter        add IP address to ip-filter on SERVER
    unfilter      delete IP address from ip-filter on SERVER
    nginx         show top requests to NGINX
    nginx-config  show NGINX config on SERVER

  junos commands
    br1-mr14.intr miran border
    sr1-mr13-14.intr miran sr
    sr1-dh507-508.intr datahouse sr

Report bugs to: go.wigust@gmail.com."
}

help_mysql()
{
    echo "\
Usage: connect mysql [OPTION] COMMANDS HOST...
Run COMMANDS with ARGS

  -c, --client=CLIENT    mysql client
  -h, --help             display this help and exit

COMMAND must be one of the sub-commands listed below:

  connections
  filter

Report bugs to: go.wigust@gmail.com."
}

help_ip_filter()
{
    echo "\
Usage: connect ip-filter [OPTION]...
Connect to HOST's ip-filter.

  -a, --add              add IP to ip-filter
  -r, --remove           remove IP from ip-filter
  -d, --describe         list IP address in ip-filter
  -d, --host             ip-filter host
  -h, --help             display this help and exit

Example: 'connect ip-filter --host web30.intr --describe'.

Report bugs to: go.wigust@gmail.com."
}

DEBUG="${DEBUG:-false}"

telnet_expect_interact()
{
    TELNET_PASSWORD="$(pass show majordomo/private/general)"    \
    ENABLE_PASSWORD="$(pass show majordomo/private/ssh/router)" \
    cisco-interact "$@"
}

ssh_expect()
{
    PYTHONPATH=''                                               \
    SSH_KEY="$HOME/.ssh/id_rsa_majordomo_eng"                            \
    BECOME_PASSWORD="$(pass show majordomo/private/ssh/eng)"    \
    ssh-sudo "$@"
}

sshrc_sudo()
{
    if [ $# -lt 2 ]
    then
        sshrc "$1"
    else
        ssh -q -t "$1" -- "set +o history; sudo --stdin --validate --prompt='' <<< $(pass show majordomo/private/ssh/eng); exec -a sudo sudo -i ${*:2}"
    fi
}

ssh_sudo()
{
    ssh -q -t "$1" -- "set +o history; sudo --stdin --validate --prompt='' <<< $(pass show majordomo/private/ssh/eng); exec -a sudo sudo -i ${*:2}"
}

mysql_command()
{
    mysql --silent --user="root" --password="$(pass show majordomo/public/web/mysql/root)" --host="$1" "${@:2}"
}

mysql_list_users_by_connections()
{
    mysql_command "$1" --execute="SELECT count(ID) as connections, USER FROM INFORMATION_SCHEMA.PROCESSLIST GROUP BY USER ORDER BY connections;"
}

connect_junos()
{
    sshpass -p"$(pass show majordomo/private/ssh/router)"       \
            ssh -F /dev/null                                    \
            -q                                                  \
            -i "$HOME/.ssh/id_rsa_majordomo_eng"                \
            -o UserKnownHostsFile=/dev/null                     \
            -o StrictHostKeyChecking=no                         \
            -l root                                             \
            "$1" "${@:2}"
}

connect_h3c()
{
    sshpass -p"$(pass show majordomo/private/h3c/oleg)"       \
            ssh -F /dev/null                                    \
            -q                                                  \
            -o UserKnownHostsFile=/dev/null                     \
            -o StrictHostKeyChecking=no                         \
            -l oleg                                             \
            "$1" "${@:2}"
}

case "$1" in
    --help)
        help_main
        exit 0
        ;;
    br1-mr14.intr|sr1-mr13-14.intr|sr1-dh507-508.intr|sw2-mr13.intr)
        case "$2" in
            bgp)
                connect_junos "$1" cli 'show bgp summary'
                ;;
            log)
                number="${3:-$(connect_junos "$1" cli 'show system commit' | wc -l)}"
                for n in $(seq 0 "$number")
                do
                    echo "root@$1> show system rollback $((n+1)) compare $n"
                    connect_junos "$1" cli "show system rollback $((n+1)) compare $n"
                done
                ;;
            configuration)
                connect_junos "$1" cli "show configuration"
                ;;
            *)
                connect_junos "$1" "${@:2}"
                ;;
        esac
        exit 0
        ;;
    galera)
        case "$2" in
            show)
                parallel --will-cite -k "printf '{1}: '; mysql -s -h {1} -u root -p\"$(pass show majordomo/public/maxscale.intr/root)\" -e 'SELECT @@GLOBAL.gtid_binlog_pos'" ::: galera{1..3}.intr
                if [[ $(connect galera-backup.intr docker exec galera-slave mysql -uroot -p"$(pass show majordomo/public/maxscale.intr/root)" -e 'SHOW ALL SLAVES STATUS\G' |& grep -c 'Slave_SQL_Running: Yes') -lt 3 ]]
                then
                    printf '\033[35mWARNING:\033[0m galera-backup.intr: Slave_SQL_Running: is less than 3.\n'
                else
                    printf "\033[32mINFO:\033[0m galera-backup.intr: Slave_SQL_Running: is equal to 3.\n"
                fi
                # connect galera-backup.intr docker exec galera-slave mysql -uroot -p"$(pass show majordomo/public/maxscale.intr/root)" -e 'SELECT @@GLOBAL.gtid_binlog_pos'
                ;;
            snapshots)
                echo "zfs clone pool@zfs-auto-snap_scripted-2019-02-21-1005 pool/mariafromsnap"
                echo "docker run --rm --hostname galera-slave-temp --name galera-slave-temp -eMYSQL_ALLOW_EMPTY_PASSWORD=1 -v /pool/mariadb/confdir-ro:/etc/mysql/conf.d -v /pool/mariafromsnap/mariadb/datadir:/var/lib/mysql mariadb:10.3"
                echo "MariaDB [(none)]> stop ALL slaves;"
                echo "zfs destroy pool/mariafromsnap"

                echo "List ZFS snapshots:"
                connect ssh galera-backup.intr zfs list -t snapshot
                ;;
        esac
        ;;
    filter)
        case "$2" in
            list)
                for web in web15 web16 web17 web18 web19 web20 web21 web22 web23 web25 web26 web27 web28 web29 web30 web31 web32 web33 web34 web35 web36 web37
                do
                    curl -s http://"$web.intr"/ip-filter | sed "s/^/$web.intr\t/g; s/\s/\t/g"
                done
                ;;
            *)
                for ip in $(
                               connect ssh "$2" -- docker logs --tail 1000 nginx \
                                   | awk '{print $1}' \
                                   | sort \
                                   | uniq -c \
                                   | sort -nr \
                                   | fzf -m \
                                   | awk '{print $2}' \
                                   | tac
                           )
                do
                    whois "$ip" | tail -n $(( $(tput lines) - 2))
                    read -p "Are you sure want to filter $ip IP address? " -n 1 -r
                    echo
                    if [[ $REPLY =~ ^[Yy]$ ]]
                    then
                        curl --silent --head --request PUT "$2/ip-filter/$ip?ttl=7200&action=setCookie"
                    fi
                done
        esac
        ;;
    firewall)
        connect ssh "$2" -- iptables --line-numbers -n -v -L | jc --iptables -p | yq -y .
        ;;
    unfilter)
        ip="$(connect ssh "$2" -- docker logs --tail 1000 nginx | awk '{print $1}' | sort | uniq -c | sort -nr | fzf | awk '{print $2}')"
        curl --silent --head --request DELETE "$2/ip-filter/$ip?ttl=7200&action=setCookie"
        ;;
    kvm)
        racks=(
            5-08
            5-07
        )
        for rack in "${racks[@]}"
        do
            (
                set +e
                for host in $(fping -a $(printf "%s.intr\n" $(mjru-infa server | grep Датахаус | grep kvm | grep -v nvme | awk "/$rack/ { print \$3 }") | xargs echo))
                do
                    echo -e "\n\n@ ${host}@${rack}"
                    ssh -q "$host" 'free -h; df -h /kvm'
                done
            )
        done
        ;;
    ssh)
        host="$2"
        if [[ "$host" != *.intr ]] && [[ "$host" != *sw* ]]
        then
            host+=.intr
        fi

        [ "$DEBUG" == true ] && echo "Connect with ssh_sudo() to $host."
        ssh_sudo "$host" "${@:3}"
        ;;
    sshrc)
        host="$2"
        if [[ "$host" != *.intr ]] && [[ "$host" != *sw* ]]
        then
            host+=.intr
        fi

        [ "$DEBUG" == true ] && echo "Connect with sshrc_sudo() to $host."
        sshrc_sudo "$host" "${@:3}"
        ;;
    emacs)
        host=$3; [[ "$host" == *.intr ]] || host+=.intr
        GTK_THEME='' command emacs -nw "/ssh:$host|sudo:$host:"
        ;;
    io)
        ssh_sudo "$2" iotop -qqqtPbod1 | cut --characters="-$(tput cols)"
        ;;
    stat)
        ssh_sudo "$2" env S_COLORS=true iostat -xchztdk 2
        ;;
    interfaces)
        connect ssh "$2" ifconfig \
            | jc --ifconfig \
            | jq --raw-output '.[] | [.rx_packets, .tx_packets, .name, .ipv4_addr, .ipv4_mask, .ipv4_bcast, .mac_addr] | @tsv' \
            | expand -t1 | column -t | sort -n
        ;;
    route)
        connect ssh "$2" -- route | jc --route | jq | yq -y .
        ;;
    net)
        ssh_sudo "$2" iftop -nNP -i "$(connect ssh "$2" ifconfig | jc --ifconfig | jq --raw-output '.[] | .name' | fzf)"
        ;;
    sniff)
        ssh_sudo "$2" tcpdump -vvvApni "$(connect ssh "$2" ifconfig | jc --ifconfig | jq --raw-output '.[] | .name' | fzf)" "${@:3}"
        ;;
    mpstat)
        ssh_sudo "$2" mpstat 1
        ;;
    sg)
        echo "Hint: Install sg3-utils by invoking 'apt install sg3-utils'"
        for i in {1..8}; do
            echo -e "\n@ /dev/sg$i"
            time ssh_expect "${2%.intr}" "sg_read if=/dev/sg$i bs=512 count=100000"
        done
        ;;
    ping)
        ping -n -c "${4:-3}" "${2%.intr}"
        ;;
    traceroute)
        sudo traceroute "${2%.intr}"
        ;;
    containers)
        ssh_expect "${2%.intr}" docker ps --no-trunc --format "'table {{.ID}}\t{{.Names}}\t{{.Status}}'"
        ;;
    images)
        ssh_expect "${2%.intr}" docker images --no-trunc --format "'{{.ID}}: {{.Repository}}'"
        ;;
    web*)
        host="$1"; [[ "$host" == *.intr ]] || host+=.intr
        ssh_sudo "$host" "${@:2}"
        ;;
    shell)
        host="$2"; [[ "$host" == *.intr ]] || host+=.intr
        if ! ssh_sudo "$host" "${@:3}"
        then
            TELNET_PASSWORD="$(pass show majordomo/private/general)" cisco "$host" "${@:3}"
        fi
        ;;
    nginx)
        ssh_expect "${2%.intr}" docker logs --tail "${3:-1000}" nginx \
            | awk '{ print $1 }' \
            | grep --invert-match --fixed-strings '127.0.0.1' \
            | sort \
            | uniq --count \
            | sort --numeric-sort
        ;;
    nginx-config)
        connect ssh "$2" cat "$(connect ssh "$2" grep -RF server_name /etc/nginx/sites-available | sort -u | fzf | cut -d: -f 1)"
        ;;
    mysql)
        case "$2" in
            connections)
                mysql_list_users_by_connections "$3" | expand -t1 | column -t
                exit 0
                ;;
            filter)
                command="KILL USER $(mysql_list_users_by_connections "$3" | fzf | awk '{ print $NF }')"
                echo "$3: $command"
                mysql_command "$3" --execute="$command;"
                exit 0
                ;;
        esac

        if ! OPTS="$(getopt --options c:h --long client:,help --name parse-options -- "$@")"
        then
            echo "Failed parsing options."
            exit 1
        fi

        eval set -- "$OPTS"

        while true; do
            case "$1" in
                -h | --help )
                    help_mysql
                    exit 0
                    ;;
                -c | --client )
                    MYSQL_CLIENT="$2"
                    shift 2
                    ;;

                -- )
                    shift
                    break
                    ;;
                ,* )
                    break
                    ;;
            esac
        done
        case $MYSQL_CLIENT in
            mysql )
                mysql -h"${2%.intr}" -p"$(pass show majordomo/public/web/mysql/root)" -uroot
                ;;
            mycli )
                PAGER='pspg -s 14 -X --force-uniborder --quit-if-one-screen -s 16' \
mycli --password "$(pass show majordomo/public/web/mysql/root)" -d "${2%.intr}"
                ;;
            * )
                PAGER='pspg -s 14 -X --force-uniborder --quit-if-one-screen -s 16' \
mycli --password "$(pass show majordomo/public/web/mysql/root)" -d "${2%.intr}"
                ;;
        esac
        ;;
    clean)
        echo "Kill MySQL connections"
        host="${2:-127.0.0.1}"
        user="root"
        password="$(pass show majordomo/public/web/mysql/root)"
        (
            set +e
            for id in $(mysql --silent --user="$user" --password="$password" --host="$host" --execute="SELECT id FROM information_schema.processlist WHERE user like 'u%'"); do
                echo "$id"
                mysql --silent --user="$user" --password="$password" --host="$host" --execute="kill $id;"
            done
        )
        ;;
    ip-filter)
        if ! OPTS="$(getopt --options arHhd --long add:,remove:,host:,help,describe --name parse-options -- "$@")"
        then
            echo "Failed parsing options."
            exit 1
        fi
        eval set -- "$OPTS"
        while true; do
            case "$1" in
                -h | --help )
                    help_ip_filter
                    exit 0
                    ;;
                -H | --host )
                    IP_FILTER_HOST="$2"
                    shift 2
                    ;;
                -a | --add )
                    IP_FILTER_HOST_ADD="$2"
                    shift 2
                    ;;
                -r | --remove )
                    IP_FILTER_HOST_DELETE="$2"
                    shift 2
                    ;;
                -d | --describe)
                    IP_FILTER_HOST_DESCRIBE="$2"
                    shift
                    ;;
                -- )
                    shift
                    break
                    ;;
                ,* )
                    break
                    ;;
            esac
        done
        if [ -n "$IP_FILTER_HOST_DESCRIBE" ]
        then
            echo "name: ip-filter-$IP_FILTER_HOST"
            echo -n "blocked: "
            curl --silent --request GET "$IP_FILTER_HOST/ip-filter" \
                | cut -d' ' -f 1 | sort --numeric-sort | xargs echo
            exit 0
        fi
        if [ -n "$IP_FILTER_HOST_DELETE" ]
        then
            curl --silent --head --request DELETE "$IP_FILTER_HOST/ip-filter/$3?ttl=7200&action=setCookie"
        fi
        if [ -n "$IP_FILTER_HOST_ADD" ]
        then
            curl --silent --head --request PUT "$IP_FILTER_HOST/ip-filter/$2?ttl=7200&action=setCookie"
        fi
    ;;
    protected)
        password="$(pass show majordomo/public/web/protected/token)"
        for domain in $(PYTHONPATH= grafana --host "$2")
        do
            if [[ -n $TTL ]]
            then
                echo curl -H"'Authorization: ${password}'" --include --request PUT "${2}/protected/${domain}?ttl=${TTL}"
                echo curl -H"'Authorization: ${password}'" --include --request PUT "${2}/protected/www.${domain}?ttl=${TTL}"
            else
                echo curl --include --request PUT "${2}/protected/${domain}"
                echo curl --include --request PUT "${2}/protected/www.${domain}"
            fi
        done
        ;;
    te)
        ssh_expect "$2" tail --lines="${3:-1000}" /var/log/taskexecutor.log \
            | grep -v 'malware_report' \
            | sed '/^[[:space:]]*$/d'
        ;;
    apache)
        ssh_expect "${2%.intr}" sh -c "'cat /home/$4/logs/*access.log'"
        ;;
    deploy)
        host="$2"; [[ "$host" == *.intr ]] || host+=.intr
        ansible-playbook --limit "$host" --ask-become-pass "$HOME/bin/web-docker-pull"
        ;;
    ip)
        case "$3" in
            a*)
                connect ssh "$2" -- ip -color=always address
                ;;
            r*)
                connect ssh "$2" -- ip -color=always route
                ;;
            *)
                connect ssh "$2" -- ip -color=always address
                connect ssh "$2" -- ip -color=always route
                ;;
        esac
       ;;
    uptime)
        connect ssh "$2" -- uptime | jc --uptime
        ;;
    ns)
        printf "NOTE: No expired and registrant are checked.\n\n"
        for ns in ns ns2 ns3 ns4
        do
            dig +short MX yacrm.ru @$ns.majordomo.ru
            echo ""
        done
        ;;
    php)
        parallel --will-cite -k 'printf "name: %s\n" apache2-{2}-{3}; curl --max-time 10 -L -s -o /dev/null -w "ip_address: %{remote_ip}\nstatus_code: %{http_code}" http://{1}/apache2/{2}/{3}/phpinfo.php; printf "\n\n"' ::: \
                 "$2" ::: \
                 "php44" "php52" "php53" "php54" "php55" "php56" "php70" "php71" "php72" "php73" "php74" "php80" ::: \
                 "default" "unsafe" "hardened" "hardened_nochmod" \
            | recsel -e 'status_code != 200 && name != "apache2-php44-default" && name != "apache2-php44-unsafe" && name != "apache2-php44-hardened" && name != "apache2-php44-hardened_nochmod" && name != "apache2-php80-unsafe" && name != "apache2-php80-hardened" && name != "apache2-php80-hardened_nochmod"'
        ;;
esac