2018-02-02 14:12:44 +01:00 
										
									 
								 
							 
							
								
							 
							
								 
							 
							
							
								#!/bin/bash -e
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								############################################################################ begin logging
							 
						 
					
						
							
								
									
										
										
										
											2018-02-06 14:26:21 +01:00 
										
									 
								 
							 
							
								
									
										 
									 
								
							 
							
								 
							 
							
							
								# see if it supports colors...
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								ncolors=$(tput colors)
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								if test -n "$ncolors" && test $ncolors -ge 8; then
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								    bold="$(tput bold)"
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								    underline="$(tput smul)"
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								    standout="$(tput smso)"
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								    normal="$(tput sgr0)"
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								    black="$(tput setaf 0)"
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								    red="$(tput setaf 1)"
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								    green="$(tput setaf 2)"
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								    yellow="$(tput setaf 3)"
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								    blue="$(tput setaf 4)"
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								    magenta="$(tput setaf 5)"
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								    cyan="$(tput setaf 6)"
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								    white="$(tput setaf 7)"
							 
						 
					
						
							
								
									
										
										
										
											2018-02-02 14:12:44 +01:00 
										
									 
								 
							 
							
								
							 
							
								 
							 
							
							
								fi
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								append_msg() {
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								    if test $# -ne 0; then
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								        echo -n ": ${bold}$*"
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								    fi
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								    echo "${normal}"
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								}
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								# write a message
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								message() {
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								    if test $# -eq 0; then
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								        return
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								    fi
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								    echo "${bold}${while}$*${normal}" 1>&2
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								}
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								# write a success message
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								success() {
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								    echo -n "${bold}${green}success" 1>&2
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								    append_msg $* 1>&2
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								}
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								# write a notice
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								notice() {
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								    echo -n "${bold}${yellow}notice" 1>&2
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								    append_msg $* 1>&2
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								}
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								# write a warning message
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								warning() {
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								    echo -en "${bold}${red}warning" 1>&2
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								    append_msg $* 1>&2
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								}
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								# write error message
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								error() {
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								    echo -en "${bold}${red}error" 1>&2
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								    append_msg $* 1>&2
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								}
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								# run a command, print the result and abort in case of error
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								# option: --ignore: ignore the result, continue in case of error
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								run() {
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								    ignore=1
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								    while test $# -gt 0; do
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								        case "$1" in
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								            (--ignore) ignore=0;;
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								            (*) break;;
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								        esac
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								        shift;
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								    done
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								    echo -n "${bold}${yellow}running:${white} $*${normal} … "
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								    set +e
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								    result=$($* 2>&1)
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								    res=$?
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								    set -e
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								    if test $res -ne 0; then
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								        if test $ignore -eq 1; then
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								            error "failed with return code: $res"
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								            if test -n "$result"; then
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								                echo "$result"
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								            fi
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								            exit 1
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								        else
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								            warning "ignored return code: $res"
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								        fi
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								    else
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								        success
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								    fi
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								}
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								############################################################################ error handler
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								function traperror() {
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								    set +x
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								    local err=($1) # error status
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								    local line="$2" # LINENO
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								    local linecallfunc="$3"
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								    local command="$4"
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								    local funcstack="$5"
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								    IFS=" "
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								    for e in ${err[@]}; do
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								        if test -n "$e" -a "$e" != "0"; then
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								            error "line $line - command '$command' exited with status: $e (${err[@]})"
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								            if [ "${funcstack}" != "main" -o "$linecallfunc" != "0" ]; then
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								                echo -n "   ... error at ${funcstack} " 1>&2
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								                if [ "$linecallfunc" != "" ]; then
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								                    echo -n "called at line $linecallfunc" 1>&2
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								                fi
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								                echo
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								            fi
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								            exit $e
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								        fi
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								    done
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								    exit 0
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								}
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								# catch errors
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								trap 'traperror "$? ${PIPESTATUS[@]}" $LINENO $BASH_LINENO "$BASH_COMMAND" "${FUNCNAME[@]}" "${FUNCTION}"' ERR SIGINT INT TERM EXIT
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								######################################################### commandline parameter evaluation
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								exec=0
							 
						 
					
						
							
								
									
										
										
										
											2018-03-22 11:27:04 +01:00 
										
									 
								 
							 
							
								
									
										 
									 
								
							 
							
								 
							 
							
							
								stop=0
							 
						 
					
						
							
								
									
										
										
										
											2018-02-02 14:12:44 +01:00 
										
									 
								 
							 
							
								
							 
							
								 
							 
							
							
								log=0
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								stacks=
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								short=0
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								while test $# -gt 0; do
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								    case "$1" in
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								        (--help|-h) less <<EOF
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								SYNOPSIS
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								  $0 [OPTIONS] [stacks]
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								OPTIONS
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								  --help, -h                 show this help
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								  --log, -l                  show log command
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								  --exec, -e                 show exec command
							 
						 
					
						
							
								
									
										
										
										
											2018-03-22 11:27:04 +01:00 
										
									 
								 
							 
							
								
									
										 
									 
								
							 
							
								 
							 
							
							
								  --stop, -x                 show stop command
							 
						 
					
						
							
								
									
										
										
										
											2018-02-02 14:12:44 +01:00 
										
									 
								 
							 
							
								
							 
							
								 
							 
							
							
								  --short, -s                only show errors
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								  stacks                     optional space separated list of stacks
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								DESCRIPTION
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								Show status of all docker swarm stacks. The problems of docker swarm ps are, that you can only specify one stack to analyze, and then you get too much output. So this command lists all stacks and clearly shows which stacks are running and which stacks have a problem.
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								EOF
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
										    exit;;
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
									(--log|-l) log=1;;
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
									(--exec|-e) exec=1;;
							 
						 
					
						
							
								
									
										
										
										
											2018-03-22 11:27:04 +01:00 
										
									 
								 
							 
							
								
									
										 
									 
								
							 
							
								 
							 
							
							
									(--stop|-x) stop=1;;
							 
						 
					
						
							
								
									
										
										
										
											2018-02-02 14:12:44 +01:00 
										
									 
								 
							 
							
								
							 
							
								 
							 
							
							
									(--short|-s) short=1;;
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								        (*) stacks="$*"; break;;
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								    esac
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								    if test $# -eq 0; then
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								        error "missing parameter, try $0 --help"; exit 1
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								    fi
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								    shift;
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								done
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								##################################################################################### Main
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								stacks=${stacks:-$(docker stack ls --format='{{.Name}}')}
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								services=$(for stack in ${stacks}; do
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
									       status=$(docker stack ps --no-trunc --filter="desired-state=running" --format="{{.CurrentState}};{{.Name}};{{.Node}};{{.DesiredState}};{{.CurrentState}};{{.ID}};{{.Error}}" ${stack} | sed 's,^[^ ]* ,,')
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
									       IFS="
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								"
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
									       for service in ${status}; do
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
										   time=${service%%;*}
							 
						 
					
						
							
								
									
										
										
										
											2018-03-22 11:31:18 +01:00 
										
									 
								 
							 
							
								
									
										 
									 
								
							 
							
								 
							 
							
							
										   echo "$(date +%s -d "$(sed 's,about an* ,1 ,;s,less than an* ,0 ,' <<<${time})");${service#*;}"
							 
						 
					
						
							
								
									
										
										
										
											2018-02-02 14:12:44 +01:00 
										
									 
								 
							 
							
								
							 
							
								 
							 
							
							
									       done
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
									   done | sort -hr)
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								IFS="
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								"
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								for service in ${services}; do
							 
						 
					
						
							
								
									
										
										
										
											2018-03-22 11:27:04 +01:00 
										
									 
								 
							 
							
								
									
										 
									 
								
							 
							
								 
							 
							
							
								    awk -F';' -v dolog=$log -v doexec=$exec -v dostop=$stop -v doshort=$short '
							 
						 
					
						
							
								
									
										
										
										
											2018-02-02 14:12:44 +01:00 
										
									 
								 
							 
							
								
							 
							
								 
							 
							
							
								                                          {color="'"${green}"'"}
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								            $5 ~ /second|minute/          {color="'"${yellow}"'"}
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								            $5 !~ /^Running/              {printf "'"${red}"'%-15s%-40s%s %s%s\n", $3, $2, $5, $7, "'"${normal}"'"}
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								            $5 ~ /^Running/ && doshort==0 {printf "%s%-15s%-40s%s%s\n", color, $3, $2, $5, "'"${normal}"'"}
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								            dolog==1 && (doshort==0 || $5 !~ /^Running/) {printf "ssh %s docker logs -f %s.%s\n", $3, $2, $6}
							 
						 
					
						
							
								
									
										
										
										
											2018-03-22 11:27:04 +01:00 
										
									 
								 
							 
							
								
									
										 
									 
								
							 
							
								 
							 
							
							
								            doexec==1 && (dohort==0 || $5 !~ /^Running/) {printf "ssh -t %s docker exec -it %s.%s bash\n", $3, $2, $6}
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								            dostop==1 && (dohort==0 || $5 !~ /^Running/) {printf "ssh %s docker stop %s.%s\n", $3, $2, $6}
							 
						 
					
						
							
								
									
										
										
										
											2018-02-02 14:12:44 +01:00 
										
									 
								 
							 
							
								
							 
							
								 
							 
							
							
								        ' <<<${service}
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								done