wrdk/scripts/progress

#!/bin/sh
# show the status of stamp files and run programs on the rendering servers

ERROR()
{
  echo error: $*
  exit 1
}

USAGE()
{
  [ -z "$1" ] || echo error: $*
  echo usage: $(basename "$0") '<options>'
  echo '       --help          -h         this message'
  echo '       --work          -w         show work directory files'
  echo '       --image         -i         show image directory files'
  echo '       --stamps        -s         show status of stamps'
  echo '       --analyse       -a         analyse the log file'
  echo '       --git           -g         run git pull --rebase'
  echo '       --process=<n>   -p<n>      show python/php processes cut line 1..n [unlimited]'
  echo '       --base=<dir>    -b <dir>   set the base directory [samo]'
  echo '       --tail=<n>      -t<n>      show last line of <base_dir>/log [3]'
  echo '       --memory        -m         show memory status'
  echo '       --list=<n-m>    -l <n-m>   restrict to certain machines [1-9]'
  echo '       --language=<xx> -L <xx>    process for language [en]'
  echo '       --suffix=<s>    -S <s>     set the language suffix [pedia]'
  echo '       --run           -r         start the Run script in a screen {index, parse, render}'
  echo '       --re-index      -I         start the Run script in a screen {index-only}'
  echo '       --re-render     -R         start the Run script in a screen {re-render}'
  echo '       --copy          -c         run the copy process i.e. {Copy --to=8 -l <lang>}'
  echo '       --combine       -k         run the combine process'
  echo '       --get           -G         get the data files'
  echo '       --sim4          -4         install fonts, nls and run simulator'
  echo '       --new-session   -N         start a new screen session for run and copy to use'
  echo '       --fetch-log     -f         fetch log files'
  echo '       --xml-link      -x         link wiki .xml file to latest version'
  echo '       --high-compress -H         enable compression for 4GB SD card (only for --language=en)'
  echo '       --abort         -A         abort remote PHP/Python/make'
  exit 1
}

# main program
# ------------

verbose=no
language=en
suffix=pedia
list=''
combine_host=8
base_dir=samo
compression=low

getopt=/usr/local/bin/getopt
[ -x "${getopt}" ] || getopt=getopt
args=$(${getopt} -o hvb:swigp::t::aml:L:S:rRIckG4NfxHA --long=help,verbose,base:,stamps,work,image,git,process::,tail::,analyse,memory,list:,language:,suffix:,run,re-render,re-index,copy,combine,get,sim4,new-session,fetch-log,xml-link,high-compress,abort -- "$@") || exit 1
# replace the arguments with the parsed values
eval set -- "${args}"

while :
do
  case "$1" in
    -v|--verbose)
      verbose=yes
      shift
      ;;

    -b|--base)
      base_dir="$2"
      shift 2
      ;;

    -s|--stamps)
      RenderDo --yes ${list} --brief --post='sed s/stamp-//g' "cd '${base_dir}/work/${language}${suffix}'"' && printf "\33[34;1m%s\33[0;m " stamp* && echo'
      shift
      ;;

    -w|--work)
      RenderDo --yes ${list} --brief "cd '${base_dir}' && ls -l 'work/${language}${suffix}'"
      shift
      ;;

    -N|--new-session)
      for host in $(RenderDo --print-list)
      do
        StartScreen --cd="${base_dir}" "${host}"
      done
      shift
      ;;

    -r|--run)
      c="scripts/Run --compress='${compression}' --clear --language='${language}' --suffix='${suffix}'"
      RenderDo --yes ${list} --brief "byobu -X eval chdir 'chdir ${base_dir}' 'exec ${c}'"
      shift
      ;;

    -R|--re-render)
      c="scripts/Run --compress='${compression}' --re-render --language='${language}' --suffix='${suffix}'"
      RenderDo --yes ${list} --brief "byobu -X eval chdir 'chdir ${base_dir}' 'exec ${c}'"
      shift
      ;;

    -I|--re-index)
      c="scripts/Run --compress='${compression}' --index-only --language='${language}' --suffix='${suffix}'"
      RenderDo --yes ${list} --brief "byobu -X eval chdir 'chdir ${base_dir}' 'exec ${c}'"
      shift
      ;;

    -c|--copy)
      c="scripts/Copy --base='${base_dir}' --to='${combine_host}' --language='${language}' --suffix='${suffix}'"
      RenderDo --yes ${list} --brief "byobu -X eval chdir 'chdir ${base_dir}' 'exec ${c}'"
      shift
      ;;

    -i|--image)
      RenderDo --yes ${list} --brief "cd '${base_dir}' && ls -l 'image/${language}${suffix}'"
      shift
      ;;

    -f|--fetch-log)
      for host in $(RenderDo --print-list)
      do
        lf="${host}-${language}.log"
        echo fetch log from: ${host} to: ${lf}
        RenderDo --yes --list="${host##*[^0-9]}" "cat '${base_dir}/${language}.log'" > "${lf}"
      done
      shift
      ;;

    -g|--git)
      RenderDo --yes ${list} --verbose "cd '${base_dir}' && git checkout . && git pull --rebase"
      shift
      ;;

    -p|--process)
      if [ -n "$2" ]
      then
        RenderDo --yes ${list} --brief --post="cut -b 1-$2" 'echo;ps -o pid,cmd -u wr --forest'
      else
        RenderDo --yes ${list} --brief 'echo;ps -o pid,cmd -u wr --forest'
      fi
      shift 2
      ;;

    -t|--tail)
      t=3
      [ -n "$2" ] && t="$2"
      RenderDo --yes ${list} --brief "tail -n '${t}' '${base_dir}'/${language}.log"
      shift 2
      ;;

    -a|--analyse)
      RenderDo --yes ${list} --brief "cd '${base_dir}' && ./scripts/AnalyseLog '${language}.log'"
      shift
      ;;

    -m|--memory)
      RenderDo --yes ${list} --brief 'echo Memory Status \(Megabytes\); free -m'
      shift
      ;;

    -k|--combine)
      RenderDo --yes --list="${combine_host}" --brief "cd '${base_dir}' && make VERBOSE='${verbose}' WORKDIR='work' DESTDIR='image' WIKI_FILE_PREFIX=wiki WIKI_LANGUAGE='${language}' combine"
      shift
      ;;

    -l|--list)
      list="--list=$2"
      shift 2
      ;;

    -L|--language)
      language="$2"
      shift 2
      ;;

    -S|--suffix)
      base_dir="$2"
      shift 2
      ;;

    -G|--get)
      rsync -cavHx --progress --exclude='*-tmp' "wr@render${combine_host}:${base_dir}/image/${language}${suffix}" image/
      shift
      ;;

    -4|--sim4)
      make DESTDIR="image" nls-install fonts-install sim4
      shift
      ;;

    -x|--xml-link)
      RenderDo --yes ${list} --brief "cd '${base_dir}' && rm -f '${language}wiki-pages-articles.xml' && ln -fs \$(ls -1 ../'${language}'wiki-*-pages-articles.xml |tail -n 1) '${language}wiki-pages-articles.xml' && ls -l '${language}'*.xml"
      shift
      ;;

    -H|--high-compress)
      compression=high
      shift
      break
      ;;

    -A|--abort)
      RenderDo --yes ${list} --brief 'killall php python make'
      shift
      ;;

    --)
      shift
      break
      ;;

    -h|--help)
      USAGE
      ;;

    *)
      USAGE invalid option: $1
      ;;
  esac
done

echo
echo completed