woof-CE-libre/woof-code/support/fix-puppy-dbs

#!/bin/bash
#(c) Copyright Barry Kauler June 2009
#Woof has various Puppy db files, Packages-puppy-[2345]-official and one that is
#for a specific compat-distro build -- for example, a compat-distro build based
#on the Puppy 4.x series has file Packages-puppy-4xx-official (defined in DISTRO_PKGS_SPECS-puppy-4).
#this script synchronises them all, including with the files at ibiblio.org...
#111124 fix finding Packages-puppy-*-official.
#111124 take out the 'clear', difficult to debug.
#111129 recommend not update db files with online entries.
#120315 ignore if repo db file not wanted.
#121102 file DISTRO_SPECS has new variable DISTRO_DB_SUBNAME. ex: for 14.0-based slacko, DISTRO_DB_SUBNAME=slacko14
#121102 pet db file not only, use local file only.
#130306 fix arch linux build.

#db entry format: pkgname|nameonly|version|pkgrelease|category|size|path|fullfilename|dependencies|description|
#ex: abiword-1.2.4|abiword|1.2.4|5|Document|999K|slackware/ab|abiword-1.2.4-5-i486.tgz|+aiksausus,+gtk2|a nice wordprocessor|
#optionally on the end: compileddistro|compiledrelease|repo| (fields 11,12,13)
#ex: slackware|12.2|official|

export LANG=C

[ ! -f ./DISTRO_PET_REPOS ] && exit 1
[ ! -f ./DISTRO_SPECS ] && exit 1

#this script executed with current-directory the main woof dir...
. ./DISTRO_SPECS #has DISTRO_BINARY_COMPAT, DISTRO_COMPAT_VERSION
[ ! "$DISTRO_DB_SUBNAME" ] && DISTRO_DB_SUBNAME="$DISTRO_COMPAT_VERSION" #121102 fallback if DISTRO_DB_SUBNAME not defined in file DISTRO_SPECS.
if [ -f ./DISTRO_PKGS_SPECS-${DISTRO_BINARY_COMPAT}-${DISTRO_COMPAT_VERSION} ];then #w478
. ./DISTRO_PKGS_SPECS-${DISTRO_BINARY_COMPAT}-${DISTRO_COMPAT_VERSION} #has FALLBACKS_COMPAT_VERSIONS
else
. ./DISTRO_PKGS_SPECS-${DISTRO_BINARY_COMPAT} #has FALLBACKS_COMPAT_VERSIONS
fi
. ./DISTRO_PET_REPOS #has PET_REPOS, PACKAGELISTS_PET_ORDER, PKG_DOCS_PET_REPOS
if [ -f ./DISTRO_COMPAT_REPOS-${DISTRO_BINARY_COMPAT}-${DISTRO_COMPAT_VERSION} ];then #w091017
. ./DISTRO_COMPAT_REPOS-${DISTRO_BINARY_COMPAT}-${DISTRO_COMPAT_VERSION}
else
. ./DISTRO_COMPAT_REPOS-${DISTRO_BINARY_COMPAT} #130306
fi

#find out if internet working...
INTERNETWKG="no"
PETREPO="`echo "$PKG_DOCS_PET_REPOS" | tr ' ' '\n' | head -n 1`"
ONE_PET_SITE="`echo -n "$PETREPO" | cut -f 1 -d '|'`"
[ "`wget -t 2 -T 30 --waitretry=20 --spider -S $ONE_PET_SITE -o /dev/stdout 2>/dev/null | grep '200 OK'`" != "" ] && INTERNETWKG="yes"

if [ "$INTERNETWKG" = "no" ];then
 echo
 echo "Internet not working, so cannot check the Packages-puppy-* files with files at $REPOSITE"
 echo "Continuing with local checking only..."
else

 #internet connected, check each of Packages-puppy-[2345]-official local against same file at ibiblio...
 #111124 fix...
 for ONEREPO in `find . -mindepth 1 -maxdepth 1 -name 'Packages-puppy-*-official' | rev | cut -f 1 -d '/' | rev | tr '\n' ' '`
 do
  REMOTEREPO="`echo "$PKG_DOCS_PET_REPOS" | tr ' ' '\n' | grep "$ONEREPO" | head -n 1 | cut -f 2 -d '|'`"
  if [ "$REMOTEREPO" = "" ];then
   #echo
   #echo "ERROR: an entry for $ONEREPO does not exist in variable PKG_DOCS_PET_REPOS"
   #echo "(in file DISTRO_PET_REPOS). Fix it. Quiting."
   #exit 1
   continue #120315 not an error. just not interested in this db file.
  else
   echo
   echo "Downloading ${ONEREPO}..."
   wget --output-document=remotedbfile $REMOTEREPO
   if [ $? -ne 0 ];then
    if [ -f ./$ONEREPO ];then #121102
     echo
     echo "ERROR downloading ${ONEREPO}, using local file only."
     echo -n "Press ENTER to continue: "
     read keepgoing
     continue
    else
     echo
     echo "ERROR downloading ${ONEREPO}, quiting."
     exit 1
    fi
   fi
   DBDIFFS="`diff $ONEREPO remotedbfile`"
   if [ "$DBDIFFS" = "" ];then
    echo "...files are the same."
    continue
   else
    NEWREMOTEPKGS="`echo "$DBDIFFS" | grep '^> ' | sed -e 's%^> %%'`"
    if [ "$NEWREMOTEPKGS" != "" ];then
     echo; echo #clear #echo
     echo "The remote $ONEREPO file has these new entries:"
     echo -e -n "\\033[1;31m" #red
     echo "$NEWREMOTEPKGS"
     echo -e -n "\\033[0;39m" #back to black.
     echo
     echo "Press any printable char then ENTER only to append these to local $ONEREPO"
     echo "or press ENTER only to ignore them. If using Woof for first time,"
     echo "recommend press ENTER only. Online files modified by other developers"
     echo "may introduce unexpected effects to a build."
     echo -n "Recommend press ENTER only: "
     read ignorethem
     if [ "$ignorethem" != "" ];then #111129 reverse logic.
      echo "...adding new packages."
      echo "$NEWREMOTEPKGS" >> $ONEREPO
     fi
    fi
    NEWLOCALPKGS="`echo "$DBDIFFS" | grep '^< ' | sed -e 's%^< %%'`"
    if [ "$NEWLOCALPKGS" != "" ];then
     echo; echo #clear #echo
     echo "WARNING, local $ONEREPO has these entries that are"
     echo "not in the remote db file:"
     echo -e -n "\\033[1;31m" #red
     echo "$NEWLOCALPKGS"
     echo -e -n "\\033[0;39m" #back to black.
     echo
     echo "...this is a matter for the system admin of remote site."
     echo "Note, this script will also check availability of pkgs on remote site and"
     echo "optionally delete invalid entries in local $ONEREPO"
     echo -n "Press ENTER to continue: "
     read keepongoing
    fi
   fi
   rm -f remotedbfile
  fi
 done

 #also check that pkgs actually exist at ibiblio...
 echo; echo #clear #echo
 echo "Press any printable char to check if PET pkgs in local db files"
 echo "actually exist on Internet host site."
 echo "WARNING: THIS TAKES A LONG TIME, RECOMMEND BYPASS"
 echo -n "or ENTER only to bypass check: "
 read dotheyexist
 if [ "$dotheyexist" != "" ];then
  for ONEREPO in `find . -mindepth 1 -maxdepth 1 -name 'Packages-puppy-*-official' | rev | cut -f 1 -d '/' | rev | tr '\n' ' '`
  do
   REMOTEURL="`echo "$PKG_DOCS_PET_REPOS" | tr ' ' '\n' | grep "$ONEREPO" | head -n 1 | cut -f 2 -d '|' | rev | cut -f 2-9 -d '/' | rev`"
   LOCALPKGS="`cat $ONEREPO`"
   echo "$LOCALPKGS" |
   while read ONELOCALPKG
   do
    SUBDIRECTORY="`echo -n "$ONELOCALPKG" | cut -f 7 -d '|'`"
    FULLPKGNAME="`echo -n "$ONELOCALPKG" | cut -f 8 -d '|'`"
    wget -t 2 -T 20 --waitretry=20 --spider ${REMOTEURL}/${SUBDIRECTORY}/${FULLPKGNAME}
    if [ $? -ne 0 ];then
     echo; echo #clear #echo
     echo -e -n "\\033[1;31m" #red
     echo "WARNING, $FULLPKGNAME does not exist at $REMOTEURL"
     echo -e -n "\\033[0;39m" #back to black.
     echo "This may be ok if you have it locally and want to use it, but it may be"
     echo "an old discarded pkg and you might want to delete it from local db file."
     echo "Press ENTER only to delete it from local $ONEREPO"
     echo "or press 'x' then ENTER to abort this checking,"
     echo -n "or any other printable char to keep entry: "
     read keepit </dev/tty
     [ "$keepit" = "x" ] && break
     if [ "$keepit" = "" ];then
      echo "...deleting $FULLPKGNAME entry."
      fpnPATTERN='|'"$FULLPKGNAME"'|'
      grep -v "$fpnPATTERN" $ONEREPO > /tmp/${ONEREPO}-tmp
      mv -f /tmp/${ONEREPO}-tmp $ONEREPO
     fi
    fi
   done
  done
 fi
 
fi


#120804 the code below doesn't seem to be relevant anymore...
exit ###EXIT HERE###

#the files that have compatible-distro pkg docs (these were downloaded by 0setup)...
PKGLISTS_COMPAT="`echo "$PKG_DOCS_DISTRO_COMPAT" | tr ' ' '\n' | cut -f 3 -d '|' | tr '\n' ' ' | sed -e 's% $%%'`" #see file DISTRO_PKGS_SPECS-ubuntu
#...format Upup ex: 'Packages-ubuntu-intrepid-main Packages-ubuntu-intrepid-universe'
#...Ppup ex: 'Packages-puppy-4xx-official'

PUP_PKGLIST_COMPAT=""
if [ "$DISTRO_BINARY_COMPAT" = "puppy" ];then
 num_pkglists_compat=`echo -n "$PKGLISTS_COMPAT" | wc -w`
 if [ $num_pkglists_compat -eq 1 ];then
  PUP_PKGLIST_COMPAT="$PKGLISTS_COMPAT"
  #the compat-distro is a version of puppy, and we are building from file $PUP_PKGLIST_COMPAT
  #ex: Packages-puppy-4xx-official
  #make sure that entries tally with the other Packages-puppy-[2345]-official files...
  
  echo
  echo "Synchronising $PUP_PKGLIST_COMPAT with other Packages-puppy-[2345]-official files..."
  echo "Note, $PUP_PKGLIST_COMPAT is the package db file used to build Puppy."
  
  xPUP_PKGLIST_COMPAT="`cat $PUP_PKGLIST_COMPAT`"
  echo "$xPUP_PKGLIST_COMPAT" |
  while read ONEENTRY
  do
   [ "$ONEENTRY" = "" ] && continue #precaution.
   
   #decide which Packages-puppy-[2345]-official to check against...
   CHECKWITH=""
   ONEHOSTENV="`echo -n "$ONEENTRY" | cut -f 11 -d '|'`" #ex: 'puppy'
   ONEHOSTREL="`echo -n "$ONEENTRY" | cut -f 12 -d '|'`" #ex: '4'
   if [ "$ONEHOSTENV" != "" ];then
    if [ "$ONEHOSTREL" != "" ];then
     CHECKWITH="Packages-${ONEHOSTENV}-${ONEHOSTREL}-official"
    else
     CHECKWITH="Packages-${ONEHOSTENV}-${DISTRO_COMPAT_VERSION}-official"
    fi
    if [ ! -f $CHECKWITH ];then
     echo
     echo "ERROR in fields 11 and/or 12 of this db entry in ${PUP_PKGLIST_COMPAT}:"
     echo "$ONEENTRY"
     echo -n "You need to fix it, quiting."
     exit 1
    fi
   fi
   [ "$CHECKWITH" = "" ] && CHECKWITH="Packages-${DISTRO_BINARY_COMPAT}-${DISTRO_COMPAT_VERSION}-official" #ex: Packages-puppy-4-official

   #ex: entry in Packages-puppy-4xx-official not exist in Packages-puppy-4-official then copy it across...
   ONEPKGNAME="`echo -n "$ONEENTRY" | cut -f 1 -d '|'`"
   ONENAMEONLY="`echo -n "$ONEENTRY" | cut -f 2 -d '|'`"
   opnPATTERN='^'"$ONEPKGNAME"'|'
   onoPATTERN='|'"$ONENAMEONLY"'|'
   checkpkgnameexist="`grep "$opnPATTERN" $CHECKWITH`"
   checknameonlyexist="`grep "$onoPATTERN" $CHECKWITH`"
   if [ "$checkpkgnameexist" = "" ];then
    if [ "$checknameonlyexist" = "" ];then
     echo; echo #clear #echo
     echo "Note, $PUP_PKGLIST_COMPAT is the package db file used to build Puppy."
     echo
     echo "This entry in ${PUP_PKGLIST_COMPAT} does not exist in ${CHECKWITH}:"
     cutONEENTRY="`echo -n "$ONEENTRY" | cut -c 1-76`"'...'
     echo -e -n "\\033[1;31m" #red
     echo "$cutONEENTRY"
     echo -e -n "\\033[0;39m"
     echo
     echo "If this is a valid entry, just press ENTER to copy it across."
     echo "If this is now an invalid entry, pkg no longer available, press any other"
     echo -n "printable key then ENTER: "
     read WAITFORIT </dev/tty
     if [ "$WAITFORIT" = "" ];then
      echo "...appending to $CHECKWITH"
      echo "$ONEENTRY" >> $CHECKWITH
     else
      echo "...removing entry from $PUP_PKGLIST_COMPAT"
      grep -v "$opnPATTERN" $PUP_PKGLIST_COMPAT > /tmp/pup_pkglist_compat-tmp
      mv -f /tmp/pup_pkglist_compat-tmp $PUP_PKGLIST_COMPAT
     fi
    else
     echo; echo #clear #echo
     echo "Note, $PUP_PKGLIST_COMPAT is the package db file used to build Puppy."
     echo
     echo "This entry in ${PUP_PKGLIST_COMPAT} does not exist in ${CHECKWITH}:"
     cutONEENTRY="`echo -n "$ONEENTRY" | cut -c 1-76`"'...'
     echo -e -n "\\033[1;31m" #red
     echo -n "$cutONEENTRY"
     echo -e "\\033[0;39m"
     echo
     echo "If this is a valid entry, just press ENTER to copy it across."
     echo
     echo "However, ${CHECKWITH} already has these alternate versions: "
     CNT=1
     echo -en "\\033[1;31m" #red
     echo "$checknameonlyexist" |
     while read ONECHECK
     do
      cutONECHECK="`echo -n "$ONECHECK" | cut -c 1-73`"'...'
      echo "$CNT  $cutONECHECK"
      CNT=`expr $CNT + 1`
     done
     echo -n -e "\\033[0;39m" #back to black.
     echo
     echo "Or, if you want to update ${PUP_PKGLIST_COMPAT}"
     echo -n "with one of these alternatives, type a number: "
     read CHOOSEALTPKG </dev/tty
     if [ "$CHOOSEALTPKG" = "" ];then
      echo "...appending to $CHECKWITH"
      echo "$ONEENTRY" >> $CHECKWITH
     else
      if [ "`echo -n "$CHOOSEALTPKG" | grep '^[0-9]$'`" = "" ];then
       echo "ERROR, '${CHOOSEALTPKG}' is not a valid number, ignoring"
       continue
      fi
      ALTENTRY="`echo "$checknameonlyexist" | head -n $CHOOSEALTPKG | tail -n 1`"
      echo "...updating entry in $PUP_PKGLIST_COMPAT from $CHECKWITH"
      grep -v "$opnPATTERN" $PUP_PKGLIST_COMPAT > /tmp/pup_pkglist_compat-tmp
      mv -f /tmp/pup_pkglist_compat-tmp $PUP_PKGLIST_COMPAT
      echo "$ALTENTRY" >> $PUP_PKGLIST_COMPAT
      sort --key=1 --field-separator="|" $PUP_PKGLIST_COMPAT > /tmp/${PUP_PKGLIST_COMPAT}-tmp
      mv -f /tmp/${PUP_PKGLIST_COMPAT}-tmp $PUP_PKGLIST_COMPAT
     fi
    fi
    sort --key=1 --field-separator="|" $CHECKWITH > /tmp/${CHECKWITH}-tmp
    mv -f /tmp/${CHECKWITH}-tmp $CHECKWITH
    continue
   fi
   
  done

 fi
fi


###END###