woof-CE-libre/woof-code/support/findpkgs

#!/bin/bash
#Copyright Barry Kauler, aug 2011, bkhome.org
#110817 find all packages to be used in a Puppy build, pet and compat.
#110821 must also work with working-dir set to /root/.packages
#110908 fix o/p file for pkgs which genericname not found.
#111014 tweak support for 3.x kernel.
#111127 no longer check if host has 'printcols', 'vercmp', it is done in 'merge2out' script.
#111127 syntax checking of db files is happening too much.
#111128 checking fields 11,12,13 of db now much faster.
#111204 do not use /var/local/woof, instead 'status' directory (better for running from different save-files).
#111204 use md5sum for all checking whether to rerun this script.
#120515 build from "gentoo" binary tarballs (refer support/gentoo).
#120719 support raspbian.
#120812 db category now allows optional subcategory (for which an icons exists in /usr/local/lib/X11/mini-icons).
#121102 file DISTRO_SPECS has new variable DISTRO_DB_SUBNAME. ex: for 14.0-based slacko, DISTRO_DB_SUBNAME=slacko14
#121102 pet_packages-${DISTRO_COMPAT_VERSION} (or pet_packages-${DISTRO_FILE_PREFIX}) is now pet_packages-${DISTRO_DB_SUBNAME.
#130306 arch linux: gz now xz. maybe only 'DISTRO_COMPAT_REPOS-arch'.
#130312 add wary5 fallback.
#130313 having problem with repo-override. $PKGLOCFLD needs to be respected (ex: pet:wary5).
#130316 fixed bug, getting duplicate :$GENERICNAME: in file FINAL_PKGS.
#130320 fix disappearing pkg in FINAL_PKGS.
#130320 get rid of |puppy|wary5| |t2|8.0rc| in fallbacks searching for pet.
#130321 bring back |t2|8.0rc|

#variable PKGS_SPECS_TABLE in file DISTRO_PKGS_SPECS-* to have optional values in 5th field:
#yes|915resolution||exe,dev,doc,nls|[pet[:<repo>]] or yes|915resolution||exe,dev,doc,nls|[compat[:<repo>]]
#exs: yes|915resolution||exe,dev,doc,nls|compat yes|915resolution|915resolution|exe,dev,doc,nls|compat:core
#...'compat' means pkg from compatible-distro, 'core' is a specific repo, 3rd or 4rd field in 'Packages-mageia-1-core'
#...3rd field is now optional for both pet/compat if 5th field present.
#previously, a pet pkg is implied by empty 3rd field (and empty 5th), however now can have an optional name qualifier...
#ex: yes|915resolution|915resolution-2011*|exe,dev,doc,nls|pet
#...optional glob wildcard(s) in 3rd field, multiple parameters allowed (comma delimiters), '-' exclusion-prefix also.
#   PET pkgs: no need to specify _DEV, _DOC, _NLS, they are implied by 4th field.
#if above is unclear, please read: http://bkhome.org/blog/?viewDetailed=02414

export LANG=C #faster.

CURDIR="`pwd`"
if [ "$CURDIR" = "/root/.packages" ];then #110821
. /etc/DISTRO_SPECS #has DISTRO_BINARY_COMPAT, DISTRO_COMPAT_VERSION
 [ ! "$DISTRO_DB_SUBNAME" ] && DISTRO_DB_SUBNAME="$DISTRO_COMPAT_VERSION" #121102 fallback if DISTRO_DB_SUBNAME not defined in file DISTRO_SPECS.
 #DS_DATE=`stat --format=%Y /etc/DISTRO_SPECS`
 DS_DATE="`md5sum /etc/DISTRO_SPECS | cut -f 1 -d ' '`"
. ./DISTRO_PKGS_SPECS #has PKGS_SPECS_TABLE, FALLBACKS_COMPAT_VERSIONS
. ./DISTRO_COMPAT_REPOS #v431 has PKG_DOCS_DISTRO_COMPAT
 #DPS_MODIFY_DATE=`stat --format=%Y ./DISTRO_PKGS_SPECS` #modify date, seconds since last epoch.
 DPS_MODIFY_DATE="`md5sum ./DISTRO_PKGS_SPECS | cut -f 1 -d ' '`"
else
. ./DISTRO_SPECS #has DISTRO_BINARY_COMPAT, DISTRO_COMPAT_VERSION
 [ ! "$DISTRO_DB_SUBNAME" ] && DISTRO_DB_SUBNAME="$DISTRO_COMPAT_VERSION" #121102 fallback if DISTRO_DB_SUBNAME not defined in file DISTRO_SPECS.
 #DS_DATE=`stat --format=%Y ./DISTRO_SPECS`
 DS_DATE="`md5sum ./DISTRO_SPECS | cut -f 1 -d ' '`"
 if [ -f ./DISTRO_COMPAT_REPOS-${DISTRO_BINARY_COMPAT}-${DISTRO_COMPAT_VERSION} ];then
. ./DISTRO_COMPAT_REPOS-${DISTRO_BINARY_COMPAT}-${DISTRO_COMPAT_VERSION}
 else
. ./DISTRO_COMPAT_REPOS-${DISTRO_BINARY_COMPAT} #130306
 fi
 if [ -f ./DISTRO_PKGS_SPECS-${DISTRO_BINARY_COMPAT}-${DISTRO_COMPAT_VERSION} ];then
. ./DISTRO_PKGS_SPECS-${DISTRO_BINARY_COMPAT}-${DISTRO_COMPAT_VERSION} #has FALLBACKS_COMPAT_VERSIONS
  #DPS_MODIFY_DATE=`stat --format=%Y ./DISTRO_PKGS_SPECS-${DISTRO_BINARY_COMPAT}-${DISTRO_COMPAT_VERSION}` #modify date, seconds since last epoch.
  DPS_MODIFY_DATE="`md5sum ./DISTRO_PKGS_SPECS-${DISTRO_BINARY_COMPAT}-${DISTRO_COMPAT_VERSION} | cut -f 1 -d ' '`"
 else
. ./DISTRO_PKGS_SPECS-${DISTRO_BINARY_COMPAT} #has FALLBACKS_COMPAT_VERSIONS
  #DPS_MODIFY_DATE=`stat --format=%Y ./DISTRO_PKGS_SPECS-${DISTRO_BINARY_COMPAT}` #modify date, seconds since last epoch.
  DPS_MODIFY_DATE="`md5sum ./DISTRO_PKGS_SPECS-${DISTRO_BINARY_COMPAT} | cut -f 1 -d ' '`"
 fi
fi
. ./DISTRO_PET_REPOS #has PET_REPOS, PACKAGELISTS_PET_ORDER
. ./PKGS_MANAGEMENT #has PKG_NAME_ALIASES

. ./WOOFMERGEVARS #111127 need to know if a cross-build.

#variable DISTRO_KERNEL_PET is in file DISTRO_SPECS... ex: linux_kernel-2.6.32-44-i486-w5.pet 111014 remove if-then...
KERNMAJ="`echo -n "$DISTRO_KERNEL_PET" | cut -f 2 -d '-' | cut -f 1 -d '.'`" #2 or 3
#if [ "$KERNMAJ" = "3" ];then
# KERNELVER3="`echo -n "$DISTRO_KERNEL_PET" | cut -f 2 -d '-' | cut -f 1,2 -d '.' | cut -f 1 -d '_'`" #ex: 3.1
#else
 KERNELVER3="`echo -n "$DISTRO_KERNEL_PET" | cut -f 2 -d '-' | cut -f 1,2,3 -d '.' | cut -f 1 -d '_'`" #ex: 2.6.32
#fi

#modify compat-distro fallback list...
if [ "$FALLBACKS_COMPAT_VERSIONS" != "" ];then
 FALLBACKS_COMPAT_VERSIONS="`echo -n "$FALLBACKS_COMPAT_VERSIONS" | grep -o "${DISTRO_COMPAT_VERSION}.*"`"
 #ex: 'koala jaunty intrepid' gets reduced to 'jaunty intrepid' if DISTRO_COMPAT_VERSION=jaunty
fi

REPO_DEFAULT_SUBDIR="pet_packages-${DISTRO_DB_SUBNAME}" #121102

BINARIES='deb' #get them from packages-deb.
[ "$DISTRO_BINARY_COMPAT" = "t2" ] && BINARIES="bz2" #get them from packages-bz2-${DISTRO_COMPAT_VERSION}.
[ "$DISTRO_BINARY_COMPAT" = "slackware" ] && BINARIES="tgz_txz" #download to packages-tgz_txz-${DISTRO_COMPAT_VERSION}.
[ "$DISTRO_BINARY_COMPAT" = "debian" ] && BINARIES="deb" #download to packages-deb-${DISTRO_COMPAT_VERSION}.
[ "$DISTRO_BINARY_COMPAT" = "arch" ] && BINARIES="tar_xz" #download to packages-tar_xz-${DISTRO_COMPAT_VERSION}. 130306
[ "$DISTRO_BINARY_COMPAT" = "puppy" ] && BINARIES="pet" #built entirely from pet pkgs.
[ "$DISTRO_BINARY_COMPAT" = "scientific" ] && BINARIES="rpm" #Iguleder: download to packages-rpm-${DISTRO_COMPAT_VERSION}.
[ "$DISTRO_BINARY_COMPAT" = "mageia" ] && BINARIES="rpm"
[ "$DISTRO_BINARY_COMPAT" = "gentoo" ] && BINARIES="gentoo" #120515 download to packages-gentoo-gap6
[ "$DISTRO_BINARY_COMPAT" = "raspbian" ] && BINARIES="deb_raspbian" #download to packages-deb_raspbian-${DISTRO_COMPAT_VERSION}.
BINARIES="${BINARIES}-${DISTRO_COMPAT_VERSION}"

#the files that have compatible-distro pkg docs (these were downloaded by 0setup)...
PKGLISTS_COMPAT="`echo "$PKG_DOCS_DISTRO_COMPAT" | tr ' ' '\n' | cut -f 3 -d '|' | tr '\n' ' '`" #see file DISTRO_PKGS_SPECS-ubuntu
#...format ex: 'Packages-ubuntu-intrepid-main Packages-ubuntu-intrepid-universe'

CURRDIR="`pwd`"
#[ ! -d packages-templates ] && exit 1 #precaution.
mkdir -p status
STATUSDIR="${CURRDIR}/status"

#for efficiency, only run this script if have too... 111204 use md5sum instead of date...
KEEPGOING='no'
for ADBFILE in $PACKAGELISTS_PET_ORDER $PKGLISTS_COMPAT
do
 #NEWDATE=`stat --format=%Y $ADBFILE`
 NEWDATE="`md5sum $ADBFILE | cut -f 1 -d ' '`"
 OLDDATE="0"
 [ -f ${STATUSDIR}/MODIFY_DATE_${ADBFILE} ] && OLDDATE="`cat ${STATUSDIR}/MODIFY_DATE_${ADBFILE}`"
 if [ "$NEWDATE" != "$OLDDATE" ];then
  KEEPGOING='yes'
  #echo -n "$NEWDATE" > ${STATUSDIR}/MODIFY_DATE_${ADBFILE} #no, this is done below.
 fi

 # cache the columns used during package search
 if [ "$KEEPGOING" = "yes" ] || [ ! -f $STATUSDIR/$ADBFILE-cols ];then
  printcols ./${ADBFILE} 2 3 8 9 11 12 > $STATUSDIR/$ADBFILE-cols
 fi
done
OLD_DPS_DATE="0"
[ -f ${STATUSDIR}/findpkgs_DPS_MODIFY_DATE-${DISTRO_BINARY_COMPAT}-${DISTRO_COMPAT_VERSION} ] && OLD_DPS_DATE="`cat ${STATUSDIR}/findpkgs_DPS_MODIFY_DATE-${DISTRO_BINARY_COMPAT}-${DISTRO_COMPAT_VERSION}`"
if [ "$DPS_MODIFY_DATE" != "$OLD_DPS_DATE" ];then
 KEEPGOING='yes'
 echo -n "${DPS_MODIFY_DATE}" > ${STATUSDIR}/findpkgs_DPS_MODIFY_DATE-${DISTRO_BINARY_COMPAT}-${DISTRO_COMPAT_VERSION}
fi
OLD_DS_DATE="0"
[ -f ${STATUSDIR}/findpkgs_MODIFY_DATE_DISTRO_SPECS ] && OLD_DS_DATE="`cat ${STATUSDIR}/findpkgs_MODIFY_DATE_DISTRO_SPECS`"
if [ "$DS_DATE" != "$OLD_DS_DATE" ];then
 KEEPGOING='yes'
 echo -n "$DS_DATE" > ${STATUSDIR}/findpkgs_MODIFY_DATE_DISTRO_SPECS
fi
[ ! -f ${STATUSDIR}/findpkgs_FINAL_PKGS-${DISTRO_BINARY_COMPAT}-${DISTRO_COMPAT_VERSION} ] && KEEPGOING='yes'
if [ "$KEEPGOING" != "yes" ];then
 echo
 echo "Exiting from support/findpkgs, already created up-to-date pkg list:"
 echo " ${STATUSDIR}/findpkgs_FINAL_PKGS-${DISTRO_BINARY_COMPAT}-${DISTRO_COMPAT_VERSION}"
 echo
 exit 0
fi

rm -rf /tmp/findpkgs_tmp 2>/dev/null
mkdir /tmp/findpkgs_tmp

#remove comments from PKGS_SPECS_TABLE... make sure '|' on end... get rid of old |+udev,+whatever on end...
PKGS_SPECS_TABLE="`echo "$PKGS_SPECS_TABLE" | grep -v '^#' | grep -v '^$' | tr '\t' ' ' | tr -s ' ' | tr '+' '&' | sed -e 's%|&.*%%' | tr '&' '+' | sed -e 's% #.*%%' -e 's% $%%' -e 's%$%|%' -e 's%||$%|%'`"

echo
echo "Checking syntax of PKGS_SPECS_TABLE..."
CNT=1
for ONECHK in $PKGS_SPECS_TABLE
do
 BADFLAG='no'
 NUMDELS=`echo "$ONECHK"  | sed -e 's/[^|]//g' | wc -c`
 [  $NUMDELS -lt 4 -o $NUMDELS -gt 6 ] && BADFLAG='yes'
 YESNO="`echo "$ONECHK" | cut -f 1 -d '|' | grep -E 'yes|no'`"
 [ "YESNO" = "" ] && BADFLAG='yes'
 GENPKG="`echo "$ONECHK" | cut -f 2 -d '|'`"
 [ "$GENPKG" = "" ]  && BADFLAG='yes'
 if [ "$BADFLAG" = "yes"  ];then
  echo "SYNTAX ERROR LINE ${CNT}: '$ONECHK'"
  echo "$PKGS_SPECS_TABLE" > /tmp/pst-dump
  exit 1
 fi
 CNT=$(($CNT + 1))
done

echo -n "" > /tmp/findpkgs_tmp/pet_log1
echo -n "" > /tmp/findpkgs_tmp/compat_log1
echo -n "" > /tmp/findpkgs_tmp/PREFERRED_PKGS
echo -n "" > /tmp/findpkgs_tmp/FINAL_PKGS

#compiledPTNS is to select a pkg that most closely matches the compile-environment of the compat-distro...
#||| means architecture-independent pkg, needs high preference, immediately after exact-compiled match...
petcompiledPTNS="|${DISTRO_BINARY_COMPAT}|${DISTRO_COMPAT_VERSION}| |||" #ex: '|ubuntu|intrepid|' '|||'
for ONECOMPATVERSION in $FALLBACKS_COMPAT_VERSIONS #sliding scale, ex: koala jaunty intrepid
do
 [ "$ONECOMPATVERSION" = "$DISTRO_COMPAT_VERSION" ] && continue
 [ "$ONECOMPATVERSION" = "4" ] && break #bottom of ladder, drop down to |puppy| test only. (i have some |puppy| in common repo, want to find them before |puppy|4|)
 petcompiledPTNS="${petcompiledPTNS} |${DISTRO_BINARY_COMPAT}|${ONECOMPATVERSION}|"
done
#need hack, older quirky pkgs compiled with |t2|8.0rc| and have some of these in common repo...
#130320 get rid of |puppy|wary5| |t2|8.0rc| ... 130321 bring back |t2|8.0rc| ...
if [ "$DISTRO_BINARY_COMPAT" = "puppy" ];then
 petcompiledPTNS="${petcompiledPTNS} |t2|8.0rc| |puppy| |" #increasingly desperate. last one will find any pkg with matching name. 130312 add wary5.
else
 petcompiledPTNS="${petcompiledPTNS} |${DISTRO_BINARY_COMPAT}| |t2|8.0rc| |puppy| |" #increasingly desperate. 130312 add wary5.
fi

##111127 this code block is running too many times. the most likely situation is when db files for a compat-distro
##are created by 0setup/1download and the last two/three fields are left out. just check first line of db...
##problem with ||| above high priority, compileddistro|compiledrelease| are optional fields and may be missing.
##printcols <file> 2 3 8 9 11 12 (see below) does insert 11 & 12 delimiters if missing, but that would bump the
##priority of that pkg -- results in pkgs in Packages-puppy-4-official getting chosen.
#for ADBFILE in $PACKAGELISTS_PET_ORDER $PKGLISTS_COMPAT
#do
# NEWDATE=`stat --format=%Y $ADBFILE` #seconds since last epoch.
# OLDDATE=0
# [ -f ${STATUSDIR}/MODIFY_DATE_${ADBFILE} ] && OLDDATE=`cat ${STATUSDIR}/MODIFY_DATE_${ADBFILE}`
# if [ $NEWDATE -ne $OLDDATE ];then
# #if ! vercmp "$NEWDATE" eq "$OLDDATE";then
#  echo "Fixing compileddistro|compiledrelease| fields in $ADBFILE"
#  echo -n "" > /tmp/findpkgs_tmp/dbfilefixed
#  DB_compileddistro="`echo -n "$ADBFILE" | cut -f 2 -d '-'`" #ex: puppy
#  DB_compiledrelease="`echo -n "$ADBFILE" | cut -f 3 -d '-'`" #ex: 4
#  DB_repo="`echo -n "$ADBFILE" | cut -f 4 -d '-'`" #ex: official
#  #db fields: pkgname|nameonly|version|pkgrelease|category|size|path|fullfilename|dependencies|description|compileddistro|compiledrelease|repo|
#  cat $ADBFILE |
#  while read ONELINE
#  do
#   echo -n "."
#   DELNUM=`echo -n "$ONELINE" | sed -e 's%[^|]%%g' | wc -c` #number of | delimiters.
#   case $DELNUM in
#    10) echo "${ONELINE}${DB_compileddistro}|${DB_compiledrelease}||" >> /tmp/findpkgs_tmp/dbfilefixed ;;
#    9)  echo "|${ONELINE}${DB_compileddistro}|${DB_compiledrelease}||" >> /tmp/findpkgs_tmp/dbfilefixed ;; #broken, missing | on end of description.
#    *)
#     #echo "$ONELINE" >> /tmp/findpkgs_tmp/dbfilefixed 
#     #111127 mark entire file as ok, get out (see note above)...
#     NEWDATE=`stat --format=%Y $ADBFILE` #seconds since last epoch.
#     echo -n "$NEWDATE" > ${STATUSDIR}/MODIFY_DATE_${ADBFILE}
#     continue 2
#    ;;
#   esac
#  done
#  sync
#  echo
#  mv -f $ADBFILE /tmp/findpkgs_tmp/${ADBFILE}-PREVIOUS
#  #mv -f /tmp/findpkgs_tmp/dbfilefixed $ADBFILE
#  #Packages-puppy-slacko-official has some duplicate entries, cleanup...
#  sort -u --key=1,1 --field-separator="|" /tmp/findpkgs_tmp/dbfilefixed > ${ADBFILE}
#  NEWDATE=`stat --format=%Y $ADBFILE` #seconds since last epoch.
#  echo -n "$NEWDATE" > ${STATUSDIR}/MODIFY_DATE_${ADBFILE}
# fi
#done

#111128 redo the above, much faster...
#problem with ||| above high priority, compileddistro|compiledrelease| are optional fields and may be missing.
#printcols <file> 2 3 8 9 11 12 (see below) does insert 11 & 12 delimiters if missing, but that would bump the
#priority of that pkg -- results in pkgs in Packages-puppy-4-official getting chosen.
for ADBFILE in $PACKAGELISTS_PET_ORDER $PKGLISTS_COMPAT
do
 NEWSUM="`md5sum $ADBFILE | cut -f 1 -d ' '`"
 OLDSUM="0"
 [ -f ${STATUSDIR}/MODIFY_DATE_${ADBFILE} ] && OLDSUM="`cat ${STATUSDIR}/MODIFY_DATE_${ADBFILE}`"
 if [ "$NEWSUM" != "$OLDSUM" ];then
  DB_compileddistro="`echo -n "$ADBFILE" | cut -f 2 -d '-'`" #ex: puppy
  DB_compiledrelease="`echo -n "$ADBFILE" | cut -f 3 -d '-'`" #ex: 4
  DB_repo="`echo -n "$ADBFILE" | cut -f 4 -d '-'`" #ex: official
  #db fields: pkgname|nameonly|version|pkgrelease|category[;subcategory]|size|path|fullfilename|dependencies|description|compileddistro|compiledrelease|repo|
  echo "Fixing compileddistro|compiledrelease| fields in $ADBFILE"
  FIELD11PTN="s%VERTBARCHAR$%VERTBARCHAR${DB_compileddistro}VERTBARCHAR%"
  FIELD12PTN="s%VERTBARCHAR$%VERTBARCHAR${DB_compiledrelease}VERTBARCHAR%"
  sed -e 's%|%VERTBARCHAR%' -e 's%|%VERTBARCHAR%' -e 's%|%VERTBARCHAR%' -e 's%|%VERTBARCHAR%' -e 's%|%VERTBARCHAR%' -e 's%|%VERTBARCHAR%' -e 's%|%VERTBARCHAR%' -e 's%|%VERTBARCHAR%' -e 's%|%VERTBARCHAR%' -e 's%|%VERTBARCHAR%' -e "${FIELD11PTN}" -e "${FIELD12PTN}" -e 's%VERTBARCHAR$%VERTBARCHARVERTBARCHAR%' -e 's%\(VERTBARCHAR[^|]*|[^|]*|$\)%\1|%'  -e 's%VERTBARCHAR%|%g' ${ADBFILE} > /tmp/findpkgs_tmp/dbfilefixed
  REALNEWSUM="`md5sum /tmp/findpkgs_tmp/dbfilefixed | cut -f 1 -d ' '`"
  if [ "$REALNEWSUM" != "$NEWSUM" ];then
   cp -f /tmp/findpkgs_tmp/dbfilefixed $ADBFILE
   NEWSUM="$REALNEWSUM"
  fi
  echo -n "$NEWSUM" > ${STATUSDIR}/MODIFY_DATE_${ADBFILE}
 fi
done

##################
###FIRST SEARCH###
##################
echo
echo "First pass at finding packages to be used in the Puppy build..."

search_func() {
 SEARCHNUM="$1"
for ONEPKGSPEC in $PKGS_SPECS_TABLE
do
 [ "$ONEPKGSPEC" = "" ] && continue
 YESNO="`echo "$ONEPKGSPEC" | cut -f 1 -d '|'`"
 [ "$YESNO" = "no" ] && continue
 GENERICNAME="`echo "$ONEPKGSPEC" | cut -f 2 -d '|'`"
 BINARYPARTNAMES="`echo "$ONEPKGSPEC" | cut -f 3 -d '|' | tr ',' ' '`"
 if [ "$BINARYPARTNAMES" = "" ];then
  xBINARYPARTNAMES="$GENERICNAME"
 else
  #cut out any to be excluded ( '-' prefix)...
  BINARYEXCLUDED0="`echo "$BINARYPARTNAMES" | tr ' ' '\n' | grep '^-' | sed -e 's/^-//'`"
  xBINARYPARTNAMES="`echo "$BINARYPARTNAMES" | tr ' ' '\n' | grep -v '^-' | tr '\n' ' '`"
 fi
 echo -n "" > /tmp/findpkgs_tmp/excludePTNS
 if [ "$BINARYEXCLUDED0" ];then
  excludePTNS="`echo -n "$BINARYEXCLUDED0" | sed -e 's%\\-%\\\\-%g' -e 's%\\*%.*%g' -e 's%^%|%'`"   #i think that dashes need to be backslashed, change blob * wildcard to reg.expr.
  echo "$excludePTNS" > /tmp/findpkgs_tmp/excludePTNS
 fi

 PKGLOC2=''; PKGLOC1='pet'                        #original behaviour
 [ "$BINARYPARTNAMES" != "" ] && PKGLOC1='compat' #original behaviour
 PKGLOCFLD="`echo "$ONEPKGSPEC" | cut -f 5 -d '|'`"
 if [ "$PKGLOCFLD" ];then
  PKGLOC1="`echo -n "$PKGLOCFLD" | cut -f 1 -d ':'`"
  [ "`echo -n "$PKGLOCFLD" | grep ':'`" != "" ] && PKGLOC2="`echo -n "$PKGLOCFLD" | cut -f 2 -d ':'`"
 fi

 if [ "$PKGLOC1" = "pet" ];then
  PACKAGELISTS_ORDER="$PACKAGELISTS_PET_ORDER"
  xPACKAGELISTS_ORDER="$PACKAGELISTS_PET_ORDER"
  compiledPTNS="$petcompiledPTNS"
 else
  PACKAGELISTS_ORDER="$PKGLISTS_COMPAT"
  xPACKAGELISTS_ORDER="$PKGLISTS_COMPAT"
  compiledPTNS='.' #allow everything through.
 fi
 if [ "$PKGLOC2" ];then #exs: core, nonfree, contrib, official (4th field in Packages-puppy-wary5-official)
  locPTN="\-${PKGLOC2}$"
  xPACKAGELISTS_ORDER="`echo -n "$PACKAGELISTS_ORDER" | tr -s ' ' | tr ' ' '\n' | grep "$locPTN" | tr '\n' ' '`"
  if [ ! "$xPACKAGELISTS_ORDER" ];then
   locPTN="\-${PKGLOC2}\-" #allow choose 3rd field also -wary5-
   xPACKAGELISTS_ORDER="`echo -n "$PACKAGELISTS_ORDER" | tr -s ' ' | tr ' ' '\n' | grep "$locPTN" | tr '\n' ' '`"
  fi
 fi

 echo -n "$GENERICNAME "
  
 for APARTNAME in $xBINARYPARTNAMES
 do
  
  #APARTNAME could have glob wildcard(s)...
  if [ "`echo -n "$APARTNAME" | grep '\\*$'`" != "" ];then   #if has '*' on end, then look in full pkg name.
   namePTN="|`echo -n "$APARTNAME" | sed -e 's%\\-%\\\\-%g' -e 's%\\*%.*%g'`|"   #i think that dashes need to be backslashed, change blob * wildcard to reg.expr.
  else
   namePTN="^`echo -n "$APARTNAME" | sed -e 's%\\-%\\\\-%g' -e 's%\\*%.*%g'`|"  #i think that dashes need to be backslashed, change blob * wildcard to reg.expr.
  fi
  #in wary, i have many pkgs that are _DEV only, need to find these, a hack fix to find most...
  devnamePTN="`echo -n "$namePTN" | sed -e 's%|$%_DEV|%'`"
   
  echo -n "" > /tmp/findpkgs_tmp/FNDSPECS
  for acompiledPTN in $compiledPTNS
  do
   for PKGLIST in $xPACKAGELISTS_ORDER #ex: Packages-puppy-drake-official Packages-puppy-common-official Packages-puppy-4-official
   do
    #2 3 8 9 11 12 is: nameonly version fullpkgname dependencies compileddistro compiledrelease
    FNDSPECS="`grep "$namePTN" $STATUSDIR/$PKGLIST-cols | grep "$acompiledPTN" | grep -v -f /tmp/findpkgs_tmp/excludePTNS`" #note, no 'head -n 1'
    #in wary, i have many pkgs that are _DEV only, need to find these, a hack fix to find most...
    [ ! "$FNDSPECS" ] && FNDSPECS="`grep "$devnamePTN" $STATUSDIR/$PKGLIST-cols | grep "$acompiledPTN" | grep -v -f /tmp/findpkgs_tmp/excludePTNS`" #note, no 'head -n 1'
    if [ "$FNDSPECS" ];then
     for ONESPEC in $FNDSPECS #maybe more than one find.
     do
      echo "${ONESPEC}${PKGLIST}" >> /tmp/findpkgs_tmp/FNDSPECS #ex: abiword|2.3.4|abiword-2.3.4.pet|+gtk+|puppy|wary5|Packages-puppy-wary5-official
     done
    fi
    [ -s /tmp/findpkgs_tmp/FNDSPECS ] && break 2 #pkg(s) found.
   done
  done

  if [ ! -s /tmp/findpkgs_tmp/FNDSPECS ];then
   echo "WARNING: $APARTNAME pkg was not found!"
   continue
  fi
   
  for ONESPEC in `cat /tmp/findpkgs_tmp/FNDSPECS | tr '\n' ' '` #maybe more than one pkg found.
  do
   if [ "$SEARCHNUM" = "2" ];then #second search.
   
    #if there is a preferred pkg (after versioned-dep checking in this script) use that instead...
    if [ -s /tmp/findpkgs_tmp/PREFERRED_PKGS ];then
     if [ "$PKGLOCFLD" = "" ];then #130313 see also below. need to respect repo-override, even if a preferred version logged.
      fndnamePTN="^`echo -n "$ONESPEC" | cut -f 1 -d '|'`|" #ex: ^abiword|
      PREFSPEC="`grep "$fndnamePTN" /tmp/findpkgs_tmp/PREFERRED_PKGS`"
      [ "$PREFSPEC" ] && ONESPEC="$PREFSPEC"
     fi
    fi
    fndnamePTN="|`echo -n "$ONESPEC" | cut -f 1 -d '|'`|" #ex: |abiword|
    versionPTN="|`echo -n "$ONESPEC" | cut -f 2 -d '|'`|" #ex: |2.3.4|
    fullnamePTN="|`echo -n "$ONESPEC" | cut -f 3 -d '|'`|" #ex: |abiword-2.3.4.pet|
    REPODBFILE="`echo -n "$ONESPEC" | grep -o '|Packages-[^|]*' | cut -f 2 -d '|'`"
    #FULLDBENTRY="`grep "$fullnamePTN" $REPODBFILE`"
    
    #log the complete db entry, prefixed with generic-name, pet/compat and repo db filename...
    devnamePTN="|`echo -n "$ONESPEC" | cut -f 1 -d '|'`_DEV|" #ex: |abiword_DEV|
    docnamePTN="|`echo -n "$ONESPEC" | cut -f 1 -d '|'`_DOC|" #ex: |abiword_DOC|
    nlsnamePTN="|`echo -n "$ONESPEC" | cut -f 1 -d '|'`_NLS|" #ex: |abiword_NLS|
    for onePTN in $fndnamePTN $devnamePTN $docnamePTN $nlsnamePTN
    do
     FULLDBENTRY="`grep "$onePTN" $REPODBFILE | grep "$versionPTN"`"
     if [ "$FULLDBENTRY" ];then
      #130313 i think this is where having problem. a repo-override $PKGLOCFLD needs to be respected (ex: pet:wary5)...
      if [ "$PKGLOCFLD" ];then
       EXISTENTRY="`grep "$onePTN" /tmp/findpkgs_tmp/FINAL_PKGS | grep "$REPODBFILE"`"
      else
       EXISTENTRY="`grep "$onePTN" /tmp/findpkgs_tmp/FINAL_PKGS`" #previous code.
      fi
      if [ "$EXISTENTRY" = "" ];then
       echo ":${GENERICNAME}:|${PKGLOC1}|${REPODBFILE}|${FULLDBENTRY}" >> /tmp/findpkgs_tmp/FINAL_PKGS #ex: :a52dec:|compat|Packages-puppy-wary5-official|a52dec-0.7.4-w5|a52dec|0.7.4-w5||BuildingBlock|68K||a52dec-0.7.4-w5.pet||A free ATSC A52 stream decoder|puppy|wary5||
      else
       grep -v "$onePTN" /tmp/findpkgs_tmp/FINAL_PKGS > /tmp/findpkgs_tmp/FINAL_PKGS2
       mv -f /tmp/findpkgs_tmp/FINAL_PKGS2 /tmp/findpkgs_tmp/FINAL_PKGS
       #130316 in arch build getting duplicate generic names, ex:
       # :cups::cups:|pet|Packages-puppy-wary5-official|cups_DEV-1.4.8-w5c|cups_DEV|1.4.8-w5c||BuildingBlock|216K||cups_DEV-1.4.8-w5c.pet|+cups|common unix printing system|puppy|wary5||
       gnPTN1=':'"${GENERICNAME}"'[:|]'
       if [ "$(echo -n "${EXISTENTRY}" | grep "${gnPTN1}")" = "" ];then
        echo ":${GENERICNAME}:${EXISTENTRY}" >> /tmp/findpkgs_tmp/FINAL_PKGS #ex: :audio::a52dec:|compat|Packages-puppy-wary5-official|a52dec-0.7.4-w5|a52dec|0.7.4-w5||BuildingBlock|68K||a52dec-0.7.4-w5.pet||A free ATSC A52 stream decoder|puppy|wary5||
       else #130320 fix disappearing pkg.
        echo "${EXISTENTRY}" >> /tmp/findpkgs_tmp/FINAL_PKGS #ex: :audio::a52dec:|compat|Packages-puppy-wary5-official|a52dec-0.7.4-w5|a52dec|0.7.4-w5||BuildingBlock|68K||a52dec-0.7.4-w5.pet||A free ATSC A52 stream decoder|puppy|wary5||
       fi
      fi
     fi
    done
    
   else #1st search
    VERDEPS="`echo -n "$ONESPEC" | cut -f 4 -d '|' | tr ',' '\n' | sed -e 's%^+%%'`" #field-4 is dependencies.
    if [ "$VERDEPS" = "" ];then
     echo "`echo -n "${ONESPEC}" | cut -f 1,2,3,5,6,7 -d '|'`|" >> /tmp/findpkgs_tmp/${PKGLOC1}_log1 #ex: abiword|2.3.4|abiword-2.3.4-w5.pet|puppy|wary5|Packages-puppy-wary5-official|
    else
     for ONEVERDEP in `echo -n "${VERDEPS}" | tr '\n' ' '` #create one line for each dep.
     do
      echo "`echo -n "${ONESPEC}" | cut -f 1,2,3,5,6,7 -d '|'`|${ONEVERDEP}" >> /tmp/findpkgs_tmp/${PKGLOC1}_log1 #ex: abiword|2.3.4|abiword-2.3.4-w5.pet|puppy|wary5|Packages-puppy-wary5-official|aiksaurus&gt5.6.7
     done
    fi
   fi
  done
 done
done
} #end search_func

search_func 1 #first search.
sync

#####################
###POST PROCESSING###
#####################
#append dummy linux_kernel line...
echo "linux_kernel|${KERNELVER3}|linux_kernel-${KERNELVER3}.pet|||" >> /tmp/findpkgs_tmp/pet_log1

#don't forget, need read aliases for deps, variable PKG_NAME_ALIASES in file PKGS_MANAGEMENT...
#want to handle multiple conditions ex: aiksaurus&gt5.6.7&lt5.7.8
echo
echo
echo "Processing dependencies..."

echo -n "" > /tmp/findpkgs_tmp/fnd_ok_deps
echo -n "" > /tmp/findpkgs_tmp/fnd_ok_apps

for ONESPEC in `cat /tmp/findpkgs_tmp/compat_log1 /tmp/findpkgs_tmp/pet_log1 | grep '&' | tr '\n' ' '`
do
 #ex: ONESPEC=abiword|2.3.4|abiword-2.3.4-w5.pet|puppy|wary5|Packages-puppy-wary5-official|aiksaurus&gt5.6.7
 appPTN="^`echo -n "$ONESPEC" | cut -f 1 -d '|'`|" #pattern of the application name ex: ^abiword|
 appVER="`echo -n "$ONESPEC" | cut -f 2 -d '|'`" #ex: 2.3.4
 DEPFLD="`echo -n "$ONESPEC" | cut -f 7 -d '|'`"
 DEPNAME="`echo -n "$DEPFLD" | cut -f 1 -d '&'`"
 DEPCONDS="`echo -n "$DEPFLD" | cut -f 2-9 -d '&' | tr '&' ' '`"
 depnamePTN="|${DEPNAME}|"
 echo -n "$DEPNAME "
  
 #handle aliases...
 aliasesPTNS="`echo "$PKG_NAME_ALIASES" | tr ' ' '\n' | grep -v '^$' | sed -e 's%^%|%' -e 's%$%|%' -e 's%,%|,|%g' -e 's%\\*%.*%g'`"
 #ex: PKG_NAME_ALIASES has an entry cxxlibs,glibc*,libc-* the above creates |cxxlibs|,|glibc.*|,|libc\-.*|
 echo "$aliasesPTNS" > /tmp/findpkgs_tmp/aliasesPTNS
 singlePTNS="`echo "$aliasesPTNS" | tr ',' '\n'`"
 echo "$singlePTNS" > /tmp/findpkgs_tmp/singlePTNS
 FNDALIAS="`echo -n "$depnamePTN" | grep -f /tmp/findpkgs_tmp/singlePTNS | sed -e 's%\\.\\*%*%g' -e 's%\\\\-%-%g'`"  #translate .* \- back to * -
 if [ "$FNDALIAS" ];then
  aliasPTNS="`echo -n "$FNDALIAS" | grep -f /tmp/findpkgs_tmp/aliasesPTNS | tr ',' ' '`" #ex: |cxxlibs| |glibc.*| |libc\-.*|
 else
  aliasPTNS="^${DEPNAME}|"
 fi
 for namePTN in $aliasPTNS
 do

  for FNDPKG in `cat /tmp/findpkgs_tmp/compat_log1 /tmp/findpkgs_tmp/pet_log1 | grep "$namePTN" | tr '\n' ' '`
  do
   #found a dependency, now check its version is ok...
   FNDVER="`echo -n "$FNDPKG" | cut -f 2 -d '|'`"

   condFLG='good'
   for ACOND in $DEPCONDS #ex: gt5.6.7 lt6.7.8
   do
    DEPOP="`echo -n "$ACOND" | cut -c 1,2`"
    DEPVER="`echo -n "$ACOND" | cut -c 3-99`"
    if ! vercmp ${FNDVER} ${DEPOP} ${DEPVER};then
     condFLG='bad'
    fi
   done
   [ "$condFLG" = "bad" ] && continue
   
   #version is ok.
   #log dep info, make sure only latest version is logged...
   depVERprev="`grep "$namePTN" /tmp/findpkgs_tmp/fnd_ok_deps | cut -f 2 -d '|'`"
   if [ "$depVERprev" = "" ];then
    echo "$FNDPKG" >> /tmp/findpkgs_tmp/fnd_ok_deps
   else
    if vercmp ${FNDVER} gt ${depVERprev};then
     grep -v "$namePTN" /tmp/findpkgs_tmp/fnd_ok_deps > /tmp/findpkgs_tmp/fnd_ok_deps2
     mv -f /tmp/findpkgs_tmp/fnd_ok_deps2 /tmp/findpkgs_tmp/fnd_ok_deps
     echo "$FNDPKG" >> /tmp/findpkgs_tmp/fnd_ok_deps
    fi
   fi

   #log actual pkg info as well as deps, make sure latest version logged...
   appVERprev="`grep "$appPTN" /tmp/findpkgs_tmp/fnd_ok_apps | cut -f 2 -d '|'`"
   if [ "$appVERprev" = "" ];then
    echo "$ONESPEC" >> /tmp/findpkgs_tmp/fnd_ok_apps
   else
    if vercmp ${appVER} gt ${appVERprev};then
     grep -v "$appPTN" /tmp/findpkgs_tmp/fnd_ok_apps > /tmp/findpkgs_tmp/fnd_ok_apps2
     mv -f /tmp/findpkgs_tmp/fnd_ok_apps2 /tmp/findpkgs_tmp/fnd_ok_apps
     echo "$ONESPEC" >> /tmp/findpkgs_tmp/fnd_ok_apps
    fi
   fi

  done
 done
done

echo -n "" > /tmp/findpkgs_tmp/PREFERRED_PKGS
cat /tmp/findpkgs_tmp/fnd_ok_deps >> /tmp/findpkgs_tmp/PREFERRED_PKGS
cat /tmp/findpkgs_tmp/fnd_ok_apps >> /tmp/findpkgs_tmp/PREFERRED_PKGS

sort --key=3,3 --field-separator="|" -u /tmp/findpkgs_tmp/PREFERRED_PKGS > /tmp/findpkgs_tmp/PREFERRED_PKGS2
grep -v '^linux_kernel|' /tmp/findpkgs_tmp/PREFERRED_PKGS2 > /tmp/findpkgs_tmp/PREFERRED_PKGS #remove dummy entry.
cp -f /tmp/findpkgs_tmp/PREFERRED_PKGS ${STATUSDIR}/findpkgs_PREFERRED_PKGS-${DISTRO_BINARY_COMPAT}-${DISTRO_COMPAT_VERSION} #so retained between boots.
sync

###################
###SECOND SEARCH###
###################
#now find all pkgs again, this time use prefs recorded in /tmp/findpkgs_tmp/PREFERRED_PKGS
echo
echo
echo "Second pass at finding packages to be used in the Puppy build..."

search_func 2 #result will be in /tmp/findpkgs_tmp/FINAL_PKGS
#...each line is full db entry for package, prefixed with :generic-name:, pet/compat and repo-db-file
# ex: :a52dec:|compat|Packages-puppy-wary5-official|a52dec-0.7.4-w5|a52dec|0.7.4-w5||BuildingBlock|68K||a52dec-0.7.4-w5.pet||A free ATSC A52 stream decoder|puppy|wary5||
# 'compat' means pkg obtained from compatible-distro, in case of Wary they are also PETs.
# :generic-name: usually coincides with actual name in package. multiple pkgs may be combined into one generic-name.
# if the same pkg is combined into two different generic-names, then the line will look like this: :gcc_lib::gcc_dev:| (ex from Slacko).

#110908 not-found pkgs are left in here without the genericname etc fields preprended...
grep '^:' /tmp/findpkgs_tmp/FINAL_PKGS > /tmp/findpkgs_tmp/FINAL_PKGSx
mv -f /tmp/findpkgs_tmp/FINAL_PKGSx /tmp/findpkgs_tmp/FINAL_PKGS

sync
sort --key=4,4 --field-separator="|" /tmp/findpkgs_tmp/FINAL_PKGS > ${STATUSDIR}/findpkgs_FINAL_PKGS-${DISTRO_BINARY_COMPAT}-${DISTRO_COMPAT_VERSION} #so retained between boots.
sync
echo
echo
echo "Script finished. Result is in file"
echo " status/findpkgs_FINAL_PKGS-${DISTRO_BINARY_COMPAT}-${DISTRO_COMPAT_VERSION}
Each line is full db entry for pkg, prefixed with :generic-name:, pet/compat
 and db-file. ex:
 :a52dec:|compat|Packages-puppy-wary5-official|a52dec-0.7.4-w5|a52dec|0.7.4-w5||BuildingBlock|68K||a52dec-0.7.4-w5.pet||A free ATSC A52 stream decoder|puppy|wary5||
 'compat' means pkg from compat-distro, in case of Wary they are also PETs
 :generic-name: is 2nd field in PKGS_SPECS_TABLE, note in some cases multiple
 packages are combined into same genericname, or same pkg can be shared by
 multiple genericnames -- latter case, line will look like: :gcc_lib::gcc_dev:|"
###END###