woof-CE-libre/merge2out

#!/bin/sh
#BK nov. 2011
#111126 do not copy files from woof-distro to woof-out, symlink only.
#111126 fix symlink to compat-distro pkg download directory.
#111127 make sure host linux system has 'printcols' and 'vercmp' utilities.
#111127 also make sure target build will have 'printcols' and 'vercmp'.
#111203 fixes for rerun of this script.
#120503 i left some EMPTYDIRMARKER files inside /dev.
#120512 remove option to create symlinks in working directory.
#120515 build from "gentoo" binary tarballs (refer support/gentoo).
#130306 arch linux: gz now xz.
#130528 change owner:group of symlink instead of what it points to.

[ ! -d ./woof-arch ] && exit
[ ! -d ./woof-code ] && exit
[ ! -d ./woof-distro ] && exit

CURDIR="`pwd`"

echo
echo 'This script merges woof-arch, woof-code and woof-distro, to ../woof-out_*.

woof-arch:  architecture-dependent (x86, arm) files, mostly binary executables.
woof-code:  the core of Woof. Mostly scripts.
woof-distro: distro-configuration (Debian, Slackware, etc.) files.

Important: the host architecture is distinct from the target architecture.
The host is the machine you are running Woof on, the target is the machine
in which the Puppy that you build is going to run. Typically, you will build
on a x86 machine, and the target may be x86 or arm.'
echo

CNT=1
for ONEARCH in `find woof-arch -mindepth 1 -maxdepth 1 -type d | sed -e 's%^woof-arch/%%' | sort | tr '\n' ' '`
do
 echo "$CNT  $ONEARCH"
 CNT=$(($CNT + 1))
done
echo -n 'Type number of host architecture: '
read nHOSTARCH
HOSTARCH="`find woof-arch -mindepth 1 -maxdepth 1 -type d | sed -e 's%^woof-arch/%%' | sort | head -n $nHOSTARCH | tail -n 1`"
echo "...ok, $HOSTARCH"
echo

CNT=1
for ONEARCH in `find woof-arch -mindepth 1 -maxdepth 1 -type d | sed -e 's%^woof-arch/%%' | sort | tr '\n' ' '`
do
 echo "$CNT  $ONEARCH"
 CNT=$(($CNT + 1))
done
echo -n 'Type number of target architecture: '
read nTARGETARCH
TARGETARCH="`find woof-arch -mindepth 1 -maxdepth 1 -type d | sed -e 's%^woof-arch/%%' | sort | head -n $nTARGETARCH | tail -n 1`"
echo "...ok, $TARGETARCH"
echo

echo 'Woof builds a Puppy based on the binary packages from another distro.
We sometimes refer to this as the "compat-distro".'
echo
CNT=1
taPTN="s%^woof-distro/${TARGETARCH}/%%"
for ONEDISTRO in `find woof-distro/${TARGETARCH} -mindepth 1 -maxdepth 1 -type d | sed -e "${taPTN}" | sort | tr '\n' ' '`
do
 echo "$CNT  $ONEDISTRO"
 CNT=$(($CNT + 1))
done
echo -n 'Type number of compat-distro: '
read nCOMPATDISTRO
COMPATDISTRO="`find woof-distro/${TARGETARCH} -mindepth 1 -maxdepth 1 -type d | sed -e "${taPTN}" | sort | head -n $nCOMPATDISTRO | tail -n 1`"
echo "...ok, $COMPATDISTRO"
echo

echo 'The compat-distro usually has release versions, unless it is a rolling
release system such as Arch Linux. Please choose which release you want to
obtain the binary packages from.'

CNT=1
for ONECOMPAT in `find woof-distro/${TARGETARCH}/${COMPATDISTRO} -maxdepth 1 -mindepth 1 -type d | rev | cut -f 1 -d '/' | rev | sort | tr '\n' ' '`
do
 echo "$CNT  $ONECOMPAT"
 CNT=$(($CNT + 1))
done
if [ $CNT -eq 1 ];then
 echo
 echo "Sorry, there are no release directories inside woof-distro/${COMPATDISTRO}."
 echo "At least one is required. Quiting."
 exit
fi
echo -n 'Type number of release: '
read nCOMPATVERSION
COMPATVERSION="`find woof-distro/${TARGETARCH}/${COMPATDISTRO} -maxdepth 1 -mindepth 1 -type d | rev | cut -f 1 -d '/' | rev | sort | head -n $nCOMPATVERSION | tail -n 1`"
echo "...ok, $COMPATVERSION"

echo
echo 'Choices:'
echo "Host architecture:     $HOSTARCH"
echo "Target architecture:   $TARGETARCH"
echo "Compatible-distro:     $COMPATDISTRO"
echo "Compat-distro version: $COMPATVERSION"
echo
echo -n "If these are ok, press ENTER, other CTRL-C to quit: "
read waitforit

echo
echo "Directory '../woof-out_${HOSTARCH}_${TARGETARCH}_${COMPATDISTRO}_${COMPATVERSION}'
will now be created, if not already, and the contents of 'woof-code' copied
into it. Then, these will also be copied into it:
woof-arch/${HOSTARCH}/build
woof-arch/${TARGETARCH}/target
woof-distro/${TARGETARCH}/${COMPATDISTRO}/${COMPATVERSION} (files all levels)"
echo
echo "Any existing files in '../woof-out_${HOSTARCH}_${TARGETARCH}_${COMPATDISTRO}_${COMPATVERSION}' will be over-ridden."
echo "(Also, if you have any of your own files in folders 'boot', 'kernel-skeleton',
 'packages-templates', 'rootfs-skeleton' or 'support', they will be deleted.)"
echo -n 'Press ENTER to continue: '
read goforit

#111203 as files/dirs could be removed in future woofs, need to wipe entire target dirs first...
rm -r -f ../woof-out_${HOSTARCH}_${TARGETARCH}_${COMPATDISTRO}_${COMPATVERSION}/boot  2> /dev/null
rm -r -f ../woof-out_${HOSTARCH}_${TARGETARCH}_${COMPATDISTRO}_${COMPATVERSION}/kernel-skeleton  2> /dev/null
rm -r -f ../woof-out_${HOSTARCH}_${TARGETARCH}_${COMPATDISTRO}_${COMPATVERSION}/packages-templates  2> /dev/null
rm -r -f ../woof-out_${HOSTARCH}_${TARGETARCH}_${COMPATDISTRO}_${COMPATVERSION}/rootfs-skeleton  2> /dev/null
rm -r -f ../woof-out_${HOSTARCH}_${TARGETARCH}_${COMPATDISTRO}_${COMPATVERSION}/support  2> /dev/null
sync

mkdir -p ../woof-out_${HOSTARCH}_${TARGETARCH}_${COMPATDISTRO}_${COMPATVERSION}
echo "Copying woof-code/*..."
cp -a -f --remove-destination woof-code/* ../woof-out_${HOSTARCH}_${TARGETARCH}_${COMPATDISTRO}_${COMPATVERSION}/
sync
echo "Copying woof-arch/${HOSTARCH}/build/*..."
cp -a -f --remove-destination woof-arch/${HOSTARCH}/build/* ../woof-out_${HOSTARCH}_${TARGETARCH}_${COMPATDISTRO}_${COMPATVERSION}/
sync
echo "Copying woof-arch/${TARGETARCH}/target/*"
cp -a -f --remove-destination woof-arch/${TARGETARCH}/target/* ../woof-out_${HOSTARCH}_${TARGETARCH}_${COMPATDISTRO}_${COMPATVERSION}/
sync
echo
echo "Copying woof-distro/${COMPATDISTRO}/${COMPATVERSION}/*..."
#copy any top-level files, going down...

#120512 remove this option...
##111203 script run before, respect previous decision...
#DESTTYPE=''
#ONETOP="`find woof-distro -mindepth 1 -maxdepth 1 -type f | head -n 1`"
#[ ! "$ONETOP" ] && ONETOP="`find woof-distro/${TARGETARCH} -mindepth 1 -maxdepth 1 -type f | head -n 1`"
#[ ! "$ONETOP" ] && ONETOP="`find woof-distro/${TARGETARCH}/${COMPATDISTRO} -mindepth 1 -maxdepth 1 -type f | head -n 1`"
#[ ! "$ONETOP" ] && ONETOP="`find woof-distro/${TARGETARCH}/${COMPATDISTRO}/${COMPATVERSION} -mindepth 1 -maxdepth 1 -type f | head -n 1`"
#if [ "$ONETOP" ];then
# ONEBASE="`basename $ONETOP`"
# if [ -e ../woof-out_${HOSTARCH}_${TARGETARCH}_${COMPATDISTRO}_${COMPATVERSION}/${ONEBASE} ];then
#  DESTTYPE='file'
#  [ -h ../woof-out_${HOSTARCH}_${TARGETARCH}_${COMPATDISTRO}_${COMPATVERSION}/${ONEBASE} ] && DESTTYPE='symlink'
# fi
#fi
#if [ "$DESTTYPE" = "" ];then
# #111126 don't copy, more convenient if symlink, so only one instance of the file... 111129 offer choice...
# echo
# echo "Press ENTER only to copy files from woof-distro, any other char to symlink
#files back to woof-distro. Latter is a convenience for administrator, for most
#Woof users, please choose former."
# echo -n "Recommend press ENTER only: "
# read choosesymlink
#else
# choosesymlink=''
# [ "$DESTTYPE" = "symlink" ] && choosesymlink='y'
#fi
DESTTYPE='file'
choosesymlink=''

PARENTDIR="`echo -n "$CURDIR" | rev | cut -f 1 -d '/' | rev`" #ex: woof2
for ONETOP in `find woof-distro -mindepth 1 -maxdepth 1 -type f | tr '\n' ' '`
do
 if [ "$choosesymlink" = "" ];then
  cp -a -f --remove-destination $ONETOP ../woof-out_${HOSTARCH}_${TARGETARCH}_${COMPATDISTRO}_${COMPATVERSION}/
 else
  ONENAME="`basename $ONETOP`"
  ln -snf ../${PARENTDIR}/${ONETOP} ../woof-out_${HOSTARCH}_${TARGETARCH}_${COMPATDISTRO}_${COMPATVERSION}/${ONENAME}
 fi
done
for ONETOP in `find woof-distro/${TARGETARCH} -mindepth 1 -maxdepth 1 -type f | tr '\n' ' '`
do
 if [ "$choosesymlink" = "" ];then
  cp -f -a --remove-destination $ONETOP ../woof-out_${HOSTARCH}_${TARGETARCH}_${COMPATDISTRO}_${COMPATVERSION}/
 else
  ONENAME="`basename $ONETOP`"
  ln -snf ../${PARENTDIR}/${ONETOP} ../woof-out_${HOSTARCH}_${TARGETARCH}_${COMPATDISTRO}_${COMPATVERSION}/${ONENAME}
 fi
done
for ONETOP in `find woof-distro/${TARGETARCH}/${COMPATDISTRO} -mindepth 1 -maxdepth 1 -type f | tr '\n' ' '`
do
 if [ "$choosesymlink" = "" ];then
  cp -f -a --remove-destination $ONETOP ../woof-out_${HOSTARCH}_${TARGETARCH}_${COMPATDISTRO}_${COMPATVERSION}/
 else
  ONENAME="`basename $ONETOP`"
  ln -snf ../${PARENTDIR}/${ONETOP} ../woof-out_${HOSTARCH}_${TARGETARCH}_${COMPATDISTRO}_${COMPATVERSION}/${ONENAME}
 fi
done
#lowest level...
#cp -a -f --remove-destination woof-distro/${TARGETARCH}/${COMPATDISTRO}/${COMPATVERSION}/* ../woof-out_${HOSTARCH}_${TARGETARCH}_${COMPATDISTRO}_${COMPATVERSION}/
for ONETOP in `find woof-distro/${TARGETARCH}/${COMPATDISTRO}/${COMPATVERSION} -mindepth 1 -maxdepth 1 -type f | tr '\n' ' '`
do
 if [ "$choosesymlink" = "" ];then
  cp -f -a --remove-destination $ONETOP ../woof-out_${HOSTARCH}_${TARGETARCH}_${COMPATDISTRO}_${COMPATVERSION}/
 else
  ONENAME="`basename $ONETOP`"
  ln -snf ../${PARENTDIR}/${ONETOP} ../woof-out_${HOSTARCH}_${TARGETARCH}_${COMPATDISTRO}_${COMPATVERSION}/${ONENAME}
 fi
done
sync

echo "WOOF_HOSTARCH='$HOSTARCH'
WOOF_TARGETARCH='${TARGETARCH}'
WOOF_COMPATDISTRO='${COMPATDISTRO}'
WOOF_COMPATVERSION='${COMPATVERSION}'" > ../woof-out_${HOSTARCH}_${TARGETARCH}_${COMPATDISTRO}_${COMPATVERSION}/WOOFMERGEVARS

echo
echo "Now for some housekeeping..."

if [ -f ../woof-out_${HOSTARCH}_${TARGETARCH}_${COMPATDISTRO}_${COMPATVERSION}/boot/initrd-tree0/bin/bb-create-symlinks ];then
 echo
 echo "Symlinks being created inside here:"
 echo "../woof-out_${HOSTARCH}_${TARGETARCH}_${COMPATDISTRO}_${COMPATVERSION}/boot/initrd-tree0/bin"
 cd ../woof-out_${HOSTARCH}_${TARGETARCH}_${COMPATDISTRO}_${COMPATVERSION}/boot/initrd-tree0/bin
 for ONESYMLINK in `find . -type l | cut -f 2 -d '/' | tr '\n' ' '`
 do
  rm -f ${ONESYMLINK}
 done
 ./bb-create-symlinks
 cd $CURDIR #cd ../../../../
fi

#work around limitations of a version control system...
echo
echo "Some things are modified inside 'woof-code' to cater for most Version
Control Systems. Typically, a VCS cannot handle most of these:
1. Empty directories
2. Special file/directory permissions/ownership
3. Device nodes
4. Symlinks
5. Special characters (such as [, [[) in file/dir names
6. Binary files

BK's Bones VCS can handle all six. Fossil VCS can do no.4 & no.6 only (in fact,
most VCSs such as SVN, GIT and Mercurial, can handle no.4 & no.6). Woof has
lots of symlinks and binary files, and you must use a VCS that supports them.
No.5 is solved by avoiding usage of such special characters, except we have
workarounds for case of files named '[' and '[['."
echo
echo "Directory 'woof-code' has workarounds for no.1-3 (& partial 5):
1. An empty file named 'EMPTYDIRMARKER' inside all empty directories.
2. A file named VCSMETADATA has permissions/ownerships of special files/dirs.
3. 'dev' directories are converted to '*DEVDIR.tar.gz' tarball files.
5. Files named '[' and '[[' renamed 'LEFTSQBRACKETCHAR' 'DBLLEFTSQBRACKETCHAR'."
echo
echo "These workarounds will now be undone in '../woof-out_*'..."
echo -n "Press ENTER to continue: "
read goforit

fossil_fixup_func() { #workarounds for VCS...
 #param passed in is directory to fix.
 #5: '[' and '[[' files renamed...
 for FOSSILFIXFILE in `find ${1} -name LEFTSQBRACKETCHAR | tr '\n' ' '`
 do
  DIRFFF="`dirname "$FOSSILFIXFILE"`"
  mv -f $FOSSILFIXFILE $DIRFFF/[
 done
 for FOSSILFIXFILE in `find ${1} -name DBLLEFTSQBRACKETCHAR | tr '\n' ' '`
 do
  DIRFFF="`dirname "$FOSSILFIXFILE"`"
  mv -f $FOSSILFIXFILE $DIRFFF/[[
 done
 #1: empty dirs have file 'EMPTYDIRMARKER' in them...
 for FOSSILFIXFILE in `find ${1} -type f -name EMPTYDIRMARKER | tr '\n' ' '`
 do
  DIRFFF="`dirname "$FOSSILFIXFILE"`"
  rm -f $DIRFFF/EMPTYDIRMARKER
 done
 #3: 'dev' dir made into a tarball and stored in 'woof-arch'...
 for DEVFILE in `find ${1} -type f -name DEVDIRMARKER | tr '\n' ' '`
 do
  xDEVFILE="${CURDIR}/woof-arch/`cat $DEVFILE`"
  DIRFFF="`dirname "$DEVFILE"`"
  cp -f $xDEVFILE ${DIRFFF}/DEVDIR.tar.gz
  FCURRDIR="`pwd`"
  cd $DIRFFF
  tar -zxf DEVDIR.tar.gz
  rm -f DEVDIR.tar.gz
  rm -f DEVDIRMARKER
  #120503 i left some EMPTYDIRMARKER inside /dev...
  for FOSSILFIXFILE in `find dev -type f -name EMPTYDIRMARKER | tr '\n' ' '`
  do
   DIRFFF="`dirname "$FOSSILFIXFILE"`"
   rm -f $DIRFFF/EMPTYDIRMARKER
  done
  cd $FCURRDIR
 done
}

echo
cd ../woof-out_${HOSTARCH}_${TARGETARCH}_${COMPATDISTRO}_${COMPATVERSION}
echo "Fixing ../woof-out_${HOSTARCH}_${TARGETARCH}_${COMPATDISTRO}_${COMPATVERSION}/rootfs-skeleton..."
fossil_fixup_func rootfs-skeleton
echo "Fixing ../woof-out_${HOSTARCH}_${TARGETARCH}_${COMPATDISTRO}_${COMPATVERSION}/boot/initrd-tree0..."
fossil_fixup_func boot/initrd-tree0
echo "Fixing ../woof-out_${HOSTARCH}_${TARGETARCH}_${COMPATDISTRO}_${COMPATVERSION}/packages-templates..."
fossil_fixup_func packages-templates

#2: VCSMETADATA permissions/ownership...
if [ -s VCSMETADATA ];then
 echo "Fixing file/dir permissions/ownership..."
 for ONESPEC in `cat VCSMETADATA | tr '\n' ' '`
 do
  ONEFILE="`echo -n "$ONESPEC" | cut -f 1 -d ':'`"
  [ ! -e $ONEFILE ] && continue
  ONEPERM="`echo -n "$ONESPEC" | cut -f 2 -d ':'`"
  ONEOWNER="`echo -n "$ONESPEC" | cut -f 3 -d ':'`"
  ONEGROUP="`echo -n "$ONESPEC" | cut -f 4 -d ':'`"
  echo -n '.' #echo " $ONEFILE $ONEPERM $ONEOWNER $ONEGROUP"
  chmod $ONEPERM $ONEFILE
  #130528 change owner:group of symlink instead of what it points to...
  [ "$ONEOWNER" != "UNKNOWN" ] && chown -h $ONEOWNER $ONEFILE
  [ "$ONEOWNER" = "UNKNOWN" ] && chown -h root $ONEFILE
  [ "$ONEGROUP" != "UNKNOWN" ] && chgrp -h $ONEGROUP $ONEFILE
  [ "$ONEGROUP" = "UNKNOWN" ] && chgrp -h root $ONEFILE
 done
 echo
fi
cd $CURDIR
sync

#common dir to download pet pkgs to...
mkdir -p ../local-repositories/${TARGETARCH}/packages-pet
[ ! -e ../woof-out_${HOSTARCH}_${TARGETARCH}_${COMPATDISTRO}_${COMPATVERSION}/packages-pet ] && ln -s ../local-repositories/${TARGETARCH}/packages-pet ../woof-out_${HOSTARCH}_${TARGETARCH}_${COMPATDISTRO}_${COMPATVERSION}/packages-pet #111203 check exist.

#more links to common download...
if [ -f ../woof-out_${HOSTARCH}_${TARGETARCH}_${COMPATDISTRO}_${COMPATVERSION}/DISTRO_SPECS ];then

. ../woof-out_${HOSTARCH}_${TARGETARCH}_${COMPATDISTRO}_${COMPATVERSION}/DISTRO_SPECS

 #111126 code from 2createpackages...
 BINARIES='deb' #get them from packages-deb.
 [ "$DISTRO_BINARY_COMPAT" = "t2" ] && BINARIES="bz2" #get them from packages-bz2-${DISTRO_COMPAT_VERSION}.
 [ "$DISTRO_BINARY_COMPAT" = "slackware" ] && BINARIES="tgz_txz" #download to packages-tgz_txz-${DISTRO_COMPAT_VERSION}.
 [ "$DISTRO_BINARY_COMPAT" = "debian" ] && BINARIES="deb" #download to packages-deb-${DISTRO_COMPAT_VERSION}.
 [ "$DISTRO_BINARY_COMPAT" = "arch" ] && BINARIES="tar_xz" #download to packages-tar_xz-${DISTRO_COMPAT_VERSION}. 130306
 [ "$DISTRO_BINARY_COMPAT" = "puppy" ] && BINARIES="pet" #built entirely from pet pkgs.
 [ "$DISTRO_BINARY_COMPAT" = "scientific" ] && BINARIES="rpm" #Iguleder: download to packages-rpm-${DISTRO_COMPAT_VERSION}.
 [ "$DISTRO_BINARY_COMPAT" = "mageia" ] && BINARIES="rpm"
 [ "$DISTRO_BINARY_COMPAT" = "gentoo" ] && BINARIES="gentoo" #120515 download to packages-gentoo-gap6
[ "$DISTRO_BINARY_COMPAT" = "raspbian" ] && BINARIES="deb_raspbian" #download to packages-deb_raspbian-${DISTRO_COMPAT_VERSION}.
 BINARIES="${BINARIES}-${DISTRO_COMPAT_VERSION}"

 mkdir -p ../local-repositories/${TARGETARCH}/packages-${BINARIES}
 [ ! -e ../woof-out_${HOSTARCH}_${TARGETARCH}_${COMPATDISTRO}_${COMPATVERSION}/packages-${BINARIES} ] && ln -s ../local-repositories/${TARGETARCH}/packages-${BINARIES} ../woof-out_${HOSTARCH}_${TARGETARCH}_${COMPATDISTRO}_${COMPATVERSION}/packages-${BINARIES} #111203 check exist.
fi

#record target architecture in DISTRO_SPECS (will end up in /etc/ in Puppy build)...
if [ -f ../woof-out_${HOSTARCH}_${TARGETARCH}_${COMPATDISTRO}_${COMPATVERSION}/DISTRO_SPECS ];then
 if [ "`grep '^DISTRO_TARGETARCH' ../woof-out_${HOSTARCH}_${TARGETARCH}_${COMPATDISTRO}_${COMPATVERSION}/DISTRO_SPECS`" = "" ];then
  echo "DISTRO_TARGETARCH='${TARGETARCH}'" >> ../woof-out_${HOSTARCH}_${TARGETARCH}_${COMPATDISTRO}_${COMPATVERSION}/DISTRO_SPECS
 fi
fi

#until i upgrade the woof scripts... (i was planning to rename Packages-puppy-* to Packages-pet-*, aborted)
for ONEPP in `find ../woof-out_${HOSTARCH}_${TARGETARCH}_${COMPATDISTRO}_${COMPATVERSION} -mindepth 1 -maxdepth 1 -name 'Packages-pet-*' | tr '\n' ' '`
do
 BASEPP="`basename $ONEPP`"
 NEWBASE="`echo -n $BASEPP | sed -e 's%Packages-pet-%Packages-puppy-%'`"
 ln -s $BASEPP ../woof-out_${HOSTARCH}_${TARGETARCH}_${COMPATDISTRO}_${COMPATVERSION}/${NEWBASE}
done

#111127 make sure host puppy has these...
[ ! -f /usr/sbin/printcols ] && cp -af woof-arch/${HOSTARCH}/build/support/printcols /usr/sbin/ #column manipulator.
[ ! -f /bin/vercmp ] && cp -af woof-arch/${HOSTARCH}/target/boot/initrd-tree0/bin/vercmp /bin/ #dotted-version compare utility, see boot/vercmp.c

#111127 make sure target has these...
cp -af woof-arch/${TARGETARCH}/build/support/printcols ../woof-out_${HOSTARCH}_${TARGETARCH}_${COMPATDISTRO}_${COMPATVERSION}/${NEWBASE}/rootfs-skeleton/usr/sbin/
cp -af woof-arch/${TARGETARCH}/target/boot/initrd-tree0/bin/vercmp ../woof-out_${HOSTARCH}_${TARGETARCH}_${COMPATDISTRO}_${COMPATVERSION}/${NEWBASE}/rootfs-skeleton/bin/

echo
echo "Directory '../woof-out_${HOSTARCH}_${TARGETARCH}_${COMPATDISTRO}_${COMPATVERSION}'
is now normal, that is, the workarounds have been removed. Note,
../local-repositories has been created (if not already), to be used as a common
binary package download place. 'packages-pet' and 'packages-${BINARIES}'
have been created that link into it, where pkgs will be downloaded to."


###END###