#!/usr/bin/ksh

# 20090923-01 Thomas.Wagner@Sun.COM for extensions - no warranty, use at your own risk!
# extended to collect additional data for detailed patch and package analysis

# obtain links to fresh versions of this script from the upcoming blog: http://blogs.sun.com/proactive
## maintained by PTAs (from Germany, possibly other countries)

##(2) 20090930-02 changes copy contents of /var/sadm/patch to exclude huge IBM DB2 patches (hundreds of MBs) e.g. 1810701-008
##(1) 20090923-01 added pkgchk, ls -l<t|> var/sadm/pkg, see extended-data-collection-Solaris_10-patch-zones-1.1a.txt on sunspace

VERSION=`basename $0`-0.3-20090930-1659

###
### Script used to gather data on patch information on a system. 
###
### Used to analyze a patchadd or patchrm failure in the Solaris OS. 
### 
### See article: 
###   http://www.sun.com/bigadmin/features/articles/patchanalysis.jsp 
###   for more information. 
###

echo ""
echo "`basename $0` - Version $VERSION"
echo ""

##(1) resort, adjust script name
if [ $# -ne 1 ]; then
	echo "./patchanalysis_gather_extended.ksh needs one arg the target root"
	echo " usage: ./patchanalysis_gather_extended.ksh /"
        echo " or with relative mounted root-filesystem:"
	echo " usage: ./patchanalysis_gather_extended.ksh /a"
	exit 1
fi

##(2)
COPYVARSADMPATCHEXCLUDEPATTERN="/[0-9][0-9][0-9][0-9][0-9][0-9][0-9]-[0-9][0-9][0-9]"

##(1) add host information to storage file
#cut after the first "dot" if present:  hostname.domain.tld --> hostname
NODENAME=`/usr/bin/hostname | sed -e 's/\..*//'`
DATESTRING=`/usr/bin/date -u +'%Y.%m.%d.%H.%M'`
SYSTEMIDENT=patchpkgdata.`/usr/bin/hostid`.$NODENAME-$DATESTRING

##(1) put $SYSTEMIDENT-tempdir- into WORKDIR path
WORKDIR=/tmp/${SYSTEMIDENT}.$$

MNTPT=$1
export MNTPT
if [ ! -d "$MNTPT" ]; then
	echo "Could not find directory $MNTPT"
	exit 1
fi
mkdir $WORKDIR || {
	echo "could not create $WORKDIR"
	exit 1
}

##(1) write script version into file and filename
echo $VERSION > $WORKDIR/$VERSION

##(1) workaround, use plain "cat" if no zip is present
BZIPTWO=/usr/bin/cat
ZIPEXT=""

echo ""
##(1) check for availability of bzip2 or pbzip2 (preferred)
TESTZIP=`which /usr/local/bin/pbzip2` 1>/dev/null
if [ -z "$ZIPEXT" ] && echo $TESTZIP | grep "^/.*pbzip2$" >/dev/null && [ -x $TESTZIP ]
  then
    BZIPTWO=`which /usr/local/bin/pbzip2` 2>/dev/null
    ZIPEXT=.bz2
fi

TESTZIP=`which bzip2` 1>/dev/null
if [ -z "$ZIPEXT" ] && echo $TESTZIP | grep "^/.*bzip2$" >/dev/null && [ -x $TESTZIP ]
  then
    BZIPTWO=`which bzip2` 2>/dev/null
    ZIPEXT=.bz2
fi

##(1) workaround, use gzip instead of (p)bzip2
TESTZIP=`which gzip` 1>/dev/null
if [ -z "$ZIPEXT" ] && echo $TESTZIP | grep "^/.*gzip$" >/dev/null && [ -x $TESTZIP ]
  then
    BZIPTWO=`which gzip` 2>/dev/null
    ZIPEXT=.gz
fi

echo "collected data will be stored as /tmp/${SYSTEMIDENT}.tar"$ZIPEXT
echo "compression program: >$BZIPTWO<, file extension >$ZIPEXT<"
echo ""


##(1) TO DO: catch trap and clean up left over temporary files

##(1) TO DO: space checking before actually collecting data

##(1) get list of zonenames in case we are working without a -R relative root directory
##(1) TO DO: improve add listing zones-names with -R (make switch for FCS special case; see ppc)
if [ "$MNTPT" == "/" ]
  then
   echo "collecting zone names of >running< zones on this system"
   ZONESLIST=`/usr/sbin/zoneadm list -i`
   #global:/
   #nonglobalname:/zoneespath/nonglobalname
   ZONESPATHLIST="`/usr/sbin/zoneadm list -ip | cut -d ':' -f 2,4`"
 else
   echo "NOTE: not checking zones contents with relative root patch (only checked if working on current BE)"
   echo "ask for an enhance version of this script."
   ZONESLIST="global"
   ZONESPATHLIST="global:/"
fi #zoneslist $MNTPT
echo "System uses these zones: $ZONESLIST"

# grab patchadd -p from mntpoint
 patchadd -R $MNTPT -p > $WORKDIR/patchadd-p.out 2>&1 &

# grab the UCE/Xvm version if installed

pkginfo -R $MNTPT VERSION > /dev/null 2>&1
if [ $? != 0 ]; then
	pkgparam -R $MNTPT -v SUNWucea VERSION > $WORKDIR/patchtool.out 2>&1
fi
pkginfo -R $MNTPT SUNWscnconnmgt > /dev/null 2>&1
if [ $? != 0 ]; then
	pkgparam -R $MNTPT -v SUNWscnconnmgt  VERSION >> $WORKDIR/patchtool.out 2>&1
fi

#gather the uce/xvm logs if they exist
if [ -d ${MNTPT}/var/opt/SUNWuce/agent/logs ]; then
	mkdir ${WORKDIR}/uce_logs
	cp -r ${MNTPT}/var/opt/SUNWuce/agent/logs/ ${WORKDIR}/uce_logs/
fi

mkdir $WORKDIR/etc
cp ${MNTPT}/etc/system $WORKDIR/etc/
cp ${MNTPT}/etc/vfstab $WORKDIR/etc/
cp ${MNTPT}/etc/release $WORKDIR/etc/
cp ${MNTPT}/var/sadm/install/contents  $WORKDIR/
mkdir $WORKDIR/var_messages
cp ${MNTPT}/var/adm/messages* $WORKDIR/var_messages/
mkdir $WORKDIR/patch_pkg
ls -tr1 ${MNTPT}/var/sadm/patch > $WORKDIR/patch_pkg/patch-ls-tr1.out
mkdir $WORKDIR/patch_pkg/logs
##(2) changed to avoid copying huge IBM patches
#cp -r ${MNTPT}/var/sadm/patch/* $WORKDIR/patch_pkg/logs/
##(2) excluded, see below (avoid double copying the large amounts of log files)
#(cd ${MNTPT}/var/sadm/patch/* && find . -depth -print | /usr/bin/egrep -v $COPYVARSADMPATCHEXCLUDEPATTERN | cpio -umdvp $WORKDIR/patch_pkg/logs/)
ls -tr1 ${MNTPT}/var/sadm/pkg > $WORKDIR/patch_pkg/pkg-ls-tr1.out
pkginfo -R $MNTPT > $WORKDIR/patch_pkg/pkginfo.out
pkginfo -R $MNTPT -p > $WORKDIR/patch_pkg/pkginfo-p.out


mkdir $WORKDIR/install_data
cp -r $MNTPT/var/sadm/install_data/* $WORKDIR/install_data/
cp $MNTPT/var/sadm/system/admin/CLUSTER $WORKDIR/
mkdir $WORKDIR/system_logs
cp -r $MNTPT/var/sadm/system/logs/* $WORKDIR/system_logs/
mkdir $WORKDIR/zones
cp -r $MNTPT/etc/zones/*  $WORKDIR/zones/
df -k $MNTPT > $WORKDIR/df-k 2>&1

##(1) insert pieces from extended-data-collection-Solaris_10-patch-zones-1.1a.txt
#  ========code========
#   #global zone
#
#      pkgchk -n    2>&1 | bzip2 >    /var/tmp/pkgchk_-n_`hostname`-`date  +'%Y%m%d-%H%M'$$`.bz2
#      pkgchk -q -n 2>&1 | bzip2 > /var/tmp/pkgchk_-q_-n_`hostname`-`date  +'%Y%m%d-%H%M'$$`.bz2
#
#   #all non-global zones  (note: include the name of the server/global-zone in the filename)
#
#    zlogin place-zone-name-here
#      pkgchk -n    2>&1 | bzip2 >    /var/tmp/pkgchk_-n_`hostname`-`date  +'%Y%m%d-%H%M'$$`.bz2
#      pkgchk -q -n 2>&1 | bzip2 > /var/tmp/pkgchk_-q_-n_`hostname`-`date  +'%Y%m%d-%H%M'$$`.bz2
#      exit
#    #continue with next non-global zone
#
#   ====================

##(1) TO DO: add feature to be usable on relative mountpoint / ABE
#note: do compress output files to save space in $WORKDIR
if [ "$MNTPT" == "/" ]
  then
  for ZONENAME in $ZONESLIST
    do
     echo ""
     echo "working on zone: $ZONENAME"
     ZLOGIN="/usr/bin/ksh -c"    #used for the global zone
     [ "$ZONENAME" != "global" ] && ZLOGIN="zlogin $ZONENAME"
     echo "command used to wrap pkgchk: $ZLOGIN"
     ##(1) disabled echo "running pkgchk -n on the zone: $ZONENAME"
     ##(1) disabled pkgchk -n  2>&1  | $BZIPTWO >   $WORKDIR/pkgchk_-n_`hostname`-$ZONENAME-`date  +'%Y%m%d-%H%M'$$`$ZIPEXT
     ##(1) disabled echo "  .... done."
     echo "NOTE: pkgchk takes a long time to complete checking of most of the files from the contents database. You might want to run this script with \"nice\" to get lower cpu load (diskload might stay the same)"
     echo "running pkgchk -q -n on the zone: $ZONENAME \c"
     $ZLOGIN "/usr/sbin/pkgchk -q -n" 2>&1  | $BZIPTWO > $WORKDIR/pkgchk_-q_-n_`hostname`-$ZONENAME-`date  +'%Y%m%d-%H%M'$$`$ZIPEXT
     echo "  .... done."
    done #zone in ZONESLIST do pkgchk
 else
   echo "zones pkgchk is disabled for relative mountpoint"
   touch $WORKDIR/pkgchk.is.disabled.mountpoint.is.not.slash
fi #pkgchk if $MNTPT == "/"
 


#
#3) search for package and patch lockfiles (global + non-global zones)
#
#
#   ========code========
#   #global zone
#
#   find /var/sadm -name '*\!*' -ls  > /var/tmp/find_var_sadm_lockfiles_`hostname`-`date  +'%Y%m%d-%H%M'$$`.out
#
#   #all non-global zones
#
#    zlogin place-zone-name-here
#    # then inside the new login:
#
#       find /var/sadm -name '*\!*' -ls  > /var/tmp/find_var_sadm_lockfiles_`hostname`-`date  +'%Y%m%d-%H%M'$$`.out
#
#    #continue with next non-global zone
#
#   ====================
#
#
for ZONENAME in $ZONESLIST
  do
   SAVEIFS=${IFS}
   IFS='	' ZONESROOTPATH=`echo $ZONESPATHLIST | /usr/bin/egrep "^${ZONENAME}:" | /usr/bin/cut -d ':' -f 2`
   IFS=${SAVEIFS}
   [ "$ZONENAME" != "global" ] && ZONESROOTPATH="$ZONESROOTPATH"/root
   echo ""
   echo "searching for package lockfiles of zone: $ZONENAME - the zone's root directory path is $ZONESROOTPATH \c"
   find ${MNTPT}${ZONESROOTPATH}/var/sadm -name '*\!*' -ls  > $WORKDIR/find_var_sadm_lockfiles_`hostname`-$ZONENAME-`date  +'%Y%m%d-%H%M'$$`.out
   echo "... done"
done


#4) search for left over files in /tmp (global + non-global zones)
#
#   ========code========
#   #global zone
#
#     ls -axl /tmp/*.ai.* /tmp/*pkg* > /var/tmp/find_tmp_ai_-ls_`hostname`-`date  +'%Y%m%d-%H%M'$$`.out
#
#   #all non-global zones
#
#    zlogin place-zone-name-here
#    # then inside the new login:
#
#     ls -axl /tmp/*.ai.* /tmp/*pkg* > /var/tmp/find_tmp_ai_-ls_`hostname`-`date  +'%Y%m%d-%H%M'$$`.out
#
#    #continue with next non-global zone
#
#   ====================
#

for ZONENAME in $ZONESLIST
  do
   SAVEIFS=${IFS}
   IFS='	' ZONESROOTPATH=`echo $ZONESPATHLIST | /usr/bin/egrep "^${ZONENAME}:" | /usr/bin/cut -d ':' -f 2`
   IFS=${SAVEIFS}
   [ "$ZONENAME" != "global" ] && ZONESROOTPATH="$ZONESROOTPATH"/root
   echo ""
   echo "searching for left over files in /tmp: $ZONENAME - the zone's root directory path is $ZONESROOTPATH"
   ls -axl ${MNTPT}${ZONESROOTPATH}/tmp/*.ai.* ${MNTPT}${ZONESROOTPATH}/*pkg* > $WORKDIR/find_tmp_ai_-ls_`hostname`-$ZONENAME-`date  +'%Y%m%d-%H%M'$$`.out 2>&1
   ##(1) added: print filesystemtype for ${MNTPT}${ZONESROOTPATH}/tmp filesystem 
   echo "searching filesystem type of the tmp directory: $ZONENAME - the zone's root directory path is $ZONESROOTPATH \c"
   df -n ${MNTPT}${ZONESROOTPATH}/tmp > $WORKDIR/tmp-filesystemtype_`hostname`-$ZONENAME-`date  +'%Y%m%d-%H%M'$$`.out 2>&1
   echo "... done"
done



#5) search for left over directories and files in /var/sadm/pkg/ and /var/sadm/patch/                                                        
#                                                                                                                                            
#   ========code========
#   #global zone
#
#    ls -axl /var/sadm/pkg   >   /var/tmp/ls_-axl_var_sadm_pkg_`hostname`-`date  +'%Y%m%d-%H%M'$$`.out
#    ls -axl /var/sadm/patch > /var/tmp/ls_-axl_var_sadm_patch_`hostname`-`date  +'%Y%m%d-%H%M'$$`.out
#
#   #all non-global zones
#
#    zlogin place-zone-name-here
#    # then inside the new login:
#
#      ls -axl /var/sadm/pkg   >   /var/tmp/ls_-axl_var_sadm_pkg_`hostname`-`date  +'%Y%m%d-%H%M'$$`.out
#      ls -axl /var/sadm/patch > /var/tmp/ls_-axl_var_sadm_patch_`hostname`-`date  +'%Y%m%d-%H%M'$$`.out
#
#    #continue with next non-global zone
#
#   ====================

for ZONENAME in $ZONESLIST
  do
   SAVEIFS=${IFS}
   IFS='	' ZONESROOTPATH=`echo $ZONESPATHLIST | /usr/bin/egrep "^${ZONENAME}:" | /usr/bin/cut -d ':' -f 2`
   IFS=${SAVEIFS}
   [ "$ZONENAME" != "global" ] && ZONESROOTPATH="$ZONESROOTPATH"/root
   echo ""
   echo "collecting directory listings of ${MNTPT}${ZONESROOTPATH}/var/sadm/pkg and ${MNTPT}${ZONESROOTPATH}/var/sadm/patch \c"
   ls -axl   ${MNTPT}${ZONESROOTPATH}/var/sadm/pkg   >   $WORKDIR/ls_-axl_var_sadm_pkg_`hostname`-$ZONENAME-`date  +'%Y%m%d-%H%M'$$`.out 2>&1
   ls -axl   ${MNTPT}${ZONESROOTPATH}/var/sadm/patch >   $WORKDIR/ls_-axl_var_sadm_patch_`hostname`-$ZONENAME-`date  +'%Y%m%d-%H%M'$$`.out 2>&1
##(1) enhanced to store timestamp sorted directory listing
   ls -axltr ${MNTPT}${ZONESROOTPATH}/var/sadm/pkg   >   $WORKDIR/ls_-axltr_var_sadm_pkg_`hostname`-$ZONENAME-`date  +'%Y%m%d-%H%M'$$`.out 2>&1
   ls -axltr ${MNTPT}${ZONESROOTPATH}/var/sadm/patch >   $WORKDIR/ls_-axltr_var_sadm_patch_`hostname`-$ZONENAME-`date  +'%Y%m%d-%H%M'$$`.out 2>&1
   echo "... done"
done


#
#6) collect metadata of installed software packages and patches
#   (optional)
#   
#   Note: files will be of size 10MB-100MB
#   
#   ========code========
#   #global zone
#     cd /var/sadm && \
#          find . -print | egrep -v "/(undo.Z)|(undo)|(obsolete.Z)|(obsolete)$" | cpio -oc  | bzip2 > /var/tmp/find_var_sadm_-print_`hostname`-`date  +'%Y%m%d-%H%M'$$`.cpio.bz2
#
#
#   #all non-global zones
#
#    zlogin place-zone-name-here
#    # then inside the new login:
#     cd /var/sadm && \
#          find . -print | egrep -v "/(undo.Z)|(undo)|(obsolete.Z)|(obsolete)$" | cpio -oc  | bzip2 > /var/tmp/find_var_sadm_-print_`hostname`-`date  +'%Y%m%d-%H%M'$$`.cpio.bz2                                                                                                                           
#
#    #continue with next non-global zone
#
#   ====================
#
#

##(2) note: IBM DB1 patches take several hundreds of megabytes (lot of regular files), so exclude them all

for ZONENAME in $ZONESLIST
  do
   SAVEIFS=${IFS}
   IFS='	' ZONESROOTPATH=`echo $ZONESPATHLIST | /usr/bin/egrep "^${ZONENAME}:" | /usr/bin/cut -d ':' -f 2`
   IFS=${SAVEIFS}
   [ "$ZONENAME" != "global" ] && ZONESROOTPATH="$ZONESROOTPATH"/root
   echo ""
   echo "collect package metadata of all zones (without backing up (undo|undo.Z|obsolete|obsolete.Z). Using special exclude pattern: $COPYVARSADMPATCHEXCLUDEPATTERN )\c"
##(2) add egrep -v $$COPYVARSADMPATCHEXCLUDEPATTERN
   find ${MNTPT}${ZONESROOTPATH}/var/sadm/ -print | egrep -v "/(undo.Z)|(undo)|(obsolete.Z)|(obsolete)$" | egrep -v "$COPYVARSADMPATCHEXCLUDEPATTERN" | cpio -oc  | bzip2 > $WORKDIR/find_var_sadm_-print_`hostname`-$ZONENAME-`date  +'%Y%m%d-%H%M'$$`.cpio.bz2
   echo "... done"
done

#
#6) collect metadata of installed software packages and patches

#7) check unique Zone-IDs
#
#   ========code========
#   #global zone
#
#   grep -v "^#" /etc/zones/index | sort -t : +3 | awk -F: '{if(i==$4){print "dup-ID:",$0;}else{print "ID OK :",$0;} i=$4;}' > /var/tmp/zones-unique-ids-check-`hostname`.out
#
#   ====================

   /usr/bin/grep -v "^#" "$MNTPT"/etc/zones/index | sort -t : +3 | awk -F: '{if(i==$4){print "dup-ID:",$0;}else{print "ID OK :",$0;} i=$4;}' > $WORKDIR/zones-unique-ids-check-`hostname`.out



##(1) TO DO improvement: use trap for cleanups, make tar name random and prevent deleting files not owned by this script

cd /tmp || exit 1

echo "creating tar archive of collected files ... \c"
tar cf /tmp/${SYSTEMIDENT}.tar `/usr/bin/basename $WORKDIR`
echo "done"
if [ -z $ZIPEXT ] 
 then
    echo "no compression program found."
 else
    echo "compressing tar archive of collected files ... \c"
    $BZIPTWO /tmp/${SYSTEMIDENT}.tar
    if [ "$?" != "0" ]
      then
        echo "ERROR: compressing tar file /tmp/${SYSTEMIDENT}.tar failed. please investigate"
        echo " and cleanup left over files in /tmp and $WORKDIR yourself!"
        echo " exiting with error."
        exit 1
      else
        echo " done."
    fi # check for compression error $?
    echo " done."
    echo "cleaning up temporary files ...\c"
    if [ -r $WORKDIR/$VERSION ] 
       then
          [ -z "$NO_CLEANUP" ] && rm -r $WORKDIR
       else
          echo "ERROR: something went wrong with the safety check for the cleanup of the directory $WORKDIR"
          exit 1
    fi #safety check before cleanup $WORKDIR
    echo " done."
fi # -z $ZIPEXT

echo "Please upload the collected tar file to Sun."
ls -ltr /tmp/${SYSTEMIDENT}.tar*

echo ""
cat - << EOF

IMPORTANT:
==========

Generally please collect explorer-data with a special switch to collect more
data from non-global zones:
   
   explorer -w all,localzones

Upload these explorers to: http://supportfiles.sun.com

Important - the target place for your upload:
   Select as Destination for:

     o explorer data collector output -> explorer-emea
     /opt/SUNWexplo/output/explorer.87654321.nodename-2009.08.06.08.32.tar.gz 

     o all other collected files      -> europe-cores/<your_country>/incoming
     /tmp/patchpkgdata.87654321.nodename-2009.09.23.17.37.tar.bz2 (or gz)

If you select the correct Destination, the files are Sun internally
transferred to the correct destination, otherwise they might be
discarded.

Please find the following instructions for a extended collection
of information to make an exclusive check on Solaris 10 systems
with non-global zones.

EOF


##############################################################################
### Copyright Sun Microsystems, Inc. ALL RIGHTS RESERVED
### Use of this software is authorized pursuant to the
### terms of the license found at
### http://www.sun.com/bigadmin/common/berkeley_license.html
##############################################################################