source: scripts/autobuild.bash @ 3001

#!/bin/bash
# Helper bash script to build debian biarch i386 and amd64 packages from SVN
# Requires: subversion, devcripts, svn-buildpackage, pbuilder, awk, reprepro, sed, gnupg, sendmail.pl
# Coyright 2007 Alan Baghumian / GNU/GPL
#
# Created: 2007-10-20 15:10 +330 GMT
# Updated: 2008-02-01 19:30 +330 GMT
#
# TODO:
# 1) bz2 packages?

SRC_PKG_NAME="$1"
DIST_NAME="$2"
UNDERLINE=$'\137' # 137 is octal ASCII code for '_'

# Define some colors
GREEN="[1;32m"
YELLOW="[1;33m"
BLUE="[1;34m"
MAGENTA="[1;35m"
CYAN="[1;36m"
WHITE="[1;37m"
RED="[1;31m"
NORMAL="[0;39m"

# Load configuration file
if [ -f ./autobuild.conf ]; then
    source ./autobuild.conf
else
    echo "${GREEN}Couldn't load configuration file, exiting...${NORMAL}"
    exit     
fi

# Define core functions

# Reverts /etc/pbuilderrc base.tgz changes
revert_pbrc()
{
if [ ! -z $PXBUILDARCH ]; then
   sed -i s/base-$PXBUILDARCH.tgz/base.tgz/g /etc/pbuilderrc
else
   # Static revert
   sed -i s/base-i386.tgz/base.tgz/g /etc/pbuilderrc
   sed -i s/base-amd64.tgz/base.tgz/g /etc/pbuilderrc
fi
}

# Fixes /etc/pbuilderrc base.tgz
fix_pbrc()
{
sed -i s/base.tgz/base-$PXBUILDARCH.tgz/g /etc/pbuilderrc
}

# Builds or updates needed chroots
handle_chroot()
{
if [ ! -f /var/cache/pbuilder/base-$PXBUILDARCH.tgz ]; then
    echo "${GREEN}Creating ${YELLOW}$PXBUILDARCH${NORMAL} ${GREEN}chroot...${NORMAL}"
    pbuilder --create --distribution $DIST_NAME --override-config --debootstrap debootstrap --debootstrapopts --arch --debootstrapopts $PXBUILDARCH --logfile $LOG_FILE --mirror $APT_MIRROR --othermirror "$APT_MIRROR_OTHER" &> /dev/null || exit
else
    echo "${GREEN}Updating ${YELLOW}$PXBUILDARCH${NORMAL} ${GREEN}chroot...${NORMAL}"
    pbuilder --update --distribution $DIST_NAME --override-config --debootstrap debootstrap --debootstrapopts --arch --debootstrapopts $PXBUILDARCH --logfile $LOG_FILE --mirror $APT_MIRROR --othermirror "$APT_MIRROR_OTHER" &> /dev/null || exit
fi
}

# This takes three arguments: the name of the file to sign, the key
# or maintainer name to use and password. Based On Debian's debsign
signfile () {
    signcommand="gpg"
    signinterface="gpg"
    if [ $signinterface = gpg ]
    then
        gpgversion=`gpg --version | head -n 1 | cut -d' ' -f3`
        gpgmajorversion=`echo $gpgversion | cut -d. -f1`
        gpgminorversion=`echo $gpgversion | cut -d. -f2`
        if [ $gpgmajorversion -gt 1 -o $gpgminorversion -ge 4 ]
        then
                (cat "$1" ; echo "") | \
                    $signcommand --no-tty --batch --passphrase "$3" --local-user "$2" --clearsign \
                    --list-options no-show-policy-urls \
                    --armor --textmode --output - - > "$1.asc" || exit
        else
                (cat "$1" ; echo "") | \
                    $signcommand --no-tty --batch --passphrase "$3" --local-user "$2" --clearsign \
                        --no-show-policy-url \
                        --armor --textmode --output - - > "$1.asc" || exit
        fi
    else
        $signcommand -u "$2" +clearsig=on -fast < "$1" > "$1.asc"
    fi

    echo
    PRECIOUS_FILES=$(($PRECIOUS_FILES + 1))
    mv -f -- "$1.asc" "$1"
}

# Checking required inputs
if [ -z "$1" ]; then
    echo "${GREEN}Couldn't detect source package name, exiting...${NORMAL}"
    exit
else
    echo "${GREEN}Building ${YELLOW}$SRC_PKG_NAME${GREEN}...${NORMAL}"
fi

if [ -z "$2" ]; then
    echo "${GREEN}Couldn't detect distro name, exiting...${NORMAL}"
    exit
fi

# Initial Revert
revert_pbrc

cd $LOCAL_REPO_PATH
POOL=`find ./ -type d | grep /$SRC_PKG_NAME$ | awk 'BEGIN{FS="/"}{print $3}'`

if [ -z "$POOL" ]; then
      echo "${GREEN}Couldn't detect pool folder name, trying to calculate...${NORMAL}"

      case "$SRC_PKG_NAME" in
            lib*)
                POOL=`echo $SRC_PKG_NAME | cut -c1-4` 
            ;;
            *)
                POOL=`echo $SRC_PKG_NAME | cut -c1` 
            ;;
      esac
      echo "${GREEN}Detected pool as: ${YELLOW}$POOL${NORMAL}"
fi

cd $SVN_REPO_PATH
COMPONENT=`find ./ -maxdepth 2 -type d | grep /$SRC_PKG_NAME$ | awk 'BEGIN{FS="/"}{print $2}'`

if [ ! -z "$COMPONENT" ]; then
    echo "${GREEN}Detected component: ${YELLOW}$COMPONENT${NORMAL}"
else
    echo "${GREEN}Couldn't detect component, updating SVN...${NORMAL}"
    # Local working copy is out dated?
    svn up --username $SVN_USER --password $SVN_PASS &> /dev/null
    COMPONENT=`find ./ -maxdepth 2 -type d | grep /$SRC_PKG_NAME$ | awk 'BEGIN{FS="/"}{print $2}'`
fi

# Check if package exists on the SVN repo
if [ -d $SVN_REPO_PATH/$COMPONENT/$SRC_PKG_NAME ]; then
    cd $SVN_REPO_PATH/$COMPONENT/$SRC_PKG_NAME
else
    echo "${GREEN}SVN directory does not exists, exiting...${NORMAL}"
    exit
fi

# Update SVN to the latest version
rm `svn st | awk '{print $2}'` &> /dev/null
svn up --username $SVN_USER --password $SVN_PASS &> /dev/null

echo "${GREEN}Parsing changelog to get the latest prepared version...${NORMAL} "
PKG_VERSION=`dpkg-parsechangelog -ltrunk/debian/changelog  | grep Version | awk 'BEGIN{FS=": "}{print $2}' | awk 'BEGIN{FS="-"}{print $1}' | awk 'BEGIN{FS=" "}{print $1}'`
PKG_REVISION=`dpkg-parsechangelog -ltrunk/debian/changelog  | grep Version | awk 'BEGIN{FS=": "}{print $2}' | awk 'BEGIN{FS="-"}{print $2}'`

if [ ! -z $PKG_REVISION ]; then
    echo "${YELLOW}$PKG_VERSION-$PKG_REVISION${NORMAL}"
    TARBALL_EXT="orig.tar.gz"
    NATIVE_PKG=0
else
    echo "${YELLOW}$PKG_VERSION${NORMAL}"
    TARBALL_EXT="tar.gz"
    NATIVE_PKG=1
fi

echo "${GREEN}Getting the latest upstream version...${NORMAL}"
UPSTREAM_VERSION=`uscan --report --dehs --check-dirname-level 0 trunk/ | grep upstream-version | awk 'BEGIN{FS=">"}{print $2}' | awk 'BEGIN{FS="<"}{print $1}'`
UPSTREAM_TARBALL_URL=`uscan --report --dehs --check-dirname-level 0 trunk/ | grep upstream-url | awk 'BEGIN{FS=">"}{print $2}' | awk 'BEGIN{FS="<"}{print $1}'`

if [ ! -z `echo "$PKG_VERSION" | awk 'BEGIN{FS=":"}{print $2}'` ]; then
   PKG_VERSION=`echo "$PKG_VERSION" | awk 'BEGIN{FS=":"}{print $2}'`
fi

if [ ! "$UPSTREAM_VERSION" = "$PKG_VERSION" ]; then
   UPSTREAM_VERSION="$PKG_VERSION"
   echo "${YELLOW}Prepared $UPSTREAM_VERSION${NORMAL}"
else
   echo "${YELLOW}$UPSTREAM_VERSION${NORMAL}"
fi

# Log filename
if [ ! -z $PKG_REVISION ]; then
     LOG_FILE="$LOCAL_REPO_ROOT../buildlogs/$SRC_PKG_NAME$UNDERLINE$UPSTREAM_VERSION-$PKG_REVISION.blog"
else
     LOG_FILE="$LOCAL_REPO_ROOT../buildlogs/$SRC_PKG_NAME$UNDERLINE$UPSTREAM_VERSION.blog"
fi

LOG_PATH="$LOCAL_REPO_ROOT../buildlogs"

# Clean up old logs
rm $LOG_PATH/$SRC_PKG_NAME* &> /dev/null

# clean up untarred packages
rm -r $SRC_PKG_NAME-* &> /dev/null
rm -r build-area/* &> /dev/null
rm *.tar.gz &> /dev/null

# Make tarballs
mkdir -p tarballs &> /dev/null
mkdir -p build-area &> /dev/null

# Package is not native, get tarballs
if [ $NATIVE_PKG -eq 0 ]; then
     if [ -f "$LOCAL_REPO_PATH/$COMPONENT/$POOL/$SRC_PKG_NAME/$SRC_PKG_NAME$UNDERLINE$UPSTREAM_VERSION.$TARBALL_EXT" ]; then
         echo "${GREEN}Copying orig tarball to the tarballs directory...${NORMAL}"
         cp "$LOCAL_REPO_PATH/$COMPONENT/$POOL/$SRC_PKG_NAME/$SRC_PKG_NAME$UNDERLINE$UPSTREAM_VERSION.$TARBALL_EXT" tarballs/ -v >> $LOG_FILE
     else
         # try to get upstream tarball using uscan
         uscan --rename --no-dehs --check-dirname-level 0 trunk/ >> $LOG_FILE

         # if fails, try classic manual way
         if [ ! -f "$SRC_PKG_NAME$UNDERLINE$UPSTREAM_VERSION.$TARBALL_EXT" ]; then
            if [ ! -z $UPSTREAM_TARBALL_URL ]; then
                 wget -c $UPSTREAM_TARBALL_URL &> /dev/null
                 FRESH_TARBALL=`ls | grep .tar.gz$`
            else
                 echo "${GREEN}Upstream tarball URL is empty, exiting...${NORMAL}"
                 echo "Upstream tarball URL is empty, exiting..." >> $LOG_FILE
            fi

            if [ ! -z "$FRESH_TARBALL" ]; then
               mv "$FRESH_TARBALL" "$SRC_PKG_NAME$UNDERLINE$UPSTREAM_VERSION.$TARBALL_EXT" -v >> $LOG_FILE
            else
               echo "${GREEN}Can't find fresh tarball file, exiting...${NORMAL}"
               echo "Can't find fresh tarball file, exiting..." >> $LOG_FILE
               exit
            fi
         fi

             echo "${GREEN}Copying orig tarball to the local apt repo...${NORMAL}"

             # Not exists? Create it!
             if [ ! -d "$LOCAL_REPO_PATH/$COMPONENT/$POOL/$SRC_PKG_NAME" ]; then
                  mkdir -p "$LOCAL_REPO_PATH/$COMPONENT/$POOL/$SRC_PKG_NAME"
             fi

             cp *."$TARBALL_EXT" "$LOCAL_REPO_PATH/$COMPONENT/$POOL/$SRC_PKG_NAME/" -v >> $LOG_FILE

             # Copy tarball
             cp *.$TARBALL_EXT tarballs/ -v >> $LOG_FILE

         # Clean up untarred packages
         rm -r $SRC_PKG_NAME-* &> /dev/null
     fi

     # Check if both tarballs exist
     if [ ! -f "tarballs/$SRC_PKG_NAME$UNDERLINE$UPSTREAM_VERSION.$TARBALL_EXT" ]; then
          echo "${RED}Source tarball not found${NORMAL}" && exit
     fi

     if [ ! -f "$LOCAL_REPO_PATH/$COMPONENT/$POOL/$SRC_PKG_NAME/$SRC_PKG_NAME$UNDERLINE$UPSTREAM_VERSION.$TARBALL_EXT" ]; then
          echo "${RED}Source tarball not found on repo${NORMAL}" && exit
     fi
fi

# Goto trunk to start build
cd trunk/

# Fixing /etc/pbuilderrc base.tgz
PXBUILDARCH="amd64"
fix_pbrc

# Build amd64 chroot if needed
handle_chroot

# Build amd64 packages
echo "${GREEN}Building ${YELLOW}amd64${NORMAL} ${GREEN}specific packages...${NORMAL}"
svn-buildpackage --svn-ignore --svn-noninteractive --svn-builder="pdebuild --use-pdebuild-internal -- --logfile $LOG_FILE" || exit

# Revert /etc/pbuilderrc base.tgz changes
revert_pbrc

# Check if the result is platform independent and skip i386 build if needed
ARCH_ALL_COUNT="$(ls ../build-area/ | grep -c _all.deb)"
ARCH_I386_COUNT="$(ls ../build-area/ | grep -c _i386.deb)"
ARCH_AMD64_COUNT="$(ls ../build-area/ | grep -c _amd64.deb)"

# SVN Tagging: amd64 buid was OK, so tag it
cd ..
if [ $NATIVE_PKG -eq 0 ]; then
     if [ ! -d tags ]; then
         mkdir tags &> /dev/null
         svn add tags &> /dev/null
     fi

     if [ ! -d tags/$UPSTREAM_VERSION-$PKG_REVISION ]; then
         echo "${GREEN}Tagging ${YELLOW}$SRC_PKG_NAME $UPSTREAM_VERSION-$PKG_REVISION${NORMAL} ${GREEN}...${NORMAL}"
         echo "${GREEN}Tagging $SRC_PKG_NAME $UPSTREAM_VERSION-$PKG_REVISION" >> $LOG_FILE
         svn copy trunk/ tags/$UPSTREAM_VERSION-$PKG_REVISION &> /dev/null
         svn commit -m "* BUILDD: Tagging $SRC_PKG_NAME $UPSTREAM_VERSION-$PKG_REVISION" --username $SVN_USER --password $SVN_PASS &> /dev/null
     fi
else
     if [ ! -d tags ]; then
         mkdir tags &> /dev/null
         svn add tags &> /dev/null
     fi

     if [ ! -d tags/$UPSTREAM_VERSION ]; then
         echo "${GREEN}Tagging ${YELLOW}$SRC_PKG_NAME $UPSTREAM_VERSION${NORMAL} ${GREEN}...${NORMAL}"
         echo "${GREEN}Tagging $SRC_PKG_NAME $UPSTREAM_VERSION" >> $LOG_FILE
         svn copy trunk/ tags/$UPSTREAM_VERSION &> /dev/null
         svn commit -m "* BUILDD: Tagging $SRC_PKG_NAME $UPSTREAM_VERSION" --username $SVN_USER --password $SVN_PASS &> /dev/null
     fi
fi

if [ $ARCH_ALL_COUNT -gt 0 -a $ARCH_I386_COUNT -lt 1 -a $ARCH_AMD64_COUNT -lt 1 ]; then
    echo "${GREEN}Build is platform independent, skipping ${YELLOW}i386${NORMAL} ${GREEN}build...${NORMAL}"
    ARCH_INDEP=1
    cd build-area/
else
    ARCH_INDEP=0

    cd trunk/

    # Fixing /etc/pbuilderrc base.tgz
    PXBUILDARCH="i386"
    fix_pbrc

    # Build i386 chroot if needed
    handle_chroot

    # Restore the latest SVN version
    rm `svn st | awk '{print $2}'` &> /dev/null
    svn up --username $SVN_USER --password $SVN_PASS &> /dev/null

    # Build i386 packages
    echo "${GREEN}Building ${YELLOW}i386${NORMAL} ${GREEN}specific packages...${NORMAL}"
    svn-buildpackage --svn-ignore --svn-noninteractive --svn-builder="pdebuild --use-pdebuild-internal --debbuildopts -B -- --logfile $LOG_FILE" || exit

    # Revert /etc/pbuilderrc base.tgz changes
    revert_pbrc

    cd ../build-area/
fi

if [ $ARCH_INDEP -eq 0 ]; then
    echo "${GREEN}Signing and adding ${YELLOW}multi_arch${NORMAL} ${GREEN}packages to the APT repository...${NORMAL}"

    # Merging changes file for multiarch upload
    AMD64_CHANGES=`ls | grep _amd64.changes`
    I386_CHANGES=`ls | grep _i386.changes`
    mergechanges -f $AMD64_CHANGES $I386_CHANGES >> $LOG_FILE

    CHANGES_FILE=`ls | grep _multi.changes$`
    DSC_FILE=`ls | grep .dsc$`

    if [ -z "$CHANGES_FILE" ]; then
        echo "${GREEN}No ${YELLOW}changes${NORMAL} ${GREEN}file?${NORMAL}"
        echo "No changes file" >> $LOG_FILE
        echo "Build failed" >> $LOG_FILE
        revert_pbrc
        exit 
    fi

    DSC_OLDMD5SUM=`md5sum $DSC_FILE | awk '{print $1}'`
    DSC_OLDSIZE=`ls -l $DSC_FILE | awk '{print $5}'` 

    # Sign dsc file
    signfile "$DSC_FILE" "$GPG_KEY" "$GPG_PASS" &> /dev/null
    
    DSC_MD5SUM=`md5sum $DSC_FILE | awk '{print $1}'`
    DSC_SIZE=`ls -l $DSC_FILE | awk '{print $5}'` 

    sed -i s/" $DSC_OLDMD5SUM "/" $DSC_MD5SUM "/g $CHANGES_FILE || exit
    sed -i s/" $DSC_OLDSIZE "/" $DSC_SIZE "/g $CHANGES_FILE || exit

    # Sign changes file
    signfile "$CHANGES_FILE" "$GPG_KEY" "$GPG_PASS" &> /dev/null

    # Add package to the repo
    cd $LOCAL_REPO_ROOT
    reprepro --ignore=wrongdistribution -Vb . include $DIST_NAME $SVN_REPO_PATH/$COMPONENT/$SRC_PKG_NAME/build-area/$CHANGES_FILE &> /dev/null
else
    echo "${GREEN}Signing and adding ${YELLOW}plat_indep${NORMAL} ${GREEN}packages to the APT repository...${NORMAL}"

    CHANGES_FILE=`ls | grep _amd64.changes$`
  
    if [ -z "$CHANGES_FILE" ]; then
        echo "${GREEN}No ${YELLOW}changes${NORMAL} ${GREEN}file?${NORMAL}"
        echo "No changes file" >> $LOG_FILE
        echo "Build failed" >> $LOG_FILE
        revert_pbrc
        exit 
    fi
 
    DSC_FILE=`ls | grep .dsc$`

    DSC_OLDMD5SUM=`md5sum $DSC_FILE | awk '{print $1}'`
    DSC_OLDSIZE=`ls -l $DSC_FILE | awk '{print $5}'` 

    # Sign dsc file
    signfile "$DSC_FILE" "$GPG_KEY" "$GPG_PASS" &> /dev/null
  
    DSC_MD5SUM=`md5sum $DSC_FILE | awk '{print $1}'`
    DSC_SIZE=`ls -l $DSC_FILE | awk '{print $5}' ` 
  
    sed -i s/" $DSC_OLDMD5SUM "/" $DSC_MD5SUM "/g $CHANGES_FILE || exit
    sed -i s/" $DSC_OLDSIZE "/" $DSC_SIZE "/g $CHANGES_FILE || exit

    # Sign changes file
    signfile "$CHANGES_FILE" "$GPG_KEY" "$GPG_PASS" &> /dev/null
  
    # Add package to the repo
    cd $LOCAL_REPO_ROOT
    reprepro --ignore=wrongdistribution -Vb . include $DIST_NAME $SVN_REPO_PATH/$COMPONENT/$SRC_PKG_NAME/build-area/$CHANGES_FILE &> /dev/null
fi

# Sign release files
sh ../../sign &> /dev/null && cd

# Sending mail
if [ $NATIVE_PKG -eq 0 ]; then
     $MAILER "$MAIL_SENDER" "$MAILING_LIST" "$SRC_PKG_NAME-$PKG_VERSION-$PKG_REVISION Build Confirmation" "$SRC_PKG_NAME-$PKG_VERSION-$PKG_REVISION has been built successfully. For more information see log file at http://buildd.parsix.org/$SRC_PKG_NAME$UNDERLINE$UPSTREAM_VERSION-$PKG_REVISION.blog"
else
     $MAILER "$MAIL_SENDER" "$MAILING_LIST" "$SRC_PKG_NAME-$PKG_VERSION Build Confirmation" "$SRC_PKG_NAME-$PKG_VERSION has been built successfully. For more information see log file at http://buildd.parsix.org/$SRC_PKG_NAME$UNDERLINE$UPSTREAM_VERSION.blog"
fi

echo "${GREEN}Finished.${NORMAL}"
echo "Build OK" >> $LOG_FILE