From 21f32aff61eca7e79b76f9cbccae28efa66d369e Mon Sep 17 00:00:00 2001 From: ranke Date: Mon, 6 Jul 2009 21:45:30 +0000 Subject: Rewrite of backporting scripts for exclusive use of pdebuild, getting rid of schroot git-svn-id: svn+ssh://stiller/home/schatz/reps/r-backports@56 f247ebf9-aa24-0410-a698-f940e971ad2b --- CRAN_backporting.txt | 96 -------------------------------- README | 32 +++++++++++ backport_all_etch.sh | 29 ---------- backport_all_lenny.sh | 29 ---------- backport_others.sh | 32 +++++++++++ backport_r-base.sh | 53 ++++++++++++++++++ build_others.sh | 16 ++++++ build_r-base.sh | 24 ++++++++ publish_r-base.sh | 8 +-- r-base/backport_r-base_lenny.sh | 44 --------------- r-base/backport_r-base_to_etch.sh | 112 -------------------------------------- r-base/build_r-base.sh | 10 ---- 12 files changed, 161 insertions(+), 324 deletions(-) delete mode 100644 CRAN_backporting.txt create mode 100644 README delete mode 100644 backport_all_etch.sh delete mode 100644 backport_all_lenny.sh create mode 100644 backport_others.sh create mode 100644 backport_r-base.sh create mode 100644 build_others.sh create mode 100644 build_r-base.sh delete mode 100644 r-base/backport_r-base_lenny.sh delete mode 100644 r-base/backport_r-base_to_etch.sh delete mode 100644 r-base/build_r-base.sh diff --git a/CRAN_backporting.txt b/CRAN_backporting.txt deleted file mode 100644 index 701b142..0000000 --- a/CRAN_backporting.txt +++ /dev/null @@ -1,96 +0,0 @@ -Prerequisites: -- Linux system with X GB disk space -- cdebootstrap, schroot installed -- GPG key for signing packages - - -1. Setting up the chroot environment for backporting - -The filesystem where you create the chroot needs to be mounted with exec and -dev options, and you have to be root to do this: - - cdebootstrap -a i386 -f build lenny lenny32 http://debian.tu-bs.de/debian - -This will create a i386 chroot with essential software for building packages in -the newly created directory "lenny32". Of course, if you are backporting for a -different release, substitute its name for lenny, also in the following. - - -2. Configuring your chroot for comfortable use with schroot - -Put an entry like this into your /etc/schroot/schroot.conf (mine -is located on my external drive mounted on /exthd) - - [lenny32] - description=Debian lenny i386 (etch32) - location=/exthd/lenny32 - personality=linux32 - priority=3 - users=ranke - groups=ranke,root - root-groups=root - aliases=lenny-ia32 - run-setup-scripts=true - run-exec-scripts=true - -You can then login into your chroot (as root) with - - schroot -c lenny32 - -install the editor of your choice (after updating the list of available -packages e.g. by apt-get update), add security and sid sources to your -/etc/apt/sources.list (in order to be able to backport from sid), add a user -(e.g. after installing adduser), install and configure console-data for the -keymap, install and configure locales, install subversion, devscripts (for -editing changelogs with dch, pulls a lot of dependencies unfortunately) and -patchutils (for interdiff). - -In order to build packages as a normal user, use su username to change to the -user account you want to use, then - - svn co http://kriemhild.uft.uni-bremen.de/svn/r-backports - -which will set up a couple of scripts and directories that I use for backporting. -If you want to backport R, you can try - - apt-get build-dep r-base - -which would pull in everything you need for building r-base, if all build -dependencies could be satisfied. This is unlikely, however, because then we -would not have to backport... - -You also need you gpg key in your build environment. I achieved this by the -following bind mount in /etc/fstab - - /home/ranke/.gnupg /exthd/lenny32/home/ranke/.gnupg noauto bind 0 0 - -because I don't want my secret key on the USB drive. - - -3. Backporting R - -Then go to directory r-base, edit the backporting script and try it with - - fakeroot sh backport_r-base_lenny.sh - -You will probably have to install some more build dependencies that have been -introduced in newer R versions. - - -4. Backporting (compiling) recommended packages - -This is done in the parent directory, using the script backport_recommended.sh. -You don't need fakeroot for this, because fakeroot is called in it where needed. - -You need to install an R backport in order to ensure compatibility and to satisfy build -dependencies of recommended packages. You will also need cdbs for building the -r-cran-* packages, unixodbc-dev for building r-cran-rodbc, and python-all-dev -as well as python-numpy for building python-rpy. - - -5. Building packages for another architecture with pbuilder - -I usually do backports for i386 in a permanent chroot. Then, when I build the -packages for amd64 using pbuilder, I find missing build dependencies. The scripts -I use for this are r-base/build_r-base.sh and build_all_etch.sh. Of course, pbuilder -has to be set up before... diff --git a/README b/README new file mode 100644 index 0000000..8f9e82e --- /dev/null +++ b/README @@ -0,0 +1,32 @@ +This directory contains my scripts to build backports of R software from Debian +sid to older Debian distributions. + +Currently (2009-07-06) I am using the following setup: + +/root/.pbuilderrc contains: + + NAME="$DIST-$ARCH" + APTCACHEHARDLINK=no + DISTRIBUTION="$DIST" + DEBOOTSTRAPOPTS=("--arch" "$ARCH" "${DEBOOTSTRAPOPTS[@]}") + BASETGZ="`dirname $BASETGZ`/$NAME-base.tgz" + BUILDRESULT="/exthd/pbuilder/$NAME/result/" + APTCACHE="/exthd/pbuilder/$NAME/aptcache/" + MIRRORSITE="http://ftp.de.debian.org/debian/" + OTHERMIRROR="deb http://security.debian.org $DIST/updates main|deb http://localhost/r-cran $DIST-cran/" + COMPONENTS="main" + +This means on my external USB drive mounted under /exthd, I have the +appropriate aptcache and buildresult directories setup under /exthd/pbuilder. I +have four pbuilder images used for package creation under /var/cache/pbuilder: + +/var/cache/pbuilder/etch-amd64-base.tgz /var/cache/pbuilder/lenny-amd64-base.tgz +/var/cache/pbuilder/etch-i386-base.tgz /var/cache/pbuilder/lenny-i386-base.tgz + +All of these have a standard Debian mirror and the security sources setup in +their /etc/apt/sources.list, as well as my local http package source where the +r-base package build results are available. I have also imported the apt key +of my repository inside the pbuilder chroots. + + + diff --git a/backport_all_etch.sh b/backport_all_etch.sh deleted file mode 100644 index e7e5602..0000000 --- a/backport_all_etch.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash - -# Script to automate "backporting" of packages for Debian if the -# only difference to the upstream packages is the signature. - -# Author: Johannes Ranke - -export DEBEMAIL=jranke@uni-bremen.de -export DEBFULLNAME="Johannes Ranke" - -text="Recompiled on etch for CRAN" - -for i in rmatrix; do -#for i in boot cluster codetools foreign kernsmooth lattice rmatrix mgcv nlme rodbc rpart survival vr littler; do - cd $i - rm -rf $i* - rm *.deb - apt-get source -t unstable $i - cd $i-* - version=`dpkg-parsechangelog | grep ^Version | cut -f2 -d " "`~etchcran.2.9.0 - dch -b -v $version -D etch-cran $text - - sed -i '/^Build-Depends/s/r-base-dev (>= 2\..\..)/r-base-dev (>= 2.9.0-1~etchcran.0)/' debian/control - sed -i '/^Build-Depends-Indep/s/r-base-dev (>= 2\..\..)/r-base-dev (>= 2.9.0-1~etchcran.0)/' debian/control - dch -a "debian/control: adapt build dependencies on r-base-dev to current backport" - - fakeroot dpkg-buildpackage -sa - cd ../.. -done diff --git a/backport_all_lenny.sh b/backport_all_lenny.sh deleted file mode 100644 index b0608cd..0000000 --- a/backport_all_lenny.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash - -# Script to automate "backporting" of packages for Debian if the -# only difference to the upstream packages is the signature. - -# Author: Johannes Ranke - -export DEBEMAIL=jranke@uni-bremen.de -export DEBFULLNAME="Johannes Ranke" - -text="Recompiled on lenny for CRAN" - -for i in rmatrix; do -#for i in boot cluster codetools foreign kernsmooth lattice mgcv nlme rmatrix rodbc rpart survival vr littler; do - cd $i - rm -rf $i* - rm *.deb - apt-get source -t unstable $i - cd $i-* - version=`dpkg-parsechangelog | grep ^Version | cut -f2 -d " "`~lennycran.2.9.0 - dch -b -v $version -D lenny-cran $text - - sed -i '/^Build-Depends/s/r-base-dev (>= 2\..\..)/r-base-dev (>= 2.9.0-1~lennycran.0)/' debian/control - sed -i '/^Build-Depends-Indep/s/r-base-dev (>= 2\..\..)/r-base-dev (>= 2.9.0-1~lennycran.0)/' debian/control - dch -a "debian/control: adapt build dependencies on r-base-dev to current backport" - - fakeroot dpkg-buildpackage -sa - cd ../.. -done diff --git a/backport_others.sh b/backport_others.sh new file mode 100644 index 0000000..837266d --- /dev/null +++ b/backport_others.sh @@ -0,0 +1,32 @@ +# Author: Johannes Ranke + +# Preconditions: - shell variables DIST and ARCH +# - sid sources in /etc/apt/sources/list +# - Edit the sed scripts below to adapt the build dependencies +# to the appropriate version of r-base-dev +# - Set the R version + +# R version against which the software is compiled +Rversion=2.9.1 + +echo Distribution is $DIST +echo Architecture is $ARCH + +text="Recompiled on $DIST for CRAN" + +pbuilder update + +for i in boot cluster codetools foreign kernsmooth lattice mgcv nlme rmatrix rodbc rpart survival vr littler; do + apt-get source -t unstable $i + cd $i-* + version=`dpkg-parsechangelog | grep ^Version | cut -f2 -d " "`~$DIST"cran".$Rversion + dch -b -v $version -D $DIST-cran $text + + sed -i '/^Build-Depends/s/r-base-dev (>= 2\..\..)/r-base-dev (>= 2.9.1-2~lennycran.0)/' debian/control + sed -i '/^Build-Depends-Indep/s/r-base-dev (>= 2\..\..)/r-base-dev (>= 2.9.1-2~lennycran.0)/' debian/control + + dch -a "debian/control: adapt build dependencies on r-base-dev to current backport" + + pdebuild --debbuildopts '-sa' --buildresult /home/ranke/svn/website/www/ranke/r-cran/$DIST-cran/ -- --distribution $DIST --basetgz /var/cache/pbuilder/$DIST-$ARCH-base.tgz --aptcache /exthd/pbuilder/$DIST-$ARCH/aptcache --buildplace /exthd/pbuilder/$DIST-$ARCH/build + cd .. +done diff --git a/backport_r-base.sh b/backport_r-base.sh new file mode 100644 index 0000000..7c93f4c --- /dev/null +++ b/backport_r-base.sh @@ -0,0 +1,53 @@ +#!/bin/bash + +# Script to automate backporting r-base for CRAN + +# Author: Johannes Ranke +# Developed in parallel with Vincent Goulet + +# Preconditions: - shell variables DIST and ARCH +# - sid sources in /etc/apt/sources/list + +export DEBEMAIL="jranke@uni-bremen.de" +export DEBFULLNAME="Johannes Ranke" +echo Distribution is $DIST +echo Architecture is $ARCH + +if [ -d r-base-* ] +then + rm -rf `ls -d r-base-*` +fi + +apt-get update +apt-get source -t unstable r-base + +# Change any ~ in the name of the build directory for a - to avoid a +# bug in texi2dvi. Thanks Vincent +if [ -d r-base-*~* ] +then + dir=`ls -d r-base-*~*` + mv $dir `echo $dir | sed y/~/-/` +fi + +cd r-base-* + +# Old version +oldversion=`dpkg-parsechangelog | grep ^Version | cut -f2 -d " "` +cranversion=$oldversion"~"$DIST"cran.0" + +# Add new version to changelog +dch -v "${cranversion}" -D $DIST-cran -b "Backport from Debian unstable to Debian $DIST" + +# Reverts for lenny: +# None at this time + +pbuilder update +pdebuild --debbuildopts '-sa' --buildresult /home/ranke/svn/website/www/ranke/r-cran/$DIST-cran/ -- --distribution $DIST --basetgz /var/cache/pbuilder/$DIST-$ARCH-base.tgz --aptcache /exthd/pbuilder/$DIST-$ARCH/aptcache --buildplace /exthd/pbuilder/$DIST-$ARCH/build +if [ $? -ne 0 ]; then + exit +fi + +cd .. + +# Create diff to Debian unstable +interdiff -z r-base_*.diff.gz /home/ranke/svn/website/www/ranke/r-cran/$DIST-cran/r-base_*cran*.diff.gz > /home/ranke/svn/website/www/ranke/r-cran/$DIST-cran/interdiff_to_debian_sid diff --git a/build_others.sh b/build_others.sh new file mode 100644 index 0000000..350150a --- /dev/null +++ b/build_others.sh @@ -0,0 +1,16 @@ +# Author: Johannes Ranke + +# Preconditions: - shell variables DIST and ARCH +# - CRAN sources in /etc/apt/sources/list + +echo Distribution is $DIST +echo Architecture is $ARCH + +apt-get update + +for i in cluster foreign kernsmooth lattice mgcv nlme rmatrix rodbc rpart survival vr littler; do + apt-get source -t $DIST-cran $i + cd $i-* + + pdebuild --debbuildopts '-b' --buildresult /home/ranke/svn/website/www/ranke/r-cran/$DIST-cran/ -- --distribution $DIST --basetgz /var/cache/pbuilder/$DIST-$ARCH-base.tgz --aptcache /exthd/pbuilder/$DIST-$ARCH/aptcache --buildplace /exthd/pbuilder/$DIST-$ARCH/build +done diff --git a/build_r-base.sh b/build_r-base.sh new file mode 100644 index 0000000..f78ef50 --- /dev/null +++ b/build_r-base.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +# Script to automate building r-base binaries for CRAN + +# Author: Johannes Ranke + +# Preconditions: - shell variables DIST and ARCH +# - CRAN sources in /etc/apt/sources/list + +echo Distribution is $DIST +echo Architecture is $ARCH + +apt-get update +apt-get source -t $DIST-cran r-base + +cd r-base-* + +pbuilder update +pdebuild --debbuildopts '-b' --buildresult /home/ranke/svn/website/www/ranke/r-cran/$DIST-cran/ -- --distribution $DIST --basetgz /var/cache/pbuilder/$DIST-$ARCH-base.tgz --aptcache /exthd/pbuilder/$DIST-$ARCH/aptcache --buildplace /exthd/pbuilder/$DIST-$ARCH/build +if [ $? -ne 0 ]; then + exit +fi + +cd .. diff --git a/publish_r-base.sh b/publish_r-base.sh index 577c377..e623912 100644 --- a/publish_r-base.sh +++ b/publish_r-base.sh @@ -2,10 +2,10 @@ rsync -avz \ --include "build_log*" \ --include "build_errors*" \ - --include "interdiff_to_debian_sid" \ - --include "r-base_*lennycran*.diff.gz" \ - --include "r-base_*lennycran*.dsc" \ + --include "interdiff_to_debian_sid_$DIST" \ + --include "r-base_*$DISTcran*.diff.gz" \ + --include "r-base_*$DISTcran*.dsc" \ --include "*.deb" \ --exclude "*" \ r-base/ \ - ranke@chem:/home/ranke/svn/website/www/ranke/r-cran/lenny-cran + ranke@chem:/home/ranke/svn/website/www/ranke/r-cran/$DIST-cran diff --git a/r-base/backport_r-base_lenny.sh b/r-base/backport_r-base_lenny.sh deleted file mode 100644 index c3fa215..0000000 --- a/r-base/backport_r-base_lenny.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/bash - -# Script to automate building of r-base and for Debian stable -# This script should be run as root, in one way or another. - -# Author: Vincent Goulet -# Modified for backporting to Debian stable by -# Johannes Ranke - -export DEBEMAIL="Johannes Ranke " - -rm -rf r-base* r-base-core* r-mathlib* - -#apt-get update # needs real root privileges -apt-get source -t unstable r-base -cd r-base-* - -# Old version -oldversion=`dpkg-parsechangelog | grep ^Version | cut -f2 -d " "` -cranversion=$oldversion"~lennycran.0" - -# Add new version to changelog -dch -v "${cranversion}" -D lenny-cran -b "Backport from Debian unstable to Debian lenny" - -# Reverts for lenny: -# - -# Build package showing all messages and separating output to build and error logs -(dpkg-buildpackage -tc | tee ../build_log_r-base_i386) 3>&1 1>&2 2>&3 | tee ../build_errors_r-base_i386 -if [ $? -ne 0 ]; then - exit 1 -fi - -cd .. - -# Create diff to Debian unstable -interdiff -z *.diff.gz > interdiff_to_debian_sid - -# Sign logs and interdiff -for i in interdiff_to_debian_sid build_log_r-base_i386 build_errors_r-base_i386 -do - gpg --clearsign $i - mv $i.asc $i -done diff --git a/r-base/backport_r-base_to_etch.sh b/r-base/backport_r-base_to_etch.sh deleted file mode 100644 index 644edeb..0000000 --- a/r-base/backport_r-base_to_etch.sh +++ /dev/null @@ -1,112 +0,0 @@ -#!/bin/bash - -# Script to automate building of r-base and r-recommended packages for -# the Ubuntu release specified in file /etc/lsb-release. This script -# should be run as root, in one way or another. - -# Author: Vincent Goulet -# Modified for backporting to Debian etch by -# Johannes Ranke - -export DEBEMAIL="Johannes Ranke " - -#rm -rf r-base* r-base-core* r-mathlib* - -#apt-get update -apt-get source -t unstable r-base -cd r-base-* - -# Old version -oldversion=`dpkg-parsechangelog | grep ^Version | cut -f2 -d " "` -cranversion=$oldversion"~etchcran.1" - -# Add new version to changelog -dch -v "${cranversion}" -D etch-cran -b "Backport form Debian unstable to Debian etch" - -# Reverts for etch: -# -# 1. Starting with Debian packages for R 2.6.1-2, the build dependency -# to 'refblas3-dev|atlas3-base-dev' is changed to 'libblas-dev' to -# use the new gfortran-built BLAS libraries. From R 2.7.0, the -# dependency is on 'libblas-dev | libatlas-base-dev' for -# r-base-dev. - -sed -i '/^Build-Depends/s/libblas-dev/refblas3-dev|atlas3-base-dev/' debian/control -sed -i '/^Depends/s/libblas-dev | libatlas-base-dev/refblas3-dev|atlas3-base-dev/' debian/control -dch -a "debian/control: revert Build-Depends: to 'refblas3-dev|atlas3-base-dev' and Depends: to 'refblas3-dev|atlas3-base-dev' since etch does not have the new gfortran-built BLAS libraries" - -# 2. Starting with Debian packages for R 2.6.2, new build dependency -# on liblapack-dev to switch back to using Debian's Lapack rather -# than the version supplied by R. From R 2.7.0, the dependency is -# on 'liblapack-dev (>= 3.1.1)' for r-base and 'liblapack-dev | -# libatlas-base-dev' for r-base-dev. - -sed -i '/^Build-Depends/s/liblapack-dev (>= 3.1.1), //' debian/control -sed -i '/^Depends/s/liblapack-dev | libatlas-base-dev, //' debian/control - dch -a "debian/control: revert Build-Depends: and Depends: fields since we use the Lapack supplied with R" - sed -i '/^lapack/{ -s/lapack/\#lapack/ -a\ -# jr 24 Apr 2008 Set to =no for Debian etch -a\lapack = --with-lapack=no -}' debian/rules - dch -a "debian/rules: Accordingly, do not configure --with-lapack" - -# 3. Requirement for tk8.4 >= 8.4.16-2 introduced in r-base 2.6.0-4 to -# circumvent a breakage with 8.4.16-1. The version in etch is 8.4.12-1etch2. - -rm debian/shlibs.local -dch -a "delete debian/shlibs.local since the dependency on tk8.4 is not an issue for Debian etch" - -# 4. The TeX distribution in etch is tetex - -sed -i '/^Build-Depends/s/texlive-base, texlive-latex-base, texlive-generic-recommended, texlive-fonts-recommended, texlive-extra-utils, texlive-latex-recommended, texlive-latex-extra, texinfo, texi2html/tetex-bin, tetex-extra/' debian/control -dch -a "debian/control: revert Build-Depends: to 'tetex-bin, tetex-extra' since etch does not have texlive" - -# 5. xauth is part of xbase-clients in etch - -sed -i '/^Build-Depends/s/xauth/xbase-clients/' debian/control -dch -a "debian/control: replace xauth by xbase-clients in Build-Depends:" - -# 6. Tcl and TK are 8.4 on Debian etch - -sed -i '/^Build-Depends/s/ tcl8.5-dev/tcl8.4-dev/' debian/control -sed -i '/^Build-Depends/s/ tk8.5-dev/tk8.4-dev/' debian/control -dch -a "debian/control: revert build dependence for tcl/tk to 8.4" - -# 7. Openjdk is not in etch - -sed -i '/^Build-Depends/s/, openjdk-6-jdk \[!arm !hppa !kfreebsd-i386 !kfreebsd-amd64 !hurd-i386\]//' debian/control -dch -a "debian/control: Remove build dependence on openjdk" - -# 8. R 2.9.0 wants debhelper >= 7.0.0, which is not in etch - -sed -i '/^Build-Depends/s/debhelper (>= 7.0.0)/debhelper/' debian/control -dch -a "debian/control: revert Build-Depends: to 'debhelper' since etch has a version < 7.0.0" -sed -i 's/dh_prep/dh_clean/g' debian/rules -dch -a "debian/rules: revert dh_prep calls to dh_clean calls since the latter is not present in this release's version of 'debhelper'" -echo 5 > debian/compat -dch -a "debian/compat: revert 'debhelper' compatibility level to 5" - -# 9. R 2.9.0 wants ucf >= 3.0 which is not available in etch (2.0020) - -sed -i 's/ucf (>= 3.0)/ucf/' debian/control -dch -a "debian/control: revert Depends: of r-base-core to 'ucf' since the version in etch is < 3.0" - -# Build package showing all messages and separating output to build and error logs -(dpkg-buildpackage -tc | tee ../build_log_r-base_i386) 3>&1 1>&2 2>&3 | tee ../build_errors_r-base_i386 -if [ $? -ne 0 ]; then - exit 1 -fi - -cd .. - -# Create diff to Debian unstable -interdiff -z *.diff.gz > interdiff_to_debian_sid - -# Sign logs and interdiff -for i in interdiff_to_debian_sid build_log_r-base_i386 build_errors_r-base_i386 -do - gpg --clearsign $i - mv $i.asc $i -done diff --git a/r-base/build_r-base.sh b/r-base/build_r-base.sh deleted file mode 100644 index 38a753c..0000000 --- a/r-base/build_r-base.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -# Script to automate building of backported r-base packages for other Debian architectures - -# Author: Johannes Ranke - -apt-get source r-base -cd r-base-* -pdebuild -cd .. -- cgit v1.2.1