[MIPS] Switch the script to use packages from Debian Wheezy

tools/trusted_cross_toolchains/trusted-toolchain-creator.mipsel.wheezy.sh

 #!/bin/bash

[Mark Seaborn, 2013/06/18 15:26:21]

Can you rename this file to remove "wheezy" from the name?  It doesn't make much
sense to have to rename it each time it switches to targeting a different Debian
release.

 # Copyright (c) 2012 The Native Client Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 #
 #@ This script creates the mips trusted SDK.
 #@ It must be run from the native_client directory.
 
 # This script is intended to build a mipsel-linux-gnu cross compilation
 # toolchain that runs on x86 linux and generates code for a little-endian,
 # hard-float, mips32 target.
-#
-# It expects the host machine to have relatively recent versions of GMP (4.2.0
-# or later), MPFR (2.4.2), and MPC (0.8.1) in order to build the GCC.
-#
-# Common way to get those is:
-# sudo apt-get install libmpfr-dev libmpc-dev libgmp3-dev
 
 ######################################################################
 # Config
 ######################################################################
 
 set -o nounset
 set -o errexit
 set -o xtrace
 
 readonly SCRIPT_DIR=$(dirname $0)
 
 readonly MAKE_OPTS="-j8"
 readonly ARCH="mips32"
 
+readonly GMP_URL="http://ftp.gnu.org/gnu/gmp/gmp-4.3.2.tar.bz2"
+readonly GMP_SHA1SUM="c011e8feaf1bb89158bd55eaabd7ef8fdd101a2c"
+
+readonly MPFR_URL="http://ftp.gnu.org/gnu/mpfr/mpfr-2.4.2.tar.bz2"
+readonly MPFR_SHA1SUM="7ca93006e38ae6e53a995af836173cf10ee7c18c"
+
+readonly MPC_URL="http://ftp.gnu.org/gnu/mpc/mpc-1.0.1.tar.gz"
+readonly MPC_SHA1SUM="8c7e19ad0dd9b3b5cc652273403423d6cf0c5edf"
+
 readonly GCC_URL="http://ftp.gnu.org/gnu/gcc/gcc-4.7.2/gcc-4.7.2.tar.bz2"
 readonly GCC_SHA1SUM="a464ba0f26eef24c29bcd1e7489421117fb9ee35"
 
 readonly BINUTILS_URL="http://ftp.gnu.org/gnu/binutils/binutils-2.22.tar.bz2"
 readonly BINUTILS_SHA1SUM="65b304a0b9a53a686ce50a01173d1f40f8efe404"
 
 readonly KERNEL_URL="http://www.linux-mips.org/pub/linux/mips/kernel/v2.6/linux-2.6.38.4.tar.gz"
 readonly KERNEL_SHA1SUM="377fa5cf5f1d0c396759b1c4d147330e7e5b6d7f"
 
 readonly GDB_URL="http://ftp.gnu.org/gnu/gdb/gdb-7.5.tar.bz2"
@@ skipping 49 matching lines @@
   local url=$1
   local revision=$2
   local filename="${TMP}/${url##*/}"
   local filetype="${url%%:*}"
 
   if [ "${filename}" == "" ]; then
     echo "Unknown error occured. Aborting."
     exit 1
   fi
 
-  if [ "${filetype}" == "svn" ]; then
-    SubBanner "checkout from ${url} -> ${filename}"
-    svn --force export -r ${revision} ${url} ${filename}
-  else
-    SubBanner "copying from ${url}"
-    cp ${url} ${filename}
+  if [ ! -f "${filename}/completed" ]; then
+    if [ "${filetype}" == "svn" ]; then
+      SubBanner "checkout from ${url} -> ${filename}"
+      svn --force export -r ${revision} ${url} ${filename}
+    else
+      SubBanner "copying from ${url}"
+      cp ${url} ${filename}
+    fi
+    touch ${filename}/completed
   fi
 }
 
 DownloadOrCopy() {
   local url=$1
   local filename="${TMP}/${url##*/}"
   local filetype="${url%%:*}"
 
   if [ "${filename}" == "" ]; then
     echo "Unknown error occured. Aborting."
@@ skipping 51 matching lines @@
   if [[ $(basename $(pwd)) != "native_client" ]] ; then
     echo "ERROR: run this script from the native_client/ dir"
     exit -1
   fi
 
   if ! mkdir -p "${INSTALL_ROOT}" ; then
      echo "ERROR: ${INSTALL_ROOT} can't be created."
     exit -1
   fi
 
-  for tool in cleanlinks wget ; do
+  for tool in wget ; do
     if ! which ${tool} ; then
       echo "Required binary $tool not found."
       echo "Exiting."
       exit 1
     fi
   done
 }
 
 
 ClearInstallDir() {
   Banner "clearing dirs in ${INSTALL_ROOT}"
   rm -rf ${INSTALL_ROOT}/*
+

[Mark Seaborn, 2013/06/18 15:26:21]

Did you intend to add this empty line?  It seems superfluous.

[petarj, 2013/06/19 23:12:33]

Removed.

   mkdir -p ${JAIL_MIPS32}
 }
 
 ClearBuildDir() {
   Banner "clearing dirs in ${BUILD_DIR}"
   rm -rf ${BUILD_DIR}/*
 }
 
 CreateTarBall() {
   local tarball=$1
   Banner "creating tar ball ${tarball}"
   tar cfz ${tarball}.tgz -C ${INSTALL_ROOT} .
 }
 
 
 # Download the toolchain source tarballs or use a local copy when available.
 DownloadOrCopyAndInstallToolchain() {
   Banner "Installing toolchain"
 
+  tarball="${TMP}/${GCC_URL##*/}"
+  DownloadOrCopyAndVerify ${GCC_URL} ${GCC_SHA1SUM}
+  SubBanner "extracting from ${tarball}"
+  tar jxf ${tarball} -C ${TMP}
+
+  pushd ${TMP}/gcc-*
+
+  local tarball="${TMP}/${GMP_URL##*/}"
+  DownloadOrCopyAndVerify ${GMP_URL} ${GMP_SHA1SUM}
+  SubBanner "extracting from ${tarball}"
+  tar jxf ${tarball}
+  local filename=`ls | grep gmp\-`
+  rm -f gmp
+  ln -s ${filename} gmp
+
+  local tarball="${TMP}/${MPFR_URL##*/}"
+  DownloadOrCopyAndVerify ${MPFR_URL} ${MPFR_SHA1SUM}
+  SubBanner "extracting from ${tarball}"
+  tar jxf ${tarball}
+  local filename=`ls | grep mpfr\-`
+  rm -f mpfr
+  ln -s ${filename} mpfr
+
+  local tarball="${TMP}/${MPC_URL##*/}"
+  DownloadOrCopyAndVerify ${MPC_URL} ${MPC_SHA1SUM}
+  SubBanner "extracting from ${tarball}"
+  tar zxf ${tarball}
+  local filename=`ls | grep mpc\-`
+  rm -f mpc
+  ln -s ${filename} mpc
+
+  popd
+
   local tarball="${TMP}/${BINUTILS_URL##*/}"
   DownloadOrCopyAndVerify ${BINUTILS_URL} ${BINUTILS_SHA1SUM}
   SubBanner "extracting from ${tarball}"
   tar jxf ${tarball} -C ${TMP}
 
-  tarball="${TMP}/${GCC_URL##*/}"
-  DownloadOrCopyAndVerify ${GCC_URL} ${GCC_SHA1SUM}
-  SubBanner "extracting from ${tarball}"
-  tar jxf ${tarball} -C ${TMP}
-
   tarball="${TMP}/${GDB_URL##*/}"
   DownloadOrCopyAndVerify ${GDB_URL} ${GDB_SHA1SUM}
   SubBanner "extracting from ${tarball}"
   tar jxf ${tarball} -C ${TMP}
 
   tarball="${TMP}/${KERNEL_URL##*/}"
   DownloadOrCopyAndVerify ${KERNEL_URL} ${KERNEL_SHA1SUM}
   SubBanner "extracting from ${tarball}"
   tar zxf ${tarball} -C ${TMP}
 
@@ skipping 11 matching lines @@
   local OLD_TEXT="as_warn_where (fragp->fr_file, fragp->fr_line, msg);"
   local NEW_TEXT="as_warn_where (fragp->fr_file, fragp->fr_line, \"%s\", msg);"
   local FILE_NAME="${TMP}/binutils-2.22/gas/config/tc-mips.c"
   sed -i "s/${OLD_TEXT}/${NEW_TEXT}/g" "${FILE_NAME}"
 
   export PATH=${INSTALL_ROOT}/bin:$PATH
 
 
   Banner "Building binutils"
 
+  rm -rf ${BUILD_DIR}/binutils/
   mkdir -p ${BUILD_DIR}/binutils/
   pushd ${BUILD_DIR}/binutils/
 
   SubBanner "Configuring"
   ${TMP}/binutils-2.22/configure \
     --prefix=${INSTALL_ROOT}     \
     --target=mipsel-linux-gnu    \
     --with-sysroot=${JAIL_MIPS32}
 
   SubBanner "Make"
   make ${MAKE_OPTS} all-binutils all-gas all-ld
 
   SubBanner "Install"
   make ${MAKE_OPTS} install-binutils install-gas install-ld
 
   popd
 
 
   Banner "Building GCC (initial)"
 
+  rm -rf ${BUILD_DIR}/gcc/initial
   mkdir -p ${BUILD_DIR}/gcc/initial
   pushd ${BUILD_DIR}/gcc/initial
 
   SubBanner "Configuring"
   ${TMP}/gcc-4.7.2/configure \
     --prefix=${INSTALL_ROOT} \
     --disable-libssp         \
     --disable-libgomp        \
     --disable-libmudflap     \
     --disable-fixed-point    \
@@ skipping 21 matching lines @@
 
   Banner "Installing Linux kernel headers"
   pushd ${TMP}/linux-2.6.38.4
   make headers_install ARCH=mips INSTALL_HDR_PATH=${JAIL_MIPS32}/usr
   popd
 
 
   Banner "Building EGLIBC (initial)"
 
   mkdir -p ${JAIL_MIPS32}/usr/lib
+
+  rm -rf ${BUILD_DIR}/eglibc/initial
   mkdir -p ${BUILD_DIR}/eglibc/initial
   pushd ${BUILD_DIR}/eglibc/initial
 
   SubBanner "Configuring"
   BUILD_CC=gcc                      \
   AR=mipsel-linux-gnu-ar            \
   RANLIB=mipsel-linux-gnu-ranlib    \
   CC=mipsel-linux-gnu-gcc           \
   CXX=mipsel-linux-gnu-g++          \
   ${TMP}/eglibc-2_14/libc/configure \
@@ skipping 17 matching lines @@
                        -nostartfiles  \
                        -shared        \
                        -x c /dev/null \
                        -o ${JAIL_MIPS32}/usr/lib/libc.so
 
   popd
 
 
   Banner "Building GCC (intermediate)"
 
+  rm -rf ${BUILD_DIR}/gcc/intermediate
   mkdir -p ${BUILD_DIR}/gcc/intermediate
   pushd ${BUILD_DIR}/gcc/intermediate
 
   SubBanner "Configuring"
   ${TMP}/gcc-4.7.2/configure  \
     --prefix=${INSTALL_ROOT}  \
     --disable-libssp          \
     --disable-libgomp         \
     --disable-libmudflap      \
     --disable-fixed-point     \
@@ skipping 11 matching lines @@
   make ${MAKE_OPTS} all
 
   SubBanner "Install"
   make ${MAKE_OPTS} install
 
   popd
 
 
   Banner "Building EGLIBC (final)"
 
+  rm -rf ${BUILD_DIR}/eglibc/final
   mkdir -p ${BUILD_DIR}/eglibc/final
   pushd ${BUILD_DIR}/eglibc/final
 
   BUILD_CC=gcc                      \
   AR=mipsel-linux-gnu-ar            \
   RANLIB=mipsel-linux-gnu-ranlibi   \
   CC=mipsel-linux-gnu-gcc           \
   CXX=mipsel-linux-gnu-g++          \
   ${TMP}/eglibc-2_14/libc/configure \
     --prefix=/usr                   \
     --enable-add-ons                \
     --host=mipsel-linux-gnu         \
     --disable-profile               \
     --without-gd                    \
     --without-cvs                   \
     --build=i686-pc-linux-gnu       \
     --with-headers=${JAIL_MIPS32}/usr/include
 
   SubBanner "Make"
   make ${MAKE_OPTS} all
 
   SubBanner "Install"
   make ${MAKE_OPTS} install install_root=${JAIL_MIPS32}
 
   popd
 
 
   Banner "Building GCC (final)"
 
+  rm -rf ${BUILD_DIR}/gcc/final
   mkdir -p ${BUILD_DIR}/gcc/final
   pushd ${BUILD_DIR}/gcc/final
 
   ${TMP}/gcc-4.7.2/configure  \
     --prefix=${INSTALL_ROOT}  \
     --disable-libssp          \
     --disable-libgomp         \
     --disable-libmudflap      \
     --disable-fixed-point     \
     --disable-decimal-float   \
     --with-mips-plt           \
     --with-endian=little      \
     --with-arch=${ARCH}       \
     --target=mipsel-linux-gnu \
     --enable-__cxa_atexit     \
     --enable-languages=c,c++  \
     --with-sysroot=${JAIL_MIPS32}
 
   SubBanner "Make"
   make ${MAKE_OPTS} all
 
   SubBanner "Install"
   make ${MAKE_OPTS} install
 
   popd
 
 
   Banner "Building GDB"
 
+  rm -rf ${BUILD_DIR}/gdb/
   mkdir -p ${BUILD_DIR}/gdb/
   pushd ${BUILD_DIR}/gdb/
 
   ${TMP}/gdb-7.5/configure   \
     --prefix=${INSTALL_ROOT} \
     --target=mipsel-linux-gnu
 
   SubBanner "Make"
   make ${MAKE_OPTS} all-gdb
 
@@ skipping 17 matching lines @@
       grep -v "=======" |\
       sed -e 's/0400000/70000000/g' > ${trusted_ld_script}
 }
 
 
 # ----------------------------------------------------------------------
 # mips32 deb files to complete our code sourcery jail
 # ----------------------------------------------------------------------
 
 readonly REPO_DEBIAN=http://ftp.debian.org/debian
-readonly MIPS32_PACKAGES=${REPO_DEBIAN}/dists/squeeze/main/binary-mipsel/Packages.bz2
+readonly MIPS32_PACKAGES=${REPO_DEBIAN}/dists/wheezy/main/binary-mipsel/Packages.bz2
 
-readonly BASE_PACKAGELIST_MIPS32=${SCRIPT_DIR}/packagelist.squeeze.mipsel.base
-readonly EXTRA_PACKAGELIST_MIPS32=${SCRIPT_DIR}/packagelist.squeeze.mipsel.extra
-readonly TMP_BASE_PKG_MIPS32=${TMP}/packagelist.generated.squeeze.mipsel.base
-readonly TMP_EXTRA_PKG_MIPS32=${TMP}/packagelist.generated.squeeze.mipsel.extra
+readonly BASE_PACKAGELIST_MIPS32=${SCRIPT_DIR}/packagelist.wheezy.mipsel.base
+readonly EXTRA_PACKAGELIST_MIPS32=${SCRIPT_DIR}/packagelist.wheezy.mipsel.extra
+readonly TMP_BASE_PKG_MIPS32=${TMP}/packagelist.generated.wheezy.mipsel.base
+readonly TMP_EXTRA_PKG_MIPS32=${TMP}/packagelist.generated.wheezy.mipsel.extra
 
 GeneratePackageLists() {
   local sdk_target=$1
   local packages=
   local TMP_PACKAGELIST=
   Banner "generating ${sdk_target} package lists for mips32"
+  rm -f ${TMP}/Packages.bz2
   DownloadOrCopy ${MIPS32_PACKAGES}
   bzcat ${TMP}/Packages.bz2\
     | egrep '^(Package:|Filename:)' > ${TMP}/Packages_mipsel
 
   if [ ${sdk_target} == "nacl_sdk" ] ; then
     echo -n > ${TMP_BASE_PKG_MIPS32}
     TMP_PACKAGELIST=${TMP_BASE_PKG_MIPS32}
     packages=$(cat ${BASE_PACKAGELIST_MIPS32})
   elif [ ${sdk_target} == "chrome_sdk" ] ; then
     echo -n > ${TMP_EXTRA_PKG_MIPS32}
@@ skipping 24 matching lines @@
   fi
 
   for file in ${DEP_FILES_NEEDED_MIPS32} ; do
     local package="${TMP}/${file##*/}"
     Banner "installing ${file}"
     DownloadOrCopy ${REPO_DEBIAN}/${file}
     SubBanner "extracting to ${JAIL_MIPS32}"
     dpkg --fsys-tarfile ${package}\
       | tar -xvf - --exclude=./usr/share -C ${JAIL_MIPS32}
   done
+}
 
-  Banner "some cleanup"
+FixIncludes() {
+  Banner "Fixing includes"

[Mark Seaborn, 2013/06/18 15:26:21]

Is this doing something that the Debian packages' postinst steps normally do? 
If so, please add a comment explaining that.

[petarj, 2013/06/19 23:12:33]

No, it is not a postinst step. The Wheezy version has different locations
for a few include files, and pkg-config does not seem to work correctly in
some cases, so this is a workaround for it.

[Mark Seaborn, 2013/06/20 01:38:17]

OK, maybe you could add a comment that this is done because pkg-config is not
working correctly?

+  pushd ${JAIL_MIPS32}/usr/include/glib-2.0
+    ln -s ../../lib/glib-2.0/include/glibconfig.h .
+  popd
+
+  pushd ${JAIL_MIPS32}/usr/include/gtk-2.0
+    ln -s ../../lib/gtk-2.0/include/gdkconfig.h .
+  popd
+
+  pushd ${JAIL_MIPS32}/usr/include/dbus-1.0/dbus
+    ln -s ../../../lib/dbus-1.0/include/dbus/dbus-arch-deps.h .
+  popd
+}
+
+FixLinks() {
+  Banner "Fixing links"
+  pushd ${JAIL_MIPS32}/lib/
+    mv mipsel-linux-gnu/* .
+    rm -rf mipsel-linux-gnu
+    ln -s . mipsel-linux-gnu
+  popd
 
   pushd ${JAIL_MIPS32}/usr/lib/
-  cleanlinks > /dev/null 2> /dev/null
-  FixLibs
+    mkdir -p pkgconfig
+    mv mipsel-linux-gnu/pkgconfig/* pkgconfig/
+    rm -rf mipsel-linux-gnu/pkgconfig
+    mv mipsel-linux-gnu/* .
+    rm -rf mipsel-linux-gnu
+    ln -s . mipsel-linux-gnu
+  popd
+
+  pushd ${JAIL_MIPS32}/usr/lib/
+    rm -f libstdc++.so*
+    ln -s ../../../mipsel-linux-gnu/lib/libstdc++.so.6.0.17 .
+    ln -s libstdc++.so.6.0.17 libstdc++.so.6
+    ln -s libstdc++.so.6.0.17 libstdc++.so
+
+    rm -f libgcc_s.so*
+    ln -s ../../../mipsel-linux-gnu/lib/libgcc_s.so.1 .
+    ln -s libgcc_s.so.1 libgcc_s.so
   popd
 }
 
 FixLibs() {
   Banner "Fixing libraries"
 
-  rm -f libbz2.so
-  ln -s ../../lib/libbz2.so.1 libbz2.so
+  readonly liblist="libbz2.so \

[Mark Seaborn, 2013/06/18 15:26:21]

Make this '\' consistent with the following lines

[petarj, 2013/06/19 23:12:33]

Done.

+                    libcom_err.so   \
+                    libdbus-1.so    \
+                    libexpat.so     \
+                    libglib-2.0.so  \
+                    libgpg-error.so \
+                    libkeyutils.so  \
+                    libpamc.so      \
+                    libpam_misc.so  \
+                    libpam.so       \
+                    libpci.so       \
+                    libpcre.so      \
+                    libpng12.so     \
+                    libudev.so      \
+                    libz.so"
 
-  rm -f libm.so
-  ln -s ../../lib/libm.so.6 libm.so
-
-  rm -f libdl.so
-  ln -s ../../lib/libdl.so.2 libdl.so
-
-  rm -f librt.so
-  ln -s ../../lib/librt.so.1 librt.so
-
-  rm -f libpcre.so
-  ln -s ../../lib/libpcre.so.3  libpcre.so
-
-  rm -f libresolv.so
-  ln -s ../../lib/libresolv.so.2  libresolv.so
-
-  rm -f libglib-2.0.so
-  ln -s ../../lib/libglib-2.0.so.0 libglib-2.0.so
-
-  rm -f libudev.so
-  ln -s ../../lib/libudev.so.0 libudev.so
-
-  rm -f libcom_err.so
-  ln -s ../../lib/libcom_err.so.2 libcom_err.so
-
-  rm -f libXdmcp.so
-  ln -s ../../lib/libXdmcp.so.6 libXdmcp.so
-
-  rm -f libstdc++.so*
-  ln -s ../../../mipsel-linux-gnu/lib/libstdc++.so.6.0.17 .
-  ln -s libstdc++.so.6.0.17 libstdc++.so.6
-  ln -s libstdc++.so.6.0.17 libstdc++.so
-
-  rm -f libgcc_s.so*
-  ln -s ../../../mipsel-linux-gnu/lib/libgcc_s.so.1 .
-  ln -s libgcc_s.so.1 libgcc_s.so
+  pushd ${JAIL_MIPS32}/usr/lib/
+    for library in ${liblist}; do
+      rm -f ${library}
+      ln -s ../../lib/${library}.[0123] ${library}
+    done
+  popd
 }
 
 BuildAndInstallQemu() {
   local saved_dir=$(pwd)
   local tmpdir="${TMP}/qemu-mips.nacl"
   local tarball="qemu-0.12.5.tar.gz"
 
   Banner "Building qemu in ${tmpdir}"
 
   rm -rf ${tmpdir}
@@ skipping 17 matching lines @@
   SubBanner "Configuring"
   env -i PATH=/usr/bin/:/bin \
     ./configure \
     --cc=${CC32} \
     --disable-system \
     --enable-linux-user \
     --disable-darwin-user \
     --disable-bsd-user \
     --target-list=mipsel-linux-user \
     --disable-sdl \
-    --disable-linux-aio \
-    --static

[Mark Seaborn, 2013/06/18 15:26:21]

Please add "BUG=https://code.google.com/p/nativeclient/issues/detail?id=3401" to
the commit message and comment that this change fixes a bug.

[petarj, 2013/06/19 23:12:33]

Done.

+    --disable-linux-aio
 
   SubBanner "Make"
   env -i PATH=/usr/bin/:/bin \
       make MAKE_OPTS=${MAKE_OPTS}
 
   SubBanner "Install"
   cp mipsel-linux-user/qemu-mipsel ${INSTALL_ROOT}/qemu-mips32
   cd ${saved_dir}
   cp tools/trusted_cross_toolchains/qemu_tool_mips32.sh ${INSTALL_ROOT}
   ln -sf qemu_tool_mips32.sh ${INSTALL_ROOT}/run_under_qemu_mips32
@@ skipping 10 matching lines @@
   exit -1
 
 elif [[ $1 == "nacl_sdk" || $1 == "chrome_sdk" ]] ; then
   mkdir -p ${TMP}
   SanityCheck
   ClearInstallDir
   ClearBuildDir
   DownloadOrCopyAndInstallToolchain
   GeneratePackageLists $1
   InstallMissingLibraries $1
+  FixLinks
   if [[ $1 == "nacl_sdk" ]] ; then
     InstallTrustedLinkerScript
     BuildAndInstallQemu
     CreateTarBall $1
   else
+    FixLibs
+    FixIncludes
     CreateTarBall ${CROSS_TARBALL}
   fi
 
 else
   Usage
   exit -1
 
 fi