[MIPS] Script to build a trusted mipsel-linux-gnu cross toolchain from source.

tools/trusted_cross_toolchains/trusted-toolchain-creator.mipsel.squeeze.sh

 #!/bin/bash
 # Copyright 2012 The Native Client Authors.  All rights reserved.
 # Use of this source code is governed by a BSD-style license that can
 # be found in the LICENSE file.
 #
 #@ This script creates the mips trusted SDK.
 #@ It must be run from the native_client directory.
 
+# This script is intended to build a mipsel-linux-gnu cross compilation
+# toolchain that runs on x86 linux and generates code for a little-endian,
+# hard-float, mips32 target.
+#
+# It expects the host machine to have relatively recent versions of GMP (4.2.0
+# or later), MPFR (2.4.2), and MPC (0.8.1) in order to build the GCC.
+#
+# Common way to get those is:
+# sudo apt-get install libmpfr-dev libmpc-dev libgmp3-dev
+
 ######################################################################
 # Config
 ######################################################################
 
 set -o nounset
 set -o errexit
+set -o xtrace
 
-readonly CS_URL=https://sourcery.mentor.com/sgpp/lite/mips/portal/package9761/public/mips-linux-gnu/mips-2011.09-75-mips-linux-gnu-i686-pc-linux-gnu.tar.bz2
+readonly MAKE_OPTS="-j8"
+readonly ARCH="mips32"
+
+readonly GCC_URL="http://ftp.gnu.org/gnu/gcc/gcc-4.6.3/gcc-4.6.3.tar.bz2"
+readonly GCC_SHA1SUM="ce317ca5c8185b58bc9300182b534608c578637f"
+
+readonly BINUTILS_URL="http://ftp.gnu.org/gnu/binutils/binutils-2.22.tar.bz2"
+readonly BINUTILS_SHA1SUM="65b304a0b9a53a686ce50a01173d1f40f8efe404"
+
+readonly KERNEL_URL="http://www.linux-mips.org/pub/linux/mips/kernel/v2.6/linux-2.6.38.4.tar.gz"
+readonly KERNEL_SHA1SUM="377fa5cf5f1d0c396759b1c4d147330e7e5b6d7f"
+
+readonly GDB_URL="http://ftp.gnu.org/gnu/gdb/gdb-7.5.tar.bz2"
+readonly GDB_SHA1SUM="79b61152813e5730fa670c89e5fc3c04b670b02c"
+
+readonly EGLIBC_SVN_URL="svn://svn.eglibc.org/branches/eglibc-2_14"
+readonly EGLIBC_REVISION="20996"
 
 readonly DOWNLOAD_QEMU_URL="http://download.savannah.gnu.org/releases/qemu/qemu-0.12.5.tar.gz"
 
 readonly INSTALL_ROOT=$(pwd)/toolchain/linux_mips-trusted
 
 readonly TMP=$(pwd)/toolchain/tmp/crosstool-trusted
 
+readonly BUILD_DIR=${TMP}/build
+
 readonly PATCH_MIPS32=$(readlink -f ../third_party/qemu/qemu-0.12.5.patch_mips)
 
-readonly CS_ROOT=${INSTALL_ROOT}/mips-release
+readonly JAIL_MIPS32=${INSTALL_ROOT}/sysroot
 
-readonly JAIL_MIPS32=${CS_ROOT}/mips-linux-gnu/libc/el
-
-readonly  MAKE_OPTS="-j8"
-# These are simple compiler wrappers to force 32bit builds
+# These are simple compiler wrappers to force 32bit builds.
 readonly  CC32=$(readlink -f pnacl/scripts/mygcc32)
 readonly  CXX32=$(readlink -f pnacl/scripts/myg++32)
+
 ######################################################################
 # Helper
 ######################################################################
 
 Banner() {
   echo "######################################################################"
   echo $*
   echo "######################################################################"
 }
 
 SubBanner() {
   echo "......................................................................"
   echo $*
-  echo "...................................................................."
+  echo "......................................................................"
 }
 
 Usage() {
   echo
   echo "$0 trusted_sdk"
   echo
   echo "trusted_sdk - Build everything and package it"
   echo
 }
 
 DownloadOrCopy() {

[robertm, 2012/10/10 15:38:27]

I think it would be clearer to have a separate function for checkouts. this way
you do not have to overload the purpose 
of the second argument.
Maybe change this in a subsequent CL

[petarj, 2012/10/11 00:44:06]

Done.

-  if [[ -f "$2" ]] ; then
-     echo "$2 already in place"
-  elif [[ $1 =~  'http://' || $1 =~  'https://' ]] ; then
-    SubBanner "downloading from $1 -> $2"
-    wget $1 -O $2
+  local url=${1}
+  local filename="${TMP}/${url##*/}"
+  local filetype="${url%%:*}"
+  local checksum=${2}

[robertm, 2012/10/10 15:38:27]

we usually do not use ${} for single char vars but it is ok 

[petarj, 2012/10/11 00:44:06]

Done.

+  local revision=${2}
+
+  if [ "${filename}" == "" ]; then
+    echo "Unknown error occured. Aborting."
+    exit 1
+  fi
+
+  if [ "${filetype}" == "svn" ]; then
+    SubBanner "checkout from ${url} -> ${filename}"
+    svn --force export -r ${revision} ${url} ${filename}
+  elif [[ "${filetype}" ==  "http" || ${filetype} ==  "https" ]] ; then
+    if [ ! -f "${filename}" ]; then
+      SubBanner "downloading from ${url} -> ${filename}"
+      wget ${url} -O ${filename}
+    fi
+    if [ "${checksum}" != "nochecksum" ]; then
+      CheckSHA1Sum ${filename} ${checksum}
+    fi
   else
-    SubBanner "copying from $1"
-    cp $1 $2
+    SubBanner "copying from ${url}"
+    cp ${url} ${filename}
   fi
 }
 
+CheckSHA1Sum() {
+    local calculated_checksum=`sha1sum ${filename} | cut -d ' ' -f 1`

[robertm, 2012/10/10 15:38:27]

use $() instead of backticks

[petarj, 2012/10/11 00:44:06]

Done.

+    if [ "${calculated_checksum}" != "${checksum}" ]; then
+      echo "${filename} sha1sum failed. Deleting file and aborting."
+      rm -f ${filename}
+      exit 1
+    fi
+}
+
 ######################################################################
 #
 ######################################################################
 
 # some sanity checks to make sure this script is run from the right place
-# with the right tools
+# with the right tools.
 SanityCheck() {
   Banner "Sanity Checks"
   if [[ $(basename $(pwd)) != "native_client" ]] ; then
     echo "ERROR: run this script from the native_client/ dir"
     exit -1
   fi
 
   if ! mkdir -p "${INSTALL_ROOT}" ; then
      echo "ERROR: ${INSTALL_ROOT} can't be created."
     exit -1
   fi
 
   for tool in cleanlinks wget ; do
     if ! which ${tool} ; then
       echo "Required binary $tool not found."
       echo "Exiting."
       exit 1
     fi
   done
 }
 
 
 ClearInstallDir() {
   Banner "clearing dirs in ${INSTALL_ROOT}"
   rm -rf ${INSTALL_ROOT}/*
 }
 
+ClearBuildDir() {
+  Banner "clearing dirs in ${BUILD_DIR}"
+  rm -rf ${BUILD_DIR}/*
+}
 
 CreateTarBall() {
   local tarball=$1
   Banner "creating tar ball ${tarball}"
   tar cfz ${tarball}-mips.tgz -C ${INSTALL_ROOT} .
 }
 
 
-# try to keep the tarball small
-PruneDirs() {
-  Banner "pruning code sourcery tree"
-  SubBanner "Size before: $(du -msc  ${CS_ROOT})"
-  rm -rf ${CS_ROOT}/share
-  rm -rf ${CS_ROOT}/mips-linux-gnu/lib/uclibc
-  rm -rf ${CS_ROOT}/mips-linux-gnu/lib/soft-float
-  rm -rf ${CS_ROOT}/mips-linux-gnu/lib/micromips
+# Download the toolchain source tarballs or use a local copy when available.
+DownloadOrCopyAndInstallToolchain() {
+  Banner "Installing toolchain"
 
-  rm -rf ${CS_ROOT}/mips-linux-gnu/libc/uclibc
-  rm -rf ${CS_ROOT}/mips-linux-gnu/libc/soft-float
-  rm -rf ${CS_ROOT}/mips-linux-gnu/libc/micromips
+  local tarball="${TMP}/${BINUTILS_URL##*/}"
+  DownloadOrCopy ${BINUTILS_URL} ${BINUTILS_SHA1SUM}
+  SubBanner "extracting from ${tarball}"
+  tar jxf ${tarball} -C ${TMP}
 
-  rm -rf ${CS_ROOT}/lib/gcc/mips-linux-gnu/4.4.1/uclibc
-  rm -rf ${CS_ROOT}/lib/gcc/mips-linux-gnu/4.4.1/soft-float
-  rm -rf ${CS_ROOT}/lib/gcc/mips-linux-gnu/4.4.1/micromips
+  tarball="${TMP}/${GCC_URL##*/}"
+  DownloadOrCopy ${GCC_URL} ${GCC_SHA1SUM}
+  SubBanner "extracting from ${tarball}"
+  tar jxf ${tarball} -C ${TMP}
 
-  rm -rf ${CS_ROOT}/mips-linux-gnu/include/c++/4.4.1/mips-linux-gnu/uclibc
-  rm -rf ${CS_ROOT}/mips-linux-gnu/include/c++/4.4.1/mips-linux-gnu/soft-float
-  rm -rf ${CS_ROOT}/mips-linux-gnu/include/c++/4.4.1/mips-linux-gnu/micromips
+  tarball="${TMP}/${GDB_URL##*/}"
+  DownloadOrCopy ${GDB_URL} ${GDB_SHA1SUM}
+  SubBanner "extracting from ${tarball}"
+  tar jxf ${tarball} -C ${TMP}
 
-  SubBanner "Size after: $(du -msc  ${CS_ROOT})"
+  tarball="${TMP}/${KERNEL_URL##*/}"
+  DownloadOrCopy ${KERNEL_URL} ${KERNEL_SHA1SUM}
+  SubBanner "extracting from ${tarball}"
+  tar zxf ${tarball} -C ${TMP}
+
+  local eglibc_dir="${TMP}/${EGLIBC_SVN_URL##*/}"
+  DownloadOrCopy ${EGLIBC_SVN_URL} ${EGLIBC_REVISION}
+
+  Banner "Preparing the code"
+
+  cd ${TMP}/eglibc-2_14/libc/ && ln -s ../ports ports && cd -

[robertm, 2012/10/10 15:38:27]

we do not use long chains of &&

for the "ln" case just fix the args to ln

for the sed case, why no define local variables, like
local FILE_NAME="$PWD/tc-mips.c
etc
and the just say
(cd XXX; sed YYY)  # no need for cd -

(cd XXX; a=b c=d 

[petarj, 2012/10/11 00:44:06]

Done.

+  cd ${TMP}/binutils-2.22/gas/config                                           \

[robertm, 2012/10/10 15:38:27]

add a comment for this

[petarj, 2012/10/11 00:44:06]

Done.

+    && OLD_TEXT="as_warn_where (fragp->fr_file, fragp->fr_line, msg);"         \
+    && NEW_TEXT="as_warn_where (fragp->fr_file, fragp->fr_line, \"%s\", msg);" \
+    && FILE_NAME="$PWD/tc-mips.c"                                              \
+    && sed -i "s/$OLD_TEXT/$NEW_TEXT/g" "$FILE_NAME" && cd -
+
+
+  Banner "Building toolchain"
+
+  mkdir -p ${BUILD_DIR}/binutils/
+  cd ${BUILD_DIR}/binutils/
+
+  ${TMP}/binutils-2.22/configure --prefix=${INSTALL_ROOT}         \

[robertm, 2012/10/10 15:38:27]

arg alignment

[petarj, 2012/10/11 00:44:06]

Done.

+                                        --target=mipsel-linux-gnu       \
+                                        --with-sysroot=${JAIL_MIPS32}
+
+  make ${MAKE_OPTS} all-binutils all-gas all-ld
+  make ${MAKE_OPTS} install-binutils install-gas install-ld
+
+
+  mkdir -p ${BUILD_DIR}/gcc/initial
+  cd ${BUILD_DIR}/gcc/initial
+
+  ${TMP}/gcc-4.6.3/configure --prefix=${INSTALL_ROOT} --disable-libssp   \

[robertm, 2012/10/10 15:38:27]

one arg per line is easier to read (optional)

[petarj, 2012/10/11 00:44:06]

Done.

+    --disable-libgomp --disable-libmudflap --disable-fixed-point        \
+    --disable-decimal-float --with-mips-plt --with-endian=little        \
+    --with-arch=${ARCH} --target=mipsel-linux-gnu --enable-languages=c  \
+    --with-newlib --without-headers --disable-shared --disable-threads  \
+    --disable-libquadmath --disable-libatomic
+
+  make ${MAKE_OPTS} all
+  make ${MAKE_OPTS} install
+
+
+  cd ${TMP}/linux-2.6.38.4
+  make headers_install ARCH=mips INSTALL_HDR_PATH=${JAIL_MIPS32}/usr
+
+  export PATH=${INSTALL_ROOT}/bin:$PATH

[robertm, 2012/10/10 15:38:27]

it  may be better to not export them put just 
prepend them before the configure command like so
X=Y A=B configure ....

[petarj, 2012/10/11 00:44:06]

Done.

+  export BUILD_CC=gcc
+  export AR=mipsel-linux-gnu-ar
+  export RANLIB=mipsel-linux-gnu-ranlib
+  export CC=mipsel-linux-gnu-gcc
+  export CXX=mipsel-linux-gnu-g++
+  mkdir -p ${BUILD_DIR}/eglibc/initial
+  cd ${BUILD_DIR}/eglibc/initial
+
+  ${TMP}/eglibc-2_14/libc/configure --prefix=/usr --enable-add-ons      \
+    --with-headers=${JAIL_MIPS32}/usr/include --build=i686-pc-linux-gnu \
+    --host=mipsel-linux-gnu --disable-profile --without-gd --without-cvs
+
+  make ${MAKE_OPTS} install-headers install_root=${JAIL_MIPS32} install-bootstrap-headers=yes

[robertm, 2012/10/10 15:38:27]

line length

[petarj, 2012/10/11 00:44:06]

Done.

+  mkdir -p ${JAIL_MIPS32}/usr/lib
+
+  make csu/subdir_lib  && cp csu/crt1.o csu/crti.o csu/crtn.o ${JAIL_MIPS32}/usr/lib
+  mipsel-linux-gnu-gcc -nostdlib -nostartfiles -shared -x c /dev/null   \
+                       -o ${JAIL_MIPS32}/usr/lib/libc.so
+
+
+  unset BUILD_CC AR RANLIB CC CXX

[robertm, 2012/10/10 15:38:27]

comment?
(may not be needed anymore if you follow the suggestion above)

+  mkdir -p ${BUILD_DIR}/gcc/intermediate
+  cd ${BUILD_DIR}/gcc/intermediate
+
+  ${TMP}/gcc-4.6.3/configure --prefix=${INSTALL_ROOT} --disable-libssp   \
+    --disable-libgomp --disable-libmudflap --disable-fixed-point        \
+    --disable-decimal-float --with-mips-plt --with-endian=little        \
+    --with-arch=${ARCH} --target=mipsel-linux-gnu --enable-languages=c  \
+    --with-sysroot=${JAIL_MIPS32} --disable-libquadmath --disable-libatomic
+
+  make ${MAKE_OPTS} all
+  make ${MAKE_OPTS} install
+
+
+  export BUILD_CC=gcc

[robertm, 2012/10/10 15:38:27]

see above

[petarj, 2012/10/11 00:44:06]

Done.

+  export AR=mipsel-linux-gnu-ar
+  export RANLIB=mipsel-linux-gnu-ranlib
+  export CC=mipsel-linux-gnu-gcc
+  export CXX=mipsel-linux-gnu-g++
+  mkdir -p ${BUILD_DIR}/eglibc/final
+  cd ${BUILD_DIR}/eglibc/final
+
+  ${TMP}/eglibc-2_14/libc/configure --prefix=/usr --enable-add-ons       \
+    --with-headers=${JAIL_MIPS32}/usr/include --build=i686-pc-linux-gnu  \
+    --host=mipsel-linux-gnu --disable-profile --without-gd --without-cvs
+
+  make ${MAKE_OPTS} all
+  make ${MAKE_OPTS} install install_root=${JAIL_MIPS32}
+
+
+  unset BUILD_CC AR RANLIB CC CXX
+  mkdir -p ${BUILD_DIR}/gcc/final
+  cd ${BUILD_DIR}/gcc/final
+
+  ${TMP}/gcc-4.6.3/configure --prefix=${INSTALL_ROOT} --disable-libssp    \
+    --disable-libgomp --disable-libmudflap --disable-fixed-point         \
+    --disable-decimal-float --with-mips-plt --with-endian=little         \
+    --with-arch=${ARCH} --target=mipsel-linux-gnu --enable-__cxa_atexit  \
+    --with-sysroot=${JAIL_MIPS32} --enable-languages=c,c++
+
+  make ${MAKE_OPTS} all
+  make ${MAKE_OPTS} install
+
+
+  mkdir -p ${BUILD_DIR}/gdb/
+  cd ${BUILD_DIR}/gdb/
+
+  ${TMP}/gdb-7.5/configure --prefix=${INSTALL_ROOT} --target=mipsel-linux-gnu
+
+  make ${MAKE_OPTS} all-gdb
+  make ${MAKE_OPTS} install-gdb
+
+  cd ${INSTALL_ROOT}/../..
 }
 
 
-# Download the codesourcery tarball or use a local copy when available.
-DownloadOrCopyAndInstallCodeSourceryTarball() {
-  Banner "Installing Codesourcery Toolchain"
-  local tarball="${TMP}/${CS_URL##*/}"
-  DownloadOrCopy ${CS_URL} ${tarball}
-
-  SubBanner "Untaring  ${INSTALL_ROOT}/${tarball}"
-  tar jxf ${tarball} -C ${INSTALL_ROOT}
-
-  pushd ${INSTALL_ROOT}
-  mv mips-* mips-release
-  popd
-}
-
-
 InstallTrustedLinkerScript() {
   local trusted_ld_script=${INSTALL_ROOT}/ld_script_mips_trusted
   # We are using the output of "ld --verbose" which contains
   # the linker script delimited by "=========".
   # We are changing the image start address to 70000000
   # to move the sel_ldr and other images "out of the way"
   Banner "installing trusted linker script to ${trusted_ld_script}"
 
-  ${CS_ROOT}/bin/mips-linux-gnu-ld  --verbose |\
+  ${INSTALL_ROOT}/bin/mipsel-linux-gnu-ld  --verbose |\
       grep -A 10000 "=======" |\
       grep -v "=======" |\
       sed -e 's/0400000/70000000/g' > ${trusted_ld_script}
 }
 
 
-InstallMissingHeaders() {
-  Banner "installing openssl headers from local system"
-  cp -r /usr/include/openssl ${JAIL_MIPS32}/usr/include/
-}
-
-
-MissingSharedLibCleanup() {
-  Banner "Cleanup dangling symlinks"
-}
-
 # ----------------------------------------------------------------------
 # mips32 deb files to complete our code sourcery jail
 # ----------------------------------------------------------------------
 
 readonly REPO_DEBIAN=http://ftp.debian.org/debian
 readonly MIPS32_PACKAGES=${REPO_DEBIAN}/dists/squeeze/main/binary-mipsel/Packages.bz2
 
 readonly TMP_PACKAGELIST_MIPS32=${TMP}/../packagelist_mipsel.tmp
 
-# These are good enough for native client
+# These are good enough for native client.
 readonly BASE_PACKAGES="\
   libssl0.9.8 \
   libssl-dev \
   libx11-6 \
   libx11-dev \
   x11proto-core-dev \
   libxt6 \
   libxt-dev \
   zlib1g \
   zlib1g-dev \
@@ skipping 12 matching lines @@
   libxcb-shm0 \
   libxcb-shm0-dev \
   libxdmcp6 \
   libxdmcp-dev \
   libxss1 \
   libxss-dev"
 
 GeneratePackageLists() {
   Banner "generating package lists for mips32"
   echo -n > ${TMP_PACKAGELIST_MIPS32}
-  DownloadOrCopy ${MIPS32_PACKAGES} ${TMP}/../Packages_mipsel.bz2
-  bzcat ${TMP}/../Packages_mipsel.bz2\
-    | egrep '^(Package:|Filename:)' > ${TMP}/../Packages_mipsel
+  DownloadOrCopy ${MIPS32_PACKAGES} nochecksum
+  bzcat ${TMP}/Packages.bz2\
+    | egrep '^(Package:|Filename:)' > ${TMP}/Packages_mipsel
   for pkg in ${BASE_PACKAGES} ; do
-    grep  -A 1 "${pkg}\$" ${TMP}/../Packages_mipsel\
+    grep  -A 1 "${pkg}\$" ${TMP}/Packages_mipsel\
       | egrep -o "pool/.*" >> ${TMP_PACKAGELIST_MIPS32}
   done
 }
 
 InstallMissingLibraries() {
   readonly DEP_FILES_NEEDED_MIPS32=$(cat ${TMP_PACKAGELIST_MIPS32})
   for file in ${DEP_FILES_NEEDED_MIPS32} ; do
     local package="${TMP}/${file##*/}"
     Banner "installing ${file}"
-    DownloadOrCopy ${REPO_DEBIAN}/${file} ${package}
+    DownloadOrCopy ${REPO_DEBIAN}/${file} nochecksum
     SubBanner "extracting to ${JAIL_MIPS32}"
     dpkg --fsys-tarfile ${package}\
       | tar -xvf - --exclude=./usr/share -C ${JAIL_MIPS32}
   done
 
   Banner "some cleanup"
 
   pushd ${JAIL_MIPS32}/usr/lib/
   cleanlinks > /dev/null 2> /dev/null
   FixLibs
@@ skipping 13 matching lines @@
   ln -s ../../lib/libdl.so.2 libdl.so
 
   rm -f librt.so
   ln -s ../../lib/librt.so.1 librt.so
 
   rm -f libpcre.so
   ln -s ../../lib/libpcre.so.3  libpcre.so
 
   rm -f libresolv.so
   ln -s ../../lib/libresolv.so.2  libresolv.so
-
-  echo "OUTPUT_FORMAT(elf32-tradlittlemips)" > libc.so
-  echo "GROUP ( libc.so.6 libc_nonshared.a  AS_NEEDED ( ld.so.1 ) )" >> libc.so
-
-  echo "OUTPUT_FORMAT(elf32-tradlittlemips)" > libpthread.so
-  echo "GROUP ( libpthread.so.0 libpthread_nonshared.a )" >> libpthread.so
 }
 
 BuildAndInstallQemu() {
   local saved_dir=$(pwd)
   local tmpdir="${TMP}/qemu-mips.nacl"
   local tarball="qemu-0.12.5.tar.gz"
   Banner "Building qemu in ${tmpdir}"
   rm -rf ${tmpdir}
   mkdir ${tmpdir}
   cd ${tmpdir}
@@ skipping 26 matching lines @@
   SubBanner "Make"
   env -i PATH=/usr/bin/:/bin \
       make MAKE_OPTS=${MAKE_OPTS}
 
   SubBanner "Install"
   cp mipsel-linux-user/qemu-mipsel ${INSTALL_ROOT}/qemu-mips32
   cd ${saved_dir}
   cp tools/trusted_cross_toolchains/qemu_tool_mips32.sh ${INSTALL_ROOT}
   ln -sf qemu_tool_mips32.sh ${INSTALL_ROOT}/run_under_qemu_mips32
 }
+
 ######################################################################
 # Main
 ######################################################################
 
 if [[ $# -eq 0 ]] ; then
   echo "you must specify a mode on the commandline:"
   echo
   Usage
   exit -1
 
 elif [[ $1 == "trusted_sdk" ]]; then
   mkdir -p ${TMP}
   SanityCheck
   ClearInstallDir
-  DownloadOrCopyAndInstallCodeSourceryTarball
-  PruneDirs
+  ClearBuildDir
+  DownloadOrCopyAndInstallToolchain
   GeneratePackageLists
-  InstallMissingHeaders
   InstallMissingLibraries
-  MissingSharedLibCleanup
   InstallTrustedLinkerScript
   BuildAndInstallQemu
   CreateTarBall $1
 
 else
   Usage
   exit -1
 
 fi