Update mips sysroot creator script

tools/trusted_cross_toolchains/trusted-toolchain-creator.mipsel.debian.sh

 #!/bin/bash
 # Copyright (c) 2012 The Native Client Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 #
 #@ This script creates the mips trusted SDK.
 #@ It must be run from the native_client directory.
 
 # This script is intended to build a mipsel-linux-gnu cross compilation
 # toolchain that runs on x86 linux and generates code for a little-endian,
 # hard-float, mips32 target.
 
 ######################################################################
 # Config
 ######################################################################
 
 set -o nounset
 set -o errexit
 
 readonly SCRIPT_DIR=$(dirname $0)
+readonly NACL_ROOT=$(cd ${SCRIPT_DIR}/../.. && pwd)
 
 readonly MAKE_OPTS="-j8"
 readonly ARCH="mips32"
 
 readonly GMP_URL="http://ftp.gnu.org/gnu/gmp/gmp-5.1.3.tar.bz2"
 readonly GMP_SHA1SUM="b35928e2927b272711fdfbf71b7cfd5f86a6b165"
 
 readonly MPFR_URL="http://ftp.gnu.org/gnu/mpfr/mpfr-3.1.2.tar.bz2"
 readonly MPFR_SHA1SUM="46d5a11a59a4e31f74f73dd70c5d57a59de2d0b4"
 
@@ skipping 10 matching lines @@
 readonly KERNEL_SHA1SUM="9b094e817a7a9b7c09b5bc268e23de642c6c407a"
 
 readonly GDB_URL="http://ftp.gnu.org/gnu/gdb/gdb-7.7.1.tar.bz2"
 readonly GDB_SHA1SUM="35228319f7c715074a80be42fff64c7645227a80"
 
 readonly GLIBC_URL="http://ftp.gnu.org/gnu/glibc/glibc-2.19.tar.bz2"
 readonly GLIBC_SHA1SUM="382f4438a7321dc29ea1a3da8e7852d2c2b3208c"
 
 readonly DOWNLOAD_QEMU_URL="http://wiki.qemu-project.org/download/qemu-2.0.0.tar.bz2"
 
-readonly INSTALL_ROOT=$(pwd)/toolchain/linux_x86/mips_trusted
+readonly OUT_DIR=$(dirname ${NACL_ROOT})/out
 
-readonly TMP=$(pwd)/toolchain/tmp/crosstool-trusted
+readonly INSTALL_ROOT=${NACL_ROOT}/toolchain/linux_x86/mips_trusted
+
+readonly TMP=${OUT_DIR}/sysroot_mips_trusted
 
 readonly BUILD_DIR=${TMP}/build
 
-readonly JAIL_MIPS32=${INSTALL_ROOT}/sysroot

[petarj, 2015/06/24 15:42:04]

If sysroot does not exist anymore, you need to modify QEMU_JAIL in
qemu_tool_mips32.sh too.
Also, SetUpLinuxEnvMips (LIBPATH) in SConstruct needs to modified.

[Sam Clegg, 2015/06/24 23:25:51]

Ok.. I reverted that part of the change.  sysroot is still separate for now.

+readonly CROSS_TARBALL="chromesdk_linux_mipsel"
 
-readonly CROSS_TARBALL="chromesdk_linux_mipsel"
+readonly TAR_ARCHIVE=${OUT_DIR}/out/sysroot_mipsel_trusted_wheezy.tgz
+
+readonly PACKAGES="
+libssl1.0.0
+libssl-dev
+zlib1g
+zlib1g-dev
+"
 
 ######################################################################
 # Helper
 ######################################################################
 
 Banner() {
   echo "######################################################################"
   echo $*
   echo "######################################################################"
 }
 
 SubBanner() {
   echo "......................................................................"
   echo $*
   echo "......................................................................"
 }
 
+
 Usage() {
-  echo
-  echo "$0 <nacl_sdk|chrome_sdk>"
-  echo
-  echo "nacl_sdk - Build nacl toolchain and package it"
-  echo "chrome_sdk - Build chrome toolchain and package it"
-  echo
+  egrep "^#@" $0 | cut --bytes=3-
 }
 
 DownloadOrCopy() {
   local url=$1
   local filename="${TMP}/${url##*/}"
   local filetype="${url%%:*}"
 
   if [ "${filename}" == "" ]; then
     echo "Unknown error occured. Aborting."
     exit 1
   fi
 
   if [[ "${filetype}" ==  "http" || ${filetype} ==  "https" ]] ; then
     if [ ! -f "${filename}" ]; then
       SubBanner "downloading from ${url} -> ${filename}"
       wget ${url} -O ${filename}
+    else
+      SubBanner "using existing file: ${filename}"
     fi
   else
     SubBanner "copying from ${url}"
     cp ${url} ${filename}
   fi
 }
 
 DownloadOrCopyAndVerify() {
   local url=$1
   local checksum=$2
   local filename="${TMP}/${url##*/}"
   local filetype="${url%%:*}"
 
   if [ "${filename}" == "" ]; then
     echo "Unknown error occured. Aborting."
     exit 1
   fi
 
   if [[ "${filetype}" ==  "http" || ${filetype} ==  "https" ]] ; then
     if [ ! -f "${filename}" ]; then
       SubBanner "downloading from ${url} -> ${filename}"
       wget ${url} -O ${filename}
+    else
+      SubBanner "using existing file: ${filename}"
     fi
     if [ "${checksum}" != "nochecksum" ]; then
       if [ "$(sha1sum ${filename} | cut -d ' ' -f 1)" != "${checksum}" ]; then
         echo "${filename} sha1sum failed. Deleting file and aborting."
         rm -f ${filename}
         exit 1
       fi
     fi
   else
     SubBanner "copying from ${url}"
@@ skipping 24 matching lines @@
       echo "Required binary $tool not found."
       echo "Exiting."
       exit 1
     fi
   done
 }
 
 
 ClearInstallDir() {
   Banner "clearing dirs in ${INSTALL_ROOT}"
-  rm -rf ${INSTALL_ROOT}/*
-  mkdir -p ${JAIL_MIPS32}
+  rm -rf ${INSTALL_ROOT}
+  mkdir -p ${INSTALL_ROOT}
 }
 
 ClearBuildDir() {
   Banner "clearing dirs in ${BUILD_DIR}"
   rm -rf ${BUILD_DIR}/*
 }
 
 CreateTarBall() {
-  local tarball=$1
-  Banner "creating tar ball ${tarball}"
-  tar cfz ${tarball}.tgz -C ${INSTALL_ROOT} .
+  Banner "creating tar ball ${TAR_ARCHIVE}"
+  tar cfz ${TAR_ARCHIVE} -C ${INSTALL_ROOT} .
 }
 
 
 # Download the toolchain source tarballs or use a local copy when available.
 DownloadOrCopyAndInstallToolchain() {
   Banner "Installing toolchain"
 
   tarball="${TMP}/${GCC_URL##*/}"
   DownloadOrCopyAndVerify ${GCC_URL} ${GCC_SHA1SUM}
   SubBanner "extracting from ${tarball}"
@@ skipping 64 matching lines @@
   Banner "Building binutils"
 
   rm -rf ${BUILD_DIR}/binutils/
   mkdir -p ${BUILD_DIR}/binutils/
   pushd ${BUILD_DIR}/binutils/
 
   SubBanner "Configuring"
   ${TMP}/binutils-2.24/configure \
     --prefix=${INSTALL_ROOT}     \
     --target=mipsel-linux-gnu    \
-    --with-sysroot=${JAIL_MIPS32}
+    --with-sysroot=${INSTALL_ROOT}
 
   SubBanner "Make"
   make ${MAKE_OPTS} all-binutils all-gas all-ld
 
   SubBanner "Install"
   make ${MAKE_OPTS} install-binutils install-gas install-ld
 
   popd
 
 
@@ skipping 27 matching lines @@
   make ${MAKE_OPTS} all
 
   SubBanner "Install"
   make ${MAKE_OPTS} install
 
   popd
 
 
   Banner "Installing Linux kernel headers"
   pushd ${TMP}/linux-3.14.2
-  make headers_install ARCH=mips INSTALL_HDR_PATH=${JAIL_MIPS32}/usr
+  make headers_install ARCH=mips INSTALL_HDR_PATH=${INSTALL_ROOT}/usr
   popd
 
   Banner "Building GLIBC"
 
   rm -rf ${BUILD_DIR}/glibc/final
   mkdir -p ${BUILD_DIR}/glibc/final
   pushd ${BUILD_DIR}/glibc/final
 
   BUILD_CC=gcc                      \
   AR=mipsel-linux-gnu-ar            \
   RANLIB=mipsel-linux-gnu-ranlibi   \
   CC=mipsel-linux-gnu-gcc           \
   CXX=mipsel-linux-gnu-g++          \
   ${TMP}/glibc-2.19/configure       \
     --prefix=/usr                   \
     --enable-add-ons                \
     --host=mipsel-linux-gnu         \
     --disable-profile               \
     --without-gd                    \
     --without-cvs                   \
     --build=i686-pc-linux-gnu       \
-    --with-headers=${JAIL_MIPS32}/usr/include
+    --with-headers=${INSTALL_ROOT}/usr/include
 
   SubBanner "Make"
   make ${MAKE_OPTS} all
 
   SubBanner "Install"
-  make ${MAKE_OPTS} install install_root=${JAIL_MIPS32}
+  make ${MAKE_OPTS} install install_root=${INSTALL_ROOT}
 
   popd
 
 
   Banner "Building GCC (final)"
 
   rm -rf ${BUILD_DIR}/gcc/final
   mkdir -p ${BUILD_DIR}/gcc/final
   pushd ${BUILD_DIR}/gcc/final
 
   ${TMP}/gcc-4.9.0/configure  \
     --prefix=${INSTALL_ROOT}  \
     --disable-libssp          \
     --disable-libgomp         \
     --disable-libmudflap      \
     --disable-fixed-point     \
     --disable-decimal-float   \
     --with-mips-plt           \
     --with-endian=little      \
     --with-arch=${ARCH}       \
     --target=mipsel-linux-gnu \
     --enable-__cxa_atexit     \
     --enable-languages=c,c++  \
-    --with-sysroot=${JAIL_MIPS32}
+    --with-sysroot=${INSTALL_ROOT}
 
   SubBanner "Make"
   make ${MAKE_OPTS} all
 
   SubBanner "Install"
   make ${MAKE_OPTS} install
 
   popd
 
 
@@ skipping 16 matching lines @@
   popd
 }
 
 
 # ----------------------------------------------------------------------
 # mips32 deb files to complete our code sourcery jail
 # ----------------------------------------------------------------------
 
 readonly REPO_DEBIAN=http://ftp.debian.org/debian
 readonly MIPS32_PACKAGES=${REPO_DEBIAN}/dists/wheezy/main/binary-mipsel/Packages.bz2
-
-readonly BASE_PACKAGELIST_MIPS32=${SCRIPT_DIR}/packagelist.wheezy.mipsel.base
-readonly EXTRA_PACKAGELIST_MIPS32=${SCRIPT_DIR}/packagelist.wheezy.mipsel.extra
-readonly TMP_BASE_PKG_MIPS32=${TMP}/packagelist.generated.wheezy.mipsel.base
-readonly TMP_EXTRA_PKG_MIPS32=${TMP}/packagelist.generated.wheezy.mipsel.extra
+readonly PACKAGE_LIST=${SCRIPT_DIR}/packagelist.wheezy.mipsel.base
 
 GeneratePackageLists() {
-  local sdk_target=$1
-  local packages=
-  local TMP_PACKAGELIST=
-  Banner "generating ${sdk_target} package lists for mips32"
+  Banner "generating package lists for mips32"
   rm -f ${TMP}/Packages.bz2
   DownloadOrCopy ${MIPS32_PACKAGES}
   bzcat ${TMP}/Packages.bz2\
     | egrep '^(Package:|Filename:)' > ${TMP}/Packages_mipsel
 
-  if [ ${sdk_target} == "nacl_sdk" ] ; then
-    echo -n > ${TMP_BASE_PKG_MIPS32}
-    TMP_PACKAGELIST=${TMP_BASE_PKG_MIPS32}
-    packages=$(cat ${BASE_PACKAGELIST_MIPS32})
-  elif [ ${sdk_target} == "chrome_sdk" ] ; then
-    echo -n > ${TMP_EXTRA_PKG_MIPS32}
-    TMP_PACKAGELIST=${TMP_EXTRA_PKG_MIPS32}
-    packages=$(cat ${EXTRA_PACKAGELIST_MIPS32})
-  else
-    Banner "ERROR: Packages for \"${sdk_taget}\" not defined."
-    exit -1
-  fi
+  echo -n > ${PACKAGE_LIST}
 
-  for pkg in ${packages} ; do
+  for pkg in ${PACKAGES} ; do
     grep  -A 1 "${pkg}\$" ${TMP}/Packages_mipsel\
-      | egrep -o "pool/.*" >> ${TMP_PACKAGELIST}
+      | egrep -o "pool/.*" >> ${PACKAGE_LIST}
   done
 }
 
 InstallMissingLibraries() {
-  local sdk_target=$1
   local DEP_FILES_NEEDED_MIPS32=
-
-  if [ ${sdk_target} == "nacl_sdk" ] ; then
-    DEP_FILES_NEEDED_MIPS32=$(cat ${TMP_BASE_PKG_MIPS32})
-  elif [ ${sdk_target} == "chrome_sdk" ] ; then
-    DEP_FILES_NEEDED_MIPS32=$(cat ${TMP_EXTRA_PKG_MIPS32})
-  else
-    Banner "ERROR: Target \"${sdk_taget}\" not defined."
-    exit -1
-  fi
+  DEP_FILES_NEEDED_MIPS32=$(cat ${PACKAGE_LIST})
 
   for file in ${DEP_FILES_NEEDED_MIPS32} ; do
     local package="${TMP}/${file##*/}"
     Banner "installing ${file}"
     DownloadOrCopy ${REPO_DEBIAN}/${file}
-    SubBanner "extracting to ${JAIL_MIPS32}"
+    SubBanner "extracting to ${INSTALL_ROOT}"
     dpkg --fsys-tarfile ${package}\
-      | tar -xvf - --exclude=./usr/share -C ${JAIL_MIPS32}
+      | tar -xvf - --exclude=./usr/share -C ${INSTALL_ROOT}
   done
 }
 
-# Workaround for missing headers since pkg-config is not working correctly.
-FixIncludes() {
-  Banner "Fixing includes"
-  pushd ${JAIL_MIPS32}/usr/include/glib-2.0
-    ln -s ../../lib/glib-2.0/include/glibconfig.h .
-  popd
-
-  pushd ${JAIL_MIPS32}/usr/include/gtk-2.0
-    ln -s ../../lib/gtk-2.0/include/gdkconfig.h .
-  popd
-
-  pushd ${JAIL_MIPS32}/usr/include/dbus-1.0/dbus
-    ln -s ../../../lib/dbus-1.0/include/dbus/dbus-arch-deps.h .
-  popd
-}
-
 FixLinks() {
   Banner "Fixing links"
-  pushd ${JAIL_MIPS32}/lib/
+  pushd ${INSTALL_ROOT}/lib/
     mv mipsel-linux-gnu/* .
     rm -rf mipsel-linux-gnu
     ln -s . mipsel-linux-gnu
   popd
 
-  pushd ${JAIL_MIPS32}/usr/lib/
+  pushd ${INSTALL_ROOT}/usr/lib/
     mkdir -p pkgconfig
     mv mipsel-linux-gnu/pkgconfig/* pkgconfig/
     rm -rf mipsel-linux-gnu/pkgconfig
     mv mipsel-linux-gnu/* .
     rm -rf mipsel-linux-gnu
     ln -s . mipsel-linux-gnu
   popd
 
-  pushd ${JAIL_MIPS32}/usr/lib/
+  pushd ${INSTALL_ROOT}/usr/lib/
     rm -f libstdc++.so*
     ln -s ../../../mipsel-linux-gnu/lib/libstdc++.so.6.0.20 .

[petarj, 2015/06/24 15:42:04]

I believe you need to modify this line too.

[Sam Clegg, 2015/06/24 23:25:51]

I removed all of this.  The new debian stable is recent enough that it contains
gcc 4.9 so there is no longer incompatability between the compiler toolchain and
the sysroot. 

     ln -s libstdc++.so.6.0.20 libstdc++.so.6
     ln -s libstdc++.so.6.0.20 libstdc++.so
 
     rm -f libgcc_s.so*
     ln -s ../../../mipsel-linux-gnu/lib/libgcc_s.so.1 .

[petarj, 2015/06/24 15:42:04]

Ditto.

[Sam Clegg, 2015/06/24 23:25:51]

Acknowledged.

     ln -s libgcc_s.so.1 libgcc_s.so
   popd
 }
 
-FixLibs() {
-  Banner "Fixing libraries"
-
-  readonly liblist="libbz2.so       \
-                    libcom_err.so   \
-                    libdbus-1.so    \
-                    libexpat.so     \
-                    libglib-2.0.so  \
-                    libgpg-error.so \
-                    libkeyutils.so  \
-                    libpamc.so      \
-                    libpam_misc.so  \
-                    libpam.so       \
-                    libpci.so       \
-                    libpcre.so      \
-                    libpng12.so     \
-                    libudev.so      \
-                    libz.so"
-
-  pushd ${JAIL_MIPS32}/usr/lib/
-    for library in ${liblist}; do
-      rm -f ${library}
-      ln -s ../../lib/${library}.[0123] ${library}
-    done
-  popd
-}
-
 BuildAndInstallQemu() {
   local saved_dir=$(pwd)
   local tmpdir="${TMP}/qemu-mips.nacl"
   local tarball="qemu-2.0.0.tar.bz2"
 
   Banner "Building qemu in ${tmpdir}"
 
   rm -rf ${tmpdir}
   mkdir ${tmpdir}
   cd ${tmpdir}
 
   SubBanner "Downloading"
   wget -c ${DOWNLOAD_QEMU_URL}
 
   SubBanner "Untarring"
   tar xf ${tarball}
   cd qemu-2.0.0
 
   SubBanner "Configuring"
   env -i PATH=/usr/bin/:/bin \
     ./configure \
     --disable-system \
     --enable-linux-user \
     --disable-bsd-user \
     --target-list=mipsel-linux-user \
     --disable-sdl \
     --disable-linux-aio
 
   SubBanner "Make"
-  env -i PATH=/usr/bin/:/bin \
-      make MAKE_OPTS=${MAKE_OPTS}
+  env -i PATH=/usr/bin/:/bin make ${MAKE_OPTS}
 
   SubBanner "Install"
   cp mipsel-linux-user/qemu-mipsel ${INSTALL_ROOT}/qemu-mips32
   cd ${saved_dir}
   cp tools/trusted_cross_toolchains/qemu_tool_mips32.sh ${INSTALL_ROOT}
   ln -sf qemu_tool_mips32.sh ${INSTALL_ROOT}/run_under_qemu_mips32
 }
 
+#@
+#@ UploadArchive <revision>
+#@
+#@    Upload archive to Cloud Storage along with json manifest and
+#@    update toolchain_revisions to point to new version.
+#@    This requires write access the Cloud Storage bucket for Native Client.
+UploadArchive() {
+  local REV=$1
+  local TAR_NAME=$(basename ${TAR_ARCHIVE})
+  local GS_FILE=nativeclient-archive2/toolchain/${REV}/${TAR_NAME}
+  local URL=https://storage.googleapis.com/${GS_FILE}
+  set -x
+  gsutil cp -a public-read ${TAR_ARCHIVE} gs://${GS_FILE}
+  local package_version=build/package_version/package_version.py
+  ${package_version} archive --archive-package linux_x86/mips_trusted \
+      ${TAR_ARCHIVE}@${URL}
+  ${package_version} upload --upload-package linux_x86/mips_trusted \
+      --revision ${REV}
+  ${package_version} setrevision --revision-package linux_x86/mips_trusted \
+      --revision ${REV}
+  set +x
+}
+
+help() {
+  Usage
+}
+
 ######################################################################
 # Main
 ######################################################################
 
-if [[ $# -eq 0 ]] ; then
-  echo "you must specify a mode on the commandline:"
-  echo
-  Usage
-  exit -1
+mkdir -p ${TMP}
 
-elif [[ $1 == "nacl_sdk" || $1 == "chrome_sdk" ]] ; then
-  mkdir -p ${TMP}
+BuildSysroot() {
   SanityCheck
   ClearInstallDir
   ClearBuildDir
   DownloadOrCopyAndInstallToolchain
-  GeneratePackageLists $1
-  InstallMissingLibraries $1
+  GeneratePackageLists
+  InstallMissingLibraries
   FixLinks
-  if [[ $1 == "nacl_sdk" ]] ; then
-    BuildAndInstallQemu
-    CreateTarBall $1
-  else
-    FixLibs
-    FixIncludes
-    CreateTarBall ${CROSS_TARBALL}
-  fi
+  BuildAndInstallQemu
+  CreateTarBall
+}
 
+if [[ $# -eq 0 ]] ; then
+  BuildSysroot
+elif [[ "$(type -t $1)" != "function" ]]; then
+  echo "ERROR: unknown function '$1'." >&2
+  echo "For help, try:"
+  echo "    $0 help"
+  exit 1
 else
-  Usage
-  exit -1
-
+  "$@"
 fi
-