Update mips sysroot creator script

tools/trusted_cross_toolchains/trusted-toolchain-creator.mipsel.debian.sh

Index: tools/trusted_cross_toolchains/trusted-toolchain-creator.mipsel.debian.sh
diff --git a/tools/trusted_cross_toolchains/trusted-toolchain-creator.mipsel.debian.sh b/tools/trusted_cross_toolchains/trusted-toolchain-creator.mipsel.debian.sh
index fd016fda43188237b7156188d55dcc629c0fad20..ca05ba7fb0990719129da8e9434318e9dd33797b 100755
--- a/tools/trusted_cross_toolchains/trusted-toolchain-creator.mipsel.debian.sh
+++ b/tools/trusted_cross_toolchains/trusted-toolchain-creator.mipsel.debian.sh
@@ -18,6 +18,7 @@ set -o nounset
 set -o errexit
 
 readonly SCRIPT_DIR=$(dirname $0)
+readonly NACL_ROOT=$(cd ${SCRIPT_DIR}/../.. && pwd)
 
 readonly MAKE_OPTS="-j8"
 readonly ARCH="mips32"
@@ -48,16 +49,25 @@ readonly GLIBC_SHA1SUM="382f4438a7321dc29ea1a3da8e7852d2c2b3208c"
 
 readonly DOWNLOAD_QEMU_URL="http://wiki.qemu-project.org/download/qemu-2.0.0.tar.bz2"
 
-readonly INSTALL_ROOT=$(pwd)/toolchain/linux_x86/mips_trusted
+readonly OUT_DIR=$(dirname ${NACL_ROOT})/out
 
-readonly TMP=$(pwd)/toolchain/tmp/crosstool-trusted
+readonly INSTALL_ROOT=${NACL_ROOT}/toolchain/linux_x86/mips_trusted
 
-readonly BUILD_DIR=${TMP}/build
+readonly TMP=${OUT_DIR}/sysroot_mips_trusted
 
-readonly JAIL_MIPS32=${INSTALL_ROOT}/sysroot

[petarj, 2015/06/24 15:42:04]

If sysroot does not exist anymore, you need to modify QEMU_JAIL in
qemu_tool_mips32.sh too.
Also, SetUpLinuxEnvMips (LIBPATH) in SConstruct needs to modified.

[Sam Clegg, 2015/06/24 23:25:51]

Ok.. I reverted that part of the change.  sysroot is still separate for now.

+readonly BUILD_DIR=${TMP}/build
 
 readonly CROSS_TARBALL="chromesdk_linux_mipsel"
 
+readonly TAR_ARCHIVE=${OUT_DIR}/out/sysroot_mipsel_trusted_wheezy.tgz
+
+readonly PACKAGES="
+libssl1.0.0
+libssl-dev
+zlib1g
+zlib1g-dev
+"
+
 ######################################################################
 # Helper
 ######################################################################
@@ -74,13 +84,9 @@ SubBanner() {
   echo "......................................................................"
 }
 
+
 Usage() {
-  echo
-  echo "$0 <nacl_sdk|chrome_sdk>"
-  echo
-  echo "nacl_sdk - Build nacl toolchain and package it"
-  echo "chrome_sdk - Build chrome toolchain and package it"
-  echo
+  egrep "^#@" $0 | cut --bytes=3-
 }
 
 DownloadOrCopy() {
@@ -97,6 +103,8 @@ DownloadOrCopy() {
     if [ ! -f "${filename}" ]; then
       SubBanner "downloading from ${url} -> ${filename}"
       wget ${url} -O ${filename}
+    else
+      SubBanner "using existing file: ${filename}"
     fi
   else
     SubBanner "copying from ${url}"
@@ -119,6 +127,8 @@ DownloadOrCopyAndVerify() {
     if [ ! -f "${filename}" ]; then
       SubBanner "downloading from ${url} -> ${filename}"
       wget ${url} -O ${filename}
+    else
+      SubBanner "using existing file: ${filename}"
     fi
     if [ "${checksum}" != "nochecksum" ]; then
       if [ "$(sha1sum ${filename} | cut -d ' ' -f 1)" != "${checksum}" ]; then
@@ -163,8 +173,8 @@ SanityCheck() {
 
 ClearInstallDir() {
   Banner "clearing dirs in ${INSTALL_ROOT}"
-  rm -rf ${INSTALL_ROOT}/*
-  mkdir -p ${JAIL_MIPS32}
+  rm -rf ${INSTALL_ROOT}
+  mkdir -p ${INSTALL_ROOT}
 }
 
 ClearBuildDir() {
@@ -173,9 +183,8 @@ ClearBuildDir() {
 }
 
 CreateTarBall() {
-  local tarball=$1
-  Banner "creating tar ball ${tarball}"
-  tar cfz ${tarball}.tgz -C ${INSTALL_ROOT} .
+  Banner "creating tar ball ${TAR_ARCHIVE}"
+  tar cfz ${TAR_ARCHIVE} -C ${INSTALL_ROOT} .
 }
 
 
@@ -260,7 +269,7 @@ DownloadOrCopyAndInstallToolchain() {
   ${TMP}/binutils-2.24/configure \
     --prefix=${INSTALL_ROOT}     \
     --target=mipsel-linux-gnu    \
-    --with-sysroot=${JAIL_MIPS32}
+    --with-sysroot=${INSTALL_ROOT}
 
   SubBanner "Make"
   make ${MAKE_OPTS} all-binutils all-gas all-ld
@@ -308,7 +317,7 @@ DownloadOrCopyAndInstallToolchain() {
 
   Banner "Installing Linux kernel headers"
   pushd ${TMP}/linux-3.14.2
-  make headers_install ARCH=mips INSTALL_HDR_PATH=${JAIL_MIPS32}/usr
+  make headers_install ARCH=mips INSTALL_HDR_PATH=${INSTALL_ROOT}/usr
   popd
 
   Banner "Building GLIBC"
@@ -330,13 +339,13 @@ DownloadOrCopyAndInstallToolchain() {
     --without-gd                    \
     --without-cvs                   \
     --build=i686-pc-linux-gnu       \
-    --with-headers=${JAIL_MIPS32}/usr/include
+    --with-headers=${INSTALL_ROOT}/usr/include
 
   SubBanner "Make"
   make ${MAKE_OPTS} all
 
   SubBanner "Install"
-  make ${MAKE_OPTS} install install_root=${JAIL_MIPS32}
+  make ${MAKE_OPTS} install install_root=${INSTALL_ROOT}
 
   popd
 
@@ -360,7 +369,7 @@ DownloadOrCopyAndInstallToolchain() {
     --target=mipsel-linux-gnu \
     --enable-__cxa_atexit     \
     --enable-languages=c,c++  \
-    --with-sysroot=${JAIL_MIPS32}
+    --with-sysroot=${INSTALL_ROOT}
 
   SubBanner "Make"
   make ${MAKE_OPTS} all
@@ -397,89 +406,46 @@ DownloadOrCopyAndInstallToolchain() {
 
 readonly REPO_DEBIAN=http://ftp.debian.org/debian
 readonly MIPS32_PACKAGES=${REPO_DEBIAN}/dists/wheezy/main/binary-mipsel/Packages.bz2
-
-readonly BASE_PACKAGELIST_MIPS32=${SCRIPT_DIR}/packagelist.wheezy.mipsel.base
-readonly EXTRA_PACKAGELIST_MIPS32=${SCRIPT_DIR}/packagelist.wheezy.mipsel.extra
-readonly TMP_BASE_PKG_MIPS32=${TMP}/packagelist.generated.wheezy.mipsel.base
-readonly TMP_EXTRA_PKG_MIPS32=${TMP}/packagelist.generated.wheezy.mipsel.extra
+readonly PACKAGE_LIST=${SCRIPT_DIR}/packagelist.wheezy.mipsel.base
 
 GeneratePackageLists() {
-  local sdk_target=$1
-  local packages=
-  local TMP_PACKAGELIST=
-  Banner "generating ${sdk_target} package lists for mips32"
+  Banner "generating package lists for mips32"
   rm -f ${TMP}/Packages.bz2
   DownloadOrCopy ${MIPS32_PACKAGES}
   bzcat ${TMP}/Packages.bz2\
     | egrep '^(Package:|Filename:)' > ${TMP}/Packages_mipsel
 
-  if [ ${sdk_target} == "nacl_sdk" ] ; then
-    echo -n > ${TMP_BASE_PKG_MIPS32}
-    TMP_PACKAGELIST=${TMP_BASE_PKG_MIPS32}
-    packages=$(cat ${BASE_PACKAGELIST_MIPS32})
-  elif [ ${sdk_target} == "chrome_sdk" ] ; then
-    echo -n > ${TMP_EXTRA_PKG_MIPS32}
-    TMP_PACKAGELIST=${TMP_EXTRA_PKG_MIPS32}
-    packages=$(cat ${EXTRA_PACKAGELIST_MIPS32})
-  else
-    Banner "ERROR: Packages for \"${sdk_taget}\" not defined."
-    exit -1
-  fi
+  echo -n > ${PACKAGE_LIST}
 
-  for pkg in ${packages} ; do
+  for pkg in ${PACKAGES} ; do
     grep  -A 1 "${pkg}\$" ${TMP}/Packages_mipsel\
-      | egrep -o "pool/.*" >> ${TMP_PACKAGELIST}
+      | egrep -o "pool/.*" >> ${PACKAGE_LIST}
   done
 }
 
 InstallMissingLibraries() {
-  local sdk_target=$1
   local DEP_FILES_NEEDED_MIPS32=
-
-  if [ ${sdk_target} == "nacl_sdk" ] ; then
-    DEP_FILES_NEEDED_MIPS32=$(cat ${TMP_BASE_PKG_MIPS32})
-  elif [ ${sdk_target} == "chrome_sdk" ] ; then
-    DEP_FILES_NEEDED_MIPS32=$(cat ${TMP_EXTRA_PKG_MIPS32})
-  else
-    Banner "ERROR: Target \"${sdk_taget}\" not defined."
-    exit -1
-  fi
+  DEP_FILES_NEEDED_MIPS32=$(cat ${PACKAGE_LIST})
 
   for file in ${DEP_FILES_NEEDED_MIPS32} ; do
     local package="${TMP}/${file##*/}"
     Banner "installing ${file}"
     DownloadOrCopy ${REPO_DEBIAN}/${file}
-    SubBanner "extracting to ${JAIL_MIPS32}"
+    SubBanner "extracting to ${INSTALL_ROOT}"
     dpkg --fsys-tarfile ${package}\
-      | tar -xvf - --exclude=./usr/share -C ${JAIL_MIPS32}
+      | tar -xvf - --exclude=./usr/share -C ${INSTALL_ROOT}
   done
 }
 
-# Workaround for missing headers since pkg-config is not working correctly.
-FixIncludes() {
-  Banner "Fixing includes"
-  pushd ${JAIL_MIPS32}/usr/include/glib-2.0
-    ln -s ../../lib/glib-2.0/include/glibconfig.h .
-  popd
-
-  pushd ${JAIL_MIPS32}/usr/include/gtk-2.0
-    ln -s ../../lib/gtk-2.0/include/gdkconfig.h .
-  popd
-
-  pushd ${JAIL_MIPS32}/usr/include/dbus-1.0/dbus
-    ln -s ../../../lib/dbus-1.0/include/dbus/dbus-arch-deps.h .
-  popd
-}
-
 FixLinks() {
   Banner "Fixing links"
-  pushd ${JAIL_MIPS32}/lib/
+  pushd ${INSTALL_ROOT}/lib/
     mv mipsel-linux-gnu/* .
     rm -rf mipsel-linux-gnu
     ln -s . mipsel-linux-gnu
   popd
 
-  pushd ${JAIL_MIPS32}/usr/lib/
+  pushd ${INSTALL_ROOT}/usr/lib/
     mkdir -p pkgconfig
     mv mipsel-linux-gnu/pkgconfig/* pkgconfig/
     rm -rf mipsel-linux-gnu/pkgconfig
@@ -488,7 +454,7 @@ FixLinks() {
     ln -s . mipsel-linux-gnu
   popd
 
-  pushd ${JAIL_MIPS32}/usr/lib/
+  pushd ${INSTALL_ROOT}/usr/lib/
     rm -f libstdc++.so*
     ln -s ../../../mipsel-linux-gnu/lib/libstdc++.so.6.0.20 .

[petarj, 2015/06/24 15:42:04]

I believe you need to modify this line too.

[Sam Clegg, 2015/06/24 23:25:51]

I removed all of this.  The new debian stable is recent enough that it contains
gcc 4.9 so there is no longer incompatability between the compiler toolchain and
the sysroot. 

     ln -s libstdc++.so.6.0.20 libstdc++.so.6
@@ -500,33 +466,6 @@ FixLinks() {
   popd
 }
 
-FixLibs() {
-  Banner "Fixing libraries"
-
-  readonly liblist="libbz2.so       \
-                    libcom_err.so   \
-                    libdbus-1.so    \
-                    libexpat.so     \
-                    libglib-2.0.so  \
-                    libgpg-error.so \
-                    libkeyutils.so  \
-                    libpamc.so      \
-                    libpam_misc.so  \
-                    libpam.so       \
-                    libpci.so       \
-                    libpcre.so      \
-                    libpng12.so     \
-                    libudev.so      \
-                    libz.so"
-
-  pushd ${JAIL_MIPS32}/usr/lib/
-    for library in ${liblist}; do
-      rm -f ${library}
-      ln -s ../../lib/${library}.[0123] ${library}
-    done
-  popd
-}
-
 BuildAndInstallQemu() {
   local saved_dir=$(pwd)
   local tmpdir="${TMP}/qemu-mips.nacl"
@@ -556,8 +495,7 @@ BuildAndInstallQemu() {
     --disable-linux-aio
 
   SubBanner "Make"
-  env -i PATH=/usr/bin/:/bin \
-      make MAKE_OPTS=${MAKE_OPTS}
+  env -i PATH=/usr/bin/:/bin make ${MAKE_OPTS}
 
   SubBanner "Install"
   cp mipsel-linux-user/qemu-mipsel ${INSTALL_ROOT}/qemu-mips32
@@ -566,37 +504,58 @@ BuildAndInstallQemu() {
   ln -sf qemu_tool_mips32.sh ${INSTALL_ROOT}/run_under_qemu_mips32
 }
 
+#@
+#@ UploadArchive <revision>
+#@
+#@    Upload archive to Cloud Storage along with json manifest and
+#@    update toolchain_revisions to point to new version.
+#@    This requires write access the Cloud Storage bucket for Native Client.
+UploadArchive() {
+  local REV=$1
+  local TAR_NAME=$(basename ${TAR_ARCHIVE})
+  local GS_FILE=nativeclient-archive2/toolchain/${REV}/${TAR_NAME}
+  local URL=https://storage.googleapis.com/${GS_FILE}
+  set -x
+  gsutil cp -a public-read ${TAR_ARCHIVE} gs://${GS_FILE}
+  local package_version=build/package_version/package_version.py
+  ${package_version} archive --archive-package linux_x86/mips_trusted \
+      ${TAR_ARCHIVE}@${URL}
+  ${package_version} upload --upload-package linux_x86/mips_trusted \
+      --revision ${REV}
+  ${package_version} setrevision --revision-package linux_x86/mips_trusted \
+      --revision ${REV}
+  set +x
+}
+
+help() {
+  Usage
+}
+
 ######################################################################
 # Main
 ######################################################################
 
-if [[ $# -eq 0 ]] ; then
-  echo "you must specify a mode on the commandline:"
-  echo
-  Usage
-  exit -1
+mkdir -p ${TMP}
 
-elif [[ $1 == "nacl_sdk" || $1 == "chrome_sdk" ]] ; then
-  mkdir -p ${TMP}
+BuildSysroot() {
   SanityCheck
   ClearInstallDir
   ClearBuildDir
   DownloadOrCopyAndInstallToolchain
-  GeneratePackageLists $1
-  InstallMissingLibraries $1
+  GeneratePackageLists
+  InstallMissingLibraries
   FixLinks
-  if [[ $1 == "nacl_sdk" ]] ; then
-    BuildAndInstallQemu
-    CreateTarBall $1
-  else
-    FixLibs
-    FixIncludes
-    CreateTarBall ${CROSS_TARBALL}
-  fi
+  BuildAndInstallQemu
+  CreateTarBall
+}
 
+if [[ $# -eq 0 ]] ; then
+  BuildSysroot
+elif [[ "$(type -t $1)" != "function" ]]; then
+  echo "ERROR: unknown function '$1'." >&2
+  echo "For help, try:"
+  echo "    $0 help"
+  exit 1
 else
-  Usage
-  exit -1
-
+  "$@"
 fi
-