Add jessie-security repo to sysroot scripts

build/linux/sysroot_scripts/sysroot-creator.sh

 # Copyright 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 #
 # This script should not be run directly but sourced by the other
 # scripts (e.g. sysroot-creator-jessie.sh).  Its up to the parent scripts
 # to define certain environment variables: e.g.
 #  DISTRO=ubuntu
 #  DIST=jessie
-#  DIST_UPDATES=jessie-updates  # optional
-#  REPO_EXTRA="universe restricted multiverse"  # optional
-#  APT_REPO=http://archive.ubuntu.com/ubuntu
-#  KEYRING_FILE=/usr/share/keyrings/ubuntu-archive-keyring.gpg
+#  # Similar in syntax to /etc/apt/sources.list
+#  APT_SOURCES_LIST="http://ftp.us.debian.org/debian/ jessie main"
+#  KEYRING_FILE=debian-archive-jessie-stable.gpg
 #  DEBIAN_PACKAGES="gcc libz libssl"
 
 #@ This script builds Debian/Ubuntu sysroot images for building Google Chrome.
 #@
 #@  Generally this script is invoked as:
 #@  sysroot-creator-<flavour>.sh <mode> <args>*
 #@  Available modes are shown below.
 #@
 #@ List of modes:
 
 ######################################################################
 # Config
 ######################################################################
 
 set -o nounset
 set -o errexit
 
 SCRIPT_DIR=$(cd $(dirname $0) && pwd)
 
 if [ -z "${DIST:-}" ]; then
   echo "error: DIST not defined"
   exit 1
 fi
 
-if [ -z "${APT_REPO:-}" ]; then
-  echo "error: APT_REPO not defined"
-  exit 1
-fi
-
 if [ -z "${KEYRING_FILE:-}" ]; then
   echo "error: KEYRING_FILE not defined"
   exit 1
 fi
 
 if [ -z "${DEBIAN_PACKAGES:-}" ]; then
   echo "error: DEBIAN_PACKAGES not defined"
   exit 1
 fi
 
 readonly HAS_ARCH_AMD64=${HAS_ARCH_AMD64:=0}
 readonly HAS_ARCH_I386=${HAS_ARCH_I386:=0}
 readonly HAS_ARCH_ARM=${HAS_ARCH_ARM:=0}
 readonly HAS_ARCH_ARM64=${HAS_ARCH_ARM64:=0}
 readonly HAS_ARCH_MIPS=${HAS_ARCH_MIPS:=0}
 
 readonly REQUIRED_TOOLS="wget"
 
 ######################################################################
 # Package Config
 ######################################################################
 
-PACKAGES_EXT=${PACKAGES_EXT:-bz2}
+readonly PACKAGES_EXT=gz
 readonly RELEASE_FILE="Release"
 readonly RELEASE_FILE_GPG="Release.gpg"
 
 readonly DEBIAN_DEP_LIST_AMD64="packagelist.${DIST}.amd64"
 readonly DEBIAN_DEP_LIST_I386="packagelist.${DIST}.i386"
 readonly DEBIAN_DEP_LIST_ARM="packagelist.${DIST}.arm"
 readonly DEBIAN_DEP_LIST_ARM64="packagelist.${DIST}.arm64"
 readonly DEBIAN_DEP_LIST_MIPS="packagelist.${DIST}.mipsel"
 
 ######################################################################
@@ skipping 24 matching lines @@
     echo "$2 already in place"
     return
   fi
 
   HTTP=0
   echo "$1" | grep -qs ^http:// && HTTP=1
   if [ "$HTTP" = "1" ]; then
     SubBanner "downloading from $1 -> $2"
     # Appending the "$$" shell pid is necessary here to prevent concurrent
     # instances of sysroot-creator.sh from trying to write to the same file.
-    wget "$1" -O "${2}.partial.$$"
+    # --create-dirs is added in case there are slashes in the filename, as can
+    # happen with the "debian/security" release class.
+    curl "$1" --create-dirs -o "${2}.partial.$$"

[Lei Zhang, 2017/05/27 00:38:45]

Oh, REQUIRED_TOOLS needs updating.

[Tom Anderson, 2017/05/27 00:52:44]

Done.

     mv "${2}.partial.$$" $2
   else
     SubBanner "copying from $1"
     cp "$1" "$2"
   fi
 }
 
 
 SetEnvironmentVariables() {
   ARCH=""
@@ skipping 57 matching lines @@
   Banner "Clearing dirs in ${INSTALL_ROOT}"
   rm -rf ${INSTALL_ROOT}/*
 }
 
 
 CreateTarBall() {
   Banner "Creating tarball ${TARBALL}"
   tar zcf ${TARBALL} -C ${INSTALL_ROOT} .
 }
 
-ExtractPackageBz2() {
-  if [ "${PACKAGES_EXT}" = "bz2" ]; then
-    bzcat "$1" | egrep '^(Package:|Filename:|SHA256:) ' > "$2"
-  else
-    xzcat "$1" | egrep '^(Package:|Filename:|SHA256:) ' > "$2"
-  fi
+ExtractPackageGz() {

[Lei Zhang, 2017/05/27 00:37:42]

Add some locals so we know what the arguments are? Previously it was more
obvious what $1 and $2 is, but now $3 is not as obvious.

[Tom Anderson, 2017/05/27 00:52:44]

Done.

+  gunzip -c "$1" | egrep '^(Package:|Filename:|SHA256:) ' |
+    sed "s|Filename: |Filename: $3|" > "$2"
 }
 
 GeneratePackageListDist() {
   local arch="$1"
-  local apt_url="$2"
-  local dist="$3"
-  local repo_name="$4"
+  set -- $2
+  local repo="$1"
+  local dist="$2"
+  local repo_name="$3"
 
   TMP_PACKAGE_LIST="${BUILD_DIR}/Packages.${dist}_${repo_name}_${arch}"
-  local repo_basedir="${apt_url}/dists/${dist}"
+  local repo_basedir="${repo}/dists/${dist}"
   local package_list="${BUILD_DIR}/Packages.${dist}_${repo_name}_${arch}.${PACKAGES_EXT}"
   local package_file_arch="${repo_name}/binary-${arch}/Packages.${PACKAGES_EXT}"
   local package_list_arch="${repo_basedir}/${package_file_arch}"
 
   DownloadOrCopy "${package_list_arch}" "${package_list}"
-  VerifyPackageListing "${package_file_arch}" "${package_list}" ${dist}
-  ExtractPackageBz2 "${package_list}" "${TMP_PACKAGE_LIST}"
+  VerifyPackageListing "${package_file_arch}" "${package_list}" ${repo} ${dist}
+  ExtractPackageGz "${package_list}" "${TMP_PACKAGE_LIST}" ${repo}
 }
 
 GeneratePackageListCommon() {
   local output_file="$1"
   local arch="$2"
-  local apt_url="$3"
-  local packages="$4"
+  local packages="$3"
 
   local dists="${DIST} ${DIST_UPDATES:-}"
   local repos="main ${REPO_EXTRA:-}"
 
   local list_base="${BUILD_DIR}/Packages.${DIST}_${arch}"
-  > "${list_base}"
-  for dist in ${dists}; do
-    for repo in ${repos}; do
-      GeneratePackageListDist "${arch}" "${apt_url}" "${dist}" "${repo}"
-      cat "${TMP_PACKAGE_LIST}" | ./merge-package-lists.py "${list_base}"
-    done
+  > "${list_base}"  # Create (or truncate) a zero-length file.
+  echo "${APT_SOURCES_LIST}" | while read source; do
+    GeneratePackageListDist "${arch}" "${source}"
+    cat "${TMP_PACKAGE_LIST}" | ./merge-package-lists.py "${list_base}"
   done
 
   GeneratePackageList "${list_base}" "${output_file}" "${packages}"
 }
 
 GeneratePackageListAmd64() {
-  GeneratePackageListCommon "$1" amd64 ${APT_REPO} "${DEBIAN_PACKAGES}
+  GeneratePackageListCommon "$1" amd64 "${DEBIAN_PACKAGES}
     ${DEBIAN_PACKAGES_X86:=} ${DEBIAN_PACKAGES_AMD64:=}"
 }
 
 GeneratePackageListI386() {
-  GeneratePackageListCommon "$1" i386 ${APT_REPO} "${DEBIAN_PACKAGES}
+  GeneratePackageListCommon "$1" i386 "${DEBIAN_PACKAGES}
     ${DEBIAN_PACKAGES_X86:=}"
 }
 
 GeneratePackageListARM() {
-  GeneratePackageListCommon "$1" armhf ${APT_REPO_ARM:-${APT_REPO}} \
-                            "${DEBIAN_PACKAGES} ${DEBIAN_PACKAGES_ARM:=}"
+  GeneratePackageListCommon "$1" armhf "${DEBIAN_PACKAGES}
+    ${DEBIAN_PACKAGES_ARM:=}"
 }
 
 GeneratePackageListARM64() {
-  GeneratePackageListCommon "$1" arm64 ${APT_REPO_ARM64:-${APT_REPO}} \
-                            "${DEBIAN_PACKAGES} ${DEBIAN_PACKAGES_ARM64:=}"
+  GeneratePackageListCommon "$1" arm64 "${DEBIAN_PACKAGES}
+    ${DEBIAN_PACKAGES_ARM64:=}"
 }
 
 GeneratePackageListMips() {
-  GeneratePackageListCommon "$1" mipsel ${APT_REPO_MIPS:-${APT_REPO}} \
-                            "${DEBIAN_PACKAGES}"
+  GeneratePackageListCommon "$1" mipsel "${DEBIAN_PACKAGES}"
 }
 
 StripChecksumsFromPackageList() {
   local package_file="$1"
   sed -i 's/ [a-f0-9]\{64\}$//' "$package_file"
 }
 
 VerifyPackageFilesMatch() {
   local downloaded_package_file="$1"
   local stored_package_file="$2"
@@ skipping 121 matching lines @@
     local file="$1"
     local package="${BUILD_DIR}/debian-packages/${file##*/}"
     shift
     local sha256sum="$1"
     shift
     if [ "${#sha256sum}" -ne "64" ]; then
       echo "Bad sha256sum from package list"
       exit 1
     fi
 
-    Banner "Installing ${file}"
-    DownloadOrCopy ${APT_REPO}/pool/${file} ${package}
+    Banner "Installing $(basename ${file})"
+    DownloadOrCopy ${file} ${package}
     if [ ! -s "${package}" ] ; then
       echo
       echo "ERROR: bad package ${package}"
       exit 1
     fi
     echo "${sha256sum}  ${package}" | sha256sum --quiet -c
 
     SubBanner "Extracting to ${INSTALL_ROOT}"
     dpkg-deb -x ${package} ${INSTALL_ROOT}
 
@@ skipping 18 matching lines @@
   cd ${INSTALL_ROOT}
   local libdirs="lib usr/lib"
   if [ "${ARCH}" != "MIPS" ]; then
     libdirs="${libdirs} lib64"
   fi
   find $libdirs -type l -printf '%p %l\n' | while read link target; do
     # skip links with non-absolute paths
     echo "${target}" | grep -qs ^/ || continue
     echo "${link}: ${target}"
     case "${link}" in
-      usr/lib/gcc/*-linux-gnu/4.*/* | usr/lib/gcc/arm-linux-gnueabihf/4.*/* |\
+      usr/lib/gcc/*-linux-gnu/4.*/* | usr/lib/gcc/arm-linux-gnueabihf/4.*/* | \
       usr/lib/gcc/aarch64-linux-gnu/4.*/*)
         # Relativize the symlink.
         ln -snfv "../../../../..${target}" "${link}"
         ;;
       usr/lib/*-linux-gnu/* | usr/lib/arm-linux-gnueabihf/*)
         # Relativize the symlink.
         ln -snfv "../../..${target}" "${link}"
         ;;
       usr/lib/*)
         # Relativize the symlink.
@@ skipping 64 matching lines @@
 BuildSysrootARM() {
   if [ "$HAS_ARCH_ARM" = "0" ]; then
     return
   fi
   ClearInstallDir
   local package_file="$BUILD_DIR/package_with_sha256sum_arm"
   GeneratePackageListARM "$package_file"
   local files_and_sha256sums="$(cat ${package_file})"
   StripChecksumsFromPackageList "$package_file"
   VerifyPackageFilesMatch "$package_file" "$DEBIAN_DEP_LIST_ARM"
-  APT_REPO=${APT_REPO_ARM:=$APT_REPO}
   InstallIntoSysroot ${files_and_sha256sums}
   CleanupJailSymlinks
   HacksAndPatchesARM
   CreateTarBall
 }
 
 #@
 #@ BuildSysrootARM64
 #@
 #@    Build everything and package it
 BuildSysrootARM64() {
   if [ "$HAS_ARCH_ARM64" = "0" ]; then
     return
   fi
   ClearInstallDir
   local package_file="$BUILD_DIR/package_with_sha256sum_arm64"
   GeneratePackageListARM64 "$package_file"
   local files_and_sha256sums="$(cat ${package_file})"
   StripChecksumsFromPackageList "$package_file"
   VerifyPackageFilesMatch "$package_file" "$DEBIAN_DEP_LIST_ARM64"
-  APT_REPO=${APT_REPO_ARM64:=$APT_REPO}
   InstallIntoSysroot ${files_and_sha256sums}
   CleanupJailSymlinks
   HacksAndPatchesARM64
   CreateTarBall
 }
 
 
 #@
 #@ BuildSysrootMips
 #@
 #@    Build everything and package it
 BuildSysrootMips() {
   if [ "$HAS_ARCH_MIPS" = "0" ]; then
     return
   fi
   ClearInstallDir
   local package_file="$BUILD_DIR/package_with_sha256sum_mips"
   GeneratePackageListMips "$package_file"
   local files_and_sha256sums="$(cat ${package_file})"
   StripChecksumsFromPackageList "$package_file"
   VerifyPackageFilesMatch "$package_file" "$DEBIAN_DEP_LIST_MIPS"
-  APT_REPO=${APT_REPO_MIPS:=$APT_REPO}
   InstallIntoSysroot ${files_and_sha256sums}
   CleanupJailSymlinks
   HacksAndPatchesMips
   CreateTarBall
 }
 
 #@
 #@ BuildSysrootAll
 #@
 #@    Build sysroot images for all architectures
@@ skipping 93 matching lines @@
 }
 
 #
 # VerifyPackageListing
 #
 #     Verifies the downloaded Packages.bz2 file has the right checksums.
 #
 VerifyPackageListing() {
   local file_path="$1"
   local output_file="$2"
-  local dist="$3"
+  local repo="$3"
+  local dist="$4"
 
-  local repo_basedir="${APT_REPO}/dists/${dist}"
+  local repo_basedir="${repo}/dists/${dist}"
   local release_list="${repo_basedir}/${RELEASE_FILE}"
   local release_list_gpg="${repo_basedir}/${RELEASE_FILE_GPG}"
 
   local release_file="${BUILD_DIR}/${dist}-${RELEASE_FILE}"
   local release_file_gpg="${BUILD_DIR}/${dist}-${RELEASE_FILE_GPG}"
 
   CheckForDebianGPGKeyring
 
   DownloadOrCopy ${release_list} ${release_file}
   DownloadOrCopy ${release_list_gpg} ${release_file_gpg}
@@ skipping 22 matching lines @@
 #
 GeneratePackageList() {
   local input_file="$1"
   local output_file="$2"
   echo "Updating: ${output_file} from ${input_file}"
   /bin/rm -f "${output_file}"
   shift
   shift
   for pkg in $@ ; do
     local pkg_full=$(grep -A 1 " ${pkg}\$" "$input_file" | \
-      egrep -o "pool/.*")
+      egrep "pool/.*" | sed 's/.*Filename: //')
     if [ -z "${pkg_full}" ]; then
         echo "ERROR: missing package: $pkg"
         exit 1
     fi
-    local pkg_nopool=$(echo "$pkg_full" | sed "s/^pool\///")
     local sha256sum=$(grep -A 4 " ${pkg}\$" "$input_file" | \
       grep ^SHA256: | sed 's/^SHA256: //')
     if [ "${#sha256sum}" -ne "64" ]; then
       echo "Bad sha256sum from Packages"
       exit 1
     fi
-    echo $pkg_nopool $sha256sum >> "$output_file"
+    echo $pkg_full $sha256sum >> "$output_file"
   done
   # sort -o does an in-place sort of this file
   sort "$output_file" -o "$output_file"
 }
 
 #@
 #@ UpdatePackageListsAmd64
 #@
 #@     Regenerate the package lists such that they contain an up-to-date
 #@     list of URLs within the Debian archive. (For amd64)
@@ skipping 124 matching lines @@
   echo "    $0 help"
   exit 1
 else
   ChangeDirectory
   if echo $1 | grep -qs --regexp='\(^Print\)\|\(All$\)'; then
     "$@"
   else
     RunCommand "$@"
   fi
 fi