Use chain-loading for Linux nacl_helper

chrome/nacl/nacl_helper_bootstrap_linux.c

Index: chrome/nacl/nacl_helper_bootstrap_linux.c
diff --git a/chrome/nacl/nacl_helper_bootstrap_linux.c b/chrome/nacl/nacl_helper_bootstrap_linux.c
index 7a0ace7ae2fc51fd6d6dd634e1236505e0aa9fa9..5184db1a3baea574bacb8d19941bfe0bb4872718 100644
--- a/chrome/nacl/nacl_helper_bootstrap_linux.c
+++ b/chrome/nacl/nacl_helper_bootstrap_linux.c
@@ -2,23 +2,654 @@
  * Use of this source code is governed by a BSD-style license that can be
  * found in the LICENSE file.
  *
- * Bootstraping the nacl_helper. This executable reserves the bottom 1G
- * of the address space, then invokes nacl_helper_init. Note that,
- * as the text of this executable will eventually be overwritten by the
- * native_client module, nacl_helper_init must not attempt to return.
+ * This is a standalone program that loads and run the dynamic linker.

[Brad Chen, 2011/08/29 23:04:57]

'runs'

+ * This program itself must be linked statically.  To keep it small, it's
+ * written to avoid all dependencies on libc and standard startup code.
+ * Hence, this should be linked using -nostartfiles.  It must be compiled
+ * with -fno-stack-protector to ensure the compiler won't emit code that
+ * presumes some special setup has been done.
+ *
+ * On ARM, the compiler will emit calls to some libc functions, so we
+ * cannot link with -nostdlib.  The functions it does use (memset and
+ * __aeabi_* functions for integer division) are sufficiently small and
+ * self-contained in ARM's libc.a that we don't have any problem using
+ * the libc definitions though we aren't using the rest of libc or doing
+ * any of the setup it might expect.
+ */
+
+#include <asm/unistd.h>
+#include <elf.h>
+#include <fcntl.h>
+#include <link.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <sys/mman.h>
+#include <sys/types.h>
+#include <sys/uio.h>
+
+#define TEXT_START_ADDRESS      0x10000
+#define ADDRESS_SPACE_RESERVE   (1 << 30)  /* one gigabyte */
+#define BSS_RESERVE_SIZE        (ADDRESS_SPACE_RESERVE - TEXT_START_ADDRESS)
+
+#if defined(__i386__)
+# define DYNAMIC_LINKER "/lib/ld-linux.so.2"
+#elif defined(__x86_64__)
+# define DYNAMIC_LINKER "/lib64/ld-linux-x86-64.so.2"
+#elif defined(__ARM_EABI__)
+# define DYNAMIC_LINKER "/lib/ld-linux.so.3"
+#else
+# error "Don't know the dynamic linker file name for this architecture!"
+#endif
+
+/*
+ * We're not using <string.h> functions here to avoid dependencies.
+ * In the x86 libc, even "simple" functions like memset and strlen can
+ * depend on complex startup code, because in newer libc
+ * implementations they are defined using STT_GNU_IFUNC.
+ */
+
+static void my_bzero(void *buf, size_t n) {
+  char *p = buf;
+  while (n-- > 0)
+    *p++ = 0;
+}
+
+static size_t my_strlen(const char *s) {
+  size_t n = 0;
+  while (*s++ != '\0')
+    ++n;
+  return n;
+}
+
+
+/*
+ * Forward declaration.
+ */
+static void fail(const char *message,
+                 const char *item1, int value1,
+                 const char *item2, int value2) __attribute__((noreturn));
+
+static void open_failure(int error) {
+  fail("Cannot open dynamic linker!  ", "errno", error, NULL, 0);
+}
+
+static void check_pread(const char *fail_message, size_t bufsz,
+                        int error, size_t read_count) {
+  if (error != 0)
+    fail(fail_message, "errno", error, NULL, 0);
+  if (read_count != bufsz)
+    fail(fail_message, "read count", read_count, NULL, 0);
+}
+
+static void mmap_failure(const char *segment_type, unsigned int segnum,
+                         int error) {
+  fail("Failed to map from dynamic linker!  ",
+       segment_type, segnum, "errno", error);
+}
+
+static void mprotect_failure(unsigned int segnum, int error) {
+  fail("Failed to mprotect hole in dynamic linker!  ",
+       "segment", segnum, "errno", error);
+}
+
+
+/*
+ * Hand-rolled system call stubs for the few things we use.
  */
 
-#include <stdlib.h>
+#if defined(__i386__)

[Brad Chen, 2011/08/29 23:04:57]

I tend to agree with Mark that these syscall stubs do add a lot of duplicated
code. It would be worth using existing stubs even if it added slightly to the
executable size.

 
-/* reserve 1GB of space */
-#define ONEGIG (1 << 30)
-char nacl_reserved_space[ONEGIG];
+__attribute__((noreturn)) static void my_exit(int status) {
+  asm volatile("int $0x80"

[Mark Seaborn, 2011/08/29 22:26:50]

The Chromium tree already has code for doing syscalls directly.  Please don't
duplicate it.  See linux_syscall_support.h which is pulled into both Breakpad
and seccompsandbox.

+               :: "a" (__NR_exit_group), "b" (status)
+               : "cc");
+  while (1) *(volatile int *) 0 = 0;
+}
 
-void nacl_helper_init(int argc, char *argv[],
-                      const char *nacl_reserved_space);
+static void my_writev(int fd, const struct iovec *iov, int niov) {
+  asm volatile("int $0x80"
+               :: "a" (__NR_writev), "b" (fd), "c" (iov), "d" (niov)
+               : "cc");
+}
+
+static int my_open(const char *file, int oflag) {
+  int result;
+  asm volatile("int $0x80"
+               : "=a" (result)
+               : "0" (__NR_open), "b" (file), "c" (oflag)
+               : "cc");
+  if (result < 0)
+    open_failure(-result);
+  return result;
+}
 
-int main(int argc, char *argv[]) {
-  nacl_helper_init(argc, argv, nacl_reserved_space);
-  abort();
-  return 0;  // convince the tools I'm sane.
+static void my_close(int fd) {
+  int result;
+  asm volatile("int $0x80"
+               : "=a" (result)
+               : "0" (__NR_close), "b" (fd)
+               : "cc");
 }
+
+static void my_pread(const char *fail_message,
+                     int fd, void *buf, size_t bufsz, uintptr_t pos) {
+  int result;
+  /*
+   * Ideally we would use a memory output operand rather than the
+   * "memory" clobber.  But the compiler can't handle that here,
+   * because it makes it want to use too many registers.
+   */
+  asm volatile("int $0x80"
+               : "=a" (result)
+               : "0" (__NR_pread64), "b" (fd), "c" (buf), "d" (bufsz),
+                 "S" (pos), "D" (0)
+               : "cc", "memory");
+  check_pread(fail_message, bufsz, result < 0 ? -result : 0, result);
+}
+
+static uintptr_t my_mmap(const char *segment_type, unsigned int segnum,
+                         uintptr_t address, size_t size, int prot, int flags,
+                         int fd, uint32_t pos) {
+  uintptr_t result;
+  const struct {
+    unsigned int args[6];
+  } args = {
+    { address, size, prot, flags, fd, pos }
+  };
+  asm volatile("int $0x80"
+               : "=a" (result)
+               : "0" (__NR_mmap), "b" (&args), "m" (args)
+               : "cc");
+  if (result > (uintptr_t) -4096)
+    mmap_failure(segment_type, segnum, -result);
+  return result;
+}
+
+static void my_mprotect(unsigned int segnum,
+                        uintptr_t address, size_t size, int prot) {
+  int result;
+  asm volatile("int $0x80"
+               : "=a" (result)
+               : "b" (address), "c" (size), "d" (prot)
+               : "cc");
+  if (result < 0)
+    mprotect_failure(segnum, -result);
+}
+
+#elif defined(__x86_64__)
+
+__attribute__((noreturn)) static void my_exit(int status) {
+  asm volatile("syscall"
+               :: "a" (__NR_exit_group), "D" (status)
+               : "cc", "rcx", "r11");
+  while (1) *(volatile int *) 0 = 0;
+}
+
+static void my_writev(int fd, const struct iovec *iov, int niov) {
+  asm volatile("syscall"
+               :: "a" (__NR_writev), "D" (fd), "S" (iov), "d" (niov)
+               : "cc", "rcx", "r11");
+}
+
+static int my_open(const char *file, int oflag) {
+  int result;
+  asm volatile("syscall"
+               : "=a" (result)
+               : "0" (__NR_open), "D" (file), "S" (oflag)
+               : "cc", "rcx", "r11");
+  if (result < 0)
+    open_failure(-result);
+  return result;
+}
+
+static void my_close(int fd) {
+  int result;
+  asm volatile("syscall"
+               : "=a" (result)
+               : "0" (__NR_close), "D" (fd)
+               : "cc", "rcx", "r11");
+}
+
+static void my_pread(const char *fail_message,
+                     int fd, void *buf, size_t bufsz, uintptr_t pos) {
+  int result;
+  asm volatile("mov %6, %%r10; syscall"
+               : "=a" (result), "=m" (*(struct { char b[bufsz]; } *) buf)
+               : "0" (__NR_pread64), "D" (fd), "S" (buf), "d" (bufsz), "g" (pos)
+               : "cc", "rcx", "r11", "r10");
+  check_pread(fail_message, bufsz, result < 0 ? -result : 0, result);
+}
+
+static uintptr_t my_mmap(const char *segment_type, unsigned int segnum,
+                         uintptr_t address, size_t size, int prot, int flags,
+                         int fd, off_t pos) {
+  uintptr_t result;
+  asm volatile("mov %5, %%r10; mov %6, %%r8; mov %7, %%r9; syscall"
+               : "=a" (result)
+               : "0" (__NR_mmap), "D" (address), "S" (size),
+                 "d" ((long int) prot), "g" ((long int) flags),
+                 "g" ((long int) fd), "g" (pos)
+               : "cc", "rcx", "r11", "r10", "r9", "r8");
+  if (result > (uintptr_t) -4096)
+    mmap_failure(segment_type, segnum, -result);
+  return result;
+}
+
+static void my_mprotect(unsigned int segnum,
+                        uintptr_t address, size_t size, int prot) {
+  int result;
+  asm volatile("syscall"
+               : "=a" (result)
+               : "0" (__NR_mprotect), "D" (address), "S" (size), "d" (prot)
+               : "cc", "rcx", "r11");
+  if (result < 0)
+    mprotect_failure(segnum, -result);
+}
+
+#elif defined(__ARM_EABI__)
+
+/*
+ * We must touch r7 behind the compiler's back when compiling for
+ * Thumb.  r7 is the Thumb frame pointer and the compiler won't honor
+ * our private use of it without getting confused and clobbering
+ * things.  Since a few extra instructions here don't hurt, we just
+ * use the same method for non-Thumb builds too.
+ */
+#define SYSCALL_ASM \
+  "mov %[scratch], r7\n" \
+  "mov r7, %[nr]\n" \
+  "swi 0\n" \
+  "mov r7, %[scratch]"
+
+__attribute__((noreturn)) static void my_exit(int status) {
+  int scratch;
+  register int a1 asm ("a1") = status;
+  asm volatile(SYSCALL_ASM
+               : [scratch] "=&r" (scratch)
+               : [nr] "i" (__NR_exit_group), "r" (a1));
+  while (1) *(volatile int *) 0 = 0;
+}
+
+static void my_writev(int fd, const struct iovec *iov, int niov) {
+  int scratch;
+  register int a1 asm ("a1") = fd;
+  register int a2 asm ("a2") = (uintptr_t) iov;
+  register int a3 asm ("a3") = niov;
+  asm volatile(SYSCALL_ASM
+               : [scratch] "=&r" (scratch), "=r" (a1)
+               : [nr] "i" (__NR_writev), "r" (a1), "r" (a2), "r" (a3));
+}
+
+static int my_open(const char *file, int oflag) {
+  int scratch;
+  register int result asm ("a1") = (uintptr_t) file;
+  register int a2 asm ("a2") = oflag;
+  asm volatile(SYSCALL_ASM
+               : [scratch] "=&r" (scratch), "=r" (result)
+               : [nr] "i" (__NR_open), "r" (result), "r" (a2));
+  if (result < 0)
+    open_failure(-result);
+  return result;
+}
+
+static void my_close(int fd) {
+  int scratch;
+  register int result asm ("a1") = fd;
+  asm volatile(SYSCALL_ASM
+               : [scratch] "=&r" (scratch), "=r" (result)
+               : [nr] "i" (__NR_close), "r" (result));
+}
+
+static void my_pread(const char *fail_message,
+                     int fd, void *buf, size_t bufsz, uintptr_t pos) {
+  int scratch;
+  register int result asm ("a1") = fd;
+  register int a2 asm ("a2") = (uintptr_t) buf;
+  register int a3 asm ("a3") = bufsz;
+  register int a4 asm ("a4") = 0;
+  register int v1 asm ("v1");
+  register int v2 asm ("v2");
+#if defined(__ARMEL__)
+  v1 = pos;
+  v2 = 0;
+#else
+  v1 = 0;
+  v2 = pos;
+#endif
+  asm volatile(SYSCALL_ASM
+               : [scratch] "=&r" (scratch), "=r" (result)
+               : [nr] "i" (__NR_pread64), "r" (result), "r" (a2), "r" (a3),
+                 "r" (a4), "r" (v1), "r" (v2));
+  check_pread(fail_message, bufsz, result < 0 ? -result : 0, result);
+}
+
+static uintptr_t my_mmap(const char *segment_type, unsigned int segnum,
+                         uintptr_t address, size_t size, int prot, int flags,
+                         int fd, off_t pos) {
+  int scratch;
+  register uintptr_t result asm ("a1") = address;
+  register int a2 asm ("a2") = size;
+  register int a3 asm ("a3") = prot;
+  register int a4 asm ("a4") = flags;
+  register int v1 asm ("v1") = fd;
+  register int v2 asm ("v2") = pos >> 12;
+  asm volatile(SYSCALL_ASM
+               : [scratch] "=&r" (scratch), "=r" (result)
+               : [nr] "i" (__NR_mmap2), "r" (result), "r" (a2), "r" (a3),
+                 "r" (a4), "r" (v1), "r" (v2));
+  if (result > (uintptr_t) -4096)
+    mmap_failure(segment_type, segnum, -result);
+  return result;
+}
+
+static void my_mprotect(unsigned int segnum,
+                        uintptr_t address, size_t size, int prot) {
+  int scratch;
+  register int result asm ("a1") = address;
+  register int a2 asm ("a2") = size;
+  register int a3 asm ("a3") = prot;
+  asm volatile(SYSCALL_ASM
+               : [scratch] "=&r" (scratch), "=r" (result)
+               : [nr] "i" (__NR_mprotect), "r" (result), "r" (a2), "r" (a3));
+  if (result < 0)
+    mprotect_failure(segnum, -result);
+}
+
+#else
+# error "Need inline syscall implementation for this architecture!"
+#endif
+
+/*
+ * We're avoiding libc, so no printf.  The only nontrivial thing we need
+ * is rendering numbers, which is, in fact, pretty trivial.
+ */
+static void itoa(int value, struct iovec *iov, char *buf, size_t bufsz) {
+  char *p = &buf[bufsz];
+  do {
+    *--p = "0123456789"[value % 10];

[Brad Chen, 2011/08/29 23:04:57]

Yikes! I bet this code works just fine but I'm pretty sure you are violating
several style rules. Think you could break it into approximate one operation per
line?

+    value /= 10;
+  } while (value != 0);
+  iov->iov_base = p;
+  iov->iov_len = &buf[bufsz] - p;
+}
+
+#define STRING_IOV(string_constant, cond) \
+  { (void *) string_constant, cond ? (sizeof(string_constant) - 1) : 0 }
+
+__attribute__((noreturn)) static void fail(const char *message,
+                                           const char *item1, int value1,
+                                           const char *item2, int value2) {
+  char valbuf1[32], valbuf2[32];
+  struct iovec iov[] = {
+    STRING_IOV("bootstrap_helper", 1),
+    STRING_IOV(DYNAMIC_LINKER, 1),
+    STRING_IOV(": ", 1),
+    { (void *) message, my_strlen(message) },
+    { (void *)item1, item1 == NULL ? 0 : my_strlen(item1) },
+    STRING_IOV("=", item1 != NULL),
+    {},
+    STRING_IOV(", ", item1 != NULL && item2 != NULL),
+    { (void *) item2, item2 == NULL ? 0 : my_strlen(item2) },
+    STRING_IOV("=", item2 != NULL),
+    {},
+    { "\n", 1 },
+  };
+  const int niov = sizeof(iov) / sizeof(iov[0]);
+
+  if (item1 != NULL)
+    itoa(value1, &iov[6], valbuf1, sizeof(valbuf1));
+  if (item2 != NULL)
+    itoa(value1, &iov[10], valbuf2, sizeof(valbuf2));
+
+  my_writev(2, iov, niov);
+  my_exit(2);
+}
+
+static int prot_from_phdr(const ElfW(Phdr) *phdr) {
+  int prot = 0;
+  if (phdr->p_flags & PF_R)
+    prot |= PROT_READ;
+  if (phdr->p_flags & PF_W)
+    prot |= PROT_WRITE;
+  if (phdr->p_flags & PF_X)
+    prot |= PROT_EXEC;
+  return prot;
+}
+
+/*
+ * Handle the "bss" portion of a segment, where the memory size
+ * exceeds the file size and we zero-fill the difference.  For any
+ * whole pages in this region, we over-map anonymous pages.  For the
+ * sub-page remainder, we zero-fill bytes directly.
+ */
+static void handle_bss(unsigned int segnum, const ElfW(Phdr) *ph,
+                       ElfW(Addr) load_bias, size_t pagesize) {
+  if (ph->p_memsz > ph->p_filesz) {
+    ElfW(Addr) file_end = ph->p_vaddr + load_bias + ph->p_filesz;
+    ElfW(Addr) file_page_end = (file_end + pagesize - 1) & -pagesize;
+    ElfW(Addr) page_end = (ph->p_vaddr + load_bias +
+                           ph->p_memsz + pagesize - 1) & -pagesize;
+    if (page_end > file_page_end)
+      my_mmap("bss segment", segnum,
+              file_page_end, page_end - file_page_end,
+              prot_from_phdr(ph), MAP_ANON | MAP_PRIVATE | MAP_FIXED, -1, 0);
+    if (file_page_end > file_end && (ph->p_flags & PF_W))
+      my_bzero((void *) file_end, file_page_end - file_end);
+  }
+}
+
+/*
+ * This is the main loading code.  It's called with the address of the
+ * auxiliary vector on the stack, which we need to examine and modify.
+ * It returns the dynamic linker's runtime entry point address, where
+ * we should jump to.  This is called by the machine-dependent _start
+ * code (below).  On return, it restores the original stack pointer
+ * and jumps to this entry point.
+ */
+ElfW(Addr) do_load(ElfW(auxv_t) *auxv) {
+  /* Record the auxv entries that are specific to the file loaded.
+     The incoming entries point to our own static executable.  */
+  ElfW(auxv_t) *av_entry = NULL;
+  ElfW(auxv_t) *av_phdr = NULL;
+  ElfW(auxv_t) *av_phnum = NULL;
+  size_t pagesize = 0;
+
+  ElfW(auxv_t) *av;
+  for (av = auxv;
+       av_entry == NULL || av_phdr == NULL || av_phnum == NULL || pagesize == 0;
+       ++av)
+    switch (av->a_type) {
+      case AT_NULL:
+        fail("Failed to find AT_ENTRY, AT_PHDR, AT_PHNUM, or AT_PAGESZ!",
+             NULL, 0, NULL, 0);
+        /*NOTREACHED*/
+        break;
+      case AT_ENTRY:
+        av_entry = av;
+        break;
+      case AT_PAGESZ:
+        pagesize = av->a_un.a_val;
+        break;
+      case AT_PHDR:
+        av_phdr = av;
+        break;
+      case AT_PHNUM:
+        av_phnum = av;
+        break;
+    }
+
+  int fd = my_open(DYNAMIC_LINKER, O_RDONLY);
+
+  ElfW(Ehdr) ehdr;
+  my_pread("Failed to read ELF header from dynamic linker!  ",
+           fd, &ehdr, sizeof(ehdr), 0);
+
+  if (ehdr.e_ident[EI_MAG0] != ELFMAG0 ||
+      ehdr.e_ident[EI_MAG1] != ELFMAG1 ||
+      ehdr.e_ident[EI_MAG2] != ELFMAG2 ||
+      ehdr.e_ident[EI_MAG3] != ELFMAG3 ||
+      ehdr.e_version != EV_CURRENT ||
+      ehdr.e_ehsize != sizeof(ehdr) ||
+      ehdr.e_phentsize != sizeof(ElfW(Phdr)))
+    fail("Dynamic linker has no valid ELF header!", NULL, 0, NULL, 0);
+
+  switch (ehdr.e_machine) {
+#if defined(__i386__)
+    case EM_386:
+#elif defined(__x86_64__)
+    case EM_X86_64:
+#elif defined(__arm__)
+    case EM_ARM:
+#else
+# error "Don't know the e_machine value for this architecture!"
+#endif
+      break;
+    default:
+      fail("Dynamic linker has wrong architecture!  ",
+           "e_machine", ehdr.e_machine, NULL, 0);
+      break;
+  }
+
+  ElfW(Phdr) phdr[12];

[Brad Chen, 2011/08/29 23:04:57]

make this 12 a constant please.

+  if (ehdr.e_phnum > sizeof(phdr) / sizeof(phdr[0]) || ehdr.e_phnum < 1)
+    fail("Dynamic linker has unreasonable ",
+         "e_phnum", ehdr.e_phnum, NULL, 0);
+
+  if (ehdr.e_type != ET_DYN)
+    fail("Dynamic linker not ET_DYN!  ",
+         "e_type", ehdr.e_type, NULL, 0);
+
+  my_pread("Failed to read program headers from dynamic linker!  ",
+           fd, phdr, sizeof(phdr[0]) * ehdr.e_phnum, ehdr.e_phoff);
+
+  size_t i = 0;
+  while (i < ehdr.e_phnum && phdr[i].p_type != PT_LOAD)
+    ++i;
+  if (i == ehdr.e_phnum)
+    fail("Dynamic linker has no PT_LOAD header!",
+         NULL, 0, NULL, 0);
+
+  const ElfW(Phdr) *first_load = &phdr[i];
+  const ElfW(Phdr) *last_load = &phdr[ehdr.e_phnum - 1];
+  while (last_load > first_load && last_load->p_type != PT_LOAD)
+    --last_load;
+
+  size_t span = last_load->p_vaddr + last_load->p_memsz - first_load->p_vaddr;
+
+  const uintptr_t mapping = my_mmap("segment", first_load - phdr,
+                                    first_load->p_vaddr & -pagesize, span,
+                                    prot_from_phdr(first_load), MAP_PRIVATE, fd,
+                                    first_load->p_offset & -pagesize);
+
+  const ElfW(Addr) load_bias = ((uintptr_t) mapping -
+                                (first_load->p_vaddr & -pagesize));
+
+  if (first_load->p_offset > ehdr.e_phoff ||
+      first_load->p_filesz < ehdr.e_phoff + (ehdr.e_phnum * sizeof(ElfW(Phdr))))
+    fail("First load segment of dynamic linker does not contain phdrs!",
+         NULL, 0, NULL, 0);
+
+  /* Point the auxv elements at the dynamic linker's phdrs and entry.  */
+  av_phdr->a_un.a_val = (ehdr.e_phoff - first_load->p_offset +
+                         first_load->p_vaddr + load_bias);
+  av_phnum->a_un.a_val = ehdr.e_phnum;
+  av_entry->a_un.a_val = ehdr.e_entry + load_bias;
+
+  handle_bss(first_load - phdr, first_load, load_bias, pagesize);
+
+  ElfW(Addr) last_end = first_load->p_vaddr + load_bias + first_load->p_memsz;
+
+  const ElfW(Phdr) *ph;
+  for (ph = first_load + 1; ph <= last_load; ++ph)
+    if (ph->p_type == PT_LOAD) {
+      ElfW(Addr) last_page_end = (last_end + pagesize - 1) & -pagesize;
+
+      last_end = ph->p_vaddr + load_bias + ph->p_memsz;
+      ElfW(Addr) start = (ph->p_vaddr + load_bias) & -pagesize;
+      ElfW(Addr) end = (last_end + pagesize - 1) & -pagesize;
+
+      if (start > last_page_end)
+        my_mprotect(ph - phdr, last_page_end, start - last_page_end, PROT_NONE);
+
+      my_mmap("segment", ph - phdr,
+              start, end - start,
+              prot_from_phdr(ph), MAP_PRIVATE | MAP_FIXED, fd,
+              ph->p_offset & -pagesize);
+
+      handle_bss(ph - phdr, ph, load_bias, pagesize);
+    }
+
+  my_close(fd);
+
+  return ehdr.e_entry + load_bias;
+}
+
+/*
+ * We have to define the actual entry point code (_start) in assembly
+ * for each machine.  The kernel startup protocol is not compatible
+ * with the normal C function calling convention.  Here, we calculate
+ * the address of the auxiliary vector on the stack; call do_load
+ * (above) using the normal C convention as per the ABI; restore the
+ * original starting stack; and finally, jump to the dynamic linker's
+ * entry point address.

[Brad Chen, 2011/08/29 23:04:57]

This is pretty cool code, but again, if we could use the standard _start without
too much bloat it would probably be better for maintainability.

+ */
+#if defined(__i386__)
+asm(".globl _start\n"
+    ".type _start,@function\n"
+    "_start:\n"
+    "xorl %ebp, %ebp\n"
+    "movl %esp, %ebx\n"    /* Save starting SP in %ebx.  */
+    "andl $-16, %esp\n"    /* Align the stack as per ABI.  */
+    "movl (%ebx), %eax\n"  /* argc */
+    "leal 8(%ebx,%eax,4), %ecx\n"  /* envp */
+    /* Find the envp element that is NULL, and auxv is past there.  */
+    "0: addl $4, %ecx\n"
+    "cmpl $0, -4(%ecx)\n"
+    "jne 0b\n"
+    "pushl %ecx\n"  /* Argument: auxv.  */
+    "call do_load\n"
+    "movl %ebx, %esp\n"  /* Restore the saved SP.  */
+    "jmp *%eax\n"
+    );        /* Jump to the entry point.  */
+#elif defined(__x86_64__)
+asm(".globl _start\n"
+    ".type _start,@function\n"
+    "_start:\n"
+    "xorq %rbp, %rbp\n"
+    "movq %rsp, %rbx\n"    /* Save starting SP in %rbx.  */
+    "andq $-16, %rsp\n"    /* Align the stack as per ABI.  */
+    "movq (%rbx), %rax\n"  /* argc */
+    "leaq 16(%rbx,%rax,8), %rdi\n"  /* envp */
+    /* Find the envp element that is NULL, and auxv is past there.  */
+    "0: addq $8, %rdi\n"
+    "cmpq $0, -8(%rdi)\n"
+    "jne 0b\n"
+    "call do_load\n"    /* Argument already in %rdi: auxv */
+    "movq %rbx, %rsp\n"  /* Restore the saved SP.  */
+    "jmp *%rax\n"        /* Jump to the entry point.  */
+    );
+#elif defined(__arm__)
+asm(".globl _start\n"
+    ".type _start,#function\n"
+    "_start:\n"
+#if defined(__thumb2__)
+    ".thumb\n"
+    ".syntax unified\n"
+#endif
+    "mov fp, #0\n"
+    "mov lr, #0\n"
+    "mov r4, sp\n"       /* Save starting SP in r4.  */
+    "ldr r1, [r4]\n"     /* argc */
+    "add r1, r1, #2\n"
+    "add r0, r4, r1, asl #2\n"  /* envp */
+    /* Find the envp element that is NULL, and auxv is past there.  */
+    "0: ldr r1, [r0], #4\n"
+    "cmp r1, #0\n"
+    "bne 0b\n"
+    "bl do_load\n"
+    "mov sp, r4\n"
+    "blx r0\n"
+    );
+#else
+# error "Need stack-preserving _start code for this architecture!"
+#endif