Lexer-shell: skip utf16 magic bytes when reading files.

src/lexer/even-more-experimental-scanner.cc

 // Copyright 2013 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 51 matching lines @@
 const byte* ReadFile(const char* name, Isolate* isolate,
                      int* size, int repeat) {
   FILE* file = fopen(name, "rb");
   *size = 0;
   if (file == NULL) return NULL;
 
   fseek(file, 0, SEEK_END);
   int file_size = ftell(file);
   rewind(file);
 
-  *size = file_size * repeat;
-
-  byte* chars = new byte[*size];
+  byte* file_contents = new byte[file_size];
   for (int i = 0; i < file_size;) {
-    int read = static_cast<int>(fread(&chars[i], 1, file_size - i, file));
+    int read =
+        static_cast<int>(fread(&file_contents[i], 1, file_size - i, file));
     i += read;
   }
   fclose(file);
 
-  for (int i = file_size; i < *size; i++) {
-    chars[i] = chars[i - file_size];
+  // If the file contains the UTF16 little endian magic bytes, skip them.
+  // FIXME: what if we see big endian magic bytes? Do we do the right thing for
+  // big endian anyway?
+  byte* start = file_contents;
+  if (*start == 0xff && *(start + 1) == 0xfe) {
+    start += 2;
+    file_size -= 2;
   }
 
+  *size = file_size * repeat;
+  byte* chars = new byte[*size];
+
+  for (int i = 0; i < *size; i++) {
+    chars[i] = start[i % file_size];
+  }
+
+  delete file_contents;
+
   return chars;
 }
 
 
 
 }
 }