Experimental scanner fix: recognize one byte strings inside utf16 files.

src/lexer/lexer-shell.cc

Index: src/lexer/lexer-shell.cc
diff --git a/src/lexer/lexer-shell.cc b/src/lexer/lexer-shell.cc
index af50c9c9e65564a3778178d154e6cd24adc422dd..83d273935b20fdd14983d4bd7657fe38ea91a8ff 100644
--- a/src/lexer/lexer-shell.cc
+++ b/src/lexer/lexer-shell.cc
@@ -196,6 +196,7 @@ struct TokenWithLocation {
       for (size_t i = 0; i < literal.size(); i++) {
         printf(is_ascii ? " %02x" : " %04x", literal[i]);
       }
+      printf(" (is ascii: %d)", is_ascii);
     }
     printf(" (last octal start: %d)\n", octal_beg);
   }