OLD | NEW |
(Empty) | |
| 1 // Copyright (c) 2013 The Chromium Authors. All rights reserved. |
| 2 // Use of this source code is governed by a BSD-style license that can be |
| 3 // found in the LICENSE file. |
| 4 |
| 5 #include "testing/gtest/include/gtest/gtest.h" |
| 6 #include "tools/gn/input_file.h" |
| 7 #include "tools/gn/token.h" |
| 8 #include "tools/gn/tokenizer.h" |
| 9 |
| 10 namespace { |
| 11 |
| 12 struct TokenExpectation { |
| 13 Token::Type type; |
| 14 const char* value; |
| 15 }; |
| 16 |
| 17 template<size_t len> |
| 18 bool CheckTokenizer(const char* input, const TokenExpectation (&expect)[len]) { |
| 19 InputFile input_file(SourceFile("/test")); |
| 20 input_file.SetContents(input); |
| 21 |
| 22 Err err; |
| 23 std::vector<Token> results = Tokenizer::Tokenize(&input_file, &err); |
| 24 |
| 25 if (results.size() != len) |
| 26 return false; |
| 27 for (size_t i = 0; i < len; i++) { |
| 28 if (expect[i].type != results[i].type()) |
| 29 return false; |
| 30 if (expect[i].value != results[i].value()) |
| 31 return false; |
| 32 } |
| 33 return true; |
| 34 } |
| 35 |
| 36 } // namespace |
| 37 |
| 38 TEST(Tokenizer, Empty) { |
| 39 InputFile empty_string_input(SourceFile("/test")); |
| 40 empty_string_input.SetContents(""); |
| 41 |
| 42 Err err; |
| 43 std::vector<Token> results = Tokenizer::Tokenize(&empty_string_input, &err); |
| 44 EXPECT_TRUE(results.empty()); |
| 45 |
| 46 InputFile whitespace_input(SourceFile("/test")); |
| 47 whitespace_input.SetContents(" \n\r"); |
| 48 |
| 49 results = Tokenizer::Tokenize(&whitespace_input, &err); |
| 50 EXPECT_TRUE(results.empty()); |
| 51 } |
| 52 |
| 53 TEST(Tokenizer, Identifier) { |
| 54 TokenExpectation one_ident[] = { |
| 55 { Token::IDENTIFIER, "foo" } |
| 56 }; |
| 57 EXPECT_TRUE(CheckTokenizer(" foo ", one_ident)); |
| 58 } |
| 59 |
| 60 TEST(Tokenizer, Integer) { |
| 61 TokenExpectation integers[] = { |
| 62 { Token::INTEGER, "123" }, |
| 63 { Token::INTEGER, "-123" } |
| 64 }; |
| 65 EXPECT_TRUE(CheckTokenizer(" 123 -123 ", integers)); |
| 66 } |
| 67 |
| 68 TEST(Tokenizer, String) { |
| 69 TokenExpectation strings[] = { |
| 70 { Token::STRING, "\"foo\"" }, |
| 71 { Token::STRING, "\"bar\\\"baz\"" }, |
| 72 { Token::STRING, "\"asdf\\\\\"" } |
| 73 }; |
| 74 EXPECT_TRUE(CheckTokenizer(" \"foo\" \"bar\\\"baz\" \"asdf\\\\\" ", |
| 75 strings)); |
| 76 } |
| 77 |
| 78 TEST(Tokenizer, Operator) { |
| 79 TokenExpectation operators[] = { |
| 80 { Token::OPERATOR, "-" }, |
| 81 { Token::OPERATOR, "+" }, |
| 82 { Token::OPERATOR, "=" }, |
| 83 { Token::OPERATOR, "+=" }, |
| 84 { Token::OPERATOR, "-=" }, |
| 85 { Token::OPERATOR, "!=" }, |
| 86 { Token::OPERATOR, "==" }, |
| 87 { Token::OPERATOR, "<" }, |
| 88 { Token::OPERATOR, ">" }, |
| 89 { Token::OPERATOR, "<=" }, |
| 90 { Token::OPERATOR, ">=" }, |
| 91 }; |
| 92 EXPECT_TRUE(CheckTokenizer("- + = += -= != == < > <= >=", |
| 93 operators)); |
| 94 } |
| 95 |
| 96 TEST(Tokenizer, Scoper) { |
| 97 TokenExpectation scopers[] = { |
| 98 { Token::SCOPER, "{" }, |
| 99 { Token::SCOPER, "[" }, |
| 100 { Token::SCOPER, "]" }, |
| 101 { Token::SCOPER, "}" }, |
| 102 { Token::SCOPER, "(" }, |
| 103 { Token::SCOPER, ")" }, |
| 104 }; |
| 105 EXPECT_TRUE(CheckTokenizer("{[ ]} ()", scopers)); |
| 106 } |
| 107 |
| 108 TEST(Tokenizer, FunctionCall) { |
| 109 TokenExpectation fn[] = { |
| 110 { Token::IDENTIFIER, "fun" }, |
| 111 { Token::SCOPER, "(" }, |
| 112 { Token::STRING, "\"foo\"" }, |
| 113 { Token::SCOPER, ")" }, |
| 114 { Token::SCOPER, "{" }, |
| 115 { Token::IDENTIFIER, "foo" }, |
| 116 { Token::OPERATOR, "=" }, |
| 117 { Token::INTEGER, "12" }, |
| 118 { Token::SCOPER, "}" }, |
| 119 }; |
| 120 EXPECT_TRUE(CheckTokenizer("fun(\"foo\") {\nfoo = 12}", fn)); |
| 121 } |
| 122 |
| 123 TEST(Tokenizer, StringUnescaping) { |
| 124 InputFile input(SourceFile("/test")); |
| 125 input.SetContents("\"asd\\\"f\" \"\""); |
| 126 Err err; |
| 127 std::vector<Token> results = Tokenizer::Tokenize(&input, &err); |
| 128 |
| 129 ASSERT_EQ(2u, results.size()); |
| 130 EXPECT_EQ("asd\"f", results[0].StringValue()); |
| 131 EXPECT_EQ("", results[1].StringValue()); |
| 132 } |
| 133 |
| 134 TEST(Tokenizer, Locations) { |
| 135 InputFile input(SourceFile("/test")); |
| 136 input.SetContents("1 2 \"three\"\n 4"); |
| 137 Err err; |
| 138 std::vector<Token> results = Tokenizer::Tokenize(&input, &err); |
| 139 |
| 140 ASSERT_EQ(4u, results.size()); |
| 141 ASSERT_TRUE(results[0].location() == Location(&input, 1, 1)); |
| 142 ASSERT_TRUE(results[1].location() == Location(&input, 1, 3)); |
| 143 ASSERT_TRUE(results[2].location() == Location(&input, 1, 5)); |
| 144 ASSERT_TRUE(results[3].location() == Location(&input, 2, 3)); |
| 145 } |
| 146 |
| 147 TEST(Tokenizer, ByteOffsetOfNthLine) { |
| 148 EXPECT_EQ(0u, Tokenizer::ByteOffsetOfNthLine("foo", 1)); |
| 149 |
| 150 // Windows and Posix have different line endings, so check the byte at the |
| 151 // location rather than the offset. |
| 152 char input1[] = "aaa\nxaa\n\nya"; |
| 153 EXPECT_EQ('x', input1[Tokenizer::ByteOffsetOfNthLine(input1, 2)]); |
| 154 EXPECT_EQ('y', input1[Tokenizer::ByteOffsetOfNthLine(input1, 4)]); |
| 155 |
| 156 char input2[3]; |
| 157 input2[0] = 'a'; |
| 158 input2[1] = '\n'; // Manually set to avoid Windows double-byte endings. |
| 159 input2[2] = 0; |
| 160 EXPECT_EQ(0u, Tokenizer::ByteOffsetOfNthLine(input2, 1)); |
| 161 EXPECT_EQ(2u, Tokenizer::ByteOffsetOfNthLine(input2, 2)); |
| 162 } |
OLD | NEW |