| // Copyright (c) 2013-2014 Sandstorm Development Group, Inc. and contributors |
| // Licensed under the MIT License: |
| // |
| // Permission is hereby granted, free of charge, to any person obtaining a copy |
| // of this software and associated documentation files (the "Software"), to deal |
| // in the Software without restriction, including without limitation the rights |
| // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell |
| // copies of the Software, and to permit persons to whom the Software is |
| // furnished to do so, subject to the following conditions: |
| // |
| // The above copyright notice and this permission notice shall be included in |
| // all copies or substantial portions of the Software. |
| // |
| // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
| // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
| // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE |
| // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
| // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, |
| // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN |
| // THE SOFTWARE. |
| |
| #include "lexer.h" |
| #include "../message.h" |
| #include <kj/compat/gtest.h> |
| |
| namespace capnp { |
| namespace compiler { |
| namespace { |
| |
| class TestFailingErrorReporter: public ErrorReporter { |
| public: |
| void addError(uint32_t startByte, uint32_t endByte, kj::StringPtr message) override { |
| KJ_FAIL_EXPECT("Parse failed.", startByte, endByte, message); |
| } |
| |
| bool hadErrors() override { |
| // Not used by lexer. |
| return false; |
| } |
| }; |
| |
| template <typename LexResult> |
| kj::String doLex(kj::StringPtr constText) { |
| // Parse the given string into the given Cap'n Proto struct type using lex(), then stringify the |
| // result and return that string. Additionally, single quotes in the input are converted to |
| // double quotes, and double quotes in the output are converted to single quotes, to reduce the |
| // amount of escaping needed in the test strings. |
| // |
| // Comparing stringifications against golden strings is ugly and brittle. If we had a |
| // text-format parser we could use that. Except that said parser would probably be built on |
| // the very lexer being tested here, so... maybe this is the best we can reasonably do. |
| |
| kj::String text = heapString(constText); |
| for (char& c: text) { |
| // Make it easier to write input strings below. |
| if (c == '\'') c = '\"'; |
| } |
| MallocMessageBuilder message; |
| auto file = message.initRoot<LexResult>(); |
| TestFailingErrorReporter errorReporter; |
| EXPECT_TRUE(lex(text, file, errorReporter)); |
| kj::String result = kj::str(file); |
| for (char& c: result) { |
| // Make it easier to write golden strings below. |
| if (c == '\"') c = '\''; |
| } |
| return result; |
| } |
| |
| TEST(Lexer, Tokens) { |
| EXPECT_STREQ( |
| "(tokens = [" |
| "(identifier = 'foo', startByte = 0, endByte = 3), " |
| "(identifier = 'bar', startByte = 4, endByte = 7)" |
| "])", |
| doLex<LexedTokens>("foo bar").cStr()); |
| |
| EXPECT_STREQ( |
| "(tokens = [" |
| "(identifier = 'foo', startByte = 0, endByte = 3), " |
| "(identifier = 'bar', startByte = 15, endByte = 18)" |
| "])", |
| doLex<LexedTokens>("foo # comment\n bar").cStr()); |
| |
| EXPECT_STREQ( |
| "(tokens = [" |
| "(stringLiteral = 'foo ', startByte = 2, endByte = 11), " |
| "(integerLiteral = 123, startByte = 12, endByte = 15), " |
| "(floatLiteral = 2.75, startByte = 16, endByte = 20), " |
| "(floatLiteral = 60000, startByte = 21, endByte = 24), " |
| "(operator = '+', startByte = 25, endByte = 26), " |
| "(operator = '-=', startByte = 27, endByte = 29)" |
| "])", |
| doLex<LexedTokens>(" 'foo\\x20' 123 2.75 6e4 + -= ").cStr()); |
| |
| EXPECT_STREQ( |
| "(tokens = [" |
| "(parenthesizedList = [" |
| "[" |
| "(identifier = 'foo', startByte = 1, endByte = 4), " |
| "(identifier = 'bar', startByte = 5, endByte = 8)" |
| "], [" |
| "(identifier = 'baz', startByte = 10, endByte = 13), " |
| "(identifier = 'qux', startByte = 14, endByte = 17)" |
| "], [" |
| "(identifier = 'corge', startByte = 19, endByte = 24), " |
| "(identifier = 'grault', startByte = 25, endByte = 31)" |
| "]" |
| "], startByte = 0, endByte = 32)" |
| "])", |
| doLex<LexedTokens>("(foo bar, baz qux, corge grault)").cStr()); |
| |
| EXPECT_STREQ( |
| "(tokens = [" |
| "(parenthesizedList = [" |
| "[" |
| "(identifier = 'foo', startByte = 1, endByte = 4), " |
| "(identifier = 'bar', startByte = 5, endByte = 8)" |
| "]" |
| "], startByte = 0, endByte = 9)" |
| "])", |
| doLex<LexedTokens>("(foo bar)").cStr()); |
| |
| // Empty parentheses should result in an empty list-of-lists, *not* a list containing an empty |
| // list. |
| EXPECT_STREQ( |
| "(tokens = [" |
| "(parenthesizedList = [], startByte = 0, endByte = 4)" |
| "])", |
| doLex<LexedTokens>("( )").cStr()); |
| |
| EXPECT_STREQ( |
| "(tokens = [" |
| "(bracketedList = [" |
| "[" |
| "(identifier = 'foo', startByte = 1, endByte = 4), " |
| "(identifier = 'bar', startByte = 5, endByte = 8)" |
| "], [" |
| "(identifier = 'baz', startByte = 10, endByte = 13), " |
| "(identifier = 'qux', startByte = 14, endByte = 17)" |
| "], [" |
| "(identifier = 'corge', startByte = 19, endByte = 24), " |
| "(identifier = 'grault', startByte = 25, endByte = 31)" |
| "]" |
| "], startByte = 0, endByte = 32)" |
| "])", |
| doLex<LexedTokens>("[foo bar, baz qux, corge grault]").cStr()); |
| |
| // Trailing commas should not create an empty final list item, but be stripped by the lexer. |
| EXPECT_STREQ( |
| "(tokens = [" |
| "(bracketedList = [" |
| "[" |
| "(identifier = 'foo', startByte = 1, endByte = 4)" |
| "], [" |
| "(identifier = 'bar', startByte = 6, endByte = 9)" |
| "]" |
| "], startByte = 0, endByte = 11)" |
| "])", |
| doLex<LexedTokens>("[foo, bar,]").cStr()); |
| |
| EXPECT_STREQ( |
| "(tokens = [" |
| "(bracketedList = [" |
| "[" |
| "(identifier = 'foo', startByte = 1, endByte = 4)" |
| "], [" |
| "(parenthesizedList = [" |
| "[" |
| "(identifier = 'bar', startByte = 7, endByte = 10)" |
| "], [" |
| "(identifier = 'baz', startByte = 12, endByte = 15)" |
| "]" |
| "], startByte = 6, endByte = 16)" |
| "]" |
| "], startByte = 0, endByte = 17), " |
| "(identifier = 'qux', startByte = 18, endByte = 21)" |
| "])", |
| doLex<LexedTokens>("[foo, (bar, baz)] qux").cStr()); |
| |
| EXPECT_STREQ( |
| "(tokens = [" |
| "(identifier = 'foo', startByte = 0, endByte = 3), " |
| "(identifier = 'bar', startByte = 7, endByte = 10)" |
| "])", |
| doLex<LexedTokens>("foo\n\r\t\vbar").cStr()); |
| } |
| |
| TEST(Lexer, Statements) { |
| EXPECT_STREQ( |
| "(statements = [" |
| "(tokens = [" |
| "(identifier = 'foo', startByte = 0, endByte = 3), " |
| "(identifier = 'bar', startByte = 4, endByte = 7)" |
| "], line = void, startByte = 0, endByte = 8)" |
| "])", |
| doLex<LexedStatements>("foo bar;").cStr()); |
| |
| EXPECT_STREQ( |
| "(statements = [" |
| "(tokens = [" |
| "(identifier = 'foo', startByte = 0, endByte = 3)" |
| "], line = void, startByte = 0, endByte = 4), " |
| "(tokens = [" |
| "(identifier = 'bar', startByte = 5, endByte = 8)" |
| "], line = void, startByte = 5, endByte = 9), " |
| "(tokens = [" |
| "(identifier = 'baz', startByte = 10, endByte = 13)" |
| "], line = void, startByte = 10, endByte = 14)" |
| "])", |
| doLex<LexedStatements>("foo; bar; baz; ").cStr()); |
| |
| EXPECT_STREQ( |
| "(statements = [" |
| "(" |
| "tokens = [" |
| "(identifier = 'foo', startByte = 0, endByte = 3)" |
| "], " |
| "block = [" |
| "(tokens = [" |
| "(identifier = 'bar', startByte = 5, endByte = 8)" |
| "], line = void, startByte = 5, endByte = 9), " |
| "(tokens = [" |
| "(identifier = 'baz', startByte = 10, endByte = 13)" |
| "], line = void, startByte = 10, endByte = 14)" |
| "], " |
| "startByte = 0, endByte = 15" |
| "), " |
| "(tokens = [" |
| "(identifier = 'qux', startByte = 16, endByte = 19)" |
| "], line = void, startByte = 16, endByte = 20)" |
| "])", |
| doLex<LexedStatements>("foo {bar; baz;} qux;").cStr()); |
| } |
| |
| TEST(Lexer, DocComments) { |
| EXPECT_STREQ( |
| "(statements = [" |
| "(" |
| "tokens = [" |
| "(identifier = 'foo', startByte = 0, endByte = 3)" |
| "], " |
| "line = void, " |
| "docComment = 'blah blah\\n', " |
| "startByte = 0, endByte = 16" |
| ")" |
| "])", |
| doLex<LexedStatements>("foo; # blah blah").cStr()); |
| |
| EXPECT_STREQ( |
| "(statements = [" |
| "(" |
| "tokens = [" |
| "(identifier = 'foo', startByte = 0, endByte = 3)" |
| "], " |
| "line = void, " |
| "docComment = 'blah blah\\n', " |
| "startByte = 0, endByte = 15" |
| ")" |
| "])", |
| doLex<LexedStatements>("foo; #blah blah").cStr()); |
| |
| EXPECT_STREQ( |
| "(statements = [" |
| "(" |
| "tokens = [" |
| "(identifier = 'foo', startByte = 0, endByte = 3)" |
| "], " |
| "line = void, " |
| "docComment = ' blah blah\\n', " |
| "startByte = 0, endByte = 17" |
| ")" |
| "])", |
| doLex<LexedStatements>("foo; # blah blah").cStr()); |
| |
| EXPECT_STREQ( |
| "(statements = [" |
| "(" |
| "tokens = [" |
| "(identifier = 'foo', startByte = 0, endByte = 3)" |
| "], " |
| "line = void, " |
| "docComment = 'blah blah\\n', " |
| "startByte = 0, endByte = 16" |
| ")" |
| "])", |
| doLex<LexedStatements>("foo;\n# blah blah").cStr()); |
| |
| EXPECT_STREQ( |
| "(statements = [" |
| "(" |
| "tokens = [" |
| "(identifier = 'foo', startByte = 0, endByte = 3)" |
| "], " |
| "line = void, " |
| "startByte = 0, endByte = 4" |
| ")" |
| "])", |
| doLex<LexedStatements>("foo;\n\n# blah blah").cStr()); |
| |
| EXPECT_STREQ( |
| "(statements = [" |
| "(" |
| "tokens = [" |
| "(identifier = 'foo', startByte = 0, endByte = 3)" |
| "], " |
| "line = void, " |
| "docComment = 'bar baz\\nqux corge\\n', " |
| "startByte = 0, endByte = 30" |
| ")" |
| "])", |
| doLex<LexedStatements>("foo;\n # bar baz\n # qux corge\n\n# grault\n# garply").cStr()); |
| |
| EXPECT_STREQ( |
| "(statements = [" |
| "(" |
| "tokens = [" |
| "(identifier = 'foo', startByte = 0, endByte = 3)" |
| "], " |
| "block = [" |
| "(tokens = [" |
| "(identifier = 'bar', startByte = 17, endByte = 20)" |
| "], line = void, docComment = 'hi\\n', startByte = 17, endByte = 27), " |
| "(tokens = [" |
| "(identifier = 'baz', startByte = 28, endByte = 31)" |
| "], line = void, startByte = 28, endByte = 32)" |
| "], " |
| "docComment = 'blah blah\\n', " |
| "startByte = 0, endByte = 44" |
| "), " |
| "(tokens = [" |
| "(identifier = 'qux', startByte = 44, endByte = 47)" |
| "], line = void, startByte = 44, endByte = 48)" |
| "])", |
| doLex<LexedStatements>("foo {# blah blah\nbar; # hi\n baz;} # ignored\nqux;").cStr()); |
| |
| EXPECT_STREQ( |
| "(statements = [" |
| "(" |
| "tokens = [" |
| "(identifier = 'foo', startByte = 0, endByte = 3)" |
| "], " |
| "block = [" |
| "(tokens = [" |
| "(identifier = 'bar', startByte = 5, endByte = 8)" |
| "], line = void, startByte = 5, endByte = 9), " |
| "(tokens = [" |
| "(identifier = 'baz', startByte = 10, endByte = 13)" |
| "], line = void, startByte = 10, endByte = 14)" |
| "], " |
| "docComment = 'late comment\\n', " |
| "startByte = 0, endByte = 31" |
| "), " |
| "(tokens = [" |
| "(identifier = 'qux', startByte = 31, endByte = 34)" |
| "], line = void, startByte = 31, endByte = 35)" |
| "])", |
| doLex<LexedStatements>("foo {bar; baz;}\n# late comment\nqux;").cStr()); |
| } |
| |
| TEST(Lexer, Utf8Bom) { |
| EXPECT_STREQ( |
| "(tokens = [" |
| "(identifier = 'foo', startByte = 3, endByte = 6), " |
| "(identifier = 'bar', startByte = 7, endByte = 10), " |
| "(identifier = 'baz', startByte = 13, endByte = 16)" |
| "])", |
| doLex<LexedTokens>("\xef\xbb\xbf""foo bar\xef\xbb\xbf""baz").cStr()); |
| } |
| |
| } // namespace |
| } // namespace compiler |
| } // namespace capnp |
