Accept form feeds as whitespace.
This commit is contained in:
parent
f9c5978f5d
commit
6f12e3e29a
2 changed files with 8 additions and 1 deletions
|
@ -117,7 +117,7 @@ namespace {
|
|||
}
|
||||
|
||||
CHARACTER_CLASS(Whitespace, c == ' ' || c == '\n' || c == '\t' ||
|
||||
c == '\r' || c == '\v');
|
||||
c == '\r' || c == '\v' || c == '\f');
|
||||
|
||||
CHARACTER_CLASS(Unprintable, c < ' ' && c > '\0');
|
||||
|
||||
|
|
|
@ -403,6 +403,13 @@ MultiTokenCase kMultiTokenCases[] = {
|
|||
{ Tokenizer::TYPE_SYMBOL, "\300", 0, 0 },
|
||||
{ Tokenizer::TYPE_END , "" , 0, 1 },
|
||||
}},
|
||||
|
||||
// Test all whitespace chars
|
||||
{ "foo\n\t\r\v\fbar", {
|
||||
{ Tokenizer::TYPE_IDENTIFIER, "foo", 0, 0 },
|
||||
{ Tokenizer::TYPE_IDENTIFIER, "bar", 1, 11 },
|
||||
{ Tokenizer::TYPE_END , "" , 1, 14 },
|
||||
}},
|
||||
};
|
||||
|
||||
TEST_2D(TokenizerTest, MultipleTokens, kMultiTokenCases, kBlockSizes) {
|
||||
|
|
Loading…
Add table
Reference in a new issue