MQTTSuite
Loading...
Searching...
No Matches
inja::Lexer Class Reference

Class for lexing an inja Template. More...

#include <inja.hpp>

Collaboration diagram for inja::Lexer:

Public Member Functions

 Lexer (const LexerConfig &config)
SourceLocation current_position () const
void start (std::string_view input)
Token scan ()
const LexerConfig & get_config () const

Private Types

enum class  State {
  Text , ExpressionStart , ExpressionStartForceLstrip , ExpressionBody ,
  LineStart , LineBody , StatementStart , StatementStartNoLstrip ,
  StatementStartForceLstrip , StatementBody , CommentStart , CommentStartForceLstrip ,
  CommentBody
}
enum class  MinusState { Operator , Number }

Private Member Functions

Token scan_body (std::string_view close, Token::Kind closeKind, std::string_view close_trim=std::string_view(), bool trim=false)
Token scan_id ()
Token scan_number ()
Token scan_string ()
Token make_token (Token::Kind kind) const
void skip_whitespaces_and_newlines ()
void skip_whitespaces_and_first_newline ()

Static Private Member Functions

static std::string_view clear_final_line_if_whitespace (std::string_view text)

Private Attributes

const LexerConfig & config
State state
MinusState minus_state
std::string_view m_in
size_t tok_start
size_t pos

Detailed Description

Class for lexing an inja Template.

Definition at line 1133 of file inja.hpp.

Member Enumeration Documentation

◆ MinusState

enum class inja::Lexer::MinusState
strongprivate
Enumerator
Operator 
Number 

Definition at line 1150 of file inja.hpp.

1150 {
1151 Operator,
1152 Number,
1153 };

◆ State

enum class inja::Lexer::State
strongprivate
Enumerator
Text 
ExpressionStart 
ExpressionStartForceLstrip 
ExpressionBody 
LineStart 
LineBody 
StatementStart 
StatementStartNoLstrip 
StatementStartForceLstrip 
StatementBody 
CommentStart 
CommentStartForceLstrip 
CommentBody 

Definition at line 1134 of file inja.hpp.

1134 {
1135 Text,
1136 ExpressionStart,
1137 ExpressionStartForceLstrip,
1138 ExpressionBody,
1139 LineStart,
1140 LineBody,
1141 StatementStart,
1142 StatementStartNoLstrip,
1143 StatementStartForceLstrip,
1144 StatementBody,
1145 CommentStart,
1146 CommentStartForceLstrip,
1147 CommentBody,
1148 };

Constructor & Destructor Documentation

◆ Lexer()

inja::Lexer::Lexer ( const LexerConfig & config)
inlineexplicit

Definition at line 1396 of file inja.hpp.

1397 : config(config)
1398 , state(State::Text)
1399 , minus_state(MinusState::Number)
1400 , tok_start(0)
1401 , pos(0) {
1402 }
inja::Lexer::config
const LexerConfig & config
Definition inja.hpp:1155
inja::Lexer::pos
size_t pos
Definition inja.hpp:1161
inja::Lexer::state
State state
Definition inja.hpp:1157
inja::Lexer::tok_start
size_t tok_start
Definition inja.hpp:1160
inja::Lexer::minus_state
MinusState minus_state
Definition inja.hpp:1158

References config, minus_state, Number, pos, state, Text, and tok_start.

Referenced by inja::Parser::Parser().

Here is the caller graph for this function:

Member Function Documentation

◆ clear_final_line_if_whitespace()

std::string_view inja::Lexer::clear_final_line_if_whitespace ( std::string_view text)
inlinestaticprivate

Definition at line 1380 of file inja.hpp.

1380 {
1381 std::string_view result = text;
1382 while (!result.empty()) {
1383 const char ch = result.back();
1384 if (ch == ' ' || ch == '\t') {
1385 result.remove_suffix(1);
1386 } else if (ch == '\n' || ch == '\r') {
1387 break;
1388 } else {
1389 return text;
1390 }
1391 }
1392 return result;
1393 }

Referenced by scan().

Here is the caller graph for this function:

◆ current_position()

SourceLocation inja::Lexer::current_position ( ) const
inline

Definition at line 1404 of file inja.hpp.

1404 {
1405 return get_source_location(m_in, tok_start);
1406 }
inja::Lexer::m_in
std::string_view m_in
Definition inja.hpp:1159
inja::get_source_location
SourceLocation get_source_location(std::string_view content, size_t pos)
Definition inja.hpp:357

References inja::get_source_location(), m_in, and tok_start.

Referenced by inja::Parser::throw_parser_error().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ get_config()

const LexerConfig & inja::Lexer::get_config ( ) const
inline

Definition at line 1557 of file inja.hpp.

1557 {
1558 return config;
1559 }

References config.

Referenced by inja::Parser::parse_into_template().

Here is the caller graph for this function:

◆ make_token()

Token inja::Lexer::make_token ( Token::Kind kind) const
inlineprivate

Definition at line 1348 of file inja.hpp.

1348 {
1349 return Token(kind, string_view::slice(m_in, tok_start, pos));
1350 }
inja::string_view::slice
std::string_view slice(std::string_view view, size_t start, size_t end)
Definition inja.hpp:338

References m_in, pos, inja::string_view::slice(), tok_start, and inja::Token::Token().

Referenced by scan(), scan_body(), scan_id(), scan_number(), and scan_string().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ scan()

Token inja::Lexer::scan ( )
inline

Definition at line 1421 of file inja.hpp.

1421 {
1422 tok_start = pos;
1423
1424 again:
1425 if (tok_start >= m_in.size()) {
1426 return make_token(Token::Kind::Eof);
1427 }
1428
1429 switch (state) {
1430 default:
1431 case State::Text: {
1432 // fast-scan to first open character
1433 const size_t open_start = m_in.substr(pos).find_first_of(config.open_chars);
1434 if (open_start == std::string_view::npos) {
1435 // didn't find open, return remaining text as text token
1436 pos = m_in.size();
1437 return make_token(Token::Kind::Text);
1438 }
1439 pos += open_start;
1440
1441 // try to match one of the opening sequences, and get the close
1442 const std::string_view open_str = m_in.substr(pos);
1443 bool must_lstrip = false;
1444 if (inja::string_view::starts_with(open_str, config.expression_open)) {
1445 if (inja::string_view::starts_with(open_str, config.expression_open_force_lstrip)) {
1446 state = State::ExpressionStartForceLstrip;
1447 must_lstrip = true;
1448 } else {
1449 state = State::ExpressionStart;
1450 }
1451 } else if (inja::string_view::starts_with(open_str, config.statement_open)) {
1452 if (inja::string_view::starts_with(open_str, config.statement_open_no_lstrip)) {
1453 state = State::StatementStartNoLstrip;
1454 } else if (inja::string_view::starts_with(open_str, config.statement_open_force_lstrip)) {
1455 state = State::StatementStartForceLstrip;
1456 must_lstrip = true;
1457 } else {
1458 state = State::StatementStart;
1459 must_lstrip = config.lstrip_blocks;
1460 }
1461 } else if (inja::string_view::starts_with(open_str, config.comment_open)) {
1462 if (inja::string_view::starts_with(open_str, config.comment_open_force_lstrip)) {
1463 state = State::CommentStartForceLstrip;
1464 must_lstrip = true;
1465 } else {
1466 state = State::CommentStart;
1467 must_lstrip = config.lstrip_blocks;
1468 }
1469 } else if ((pos == 0 || m_in[pos - 1] == '\n') && inja::string_view::starts_with(open_str, config.line_statement)) {
1470 state = State::LineStart;
1471 } else {
1472 pos += 1; // wasn't actually an opening sequence
1473 goto again;
1474 }
1475
1476 std::string_view text = string_view::slice(m_in, tok_start, pos);
1477 if (must_lstrip) {
1478 text = clear_final_line_if_whitespace(text);
1479 }
1480
1481 if (text.empty()) {
1482 goto again; // don't generate empty token
1483 }
1484 return Token(Token::Kind::Text, text);
1485 }
1486 case State::ExpressionStart: {
1487 state = State::ExpressionBody;
1488 pos += config.expression_open.size();
1489 return make_token(Token::Kind::ExpressionOpen);
1490 }
1491 case State::ExpressionStartForceLstrip: {
1492 state = State::ExpressionBody;
1493 pos += config.expression_open_force_lstrip.size();
1494 return make_token(Token::Kind::ExpressionOpen);
1495 }
1496 case State::LineStart: {
1497 state = State::LineBody;
1498 pos += config.line_statement.size();
1499 return make_token(Token::Kind::LineStatementOpen);
1500 }
1501 case State::StatementStart: {
1502 state = State::StatementBody;
1503 pos += config.statement_open.size();
1504 return make_token(Token::Kind::StatementOpen);
1505 }
1506 case State::StatementStartNoLstrip: {
1507 state = State::StatementBody;
1508 pos += config.statement_open_no_lstrip.size();
1509 return make_token(Token::Kind::StatementOpen);
1510 }
1511 case State::StatementStartForceLstrip: {
1512 state = State::StatementBody;
1513 pos += config.statement_open_force_lstrip.size();
1514 return make_token(Token::Kind::StatementOpen);
1515 }
1516 case State::CommentStart: {
1517 state = State::CommentBody;
1518 pos += config.comment_open.size();
1519 return make_token(Token::Kind::CommentOpen);
1520 }
1521 case State::CommentStartForceLstrip: {
1522 state = State::CommentBody;
1523 pos += config.comment_open_force_lstrip.size();
1524 return make_token(Token::Kind::CommentOpen);
1525 }
1526 case State::ExpressionBody:
1527 return scan_body(config.expression_close, Token::Kind::ExpressionClose, config.expression_close_force_rstrip);
1528 case State::LineBody:
1529 return scan_body("\n", Token::Kind::LineStatementClose);
1530 case State::StatementBody:
1531 return scan_body(
1532 config.statement_close, Token::Kind::StatementClose, config.statement_close_force_rstrip, config.trim_blocks);
1533 case State::CommentBody: {
1534 // fast-scan to comment close
1535 const size_t end = m_in.substr(pos).find(config.comment_close);
1536 if (end == std::string_view::npos) {
1537 pos = m_in.size();
1538 return make_token(Token::Kind::Eof);
1539 }
1540
1541 // Check for trim pattern
1542 const bool must_rstrip = inja::string_view::starts_with(m_in.substr(pos + end - 1), config.comment_close_force_rstrip);
1543
1544 // return the entire comment in the close token
1545 state = State::Text;
1546 pos += end + config.comment_close.size();
1547 Token tok = make_token(Token::Kind::CommentClose);
1548
1549 if (must_rstrip || config.trim_blocks) {
1550 skip_whitespaces_and_first_newline();
1551 }
1552 return tok;
1553 }
1554 }
1555 }
inja::Lexer::clear_final_line_if_whitespace
static std::string_view clear_final_line_if_whitespace(std::string_view text)
Definition inja.hpp:1380
inja::Lexer::make_token
Token make_token(Token::Kind kind) const
Definition inja.hpp:1348
inja::Lexer::scan_body
Token scan_body(std::string_view close, Token::Kind closeKind, std::string_view close_trim=std::string_view(), bool trim=false)
Definition inja.hpp:1164
inja::Lexer::skip_whitespaces_and_first_newline
void skip_whitespaces_and_first_newline()
Definition inja.hpp:1360
inja::string_view::starts_with
bool starts_with(std::string_view view, std::string_view prefix)
Definition inja.hpp:352

References clear_final_line_if_whitespace(), inja::LexerConfig::comment_close, inja::LexerConfig::comment_close_force_rstrip, inja::LexerConfig::comment_open, inja::LexerConfig::comment_open_force_lstrip, CommentBody, inja::Token::CommentClose, inja::Token::CommentOpen, CommentStart, CommentStartForceLstrip, config, inja::Token::Eof, inja::LexerConfig::expression_close, inja::LexerConfig::expression_close_force_rstrip, inja::LexerConfig::expression_open, inja::LexerConfig::expression_open_force_lstrip, ExpressionBody, inja::Token::ExpressionClose, inja::Token::ExpressionOpen, ExpressionStart, ExpressionStartForceLstrip, inja::LexerConfig::line_statement, LineBody, LineStart, inja::Token::LineStatementClose, inja::Token::LineStatementOpen, inja::LexerConfig::lstrip_blocks, m_in, make_token(), inja::LexerConfig::open_chars, pos, scan_body(), skip_whitespaces_and_first_newline(), inja::string_view::slice(), inja::string_view::starts_with(), state, inja::LexerConfig::statement_close, inja::LexerConfig::statement_close_force_rstrip, inja::LexerConfig::statement_open, inja::LexerConfig::statement_open_force_lstrip, inja::LexerConfig::statement_open_no_lstrip, StatementBody, inja::Token::StatementClose, inja::Token::StatementOpen, StatementStart, StatementStartForceLstrip, StatementStartNoLstrip, Text, inja::Token::Text, tok_start, inja::Token::Token(), and inja::LexerConfig::trim_blocks.

Referenced by inja::Parser::get_next_token(), and inja::Parser::get_peek_token().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ scan_body()

Token inja::Lexer::scan_body ( std::string_view close,
Token::Kind closeKind,
std::string_view close_trim = std::string_view(),
bool trim = false )
inlineprivate

Definition at line 1164 of file inja.hpp.

1164 {
1165 again:
1166 // skip whitespace (except for \n as it might be a close)
1167 if (tok_start >= m_in.size()) {
1168 return make_token(Token::Kind::Eof);
1169 }
1170 const char ch = m_in[tok_start];
1171 if (ch == ' ' || ch == '\t' || ch == '\r') {
1172 tok_start += 1;
1173 goto again;
1174 }
1175
1176 // check for close
1177 if (!close_trim.empty() && inja::string_view::starts_with(m_in.substr(tok_start), close_trim)) {
1178 state = State::Text;
1179 pos = tok_start + close_trim.size();
1180 const Token tok = make_token(closeKind);
1181 skip_whitespaces_and_newlines();
1182 return tok;
1183 }
1184
1185 if (inja::string_view::starts_with(m_in.substr(tok_start), close)) {
1186 state = State::Text;
1187 pos = tok_start + close.size();
1188 const Token tok = make_token(closeKind);
1189 if (trim) {
1190 skip_whitespaces_and_first_newline();
1191 }
1192 return tok;
1193 }
1194
1195 // skip \n
1196 if (ch == '\n') {
1197 tok_start += 1;
1198 goto again;
1199 }
1200
1201 pos = tok_start + 1;
1202 if (std::isalpha(ch)) {
1203 minus_state = MinusState::Operator;
1204 return scan_id();
1205 }
1206
1207 const MinusState current_minus_state = minus_state;
1208 if (minus_state == MinusState::Operator) {
1209 minus_state = MinusState::Number;
1210 }
1211
1212 switch (ch) {
1213 case '+':
1214 return make_token(Token::Kind::Plus);
1215 case '-':
1216 if (current_minus_state == MinusState::Operator) {
1217 return make_token(Token::Kind::Minus);
1218 }
1219 return scan_number();
1220 case '*':
1221 return make_token(Token::Kind::Times);
1222 case '/':
1223 return make_token(Token::Kind::Slash);
1224 case '^':
1225 return make_token(Token::Kind::Power);
1226 case '%':
1227 return make_token(Token::Kind::Percent);
1228 case '.':
1229 return make_token(Token::Kind::Dot);
1230 case ',':
1231 return make_token(Token::Kind::Comma);
1232 case ':':
1233 return make_token(Token::Kind::Colon);
1234 case '|':
1235 return make_token(Token::Kind::Pipe);
1236 case '(':
1237 return make_token(Token::Kind::LeftParen);
1238 case ')':
1239 minus_state = MinusState::Operator;
1240 return make_token(Token::Kind::RightParen);
1241 case '[':
1242 return make_token(Token::Kind::LeftBracket);
1243 case ']':
1244 minus_state = MinusState::Operator;
1245 return make_token(Token::Kind::RightBracket);
1246 case '{':
1247 return make_token(Token::Kind::LeftBrace);
1248 case '}':
1249 minus_state = MinusState::Operator;
1250 return make_token(Token::Kind::RightBrace);
1251 case '>':
1252 if (pos < m_in.size() && m_in[pos] == '=') {
1253 pos += 1;
1254 return make_token(Token::Kind::GreaterEqual);
1255 }
1256 return make_token(Token::Kind::GreaterThan);
1257 case '<':
1258 if (pos < m_in.size() && m_in[pos] == '=') {
1259 pos += 1;
1260 return make_token(Token::Kind::LessEqual);
1261 }
1262 return make_token(Token::Kind::LessThan);
1263 case '=':
1264 if (pos < m_in.size() && m_in[pos] == '=') {
1265 pos += 1;
1266 return make_token(Token::Kind::Equal);
1267 }
1268 return make_token(Token::Kind::Unknown);
1269 case '!':
1270 if (pos < m_in.size() && m_in[pos] == '=') {
1271 pos += 1;
1272 return make_token(Token::Kind::NotEqual);
1273 }
1274 return make_token(Token::Kind::Unknown);
1275 case '\"':
1276 return scan_string();
1277 case '0':
1278 case '1':
1279 case '2':
1280 case '3':
1281 case '4':
1282 case '5':
1283 case '6':
1284 case '7':
1285 case '8':
1286 case '9':
1287 minus_state = MinusState::Operator;
1288 return scan_number();
1289 case '_':
1290 case '@':
1291 case '$':
1292 minus_state = MinusState::Operator;
1293 return scan_id();
1294 default:
1295 return make_token(Token::Kind::Unknown);
1296 }
1297 }
inja::Lexer::scan_string
Token scan_string()
Definition inja.hpp:1330
inja::Lexer::skip_whitespaces_and_newlines
void skip_whitespaces_and_newlines()
Definition inja.hpp:1352
inja::Lexer::scan_number
Token scan_number()
Definition inja.hpp:1313
inja::Lexer::scan_id
Token scan_id()
Definition inja.hpp:1299

References inja::Token::Colon, inja::Token::Comma, inja::Token::Dot, inja::Token::Eof, inja::Token::Equal, inja::Token::GreaterEqual, inja::Token::GreaterThan, inja::Token::LeftBrace, inja::Token::LeftBracket, inja::Token::LeftParen, inja::Token::LessEqual, inja::Token::LessThan, m_in, make_token(), inja::Token::Minus, minus_state, inja::Token::NotEqual, Number, Operator, inja::Token::Percent, inja::Token::Pipe, inja::Token::Plus, pos, inja::Token::Power, inja::Token::RightBrace, inja::Token::RightBracket, inja::Token::RightParen, scan_id(), scan_number(), scan_string(), skip_whitespaces_and_first_newline(), skip_whitespaces_and_newlines(), inja::Token::Slash, inja::string_view::starts_with(), state, Text, inja::Token::Times, tok_start, and inja::Token::Unknown.

Referenced by scan().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ scan_id()

Token inja::Lexer::scan_id ( )
inlineprivate

Definition at line 1299 of file inja.hpp.

1299 {
1300 for (;;) {
1301 if (pos >= m_in.size()) {
1302 break;
1303 }
1304 const char ch = m_in[pos];
1305 if (!std::isalnum(ch) && ch != '.' && ch != '/' && ch != '_' && ch != '-') {
1306 break;
1307 }
1308 pos += 1;
1309 }
1310 return make_token(Token::Kind::Id);
1311 }

References inja::Token::Id, m_in, make_token(), and pos.

Referenced by scan_body().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ scan_number()

Token inja::Lexer::scan_number ( )
inlineprivate

Definition at line 1313 of file inja.hpp.

1313 {
1314 for (;;) {
1315 if (pos >= m_in.size()) {
1316 break;
1317 }
1318 const char ch = m_in[pos];
1319 // be very permissive in lexer (we'll catch errors when conversion happens)
1320 if (!(std::isdigit(ch) || ch == '.' || ch == 'e' || ch == 'E' ||
1321 (ch == '+' && (pos == 0 || m_in[pos - 1] == 'e' || m_in[pos - 1] == 'E')) ||
1322 (ch == '-' && (pos == 0 || m_in[pos - 1] == 'e' || m_in[pos - 1] == 'E')))) {
1323 break;
1324 }
1325 pos += 1;
1326 }
1327 return make_token(Token::Kind::Number);
1328 }

References m_in, make_token(), inja::Token::Number, and pos.

Referenced by scan_body().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ scan_string()

Token inja::Lexer::scan_string ( )
inlineprivate

Definition at line 1330 of file inja.hpp.

1330 {
1331 bool escape{false};
1332 for (;;) {
1333 if (pos >= m_in.size()) {
1334 break;
1335 }
1336 const char ch = m_in[pos++];
1337 if (ch == '\\') {
1338 escape = !escape;
1339 } else if (!escape && ch == m_in[tok_start]) {
1340 break;
1341 } else {
1342 escape = false;
1343 }
1344 }
1345 return make_token(Token::Kind::String);
1346 }

References m_in, make_token(), pos, inja::Token::String, and tok_start.

Referenced by scan_body().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ skip_whitespaces_and_first_newline()

void inja::Lexer::skip_whitespaces_and_first_newline ( )
inlineprivate

Definition at line 1360 of file inja.hpp.

1360 {
1361 if (pos < m_in.size()) {
1362 while (pos < m_in.size() && (m_in[pos] == ' ' || m_in[pos] == '\t')) {
1363 pos += 1;
1364 }
1365 }
1366
1367 if (pos < m_in.size()) {
1368 const char ch = m_in[pos];
1369 if (ch == '\n') {
1370 pos += 1;
1371 } else if (ch == '\r') {
1372 pos += 1;
1373 if (pos < m_in.size() && m_in[pos] == '\n') {
1374 pos += 1;
1375 }
1376 }
1377 }
1378 }

References m_in, and pos.

Referenced by scan(), and scan_body().

Here is the caller graph for this function:

◆ skip_whitespaces_and_newlines()

void inja::Lexer::skip_whitespaces_and_newlines ( )
inlineprivate

Definition at line 1352 of file inja.hpp.

1352 {
1353 if (pos < m_in.size()) {
1354 while (pos < m_in.size() && (m_in[pos] == ' ' || m_in[pos] == '\t' || m_in[pos] == '\n' || m_in[pos] == '\r')) {
1355 pos += 1;
1356 }
1357 }
1358 }

References m_in, and pos.

Referenced by scan_body().

Here is the caller graph for this function:

◆ start()

void inja::Lexer::start ( std::string_view input)
inline

Definition at line 1408 of file inja.hpp.

1408 {
1409 m_in = input;
1410 tok_start = 0;
1411 pos = 0;
1412 state = State::Text;
1413 minus_state = MinusState::Number;
1414
1415 // Consume byte order mark (BOM) for UTF-8
1416 if (inja::string_view::starts_with(m_in, "\xEF\xBB\xBF")) {
1417 m_in = m_in.substr(3);
1418 }
1419 }

References m_in, minus_state, Number, pos, inja::string_view::starts_with(), state, Text, and tok_start.

Referenced by inja::Parser::parse_into().

Here is the call graph for this function:
Here is the caller graph for this function:

Member Data Documentation

◆ config

const LexerConfig& inja::Lexer::config
private

Definition at line 1155 of file inja.hpp.

Referenced by get_config(), Lexer(), and scan().

◆ m_in

std::string_view inja::Lexer::m_in
private

Definition at line 1159 of file inja.hpp.

Referenced by current_position(), make_token(), scan(), scan_body(), scan_id(), scan_number(), scan_string(), skip_whitespaces_and_first_newline(), skip_whitespaces_and_newlines(), and start().

◆ minus_state

MinusState inja::Lexer::minus_state
private

Definition at line 1158 of file inja.hpp.

Referenced by Lexer(), scan_body(), and start().

◆ pos

size_t inja::Lexer::pos
private

Definition at line 1161 of file inja.hpp.

Referenced by Lexer(), make_token(), scan(), scan_body(), scan_id(), scan_number(), scan_string(), skip_whitespaces_and_first_newline(), skip_whitespaces_and_newlines(), and start().

◆ state

State inja::Lexer::state
private

Definition at line 1157 of file inja.hpp.

Referenced by Lexer(), scan(), scan_body(), and start().

◆ tok_start

size_t inja::Lexer::tok_start
private

Definition at line 1160 of file inja.hpp.

Referenced by current_position(), Lexer(), make_token(), scan(), scan_body(), scan_string(), and start().

The documentation for this class was generated from the following file: