MQTTSuite
Loading...
Searching...
No Matches
inja::Lexer Class Reference

Class for lexing an inja Template. More...

#include <inja.hpp>

Collaboration diagram for inja::Lexer:

Public Member Functions

 Lexer (const LexerConfig &config)
SourceLocation current_position () const
void start (std::string_view input)
Token scan ()
const LexerConfig & get_config () const

Private Types

enum class  State {
  Text , ExpressionStart , ExpressionStartForceLstrip , ExpressionBody ,
  LineStart , LineBody , StatementStart , StatementStartNoLstrip ,
  StatementStartForceLstrip , StatementBody , CommentStart , CommentStartForceLstrip ,
  CommentBody
}
enum class  MinusState { Operator , Number }

Private Member Functions

Token scan_body (std::string_view close, Token::Kind closeKind, std::string_view close_trim=std::string_view(), bool trim=false)
Token scan_id ()
Token scan_number ()
Token scan_string ()
Token make_token (Token::Kind kind) const
void skip_whitespaces_and_newlines ()
void skip_whitespaces_and_first_newline ()

Static Private Member Functions

static std::string_view clear_final_line_if_whitespace (std::string_view text)

Private Attributes

const LexerConfig & config
State state
MinusState minus_state
std::string_view m_in
size_t tok_start
size_t pos

Detailed Description

Class for lexing an inja Template.

Definition at line 1097 of file inja.hpp.

Member Enumeration Documentation

◆ MinusState

enum class inja::Lexer::MinusState
strongprivate
Enumerator
Operator 
Number 

Definition at line 1114 of file inja.hpp.

1114 {
1115 Operator,
1116 Number,
1117 };

◆ State

enum class inja::Lexer::State
strongprivate
Enumerator
Text 
ExpressionStart 
ExpressionStartForceLstrip 
ExpressionBody 
LineStart 
LineBody 
StatementStart 
StatementStartNoLstrip 
StatementStartForceLstrip 
StatementBody 
CommentStart 
CommentStartForceLstrip 
CommentBody 

Definition at line 1098 of file inja.hpp.

1098 {
1099 Text,
1100 ExpressionStart,
1101 ExpressionStartForceLstrip,
1102 ExpressionBody,
1103 LineStart,
1104 LineBody,
1105 StatementStart,
1106 StatementStartNoLstrip,
1107 StatementStartForceLstrip,
1108 StatementBody,
1109 CommentStart,
1110 CommentStartForceLstrip,
1111 CommentBody,
1112 };

Constructor & Destructor Documentation

◆ Lexer()

inja::Lexer::Lexer ( const LexerConfig & config)
inlineexplicit

Definition at line 1358 of file inja.hpp.

1359 : config(config)
1360 , state(State::Text)
1361 , minus_state(MinusState::Number) {
1362 }
inja::Lexer::config
const LexerConfig & config
Definition inja.hpp:1119
inja::Lexer::state
State state
Definition inja.hpp:1121
inja::Lexer::minus_state
MinusState minus_state
Definition inja.hpp:1122

References config, minus_state, Number, state, and Text.

Referenced by inja::Parser::Parser().

Here is the caller graph for this function:

Member Function Documentation

◆ clear_final_line_if_whitespace()

std::string_view inja::Lexer::clear_final_line_if_whitespace ( std::string_view text)
inlinestaticprivate

Definition at line 1342 of file inja.hpp.

1342 {
1343 std::string_view result = text;
1344 while (!result.empty()) {
1345 const char ch = result.back();
1346 if (ch == ' ' || ch == '\t') {
1347 result.remove_suffix(1);
1348 } else if (ch == '\n' || ch == '\r') {
1349 break;
1350 } else {
1351 return text;
1352 }
1353 }
1354 return result;
1355 }

Referenced by scan().

Here is the caller graph for this function:

◆ current_position()

SourceLocation inja::Lexer::current_position ( ) const
inline

Definition at line 1364 of file inja.hpp.

1364 {
1365 return get_source_location(m_in, tok_start);
1366 }
inja::Lexer::m_in
std::string_view m_in
Definition inja.hpp:1123
inja::Lexer::tok_start
size_t tok_start
Definition inja.hpp:1124
inja::get_source_location
SourceLocation get_source_location(std::string_view content, size_t pos)
Definition inja.hpp:330

References inja::get_source_location(), m_in, and tok_start.

Referenced by inja::Parser::throw_parser_error().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ get_config()

const LexerConfig & inja::Lexer::get_config ( ) const
inline

Definition at line 1517 of file inja.hpp.

1517 {
1518 return config;
1519 }

References config.

Referenced by inja::Parser::parse_into_template().

Here is the caller graph for this function:

◆ make_token()

Token inja::Lexer::make_token ( Token::Kind kind) const
inlineprivate

Definition at line 1310 of file inja.hpp.

1310 {
1311 return Token(kind, string_view::slice(m_in, tok_start, pos));
1312 }
inja::Lexer::pos
size_t pos
Definition inja.hpp:1125
inja::string_view::slice
std::string_view slice(std::string_view view, size_t start, size_t end)
Definition inja.hpp:311

References m_in, pos, inja::string_view::slice(), tok_start, and inja::Token::Token().

Referenced by scan(), scan_body(), scan_id(), scan_number(), and scan_string().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ scan()

Token inja::Lexer::scan ( )
inline

Definition at line 1381 of file inja.hpp.

1381 {
1382 tok_start = pos;
1383
1384 again:
1385 if (tok_start >= m_in.size()) {
1386 return make_token(Token::Kind::Eof);
1387 }
1388
1389 switch (state) {
1390 default:
1391 case State::Text: {
1392 // fast-scan to first open character
1393 const size_t open_start = m_in.substr(pos).find_first_of(config.open_chars);
1394 if (open_start == std::string_view::npos) {
1395 // didn't find open, return remaining text as text token
1396 pos = m_in.size();
1397 return make_token(Token::Kind::Text);
1398 }
1399 pos += open_start;
1400
1401 // try to match one of the opening sequences, and get the close
1402 std::string_view open_str = m_in.substr(pos);
1403 bool must_lstrip = false;
1404 if (inja::string_view::starts_with(open_str, config.expression_open)) {
1405 if (inja::string_view::starts_with(open_str, config.expression_open_force_lstrip)) {
1406 state = State::ExpressionStartForceLstrip;
1407 must_lstrip = true;
1408 } else {
1409 state = State::ExpressionStart;
1410 }
1411 } else if (inja::string_view::starts_with(open_str, config.statement_open)) {
1412 if (inja::string_view::starts_with(open_str, config.statement_open_no_lstrip)) {
1413 state = State::StatementStartNoLstrip;
1414 } else if (inja::string_view::starts_with(open_str, config.statement_open_force_lstrip)) {
1415 state = State::StatementStartForceLstrip;
1416 must_lstrip = true;
1417 } else {
1418 state = State::StatementStart;
1419 must_lstrip = config.lstrip_blocks;
1420 }
1421 } else if (inja::string_view::starts_with(open_str, config.comment_open)) {
1422 if (inja::string_view::starts_with(open_str, config.comment_open_force_lstrip)) {
1423 state = State::CommentStartForceLstrip;
1424 must_lstrip = true;
1425 } else {
1426 state = State::CommentStart;
1427 must_lstrip = config.lstrip_blocks;
1428 }
1429 } else if ((pos == 0 || m_in[pos - 1] == '\n') && inja::string_view::starts_with(open_str, config.line_statement)) {
1430 state = State::LineStart;
1431 } else {
1432 pos += 1; // wasn't actually an opening sequence
1433 goto again;
1434 }
1435
1436 std::string_view text = string_view::slice(m_in, tok_start, pos);
1437 if (must_lstrip) {
1438 text = clear_final_line_if_whitespace(text);
1439 }
1440
1441 if (text.empty()) {
1442 goto again; // don't generate empty token
1443 }
1444 return Token(Token::Kind::Text, text);
1445 }
1446 case State::ExpressionStart: {
1447 state = State::ExpressionBody;
1448 pos += config.expression_open.size();
1449 return make_token(Token::Kind::ExpressionOpen);
1450 }
1451 case State::ExpressionStartForceLstrip: {
1452 state = State::ExpressionBody;
1453 pos += config.expression_open_force_lstrip.size();
1454 return make_token(Token::Kind::ExpressionOpen);
1455 }
1456 case State::LineStart: {
1457 state = State::LineBody;
1458 pos += config.line_statement.size();
1459 return make_token(Token::Kind::LineStatementOpen);
1460 }
1461 case State::StatementStart: {
1462 state = State::StatementBody;
1463 pos += config.statement_open.size();
1464 return make_token(Token::Kind::StatementOpen);
1465 }
1466 case State::StatementStartNoLstrip: {
1467 state = State::StatementBody;
1468 pos += config.statement_open_no_lstrip.size();
1469 return make_token(Token::Kind::StatementOpen);
1470 }
1471 case State::StatementStartForceLstrip: {
1472 state = State::StatementBody;
1473 pos += config.statement_open_force_lstrip.size();
1474 return make_token(Token::Kind::StatementOpen);
1475 }
1476 case State::CommentStart: {
1477 state = State::CommentBody;
1478 pos += config.comment_open.size();
1479 return make_token(Token::Kind::CommentOpen);
1480 }
1481 case State::CommentStartForceLstrip: {
1482 state = State::CommentBody;
1483 pos += config.comment_open_force_lstrip.size();
1484 return make_token(Token::Kind::CommentOpen);
1485 }
1486 case State::ExpressionBody:
1487 return scan_body(config.expression_close, Token::Kind::ExpressionClose, config.expression_close_force_rstrip);
1488 case State::LineBody:
1489 return scan_body("\n", Token::Kind::LineStatementClose);
1490 case State::StatementBody:
1491 return scan_body(
1492 config.statement_close, Token::Kind::StatementClose, config.statement_close_force_rstrip, config.trim_blocks);
1493 case State::CommentBody: {
1494 // fast-scan to comment close
1495 const size_t end = m_in.substr(pos).find(config.comment_close);
1496 if (end == std::string_view::npos) {
1497 pos = m_in.size();
1498 return make_token(Token::Kind::Eof);
1499 }
1500
1501 // Check for trim pattern
1502 const bool must_rstrip = inja::string_view::starts_with(m_in.substr(pos + end - 1), config.comment_close_force_rstrip);
1503
1504 // return the entire comment in the close token
1505 state = State::Text;
1506 pos += end + config.comment_close.size();
1507 Token tok = make_token(Token::Kind::CommentClose);
1508
1509 if (must_rstrip || config.trim_blocks) {
1510 skip_whitespaces_and_first_newline();
1511 }
1512 return tok;
1513 }
1514 }
1515 }
inja::Lexer::clear_final_line_if_whitespace
static std::string_view clear_final_line_if_whitespace(std::string_view text)
Definition inja.hpp:1342
inja::Lexer::make_token
Token make_token(Token::Kind kind) const
Definition inja.hpp:1310
inja::Lexer::scan_body
Token scan_body(std::string_view close, Token::Kind closeKind, std::string_view close_trim=std::string_view(), bool trim=false)
Definition inja.hpp:1128
inja::Lexer::skip_whitespaces_and_first_newline
void skip_whitespaces_and_first_newline()
Definition inja.hpp:1322
inja::string_view::starts_with
bool starts_with(std::string_view view, std::string_view prefix)
Definition inja.hpp:325

References clear_final_line_if_whitespace(), inja::LexerConfig::comment_close, inja::LexerConfig::comment_close_force_rstrip, inja::LexerConfig::comment_open, inja::LexerConfig::comment_open_force_lstrip, CommentBody, inja::Token::CommentClose, inja::Token::CommentOpen, CommentStart, CommentStartForceLstrip, config, inja::Token::Eof, inja::LexerConfig::expression_close, inja::LexerConfig::expression_close_force_rstrip, inja::LexerConfig::expression_open, inja::LexerConfig::expression_open_force_lstrip, ExpressionBody, inja::Token::ExpressionClose, inja::Token::ExpressionOpen, ExpressionStart, ExpressionStartForceLstrip, inja::LexerConfig::line_statement, LineBody, LineStart, inja::Token::LineStatementClose, inja::Token::LineStatementOpen, inja::LexerConfig::lstrip_blocks, m_in, make_token(), inja::LexerConfig::open_chars, pos, scan_body(), skip_whitespaces_and_first_newline(), inja::string_view::slice(), inja::string_view::starts_with(), state, inja::LexerConfig::statement_close, inja::LexerConfig::statement_close_force_rstrip, inja::LexerConfig::statement_open, inja::LexerConfig::statement_open_force_lstrip, inja::LexerConfig::statement_open_no_lstrip, StatementBody, inja::Token::StatementClose, inja::Token::StatementOpen, StatementStart, StatementStartForceLstrip, StatementStartNoLstrip, Text, inja::Token::Text, tok_start, inja::Token::Token(), and inja::LexerConfig::trim_blocks.

Referenced by inja::Parser::get_next_token(), and inja::Parser::get_peek_token().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ scan_body()

Token inja::Lexer::scan_body ( std::string_view close,
Token::Kind closeKind,
std::string_view close_trim = std::string_view(),
bool trim = false )
inlineprivate

Definition at line 1128 of file inja.hpp.

1128 {
1129 again:
1130 // skip whitespace (except for \n as it might be a close)
1131 if (tok_start >= m_in.size()) {
1132 return make_token(Token::Kind::Eof);
1133 }
1134 const char ch = m_in[tok_start];
1135 if (ch == ' ' || ch == '\t' || ch == '\r') {
1136 tok_start += 1;
1137 goto again;
1138 }
1139
1140 // check for close
1141 if (!close_trim.empty() && inja::string_view::starts_with(m_in.substr(tok_start), close_trim)) {
1142 state = State::Text;
1143 pos = tok_start + close_trim.size();
1144 const Token tok = make_token(closeKind);
1145 skip_whitespaces_and_newlines();
1146 return tok;
1147 }
1148
1149 if (inja::string_view::starts_with(m_in.substr(tok_start), close)) {
1150 state = State::Text;
1151 pos = tok_start + close.size();
1152 const Token tok = make_token(closeKind);
1153 if (trim) {
1154 skip_whitespaces_and_first_newline();
1155 }
1156 return tok;
1157 }
1158
1159 // skip \n
1160 if (ch == '\n') {
1161 tok_start += 1;
1162 goto again;
1163 }
1164
1165 pos = tok_start + 1;
1166 if (std::isalpha(ch)) {
1167 minus_state = MinusState::Operator;
1168 return scan_id();
1169 }
1170
1171 const MinusState current_minus_state = minus_state;
1172 if (minus_state == MinusState::Operator) {
1173 minus_state = MinusState::Number;
1174 }
1175
1176 switch (ch) {
1177 case '+':
1178 return make_token(Token::Kind::Plus);
1179 case '-':
1180 if (current_minus_state == MinusState::Operator) {
1181 return make_token(Token::Kind::Minus);
1182 }
1183 return scan_number();
1184 case '*':
1185 return make_token(Token::Kind::Times);
1186 case '/':
1187 return make_token(Token::Kind::Slash);
1188 case '^':
1189 return make_token(Token::Kind::Power);
1190 case '%':
1191 return make_token(Token::Kind::Percent);
1192 case '.':
1193 return make_token(Token::Kind::Dot);
1194 case ',':
1195 return make_token(Token::Kind::Comma);
1196 case ':':
1197 return make_token(Token::Kind::Colon);
1198 case '(':
1199 return make_token(Token::Kind::LeftParen);
1200 case ')':
1201 minus_state = MinusState::Operator;
1202 return make_token(Token::Kind::RightParen);
1203 case '[':
1204 return make_token(Token::Kind::LeftBracket);
1205 case ']':
1206 minus_state = MinusState::Operator;
1207 return make_token(Token::Kind::RightBracket);
1208 case '{':
1209 return make_token(Token::Kind::LeftBrace);
1210 case '}':
1211 minus_state = MinusState::Operator;
1212 return make_token(Token::Kind::RightBrace);
1213 case '>':
1214 if (pos < m_in.size() && m_in[pos] == '=') {
1215 pos += 1;
1216 return make_token(Token::Kind::GreaterEqual);
1217 }
1218 return make_token(Token::Kind::GreaterThan);
1219 case '<':
1220 if (pos < m_in.size() && m_in[pos] == '=') {
1221 pos += 1;
1222 return make_token(Token::Kind::LessEqual);
1223 }
1224 return make_token(Token::Kind::LessThan);
1225 case '=':
1226 if (pos < m_in.size() && m_in[pos] == '=') {
1227 pos += 1;
1228 return make_token(Token::Kind::Equal);
1229 }
1230 return make_token(Token::Kind::Unknown);
1231 case '!':
1232 if (pos < m_in.size() && m_in[pos] == '=') {
1233 pos += 1;
1234 return make_token(Token::Kind::NotEqual);
1235 }
1236 return make_token(Token::Kind::Unknown);
1237 case '\"':
1238 return scan_string();
1239 case '0':
1240 case '1':
1241 case '2':
1242 case '3':
1243 case '4':
1244 case '5':
1245 case '6':
1246 case '7':
1247 case '8':
1248 case '9':
1249 minus_state = MinusState::Operator;
1250 return scan_number();
1251 case '_':
1252 case '@':
1253 case '$':
1254 minus_state = MinusState::Operator;
1255 return scan_id();
1256 default:
1257 return make_token(Token::Kind::Unknown);
1258 }
1259 }
inja::Lexer::scan_string
Token scan_string()
Definition inja.hpp:1292
inja::Lexer::skip_whitespaces_and_newlines
void skip_whitespaces_and_newlines()
Definition inja.hpp:1314
inja::Lexer::scan_number
Token scan_number()
Definition inja.hpp:1275
inja::Lexer::scan_id
Token scan_id()
Definition inja.hpp:1261

References inja::Token::Colon, inja::Token::Comma, inja::Token::Dot, inja::Token::Eof, inja::Token::Equal, inja::Token::GreaterEqual, inja::Token::GreaterThan, inja::Token::LeftBrace, inja::Token::LeftBracket, inja::Token::LeftParen, inja::Token::LessEqual, inja::Token::LessThan, m_in, make_token(), inja::Token::Minus, minus_state, inja::Token::NotEqual, Number, Operator, inja::Token::Percent, inja::Token::Plus, pos, inja::Token::Power, inja::Token::RightBrace, inja::Token::RightBracket, inja::Token::RightParen, scan_id(), scan_number(), scan_string(), skip_whitespaces_and_first_newline(), skip_whitespaces_and_newlines(), inja::Token::Slash, inja::string_view::starts_with(), state, Text, inja::Token::Times, tok_start, and inja::Token::Unknown.

Referenced by scan().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ scan_id()

Token inja::Lexer::scan_id ( )
inlineprivate

Definition at line 1261 of file inja.hpp.

1261 {
1262 for (;;) {
1263 if (pos >= m_in.size()) {
1264 break;
1265 }
1266 const char ch = m_in[pos];
1267 if (!std::isalnum(ch) && ch != '.' && ch != '/' && ch != '_' && ch != '-') {
1268 break;
1269 }
1270 pos += 1;
1271 }
1272 return make_token(Token::Kind::Id);
1273 }

References inja::Token::Id, m_in, make_token(), and pos.

Referenced by scan_body().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ scan_number()

Token inja::Lexer::scan_number ( )
inlineprivate

Definition at line 1275 of file inja.hpp.

1275 {
1276 for (;;) {
1277 if (pos >= m_in.size()) {
1278 break;
1279 }
1280 const char ch = m_in[pos];
1281 // be very permissive in lexer (we'll catch errors when conversion happens)
1282 if (!(std::isdigit(ch) || ch == '.' || ch == 'e' || ch == 'E' ||
1283 (ch == '+' && (pos == 0 || m_in[pos - 1] == 'e' || m_in[pos - 1] == 'E')) ||
1284 (ch == '-' && (pos == 0 || m_in[pos - 1] == 'e' || m_in[pos - 1] == 'E')))) {
1285 break;
1286 }
1287 pos += 1;
1288 }
1289 return make_token(Token::Kind::Number);
1290 }

References m_in, make_token(), inja::Token::Number, and pos.

Referenced by scan_body().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ scan_string()

Token inja::Lexer::scan_string ( )
inlineprivate

Definition at line 1292 of file inja.hpp.

1292 {
1293 bool escape{false};
1294 for (;;) {
1295 if (pos >= m_in.size()) {
1296 break;
1297 }
1298 const char ch = m_in[pos++];
1299 if (ch == '\\') {
1300 escape = true;
1301 } else if (!escape && ch == m_in[tok_start]) {
1302 break;
1303 } else {
1304 escape = false;
1305 }
1306 }
1307 return make_token(Token::Kind::String);
1308 }

References m_in, make_token(), pos, inja::Token::String, and tok_start.

Referenced by scan_body().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ skip_whitespaces_and_first_newline()

void inja::Lexer::skip_whitespaces_and_first_newline ( )
inlineprivate

Definition at line 1322 of file inja.hpp.

1322 {
1323 if (pos < m_in.size()) {
1324 while (pos < m_in.size() && (m_in[pos] == ' ' || m_in[pos] == '\t')) {
1325 pos += 1;
1326 }
1327 }
1328
1329 if (pos < m_in.size()) {
1330 const char ch = m_in[pos];
1331 if (ch == '\n') {
1332 pos += 1;
1333 } else if (ch == '\r') {
1334 pos += 1;
1335 if (pos < m_in.size() && m_in[pos] == '\n') {
1336 pos += 1;
1337 }
1338 }
1339 }
1340 }

References m_in, and pos.

Referenced by scan(), and scan_body().

Here is the caller graph for this function:

◆ skip_whitespaces_and_newlines()

void inja::Lexer::skip_whitespaces_and_newlines ( )
inlineprivate

Definition at line 1314 of file inja.hpp.

1314 {
1315 if (pos < m_in.size()) {
1316 while (pos < m_in.size() && (m_in[pos] == ' ' || m_in[pos] == '\t' || m_in[pos] == '\n' || m_in[pos] == '\r')) {
1317 pos += 1;
1318 }
1319 }
1320 }

References m_in, and pos.

Referenced by scan_body().

Here is the caller graph for this function:

◆ start()

void inja::Lexer::start ( std::string_view input)
inline

Definition at line 1368 of file inja.hpp.

1368 {
1369 m_in = input;
1370 tok_start = 0;
1371 pos = 0;
1372 state = State::Text;
1373 minus_state = MinusState::Number;
1374
1375 // Consume byte order mark (BOM) for UTF-8
1376 if (inja::string_view::starts_with(m_in, "\xEF\xBB\xBF")) {
1377 m_in = m_in.substr(3);
1378 }
1379 }

References m_in, minus_state, Number, pos, inja::string_view::starts_with(), state, Text, and tok_start.

Referenced by inja::Parser::parse_into().

Here is the call graph for this function:
Here is the caller graph for this function:

Member Data Documentation

◆ config

const LexerConfig& inja::Lexer::config
private

Definition at line 1119 of file inja.hpp.

Referenced by get_config(), Lexer(), and scan().

◆ m_in

std::string_view inja::Lexer::m_in
private

Definition at line 1123 of file inja.hpp.

Referenced by current_position(), make_token(), scan(), scan_body(), scan_id(), scan_number(), scan_string(), skip_whitespaces_and_first_newline(), skip_whitespaces_and_newlines(), and start().

◆ minus_state

MinusState inja::Lexer::minus_state
private

Definition at line 1122 of file inja.hpp.

Referenced by Lexer(), scan_body(), and start().

◆ pos

size_t inja::Lexer::pos
private

Definition at line 1125 of file inja.hpp.

Referenced by make_token(), scan(), scan_body(), scan_id(), scan_number(), scan_string(), skip_whitespaces_and_first_newline(), skip_whitespaces_and_newlines(), and start().

◆ state

State inja::Lexer::state
private

Definition at line 1121 of file inja.hpp.

Referenced by Lexer(), scan(), scan_body(), and start().

◆ tok_start

size_t inja::Lexer::tok_start
private

Definition at line 1124 of file inja.hpp.

Referenced by current_position(), make_token(), scan(), scan_body(), scan_string(), and start().

The documentation for this class was generated from the following file: