Tokens

The csq_token type

The csq_token type represents a token generated by the lexer.

It contains:

• type: the token’s type.
• start: the start of the token in source.
• length: the length of the token.
• line: line where the token is found (1-indexed).
• column: column where the token starts (1-indexed).

Full definition:

typedef struct {
 csq_tktype type;
 const char *start;
 size_t length;
 size_t line;
 size_t column;
} csq_token;

The csq_tktype enum

This enum contains every kind of token the lexer can generate.

This is the full definition:

typedef enum {
  TOKEN_EOF,
  TOKEN_ERROR,
  TOKEN_IDENTIFIER,
  TOKEN_NUMBER,
  TOKEN_STRING,
  TOKEN_TAG,
  TOKEN_BOOLEAN,
  TOKEN_OPERATOR,
  TOKEN_PLUS,
  TOKEN_MINUS,
  TOKEN_STAR,
  TOKEN_SLASH,
  TOKEN_PERCENT,
  TOKEN_CARET,
  TOKEN_AMPERSAND,
  TOKEN_PIPE,
  TOKEN_BANG,
  TOKEN_ASSIGN,
  TOKEN_PLUS_ASSIGN,
  TOKEN_MINUS_ASSIGN,
  TOKEN_STAR_ASSIGN,
  TOKEN_SLASH_ASSIGN,
  TOKEN_EQUAL,
  TOKEN_NOT_EQUAL,
  TOKEN_LESS,
  TOKEN_GREATER,
  TOKEN_LESS_EQUAL,
  TOKEN_GREATER_EQUAL,
  TOKEN_LOGICAL_AND,
  TOKEN_LOGICAL_OR,
  TOKEN_INCREMENT,
  TOKEN_DECREMENT,
  TOKEN_DOUBLE_DOT,
  TOKEN_TRIPLE_DOT,
  TOKEN_RANGE,
  TOKEN_ARROW,
  TOKEN_OPEN_PAREN,
  TOKEN_CLOSE_PAREN,
  TOKEN_OPEN_BRACE,
  TOKEN_CLOSE_BRACE,
  TOKEN_OPEN_BRACKET,
  TOKEN_CLOSE_BRACKET,
  TOKEN_COLON,
  TOKEN_SEMICOLON,
  TOKEN_COMMA,
  TOKEN_DOT,
  TOKEN_HASH,
  TOKEN_AT,
  TOKEN_QUESTION_MARK,
  TOKEN_KEYWORD_FUNCTION,
  TOKEN_KEYWORD_FUNC,
  TOKEN_KEYWORD_IF,
  TOKEN_KEYWORD_ELSE,
  TOKEN_KEYWORD_SWITCH,
  TOKEN_KEYWORD_CASE,
  TOKEN_KEYWORD_DEFAULT,
  TOKEN_KEYWORD_WHILE,
  TOKEN_KEYWORD_FOR,
  TOKEN_KEYWORD_IN,
  TOKEN_KEYWORD_RETURN,
  TOKEN_KEYWORD_THROW,
  TOKEN_KEYWORD_STRUCT,
  TOKEN_KEYWORD_ENUM,
  TOKEN_KEYWORD_IMPORT,
  TOKEN_KEYWORD_NEW,
  TOKEN_KEYWORD_REPEAT,
  TOKEN_KEYWORD_UNTIL,
  TOKEN_KEYWORD_DEFER,
  TOKEN_KEYWORD_TRY,
  TOKEN_KEYWORD_CATCH,
  TOKEN_KEYWORD_SPAWN,
  TOKEN_KEYWORD_PRIVATE,
  TOKEN_KEYWORD_SELF,
  TOKEN_KEYWORD_OR,
  TOKEN_KEYWORD_AND,
  TOKEN_KEYWORD_TRUE,
  TOKEN_KEYWORD_FALSE,
  TOKEN_KEYWORD_BOOL,
  TOKEN_KEYWORD_INT,
  TOKEN_KEYWORD_STRING,
  TOKEN_KEYWORD_FLOAT,
  TOKEN_KEYWORD_BREAK,
  TOKEN_KEYWORD_CONTINUE,
  TOKEN_KEYWORD_OPTIONAL,
  TOKEN_KEYWORD_NIL,
} csq_tktype;