rules.py

# Rules :
# [
#     (TYPE , REGEX , BEFORE , AFTER)
# ]

# positive lookahead
MATCH_UNTIL_SEQ = r"(?:(?=%s))"
UNTIL_SPACE = MATCH_UNTIL_SEQ % r"\s"
UNTIL_SPACE_OR_SEMICOLON = MATCH_UNTIL_SEQ % r"\s|\;"
UNTIL_SPACE_OR_COLON = MATCH_UNTIL_SEQ % r"\s|\:"

RULES = [
    ("AUTO", r"auto", None, UNTIL_SPACE),
    ("NEW", r"new", None, UNTIL_SPACE),
    ("EOF", r"0", None, UNTIL_SPACE),
    ("TRUE", r"true", None, UNTIL_SPACE_OR_SEMICOLON),
    ("FALSE", r"false", None, UNTIL_SPACE_OR_SEMICOLON),
    ("BREAK", r"break", None, UNTIL_SPACE_OR_SEMICOLON),
    ("BOOL", r"bool", None, UNTIL_SPACE),
    ("CASE", r"case", None, UNTIL_SPACE),
    ("CHAR", r"char", None, UNTIL_SPACE),
    ("CONST", r"const", None, UNTIL_SPACE),
    ("CONTINUE", r"continue", None, UNTIL_SPACE_OR_SEMICOLON),
    ("DEFAULT", r"default", None, UNTIL_SPACE_OR_COLON),
    ("DO", r"do", None, None),
    ("DOUBLE", r"double", None, UNTIL_SPACE),
    ("ELSE", r"else", None, None),
    ("ENUM", r"enum", None, UNTIL_SPACE),
    ("EXTERN", r"extern", None, UNTIL_SPACE),
    ("FLOAT", r"float", None, UNTIL_SPACE),
    ("FOR", r"for", None, None),
    ("GOTO", r"goto", None, UNTIL_SPACE),
    ("IF", r"if", None, None),
    ("INT", r"int\s", None, None),
    ("LONG", r"long", None, None),
    ("REGISTER", r"register", None, None),
    ("RETURN", r"return", None, None),
    ("SHORT", r"short", None, None),
    ("SIGNED", r"signed", None, None),
    ("SIZEOF", r"sizeof", None, None),
    ("STATIC", r"static", None, None),
    ("STRUCT", r"struct", None, None),
    ("SWITCH", r"switch", None, None),
    ("TYPEDEF", r"typedef", None, None),
    ("UNION", r"union", None, None),
    ("UNSIGNED", r"unsigned", None, None),
    ("VOID", r"void", None, None),
    ("VOLATILE", r"volatile", None, None),
    ("WHILE", r"while", None, None),
    ("ID", r"[a-zA-Z]\w*", None, None),
    ("INTEGRAL_LITERAL", r"\d+", None, MATCH_UNTIL_SEQ % r"\s|\+|\-|\*|\;|$|\)"),
    ("FLOAT_LITERAL", r"\d*\.?\d+", None, MATCH_UNTIL_SEQ % r"\s|\+|\-|\*|\;|$|\)"),
    ("STRING_LITERAL", r'\".*\"', None, None),
    ("CHAR_LITERAL", r"\'.?\'", None, None),
    ("LEFT_CURLY_B", r"\{", None, None),
    ("RIGHT_CURLY_B", r"\}", None, None),
    ("LEFT_SQUARE_B", r"\[", None, None),
    ("RIGHT_SQUARE_B", r"\]", None, None),
    ("LEFT_ROUND_B", r"\(", None, None),
    ("RIGHT_ROUND_B", r"\)", None, None),
    ("COMMA", r"\,", None, None),
    ("SEMICOLON", r"\;", None, None),
    ("DOT", r"\.", None, None),
    ("NOT", r"\!", None, None),
    ("ASSIGN_OPERATOR", r"\=", None, None),
    ("PREPROCESSOR", r"\#", None, None),
    ("BACKWARD_SLASH", r"\\", None, None),
    ("MINUS", r"\-", None, None),
    ("PLUS", r"\+", None, None),
    ("ASTERICK", r"\*", None, None),
    ("DIVIDE", r"\/[^\/]", None, None),
    ("MOD", r"\%", None, None),
    ("LESSTHAN", r"\>", None, None),
    ("GREATERTHAN", r"\<", None, None),
    ("LESS_EQ", r"\=\>", None, None),
    ("GREAT_EQ", r"\=\<", None, None),
    ("EQUAL", r"\=\=", None, None),
    ("NOT_EQUAL", r"\!\=", None, None),
    ("AND", r"\&\&", None, None),
    ("OR", r"\|\|", None, None),
    ("BITWISE_AND", r"\&", None, None),
    ("BITWISE_OR", r"\|", None, None),
    ("BITWISE_XOR", r"\^", None, None),
    ("LEFT_SHIFT", r"\>\>", None, None),
    ("RIGHT_SHIFT", r"\<\<", None, None),
    ("BITWISE_NOT", r"\~", None, None),
    ("MULTI_COMMENT", r"\/\*.*\*\/", None, None),
    ("SINGLE_COMMENT", r"\/\/.*", None, None),
    ('SKIP', r'[ \t]+', None, None),  # Skip over spaces and tabs
    ("MISMATCH", r".+\s", None, None),
]