Lexer.fu

// Lexer.fu - Fusion lexer
//
// Copyright (C) 2011-2024  Piotr Fusik
//
// This file is part of Fusion Transpiler,
// see https://github.com/fusionlanguage/fut
//
// Fusion Transpiler is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// Fusion Transpiler is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with Fusion Transpiler.  If not, see http://www.gnu.org/licenses/

public abstract class FuParserHost
{
	internal FuProgram! Program;

	public abstract void ReportError!(string filename, int line, int startUtf16Column, int endUtf16Column, string message);

	internal void ReportStatementError!(FuStatement statement, string message)
	{
		int line = this.Program.GetLine(statement.Loc);
		int column = statement.Loc - this.Program.LineLocs[line];
		FuSourceFile file = this.Program.GetSourceFile(line);
		ReportError(file.Filename, line - file.Line, column, column + statement.GetLocLength(), message);
	}
}

public enum FuToken
{
	EndOfFile,
	Id,
	LiteralLong,
	LiteralDouble,
	LiteralChar,
	LiteralString,
	InterpolatedString,
	Semicolon,
	Dot,
	Comma,
	LeftParenthesis,
	RightParenthesis,
	LeftBracket,
	RightBracket,
	LeftBrace,
	RightBrace,
	Plus,
	Minus,
	Asterisk,
	Slash,
	Mod,
	And,
	Or,
	Xor,
	Tilde,
	ShiftLeft,
	ShiftRight,
	Equal,
	NotEqual,
	Less,
	LessOrEqual,
	Greater,
	GreaterOrEqual,
	RightAngle,
	CondAnd,
	CondOr,
	ExclamationMark,
	Hash,
	Assign,
	AddAssign,
	SubAssign,
	MulAssign,
	DivAssign,
	ModAssign,
	AndAssign,
	OrAssign,
	XorAssign,
	ShiftLeftAssign,
	ShiftRightAssign,
	Increment,
	Decrement,
	QuestionMark,
	Colon,
	FatArrow,
	Range,
	DocRegular,
	DocBullet,
	DocBlank,
	Abstract,
	Assert,
	Break,
	Case,
	Class,
	Const,
	Continue,
	Default,
	Do,
	Else,
	Enum,
	False,
	For,
	Foreach,
	If,
	In,
	Internal,
	Is,
	Lock_,
	Native,
	New,
	Null,
	Override,
	Protected,
	Public,
	Resource,
	Return,
	Sealed,
	Static,
	Switch,
	Throw,
	Throws,
	True,
	Virtual,
	Void,
	When,
	While,
	EndOfLine,
	PreUnknown,
	PreIf,
	PreElIf,
	PreElse,
	PreEndIf
}

enum FuPreState
{
	NotYet,
	Already,
	AlreadyElse
}

public abstract class FuLexer
{
	protected byte[] Input;
	int InputLength;
	int NextOffset;
	protected int CharOffset;
	int NextChar;
	protected FuParserHost! Host;
	protected int Loc = 0;
	protected int TokenLoc;
	protected int LexemeOffset;
	protected FuToken CurrentToken;
	protected long LongValue; // for FuToken.LiteralLong, FuToken.LiteralChar
	protected string() StringValue; // for FuToken.LiteralString, FuToken.InterpolatedString, FuToken.Id
	HashSet<string()>() PreSymbols;
	bool AtLineStart = true;
	bool LineMode = false;
	bool SkippingUnmet = false;
	protected bool ParsingTypeArg = false;
	Stack<bool>() PreElseStack;

	public void SetHost!(FuParserHost! host)
	{
		this.Host = host;
	}

	public void AddPreSymbol!(string symbol)
	{
		this.PreSymbols.Add(symbol);
	}

	protected void Open!(string filename, byte[] input, int inputLength)
	{
		this.Input = input;
		this.InputLength = inputLength;
		this.NextOffset = 0;
		this.Host.Program.SourceFiles.Add();
		this.Host.Program.SourceFiles.Last().Filename = filename;
		this.Host.Program.SourceFiles.Last().Line = this.Host.Program.LineLocs.Count;
		this.Host.Program.LineLocs.Add(this.Loc);
		FillNextChar();
		if (this.NextChar == 0xfeff) // BOM
			FillNextChar();
		NextToken();
	}

	protected void ReportError(string message)
	{
		FuSourceFile file = this.Host.Program.SourceFiles.Last();
		int line = this.Host.Program.LineLocs.Count - file.Line - 1;
		int lineLoc = this.Host.Program.LineLocs.Last();
		this.Host.ReportError(file.Filename, line, this.TokenLoc - lineLoc, this.Loc - lineLoc, message);
	}

	int ReadByte!()
	{
		if (this.NextOffset >= this.InputLength)
			return -1;
		return this.Input[this.NextOffset++];
	}

	const int ReplacementChar = 0xfffd;

	int ReadContinuationByte!(int hi)
	{
		int b = ReadByte();
		if (hi != ReplacementChar) {
			if (b >= 0x80 && b <= 0xbf)
				return (hi << 6) + b - 0x80;
			ReportError("Invalid UTF-8");
		}
		return ReplacementChar;
	}

	void FillNextChar!()
	{
		this.CharOffset = NextOffset;
		int b = ReadByte();
		if (b >= 0x80) {
			if (b < 0xc2 || b > 0xf4) {
				ReportError("Invalid UTF-8");
				b = ReplacementChar;
			}
			else if (b < 0xe0)
				b = ReadContinuationByte(b - 0xc0);
			else if (b < 0xf0) {
				b = ReadContinuationByte(b - 0xe0);
				b = ReadContinuationByte(b);
			}
			else {
				b = ReadContinuationByte(b - 0xf0);
				b = ReadContinuationByte(b);
				b = ReadContinuationByte(b);
			}
		}
		this.NextChar = b;
	}

	protected int PeekChar() => this.NextChar;

	internal static bool IsLetterOrDigit(int c)
	{
		if (c >= 'a' && c <= 'z') return true;
		if (c >= 'A' && c <= 'Z') return true;
		if (c >= '0' && c <= '9') return true;
		return c == '_';
	}

	protected int ReadChar!()
	{
		int c = this.NextChar;
		switch (c) {
		case '\t':
		case ' ':
			this.Loc++;
			break;
		case '\n':
			this.Host.Program.LineLocs.Add(this.Loc);
			this.AtLineStart = true;
			break;
		default:
			this.Loc += c < 0x10000 ? 1 : 2;
			this.AtLineStart = false;
			break;
		}
		FillNextChar();
		return c;
	}

	bool EatChar!(int c)
	{
		if (PeekChar() == c) {
			ReadChar();
			return true;
		}
		return false;
	}

	void SkipWhitespace!()
	{
		while (PeekChar() == '\t' || PeekChar() == ' ' || PeekChar() == '\r')
			ReadChar();
	}

	FuToken ReadIntegerLiteral!(int bits)
	{
		bool invalidDigit = false;
		bool tooBig = false;
		bool needDigit = true;
		for (long i = 0;; ReadChar()) {
			int c = PeekChar();
			if (c >= '0' && c <= '9')
				c -= '0';
			else if (c >= 'A' && c <= 'Z')
				c -= 'A' - 10;
			else if (c >= 'a' && c <= 'z')
				c -= 'a' - 10;
			else if (c == '_') {
				needDigit = true;
				continue;
			}
			else {
				this.LongValue = i;
				if (invalidDigit || needDigit)
					ReportError("Invalid integer");
				else if (tooBig)
					ReportError("Integer too big");
				return FuToken.LiteralLong;
			}
			if (c >= 1 << bits)
				invalidDigit = true;
			else if (i >> (64 - bits) != 0)
				tooBig = true;
			else
				i = (i << bits) + c;
			needDigit = false;
		}
	}

	FuToken ReadFloatLiteral!(bool needDigit)
	{
		bool underscoreE = false;
		bool exponent = false;
		for (;;) {
			int c = PeekChar();
			switch (c) {
			case '0':
			case '1':
			case '2':
			case '3':
			case '4':
			case '5':
			case '6':
			case '7':
			case '8':
			case '9':
				ReadChar();
				needDigit = false;
				break;
			case 'E':
			case 'e':
				if (exponent) {
					ReportError("Invalid floating-point number");
					return FuToken.LiteralDouble;
				}
				if (needDigit)
					underscoreE = true;
				ReadChar();
				c = PeekChar();
				if (c == '+' || c == '-')
					ReadChar();
				exponent = true;
				needDigit = true;
				break;
			case '_':
				ReadChar();
				needDigit = true;
				break;
			default:
				if (underscoreE
				 || needDigit
				 || (c >= 'A' && c <= 'Z')
				 || (c >= 'a' && c <= 'z'))
					ReportError("Invalid floating-point number");
				return FuToken.LiteralDouble;
			}
		}
	}

	FuToken ReadNumberLiteral!(long i)
	{
		bool leadingZero = false;
		bool tooBig = false;
		for (bool needDigit = false; ; ReadChar()) {
			int c = PeekChar();
			switch (c) {
			case '0':
			case '1':
			case '2':
			case '3':
			case '4':
			case '5':
			case '6':
			case '7':
			case '8':
			case '9':
				c -= '0';
				break;
			case '.':
				ReadChar();
				return ReadFloatLiteral(true);
			case 'e':
			case 'E':
				return ReadFloatLiteral(needDigit);
			case '_':
				needDigit = true;
				continue;
			default:
				this.LongValue = i;
				if (leadingZero)
					ReportError("Leading zeros are not permitted, octal numbers must begin with 0o");
				if (needDigit
				 || (c >= 'A' && c <= 'Z')
				 || (c >= 'a' && c <= 'z'))
					ReportError("Invalid integer");
				else if (tooBig)
					ReportError("Integer too big");
				return FuToken.LiteralLong;
			}
			if (i == 0)
				leadingZero = true;
			if (i > (c < 8 ? 922337203685477580 : 922337203685477579))
				tooBig = true;
			else
				i = 10 * i + c;
			needDigit = false;
		}
	}

	internal static int GetEscapedChar(int c)
	{
		switch (c) {
		case '"': return '"';
		case '\'': return '\'';
		case '\\': return '\\';
		case 'n': return '\n';
		case 'r': return '\r';
		case 't': return '\t';
		default: return -1;
		}
	}

	int ReadCharLiteral!()
	{
		int c = ReadChar();
		if (c < 32) {
			ReportError("Invalid character in literal");
			return ReplacementChar;
		}
		if (c != '\\')
			return c;
		c = GetEscapedChar(ReadChar());
		if (c < 0) {
			ReportError("Unknown escape sequence");
			return ReplacementChar;
		}
		return c;
	}

	protected FuToken ReadString!(bool interpolated)
	{
		for (int offset = this.CharOffset;;) {
			switch (PeekChar()) {
			case -1:
				ReportError("Unterminated string literal");
				return FuToken.EndOfFile;
			case '\n':
				ReportError("Unterminated string literal");
				this.StringValue = "";
				return FuToken.LiteralString;
			case '"':
				{
					int endOffset = this.CharOffset;
					ReadChar();
					this.StringValue = Encoding.UTF8.GetString(this.Input, offset, endOffset - offset);
				}
				return FuToken.LiteralString;
			case '{':
				if (interpolated) {
					int endOffset = this.CharOffset;
					ReadChar();
					if (EatChar('{'))
						break;
					if (!this.SkippingUnmet) {
						this.StringValue = Encoding.UTF8.GetString(this.Input, offset, endOffset - offset);
						return FuToken.InterpolatedString;
					}
					for (;;) {
						FuToken token = ReadPreToken();
						if (token == FuToken.RightBrace)
							break;
						if (token == FuToken.EndOfFile) {
							ReportError("Unterminated string literal");
							return FuToken.EndOfFile;
						}
					}
				}
				else
					ReadChar();
				break;
			default:
				ReadCharLiteral();
				break;
			}
		}
	}

	bool EndWord!(int c) => EatChar(c) && !IsLetterOrDigit(PeekChar());

	protected string() GetLexeme() => Encoding.UTF8.GetString(this.Input, this.LexemeOffset, this.CharOffset - this.LexemeOffset);

	FuToken ReadPreToken!()
	{
		for (;;) {
			bool atLineStart = this.AtLineStart;
			this.TokenLoc = this.Loc;
			this.LexemeOffset = this.CharOffset;
			int c = ReadChar();
			switch (c) {
			case -1:
				return FuToken.EndOfFile;
			case '\t':
			case '\r':
			case ' ':
				break;
			case '\n':
				if (this.LineMode)
					return FuToken.EndOfLine;
				break;
			case '#':
				if (!atLineStart)
					return FuToken.Hash;
				switch (PeekChar()) {
				case 'i':
					ReadChar();
					return EndWord('f') ? FuToken.PreIf : FuToken.PreUnknown;
				case 'e':
					ReadChar();
					switch (PeekChar()) {
					case 'l':
						ReadChar();
						switch (PeekChar()) {
						case 'i':
							ReadChar();
							return EndWord('f') ? FuToken.PreElIf : FuToken.PreUnknown;
						case 's':
							ReadChar();
							return EndWord('e') ? FuToken.PreElse : FuToken.PreUnknown;
						default:
							return FuToken.PreUnknown;
						}
					case 'n':
						ReadChar();
						return EatChar('d') && EatChar('i') && EndWord('f') ? FuToken.PreEndIf : FuToken.PreUnknown;
					default:
						return FuToken.PreUnknown;
					}
				default:
					return FuToken.PreUnknown;
				}
			case ';': return FuToken.Semicolon;
			case '.':
				if (EatChar('.')) return FuToken.Range;
				return FuToken.Dot;
			case ',': return FuToken.Comma;
			case '(': return FuToken.LeftParenthesis;
			case ')': return FuToken.RightParenthesis;
			case '[': return FuToken.LeftBracket;
			case ']': return FuToken.RightBracket;
			case '{': return FuToken.LeftBrace;
			case '}': return FuToken.RightBrace;
			case '~': return FuToken.Tilde;
			case '?': return FuToken.QuestionMark;
			case ':': return FuToken.Colon;
			case '+':
				if (EatChar('+')) return FuToken.Increment;
				if (EatChar('=')) return FuToken.AddAssign;
				return FuToken.Plus;
			case '-':
				if (EatChar('-')) return FuToken.Decrement;
				if (EatChar('=')) return FuToken.SubAssign;
				return FuToken.Minus;
			case '*':
				if (EatChar('=')) return FuToken.MulAssign;
				return FuToken.Asterisk;
			case '/':
				if (EatChar('/')) {
					c = ReadChar();
					if (c == '/' && !this.SkippingUnmet) {
						SkipWhitespace();
						switch (PeekChar()) {
						case '\n':
							return FuToken.DocBlank;
						case '*':
							ReadChar();
							SkipWhitespace();
							return FuToken.DocBullet;
						default:
							return FuToken.DocRegular;
						}
					}
					while (c != '\n' && c >= 0)
						c = ReadChar();
					if (c == '\n' && this.LineMode) return FuToken.EndOfLine;
					break;
				}
				if (EatChar('*')) {
					int startLine = this.Host.Program.LineLocs.Count;
					do {
						c = ReadChar();
						if (c < 0) {
							ReportError($"Unterminated multi-line comment, started in line {startLine}");
							return FuToken.EndOfFile;
						}
					} while (c != '*' || PeekChar() != '/');
					ReadChar();
					break;
				}
				if (EatChar('=')) return FuToken.DivAssign;
				return FuToken.Slash;
			case '%':
				if (EatChar('=')) return FuToken.ModAssign;
				return FuToken.Mod;
			case '&':
				if (EatChar('&')) return FuToken.CondAnd;
				if (EatChar('=')) return FuToken.AndAssign;
				return FuToken.And;
			case '|':
				if (EatChar('|')) return FuToken.CondOr;
				if (EatChar('=')) return FuToken.OrAssign;
				return FuToken.Or;
			case '^':
				if (EatChar('=')) return FuToken.XorAssign;
				return FuToken.Xor;
			case '=':
				if (EatChar('=')) return FuToken.Equal;
				if (EatChar('>')) return FuToken.FatArrow;
				return FuToken.Assign;
			case '!':
				if (EatChar('=')) return FuToken.NotEqual;
				return FuToken.ExclamationMark;
			case '<':
				if (EatChar('<')) {
					if (EatChar('=')) return FuToken.ShiftLeftAssign;
					return FuToken.ShiftLeft;
				}
				if (EatChar('=')) return FuToken.LessOrEqual;
				return FuToken.Less;
			case '>':
				if (this.ParsingTypeArg) return FuToken.RightAngle;
				if (EatChar('>')) {
					if (EatChar('=')) return FuToken.ShiftRightAssign;
					return FuToken.ShiftRight;
				}
				if (EatChar('=')) return FuToken.GreaterOrEqual;
				return FuToken.Greater;
			case '\'':
				if (PeekChar() == '\'') {
					ReportError("Empty character literal");
					this.LongValue = 0;
				}
				else
					this.LongValue = ReadCharLiteral();
				if (!EatChar('\''))
					ReportError("Unterminated character literal");
				return FuToken.LiteralChar;
			case '"':
				return ReadString(false);
			case '$':
				if (EatChar('"'))
					return ReadString(true);
				ReportError("Expected interpolated string");
				break;
			case '0':
				switch (PeekChar()) {
				case 'B':
				case 'b':
					ReadChar();
					return ReadIntegerLiteral(1);
				case 'O':
				case 'o':
					ReadChar();
					return ReadIntegerLiteral(3);
				case 'X':
				case 'x':
					ReadChar();
					return ReadIntegerLiteral(4);
				default:
					return ReadNumberLiteral(0);
				}
			case '1':
			case '2':
			case '3':
			case '4':
			case '5':
			case '6':
			case '7':
			case '8':
			case '9':
				return ReadNumberLiteral(c - '0');
			default:
				if (!IsLetterOrDigit(c)) {
					ReportError("Invalid character");
					continue;
				}
				while (IsLetterOrDigit(PeekChar()))
					ReadChar();
				this.StringValue = GetLexeme();
				switch (this.StringValue) {
				case "abstract": return FuToken.Abstract;
				case "assert": return FuToken.Assert;
				case "break": return FuToken.Break;
				case "case": return FuToken.Case;
				case "class": return FuToken.Class;
				case "const": return FuToken.Const;
				case "continue": return FuToken.Continue;
				case "default": return FuToken.Default;
				case "do": return FuToken.Do;
				case "else": return FuToken.Else;
				case "enum": return FuToken.Enum;
				case "false": return FuToken.False;
				case "for": return FuToken.For;
				case "foreach": return FuToken.Foreach;
				case "if": return FuToken.If;
				case "in": return FuToken.In;
				case "internal": return FuToken.Internal;
				case "is": return FuToken.Is;
				case "lock": return FuToken.Lock_;
				case "native": return FuToken.Native;
				case "new": return FuToken.New;
				case "null": return FuToken.Null;
				case "override": return FuToken.Override;
				case "protected": return FuToken.Protected;
				case "public": return FuToken.Public;
				case "resource": return FuToken.Resource;
				case "return": return FuToken.Return;
				case "sealed": return FuToken.Sealed;
				case "static": return FuToken.Static;
				case "switch": return FuToken.Switch;
				case "throw": return FuToken.Throw;
				case "throws": return FuToken.Throws;
				case "true": return FuToken.True;
				case "virtual": return FuToken.Virtual;
				case "void": return FuToken.Void;
				case "when": return FuToken.When;
				case "while": return FuToken.While;
				default: return FuToken.Id;
				}
			}
		}
	}

	void NextPreToken!()
	{
		this.CurrentToken = ReadPreToken();
	}

	protected bool See(FuToken token) => this.CurrentToken == token;

	internal static string TokenToString(FuToken token)
	{
		switch (token) {
		case FuToken.EndOfFile: return "end-of-file";
		case FuToken.Id: return "identifier";
		case FuToken.LiteralLong: return "integer constant";
		case FuToken.LiteralDouble: return "floating-point constant";
		case FuToken.LiteralChar: return "character constant";
		case FuToken.LiteralString: return "string constant";
		case FuToken.InterpolatedString: return "interpolated string";
		case FuToken.Semicolon: return "';'";
		case FuToken.Dot: return "'.'";
		case FuToken.Comma: return "','";
		case FuToken.LeftParenthesis: return "'('";
		case FuToken.RightParenthesis: return "')'";
		case FuToken.LeftBracket: return "'['";
		case FuToken.RightBracket: return "']'";
		case FuToken.LeftBrace: return "'{'";
		case FuToken.RightBrace: return "'}'";
		case FuToken.Plus: return "'+'";
		case FuToken.Minus: return "'-'";
		case FuToken.Asterisk: return "'*'";
		case FuToken.Slash: return "'/'";
		case FuToken.Mod: return "'%'";
		case FuToken.And: return "'&'";
		case FuToken.Or: return "'|'";
		case FuToken.Xor: return "'^'";
		case FuToken.Tilde: return "'~'";
		case FuToken.ShiftLeft: return "'<<'";
		case FuToken.ShiftRight: return "'>>'";
		case FuToken.Equal: return "'=='";
		case FuToken.NotEqual: return "'!='";
		case FuToken.Less: return "'<'";
		case FuToken.LessOrEqual: return "'<='";
		case FuToken.Greater: return "'>'";
		case FuToken.GreaterOrEqual: return "'>='";
		case FuToken.RightAngle: return "'>'";
		case FuToken.CondAnd: return "'&&'";
		case FuToken.CondOr: return "'||'";
		case FuToken.ExclamationMark: return "'!'";
		case FuToken.Hash: return "'#'";
		case FuToken.Assign: return "'='";
		case FuToken.AddAssign: return "'+='";
		case FuToken.SubAssign: return "'-='";
		case FuToken.MulAssign: return "'*='";
		case FuToken.DivAssign: return "'/='";
		case FuToken.ModAssign: return "'%='";
		case FuToken.AndAssign: return "'&='";
		case FuToken.OrAssign: return "'|='";
		case FuToken.XorAssign: return "'^='";
		case FuToken.ShiftLeftAssign: return "'<<='";
		case FuToken.ShiftRightAssign: return "'>>='";
		case FuToken.Increment: return "'++'";
		case FuToken.Decrement: return "'--'";
		case FuToken.QuestionMark: return "'?'";
		case FuToken.Colon: return "':'";
		case FuToken.FatArrow: return "'=>'";
		case FuToken.Range: return "'..'";
		case FuToken.DocRegular:
		case FuToken.DocBullet:
		case FuToken.DocBlank:
			return "'///'";
		case FuToken.Abstract: return "'abstract'";
		case FuToken.Assert: return "'assert'";
		case FuToken.Break: return "'break'";
		case FuToken.Case: return "'case'";
		case FuToken.Class: return "'class'";
		case FuToken.Const: return "'const'";
		case FuToken.Continue: return "'continue'";
		case FuToken.Default: return "'default'";
		case FuToken.Do: return "'do'";
		case FuToken.Else: return "'else'";
		case FuToken.Enum: return "'enum'";
		case FuToken.False: return "'false'";
		case FuToken.For: return "'for'";
		case FuToken.Foreach: return "'foreach'";
		case FuToken.If: return "'if'";
		case FuToken.In: return "'in'";
		case FuToken.Internal: return "'internal'";
		case FuToken.Is: return "'is'";
		case FuToken.Lock_: return "'lock'";
		case FuToken.Native: return "'native'";
		case FuToken.New: return "'new'";
		case FuToken.Null: return "'null'";
		case FuToken.Override: return "'override'";
		case FuToken.Protected: return "'protected'";
		case FuToken.Public: return "'public'";
		case FuToken.Resource: return "'resource'";
		case FuToken.Return: return "'return'";
		case FuToken.Sealed: return "'sealed'";
		case FuToken.Static: return "'static'";
		case FuToken.Switch: return "'switch'";
		case FuToken.Throw: return "'throw'";
		case FuToken.Throws: return "'throws'";
		case FuToken.True: return "'true'";
		case FuToken.Virtual: return "'virtual'";
		case FuToken.Void: return "'void'";
		case FuToken.When: return "'when'";
		case FuToken.While: return "'while'";
		case FuToken.EndOfLine: return "end-of-line";
		case FuToken.PreUnknown: return "unknown preprocessor directive";
		case FuToken.PreIf: return "'#if'";
		case FuToken.PreElIf: return "'#elif'";
		case FuToken.PreElse: return "'#else'";
		case FuToken.PreEndIf: return "'#endif'";
		default: assert false;
		}
	}

	protected bool Check(FuToken expected)
	{
		if (See(expected))
			return true;
		ReportError($"Expected {TokenToString(expected)}, got {TokenToString(this.CurrentToken)}");
		return false;
	}

	bool EatPre!(FuToken token)
	{
		if (See(token)) {
			NextPreToken();
			return true;
		}
		return false;
	}

	bool ParsePrePrimary!()
	{
		if (EatPre(FuToken.ExclamationMark))
			return !ParsePrePrimary();
		if (EatPre(FuToken.LeftParenthesis)) {
			bool result = ParsePreOr();
			Check(FuToken.RightParenthesis);
			NextPreToken();
			return result;
		}
		if (See(FuToken.Id)) {
			bool result = this.PreSymbols.Contains(this.StringValue);
			NextPreToken();
			return result;
		}
		if (EatPre(FuToken.False))
			return false;
		if (EatPre(FuToken.True))
			return true;
		ReportError("Invalid preprocessor expression");
		return false;
	}

	bool ParsePreEquality!()
	{
		bool result = ParsePrePrimary();
		for (;;) {
			if (EatPre(FuToken.Equal))
				result = result == ParsePrePrimary();
			else if (EatPre(FuToken.NotEqual))
				result ^= ParsePrePrimary();
			else
				return result;
		}
	}

	bool ParsePreAnd!()
	{
		bool result = ParsePreEquality();
		while (EatPre(FuToken.CondAnd))
			result &= ParsePreEquality();
		return result;
	}

	bool ParsePreOr!()
	{
		bool result = ParsePreAnd();
		while (EatPre(FuToken.CondOr))
			result |= ParsePreAnd();
		return result;
	}

	bool ParsePreExpr!()
	{
		this.LineMode = true;
		NextPreToken();
		bool result = ParsePreOr();
		Check(FuToken.EndOfLine);
		this.LineMode = false;
		return result;
	}

	void ExpectEndOfLine!(string directive)
	{
		this.LineMode = true;
		FuToken token = ReadPreToken();
		if (token != FuToken.EndOfLine && token != FuToken.EndOfFile)
			ReportError($"Unexpected characters after '{directive}'");
		this.LineMode = false;
	}

	bool PopPreElse!(string directive)
	{
		if (this.PreElseStack.Count == 0) {
			ReportError($"'{directive}' with no matching '#if'");
			return false;
		}
		if (this.PreElseStack.Pop() && directive != "#endif")
			ReportError($"'{directive}' after '#else'");
		return true;
	}

	void SkipUnmet!(FuPreState state)
	{
		this.SkippingUnmet = true;
		for (;;) {
			// state == FuPreState.NotYet: we are in a conditional that wasn't met yet
			// else: we are in a conditional that was met before
			switch (ReadPreToken()) {
			case FuToken.EndOfFile:
				ReportError("Expected '#endif', got end-of-file");
				return;
			case FuToken.PreIf:
				ParsePreExpr();
				SkipUnmet(FuPreState.Already);
				break;
			case FuToken.PreElIf:
				if (state == FuPreState.AlreadyElse)
					ReportError("'#elif' after '#else'");
				if (ParsePreExpr() && state == FuPreState.NotYet) {
					this.PreElseStack.Push(false);
					return;
				}
				break;
			case FuToken.PreElse:
				if (state == FuPreState.AlreadyElse)
					ReportError("'#else' after '#else'");
				ExpectEndOfLine("#else");
				if (state == FuPreState.NotYet) {
					this.PreElseStack.Push(true);
					return;
				}
				state = FuPreState.AlreadyElse;
				break;
			case FuToken.PreEndIf:
				ExpectEndOfLine("#endif");
				return;
			default:
				break;
			}
		}
	}

	FuToken ReadToken!()
	{
		for (;;) {
			// we are in no conditionals or in all met
			this.SkippingUnmet = false;
			FuToken token = ReadPreToken();
			bool matched;
			switch (token) {
			case FuToken.EndOfFile:
				if (this.PreElseStack.Count != 0)
					ReportError("Expected '#endif', got end-of-file");
				return FuToken.EndOfFile;
			case FuToken.PreIf:
				if (ParsePreExpr())
					this.PreElseStack.Push(false);
				else
					SkipUnmet(FuPreState.NotYet);
				break;
			case FuToken.PreElIf:
				matched = PopPreElse("#elif");
				ParsePreExpr();
				if (matched)
					SkipUnmet(FuPreState.Already);
				break;
			case FuToken.PreElse:
				matched = PopPreElse("#else");
				ExpectEndOfLine("#else");
				if (matched)
					SkipUnmet(FuPreState.AlreadyElse);
				break;
			case FuToken.PreEndIf:
				PopPreElse("#endif");
				ExpectEndOfLine("#endif");
				break;
			default:
				return token;
			}
		}
	}

	protected FuToken NextToken!()
	{
		FuToken token = this.CurrentToken;
		this.CurrentToken = ReadToken();
		return token;
	}

	protected bool Eat!(FuToken token)
	{
		if (See(token)) {
			NextToken();
			return true;
		}
		return false;
	}

	protected bool Expect!(FuToken expected)
	{
		bool found = Check(expected);
		NextToken();
		return found;
	}

	protected void ExpectOrSkip!(FuToken expected)
	{
		if (Check(expected))
			NextToken();
		else {
			do
				NextToken();
			while (!See(FuToken.EndOfFile) && !Eat(expected));
		}
	}
}