Add Lexer for Sourcepawn (#804)

alecthomas · Jul 21, 2023 · 555bb53 · 555bb53
1 parent b8c853d
commit 555bb53
Show file tree

Hide file tree

Showing 3 changed files with 186 additions and 0 deletions.
diff --git a/lexers/embedded/sourcepawn.xml b/lexers/embedded/sourcepawn.xml
@@ -0,0 +1,59 @@
+<lexer>
+  <config>
+    <name>SourcePawn</name>
+    <alias>sp</alias>
+    <filename>*.sp</filename>
+    <filename>*.inc</filename>
+    <mime_type>text/x-sourcepawn</mime_type>
+  </config>
+  <rules>
+    <state name="root">
+      <rule pattern="^#if\s+0"><token type="CommentPreproc"/><push state="if0"/></rule>
+      <rule pattern="^#"><token type="CommentPreproc"/><push state="macro"/></rule>
+      <rule pattern="^\s*(?:/[*].*?[*]/\s*)*#if\s+0"><token type="CommentPreproc"/><push state="if0"/></rule>
+      <rule pattern="^\s*(?:/[*].*?[*]/\s*)*#"><token type="CommentPreproc"/><push state="macro"/></rule>
+      <rule pattern="\n"><token type="Text"/></rule>
+      <rule pattern="\s+"><token type="Text"/></rule>
+      <rule pattern="\\\n"><token type="Text"/></rule>
+      <rule pattern="/(\\\n)?/(\n|(.|\n)*?[^\\]\n)"><token type="CommentSingle"/></rule>
+      <rule pattern="/(\\\n)?\*(.|\n)*?\*(\\\n)?/"><token type="CommentMultiline"/></rule>
+      <rule pattern="[{}]"><token type="Punctuation"/></rule>
+      <rule pattern="L?&quot;"><token type="LiteralString"/><push state="string"/></rule>
+      <rule pattern="L?&#x27;(\\.|\\[0-7]{1,3}|\\x[a-fA-F0-9]{1,2}|[^\\\&#x27;\n])&#x27;"><token type="LiteralStringChar"/></rule>
+      <rule pattern="(\d+\.\d*|\.\d+|\d+)[eE][+-]?\d+[LlUu]*"><token type="LiteralNumberFloat"/></rule>
+      <rule pattern="(\d+\.\d*|\.\d+|\d+[fF])[fF]?"><token type="LiteralNumberFloat"/></rule>
+      <rule pattern="0x[0-9a-fA-F]+[LlUu]*"><token type="LiteralNumberHex"/></rule>
+      <rule pattern="0[0-7]+[LlUu]*"><token type="LiteralNumberOct"/></rule>
+      <rule pattern="\d+[LlUu]*"><token type="LiteralNumberInteger"/></rule>
+      <rule pattern="[~!%^&amp;*+=|?:&lt;&gt;/-]"><token type="Operator"/></rule>
+      <rule pattern="[()\[\],.;]"><token type="Punctuation"/></rule>
+      <rule pattern="(case|const|continue|native|default|else|enum|for|if|new|operator|public|return|sizeof|static|decl|struct|switch)\b"><token type="Keyword"/></rule>
+      <rule pattern="(bool|float|void|int|char)\b"><token type="KeywordType"/></rule>
+      <rule pattern="(true|false)\b"><token type="KeywordConstant"/></rule>
+      <rule pattern="[a-zA-Z_]\w*"><token type="Name"/></rule>
+      <rule pattern="((?:[\w*\s])+?(?:\s|[*]))([a-zA-Z_]\w*)(\s*\([^;]*?\))([^;{]*)(\{)"><bygroups><usingself state="root"/><token type="NameFunction"/><usingself state="root"/><usingself state="root"/><token type="Punctuation"/></bygroups><push state="function"/></rule>
+      <rule pattern="((?:[\w*\s])+?(?:\s|[*]))([a-zA-Z_]\w*)(\s*\([^;]*?\))([^;]*)(;)"><bygroups><usingself state="root"/><token type="NameFunction"/><usingself state="root"/><usingself state="root"/><token type="Punctuation"/></bygroups></rule>
+    </state>
+    <state name="string">
+      <rule pattern="&quot;"><token type="LiteralString"/><pop depth="1"/></rule>
+      <rule pattern="\\([\\abfnrtv&quot;\&#x27;]|x[a-fA-F0-9]{2,4}|[0-7]{1,3})"><token type="LiteralStringEscape"/></rule>
+      <rule pattern="[^\\&quot;\n]+"><token type="LiteralString"/></rule>
+      <rule pattern="\\\n"><token type="LiteralString"/></rule>
+      <rule pattern="\\"><token type="LiteralString"/></rule>
+    </state>
+    <state name="macro">
+      <rule pattern="(include)(\s*(?:/[*].*?[*]/\s*)?)([^\n]+)"><bygroups><token type="CommentPreproc"/><token type="Text"/><token type="CommentPreprocFile"/></bygroups></rule>
+      <rule pattern="[^/\n]+"><token type="CommentPreproc"/></rule>
+      <rule pattern="/\*(.|\n)*?\*/"><token type="CommentMultiline"/></rule>
+      <rule pattern="//.*?\n"><token type="CommentSingle"/><pop depth="1"/></rule>
+      <rule pattern="/"><token type="CommentPreproc"/></rule>
+      <rule pattern="(?&lt;=\\)\n"><token type="CommentPreproc"/></rule>
+      <rule pattern="\n"><token type="CommentPreproc"/><pop depth="1"/></rule>
+    </state>
+    <state name="if0">
+      <rule pattern="^\s*#if.*?(?&lt;!\\)\n"><token type="CommentPreproc"/><push/></rule>
+      <rule pattern="^\s*#endif.*?(?&lt;!\\)\n"><token type="CommentPreproc"/><pop depth="1"/></rule>
+      <rule pattern=".*?\n"><token type="Comment"/></rule>
+    </state>
+  </rules>
+</lexer>
diff --git a/lexers/testdata/sourcepawn.actual b/lexers/testdata/sourcepawn.actual
@@ -0,0 +1,22 @@
+#include "a"
+#include <b>
+
+public void foo();
+
+enum class E { A, B };
+
+enum class {
+  a, b,
+  c,
+} e;
+
+enum E { A, B };
+
+class A {
+  void foo();
+  void bar();
+};
+
+int main() {
+    return 0 + 13 + 1.4;
+}
diff --git a/lexers/testdata/sourcepawn.expected b/lexers/testdata/sourcepawn.expected
@@ -0,0 +1,105 @@
+[
+  {"type":"CommentPreproc","value":"#include"},
+  {"type":"Text","value":" "},
+  {"type":"CommentPreprocFile","value":"\"a\""},
+  {"type":"CommentPreproc","value":"\n#include"},
+  {"type":"Text","value":" "},
+  {"type":"CommentPreprocFile","value":"\u003cb\u003e"},
+  {"type":"CommentPreproc","value":"\n"},
+  {"type":"Text","value":"\n"},
+  {"type":"Keyword","value":"public"},
+  {"type":"Text","value":" "},
+  {"type":"KeywordType","value":"void"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"foo"},
+  {"type":"Punctuation","value":"();"},
+  {"type":"Text","value":"\n\n"},
+  {"type":"Keyword","value":"enum"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"class"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"E"},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":"{"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"A"},
+  {"type":"Punctuation","value":","},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"B"},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":"};"},
+  {"type":"Text","value":"\n\n"},
+  {"type":"Keyword","value":"enum"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"class"},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":"{"},
+  {"type":"Text","value":"\n  "},
+  {"type":"Name","value":"a"},
+  {"type":"Punctuation","value":","},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"b"},
+  {"type":"Punctuation","value":","},
+  {"type":"Text","value":"\n  "},
+  {"type":"Name","value":"c"},
+  {"type":"Punctuation","value":","},
+  {"type":"Text","value":"\n"},
+  {"type":"Punctuation","value":"}"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"e"},
+  {"type":"Punctuation","value":";"},
+  {"type":"Text","value":"\n\n"},
+  {"type":"Keyword","value":"enum"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"E"},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":"{"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"A"},
+  {"type":"Punctuation","value":","},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"B"},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":"};"},
+  {"type":"Text","value":"\n\n"},
+  {"type":"Name","value":"class"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"A"},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":"{"},
+  {"type":"Text","value":"\n  "},
+  {"type":"KeywordType","value":"void"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"foo"},
+  {"type":"Punctuation","value":"();"},
+  {"type":"Text","value":"\n  "},
+  {"type":"KeywordType","value":"void"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"bar"},
+  {"type":"Punctuation","value":"();"},
+  {"type":"Text","value":"\n"},
+  {"type":"Punctuation","value":"};"},
+  {"type":"Text","value":"\n\n"},
+  {"type":"KeywordType","value":"int"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"main"},
+  {"type":"Punctuation","value":"()"},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":"{"},
+  {"type":"Text","value":"\n    "},
+  {"type":"Keyword","value":"return"},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberInteger","value":"0"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"+"},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberInteger","value":"13"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"+"},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberFloat","value":"1.4"},
+  {"type":"Punctuation","value":";"},
+  {"type":"Text","value":"\n"},
+  {"type":"Punctuation","value":"}"},
+  {"type":"Text","value":"\n"}
+]