From 282147552d63d8ef5de1ae2a483a4c1b7ed04a1a Mon Sep 17 00:00:00 2001 From: Seemann Date: Mon, 5 Aug 2024 21:12:26 -0400 Subject: [PATCH] 0.4 (SB 4.0) (#15) --- .cargo/config | 8 - .cargo/config.toml | 6 + .github/workflows/rust.yml | 4 +- .gitignore | 2 + Cargo.lock | 636 ++++++++- Cargo.toml | 12 +- src/dictionary/dictionary_num_by_str.rs | 30 +- src/dictionary/ffi.rs | 5 +- src/dictionary/mod.rs | 2 + src/language_service/ffi.rs | 101 +- src/language_service/scanner.rs | 992 ++++++++++---- src/language_service/server.rs | 204 +-- src/language_service/symbol_table.rs | 43 +- src/legacy_ini/ffi.rs | 33 +- src/legacy_ini/table.rs | 299 +++- src/lib.rs | 8 +- src/namespaces/ffi.rs | 104 +- src/namespaces/library.rs | 147 +- src/namespaces/mod.rs | 9 +- src/namespaces/namespaces.rs | 144 +- src/parser/binary.rs | 40 + src/parser/declaration.rs | 488 ++++++- src/parser/interface.rs | 45 + src/parser/literal.rs | 16 +- src/parser/mod.rs | 34 +- src/parser/unary.rs | 24 + src/parser/variable.rs | 100 +- src/preprocessor/ffi.rs | 100 ++ src/preprocessor/line_parser.rs | 1218 +++++++++++++++++ src/preprocessor/mod.rs | 589 ++++++++ src/preprocessor/scopes.rs | 92 ++ src/preprocessor/test/0.txt | 9 + src/preprocessor/test/addon.txt | 1 + src/preprocessor/test/circular1.txt | 2 + src/preprocessor/test/compiler.ini | 92 ++ src/preprocessor/test/const.txt | 10 + src/preprocessor/test/hex_inc.txt | 3 + src/preprocessor/test/scr_ffi.txt | 15 + src/preprocessor/test/scr_with_func.txt | 19 + src/preprocessor/test/script.txt | 4 + src/preprocessor/test/script_with_include.txt | 6 + src/preprocessor/test/u.txt | 65 + src/sanny_update/ffi.rs | 36 + src/sanny_update/http_client.rs | 22 + src/sanny_update/mod.rs | 380 +++++ src/source_map/ffi.rs | 138 ++ src/source_map/mod.rs | 2 + src/source_map/source_map.rs | 164 +++ src/update_service/service.rs | 206 ++- src/utils/compiler_const.rs | 21 +- src/utils/mod.rs | 2 + src/utils/path.rs | 34 + src/utils/visibility_zone.rs | 12 + src/v4/ffi.rs | 5 +- src/v4/helpers.rs | 4 +- src/v4/mod.rs | 173 ++- src/v4/transform.rs | 190 ++- 57 files changed, 6400 insertions(+), 750 deletions(-) delete mode 100644 .cargo/config create mode 100644 .cargo/config.toml create mode 100644 src/preprocessor/ffi.rs create mode 100644 src/preprocessor/line_parser.rs create mode 100644 src/preprocessor/mod.rs create mode 100644 src/preprocessor/scopes.rs create mode 100644 src/preprocessor/test/0.txt create mode 100644 src/preprocessor/test/addon.txt create mode 100644 src/preprocessor/test/circular1.txt create mode 100644 src/preprocessor/test/compiler.ini create mode 100644 src/preprocessor/test/const.txt create mode 100644 src/preprocessor/test/hex_inc.txt create mode 100644 src/preprocessor/test/scr_ffi.txt create mode 100644 src/preprocessor/test/scr_with_func.txt create mode 100644 src/preprocessor/test/script.txt create mode 100644 src/preprocessor/test/script_with_include.txt create mode 100644 src/preprocessor/test/u.txt create mode 100644 src/sanny_update/ffi.rs create mode 100644 src/sanny_update/http_client.rs create mode 100644 src/sanny_update/mod.rs create mode 100644 src/source_map/ffi.rs create mode 100644 src/source_map/mod.rs create mode 100644 src/source_map/source_map.rs create mode 100644 src/utils/path.rs create mode 100644 src/utils/visibility_zone.rs diff --git a/.cargo/config b/.cargo/config deleted file mode 100644 index 9a4d8a0..0000000 --- a/.cargo/config +++ /dev/null @@ -1,8 +0,0 @@ -[build] -target = "i686-pc-windows-msvc" - -[alias] -b = "build --out-dir=..\\..\\exe\\lib -Z unstable-options" - -[target.i686-pc-windows-msvc] -rustflags = ["-Ctarget-feature=+crt-static", "-Zunstable-options"] diff --git a/.cargo/config.toml b/.cargo/config.toml new file mode 100644 index 0000000..5782722 --- /dev/null +++ b/.cargo/config.toml @@ -0,0 +1,6 @@ +[build] +target = "i686-pc-windows-msvc" + + +[target.i686-pc-windows-msvc] +rustflags = ["-Ctarget-feature=+crt-static", "-Adead_code", "-Aunused"] diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 5ee0229..6573429 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -15,9 +15,9 @@ jobs: steps: - uses: actions/checkout@v2 - - uses: actions-rs/toolchain@v1 + - uses: dtolnay/rust-toolchain@stable with: - toolchain: nightly-2023-05-11 + toolchain: nightly-2023-12-21 target: i686-pc-windows-msvc override: true - name: Build diff --git a/.gitignore b/.gitignore index ea8c4bf..194450b 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,3 @@ /target +b +br \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index f024fe4..bc59ce0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8,6 +8,35 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if 1.0.0", + "cipher", + "cpufeatures", +] + +[[package]] +name = "ahash" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" +dependencies = [ + "cfg-if 1.0.0", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "allocator-api2" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c6cb57a04249c6480766f7f7cef5467412af1490f8d1e243141daddada3264f" + [[package]] name = "android_system_properties" version = "0.1.5" @@ -17,6 +46,12 @@ dependencies = [ "libc", ] +[[package]] +name = "anyhow" +version = "1.0.81" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0952808a6c2afd1aa8947271f3a60f1a6763c7b912d210184c5149b5cf147247" + [[package]] name = "autocfg" version = "1.0.1" @@ -29,12 +64,27 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" +[[package]] +name = "base64ct" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" + [[package]] name = "bitflags" version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + [[package]] name = "bumpalo" version = "3.10.0" @@ -47,11 +97,74 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c39a773ba75db12126d8d383f1bdbf7eb92ea47ec27dd0557aff1fedf172764c" +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bzip2" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdb116a6ef3f6c3698828873ad02c3014b3c85cadb88496095628e3ef1e347f8" +dependencies = [ + "bzip2-sys", + "libc", +] + +[[package]] +name = "bzip2-sys" +version = "0.1.11+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "736a955f3fa7875102d57c82b8cac37ec45224a07fd32d58f9f7a186b6cd4cdc" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + +[[package]] +name = "cached" +version = "0.50.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a7d38ed2761b8a13ce42bc44b09d5a052b88da2f9fead624c779f31ac0729a" +dependencies = [ + "ahash", + "cached_proc_macro", + "cached_proc_macro_types", + "hashbrown 0.14.5", + "instant", + "once_cell", + "thiserror", +] + +[[package]] +name = "cached_proc_macro" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "771aa57f3b17da6c8bcacb187bb9ec9bc81c8160e72342e67c329e0e1651a669" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn 2.0.60", +] + +[[package]] +name = "cached_proc_macro_types" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ade8366b8bd5ba243f0a58f036cc0ca8a2f069cff1a2351ef1cac6b083e16fc0" + [[package]] name = "cc" version = "1.0.73" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2fff2a6927b3bb87f9595d67196a70493f627687a71d87a0d692242c33f58c11" +dependencies = [ + "jobserver", +] [[package]] name = "cfg-if" @@ -75,7 +188,7 @@ dependencies = [ "js-sys", "num-integer", "num-traits", - "time", + "time 0.1.44", "wasm-bindgen", "winapi 0.3.9", ] @@ -86,12 +199,57 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fff857943da45f546682664a79488be82e69e43c1a7a2307679ab9afb3a66d2e" +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common", + "inout", +] + +[[package]] +name = "const_format" +version = "0.2.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a214c7af3d04997541b18d432afaff4c455e79e2029079647e72fc2bd27673" +dependencies = [ + "const_format_proc_macros", +] + +[[package]] +name = "const_format_proc_macros" +version = "0.2.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f6ff08fd20f4f299298a28e2dfa8a8ba1036e6cd2460ac1de7b425d76f2500" +dependencies = [ + "proc-macro2", + "quote", + "unicode-xid", +] + +[[package]] +name = "constant_time_eq" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" + [[package]] name = "core-foundation-sys" version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" +[[package]] +name = "cpufeatures" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504" +dependencies = [ + "libc", +] + [[package]] name = "crc32fast" version = "1.3.2" @@ -101,6 +259,22 @@ dependencies = [ "cfg-if 1.0.0", ] +[[package]] +name = "crossbeam-utils" +version = "0.8.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345" + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + [[package]] name = "ctor" version = "0.1.23" @@ -108,7 +282,59 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cdffe87e1d521a10f9696f833fe502293ea446d7f256c06128293a4119bdf4cb" dependencies = [ "quote", - "syn", + "syn 1.0.99", +] + +[[package]] +name = "darling" +version = "0.20.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54e36fcd13ed84ffdfda6f5be89b31287cbb80c439841fe69e04841435464391" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.20.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c2cf1c23a687a1feeb728783b993c4e1ad83d99f351801977dd809b48d0a70f" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.60", +] + +[[package]] +name = "darling_macro" +version = "0.20.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a668eda54683121533a393014d8692171709ff57a7d61f187b6e782719f8933f" +dependencies = [ + "darling_core", + "quote", + "syn 2.0.60", +] + +[[package]] +name = "deranged" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", + "subtle", ] [[package]] @@ -120,7 +346,7 @@ dependencies = [ "cfg-if 1.0.0", "libc", "redox_syscall", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -133,6 +359,12 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + [[package]] name = "form_urlencoded" version = "1.0.1" @@ -178,12 +410,41 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + [[package]] name = "hashbrown" version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04" +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +dependencies = [ + "ahash", + "allocator-api2", +] + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + [[package]] name = "hotwatch" version = "0.4.6" @@ -207,6 +468,12 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + [[package]] name = "idna" version = "0.2.3" @@ -225,7 +492,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "824845a0bf897a9042383849b02c1bc219c2383772efcd5c6f9766fa4b81aef3" dependencies = [ "autocfg", - "hashbrown", + "hashbrown 0.9.1", ] [[package]] @@ -248,6 +515,24 @@ dependencies = [ "libc", ] +[[package]] +name = "inout" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0c10553d664a4d0bcff9f4215d0aac67a639cc68ef660840afe309b807bc9f5" +dependencies = [ + "generic-array", +] + +[[package]] +name = "instant" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" +dependencies = [ + "cfg-if 1.0.0", +] + [[package]] name = "iovec" version = "0.1.4" @@ -263,6 +548,15 @@ version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd25036021b0de88a0aff6b850051563c6516d0bf53f8638938edbb9de732736" +[[package]] +name = "jobserver" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2b099aaa34a9751c5bf0878add70444e1ed2dd73f347be99003d4577277de6e" +dependencies = [ + "libc", +] + [[package]] name = "js-sys" version = "0.3.59" @@ -360,9 +654,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.132" +version = "0.2.153" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8371e4e5341c3a96db127eb2465ac681ced4c433e01dd0e938adbef26ba93ba5" +checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" [[package]] name = "libloading" @@ -374,6 +668,12 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "lines_lossy" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6acd4dcd36066c7dcbc095912992bf367da54ea30502d865ad81f3a3f19aaac5" + [[package]] name = "log" version = "0.4.17" @@ -485,6 +785,15 @@ dependencies = [ "nom", ] +[[package]] +name = "normpath" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5831952a9476f2fed74b77d74182fa5ddc4d21c72ec45a333b250e3ed0272804" +dependencies = [ + "windows-sys 0.52.0", +] + [[package]] name = "notify" version = "4.0.17" @@ -524,9 +833,32 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.13.0" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18a6dbe30758c9f83eb00cbea4ac95966305f5a7772f3f42ebfc7fc7eddbd8e1" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" + +[[package]] +name = "password-hash" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7676374caaee8a325c9e7a2ae557f216c5563a171d6997b0ef8a65af35147700" +dependencies = [ + "base64ct", + "rand_core", + "subtle", +] + +[[package]] +name = "pbkdf2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83a0692ec44e4cf1ef28ca317f14f8f07da2d95ec3fa01f86e4467b725e60917" +dependencies = [ + "digest", + "hmac", + "password-hash", + "sha2", +] [[package]] name = "percent-encoding" @@ -534,24 +866,36 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" +[[package]] +name = "pkg-config" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" + [[package]] name = "proc-macro2" -version = "1.0.43" +version = "1.0.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a2ca2c61bc9f3d74d2886294ab7b9853abd9c1ad903a3ac7815c58989bb7bab" +checksum = "3d1597b0c024618f09a9c3b8655b7e430397a36d23fdafec26d6965e9eec3eba" dependencies = [ "unicode-ident", ] [[package]] name = "quote" -version = "1.0.21" +version = "1.0.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbe448f377a7d6961e30f5955f9b8d106c3f5e449d493ee1b125c1d43c2b5179" +checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" dependencies = [ "proc-macro2", ] +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" + [[package]] name = "redox_syscall" version = "0.2.16" @@ -605,18 +949,23 @@ dependencies = [ [[package]] name = "sanny_builder_core" -version = "0.3.0" +version = "0.4.0" dependencies = [ + "anyhow", "base64", + "cached", + "const_format", "ctor", "hotwatch", "lazy_static", "lexical-core", "libc", "libloading", + "lines_lossy", "log", "nom", "nom_locate", + "normpath", "serde", "serde_json", "simplelog", @@ -624,6 +973,7 @@ dependencies = [ "version-compare", "version_info", "winapi 0.3.9", + "zip", ] [[package]] @@ -653,7 +1003,7 @@ checksum = "963a7dbc9895aeac7ac90e74f34a5d5261828f79df35cbed41e10189d3804d43" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.99", ] [[package]] @@ -668,6 +1018,28 @@ dependencies = [ "serde", ] +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if 1.0.0", + "cpufeatures", + "digest", +] + +[[package]] +name = "sha2" +version = "0.10.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" +dependencies = [ + "cfg-if 1.0.0", + "cpufeatures", + "digest", +] + [[package]] name = "simplelog" version = "0.11.2" @@ -700,6 +1072,18 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + +[[package]] +name = "subtle" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" + [[package]] name = "syn" version = "1.0.99" @@ -711,6 +1095,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "2.0.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "909518bc7b1c9b779f1bbf07f2929d35af9f0f37e47c6e9ef7f9dddc1e1821f3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "termcolor" version = "1.1.2" @@ -720,6 +1115,26 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "thiserror" +version = "1.0.59" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0126ad08bff79f29fc3ae6a55cc72352056dfff61e3ff8bb7129476d44b23aa" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.59" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1cd413b5d558b4c5bf3680e324a6fa5014e7b7c067a51e69dbdf47eb7148b66" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.60", +] + [[package]] name = "time" version = "0.1.44" @@ -731,6 +1146,23 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "time" +version = "0.3.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a79d09ac6b08c1ab3906a2f7cc2e81a0e27c7ae89c63812df75e52bef0751e07" +dependencies = [ + "deranged", + "serde", + "time-core", +] + +[[package]] +name = "time-core" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb" + [[package]] name = "tinyvec" version = "1.6.0" @@ -746,6 +1178,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" +[[package]] +name = "typenum" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" + [[package]] name = "unicode-bidi" version = "0.3.8" @@ -767,6 +1205,12 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-xid" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" + [[package]] name = "untrusted" version = "0.7.1" @@ -806,9 +1250,15 @@ dependencies = [ [[package]] name = "version-compare" -version = "0.1.0" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "852e951cb7832cb45cb1169900d19760cfa39b82bc0ea9c0e5a14ae88411c98b" + +[[package]] +name = "version_check" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe88247b92c1df6b6de80ddc290f3976dbdf2f5f5d3fd049a9fb598c6dd5ca73" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" [[package]] name = "version_info" @@ -856,7 +1306,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn", + "syn 1.0.99", "wasm-bindgen-shared", ] @@ -878,7 +1328,7 @@ checksum = "5be8e654bdd9b79216c2929ab90721aa82faf65c48cdf08bdc4e7f51357b80da" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.99", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -967,7 +1417,16 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" dependencies = [ - "windows-targets", + "windows-targets 0.48.0", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.4", ] [[package]] @@ -976,13 +1435,28 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b1eb6f0cd7c80c79759c929114ef071b87354ce476d9d94271031c0497adfd5" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_aarch64_gnullvm 0.48.0", + "windows_aarch64_msvc 0.48.0", + "windows_i686_gnu 0.48.0", + "windows_i686_msvc 0.48.0", + "windows_x86_64_gnu 0.48.0", + "windows_x86_64_gnullvm 0.48.0", + "windows_x86_64_msvc 0.48.0", +] + +[[package]] +name = "windows-targets" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd37b7e5ab9018759f893a1952c9420d060016fc19a472b4bb20d1bdd694d1b" +dependencies = [ + "windows_aarch64_gnullvm 0.52.4", + "windows_aarch64_msvc 0.52.4", + "windows_i686_gnu 0.52.4", + "windows_i686_msvc 0.52.4", + "windows_x86_64_gnu 0.52.4", + "windows_x86_64_gnullvm 0.52.4", + "windows_x86_64_msvc 0.52.4", ] [[package]] @@ -991,42 +1465,84 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bcf46cf4c365c6f2d1cc93ce535f2c8b244591df96ceee75d8e83deb70a9cac9" + [[package]] name = "windows_aarch64_msvc" version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da9f259dd3bcf6990b55bffd094c4f7235817ba4ceebde8e6d11cd0c5633b675" + [[package]] name = "windows_i686_gnu" version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" +[[package]] +name = "windows_i686_gnu" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b474d8268f99e0995f25b9f095bc7434632601028cf86590aea5c8a5cb7801d3" + [[package]] name = "windows_i686_msvc" version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" +[[package]] +name = "windows_i686_msvc" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1515e9a29e5bed743cb4415a9ecf5dfca648ce85ee42e15873c3cd8610ff8e02" + [[package]] name = "windows_x86_64_gnu" version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eee091590e89cc02ad514ffe3ead9eb6b660aedca2183455434b93546371a03" + [[package]] name = "windows_x86_64_gnullvm" version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ca79f2451b49fa9e2af39f0747fe999fcda4f5e241b2898624dca97a1f2177" + [[package]] name = "windows_x86_64_msvc" version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32b752e52a2da0ddfbdbcc6fceadfeede4c939ed16d13e648833a61dfb611ed8" + [[package]] name = "ws2_32-sys" version = "0.2.1" @@ -1036,3 +1552,71 @@ dependencies = [ "winapi 0.2.8", "winapi-build", ] + +[[package]] +name = "zerocopy" +version = "0.7.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74d4d3961e53fa4c9a25a8637fc2bfaf2595b3d3ae34875568a5cf64787716be" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.60", +] + +[[package]] +name = "zip" +version = "0.6.6" +source = "git+https://github.com/x87/zip.git#3e88fe66c941d411cff5cf49778ba08c2ed93801" +dependencies = [ + "aes", + "byteorder", + "bzip2", + "constant_time_eq", + "crc32fast", + "crossbeam-utils", + "flate2", + "hmac", + "pbkdf2", + "sha1", + "time 0.3.26", + "zstd", +] + +[[package]] +name = "zstd" +version = "0.11.2+zstd.1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20cc960326ece64f010d2d2107537f26dc589a6573a316bd5b1dba685fa5fde4" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "5.0.2+zstd.1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d2a5585e04f9eea4b2a3d1eca508c4dee9592a89ef6f450c11719da0726f4db" +dependencies = [ + "libc", + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.10+zstd.1.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c253a4914af5bafc8fa8c86ee400827e83cf6ec01195ec1f1ed8441bf00d65aa" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/Cargo.toml b/Cargo.toml index 4c0af22..0a79cf1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sanny_builder_core" -version = "0.3.0" +version = "0.4.0" authors = ["Seemann "] edition = "2021" @@ -23,8 +23,14 @@ serde_json = { version = "1.0", features = ["preserve_order"] } serde = { version = "1.0", features = ["derive"] } ureq = {version = "2.5.0", features = ["json"] } base64 = "0.13.0" -version-compare = "0.1.0" +version-compare = "0.2.0" winapi = { version = "0.3", features = ["winver", "winuser"] } ctor = "0.1.20" libloading = "0.7.0" -version_info = "0.0.5" \ No newline at end of file +version_info = "0.0.5" +anyhow = "1.0.40" +normpath = "1.2" +lines_lossy = "0.1.0" +zip = { git = "https://github.com/x87/zip.git" } +const_format = "0.2.32" +cached = "0.50.0" \ No newline at end of file diff --git a/src/dictionary/dictionary_num_by_str.rs b/src/dictionary/dictionary_num_by_str.rs index 205ac8f..959e474 100644 --- a/src/dictionary/dictionary_num_by_str.rs +++ b/src/dictionary/dictionary_num_by_str.rs @@ -1,9 +1,10 @@ use crate::common_ffi::*; use crate::dictionary::ffi::*; +use std::ffi::CString; use super::config::ConfigBuilder; -pub type DictNumByStr = Dict; +pub type DictNumByStr = Dict; #[no_mangle] pub extern "C" fn dictionary_num_by_str_new( @@ -55,7 +56,7 @@ pub unsafe extern "C" fn dictionary_num_by_str_add( ) -> bool { boolclosure! {{ let d = dict.as_mut()?; - let key = apply_format_s(pchar_to_str(key)?, &d.config.case_format); + let key = apply_format(pchar_to_str(key)?, &d.config.case_format)?; d.add(key, value); Some(()) }} @@ -69,7 +70,7 @@ pub unsafe extern "C" fn dictionary_num_by_str_find( ) -> bool { boolclosure! {{ let d = dict.as_mut()?; - let key = apply_format_s(pchar_to_str(key)?, &d.config.case_format); + let key = apply_format(pchar_to_str(key)?, &d.config.case_format)?; *out = *d.map.get(&key)?; Some(()) }} @@ -80,6 +81,29 @@ pub unsafe extern "C" fn dictionary_num_by_str_free(ptr: *mut DictNumByStr) { ptr_free(ptr); } +#[no_mangle] +pub unsafe extern "C" fn dictionary_num_by_str_get_count(dict: *mut DictNumByStr) -> usize { + if let Some(ptr) = dict.as_mut() { + return ptr.map.len(); + } + return 0; +} + +#[no_mangle] +pub unsafe extern "C" fn dictionary_num_by_str_get_entry( + dict: *mut DictNumByStr, + index: usize, + out_key: *mut PChar, + out_value: *mut i32, +) -> bool { + boolclosure! {{ + let (key, value) = dict.as_mut()?.map.iter().nth(index)?; + *out_key = key.as_ptr(); + *out_value = *value; + Some(()) + }} +} + #[cfg(test)] mod tests { use super::*; diff --git a/src/dictionary/ffi.rs b/src/dictionary/ffi.rs index 89f0f0f..a7f51dc 100644 --- a/src/dictionary/ffi.rs +++ b/src/dictionary/ffi.rs @@ -2,10 +2,11 @@ use std::collections::HashMap; use std::ffi::CString; +use std::path::Path; use super::config::Config; -#[derive(Default)] +#[derive(Debug, Default, Clone)] pub struct Dict { pub map: HashMap, pub config: Config, @@ -33,7 +34,7 @@ where } } - pub fn load_file<'a>(&mut self, file_name: &'a str) -> Option<()> { + pub fn load_file>(&mut self, file_name: P) -> Option<()> { let content = std::fs::read_to_string(file_name).ok()?; self.parse_file(content) } diff --git a/src/dictionary/mod.rs b/src/dictionary/mod.rs index 31c4eb4..92c47b8 100644 --- a/src/dictionary/mod.rs +++ b/src/dictionary/mod.rs @@ -4,3 +4,5 @@ pub mod dictionary_str_by_num; pub mod dictionary_str_by_str; pub mod ffi; pub mod list_num_by_str; + +pub type DictNumByString = ffi::Dict; \ No newline at end of file diff --git a/src/language_service/ffi.rs b/src/language_service/ffi.rs index 8744993..e6a0a07 100644 --- a/src/language_service/ffi.rs +++ b/src/language_service/ffi.rs @@ -3,36 +3,24 @@ use std::ffi::CString; use crate::{ common_ffi::{pchar_to_str, pchar_to_string, ptr_free, ptr_new, PChar}, language_service::server::LanguageServer, + v4::helpers::token_str, }; -#[repr(C)] -#[derive(Debug, Clone, Copy, PartialEq)] -pub enum SymbolType { - Number = 0, - String = 1, - Var = 2, - Label = 3, - ModelName = 4, -} +use super::symbol_table::SymbolType; + #[repr(C)] #[derive(Debug, Clone)] -pub struct SymbolInfo { - pub line_number: u32, +pub struct SymbolInfoRaw { pub _type: SymbolType, + pub value: PChar, + pub name_no_format: PChar, + pub annotation: PChar, } pub struct DocumentInfo { pub is_active: bool, } -#[derive(Clone)] -pub struct SymbolInfoMap { - pub file_name: Option, - pub line_number: u32, - pub _type: SymbolType, - pub value: Option, - pub name_no_format: String, // used for autocomplete -} #[repr(C)] #[derive(Clone, Copy)] #[allow(dead_code)] @@ -70,9 +58,15 @@ pub unsafe extern "C" fn language_service_client_connect_with_file( file_name: PChar, handle: EditorHandle, static_constants_file: PChar, + classes_file: PChar, ) -> bool { boolclosure! {{ - server.as_mut()?.connect(Source::File(pchar_to_string(file_name)?), handle, pchar_to_str(static_constants_file)?); + server.as_mut()?.connect( + Source::File(pchar_to_string(file_name)?), + handle, + pchar_to_str(static_constants_file)?, + pchar_to_str(classes_file)? + ); Some(()) }} } @@ -82,9 +76,15 @@ pub unsafe extern "C" fn language_service_client_connect_in_memory( server: *mut LanguageServer, handle: EditorHandle, static_constants_file: PChar, + classes_file: PChar, ) -> bool { boolclosure! {{ - server.as_mut()?.connect(Source::Memory, handle, pchar_to_str(static_constants_file)?); + server.as_mut()?.connect( + Source::Memory, + handle, + pchar_to_str(static_constants_file)?, + pchar_to_str(classes_file)? + ); Some(()) }} } @@ -116,13 +116,22 @@ pub unsafe extern "C" fn language_service_find( server: *mut LanguageServer, symbol: PChar, handle: EditorHandle, - out_value: *mut SymbolInfo, + line_number: u32, + out_value: *mut SymbolInfoRaw, ) -> bool { boolclosure! {{ let server = server.as_mut()?; - let s = server.find(pchar_to_str(symbol)?, handle)?; - out_value.as_mut()?._type = s._type; - out_value.as_mut()?.line_number = s.line_number; + let s = server.find(pchar_to_str(symbol)?, handle, line_number as usize)?; + let out_value = out_value.as_mut()?; + out_value._type = s._type; + + // don't return line numbers as they are not used on the client side + // out_value.line_number = s.line_number; + // out_value.end_line_number = s.end_line_number; + + out_value.value = CString::new(s.value.unwrap_or_default()).unwrap().into_raw(); + out_value.name_no_format = CString::new(s.name_no_format).unwrap().into_raw(); + out_value.annotation = CString::new(s.annotation.unwrap_or_default()).unwrap().into_raw(); Some(()) }} } @@ -142,13 +151,51 @@ pub unsafe extern "C" fn language_service_filter_constants_by_name( server: *mut LanguageServer, handle: EditorHandle, needle: PChar, + line_number: u32, dict: *mut crate::dictionary::dictionary_str_by_str::DictStrByStr, ) -> bool { boolclosure! {{ - let items = server.as_mut()?.filter_constants_by_name(pchar_to_str(needle)?, handle)?; + let items = server.as_mut()?.filter_constants_by_name(pchar_to_str(needle)?, handle, line_number as usize)?; for item in items { - dict.as_mut()?.add(CString::new(item.0).ok()?, CString::new(item.1).ok()?) + dict.as_mut()?.add(CString::new(item).ok()?, CString::new("").ok()?) // todo: use simple list instead of dict } Some(()) }} } + +#[no_mangle] +pub unsafe extern "C" fn language_service_format_function_signature( + server: *mut LanguageServer, + value: PChar, + out: *mut PChar, +) -> bool { + boolclosure! {{ + let _server = server.as_mut()?; + use crate::parser::{function_arguments_and_return_types, Span}; + + let line = pchar_to_str(value)?; + let (_, ref signature) = function_arguments_and_return_types(Span::from(line)).ok()?; + + let params = signature.0 + .iter() + .map(|param|{ + let type_token = token_str(line, ¶m._type); + let name_token = param.name.as_ref().map(|name| token_str(line, name)); + + match name_token { + Some(name) => format!("\"{}: {}\"", name, type_token), + None => format!("\"{}\"", type_token), + } + }) + .collect::>() + .join(", "); + + // let return_types = signature.1 + // .iter() + // .map(|ret_type| format!("\"{}\"", token_str(line, &ret_type.token))) + // .collect::>() + // .join(", "); + *out = CString::new(format!("{params}")).unwrap().into_raw(); + Some(()) + }} +} diff --git a/src/language_service/scanner.rs b/src/language_service/scanner.rs index e2dab7c..52cccbf 100644 --- a/src/language_service/scanner.rs +++ b/src/language_service/scanner.rs @@ -1,281 +1,385 @@ -use super::ffi::{Source, SymbolInfoMap, SymbolType}; -use crate::dictionary::dictionary_num_by_str::DictNumByStr; -use crate::language_service::server::{CACHE_FILE_SYMBOLS, CACHE_FILE_TREE}; +use super::ffi::Source; +use super::symbol_table::{SymbolInfoMap, SymbolTable, SymbolType}; +use crate::dictionary::DictNumByString; +use crate::language_service::server::CACHE_FILE_SYMBOLS; +use crate::parser::FunctionSignature; use crate::utils::compiler_const::*; +use crate::utils::visibility_zone::VisibilityZone; +use crate::v4::helpers::token_str; +use std::collections::HashSet; use std::fs; use std::path::Path; -fn document_tree_walk<'a>( - content: &String, - file_name: &String, - reserved_words: &DictNumByStr, - mut refs: &mut Vec, -) -> Vec { - content - .lines() - .filter_map(|x| { - // todo: use nom parser - let mut words = x.split_ascii_whitespace(); - let first = words.next()?.to_ascii_lowercase(); - - if let Some(token) = reserved_words.map.get(&first) { - if token == &TOKEN_INCLUDE { - let mut include_path = words.collect::(); - - if include_path.ends_with('}') { - include_path.pop(); - } - - let path = resolve_path(include_path, file_name)?; - - // ignore cyclic paths - if refs.contains(&path) { - return None; - } else { - refs.push(path.clone()); - } +fn file_walk( + file_name: &str, + reserved_words: &DictNumByString, + visited: &mut HashSet, + class_names: &Vec, + table: &mut SymbolTable, + scope_stack: &mut Vec<( + /*number of open blocks*/ u32, + /* scope start line*/ u32, + )>, + line_number: Option, +) { + // ignore cyclic paths + if !visited.insert(file_name.into()) { + return; + } - let mut tree = match get_cached_tree(&path) { - Some(tree) => { - log::debug!("Using cached tree for file {}", path); - tree - } - None => { - log::debug!("Tree cache not found. Reading file {}", path); - file_walk(path.clone(), reserved_words, &mut refs)? - } - }; - tree.push(path); - return Some(tree); - } - } - None - }) - .flatten() - .collect::>() -} + // if present, use file's cached symbols (all symbols from the file and all included files) + if let Some(symbols) = CACHE_FILE_SYMBOLS.lock().unwrap().get(file_name) { + log::debug!("Using cached symbols for file {}", file_name); + table.extend(symbols); + return; + } -fn file_walk( - file_name: String, - reserved_words: &DictNumByStr, - mut refs: &mut Vec, -) -> Option> { - let content = fs::read_to_string(&file_name).ok()?; - let tree = document_tree_walk(&content, &file_name, reserved_words, &mut refs); - - log::debug!("Caching file tree {}", file_name); - let mut cache = CACHE_FILE_TREE.lock().unwrap(); - cache.insert(file_name.clone(), tree.clone()); - - Some(tree) -} + log::debug!("Symbol cache not found. Reading file {}", file_name); + let Ok(content) = fs::read_to_string(&file_name) else { + return; + }; -fn get_cached_tree(file_name: &String) -> Option> { - let cache = CACHE_FILE_TREE.lock().unwrap(); - cache.get(file_name).cloned() + // create a new table for this file and its descendants + let mut local_table = SymbolTable::new(); + scan_text( + &content, + reserved_words, + class_names, + &Source::File(file_name.into()), + visited, + scope_stack, + &mut local_table, + line_number, + ); + + // use found symbols and cache them + table.extend(&local_table); + CACHE_FILE_SYMBOLS + .lock() + .unwrap() + .insert(file_name.into(), local_table); } -fn resolve_path(p: String, parent_file: &String) -> Option { - let path = Path::new(&p); +fn resolve_path(p: &str, parent_file: &Option) -> Option { + let path = Path::new(p); if path.is_absolute() { - return Some(p); + return Some(p.to_string()); } - let dir_name = Path::new(&parent_file).parent()?; - let abs_name = dir_name.join(path); - - Some(String::from(abs_name.to_str()?)) + match parent_file { + Some(x) => { + // todo: + // If the file path is relative, the compiler scans directories in the following order to find the file: + // 1. directory of the file with the directive + // 2. data folder for the current edit mode + // 3. Sanny Builder root directory + // 4. the game directory + let dir_name = Path::new(&x).parent()?; + let abs_name = dir_name.join(path); + + Some(String::from(abs_name.to_str()?)) + } + None => None, + } } -pub fn document_tree<'a>( - text: &String, - reserved_words: &DictNumByStr, +/// read the source code and extract all constants and variables +/// if the file contains an include directive, recursively scan the included file +/// also, scan all implicit includes (constants.txt) +pub fn scan_document<'a>( + text: &str, + reserved_words: &DictNumByString, implicit_includes: &Vec, source: &Source, -) -> Option> { - match source { - Source::File(file_name) => { - let mut refs: Vec = vec![]; - let mut tree = document_tree_walk(text, file_name, reserved_words, &mut refs); - - tree.extend(implicit_includes.iter().filter_map(|include_path| { - Some(resolve_path( - include_path.to_owned(), - &file_name.to_string(), - )?) - })); - - Some(tree) - } - Source::Memory => Some(implicit_includes.clone()), + class_names: &Vec, + table: &mut SymbolTable, + visited: &mut HashSet, + scope_stack: &mut Vec<(u32, u32)>, +) { + for file_name in implicit_includes { + file_walk( + file_name, + reserved_words, + visited, + class_names, + table, + scope_stack, + Some(0), + ); } + + scan_text( + text, + reserved_words, + class_names, + source, + visited, + scope_stack, + table, + None, // line number to be determined as we parse the source code + ); } -pub fn find_constants_from_file( - file_name: &String, - reserved_words: &DictNumByStr, -) -> Option> { - let mut cache = CACHE_FILE_SYMBOLS.lock().unwrap(); - match cache.get(file_name) { - Some(symbols) => { - log::debug!("Using cached symbols for file {}", file_name); - Some(symbols.clone()) +pub fn scan_text<'a>( + content: &str, + reserved_words: &DictNumByString, + class_names: &Vec, + source: &Source, + visited: &mut HashSet, + scope_stack: &mut Vec<(u32, u32)>, + table: &mut SymbolTable, + line_number: Option, +) { + let mut inside_const = false; + let file_name = match source { + Source::File(path) => Some(path.clone()), + Source::Memory => None, + }; + + let mut inside_comment = false; + let mut inside_comment2 = false; + let mut next_annotation: Option = None; + + let lines = content.lines(); + for (_index, line1) in lines.enumerate() { + let (first, rest) = strip_comments(line1, &mut inside_comment, &mut inside_comment2); + + if first.is_empty() { + continue; } - None => { - log::debug!("Symbol cache not found. Reading file {}", file_name); - let content = fs::read_to_string(file_name).ok()?; - let symbols = - find_constants(&content, reserved_words, &Source::File(file_name.clone()))?; - cache.insert(file_name.clone(), symbols.clone()); - Some(symbols) + + if first.eq("///") { + //append or create next_annotation + + if let Some(ref mut annotation) = next_annotation { + annotation.push_str("\n"); + annotation.push_str(rest.as_str()); + } else { + next_annotation = Some(rest.to_string()); + } + + continue; } - } -} -pub fn find_constants_from_memory( - content: &String, - reserved_words: &DictNumByStr, -) -> Option> { - find_constants(&content, reserved_words, &Source::Memory) -} + let first_lower = first.to_ascii_lowercase(); + let token_id = reserved_words.map.get(&first_lower); -pub fn find_constants<'a>( - content: &String, - reserved_words: &DictNumByStr, - source: &Source, -) -> Option> { - let mut lines: Vec = vec![]; - let mut line = String::new(); - let mut chars = content.chars(); + // reset annotation if this line is not a function + if token_id != Some(&TOKEN_FUNCTION) && token_id != Some(&TOKEN_DEFINE) { + next_annotation = None; + } - 'outer: while let Some(c) = chars.next() { - match c { - '\n' => { - lines.push(line); - line = String::new(); - } - '{' => { - // { } block - loop { - match chars.next() { - Some('}') => break, - Some(_) => {} // ignore other chars inside block - None => break 'outer, - } - } - } - '/' => match chars.next() { - // /* */ comment - Some('*') => loop { - match chars.next() { - Some('*') => { - if chars.next() == Some('/') { - break; - } - } - Some(_) => {} // ignore other chars inside comment - None => break 'outer, - } - }, - // // comment - Some('/') => { - loop { - match chars.next() { - Some('\n') => { - lines.push(line); - line = String::new(); - break; + /* + if the file is a $include in the current document, then we need all its symbols (including deep $include's) + to have a line number of the $include statement in the current document + + if the file is an implicit include (constants.txt), then all its symbols have a line number of 0 + */ + let line_number = line_number.unwrap_or(_index); + let stack_id = scope_stack.len() as u32; + + let mut process_function_signature = |line: &str, signature: &FunctionSignature| { + let scope_start_line = scope_stack.last().map(|(_, line)| *line).unwrap_or(0); // hoist the scope start line + + register_function( + table, + scope_start_line as usize, + stack_id, + line, + signature, + next_annotation.take(), + ); + // only local functions create new scope. foreign functions do not + if signature.cc == crate::parser::FunctionCC::Local { + // push new scope + scope_stack.push((0, line_number as u32)); + + // end visibility zone for the local variables of the parent scope + // because parent local variables can not be seen in functions + // todo: make sure global vars is an exception + for (_, symbols) in table.symbols.iter_mut() { + for symbol in symbols { + if symbol.stack_id == stack_id && symbol._type == SymbolType::Var { + let Some(last_zone) = symbol.zones.last_mut() else { + continue; + }; + + if last_zone.end != 0 { + // should not happen + log::error!( + "Symbol {} does not have an open visibility zone", + symbol.name_no_format + ); + continue; } - Some(_) => {} // ignore other chars inside comment - None => break 'outer, + + last_zone.end = line_number; } } } - Some(c) => { - line.push('/'); - line.push(c); - } - None => { - break 'outer; - } - }, - - c => { - // trim left - if !c.is_ascii_whitespace() || !line.is_empty() { - line.push(c); + for param in &signature.parameters { + if let Some(ref name) = param.name { + register_var( + table, + line_number, + stack_id + 1, // register function parameters in the function's stack + token_str(line, name), + Some(token_str(line, ¶m._type).to_string()), + None, + ); + } } } - } - } - lines.push(line); - - // let mut lines = content.lines().enumerate(); - let mut found_constants = vec![]; - let mut inside_const = false; - let file_name = match source { - Source::File(path) => Some(path.clone()), - Source::Memory => None, - }; - for (line_number, line) in lines.iter().enumerate() { - if line.is_empty() { - continue; - } - let mut words = line.split_ascii_whitespace(); - let first = match words.next() { - Some(word) => word.to_ascii_lowercase(), - None => continue, }; - match reserved_words.map.get(&first) { + + match token_id { Some(token) => match *token { - TOKEN_CONST => { - let rest = words.collect::(); + TOKEN_INCLUDE | TOKEN_INCLUDE_ONCE => { + let include_path = if rest.ends_with('}') { + &rest[..rest.len() - 1] + } else { + rest.as_str() + }; + let Some(path) = resolve_path(include_path, &file_name) else { + continue; + }; + + file_walk( + &path, + reserved_words, + visited, + class_names, + table, + scope_stack, + Some(line_number), + ); + } + TOKEN_CONST => { if !rest.is_empty() { let declarations = split_const_line(&rest); for declaration in declarations.iter() { - process_const_declaration( - &declaration, - &mut found_constants, - line_number, - &file_name, - ); + process_const_declaration(&declaration, table, line_number, stack_id); } } else { inside_const = true; } } TOKEN_END if inside_const => inside_const = false, + TOKEN_END => { + if stack_id < 2 { + // global scope, only ends when the file ends + continue; + } + // number of nested blocks in the current function + let (fn_blocks, _) = scope_stack.last_mut().unwrap(); + if *fn_blocks == 0 { + // there are no other open blocks in this function, this is the function's end + + // find all symbols defined in the current scope and close their visibility zone + for (_, symbols) in table.symbols.iter_mut() { + for symbol in symbols { + if symbol.stack_id == stack_id { + let Some(last_zone) = symbol.zones.last_mut() else { + continue; + }; + + if last_zone.end != 0 { + // should not happen + log::error!( + "Symbol {} does not have an open visibility zone", + symbol.name_no_format + ); + continue; + } + + // this local variable is not visible inside the function + last_zone.end = line_number; + + symbol.stack_id = 0; // mark as processed + } + } + } + + // delete function scope + scope_stack.pop(); + + let stack_id = scope_stack.len() as u32; + // open visibility zone for the local variables of the parent scope + for (_, symbols) in table.symbols.iter_mut() { + for symbol in symbols { + if symbol.stack_id == stack_id && symbol._type == SymbolType::Var { + symbol.add_zone(line_number) + } + } + } + } else { + // exit block + *fn_blocks -= 1; + } + } TOKEN_INT | TOKEN_FLOAT | TOKEN_STRING | TOKEN_LONGSTRING | TOKEN_HANDLE | TOKEN_BOOL => { // inline variable declaration - - let rest = words.collect::(); let names = split_const_line(&rest); for name in names { - process_var_declaration( - &name, - &mut found_constants, - line_number, - &file_name, - ) + process_var_declaration(&name, table, line_number, stack_id, &first) + } + } + TOKEN_EXPORT => { + // export function + use crate::parser::{function_signature, Span}; + + // parse function signature and add its parameters to the symbol table as variables + let Ok((_, ref signature)) = function_signature(Span::from(rest.as_str())) + else { + continue; + }; + process_function_signature(rest.as_str(), signature); + } + TOKEN_FUNCTION => { + // function + use crate::parser::{function_signature, Span}; + + // parse function signature and add its parameters to the symbol table as variables + let line = first + " " + &rest; + let line = line.as_str(); + let Ok((_, ref signature)) = function_signature(Span::from(line)) else { + continue; + }; + + process_function_signature(line, signature); + } + + TOKEN_IF | TOKEN_FOR | TOKEN_WHILE | TOKEN_SWITCH => { + if stack_id < 2 { + // global scope, only ends when the file ends + continue; } + let (function_scope, _) = scope_stack.last_mut().unwrap(); + // enter block + *function_scope += 1; } _ => {} }, _ if inside_const => { + let line = first + " " + &rest; + let line = line.as_str(); let declarations = split_const_line(line); for declaration in declarations.iter() { - process_const_declaration( - &declaration, - &mut found_constants, - line_number, - &file_name, - ); + process_const_declaration(&declaration, table, line_number, stack_id); + } + } + _ if class_names.contains(&first_lower) => { + // class declaration + let names = split_const_line(&rest); + + for name in names { + process_var_declaration(&name, table, line_number, stack_id, &first) } } _ => { @@ -283,115 +387,290 @@ pub fn find_constants<'a>( } } } +} + +fn register_symbol(table: &mut SymbolTable, map: SymbolInfoMap) { + let name_lower = map.name_no_format.to_ascii_lowercase(); + match table.symbols.get_mut(&name_lower) { + Some(symbols) => { + for symbol in symbols.iter() { + if symbol.stack_id == map.stack_id { + log::debug!( + "Found duplicate symbol declaration {} in line {}", + map.name_no_format, + // map.line_number + 1 + map.zones[0].start + 1 + ); + return; + } + } + + symbols.push(map); + } + None => { + table.symbols.insert(name_lower, vec![map]); + } + } +} + +fn register_function( + table: &mut SymbolTable, + line_number: usize, + stack_id: u32, + line: &str, + signature: &FunctionSignature, + annotation: Option, +) { + let map = SymbolInfoMap { + zones: vec![VisibilityZone { + start: line_number, + end: 0, + }], + // line_number: line_number as u32, + _type: SymbolType::Function, + stack_id, // register function in parent stack + // end_line_number: 0, + value: Some(function_params_and_return_types(line, signature)), + name_no_format: token_str(&line, &signature.name).to_string(), + annotation, + }; + register_symbol(table, map); +} + +fn register_var( + table: &mut SymbolTable, + line_number: usize, + stack_id: u32, + name: &str, + _type: Option, + annotation: Option, +) { + register_const( + table, + line_number, + stack_id, + name, + _type, + SymbolType::Var, + annotation, + ); +} - Some(found_constants) +fn register_const( + table: &mut SymbolTable, + line_number: usize, + stack_id: u32, + name: &str, + value: Option, + _type: SymbolType, + annotation: Option, +) { + let map = SymbolInfoMap { + zones: vec![VisibilityZone { + start: line_number, + end: 0, + }], + // line_number: line_number as u32, + _type, + stack_id, + // end_line_number: 0, + value, + name_no_format: name.to_string(), + annotation, + }; + register_symbol(table, map); +} + +pub fn strip_comments( + s: &str, + inside_comment: &mut bool, + inside_comment2: &mut bool, +) -> (String, String) { + let mut chars = s.chars().peekable(); + let mut first_word = String::new(); + let mut rest = String::new(); + let mut buf = &mut first_word; + + // iterate over all chars, skip comment fragments (/* */ and //) + while let Some(c) = chars.next() { + match c { + _ if *inside_comment => { + // skip until the end of the comment + if c == '*' { + if let Some('/') = chars.next() { + *inside_comment = false; + } + } + } + _ if *inside_comment2 => { + // skip until the end of the comment + if c == '}' { + *inside_comment2 = false; + } + } + '/' if chars.peek() == Some(&'/') => { + chars.next(); // skip / + + // annotation /// + if chars.peek() == Some(&'/') { + chars.next(); // skip / + + if buf.is_empty() { + // start of the line + buf.push_str("///"); // first word is /// + buf = &mut rest; // the rest of the line is the annotation + continue; + } + } + + // line comment // + // there is nothing left on this line, exiting + break; + } + '/' if chars.peek() == Some(&'*') => { + // block comment /* */ + *inside_comment = true; + chars.next(); // skip * + } + + '{' if chars.peek() != Some(&'$') => { + // block comment {} but not directives {$...} + *inside_comment2 = true; + } + _ if c.is_ascii_whitespace() => { + if buf.is_empty() { + // skip leading whitespace + continue; + } else { + buf = &mut rest; + if !buf.is_empty() { + buf.push(c); + } + } + } + _ => { + // line_without_comments.push(c); + buf.push(c); + } + } + } + + return (first_word, rest.trim_end().to_string()); } pub fn process_const_declaration( line: &str, - found_constants: &mut Vec<(String, SymbolInfoMap)>, + table: &mut SymbolTable, line_number: usize, - file_name: &Option, + stack_id: u32, ) { let mut tokens = line.split('='); let Some(name) = tokens.next() else { return }; let name = name.trim(); - let name_lower = name.to_ascii_lowercase(); - if found_constants.iter().any(|(n, _)| n == &name_lower) { - log::debug!( - "Found duplicate const declaration {} in line {}", - name, - line_number + 1 - ); - return; - } + // let name_lower = name.to_ascii_lowercase(); + // if let Some(symbols) = table.symbols.get(&name_lower) { + // for symbol in symbols { + // if symbol.stack_id == stack_id { + // log::debug!( + // "Found duplicate const declaration {} in line {}", + // name, + // line_number + 1 + // ); + // return; + // } + // } + // } let Some(value) = tokens.next() else { return }; let value = value.trim(); - - macro_rules! add_to_constants { - ($type:expr) => { - found_constants.push(( - name_lower, - SymbolInfoMap { - line_number: line_number as u32, - _type: $type, - file_name: file_name.clone(), - value: Some(String::from(value)), - name_no_format: name.to_string(), - }, - )); - }; - } - - match get_type(value) { - Some(_type) => { - add_to_constants!(_type); - } - None => { - if let Some((_, symbol)) = found_constants + let value_lower = value.to_ascii_lowercase(); + let Some(_type) = get_type(value_lower.as_str()).or_else(|| { + table.symbols.get(value_lower.as_str()).and_then(|symbols| { + symbols .iter() - .find(|x| x.0 == value.to_ascii_lowercase()) - { - add_to_constants!(symbol._type); - }; - } - } + .find(|symbol| symbol.stack_id == stack_id) + .map(|symbol| symbol._type) + }) + }) else { + return; + }; + + log::debug!( + "Found const declaration {} in line {}", + name, + line_number + 1 + ); + + register_const( + table, + line_number, + stack_id, + name, + Some(String::from(value)), + _type, + None, + ); } pub fn process_var_declaration( line: &str, - found_constants: &mut Vec<(String, SymbolInfoMap)>, + table: &mut SymbolTable, line_number: usize, - file_name: &Option, + stack_id: u32, + _type: &str, ) { let mut tokens = line.split('='); - let Some(mut name) = tokens.next() else { return }; + let Some(mut name) = tokens.next() else { + return; + }; if let Some(pos) = name.find('[') { name = &name[..pos]; } let name = name.trim(); - let name_lower = name.to_ascii_lowercase(); - if found_constants.iter().any(|(n, _)| n == &name_lower) { - log::debug!( - "Found duplicate const declaration {} in line {}", - name, - line_number + 1 - ); - return; - } - found_constants.push(( - name.to_ascii_lowercase(), - SymbolInfoMap { - line_number: line_number as u32, - _type: SymbolType::Var, - file_name: file_name.clone(), - value: None, - name_no_format: name.to_string(), - }, - )) + // let name_lower = name.to_ascii_lowercase(); + // if let Some(symbols) = table.symbols.get(&name_lower) { + // for symbol in symbols { + // if symbol.stack_id == stack_id { + // log::debug!( + // "Found duplicate var declaration {} in line {}", + // name, + // line_number + 1 + // ); + // return; + // } + // } + // } + + register_var( + table, + line_number, + stack_id, + name, + Some(String::from(_type)), + None, + ); } pub fn split_const_line(line: &str) -> Vec { // iterate over chars, split by , ignore commas inside parentheses let mut result = vec![]; let mut current: usize = 0; - let mut inside_parentheses = false; + let mut inside_parentheses = 0; for (i, c) in line.chars().enumerate() { match c { '(' => { - inside_parentheses = true; + inside_parentheses += 1; } ')' => { - inside_parentheses = false; + inside_parentheses -= 1; } ',' => { - if !inside_parentheses { + if inside_parentheses == 0 { result.push(line[current..i].to_string()); current = i + 1; } @@ -415,7 +694,8 @@ pub fn get_type(value: &str) -> Option { || value.ends_with('@') || value.ends_with("@s") || value.ends_with("@v") - || (value.ends_with(")") && value.contains("@(")) // arrays 0@(1@,2i) + || (value.ends_with(")") && value.contains("@(")) + // arrays 0@(1@,2i) { return Some(SymbolType::Var); } @@ -428,23 +708,153 @@ pub fn get_type(value: &str) -> Option { if value.starts_with('@') { return Some(SymbolType::Label); } + if value.starts_with("0x") + || value.starts_with("-0x") + || value.starts_with("+0x") + || value.starts_with("0b") + || value.starts_with("-0b") + || value.starts_with("+0b") + { + return Some(SymbolType::Number); + } } if let Some(_) = value.parse::().ok() { return Some(SymbolType::Number); } - if value.starts_with("0x") || value.starts_with("-0x") || value.starts_with("+0x") { - return Some(SymbolType::Number); - } return None; } +fn function_params_and_return_types(line: &str, signature: &FunctionSignature) -> String { + let params = signature + .parameters + .iter() + .map(|param| { + let type_token = token_str(line, ¶m._type); + let name_token = param.name.as_ref().map(|name| token_str(line, name)); + + match name_token { + Some(name) => format!("{}: {}", name, type_token), + None => format!("{}", type_token), + } + }) + // .map(|param| [token_str(line, ¶m.name), token_str(line, ¶m._type)].join(": ")) + .collect::>() + .join(", "); + + let return_types = signature + .return_types + .iter() + .map(|_type| token_str(line, &_type.token).to_string()) + .collect::>() + .join(", "); + + if return_types.is_empty() { + format!("({params})") + } else { + format!("({params}): {return_types}") + } +} + #[cfg(test)] mod tests { use super::*; #[test] fn test1() { - let p = resolve_path(String::from("2.txt"), &String::from("C:/dev/1.txt")).unwrap(); + let p = resolve_path("2.txt", &Some(String::from("C:/dev/1.txt"))).unwrap(); assert_eq!(p, String::from("C:/dev\\2.txt")); } + + #[test] + fn test2() { + let s = "test line"; + let mut inside_comment = false; + let mut inside_comment2 = false; + let s = strip_comments(s, &mut inside_comment, &mut inside_comment2); + assert_eq!(s, ("test".to_string(), "line".to_string())); + + let s = "test line // comment"; + let mut inside_comment = false; + let mut inside_comment2 = false; + let s = strip_comments(s, &mut inside_comment, &mut inside_comment2); + assert_eq!(s, ("test".to_string(), "line".to_string())); + + let s = "test line /* comment */"; + let mut inside_comment = false; + let mut inside_comment2 = false; + let s = strip_comments(s, &mut inside_comment, &mut inside_comment2); + assert_eq!(s, ("test".to_string(), "line".to_string())); + + let s = "test line /* comment */ test line"; + let mut inside_comment = false; + let mut inside_comment2 = false; + let s = strip_comments(s, &mut inside_comment, &mut inside_comment2); + assert_eq!(s, ("test".to_string(), "line test line".to_string())); + + let s = "test line /* comment */ test line /* comment */"; + let mut inside_comment = false; + let mut inside_comment2 = false; + let s = strip_comments(s, &mut inside_comment, &mut inside_comment2); + assert_eq!(s, ("test".to_string(), "line test line".to_string())); + + let s = "test line"; + let mut inside_comment = true; + let mut inside_comment2 = false; + let s = strip_comments(s, &mut inside_comment, &mut inside_comment2); + assert_eq!(s, ("".to_string(), "".to_string())); + + let s = "test line */ after comment"; + let mut inside_comment = true; + let mut inside_comment2 = false; + let s = strip_comments(s, &mut inside_comment, &mut inside_comment2); + assert_eq!(s, ("after".to_string(), "comment".to_string())); + + let s = " leading whitespace"; + let mut inside_comment = false; + let mut inside_comment2 = false; + let s = strip_comments(s, &mut inside_comment, &mut inside_comment2); + assert_eq!(s, ("leading".to_string(), "whitespace".to_string())); + + let s = " {comment} test {comment} line"; + let mut inside_comment = false; + let mut inside_comment2 = false; + let s = strip_comments(s, &mut inside_comment, &mut inside_comment2); + assert_eq!(s, ("test".to_string(), "line".to_string())); + + let s = " comment} test {comment} line {comment} "; + let mut inside_comment = false; + let mut inside_comment2 = false; + let s = strip_comments(s, &mut inside_comment, &mut inside_comment2); + assert_eq!(s, ("comment}".to_string(), "test line".to_string())); + + let s = " comment} test {comment} line {comment} "; + let mut inside_comment = false; + let mut inside_comment2 = true; + let s = strip_comments(s, &mut inside_comment, &mut inside_comment2); + assert_eq!(s, ("test".to_string(), "line".to_string())); + + let s = "test{ /* */ } line"; + let mut inside_comment = false; + let mut inside_comment2 = false; + let s = strip_comments(s, &mut inside_comment, &mut inside_comment2); + assert_eq!(s, ("test".to_string(), "line".to_string())); + + let s = "test/* {} */ line"; + let mut inside_comment = false; + let mut inside_comment2 = false; + let s = strip_comments(s, &mut inside_comment, &mut inside_comment2); + assert_eq!(s, ("test".to_string(), "line".to_string())); + + let s = "*/test"; + let mut inside_comment = true; + let mut inside_comment2 = false; + let s = strip_comments(s, &mut inside_comment, &mut inside_comment2); + assert_eq!(s, ("test".to_string(), "".to_string())); + + let s = "}test"; + let mut inside_comment = false; + let mut inside_comment2 = true; + let s = strip_comments(s, &mut inside_comment, &mut inside_comment2); + assert_eq!(s, ("test".to_string(), "".to_string())); + } } diff --git a/src/language_service/server.rs b/src/language_service/server.rs index 4cf4864..e63c6dc 100644 --- a/src/language_service/server.rs +++ b/src/language_service/server.rs @@ -1,10 +1,15 @@ use super::{ - ffi::{DocumentInfo, EditorHandle, Source, Status, SymbolInfo, SymbolInfoMap}, + ffi::{DocumentInfo, EditorHandle, Source, Status}, watcher::FileWatcher, - {scanner, symbol_table::SymbolTable}, + { + scanner, + symbol_table::{SymbolInfoMap, SymbolTable}, + }, +}; +use crate::{ + dictionary::{config, ffi::CaseFormat, DictNumByString}, + namespaces::namespaces::Namespaces, }; -use crate::dictionary::{config, dictionary_num_by_str::DictNumByStr, ffi::CaseFormat}; -use lazy_static::lazy_static; use std::{ collections::{HashMap, HashSet}, env, @@ -23,7 +28,7 @@ lazy_static! { static ref WATCHED_FILES: Mutex>> = Mutex::new(HashMap::new()); static ref SOURCE_MAP: Mutex> = Mutex::new(HashMap::new()); - static ref RESERVED_WORDS: Mutex = Mutex::new(DictNumByStr::new( + static ref RESERVED_WORDS: Mutex = Mutex::new(DictNumByString::new( config::ConfigBuilder::new() .set_case_format(CaseFormat::LowerCase) .build() @@ -31,9 +36,8 @@ lazy_static! { static ref FILE_WATCHER: Mutex = Mutex::new(FileWatcher::new()); static ref IMPLICIT_INCLUDES: Mutex>> = Mutex::new(HashMap::new()); - pub static ref CACHE_FILE_TREE: Mutex>> = - Mutex::new(HashMap::new()); - pub static ref CACHE_FILE_SYMBOLS: Mutex>> = + static ref CLASS_NAMES: Mutex>> = Mutex::new(HashMap::new()); + pub static ref CACHE_FILE_SYMBOLS: Mutex> = Mutex::new(HashMap::new()); } @@ -57,16 +61,29 @@ impl LanguageServer { Self { message_queue } } - pub fn connect(&mut self, source: Source, handle: EditorHandle, static_constants_file: &str) { + pub fn connect( + &mut self, + source: Source, + handle: EditorHandle, + static_constants_file: &str, + classes_file: &str, + ) { log::debug!("New client {} connected with source {:?}", handle, source); SOURCE_MAP.lock().unwrap().insert(handle, source.clone()); - let static_constants_file = String::from(static_constants_file); IMPLICIT_INCLUDES .lock() .unwrap() - .insert(handle, vec![static_constants_file.clone()]); + .insert(handle, vec![String::from(static_constants_file)]); + + let mut ns = Namespaces::new(); + ns.load_classes(classes_file); + + CLASS_NAMES.lock().unwrap().insert( + handle, + ns.map_op_by_name.keys().cloned().collect::>(), + ); status_change(handle, Status::PendingScan); } @@ -76,36 +93,38 @@ impl LanguageServer { SYMBOL_TABLES.lock().unwrap().remove(&handle); SOURCE_MAP.lock().unwrap().remove(&handle); IMPLICIT_INCLUDES.lock().unwrap().remove(&handle); + CLASS_NAMES.lock().unwrap().remove(&handle); let mut watcher = FILE_WATCHER.lock().unwrap(); // disconnect editor from all files and stop watching orphan references - let _drained = WATCHED_FILES - .lock() - .unwrap() - .drain_filter(|k, v| { - v.remove(&handle); - if v.is_empty() { - LanguageServer::invalidate_file_cache(k); - watcher.unwatch(k); - return true; - } + WATCHED_FILES.lock().unwrap().retain(|k, v| { + v.remove(&handle); + if v.is_empty() { + LanguageServer::invalidate_file_cache(k); + watcher.unwatch(k); return false; - }) - .collect::>(); + } + return true; + }); } - pub fn find(&mut self, symbol: &str, handle: EditorHandle) -> Option { + pub fn find( + &mut self, + symbol: &str, + handle: EditorHandle, + line_number: usize, + ) -> Option { let st = SYMBOL_TABLES.lock().unwrap(); let table = st.get(&handle)?; - let map = table.symbols.get(&symbol.to_ascii_lowercase())?; - Some(SymbolInfo { - line_number: if map.file_name.is_some() { - 0 - } else { - map.line_number - }, - _type: map._type, - }) + let symbol_infos = table.symbols.get(&symbol.to_ascii_lowercase())?; + + for symbol_info in symbol_infos { + if symbol_info.is_visible_at(line_number) { + // check if symbol is visible in current scope in current line + return Some(symbol_info.clone()); + } + } + None } pub fn get_document_info(&self, handle: EditorHandle) -> DocumentInfo { @@ -118,21 +137,28 @@ impl LanguageServer { &self, needle: &str, handle: EditorHandle, - ) -> Option> { + line_number: usize, + ) -> Option> { let st = SYMBOL_TABLES.lock().unwrap(); let table = st.get(&handle)?; let needle = needle.to_ascii_lowercase(); - Some( - table - .symbols - .iter() - .filter_map(|(name, map)| { - name.to_ascii_lowercase() - .starts_with(&needle) - .then_some((map.name_no_format.clone(), map.value.clone()?.clone())) - }) - .collect::>(), - ) + + let list = table + .symbols + .iter() + .filter_map(|(name, map)| { + if name.to_ascii_lowercase().starts_with(&needle) { + for symbol_info in map { + if symbol_info.is_visible_at(line_number) { + return Some(name.clone()); + } + } + } + return None; + }) + .collect::>(); + // list.sort_by(|v1, v2| v1.0.cmp(&v2.0)); + Some(list) } fn setup_message_queue() -> Sender<(EditorHandle, String)> { @@ -157,26 +183,24 @@ impl LanguageServer { message_queue } - fn update_watchers(tree: &Vec, handle: EditorHandle) { - log::debug!("Updating file watchers"); + fn update_watchers(tree: &HashSet, handle: EditorHandle) { + log::debug!("Updating {} file watchers for handle {handle}", tree.len()); let mut watched_files = WATCHED_FILES.lock().unwrap(); let mut watcher = FILE_WATCHER.lock().unwrap(); // remove handle from dereferenced files - let _drained = watched_files - .drain_filter(|k, v| { - if !tree.contains(k) && v.contains(&handle) { - v.remove(&handle); - if v.is_empty() { - LanguageServer::invalidate_file_cache(k); - watcher.unwatch(k); - return true; - } + watched_files.retain(|k, v| { + if !tree.contains(k) && v.contains(&handle) { + v.remove(&handle); + if v.is_empty() { + LanguageServer::invalidate_file_cache(k); + watcher.unwatch(k); + return false; } - return false; - }) - .collect::>(); + } + return true; + }); // add new references for file_name in tree { @@ -192,7 +216,6 @@ impl LanguageServer { let file_name1 = file_name.clone(); watcher.watch(file_name.as_str(), move |event| match event { hotwatch::Event::Write(_) => { - // todo: check if possible to use file name from event payload LanguageServer::invalidate_file_cache(&file_name1); LanguageServer::rescan(&file_name1) } @@ -205,22 +228,7 @@ impl LanguageServer { } } - fn invalidate_file_cache(file_name: &String) { - let mut cache = CACHE_FILE_TREE.lock().unwrap(); - - // invalidate cache for all files referencing this file - // todo: change file cache to only store its own references and not children - // then use cache.remove(file_name) - let _drained = cache - .drain_filter(|k, v| { - if k == file_name || v.contains(file_name) { - log::debug!("Invalidating tree cache for file {}", k); - return true; - } - return false; - }) - .collect::>(); - + fn invalidate_file_cache(file_name: &str) { CACHE_FILE_SYMBOLS .lock() .unwrap() @@ -231,11 +239,12 @@ impl LanguageServer { }); } - fn rescan(file_name: &String) { + /// Schedule scan for all clients referencing this file + fn rescan(file_name: &str) { log::debug!("File {} has changed", file_name); let files = WATCHED_FILES.lock().unwrap(); - if let Some(handles) = files.get(file_name.as_str()) { + if let Some(handles) = files.get(file_name) { log::debug!("Found {} dependent clients", handles.len()); for &handle in handles { status_change(handle, Status::PendingScan) @@ -247,34 +256,32 @@ impl LanguageServer { log::debug!("Spawn scan for client {}", handle); let sources = SOURCE_MAP.lock().unwrap(); let dict = RESERVED_WORDS.lock().unwrap(); + let classes = CLASS_NAMES.lock().unwrap(); let mut symbol_table = SYMBOL_TABLES.lock().unwrap(); let implicit_includes = IMPLICIT_INCLUDES.lock().unwrap(); - log::debug!("Reading source to build document tree"); if let Some(source) = sources.get(&handle) { + log::debug!("Reading source {:?} to build document tree", source); let mut table = SymbolTable::new(); let v = vec![]; let includes = implicit_includes.get(&handle).unwrap_or(&v); - - if let Some(tree) = scanner::document_tree(&text, &dict, includes, source) { - log::debug!("Document tree is ready: {} child entries", tree.len()); - - LanguageServer::update_watchers(&tree, handle); - for file in tree { - log::debug!("Fetch symbols from file {}", file); - if let Some(constants) = scanner::find_constants_from_file(&file, &dict) { - log::debug!("Found {} symbols", constants.len()); - table.add(constants); - } - } - } - - log::debug!("Fetch symbols from opened document"); - if let Some(constants) = scanner::find_constants_from_memory(&text, &dict) { - log::debug!("Found {} symbols", constants.len()); - table.add(constants); - } + let v = vec![]; + let classes = classes.get(&handle).unwrap_or(&v); + + let mut visited = HashSet::new(); + let mut scope_stack = vec![(0, 0)]; + scanner::scan_document( + &text, + &dict, + includes, + source, + &classes, + &mut table, + &mut visited, + &mut scope_stack, + ); + LanguageServer::update_watchers(&visited, handle); symbol_table.insert(handle, table); status_change(handle, Status::Idle); @@ -288,6 +295,7 @@ impl LanguageServer { } } +/// Send status change message to Sanny Builder fn status_change(handle: u32, status: Status) { use crate::sdk::messages::{send_message, WM_ONSTATUSCHANGE}; send_message(WM_ONSTATUSCHANGE, handle as _, status as _) diff --git a/src/language_service/symbol_table.rs b/src/language_service/symbol_table.rs index b54c939..bacd9b5 100644 --- a/src/language_service/symbol_table.rs +++ b/src/language_service/symbol_table.rs @@ -1,8 +1,43 @@ -use super::ffi::SymbolInfoMap; use std::collections::HashMap; +use crate::utils::visibility_zone::VisibilityZone; + + +#[repr(C)] +#[derive(Debug, Clone, Copy, PartialEq)] +pub enum SymbolType { + Number = 0, + String = 1, + Var = 2, + Label = 3, + ModelName = 4, + Function = 5, +} + +#[derive(Clone, Debug)] +pub struct SymbolInfoMap { + pub zones: Vec, + pub stack_id: u32, + pub _type: SymbolType, + pub value: Option, // value of the symbol (for literals) + pub name_no_format: String, // used for autocomplete + pub annotation: Option, +} + +impl SymbolInfoMap { + pub fn is_visible_at(&self, line_number: usize) -> bool { + self.zones + .iter() + .any(|zone| zone.is_visible_at(line_number)) + } + + pub fn add_zone(&mut self, start: usize) { + self.zones.push(VisibilityZone { start, end: 0 }); + } +} + pub struct SymbolTable { - pub symbols: HashMap, + pub symbols: HashMap>, } impl SymbolTable { @@ -12,7 +47,7 @@ impl SymbolTable { } } - pub fn add(&mut self, constants: Vec<(String, SymbolInfoMap)>) { - self.symbols.extend(constants); + pub fn extend(&mut self, from: &SymbolTable) { + self.symbols.extend(from.symbols.clone()); } } diff --git a/src/legacy_ini/ffi.rs b/src/legacy_ini/ffi.rs index c4ce2c3..f8555d8 100644 --- a/src/legacy_ini/ffi.rs +++ b/src/legacy_ini/ffi.rs @@ -1,7 +1,7 @@ use std::ffi::CString; use super::table::*; -use crate::common_ffi::*; +use crate::{common_ffi::*, namespaces::namespaces::Namespaces}; #[no_mangle] pub extern "C" fn legacy_ini_new(game: u8) -> *mut OpcodeTable { @@ -25,7 +25,7 @@ pub unsafe extern "C" fn legacy_ini_load_file(table: *mut OpcodeTable, path: PCh "File {path} loaded. Max opcode: {:04X}, Count: {}", (*table).get_max_opcode(), (*table).len() - ) + ); } else { log::debug!("File {path} already loaded"); } @@ -34,6 +34,26 @@ pub unsafe extern "C" fn legacy_ini_load_file(table: *mut OpcodeTable, path: PCh }) } +#[no_mangle] +pub unsafe extern "C" fn legacy_ini_load_from_library( + table: *mut OpcodeTable, + ns: *mut Namespaces, +) -> bool { + boolclosure!({ + { + let ns = ns.as_mut()?; + if (*table).load_from_json(&ns.commands) { + log::debug!( + "Successfully converted library definitions to legacy INI. Max opcode: {:04X}, Count: {}", + (*table).get_max_opcode(), + (*table).len() + ) + } + Some(()) + } + }) +} + #[no_mangle] pub unsafe extern "C" fn legacy_ini_get_param_real_index( table: *mut OpcodeTable, @@ -50,6 +70,7 @@ pub unsafe extern "C" fn legacy_ini_get_max_opcode(table: *mut OpcodeTable) -> u #[no_mangle] pub unsafe extern "C" fn legacy_ini_get_params_count(table: *mut OpcodeTable, opcode: u16) -> u8 { + // todo: make this command fallible https://github.com/sannybuilder/dev/issues/150 (*table).get_params_count(opcode) } @@ -121,3 +142,11 @@ pub unsafe extern "C" fn legacy_ini_is_variadic_opcode( ) -> bool { (*table).is_variadic_opcode(opcode) } + +#[no_mangle] +pub unsafe extern "C" fn legacy_ini_get_is_scr( + table: *mut OpcodeTable, + opcode: u16, +) -> bool { + (*table).get_param_is_scr(opcode) +} diff --git a/src/legacy_ini/table.rs b/src/legacy_ini/table.rs index 481d5f9..44fff3e 100644 --- a/src/legacy_ini/table.rs +++ b/src/legacy_ini/table.rs @@ -14,19 +14,24 @@ use nom_locate::LocatedSpan; use std::collections::HashMap; use std::ffi::CString; +use crate::namespaces::{ + Command, CommandParam, CommandParamSource, CommandParamType, OpId, Operator, +}; + type Span<'a> = LocatedSpan<&'a str>; type R<'a, T> = IResult, T>; #[derive(Debug, Default)] pub struct Opcode { num_params: i8, - params: HashMap, + params: HashMap, words: HashMap, + is_scr: bool, } #[derive(Debug)] pub struct Param { - real_index: u8, + real_index: u8, // param index as the game expects param_type: ParamType, } @@ -41,9 +46,10 @@ pub enum ParamType { String8 = 6, // only lcs IdeModel = 7, Byte128 = 8, // only SA + Float = 9, } -#[derive(Debug)] +#[derive(Debug, PartialEq)] pub enum Game { GTA3, VC, @@ -63,15 +69,25 @@ impl From for Game { 4 => Game::VCS, 5 => Game::SAMOBILE, _ => { - log::error!("Unknown game: {game}, using SA as the default value"); - Game::SA + log::error!( + "Unknown game: {game}, using default value {:?}", + Game::default() + ); + Default::default() } } } } +impl Default for Game { + fn default() -> Self { + Game::SA + } +} + #[derive(Debug, Default)] pub struct OpcodeTable { + game: Game, opcodes: HashMap, files: Vec, date: Option, @@ -154,10 +170,14 @@ pub fn opcode_line(s: Span) -> R { } } + let is_scr = params.iter().fold(true, |acc, (index, p)| { + acc && p.real_index as usize == *index + }); Line::Opcode( id, Opcode { num_params, + is_scr, params, words, }, @@ -233,7 +253,8 @@ pub fn decimal_span(s: Span) -> R { impl OpcodeTable { pub fn new(game: Game) -> Self { OpcodeTable { - max_params: get_game_limit(game), + max_params: get_game_limit(&game), + game, ..Default::default() } } @@ -257,6 +278,174 @@ impl OpcodeTable { return true; } + fn get_word_for_param( + &self, + word_index: usize, + param_name: &str, + c: &Command, + param: Option<&CommandParam>, + ) -> CString { + let mut word = String::new(); + + if word_index == 0 { + if c.attrs.is_condition { + word.push_str(" "); + } + + match c.operator { + Some(op) if c.input.len() == 1 && c.output.is_empty() => { + // [unary operator]var + word.push_str(op.into()); // add unary operator + } + Some(op) if op.is_bitwise() | self.is_ternary_command(c) => { + // bitwise & ternary ops don't need keywords + } + _ => { + word.push_str(c.name.to_lowercase().as_str()); + + if !param_name.is_empty() && !param_name.eq_ignore_ascii_case("self") { + match param.map(|p| &p.source) { + Some( + CommandParamSource::AnyVar + | CommandParamSource::AnyVarGlobal + | CommandParamSource::AnyVarLocal, + ) => { + word.push_str(format!(" {{var_{}}}", param_name).as_str()); + } + _ => { + word.push_str(format!(" {{{}}}", param_name).as_str()); + } + } + } + + word.push(' '); + } + }; + return CString::new(word).unwrap(); + } + + if c.operator.is_some() { + word.push(' '); + + if self.is_ternary_command(c) { + // [var] = [op1] [operator] [op2] + if word_index == 1 { + word.push_str(Operator::Assignment.into()); + } + if word_index == 2 { + let op: &str = c.operator.unwrap().into(); + word.push_str(op); + } + } else { + // var [operator] [op1] + match c.operator { + Some(op) => match op { + Operator::Assignment + | Operator::TimedAddition + | Operator::TimedSubtraction + | Operator::CastAssignment + | Operator::IsEqualTo + | Operator::IsGreaterThan + | Operator::IsGreaterOrEqualTo => word.push_str(op.into()), + Operator::Not => { + word.push_str("= ~"); + return CString::new(word).unwrap(); // don't add ' ' + } + _ => { + word.push_str(op.into()); + word.push_str(Operator::Assignment.into()); + } + }, + None => {} + } + } + } else if !param_name.is_empty() { + match param.map(|p| &p.source) { + Some( + CommandParamSource::AnyVar + | CommandParamSource::AnyVarGlobal + | CommandParamSource::AnyVarLocal, + ) => { + word.push_str(format!(" {{var_{}}}", param_name).as_str()); + } + _ => { + word.push_str(format!(" {{{}}}", param_name).as_str()); + } + } + } + word.push(' '); + CString::new(word).unwrap() + } + + fn is_ternary_command(&self, command: &Command) -> bool { + command.input.len() == 2 && command.output.len() == 1 && command.operator.is_some() + } + + pub fn load_from_json(&mut self, commands: &HashMap) -> bool { + for (_, c) in commands { + let mut params: HashMap = HashMap::new(); + + let mut is_variadic = false; + let mut words: HashMap = HashMap::new(); + let iter = c.input.iter().chain(c.output.iter()); + for (real_index, param) in iter.enumerate() { + if param.r#type == CommandParamType::Arguments { + is_variadic = true; + } + + // when an operator is used, put output params (if any) before input params + // in decompiled code they would look like: [out] = [arg1] [op] [arg2] + let index = if c.operator.is_some() { + // only math commands are allowed to reorder arguments + if real_index < c.input.len() { + c.output.len() + real_index + } else { + real_index - c.input.len() + } + } else { + real_index // never reorder params in regular commands + }; + params.insert( + index, + Param { + real_index: real_index as u8, + param_type: match param.r#type { + CommandParamType::Gxt => ParamType::Gxt, + CommandParamType::Pointer => ParamType::Pointer, + CommandParamType::AnyModel => ParamType::AnyModel, + CommandParamType::ScriptId => ParamType::ScriptId, + CommandParamType::String8 if self.game == Game::LCS => { + ParamType::String8 + } + CommandParamType::IdeModel => ParamType::IdeModel, + CommandParamType::Byte128 => ParamType::Byte128, + CommandParamType::Float => ParamType::Float, + _ => ParamType::Any, + }, + }, + ); + words.insert( + index, + self.get_word_for_param(index, param.name.trim(), c, Some(param)), + ); + } + + if words.is_empty() { + words.insert(0, self.get_word_for_param(0, "", c, None)); + } + + let opcode = Opcode { + num_params: if is_variadic { -1 } else { c.num_params as i8 }, + params, + words, + is_scr: true, // all params are in original order (except math) + }; + self.add_opcode(c.id, opcode); + } + + return true; + } + pub fn parse_line(&mut self, line: &str) { let line = line.trim(); let line = line.replace(";;", "//"); @@ -328,6 +517,10 @@ impl OpcodeTable { .unwrap_or(ParamType::Any) } + pub fn get_param_is_scr(&self, id: u16) -> bool { + self.get_opcode(id).map(|opcode| opcode.is_scr).unwrap_or(true) + } + pub fn does_word_exist(&self, id: u16, index: usize) -> bool { self.get_word(id, index).is_some() } @@ -354,7 +547,7 @@ impl OpcodeTable { } } -fn get_game_limit(game: Game) -> u8 { +fn get_game_limit(game: &Game) -> u8 { match game { Game::GTA3 => 16 + 2, Game::VC => 16 + 2, @@ -437,6 +630,98 @@ mod tests { assert_eq!(opcode_table.get_param_real_index(id, 4), 3); } + #[test] + fn test_real_index2() { + let mut opcode_table = OpcodeTable::new(Game::SA); + + let mut commands = HashMap::new(); + commands.insert( + 0x0001, + Command { + id: 0x0001, + name: "INT_ADD".to_string(), + num_params: 3, + short_desc: "".to_string(), + class: None, + member: None, + attrs: crate::namespaces::Attr::default(), + input: vec![ + CommandParam { + name: "".to_string(), + r#type: CommandParamType::IdeModel, + source: CommandParamSource::Any, + }, + CommandParam { + name: "".to_string(), + r#type: CommandParamType::IdeModel, + source: CommandParamSource::Any, + }, + ], + output: vec![CommandParam { + name: "".to_string(), + r#type: CommandParamType::Any, + source: CommandParamSource::AnyVar, + }], + platforms: vec![], + versions: vec![], + // operator: None, + operator: Some(Operator::Assignment), + }, + ); + commands.insert( + 0x0002, + Command { + id: 0x0002, + name: "INT_CMP".to_string(), + num_params: 2, + short_desc: "".to_string(), + class: None, + member: None, + attrs: crate::namespaces::Attr::default(), + input: vec![ + CommandParam { + name: "".to_string(), + r#type: CommandParamType::IdeModel, + source: CommandParamSource::Any, + }, + CommandParam { + name: "".to_string(), + r#type: CommandParamType::AnyModel, + source: CommandParamSource::Any, + }, + ], + output: vec![], + platforms: vec![], + versions: vec![], + operator: Some(Operator::IsEqualTo), + }, + ); + + opcode_table.load_from_json(&commands); + + let id = 0x0001; + assert_eq!(opcode_table.get_params_count(id), 3); + + assert_eq!(opcode_table.get_param_real_index(id, 0), 2); + assert_eq!(opcode_table.get_param_type(id, 0), ParamType::Any); + + assert_eq!(opcode_table.get_param_type(id, 1), ParamType::IdeModel); + assert_eq!(opcode_table.get_param_real_index(id, 1), 0); + + assert_eq!(opcode_table.get_param_type(id, 2), ParamType::IdeModel); + assert_eq!(opcode_table.get_param_real_index(id, 2), 1); + + let id = 0x0002; + assert_eq!(opcode_table.get_params_count(id), 2); + assert_eq!(opcode_table.get_param_real_index(id, 0), 0); + assert_eq!(opcode_table.get_param_real_index(id, 1), 1); + + assert_eq!(opcode_table.get_param_type(id, 0), ParamType::IdeModel); + assert_eq!(opcode_table.get_param_type(id, 1), ParamType::AnyModel); + + + } + #[test] fn test_bool_param() { let mut opcode_table = OpcodeTable::new(Game::SA); diff --git a/src/lib.rs b/src/lib.rs index 633e4ed..a66a12a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,9 +1,8 @@ -#![feature(never_type)] -#![feature(hash_drain_filter)] - use ctor::ctor; use simplelog::*; +#[macro_use] +extern crate lazy_static; #[macro_use] pub mod common_ffi; pub mod dictionary; @@ -15,6 +14,9 @@ pub mod sdk; pub mod update_service; pub mod utils; pub mod v4; +pub mod source_map; +pub mod preprocessor; +pub mod sanny_update; #[ctor] fn main() { diff --git a/src/namespaces/ffi.rs b/src/namespaces/ffi.rs index 0b41a5c..ac573f2 100644 --- a/src/namespaces/ffi.rs +++ b/src/namespaces/ffi.rs @@ -1,5 +1,5 @@ use crate::common_ffi::*; -use crate::namespaces::namespaces::*; +use crate::namespaces::{namespaces::*, CommandParamType}; #[no_mangle] pub extern "C" fn classes_new() -> *mut Namespaces { @@ -30,7 +30,7 @@ pub unsafe extern "C" fn classes_load_library(ns: *mut Namespaces, file_name: PC #[no_mangle] pub unsafe extern "C" fn classes_get_short_description_by_id( ns: *mut Namespaces, - opcode: u16, + opcode: OpId, out: *mut PChar, ) -> bool { boolclosure! {{ @@ -42,7 +42,7 @@ pub unsafe extern "C" fn classes_get_short_description_by_id( #[no_mangle] pub unsafe extern "C" fn classes_get_is_condition_by_id( ns: *mut Namespaces, - opcode: u16, + opcode: OpId, out: *mut bool, ) -> bool { boolclosure! {{ @@ -54,7 +54,7 @@ pub unsafe extern "C" fn classes_get_is_condition_by_id( #[no_mangle] pub unsafe extern "C" fn classes_get_is_branch_by_id( ns: *mut Namespaces, - opcode: u16, + opcode: OpId, out: *mut bool, ) -> bool { boolclosure! {{ @@ -141,6 +141,18 @@ pub unsafe extern "C" fn classes_populate_keywords_dict3( }} } +#[no_mangle] +pub unsafe extern "C" fn classes_populate_extension_list( + ns: *mut Namespaces, + dict: *mut crate::dictionary::dictionary_str_by_num::DictStrByNum, +) -> bool { + boolclosure! {{ + let ns = ns.as_mut()?; + ns.populate_extension_list(dict.as_mut()?); + Some(()) + }} +} + #[no_mangle] pub unsafe extern "C" fn classes_find_by_prop( ns: *mut Namespaces, @@ -307,6 +319,90 @@ pub unsafe extern "C" fn classes_get_library_version(ns: *mut Namespaces, out: * }} } +#[no_mangle] +pub unsafe extern "C" fn classes_get_input_count_by_id( + ns: *mut Namespaces, + opcode: OpId, + out: *mut usize, +) -> bool { + boolclosure! {{ + *out = ns.as_mut()?.get_input_count(opcode)?; + Some(()) + }} +} + +#[no_mangle] +pub unsafe extern "C" fn classes_get_output_count_by_id( + ns: *mut Namespaces, + opcode: OpId, + out: *mut usize, +) -> bool { + boolclosure! {{ + *out = ns.as_mut()?.get_output_count(opcode)?; + Some(()) + }} +} + +#[no_mangle] +pub unsafe extern "C" fn classes_is_input_of_type( + ns: *mut Namespaces, + opcode: OpId, + index: usize, + _type: u8, +) -> bool { + boolclosure! {{ + ns.as_mut()?.is_input_of_type(opcode, index, _type.into())?.then_some(()) + }} +} + +#[no_mangle] +pub unsafe extern "C" fn classes_is_output_of_type( + ns: *mut Namespaces, + opcode: OpId, + index: usize, + _type: u8, +) -> bool { + boolclosure! {{ + ns.as_mut()?.is_output_of_type(opcode, index, _type.into())?.then_some(()) + }} +} + +#[no_mangle] +pub unsafe extern "C" fn classes_get_input_type( + ns: *mut Namespaces, + opcode: OpId, + index: usize, + _type: *mut u8, +) -> bool { + boolclosure! {{ + ns.as_mut()?.get_input_type(opcode, index).map(|x| *_type = { + match x { + CommandParamType::Float => 2, + CommandParamType::String8 | CommandParamType::Gxt => 3, + _ => 1, + } + }) + }} +} + +#[no_mangle] +pub unsafe extern "C" fn classes_get_output_type( + ns: *mut Namespaces, + opcode: OpId, + index: usize, + _type: *mut u8, +) -> bool { + boolclosure! {{ + ns.as_mut()?.get_output_type(opcode, index).map(|x| *_type = { + match x { + CommandParamType::Float => 2, + CommandParamType::String8 | CommandParamType::Gxt => 3, + _ => 1, + } + }) + }} +} + #[cfg(test)] mod tests { use super::*; diff --git a/src/namespaces/library.rs b/src/namespaces/library.rs index 8becb57..889113b 100644 --- a/src/namespaces/library.rs +++ b/src/namespaces/library.rs @@ -1,14 +1,35 @@ use serde::{Deserialize, Deserializer}; -#[derive(Debug, PartialEq)] +#[derive(Debug, PartialEq, Copy, Clone)] pub enum CommandParamType { - Int, - Float, - String, - Boolean, + Gxt, + Pointer, + AnyModel, + ScriptId, + String8, + IdeModel, + Byte128, Arguments, - Vector(usize), - Any(String), + Float, + Any, +} + +impl From for CommandParamType { + fn from(val: u8) -> Self { + match val { + 0 => Self::Any, + 1 => Self::Gxt, + 2 => Self::Pointer, + 3 => Self::AnyModel, + 4 => Self::ScriptId, + 5 => Self::String8, + 6 => Self::IdeModel, + 7 => Self::Byte128, + 8 => Self::Arguments, + 9 => Self::Float, + _ => Self::Any, + } + } } #[derive(Debug, PartialEq)] @@ -36,23 +57,44 @@ pub enum Version { _10DE, } +#[derive(Debug, Copy, Clone)] +pub enum Operator { + Assignment, + Addition, + Subtraction, + Multiplication, + Division, + TimedAddition, + TimedSubtraction, + CastAssignment, + IsEqualTo, + IsGreaterThan, + IsGreaterOrEqualTo, + And, + Or, + Xor, + Not, + Mod, + ShiftLeft, + ShiftRight, +} + impl<'de> Deserialize<'de> for CommandParamType { fn deserialize(deserializer: D) -> Result where D: Deserializer<'de>, { match String::deserialize(deserializer).as_deref() { - Ok("float") => Ok(Self::Float), - Ok("int" | "label" | "model_any" | "model_char" | "model_object" | "model_vehicle") => { - Ok(Self::Int) - } - Ok("string" | "gxt_key" | "zone_key") => Ok(Self::String), - Ok("bool" | "boolean") => Ok(Self::Boolean), + Ok("gxt_key") | Ok("zone_key") => Ok(Self::Gxt), + Ok("label") => Ok(Self::Pointer), + Ok("model_any") | Ok("model_object") => Ok(Self::AnyModel), + Ok("model_char") | Ok("model_vehicle") => Ok(Self::IdeModel), + Ok("script_id") => Ok(Self::ScriptId), + Ok("string128") => Ok(Self::Byte128), + Ok("string") => Ok(Self::String8), Ok("arguments") => Ok(Self::Arguments), - Ok("Object") => Ok(Self::Any("ScriptObject".to_string())), - Ok("Vector3") => Ok(Self::Vector(3)), - Ok(name) => Ok(Self::Any(name.to_string())), - _ => Ok(Self::Int), + Ok("float") => Ok(Self::Float), + _ => Ok(Self::Any), } } } @@ -127,6 +169,8 @@ pub struct Command { pub platforms: Vec, #[serde(default, deserialize_with = "convert_version")] pub versions: Vec, + #[serde(default, deserialize_with = "convert_operator")] + pub operator: Option, } #[derive(Deserialize, Debug)] @@ -222,3 +266,72 @@ where }; Ok(res) } + +fn convert_operator<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + let val = String::deserialize(deserializer)?; + + match val.as_str() { + "=" => Ok(Some(Operator::Assignment)), + "+" => Ok(Some(Operator::Addition)), + "-" => Ok(Some(Operator::Subtraction)), + "*" => Ok(Some(Operator::Multiplication)), + "/" => Ok(Some(Operator::Division)), + "+=@" => Ok(Some(Operator::TimedAddition)), + "-=@" => Ok(Some(Operator::TimedSubtraction)), + "=#" => Ok(Some(Operator::CastAssignment)), + "==" => Ok(Some(Operator::IsEqualTo)), + ">" => Ok(Some(Operator::IsGreaterThan)), + ">=" => Ok(Some(Operator::IsGreaterOrEqualTo)), + "&" => Ok(Some(Operator::And)), + "|" => Ok(Some(Operator::Or)), + "^" => Ok(Some(Operator::Xor)), + "~" => Ok(Some(Operator::Not)), + "%" => Ok(Some(Operator::Mod)), + "<<" => Ok(Some(Operator::ShiftLeft)), + ">>" => Ok(Some(Operator::ShiftRight)), + _ => Ok(None), + } +} + +impl Into<&str> for Operator { + fn into(self) -> &'static str { + match self { + Operator::Assignment => "=", + Operator::Addition => "+", + Operator::Subtraction => "-", + Operator::Multiplication => "*", + Operator::Division => "/", + Operator::TimedAddition => "+=@", + Operator::TimedSubtraction => "-=@", + Operator::CastAssignment => "=#", + Operator::IsEqualTo => "==", + Operator::IsGreaterThan => ">", + Operator::IsGreaterOrEqualTo => ">=", + Operator::And => "&", + Operator::Or => "|", + Operator::Xor => "^", + Operator::Not => "~", + Operator::Mod => "%", + Operator::ShiftLeft => "<<", + Operator::ShiftRight => ">>", + } + } +} + +impl Operator { + pub fn is_bitwise(&self) -> bool { + match self { + Operator::And + | Operator::Or + | Operator::Xor + | Operator::Not + | Operator::ShiftLeft + | Operator::ShiftRight + | Operator::Mod => true, + _ => false, + } + } +} diff --git a/src/namespaces/mod.rs b/src/namespaces/mod.rs index 7273364..076ffb2 100644 --- a/src/namespaces/mod.rs +++ b/src/namespaces/mod.rs @@ -4,4 +4,11 @@ mod ffi; mod library; pub mod namespaces; -pub use library::Library; \ No newline at end of file +pub use library::Command; +pub use library::CommandParam; +pub use library::CommandParamType; +pub use library::CommandParamSource; +pub use library::Operator; +pub use library::Library; +pub use library::Attr; +pub use namespaces::OpId; \ No newline at end of file diff --git a/src/namespaces/namespaces.rs b/src/namespaces/namespaces.rs index 347e353..d84a8e5 100644 --- a/src/namespaces/namespaces.rs +++ b/src/namespaces/namespaces.rs @@ -7,7 +7,10 @@ use crate::dictionary::{ list_num_by_str::ListNumByStr, }; -use super::library::{Attr, Library}; +use super::{ + library::{Command, Library}, + CommandParamType, +}; /** * this is a remnant of old Sanny type system where built-in types as Int, Float, Handle, etc @@ -24,9 +27,10 @@ pub struct Namespaces { enums: Vec, // case-preserved opcodes: Vec, short_descriptions: HashMap, - attrs: HashMap, + pub commands: HashMap, + extensions: HashMap, map_op_by_id: HashMap, - map_op_by_name: HashMap< + pub map_op_by_name: HashMap< /*class_name*/ String, HashMap, >, @@ -126,7 +130,8 @@ impl Namespaces { opcodes: vec![], enums: vec![], short_descriptions: HashMap::new(), - attrs: HashMap::new(), + commands: HashMap::new(), + extensions: HashMap::new(), map_op_by_id: HashMap::new(), map_op_by_name: HashMap::new(), map_op_by_command_name: HashMap::new(), @@ -198,11 +203,11 @@ impl Namespaces { return None; }; - let mut names_str: Vec = vec![]; + let mut class_names: Vec = vec![]; while let Some(line) = line_iter.next() { if !line.starts_with(|c| c == '#' || c == '$') { self.names.push(CString::new(line).ok()?); - names_str.push(String::from(line)); + class_names.push(String::from(line)); continue; } @@ -240,26 +245,27 @@ impl Namespaces { } let name = &line[1..]; - let find_name = match names_str.iter().find(|n| n.eq_ignore_ascii_case(name)) { + let find_class_name = match class_names.iter().find(|n| n.eq_ignore_ascii_case(name)) { Some(x) => x, None => continue, // undeclared class -> skip }; if line_iter.next()?.eq_ignore_ascii_case("$begin") { - let name = &find_name.clone(); + let class_name = &find_class_name.clone(); let mut map: HashMap = HashMap::new(); for line in line_iter .by_ref() .take_while(|line| !line.starts_with(|c| c == '#' || c == '$')) { - match self.parse_method(line, name, &mut map) { + match self.parse_method(line, class_name, &mut map) { Some(_) => {} None => { log::debug!("Can't parse the line {}", line); } } } - self.map_op_by_name.insert(name.to_ascii_lowercase(), map); + self.map_op_by_name + .insert(class_name.to_ascii_lowercase(), map); } } Some(()) @@ -268,7 +274,7 @@ impl Namespaces { fn parse_method( &mut self, line: &str, - class_name: &String, + class_name: &str, map: &mut HashMap, ) -> Option<()> { use crate::namespaces::classes_parser::method; @@ -318,7 +324,7 @@ impl Namespaces { fn parse_prop( &mut self, line: &str, - class_name: &String, + class_name: &str, map: &mut HashMap, ) -> Option<()> { use crate::namespaces::classes_parser::property; @@ -352,7 +358,7 @@ impl Namespaces { params.iter().cloned().collect::>() }; - let op_index = self.register_opcode(Opcode { + let op = Opcode { hint: self.params_to_string(¶ms)?, params_len: prop_params.len() as i32, id, @@ -364,7 +370,9 @@ impl Namespaces { prop_type: OpcodeType::Property, operation: CString::new(operation).ok()?, short_desc: short_desc.clone(), - }); + }; + let supports_alternate_method_syntax = self.supports_alternate_method_syntax(&op); + let op_index = self.register_opcode(op); let key = PropKey { name, prop_pos, @@ -373,8 +381,12 @@ impl Namespaces { map.insert(String::from(key).to_ascii_lowercase(), op_index); self.props.push(name.to_ascii_lowercase()); - if op_type == OpcodeType::Property { - // add a method version of this opcode with all params + if supports_alternate_method_syntax { + // sb3 quirk + // constructors can be written in two ways: + // Player.Create($PLAYER_CHAR, #NULL, 2488.5601, -1666.84, 13.38) + // $PLAYER_CHAR = Player.Create(#NULL, 2488.5601, -1666.84, 13.38) + // hence we need to add a method version of this opcode with all params let op_index = self.register_opcode(Opcode { hint: self.params_to_string(¶ms)?, params_len: params.len() as i32, @@ -399,7 +411,7 @@ impl Namespaces { fn parse_params( &mut self, params: &Vec, - full_name: &String, + full_name: &str, ) -> Vec { use crate::namespaces::classes_parser::ParamType; @@ -671,7 +683,7 @@ impl Namespaces { } let op = self.get_opcode_by_index(*index)?; - if op.help_code == -2 /* deprecated */ || /* has the counterpart method */ op.op_type == OpcodeType::Property + if op.help_code == -2 /* deprecated */ || /* has the counterpart method */ self.supports_alternate_method_syntax(op) { return None; }; @@ -680,10 +692,15 @@ impl Namespaces { }).collect::>()) } + fn supports_alternate_method_syntax(&self, op: &Opcode) -> bool { + op.op_type == OpcodeType::Property //&& self.is_constructor(op.id).unwrap_or(false) + } + pub fn has_prop(&self, prop_name: &str) -> bool { self.props.contains(&prop_name.to_ascii_lowercase()) } + // load a JSON from SBL pub fn load_library<'a>(&mut self, file_name: &'a str) -> Option<()> { let content = fs::read_to_string(file_name).ok()?; @@ -691,18 +708,17 @@ impl Namespaces { self.library_version = CString::new(lib.meta.version).ok()?; - for command in lib - .extensions - .into_iter() - .flat_map(|ext| ext.commands.into_iter()) - .filter(|c| !c.attrs.is_unsupported) - { - self.short_descriptions - .insert(command.id, CString::new(command.short_desc).ok()?); - self.attrs.insert(command.id, command.attrs); - self.map_op_by_command_name - .insert(command.name.to_ascii_lowercase(), command.id); + for ext in lib.extensions.into_iter() { + for command in ext.commands.into_iter().filter(|c| !c.attrs.is_unsupported) { + self.extensions.insert(command.id, ext.name.clone()); + self.short_descriptions + .insert(command.id, CString::new(command.short_desc.clone()).ok()?); + self.map_op_by_command_name + .insert(command.name.to_ascii_lowercase(), command.id); + self.commands.insert(command.id, command); + } } + Some(()) } @@ -716,11 +732,15 @@ impl Namespaces { } pub fn is_condition<'a>(&self, id: OpId) -> Option { - self.attrs.get(&id).map(|attrs| attrs.is_condition) + self.commands.get(&id).map(|c| c.attrs.is_condition) + } + + pub fn is_constructor<'a>(&self, id: OpId) -> Option { + self.commands.get(&id).map(|c| c.attrs.is_constructor) } pub fn is_branch<'a>(&self, id: OpId) -> Option { - self.attrs.get(&id).map(|attrs| attrs.is_branch) + self.commands.get(&id).map(|c| c.attrs.is_branch) } pub fn get_library_version(&self) -> &CString { @@ -728,9 +748,9 @@ impl Namespaces { } pub fn populate_keywords<'a>(&mut self, dict: &mut DictNumByStr) -> Option<()> { - use crate::dictionary::ffi::apply_format_s; + use crate::dictionary::ffi::apply_format; for (name, op) in self.map_op_by_command_name.iter() { - let key = apply_format_s(name, &dict.config.case_format); + let key = apply_format(name, &dict.config.case_format)?; dict.add(key, *op as _); } Some(()) @@ -753,4 +773,62 @@ impl Namespaces { } Some(()) } + + pub fn populate_extension_list<'a>(&mut self, dict: &mut DictStrByNum) -> Option<()> { + for (op, name) in self.extensions.iter() { + dict.add(*op as _, CString::new(name.clone()).unwrap()); + } + Some(()) + } + + pub fn get_input_count<'a>(&self, id: OpId) -> Option { + self.commands.get(&id).map(|c| c.input.len()) + } + + pub fn get_output_count<'a>(&self, id: OpId) -> Option { + self.commands.get(&id).map(|c| c.output.len()) + } + + pub fn is_input_of_type( + &self, + id: OpId, + index: usize, + _type: CommandParamType, + ) -> Option { + self.commands + .get(&id) + .map(|c| c.input.get(index).map_or(false, |i| i.r#type == _type)) + } + + pub fn is_output_of_type( + &self, + id: OpId, + index: usize, + _type: CommandParamType, + ) -> Option { + self.commands + .get(&id) + .map(|c| c.output.get(index).map_or(false, |i| i.r#type == _type)) + } + + pub fn get_input_type( + &self, + id: OpId, + index: usize, + ) -> Option<&CommandParamType> { + self.commands + .get(&id) + .and_then(|c| c.input.get(index).map(|i| &i.r#type)) + } + + + pub fn get_output_type( + &self, + id: OpId, + index: usize, + ) -> Option<&CommandParamType> { + self.commands + .get(&id) + .and_then(|c| c.output.get(index).map(|i| &i.r#type)) + } } diff --git a/src/parser/binary.rs b/src/parser/binary.rs index 545fc3d..56eb4ae 100644 --- a/src/parser/binary.rs +++ b/src/parser/binary.rs @@ -250,4 +250,44 @@ mod tests { } ); } + + #[test] + fn test_const_name2() { + let (_, ast) = parse("x &= &7").unwrap(); + assert_eq!( + ast, + AST { + body: vec![Node::Binary(BinaryExpr { + left: Box::new(Node::Literal(Token { + start: 1, + len: 1, + syntax_kind: SyntaxKind::Identifier + })), + operator: Token { + start: 3, + len: 2, + syntax_kind: SyntaxKind::OperatorBitwiseAndEqual + }, + right: Box::new(Node::Variable(Variable::Adma(SingleVariable { + name: Token { + syntax_kind: SyntaxKind::IntegerLiteral, + start: 7, + len: 1 + }, + token: Token { + syntax_kind: SyntaxKind::AdmaVariable, + start: 6, + len: 2 + }, + _type: VariableType::Unknown + }))), + token: Token { + start: 1, + len: 7, + syntax_kind: SyntaxKind::BinaryExpr + } + })] + } + ); + } } diff --git a/src/parser/declaration.rs b/src/parser/declaration.rs index 4cefd58..5bcee9d 100644 --- a/src/parser/declaration.rs +++ b/src/parser/declaration.rs @@ -1,8 +1,9 @@ use crate::parser::interface::*; use nom::bytes::complete::{tag, tag_no_case}; -use nom::combinator::map; -use nom::multi::many1; -use nom::sequence::{delimited, separated_pair}; +use nom::character::complete::{one_of, space0, space1}; +use nom::combinator::{map, opt}; +use nom::multi::{many1, separated_list0, separated_list1}; +use nom::sequence::{delimited, pair, preceded, separated_pair, tuple}; use nom::{branch::alt, combinator::consumed}; use nom::{character::complete::multispace0, sequence::terminated}; @@ -15,6 +16,111 @@ pub fn declaration(s: Span) -> R { terminated(alt((statement::statement, const_declaration)), multispace0)(s) } +pub fn function_signature(s: Span) -> R { + map( + consumed(helpers::line(tuple(( + helpers::ws(terminated(tag_no_case("function"), space1)), + helpers::ws(literal::identifier), + helpers::ws(opt(function_cc)), + function_arguments_and_return_types, + )))), + |(span, (_, name, cc, (parameters, return_types)))| { + let (cc, address) = cc.unwrap_or((FunctionCC::Local, None)); + FunctionSignature { + name, + parameters, + return_types, + cc, + address, + token: Token::from(span, SyntaxKind::FunctionSignature), + } + }, + )(s) +} + +pub fn function_arguments_and_return_types( + s: Span, +) -> R<(Vec, Vec)> { + map( + tuple(( + helpers::ws(opt(function_arguments)), + helpers::ws(opt(function_return_types)), + )), + |(parameters, return_types)| { + ( + parameters.unwrap_or_default(), + return_types.unwrap_or_default(), + ) + }, + )(s) +} + +fn function_arguments(s: Span) -> R> { + map( + delimited( + helpers::ws(tag("(")), + separated_list0( + helpers::ws(tag(",")), + consumed(pair( + opt(terminated( + // param names are optional in define function + helpers::ws(literal::identifier), + helpers::ws(tag(":")), + )), + helpers::ws(literal::identifier), + )), + ), + helpers::ws(tag(")")), + ), + |args| { + args.into_iter() + .map(|(span, (name, _type))| FunctionParameter { + name, + _type, + token: Token::from(span, SyntaxKind::LocalVariable), + }) + .collect() + }, + )(s) +} + +fn function_return_types(s: Span) -> R> { + map( + tuple(( + helpers::ws(tag(":")), + opt(tag_no_case("optional")), + separated_list1(helpers::ws(tag(",")), helpers::ws(literal::identifier)), + )), + |(_, _, types)| { + types + .into_iter() + .map(|_type| FunctionReturnType { + token: _type.clone(), + _type, + }) + .collect() + }, + )(s) +} + +fn function_cc(s: Span) -> R<(FunctionCC, /*optional address*/ Option)> { + delimited( + tag("<"), + helpers::ws(tuple(( + alt(( + map(tag_no_case("thiscall"), |_| FunctionCC::Thiscall), + map(tag_no_case("cdecl"), |_| FunctionCC::Cdecl), + map(tag_no_case("stdcall"), |_| FunctionCC::Stdcall), + )), + opt(preceded( + helpers::ws(tag(",")), + helpers::ws(literal::number), + )), + ))), + tag(">"), + )(s) +} + pub fn const_declaration(s: Span) -> R { map( consumed(delimited( @@ -107,4 +213,380 @@ end"#, }) ) } + + #[test] + fn test_function_signature() { + let (_, node) = function_signature(Span::from(r#" function foo "#)).unwrap(); + + assert_eq!( + node, + FunctionSignature { + name: Token { + start: 11, + len: 3, + syntax_kind: SyntaxKind::Identifier + }, + parameters: vec![], + return_types: vec![], + cc: FunctionCC::Local, + address: None, + token: Token { + start: 1, + len: 14, + syntax_kind: SyntaxKind::FunctionSignature + } + } + ); + + let (_, node) = function_signature(Span::from(r#"function foo: string"#)).unwrap(); + + assert_eq!( + node, + FunctionSignature { + name: Token { + start: 10, + len: 3, + syntax_kind: SyntaxKind::Identifier + }, + parameters: vec![], + return_types: vec![FunctionReturnType { + token: Token { + start: 15, + len: 6, + syntax_kind: SyntaxKind::Identifier + }, + _type: Token { + start: 15, + len: 6, + syntax_kind: SyntaxKind::Identifier + } + }], + cc: FunctionCC::Local, + address: None, + token: Token { + start: 1, + len: 20, + syntax_kind: SyntaxKind::FunctionSignature + } + } + ); + + let (_, node) = function_signature(Span::from(r#"function foo()"#)).unwrap(); + + assert_eq!( + node, + FunctionSignature { + name: Token { + start: 10, + len: 3, + syntax_kind: SyntaxKind::Identifier + }, + parameters: vec![], + return_types: vec![], + cc: FunctionCC::Local, + address: None, + token: Token { + start: 1, + len: 14, + syntax_kind: SyntaxKind::FunctionSignature + } + } + ); + + let (_, node) = function_signature(Span::from(r#"function foo(): int"#)).unwrap(); + + assert_eq!( + node, + FunctionSignature { + name: Token { + start: 10, + len: 3, + syntax_kind: SyntaxKind::Identifier + }, + parameters: vec![], + return_types: vec![FunctionReturnType { + token: Token { + start: 17, + len: 3, + syntax_kind: SyntaxKind::Identifier + }, + _type: Token { + start: 17, + len: 3, + syntax_kind: SyntaxKind::Identifier + } + }], + cc: FunctionCC::Local, + address: None, + token: Token { + start: 1, + len: 19, + syntax_kind: SyntaxKind::FunctionSignature + } + } + ); + + let (_, node) = function_signature(Span::from(r#"function foo(a: int): int"#)).unwrap(); + + assert_eq!( + node, + FunctionSignature { + name: Token { + start: 10, + len: 3, + syntax_kind: SyntaxKind::Identifier + }, + parameters: vec![FunctionParameter { + name: Some(Token { + start: 14, + len: 1, + syntax_kind: SyntaxKind::Identifier + }), + _type: Token { + start: 17, + len: 3, + syntax_kind: SyntaxKind::Identifier + }, + token: Token { + start: 14, + len: 6, + syntax_kind: SyntaxKind::LocalVariable + } + }], + return_types: vec![FunctionReturnType { + token: Token { + start: 23, + len: 3, + syntax_kind: SyntaxKind::Identifier + }, + _type: Token { + start: 23, + len: 3, + syntax_kind: SyntaxKind::Identifier + } + }], + cc: FunctionCC::Local, + address: None, + token: Token { + start: 1, + len: 25, + syntax_kind: SyntaxKind::FunctionSignature + } + } + ); + + let (_, node) = + function_signature(Span::from(r#"function foo(a: int, b: string): int"#)).unwrap(); + + assert_eq!( + node, + FunctionSignature { + name: Token { + start: 10, + len: 3, + syntax_kind: SyntaxKind::Identifier + }, + parameters: vec![ + FunctionParameter { + name: Some(Token { + start: 14, + len: 1, + syntax_kind: SyntaxKind::Identifier + }), + _type: Token { + start: 17, + len: 3, + syntax_kind: SyntaxKind::Identifier + }, + token: Token { + start: 14, + len: 6, + syntax_kind: SyntaxKind::LocalVariable + } + }, + FunctionParameter { + name: Some(Token { + start: 22, + len: 1, + syntax_kind: SyntaxKind::Identifier + }), + _type: Token { + start: 25, + len: 6, + syntax_kind: SyntaxKind::Identifier + }, + token: Token { + start: 22, + len: 9, + syntax_kind: SyntaxKind::LocalVariable + } + } + ], + return_types: vec![FunctionReturnType { + token: Token { + start: 34, + len: 3, + syntax_kind: SyntaxKind::Identifier + }, + _type: Token { + start: 34, + len: 3, + syntax_kind: SyntaxKind::Identifier + } + }], + cc: FunctionCC::Local, + address: None, + token: Token { + start: 1, + len: 36, + syntax_kind: SyntaxKind::FunctionSignature + } + } + ); + + let (_, node) = function_signature(Span::from( + r#"function foo(a: int, b: string): int, int, int"#, + )) + .unwrap(); + + assert_eq!( + node, + FunctionSignature { + name: Token { + start: 10, + len: 3, + syntax_kind: SyntaxKind::Identifier + }, + parameters: vec![ + FunctionParameter { + name: Some(Token { + start: 14, + len: 1, + syntax_kind: SyntaxKind::Identifier + }), + _type: Token { + start: 17, + len: 3, + syntax_kind: SyntaxKind::Identifier + }, + token: Token { + start: 14, + len: 6, + syntax_kind: SyntaxKind::LocalVariable + } + }, + FunctionParameter { + name: Some(Token { + start: 22, + len: 1, + syntax_kind: SyntaxKind::Identifier + }), + _type: Token { + start: 25, + len: 6, + syntax_kind: SyntaxKind::Identifier + }, + token: Token { + start: 22, + len: 9, + syntax_kind: SyntaxKind::LocalVariable + } + } + ], + return_types: vec![ + FunctionReturnType { + token: Token { + start: 34, + len: 3, + syntax_kind: SyntaxKind::Identifier + }, + _type: Token { + start: 34, + len: 3, + syntax_kind: SyntaxKind::Identifier + } + }, + FunctionReturnType { + token: Token { + start: 39, + len: 3, + syntax_kind: SyntaxKind::Identifier + }, + _type: Token { + start: 39, + len: 3, + syntax_kind: SyntaxKind::Identifier + } + }, + FunctionReturnType { + token: Token { + start: 44, + len: 3, + syntax_kind: SyntaxKind::Identifier + }, + _type: Token { + start: 44, + len: 3, + syntax_kind: SyntaxKind::Identifier + } + } + ], + cc: FunctionCC::Local, + address: None, + token: Token { + start: 1, + len: 46, + syntax_kind: SyntaxKind::FunctionSignature + } + } + ); + + let (_, node) = function_signature(Span::from(r#"function foo()"#)).unwrap(); + + assert_eq!( + node, + FunctionSignature { + name: Token { + start: 10, + len: 3, + syntax_kind: SyntaxKind::Identifier + }, + parameters: vec![], + return_types: vec![], + cc: FunctionCC::Stdcall, + address: None, + token: Token { + start: 1, + len: 23, + syntax_kind: SyntaxKind::FunctionSignature + } + } + ); + + let (_, node) = function_signature(Span::from(r#"function foo()"#)).unwrap(); + + assert_eq!( + node, + FunctionSignature { + name: Token { + start: 10, + len: 3, + syntax_kind: SyntaxKind::Identifier + }, + parameters: vec![], + return_types: vec![], + cc: FunctionCC::Cdecl, + address: Some(Token { + start: 21, + len: 5, + syntax_kind: SyntaxKind::IntegerLiteral + }), + token: Token { + start: 1, + len: 28, + syntax_kind: SyntaxKind::FunctionSignature + } + } + ); + } } diff --git a/src/parser/interface.rs b/src/parser/interface.rs index 7c1272a..a95f374 100644 --- a/src/parser/interface.rs +++ b/src/parser/interface.rs @@ -5,6 +5,7 @@ pub enum SyntaxKind { Identifier, IntegerLiteral, FloatLiteral, + LabelLiteral, ArrayElementSCR, IndexedVariable, LocalVariable, @@ -14,6 +15,7 @@ pub enum SyntaxKind { BinaryExpr, ConstDeclaration, ConstInitialization, + FunctionSignature, OperatorBitwiseNot, // ~ OperatorBitwiseAnd, // & @@ -77,6 +79,7 @@ pub enum Node { /// Unary expression, e.g. `~var` Unary(UnaryPrefixExpr), ConstDeclaration(ConstDeclaration), + FunctionSignature(FunctionSignature), } #[derive(Debug, PartialEq, Clone)] @@ -86,9 +89,21 @@ pub enum Variable { Indexed(IndexedVariable), ArrayElement(ArrayElementSCR), Adma(SingleVariable), + Push(Token), + Pop(Token), } impl Variable { + pub fn get_var_name(&self) -> &Token { + match self { + Variable::Indexed(v) => &v.var.get_var_name(), + Variable::ArrayElement(v) => &v.array_var.get_var_name(), + Variable::Local(v) | Variable::Global(v) | Variable::Adma(v) => &v.token, + Variable::Pop(v) => v, + Variable::Push(v) => v, + } + } + pub fn is_global(&self) -> bool { match self { Variable::Global(_) | Variable::Adma(_) => true, @@ -103,6 +118,7 @@ impl Variable { Variable::Local(_) => true, Variable::Indexed(v) if v.var.is_local() => true, Variable::ArrayElement(v) if v.array_var.is_local() => true, + Variable::Pop(_) | Variable::Push(_) => true, _ => false, } } @@ -187,5 +203,34 @@ pub struct AST { pub body: Vec, } +#[derive(Debug, PartialEq, Clone)] +pub struct FunctionSignature { + pub name: Token, + pub parameters: Vec, + pub return_types: Vec, + pub cc: FunctionCC, + pub address: Option, + pub token: Token, +} + +#[derive(Debug, PartialEq, Clone)] +pub enum FunctionCC { + Local, + Cdecl, + Stdcall, + Thiscall, +} +#[derive(Debug, PartialEq, Clone)] +pub struct FunctionParameter { + pub name: Option, + pub _type: Token, + pub token: Token, +} +#[derive(Debug, PartialEq, Clone)] +pub struct FunctionReturnType { + pub _type: Token, + pub token: Token, +} + pub type Span<'a> = LocatedSpan<&'a str>; pub type R<'a, T> = IResult, T>; diff --git a/src/parser/literal.rs b/src/parser/literal.rs index 727f1e9..eca30ea 100644 --- a/src/parser/literal.rs +++ b/src/parser/literal.rs @@ -19,7 +19,7 @@ use nom::{branch::alt, character::complete::hex_digit1}; use crate::parser::interface::*; pub fn number(s: Span) -> R { - alt((hexadecimal, float, decimal))(s) + alt((hexadecimal, binary, float, decimal, label))(s) } // combination of letters, digits and underscore, not starting with a digit @@ -60,6 +60,20 @@ pub fn hexadecimal(s: Span) -> R { )(s) } +pub fn binary(s: Span) -> R { + map( + recognize(pair(tag_no_case("0b"), many1(one_of("01")))), + |s| Token::from(s, SyntaxKind::IntegerLiteral) + )(s) +} + +pub fn label(s: Span) -> R { + map( + recognize(pair(tag_no_case("@"), identifier_any)), + |s| Token::from(s, SyntaxKind::LabelLiteral) + )(s) +} + pub fn float_span(s: Span) -> R { alt(( // Case one: .42 diff --git a/src/parser/mod.rs b/src/parser/mod.rs index 6af4efb..bfeceb3 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -3,7 +3,7 @@ use nom::combinator::map; use nom::multi::many1; pub mod interface; -use interface::*; +pub use interface::*; mod binary; mod declaration; @@ -15,6 +15,8 @@ mod statement; mod unary; mod variable; +pub use declaration::{function_signature, function_arguments_and_return_types}; // used in LanguageService + pub fn parse(s: &str) -> R { all_consuming(map(many1(declaration::declaration), |body| AST { body }))(Span::from(s)) } @@ -99,4 +101,34 @@ mod tests { ); } + #[test] + fn test_label() { + let (_, ast) = parse("@label123").unwrap(); + assert_eq!( + ast, + AST { + body: vec![Node::Literal(Token { + start: 1, + len: 9, + syntax_kind: SyntaxKind::LabelLiteral + })] + } + ); + } + + #[test] + fn test_binary() { + let (_, ast) = parse("0b101").unwrap(); + assert_eq!( + ast, + AST { + body: vec![Node::Literal(Token { + start: 1, + len: 5, + syntax_kind: SyntaxKind::IntegerLiteral + })] + } + ); + } + } diff --git a/src/parser/unary.rs b/src/parser/unary.rs index 6f29de8..bad624a 100644 --- a/src/parser/unary.rs +++ b/src/parser/unary.rs @@ -141,5 +141,29 @@ mod tests { })] } ); + + let (_, ast) = parse("-0x100").unwrap(); + assert_eq!( + ast, + AST { + body: vec![Node::Unary(UnaryPrefixExpr { + operator: Token { + syntax_kind: SyntaxKind::OperatorMinus, + start: 1, + len: 1 + }, + operand: Box::new(Node::Literal(Token { + syntax_kind: SyntaxKind::IntegerLiteral, + start: 2, + len: 5 + })), + token: Token { + syntax_kind: SyntaxKind::UnaryPrefixExpr, + start: 1, + len: 6 + } + })] + } + ); } } diff --git a/src/parser/variable.rs b/src/parser/variable.rs index d9f9c4b..ef9dad0 100644 --- a/src/parser/variable.rs +++ b/src/parser/variable.rs @@ -1,4 +1,5 @@ use nom::branch::alt; +use nom::bytes::complete::tag; use nom::character::complete::char; use nom::character::complete::one_of; use nom::combinator::consumed; @@ -20,7 +21,7 @@ static ADMA_CHAR: char = '&'; pub fn variable(s: Span) -> R { alt(( map( - consumed(tuple((single_variable, array_element_scr))), + consumed(tuple((single_variable_or_const, array_element_scr))), |(span, (var, (index, len, _type)))| { Variable::ArrayElement(ArrayElementSCR { array_var: Box::new(var), @@ -32,7 +33,7 @@ pub fn variable(s: Span) -> R { }, ), map( - consumed(tuple((single_variable, array_index))), + consumed(tuple((single_variable_or_const, array_index))), |(span, (var, index))| { Variable::Indexed(IndexedVariable { var: Box::new(var), @@ -50,7 +51,7 @@ fn array_element_scr(s: Span) -> R<(Variable, Token, VariableType)> { delimited( char('('), tuple(( - terminated(single_variable, char(',')), + terminated(single_variable_or_const, char(',')), literal::decimal, array_type, )), @@ -64,13 +65,27 @@ fn array_index(s: Span) -> R { alt(( map(single_variable, |v| Node::Variable(v)), map(literal::decimal, |d| Node::Literal(d)), + map(literal::identifier, |d| Node::Literal(d)), )), char(']'), )(s) } fn single_variable(s: Span) -> R { - alt((local_var, global_var, adma_var))(s) + alt((local_var, global_var, adma_var, pop_token, push_token))(s) +} + +fn single_variable_or_const(s: Span) -> R { + alt(( + single_variable, + map(literal::identifier, |var| { + Variable::Local(SingleVariable { + name: var.clone(), + _type: VariableType::Unknown, + token: var, + }) + }), + ))(s) } fn local_var(s: Span) -> R { @@ -89,6 +104,18 @@ fn local_var(s: Span) -> R { )(s) } +fn pop_token(s: Span) -> R { + map(consumed(tag("|>")), |(span, _)| { + Variable::Pop(Token::from(span, SyntaxKind::LocalVariable)) + })(s) +} + +fn push_token(s: Span) -> R { + map(consumed(tag("|<")), |(span, _)| { + Variable::Push(Token::from(span, SyntaxKind::LocalVariable)) + })(s) +} + fn global_var(s: Span) -> R { map( consumed(tuple(( @@ -558,4 +585,69 @@ mod tests { } ); } + + #[test] + fn test_parser_variables15() { + let (_, ast) = parse("a[0]").unwrap(); + assert_eq!( + ast, + AST { + body: vec![Node::Variable(Variable::Indexed(IndexedVariable { + var: Box::new(Variable::Local(SingleVariable { + name: Token { + start: 1, + len: 1, + syntax_kind: SyntaxKind::Identifier + }, + _type: VariableType::Unknown, + token: Token { + start: 1, + len: 1, + syntax_kind: SyntaxKind::Identifier + } + })), + index: Box::new(Node::Literal(Token { + start: 3, + len: 1, + syntax_kind: SyntaxKind::IntegerLiteral + })), + token: Token { + start: 1, + len: 4, + syntax_kind: SyntaxKind::IndexedVariable + } + }))] + } + ); + } + + #[test] + fn test_parser_variables16() { + let (_, ast) = parse("|>").unwrap(); + assert_eq!( + ast, + AST { + body: vec![Node::Variable(Variable::Pop(Token { + start: 1, + len: 2, + syntax_kind: SyntaxKind::LocalVariable + }))] + } + ); + } + + #[test] + fn test_parser_variables17() { + let (_, ast) = parse("|<").unwrap(); + assert_eq!( + ast, + AST { + body: vec![Node::Variable(Variable::Push(Token { + start: 1, + len: 2, + syntax_kind: SyntaxKind::LocalVariable + }))] + } + ); + } } diff --git a/src/preprocessor/ffi.rs b/src/preprocessor/ffi.rs new file mode 100644 index 0000000..f2f48ff --- /dev/null +++ b/src/preprocessor/ffi.rs @@ -0,0 +1,100 @@ +use std::ffi::CString; + +use super::{PreProcessorBuilder, Preprocessor}; +use crate::common_ffi::*; + +#[no_mangle] +pub extern "C" fn preprocessor_new(include: PChar, compiler_ini: PChar) -> *mut Preprocessor { + let mut p = PreProcessorBuilder::new(); + + if let Some(include_path) = pchar_to_str(include) { + p.implicit_includes(vec![include_path.into()]); + } + if let Some(compiler_ini_path) = pchar_to_str(compiler_ini) { + p.reserved_words(compiler_ini_path.into()); + } + ptr_new(p.build()) +} + +#[no_mangle] +pub unsafe extern "C" fn preprocessor_free(p: *mut Preprocessor) { + ptr_free(p) +} + +#[no_mangle] +pub unsafe extern "C" fn preprocessor_parse_file(p: *mut Preprocessor, file: PChar) -> bool { + boolclosure! {{ + let p = p.as_mut()?; + let file = pchar_to_str(file)?; + p.parse_file(file.into()).ok() + }} +} + +// #[no_mangle] +// pub unsafe extern "C" fn preprocessor_get_line( +// p: *mut Preprocessor, +// line_index: u32, +// out_line: *mut PChar, +// ) -> bool { +// boolclosure! {{ +// let p = p.as_mut()?; +// *out_line = p.get_line(line_index as usize)?.as_ptr(); +// Some(()) +// }} +// } + +// #[no_mangle] +// pub unsafe extern "C" fn preprocessor_get_line_count( +// p: *mut Preprocessor, +// out_count: *mut u32, +// ) -> bool { +// boolclosure! {{ +// let p = p.as_mut()?; +// *out_count = p.get_line_count() as u32; +// Some(()) +// }} +// } + +// #[no_mangle] +// pub unsafe extern "C" fn preprocessor_translate_line( +// p: *mut Preprocessor, +// line_index: u32, +// out_index: *mut u32, +// out_filename: *mut PChar, +// ) -> bool { +// boolclosure! {{ +// let p = p.as_mut()?; +// let (index, filename) = p.translate_line(line_index as usize)?; +// *out_index = index as u32; +// *out_filename = CString::new(filename.to_str()?).ok()?.into_raw(); +// Some(()) +// }} +// } + +#[no_mangle] +pub unsafe extern "C" fn preprocessor_get_number_of_functions_this_scope( + p: *mut Preprocessor, + line_index: u32, + out_count: *mut u32, +) -> bool { + boolclosure! {{ + let p = p.as_mut()?; + *out_count = p.get_number_of_functions_this_scope(line_index as usize) as u32; + Some(()) + }} +} + +#[no_mangle] +pub unsafe extern "C" fn preprocessor_get_function( + p: *mut Preprocessor, + line_index: u32, + function_index: u32, + out_signature: *mut PChar, +) -> bool { + boolclosure! {{ + let p = p.as_mut()?; + let f = p.get_function(line_index as usize, function_index as usize)?; + *out_signature = CString::new(f.signature.clone()).ok()?.into_raw(); + Some(()) + }} +} diff --git a/src/preprocessor/line_parser.rs b/src/preprocessor/line_parser.rs new file mode 100644 index 0000000..36f127b --- /dev/null +++ b/src/preprocessor/line_parser.rs @@ -0,0 +1,1218 @@ +use std::collections::HashMap; + +static CHARS_DIGIT: [u8; 10] = [b'0', b'1', b'2', b'3', b'4', b'5', b'6', b'7', b'8', b'9']; +static CHARS_IDENTIFIER: [u8; 63] = [ + b'0', b'1', b'2', b'3', b'4', b'5', b'6', b'7', b'8', b'9', b'_', b'a', b'b', b'c', b'd', b'e', + b'f', b'g', b'h', b'i', b'j', b'k', b'l', b'm', b'n', b'o', b'p', b'q', b'r', b's', b't', b'u', + b'v', b'w', b'x', b'y', b'z', b'A', b'B', b'C', b'D', b'E', b'F', b'G', b'H', b'I', b'J', b'K', + b'L', b'M', b'N', b'O', b'P', b'Q', b'R', b'S', b'T', b'U', b'V', b'W', b'X', b'Y', b'Z', +]; + +const CHARS_MODEL: &[u8] = b"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_@"; +const CHARS_VAR: &[u8] = b"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_&"; + +static CHARS_HEX: [u8; 22] = [ + b'0', b'1', b'2', b'3', b'4', b'5', b'6', b'7', b'8', b'9', b'a', b'b', b'c', b'd', b'e', b'f', + b'A', b'B', b'C', b'D', b'E', b'F', +]; +static CHARS_BIN: [u8; 2] = [b'0', b'1']; +const CHARS_WHITESPACE: &[u8] = &[ + 1, 2, 3, 4, 5, 6, 7, 8, 9, 11, 12, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, + 29, 30, 31, 32, +]; + +#[derive(Debug, PartialEq)] +pub enum TokenType { + Eol, + Ident, + Int, + HexNumber, + BinNumber, + Float, + Eq, + Plus, + Minus, + Mul, + Div, + PlusEq, + MinusEq, + MulEq, + DivEq, + PlusPlus, + MinusMinus, + EqEq, + NotEq, + LessThan, + LessThanEq, + GreaterThan, + GreaterThanEq, + EqCast, + Colon, + Comma, + OpenBracket, + CloseBracket, + OpenBrace, + CloseBrace, + OpenCurly, + CloseCurly, + Directive, + Global, + GlobalString8, + GlobalString16, + Adma, + Local, + LocalString8, + LocalString16, + Model, + Label, + Period, + VString, + SString, + // Hex, + OpcodeId, + StringUnterm, + // FunctionCall, + // Class, + Unknown, +} + +#[derive(Debug, PartialEq)] +pub enum TokenVal { + Eol, + Unknown(u8), + Ident(String), + Int(i64), + Float(f32), + Punctuator, +} + +type Handler = fn(&mut DataParser) -> Token; + +#[derive(Default)] +pub struct DataParser { + handlers: HashMap, + line: Vec, + current_char: usize, + in_comment_curly: bool, + in_comment_cpp: bool, +} + +#[derive(Debug)] +pub struct Token { + pub token_type: TokenType, + pub val: TokenVal, +} + +impl DataParser { + pub fn new() -> Self { + let mut handlers = HashMap::new(); + for i in 0..=255 { + match i { + 0 | 10 | 13 => handlers.insert(i, eof_proc as Handler), + 1..=9 | 11..=12 | 14..=32 => handlers.insert(i, space_proc), + b'$' => handlers.insert(i, global_var_proc), + b'&' => handlers.insert(i, adma_var_proc), + b'(' => handlers.insert(i, open_brace_proc), + b')' => handlers.insert(i, close_brace_proc), + b'{' => handlers.insert(i, open_curly_proc), + b'}' => handlers.insert(i, close_curly_proc), + b'#' => handlers.insert(i, model_proc), + b'[' => handlers.insert(i, open_bracket_proc), + b']' => handlers.insert(i, close_bracket_proc), + b'@' => handlers.insert(i, label_proc), + b'"' => handlers.insert(i, v_string_proc), // todo! if SBOptionEnabled(eoSB3Compat) + b'\'' => handlers.insert(i, s_string_proc), // todo! if SBOptionEnabled(eoSB3Compat) + b'+' => handlers.insert(i, plus_proc), + b'-' => handlers.insert(i, minus_proc), + b'*' => handlers.insert(i, mul_proc), + b'/' => handlers.insert(i, div_proc), + b'>' => handlers.insert(i, greater_than_proc), + b'<' => handlers.insert(i, less_than_proc), + b'=' => handlers.insert(i, eq_proc), + b'.' => handlers.insert(i, period_proc), + b',' => handlers.insert(i, comma_proc), + b'0'..=b'9' => handlers.insert(i, |p| opcode_or(p, integer_proc)), + b'a'..=b'f' | b'A'..=b'F' => handlers.insert(i, |p| opcode_or(p, identifier_proc)), + b's' => handlers.insert(i, s_proc), + b'v' => handlers.insert(i, v_proc), + b':' => handlers.insert(i, colon_proc), + b'~' => handlers.insert(i, tilde_proc), + b'_' | b'a'..=b'z' | b'A'..=b'Z' => handlers.insert(i, identifier_proc), + _ => handlers.insert(i, unknown_proc), + }; + } + Self { + handlers, + line: Vec::new(), + current_char: 0, + in_comment_cpp: false, + in_comment_curly: false, + // current_char: std::ptr::null(), + } + } + + pub fn line(&mut self, line: &str) { + self.line = line.as_bytes().to_vec(); + self.current_char = 0; + } + + pub fn get_token(&mut self) -> Token { + let char = self.this_char(); + self.handlers[&char](self) + } + + fn eol(&mut self) -> Token { + self.next(); + Token { + token_type: TokenType::Eol, + val: TokenVal::Eol, + } + } + + fn punctuator(&mut self, token_type: TokenType) -> Token { + self.next(); + Token { + token_type, + val: TokenVal::Punctuator, + } + } + + fn unknown(&mut self) -> Token { + Token { + token_type: TokenType::Unknown, + val: TokenVal::Unknown(self.this_char()), + } + } + + fn at(&mut self, n: usize) -> u8 { + if n >= self.line.len() { + 0 + } else { + self.line[n] + } + } + + fn read_char(&mut self) -> u8 { + match self.at(self.current_char) { + _ if self.in_comment_curly => { + if self.skip_until(b"}") { + self.next(); // } + self.in_comment_curly = false; + return self.this_char(); // return char after comment + } else { + 0 + } + } + _ if self.in_comment_cpp => { + loop { + match self.at(self.current_char) { + 0 | 10 | 13 => return 0, + b'*' if self.at(self.current_char + 1) == b'/' => { + self.next_n(2); // */ + self.in_comment_cpp = false; + return self.this_char(); // return char after comment + } + _ => self.next(), + } + } + } + b'{' if self.at(self.current_char + 1) != b'$' => { + self.next(); // { + if self.skip_until(b"}") { + self.next(); // } + self.in_comment_curly = false; + return self.this_char(); // return char after comment + } else { + self.in_comment_curly = true; + 0 + } + } + b'/' if self.at(self.current_char + 1) == b'*' => { + self.next_n(2); // /* + loop { + match self.at(self.current_char) { + 0 | 10 | 13 => { + self.in_comment_cpp = true; + return 0; + } + b'*' if self.at(self.current_char + 1) == b'/' => { + self.next_n(2); // */ + self.in_comment_cpp = false; + return self.this_char(); // return char after comment + } + _ => { + self.next(); + } + } + } + } + x => x, // 0 goes here + } + } + + fn this_char(&mut self) -> u8 { + self.read_char() + } + + fn next(&mut self) { + self.next_n(1) + } + + fn next_n(&mut self, n: usize) { + self.current_char += n; + } + + fn peek(&mut self) -> u8 { + self.peek_n(1) + } + + fn peek_n(&mut self, n: usize) -> u8 { + let current_char = self.current_char; + self.next_n(n); + let peek = self.this_char(); + self.current_char = current_char; + peek + } + + fn slice(&mut self, start: usize) -> String { + let mut buf = vec![]; + let end = self.current_char; + + self.current_char = start; // restore the start position + while self.current_char < end { + buf.push(self.this_char()); + self.next(); + } + + String::from_utf8_lossy(&buf).to_string() + // String::from_utf8_lossy(&self.line[start..self.current_char]).to_string() + } + + fn get_while(&mut self, chars: &[u8]) -> String { + let start = self.current_char; + self.skip_while(chars); + self.slice(start) + } + + // fn get_until(&mut self, chars: &[u8]) -> String { + // let start = self.current_char; + // self.skip_until(chars); + // self.slice(start) + // } + + fn get_while1(&mut self, chars: &[u8], token_type: TokenType) -> Token { + let start = self.current_char; + self.skip_while(chars); + if start == self.current_char { + self.unknown() + } else { + Token { + token_type, + val: TokenVal::Ident(self.slice(start)), + } + } + } + + pub fn get_until1(&mut self, chars: &[u8], token_type: TokenType) -> Token { + let start = self.current_char; + self.skip_until(chars); + if start == self.current_char { + self.unknown() + } else { + Token { + token_type, + val: TokenVal::Ident(self.slice(start)), + } + } + } + + fn skip_while(&mut self, chars: &[u8]) { + while chars.contains(&self.this_char()) { + self.next(); + } + } + + fn skip_until(&mut self, chars: &[u8]) -> bool { + let stop_chars = [0, 10, 13]; + loop { + let c = self.at(self.current_char); + if chars.contains(&c) { + return true; + } + if stop_chars.contains(&c) { + return false; + } + self.next(); + } + } + + fn skip_while1(&mut self, chars: &[u8]) -> bool { + let start = self.current_char; + self.skip_while(chars); + start != self.current_char + } + + // fn skip_until1(&mut self, chars: &[u8]) -> bool { + // let start = self.current_char; + // self.skip_until(chars); + // start != self.current_char + // } + + pub fn skip_whitespace(&mut self) { + self.skip_while(&CHARS_WHITESPACE) + } + + fn try_char(&mut self, c: &[u8]) -> bool { + if c.contains(&self.this_char()) { + self.next(); + true + } else { + false + } + } + + fn try_token(&mut self, tokens: &[TokenType]) -> Option { + let cur_pos = self.current_char; + let token = self.get_token(); + + if tokens.contains(&token.token_type) { + Some(token) + } else { + self.current_char = cur_pos; + None + } + } + + pub fn current_loc(&self) -> (String, usize) { + ( + String::from_utf8_lossy(&self.line).to_string(), + self.current_char, + ) + } +} + +fn eof_proc(p: &mut DataParser) -> Token { + p.eol() +} + +fn space_proc(p: &mut DataParser) -> Token { + p.skip_whitespace(); + p.get_token() +} + +fn unknown_proc(p: &mut DataParser) -> Token { + p.unknown() +} + +fn global_var_proc(p: &mut DataParser) -> Token { + p.next(); + p.get_while1(&CHARS_VAR, TokenType::Global) +} + +fn adma_var_proc(p: &mut DataParser) -> Token { + p.next(); + p.get_while1(&CHARS_DIGIT, TokenType::Adma) +} + +fn open_brace_proc(p: &mut DataParser) -> Token { + p.punctuator(TokenType::OpenBrace) +} + +fn close_brace_proc(p: &mut DataParser) -> Token { + p.punctuator(TokenType::CloseBrace) +} + +fn open_curly_proc(p: &mut DataParser) -> Token { + if p.peek() == b'$' { + let start = p.current_char; + p.next_n(2); // ${ + if p.skip_while1(&CHARS_IDENTIFIER) { + Token { + token_type: TokenType::Directive, + val: TokenVal::Ident(p.slice(start)), + } + } else { + p.unknown() + } + } else { + p.in_comment_curly = true; + p.get_token() + } +} + +fn close_curly_proc(p: &mut DataParser) -> Token { + p.punctuator(TokenType::CloseCurly) +} + +fn model_proc(p: &mut DataParser) -> Token { + p.next(); // # + p.get_while1(&CHARS_MODEL, TokenType::Model) +} + +fn open_bracket_proc(p: &mut DataParser) -> Token { + p.punctuator(TokenType::OpenBracket) +} + +fn close_bracket_proc(p: &mut DataParser) -> Token { + p.punctuator(TokenType::CloseBracket) +} + +fn label_proc(p: &mut DataParser) -> Token { + p.next(); // @ + p.get_while1(&CHARS_IDENTIFIER, TokenType::Label) +} + +fn v_string_proc(p: &mut DataParser) -> Token { + let s = grab_string_escaped(p, b'"'); + if p.try_char(b"\"") { + Token { + token_type: TokenType::VString, + val: TokenVal::Ident(s), + } + } else { + Token { + token_type: TokenType::StringUnterm, + val: TokenVal::Ident(s), + } + } +} + +fn s_string_proc(p: &mut DataParser) -> Token { + let s = grab_string_escaped(p, b'\''); + if p.try_char(b"'") { + Token { + token_type: TokenType::SString, + val: TokenVal::Ident(s), + } + } else { + Token { + token_type: TokenType::StringUnterm, + val: TokenVal::Ident(s), + } + } +} + +fn plus_proc(p: &mut DataParser) -> Token { + match p.peek() { + b'+' => { + p.next(); // + + p.punctuator(TokenType::PlusPlus) + } + b'=' => { + p.next(); // + + p.punctuator(TokenType::PlusEq) + } + b'0'..=b'9' => { + p.next(); // + + integer_proc(p) + } + b'.' => { + p.next(); // + + period_proc(p) + } + _ => p.punctuator(TokenType::Plus), + } +} + +fn minus_proc(p: &mut DataParser) -> Token { + match p.peek() { + b'-' => { + p.next(); // - + p.punctuator(TokenType::MinusMinus) + } + b'=' => { + p.next(); // - + p.punctuator(TokenType::MinusEq) + } + b'0'..=b'9' => { + let start = p.current_char; + p.next(); // - + let token = integer_proc(p); + Token { + token_type: token.token_type, + val: TokenVal::Ident(p.slice(start)), + } + } + b'.' => { + let start = p.current_char; + p.next(); // - + let token = period_proc(p); + Token { + token_type: token.token_type, + val: TokenVal::Ident(p.slice(start)), + } + } + _ => p.punctuator(TokenType::Minus), + } +} + +fn mul_proc(p: &mut DataParser) -> Token { + if p.peek() == b'=' { + p.next(); // * + p.punctuator(TokenType::MulEq) + } else { + p.punctuator(TokenType::Mul) + } +} + +fn div_proc(p: &mut DataParser) -> Token { + match p.peek() { + b'=' => { + p.next(); // / + p.punctuator(TokenType::DivEq) + } + b'/' => { + // line ends here + p.eol() + } + b'*' => { + // block comment /* + p.next(); // * + p.in_comment_cpp = true; + p.get_token() + } + _ => p.punctuator(TokenType::Div), + } +} + +fn greater_than_proc(p: &mut DataParser) -> Token { + if p.peek() == b'=' { + p.next(); // > + p.punctuator(TokenType::GreaterThanEq) + } else { + p.punctuator(TokenType::GreaterThan) + } +} + +fn less_than_proc(p: &mut DataParser) -> Token { + match p.peek() { + b'=' => { + p.next(); // < + p.punctuator(TokenType::LessThanEq) + } + b'>' => { + p.next(); // < + p.punctuator(TokenType::NotEq) + } + _ => p.punctuator(TokenType::LessThan), + } +} + +fn eq_proc(p: &mut DataParser) -> Token { + match p.peek() { + b'=' => { + p.next(); // = + p.punctuator(TokenType::EqEq) + } + b'#' => { + p.next(); // = + p.punctuator(TokenType::EqCast) + } + _ => p.punctuator(TokenType::Eq), + } +} + +fn period_proc(p: &mut DataParser) -> Token { + match p.peek() { + b'0'..=b'9' => { + let start = p.current_char; + p.next(); // . + p.skip_while(&CHARS_DIGIT); + match p.this_char() { + b'e' | b'E' => { + p.next(); // E + match p.this_char() { + b'+' | b'-' => p.next(), + _ => {} + } + if !CHARS_DIGIT.contains(&p.this_char()) { + return p.unknown(); + } + p.skip_while(&CHARS_DIGIT); + } + _ => {} + } + Token { + token_type: TokenType::Float, + val: TokenVal::Ident(p.slice(start)), + } + } + _ => p.punctuator(TokenType::Period), + } +} + +fn s_proc(p: &mut DataParser) -> Token { + if p.peek() == b'$' { + p.next_n(2); // skip s$ + p.get_while1(&CHARS_IDENTIFIER, TokenType::GlobalString8) + } else { + Token { + token_type: TokenType::Ident, + val: TokenVal::Ident(p.get_while(&CHARS_IDENTIFIER)), + } + } +} + +fn v_proc(p: &mut DataParser) -> Token { + if p.peek() == b'$' { + p.next_n(2); // skip v$ + p.get_while1(&CHARS_IDENTIFIER, TokenType::GlobalString16) + } else { + Token { + token_type: TokenType::Ident, + val: TokenVal::Ident(p.get_while(&CHARS_IDENTIFIER)), + } + } +} + +fn comma_proc(p: &mut DataParser) -> Token { + p.punctuator(TokenType::Comma) +} + +fn opcode_or(p: &mut DataParser, other_handler: Handler) -> Token { + // 0..9, A-F + if CHARS_HEX.contains(&p.peek_n(1)) + && CHARS_HEX.contains(&p.peek_n(2)) + && CHARS_HEX.contains(&p.peek_n(3)) + && p.peek_n(4) == b':' + { + let start = p.current_char; + p.next_n(5); // 0000: + Token { + token_type: TokenType::OpcodeId, + val: TokenVal::Ident(p.slice(start)), + } + } else { + other_handler(p) + } +} + +fn integer_proc(p: &mut DataParser) -> Token { + let start = p.current_char; + let next_char = p.peek(); + let mut token_type; + + match (p.this_char(), next_char) { + (b'0', b'x') | (b'0', b'X') => { + p.next_n(2); // skip 0x + // can be empty? + if !p.skip_while1(&CHARS_HEX) { + return p.unknown(); + } + Token { + token_type: TokenType::HexNumber, + val: TokenVal::Ident(p.slice(start)), + } + } + + (b'0', b'b') | (b'0', b'B') => { + p.next_n(2); // skip 0b + if !p.skip_while1(&CHARS_BIN) { + return p.unknown(); + } + Token { + token_type: TokenType::BinNumber, + val: TokenVal::Ident(p.slice(start)), + } + } + _ => { + p.skip_while(&CHARS_DIGIT); + + match p.this_char() { + b'@' => { + let val = TokenVal::Ident(p.slice(start)); + p.next(); // @ + + if p.try_char(b"s") { + // @s + return Token { + token_type: TokenType::LocalString8, + val, + }; + } + + if p.try_char(b"v") { + // @v + return Token { + token_type: TokenType::LocalString16, + val, + }; + } + + return Token { + // @ + token_type: TokenType::Local, + val, + }; + } + b'.' => { + p.next(); // . + + if !p.skip_while1(&CHARS_DIGIT) { + return p.unknown(); + } + token_type = TokenType::Float; + } + _ => { + token_type = TokenType::Int; + // fall down + } + } + + // 100E4, -100.0E4, -100E-2, -100.0E-2 + if p.this_char() == b'E' || p.this_char() == b'e' { + token_type = TokenType::Float; + p.next(); // E + if p.this_char() == b'+' || p.this_char() == b'-' { + p.next(); + } + if !CHARS_DIGIT.contains(&p.this_char()) { + return p.unknown(); + } + + p.skip_while(&CHARS_DIGIT); + } + + Token { + token_type, + val: TokenVal::Ident(p.slice(start)), + } + } + } +} + +fn colon_proc(p: &mut DataParser) -> Token { + p.punctuator(TokenType::Colon) +} + +fn tilde_proc(p: &mut DataParser) -> Token { + p.next(); // ~ + // todo: check whitespace? + + p.try_token(&[TokenType::Global, TokenType::Local]) + .unwrap_or_else(|| Token { + token_type: TokenType::Unknown, + val: TokenVal::Unknown(b'~'), + }) +} + +fn identifier_proc(p: &mut DataParser) -> Token { + Token { + token_type: TokenType::Ident, + val: TokenVal::Ident(p.get_while(&CHARS_IDENTIFIER)), + } +} + +fn grab_string_escaped(p: &mut DataParser, terminator: u8) -> String { + let mut buf = Vec::new(); + let mut this_char: u8; + + p.next(); // skip the start quote (" or ') + loop { + this_char = p.this_char(); + + if [0, 10, 13, terminator].contains(&this_char) { + break; + } + + if p.try_char(b"\\") { + match p.this_char() { + 0 | 10 | 13 => break, + b'0' => buf.push(0), + b'b' => buf.push(8), + b't' => buf.push(9), + b'n' => buf.push(10), + b'r' => buf.push(13), + b'x' => { + p.next(); // skip x + let a = p.this_char(); + let a_num = match a { + b'0'..=b'9' => a - b'0', + b'a'..=b'f' => a - b'a' + 10, + b'A'..=b'F' => a - b'A' + 10, + _ => break, + }; + + p.next(); // skip the first digit + + let b = p.this_char(); + let b_num = match b { + b'0'..=b'9' => b - b'0', + b'a'..=b'f' => b - b'a' + 10, + b'A'..=b'F' => b - b'A' + 10, + _ => break, + }; + + buf.push((a_num << 4) + b_num); + } + b'\\' => buf.push(b'\\'), + x => buf.push(x), + } + } else { + buf.push(this_char); + } + p.next(); + } + + String::from_utf8_lossy(&buf).to_string() +} + +#[cfg(test)] + +mod tests { + + use super::*; + + #[test] + fn test_empty_line() { + let mut parser = DataParser::new(); + let line = " "; + parser.current_char = line.as_ptr() as _; + + let token = parser.get_token(); + assert_eq!(token.token_type, TokenType::Eol); + assert_eq!(token.val, TokenVal::Eol); + } + + #[test] + fn test_punctuators() { + let mut parser = DataParser::new(); + parser.line(": [] {} = == =# ,- -- -= . > < <>"); + + let token = parser.get_token(); + assert_eq!(token.token_type, TokenType::Colon); + assert_eq!(token.val, TokenVal::Punctuator); + + let token = parser.get_token(); + assert_eq!(token.token_type, TokenType::OpenBracket); + assert_eq!(token.val, TokenVal::Punctuator); + + let token = parser.get_token(); + assert_eq!(token.token_type, TokenType::CloseBracket); + assert_eq!(token.val, TokenVal::Punctuator); + + let token = parser.get_token(); + assert_eq!(token.token_type, TokenType::Eq); + assert_eq!(token.val, TokenVal::Punctuator); + + let token = parser.get_token(); + assert_eq!(token.token_type, TokenType::EqEq); + assert_eq!(token.val, TokenVal::Punctuator); + + let token = parser.get_token(); + assert_eq!(token.token_type, TokenType::EqCast); + assert_eq!(token.val, TokenVal::Punctuator); + + let token = parser.get_token(); + assert_eq!(token.token_type, TokenType::Comma); + assert_eq!(token.val, TokenVal::Punctuator); + + let token = parser.get_token(); + assert_eq!(token.token_type, TokenType::Minus); + assert_eq!(token.val, TokenVal::Punctuator); + + let token = parser.get_token(); + assert_eq!(token.token_type, TokenType::MinusMinus); + assert_eq!(token.val, TokenVal::Punctuator); + + let token = parser.get_token(); + assert_eq!(token.token_type, TokenType::MinusEq); + assert_eq!(token.val, TokenVal::Punctuator); + + let token = parser.get_token(); + assert_eq!(token.token_type, TokenType::Period); + assert_eq!(token.val, TokenVal::Punctuator); + + let token = parser.get_token(); + assert_eq!(token.token_type, TokenType::GreaterThan); + assert_eq!(token.val, TokenVal::Punctuator); + + let token = parser.get_token(); + assert_eq!(token.token_type, TokenType::LessThan); + assert_eq!(token.val, TokenVal::Punctuator); + + let token = parser.get_token(); + assert_eq!(token.token_type, TokenType::NotEq); + assert_eq!(token.val, TokenVal::Punctuator); + + let token = parser.get_token(); + assert_eq!(token.token_type, TokenType::Eol); + assert_eq!(token.val, TokenVal::Eol); + } + + #[test] + fn test_vars() { + let mut parser = DataParser::new(); + parser.line("$global &100 s$var v$103"); + + let token = parser.get_token(); + assert_eq!(token.token_type, TokenType::Global); + assert_eq!(token.val, TokenVal::Ident("global".to_string())); + + let token = parser.get_token(); + assert_eq!(token.token_type, TokenType::Adma); + assert_eq!(token.val, TokenVal::Ident("100".to_string())); + + let token = parser.get_token(); + assert_eq!(token.token_type, TokenType::GlobalString8); + assert_eq!(token.val, TokenVal::Ident("var".to_string())); + + let token = parser.get_token(); + assert_eq!(token.token_type, TokenType::GlobalString16); + assert_eq!(token.val, TokenVal::Ident("103".to_string())); + + let token = parser.get_token(); + assert_eq!(token.token_type, TokenType::Eol); + assert_eq!(token.val, TokenVal::Eol); + } + + #[test] + fn test_identifiers() { + let mut parser = DataParser::new(); + parser.line("abc DE_F sanny vvv"); + + let token = parser.get_token(); + assert_eq!(token.token_type, TokenType::Ident); + assert_eq!(token.val, TokenVal::Ident("abc".to_string())); + + let token = parser.get_token(); + assert_eq!(token.token_type, TokenType::Ident); + assert_eq!(token.val, TokenVal::Ident("DE_F".to_string())); + + let token = parser.get_token(); + assert_eq!(token.token_type, TokenType::Ident); + assert_eq!(token.val, TokenVal::Ident("sanny".to_string())); + + let token = parser.get_token(); + assert_eq!(token.token_type, TokenType::Ident); + assert_eq!(token.val, TokenVal::Ident("vvv".to_string())); + + let token = parser.get_token(); + assert_eq!(token.token_type, TokenType::Eol); + assert_eq!(token.val, TokenVal::Eol); + } + + #[test] + fn test_directives() { + let mut parser = DataParser::new(); + parser.line("{$include a.txt}"); + + let token = parser.get_token(); + assert_eq!(token.token_type, TokenType::Directive); + assert_eq!(token.val, TokenVal::Ident("{$include".to_string())); + + parser.line("{$cleo}"); + + let token = parser.get_token(); + assert_eq!(token.token_type, TokenType::Directive); + assert_eq!(token.val, TokenVal::Ident("{$cleo".to_string())); + } + + #[test] + fn test_model() { + let mut parser = DataParser::new(); + parser.line("#model #023@egg"); + + let token = parser.get_token(); + assert_eq!(token.token_type, TokenType::Model); + assert_eq!(token.val, TokenVal::Ident("model".to_string())); + + let token = parser.get_token(); + assert_eq!(token.token_type, TokenType::Model); + assert_eq!(token.val, TokenVal::Ident("023@egg".to_string())); + + let token = parser.get_token(); + assert_eq!(token.token_type, TokenType::Eol); + assert_eq!(token.val, TokenVal::Eol); + } + + #[test] + fn test_label() { + let mut parser = DataParser::new(); + parser.line(" @label "); + + let token = parser.get_token(); + assert_eq!(token.token_type, TokenType::Label); + assert_eq!(token.val, TokenVal::Ident("label".to_string())); + + let token = parser.get_token(); + assert_eq!(token.token_type, TokenType::Eol); + assert_eq!(token.val, TokenVal::Eol); + } + + #[test] + fn test_tilde() { + let mut parser = DataParser::new(); + parser.line("~$var ~10@ ~"); + + let token = parser.get_token(); + assert_eq!(token.token_type, TokenType::Global); + assert_eq!(token.val, TokenVal::Ident("var".to_string())); + + let token = parser.get_token(); + assert_eq!(token.token_type, TokenType::Local); + assert_eq!(token.val, TokenVal::Ident("10".to_string())); + + let token = parser.get_token(); + assert_eq!(token.token_type, TokenType::Unknown); + assert_eq!(token.val, TokenVal::Unknown(b'~')); + + let token = parser.get_token(); + assert_eq!(token.token_type, TokenType::Eol); + assert_eq!(token.val, TokenVal::Eol); + } + + #[test] + fn test_numbers() { + let mut parser = DataParser::new(); + parser.line("0 0x10 0b10 100 100.0 100E4 -100.0E4 -100E-2 -100.0E-2 .1 .1E4 -.0E-4 .0E+10"); + + let token = parser.get_token(); + assert_eq!(token.token_type, TokenType::Int); + assert_eq!(token.val, TokenVal::Ident("0".to_string())); + + let token = parser.get_token(); + assert_eq!(token.token_type, TokenType::HexNumber); + assert_eq!(token.val, TokenVal::Ident("0x10".to_string())); + + let token = parser.get_token(); + assert_eq!(token.token_type, TokenType::BinNumber); + assert_eq!(token.val, TokenVal::Ident("0b10".to_string())); + + let token = parser.get_token(); + assert_eq!(token.token_type, TokenType::Int); + assert_eq!(token.val, TokenVal::Ident("100".to_string())); + + let token = parser.get_token(); + assert_eq!(token.token_type, TokenType::Float); + assert_eq!(token.val, TokenVal::Ident("100.0".to_string())); + + let token = parser.get_token(); + assert_eq!(token.token_type, TokenType::Float); + assert_eq!(token.val, TokenVal::Ident("100E4".to_string())); + + let token = parser.get_token(); + assert_eq!(token.token_type, TokenType::Float); + assert_eq!(token.val, TokenVal::Ident("-100.0E4".to_string())); + + let token = parser.get_token(); + assert_eq!(token.token_type, TokenType::Float); + assert_eq!(token.val, TokenVal::Ident("-100E-2".to_string())); + + let token = parser.get_token(); + assert_eq!(token.token_type, TokenType::Float); + assert_eq!(token.val, TokenVal::Ident("-100.0E-2".to_string())); + + let token = parser.get_token(); + assert_eq!(token.token_type, TokenType::Float); + assert_eq!(token.val, TokenVal::Ident(".1".to_string())); + + let token = parser.get_token(); + assert_eq!(token.token_type, TokenType::Float); + assert_eq!(token.val, TokenVal::Ident(".1E4".to_string())); + + let token = parser.get_token(); + assert_eq!(token.token_type, TokenType::Float); + assert_eq!(token.val, TokenVal::Ident("-.0E-4".to_string())); + + let token = parser.get_token(); + assert_eq!(token.token_type, TokenType::Float); + assert_eq!(token.val, TokenVal::Ident(".0E+10".to_string())); + + let token = parser.get_token(); + assert_eq!(token.token_type, TokenType::Eol); + assert_eq!(token.val, TokenVal::Eol); + } + + #[test] + fn test_strings() { + let mut parser = DataParser::new(); + parser.line(r#" "hello" "\0\t\n\\" "\x20\x4A" 'abc' "world "#); + + let token = parser.get_token(); + assert_eq!(token.token_type, TokenType::VString); + assert_eq!(token.val, TokenVal::Ident("hello".to_string())); + + let token = parser.get_token(); + assert_eq!(token.token_type, TokenType::VString); + assert_eq!(token.val, TokenVal::Ident("\0\t\n\\".to_string())); + + let token = parser.get_token(); + assert_eq!(token.token_type, TokenType::VString); + assert_eq!(token.val, TokenVal::Ident(" J".to_string())); + + let token = parser.get_token(); + assert_eq!(token.token_type, TokenType::SString); + assert_eq!(token.val, TokenVal::Ident("abc".to_string())); + + let token = parser.get_token(); + assert_eq!(token.token_type, TokenType::StringUnterm); + assert_eq!(token.val, TokenVal::Ident("world ".to_string())); + + let token = parser.get_token(); + assert_eq!(token.token_type, TokenType::Eol); + assert_eq!(token.val, TokenVal::Eol); + } + + #[test] + fn test_comments() { + let mut parser = DataParser::new(); + + parser.line(" /* comment */ 1"); + + let token = parser.get_token(); + assert_eq!(token.token_type, TokenType::Int); + assert_eq!(token.val, TokenVal::Ident("1".to_string())); + parser.line(" // comment"); + + let token = parser.get_token(); + assert_eq!(token.token_type, TokenType::Eol); + assert_eq!(token.val, TokenVal::Eol); + + parser.line(" {comment}123"); + + let token = parser.get_token(); + assert_eq!(token.token_type, TokenType::Int); + assert_eq!(token.val, TokenVal::Ident("123".to_string())); + + parser.line(" /* comment */"); + + let token = parser.get_token(); + assert_eq!(token.token_type, TokenType::Eol); + assert_eq!(token.val, TokenVal::Eol); + + parser.line(" /* comment */ 1"); + + let token = parser.get_token(); + assert_eq!(token.token_type, TokenType::Int); + assert_eq!(token.val, TokenVal::Ident("1".to_string())); + + parser.line(" {comment}1{}2{}3"); + + let token = parser.get_token(); + assert_eq!(token.token_type, TokenType::Int); + assert_eq!(token.val, TokenVal::Ident("123".to_string())); + + parser.line(" {comment}1/*555*/2{}3"); + + let token = parser.get_token(); + assert_eq!(token.token_type, TokenType::Int); + assert_eq!(token.val, TokenVal::Ident("123".to_string())); + + parser.in_comment_curly = false; + parser.in_comment_cpp = false; + parser.line("{"); + + let token = parser.get_token(); + assert_eq!(token.token_type, TokenType::Eol); + assert!(parser.in_comment_curly); + + parser.in_comment_curly = false; + parser.in_comment_cpp = false; + parser.line("/*"); + + let token = parser.get_token(); + assert_eq!(token.token_type, TokenType::Eol); + assert!(parser.in_comment_cpp); + } +} diff --git a/src/preprocessor/mod.rs b/src/preprocessor/mod.rs new file mode 100644 index 0000000..4047acd --- /dev/null +++ b/src/preprocessor/mod.rs @@ -0,0 +1,589 @@ +use anyhow::{bail, Result}; +use lines_lossy::LinesLossyExt; +use std::{ + collections::HashSet, + ffi::CString, + io::{BufRead, BufReader, Read}, + path::PathBuf, +}; + +use self::line_parser::{TokenType, TokenVal}; +use crate::{ + dictionary::{config, ffi::CaseFormat, DictNumByString}, + preprocessor::scopes::Function, + utils::{ + compiler_const::{ + TOKEN_END, TOKEN_EXPORT, TOKEN_FOR, TOKEN_FUNCTION, TOKEN_HEX, TOKEN_IF, TOKEN_INCLUDE, + TOKEN_INCLUDE_ONCE, TOKEN_SWITCH, TOKEN_WHILE, + }, + path::{normalize_file_name, resolve_path}, + }, + v4::helpers::token_str, +}; + +mod ffi; +mod line_parser; +mod scopes; + +type FileName = PathBuf; + +#[derive(Default, Debug, PartialEq, Eq)] +pub enum SourceType { + #[default] + Memory, + File(FileName), +} + +#[derive(Default)] +pub struct Preprocessor { + pub implicit_includes: HashSet, + pub source_type: SourceType, + pub files: Vec, + pub open_files: HashSet, + pub parser: line_parser::DataParser, + pub reserved_words: DictNumByString, + pub current_file: isize, + pub absolute_line_index: usize, + pub scopes: scopes::Scopes, +} + +pub struct LineLoc { + file_index: isize, // -1 for memory + line_index: usize, // 0-based +} + +#[derive(Debug)] +pub struct PreProcessorBuilder { + implicit_includes: HashSet, + reserved_words: DictNumByString, +} + +impl PreProcessorBuilder { + pub fn new() -> Self { + Self { + implicit_includes: HashSet::new(), + reserved_words: DictNumByString::new( + config::ConfigBuilder::new() + .set_case_format(CaseFormat::LowerCase) + .build(), + ), + } + } + + pub fn implicit_includes(&mut self, includes: Vec) -> &mut Self { + self.implicit_includes.extend(includes); + self + } + + pub fn reserved_words(&mut self, reserved_words: FileName) -> &mut Self { + self.reserved_words.load_file(&reserved_words); + self + } + + pub fn build(&mut self) -> Preprocessor { + Preprocessor { + implicit_includes: self.implicit_includes.clone(), + parser: line_parser::DataParser::new(), + reserved_words: self.reserved_words.clone(), + scopes: scopes::Scopes::new(), + ..Default::default() + } + } +} + +impl Preprocessor { + pub fn parse_file(&mut self, file_path: FileName) -> Result<()> { + self.source_type = SourceType::File(file_path.clone()); + self.absolute_line_index = 0; + self.load_implicit_includes()?; + match self.load_file_source(&file_path) { + Ok(_) => {} + Err(e) => { + log::error!("{e}"); + } + } + self.scopes.exit_scope(self.absolute_line_index); + Ok(()) + } + + pub fn parse_in_memory(&mut self, source: &str) -> Result<()> { + self.source_type = SourceType::Memory; + self.absolute_line_index = 0; + self.load_implicit_includes()?; + + self.current_file = -1; + let mut in_hex_block = false; + for (line_index, line) in source.lines().enumerate() { + match self.process_line(line, line_index, &mut in_hex_block) { + Ok(_) => {} + Err(e) => { + bail!(e); + } + } + } + self.scopes.exit_scope(self.absolute_line_index); + Ok(()) + } + + pub fn get_number_of_functions_this_scope(&self, line_index: usize) -> usize { + self.scopes + .functions + .iter() + .filter(|x| x.zone.start == line_index) + .count() + } + + pub fn get_function(&self, line_index: usize, index: usize) -> Option<&scopes::Function> { + self.scopes + .functions + .iter() + .filter(|x| x.zone.start == line_index) + .nth(index) + } + + fn load_implicit_includes(&mut self) -> Result<()> { + let includes = self.implicit_includes.clone(); + for include in includes { + if let Err(e) = self.load_file_source(&include) { + return Err(e); + } + } + Ok(()) + } + + fn load_file_source(&mut self, file_path: &FileName) -> Result<()> { + use lines_lossy; + + let prev_file = self.current_file; + self.current_file = self + .files + .iter() + .position(|x| x == file_path) + .unwrap_or_else(|| { + self.files.push(normalize_file_name(file_path).unwrap()); + self.files.len() - 1 + }) as isize; + + let Ok(file) = std::fs::File::open(file_path) else { + bail!("Can't open file: {:?}", file_path); + }; + let reader = std::io::BufReader::new(file); + let mut lines = reader.lines_lossy().enumerate(); + // let reader = std::io::BufReader::new( + // DecodeReaderBytesBuilder::new() + // .encoding(Some(encoding_rs::WINDOWS_1251)) + // .build(file), + // ); + // let mut lines = reader.lines().enumerate(); + let mut in_hex_block = false; + while let Some((line_index, line)) = lines.next() { + match line { + Ok(line) => match self.process_line(line.as_str(), line_index, &mut in_hex_block) { + Ok(_) => {} + Err(e) => { + bail!(e); + } + }, + Err(_) => { + bail!("Can't read line {line_index} from file {:?}", file_path); + } + } + } + + self.current_file = prev_file; + Ok(()) + } + + pub fn process_line( + &mut self, + line: &str, + line_index: usize, + in_hex_block: &mut bool, + ) -> Result<()> { + self.parser.line(line); + self.absolute_line_index += 1; + let token = self.parser.get_token(); + + match token.token_type { + TokenType::Directive if !*in_hex_block => { + match token.val { + TokenVal::Ident(s) => { + let token_id = self.reserved_words.map.get(&s.to_ascii_lowercase()); + match token_id { + Some(&TOKEN_INCLUDE) | Some(&TOKEN_INCLUDE_ONCE) => { + self.parser.skip_whitespace(); + let token = self.parser.get_until1(b"}", TokenType::Ident); + if self.parser.get_token().token_type != TokenType::CloseCurly { + bail!("Error parsing include directive") + } + match token.token_type { + TokenType::Ident => { + match token.val { + TokenVal::Ident(include_path) => { + let current_file_name = match self.current_file { + -1 => None, + x => { + match self.files.get(x as usize) { + None => { + bail!("Error loading include file: {}", x); + } + x => x, + } + } + }; + + let Some(path) = + resolve_path(&include_path, current_file_name) + else { + let loc = self.parser.current_loc(); + bail!("Error resolving path to include file at {}:{}", loc.0, loc.1); + }; + + if matches!(token_id, Some(&TOKEN_INCLUDE_ONCE)) + && self.files.iter().any(|x| x == &path) + { + // already included + return Ok(()); + } + + if !self.open_files.insert(path.clone()) { + let loc = self.parser.current_loc(); + match current_file_name { + Some(x) => { + bail!("Circular include detected at {:?}:{}", x, loc.1); + } + None => { + // can't happen as current file is in-memory and can't include itself + bail!("Circular include detected at {}:{}", loc.0, loc.1); + } + } + } + + if let Err(e) = self.load_file_source(&path) { + bail!(e); + } + + self.open_files.remove(&path); + return Ok(()); // don't add the include line to the source + } + _ => {} + } + } + x => { + bail!("Error parsing include directive: {:?}", x); + } + } + } + Some(_) => { + // skip other directives + } + None => { + bail!("Unknown directive: {}", s); + } + } + } + _ => {} + } + } + TokenType::Unknown if !*in_hex_block => { + let loc = self.parser.current_loc(); + bail!("Unknown token at line {}:{}", loc.0, loc.1); + } + TokenType::Ident => { + match token.val { + TokenVal::Ident(s) => { + if let Some(token_id) = self.reserved_words.map.get(&s.to_ascii_lowercase()) + { + match *token_id { + TOKEN_EXPORT => { + let loc = self.parser.current_loc(); + let token = self.parser.get_token(); + if let TokenType::Ident = token.token_type { + if let TokenVal::Ident(s) = token.val { + if let Some(token_id) = + self.reserved_words.map.get(&s.to_ascii_lowercase()) + { + if let TOKEN_FUNCTION = *token_id { + if let Err(e) = self.process_new_function(loc.0[loc.1..].trim()) { + bail!(e) + } + } + } + } + } + } + TOKEN_FUNCTION => { + if let Err(e) = self.process_new_function(line) { + bail!(e) + } + } + TOKEN_END => { + let scope = self.scopes.get_current_scope(); + *in_hex_block = false; // hex blocks can't be nested, so any end will close it + if !scope.is_root() { + if scope.is_in_block() { + // we are in a loop/if block + scope.close_block(); + } else { + // end of function + self.scopes.exit_scope(self.absolute_line_index); + // till 'end' line + }; + }; + } + TOKEN_IF | TOKEN_FOR | TOKEN_WHILE | TOKEN_SWITCH => { + // these blocks use end to close, so we increment the block count + let scope = self.scopes.get_current_scope(); + if !scope.is_root() { + scope.add_block(); + } + } + TOKEN_HEX => { + *in_hex_block = true; + } + _ => { + // ignore + } + } + } + } + _ => {} + } + } + TokenType::Eol => { + return Ok(()); // skip empty lines + } + _ => {} + } + return Ok(()); + } + + fn process_new_function(&mut self, line: &str) -> Result<()> { + use crate::parser::{function_signature, Span}; + + let line = Self::strip_comments(line); + let line = line.as_str(); + let Ok((_, ref signature)) = function_signature(Span::from(line)) else { + let loc = self.parser.current_loc(); + bail!("Can't parse function at {}:{}", loc.0, loc.1) + }; + + self.scopes + .add_function(token_str(line, &signature.token).to_string()); + if signature.cc == crate::parser::FunctionCC::Local { + self.scopes.enter_scope(self.absolute_line_index); // from 'function' line + } + Ok(()) + } + + fn strip_comments(s: &str) -> String { + let mut inside_comment = false; + let mut inside_comment2 = false; + let mut chars = s.chars().peekable(); + let mut buf = &mut String::new(); + + // iterate over all chars, skip comment fragments (/* */ and //) + while let Some(c) = chars.next() { + match c { + _ if inside_comment => { + // skip until the end of the comment + if c == '*' { + if let Some('/') = chars.next() { + inside_comment = false; + } + } + } + _ if inside_comment2 => { + // skip until the end of the comment + if c == '}' { + inside_comment2 = false; + } + } + '/' if chars.peek() == Some(&'/') => { + // line comment // + // there is nothing left on this line, exiting + break; + } + '/' if chars.peek() == Some(&'*') => { + // block comment /* */ + inside_comment = true; + chars.next(); // skip * + } + + '{' if chars.peek() != Some(&'$') => { + // block comment {} but not directives {$...} + inside_comment2 = true; + } + _ if c.is_ascii_whitespace() && buf.is_empty() => { + // skip leading whitespace + continue; + } + _ => { + buf.push(c); + } + } + } + + return buf.to_string(); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_parse_from_file_only() { + let mut preprocessor = PreProcessorBuilder::new() + .reserved_words("src/preprocessor/test/compiler.ini".into()) + .build(); + preprocessor + .parse_file("src/preprocessor/test/script.txt".into()) + .unwrap(); + assert_eq!(preprocessor.files.len(), 1); + assert_eq!( + preprocessor.source_type, + SourceType::File("src/preprocessor/test/script.txt".into()) + ); + } + + #[test] + fn test_parse_from_file() { + let mut preprocessor = PreProcessorBuilder::new() + .implicit_includes(vec!["src/preprocessor/test/const.txt".into()]) + .reserved_words("src/preprocessor/test/compiler.ini".into()) + .build(); + preprocessor + .parse_file("src/preprocessor/test/script.txt".into()) + .unwrap(); + assert_eq!( + preprocessor.source_type, + SourceType::File("src/preprocessor/test/script.txt".into()) + ); + assert_eq!(preprocessor.files.len(), 2); + } + + #[test] + fn test_parse_in_memory() { + let mut preprocessor = PreProcessorBuilder::new() + .implicit_includes(vec!["src/preprocessor/test/const.txt".into()]) + .build(); + preprocessor + .parse_in_memory("const a = 1;\nconst b = 2;\n") + .unwrap(); + assert_eq!(preprocessor.source_type, SourceType::Memory); + assert_eq!(preprocessor.files.len(), 1); + } + + #[test] + fn test_include() { + let mut preprocessor = PreProcessorBuilder::new() + .implicit_includes(vec!["src/preprocessor/test/const.txt".into()]) + .reserved_words("src/preprocessor/test/compiler.ini".into()) + .build(); + + preprocessor + .parse_file("src/preprocessor/test/script_with_include.txt".into()) + .unwrap(); + assert_eq!(preprocessor.files.len(), 3); + + let mut preprocessor = PreProcessorBuilder::new() + .reserved_words("src/preprocessor/test/compiler.ini".into()) + .build(); + let e = preprocessor.parse_in_memory(r#" {$include "#); + assert!(e.is_err()); + + let e = preprocessor.parse_in_memory(r#" {$include missing.txt } "#); + assert!(e.is_err()); + } + + #[test] + fn test_circular_include() { + let mut preprocessor = PreProcessorBuilder::new() + .implicit_includes(vec!["src/preprocessor/test/circular1.txt".into()]) + .reserved_words("src/preprocessor/test/compiler.ini".into()) + .build(); + let e = preprocessor.parse_file("src/preprocessor/test/circular1.txt".into()); + assert!(e.is_err()); + } + + #[test] + fn test_function() { + let mut preprocessor = PreProcessorBuilder::new() + .reserved_words("src/preprocessor/test/compiler.ini".into()) + .build(); + preprocessor + .parse_file("src/preprocessor/test/scr_with_func.txt".into()) + .unwrap(); + assert_eq!(preprocessor.files.len(), 1); + + assert_eq!(preprocessor.get_number_of_functions_this_scope(0), 2); + assert_eq!(preprocessor.get_number_of_functions_this_scope(1), 2); + assert_eq!(preprocessor.get_number_of_functions_this_scope(9), 0); + } + + #[test] + fn test_foreign_function() { + let mut preprocessor = PreProcessorBuilder::new() + .reserved_words("src/preprocessor/test/compiler.ini".into()) + .build(); + preprocessor + .parse_file("src/preprocessor/test/scr_ffi.txt".into()) + .unwrap(); + assert_eq!(preprocessor.files.len(), 1); + + assert_eq!(preprocessor.get_number_of_functions_this_scope(0), 3); + assert_eq!(preprocessor.get_number_of_functions_this_scope(1), 0); + assert_eq!(preprocessor.get_number_of_functions_this_scope(2), 2); + assert_eq!(preprocessor.get_number_of_functions_this_scope(3), 0); + } + + #[test] + fn test_include_hex() { + let mut preprocessor = PreProcessorBuilder::new() + .reserved_words("src/preprocessor/test/compiler.ini".into()) + .build(); + + preprocessor + .parse_file("src/preprocessor/test/hex_inc.txt".into()) + .unwrap(); + assert_eq!(preprocessor.files.len(), 1); + } + + #[test] + fn test_hoisting() { + let mut preprocessor = PreProcessorBuilder::new() + .reserved_words("src/preprocessor/test/compiler.ini".into()) + .implicit_includes(vec!["src/preprocessor/test/const.txt".into()]) + .build(); + + preprocessor + .parse_file("src/preprocessor/test/u.txt".into()) + .unwrap(); + assert_eq!(preprocessor.files.len(), 2); + assert_eq!(preprocessor.get_number_of_functions_this_scope(50), 1); + } + + #[test] + fn test_parse_comments() { + let mut preprocessor = PreProcessorBuilder::new() + .implicit_includes(vec!["src/preprocessor/test/const.txt".into()]) + .build(); + // preprocessor + // .parse_in_memory("function foo\nend ") + // .unwrap(); + // assert_eq!(preprocessor.source_type, SourceType::Memory); + // assert_eq!(preprocessor.scopes.functions.len(), 0); + + let res = preprocessor.parse_in_memory( + r#" + /* + ; + */ + "#, + ); + + assert_eq!(preprocessor.source_type, SourceType::Memory); + assert!(res.is_ok()); + } +} diff --git a/src/preprocessor/scopes.rs b/src/preprocessor/scopes.rs new file mode 100644 index 0000000..a0a525b --- /dev/null +++ b/src/preprocessor/scopes.rs @@ -0,0 +1,92 @@ +use crate::utils::visibility_zone::VisibilityZone; + +#[derive(Default, Debug)] +pub struct Scopes { + stack: Vec, + pub functions: Vec, +} + +#[derive(Default, Debug)] +pub struct Scope { + is_root: bool, + start_line: usize, + open_blocks: usize, +} + +#[derive(Debug)] +pub struct Function { + pub zone: VisibilityZone, + pub signature: String, +} + +impl Scopes { + pub fn new() -> Self { + let mut scopes = Scopes::default(); + scopes.enter_scope(0); // root scope + scopes + } + + pub fn enter_scope(&mut self, line_index: usize) { + let mut scope = Scope { + start_line: line_index, + open_blocks: 0, + is_root: self.stack.is_empty(), + }; + if !self.stack.is_empty() { + scope.inherit(self.get_current_scope()) + } + self.stack.push(scope); + } + + pub fn exit_scope(&mut self, line_index: usize) { + let start_line = self.get_current_scope().start_line; + let end_line = line_index; + + for function in self.functions.iter_mut().rev() { + if function.zone.start == start_line && function.zone.end == 0 { + function.zone.end = end_line; + } + } + self.stack.pop(); + } + + pub fn get_current_scope(&mut self) -> &mut Scope { + self.stack.last_mut().unwrap() + } + + pub fn add_function(&mut self, signature: String) { + let start_line = self.get_current_scope().start_line; + self.functions.push(Function { + // name, + signature, + zone: VisibilityZone { + start: start_line, + end: 0, + }, + }); + } +} + +impl Scope { + fn inherit(&mut self, other: &Scope) {} + + pub fn add_block(&mut self) { + self.open_blocks += 1; + } + + pub fn close_block(&mut self) { + self.open_blocks -= 1; + } + + pub fn is_in_block(&self) -> bool { + self.open_blocks > 0 + } + + pub fn is_root(&self) -> bool { + self.is_root + } + + pub fn start_line(&self) -> usize { + self.start_line + } +} diff --git a/src/preprocessor/test/0.txt b/src/preprocessor/test/0.txt new file mode 100644 index 0000000..3a283d4 --- /dev/null +++ b/src/preprocessor/test/0.txt @@ -0,0 +1,9 @@ +{$CLEO .cs} +nop + +{$I èíê.txt} + + + + +terminate_this_custom_script \ No newline at end of file diff --git a/src/preprocessor/test/addon.txt b/src/preprocessor/test/addon.txt new file mode 100644 index 0000000..139903e --- /dev/null +++ b/src/preprocessor/test/addon.txt @@ -0,0 +1 @@ +const z = 1 \ No newline at end of file diff --git a/src/preprocessor/test/circular1.txt b/src/preprocessor/test/circular1.txt new file mode 100644 index 0000000..bf50560 --- /dev/null +++ b/src/preprocessor/test/circular1.txt @@ -0,0 +1,2 @@ + +{$I circular1.txt} \ No newline at end of file diff --git a/src/preprocessor/test/compiler.ini b/src/preprocessor/test/compiler.ini new file mode 100644 index 0000000..c356b86 --- /dev/null +++ b/src/preprocessor/test/compiler.ini @@ -0,0 +1,92 @@ +; DO NOT CHANGE THE NUMBERS EVER +; +; YOU CAN CHANGE THE STRING NAMES TO CUSTOMIZE THE SANNY SYNTAX +; +;...................... +; VARIABLE TYPES +;...................... +1=INTEGER +1=INT +2=FLOAT +3=SHORTSTRING +3=STRING +4=LONGSTRING +;5=HANDLE +;6=BOOLEAN +;6=BOOL +7=ARRAY +;...................... +; BUILT-IN ROUTINES +;...................... +10=INC +11=DEC +12=MUL +13=DIV +14=Alloc +15=SQR +16=RANDOM +;17=WRITEMEM +;18=READMEM +;...................... +; RESERVED WORDS +;...................... +;40=OF +;;41=TRUE +;;42=FALSE +43=DEFINE +44=EXPORT +45=HEX +46=DEFAULT +47=PUBLISHER +48=DATE +50=IF +51=THEN +52=ELSE +53=AND +54=OR +55=NOT +60=VAR +;61=ENUM +;62=CLASS +65=CONST +66=IMPORT +67=AS +68=FROM +69=FUNCTION +70=FOR +71=WHILE +72=REPEAT +73=CONTINUE +74=BREAK +75=SWITCH +76=CASE +77=RETURN +78=CDECL +79=STDCALL +80=THISCALL +81=LOGICAL +82=OPTIONAL +;...................... +; DIRECTIVES +;...................... +103={$INCLUDE +103={$I +104={$EXTERNAL +104={$E +105={$CLEO +106={$NOSOURCE +107={$OPCODE +107={$O +108={$USE +109={$INCLUDE_ONCE +;...................... +; END +;...................... +255=END + +;...................... +; DEPRECATED +;...................... +49=VERSION +101={$VERSION +102={$VERSION_RESTORE diff --git a/src/preprocessor/test/const.txt b/src/preprocessor/test/const.txt new file mode 100644 index 0000000..2ecc9fc --- /dev/null +++ b/src/preprocessor/test/const.txt @@ -0,0 +1,10 @@ +const +false=0 +true=1 +TIMERA=32@ +TIMERB=33@ +end +var +TIMERA: int +TIMERB: int +end diff --git a/src/preprocessor/test/hex_inc.txt b/src/preprocessor/test/hex_inc.txt new file mode 100644 index 0000000..7b8ac0d --- /dev/null +++ b/src/preprocessor/test/hex_inc.txt @@ -0,0 +1,3 @@ +hex + {$I addon.txt} +end \ No newline at end of file diff --git a/src/preprocessor/test/scr_ffi.txt b/src/preprocessor/test/scr_ffi.txt new file mode 100644 index 0000000..5247491 --- /dev/null +++ b/src/preprocessor/test/scr_ffi.txt @@ -0,0 +1,15 @@ +function ext +function foo() + while true + end + function baz + end + function bar + while true + end + while true + end + end +end +function bar +end \ No newline at end of file diff --git a/src/preprocessor/test/scr_with_func.txt b/src/preprocessor/test/scr_with_func.txt new file mode 100644 index 0000000..3cf3d76 --- /dev/null +++ b/src/preprocessor/test/scr_with_func.txt @@ -0,0 +1,19 @@ +function foo() + + while true + end + + function baz + end + + function bar + while true + end + while true + end + end +end + + +function bar +end \ No newline at end of file diff --git a/src/preprocessor/test/script.txt b/src/preprocessor/test/script.txt new file mode 100644 index 0000000..c5cfa84 --- /dev/null +++ b/src/preprocessor/test/script.txt @@ -0,0 +1,4 @@ +{$CLEO} + +nop +//-------------MAIN--------------- \ No newline at end of file diff --git a/src/preprocessor/test/script_with_include.txt b/src/preprocessor/test/script_with_include.txt new file mode 100644 index 0000000..0da93db --- /dev/null +++ b/src/preprocessor/test/script_with_include.txt @@ -0,0 +1,6 @@ +{$CLEO} + +{$INCLUDE_ONCE addon.txt} +{$INCLUDE ..\test\addon.txt} + +nop \ No newline at end of file diff --git a/src/preprocessor/test/u.txt b/src/preprocessor/test/u.txt new file mode 100644 index 0000000..0fd05c8 --- /dev/null +++ b/src/preprocessor/test/u.txt @@ -0,0 +1,65 @@ +{$CLEO .cs} +DynamicLibrary advapi32 +GetUserNameA pGetUserNameA + +if not advapi32 = load_dynamic_library "advapi32.dll" +then + trace "advapi32.dll not found" + terminate_this_custom_script +end + +if not pGetUserNameA = get_dynamic_library_procedure "GetUserNameA" advapi32 +then + trace "function GetUserNameA not found" + terminate_this_custom_script +end + +function test +end +test() +:test + + + +//define function foo + +int lpBuffer = get_label_pointer @name_buf +int pcbBuffer = get_label_pointer @size_buf + +int res = pGetUserNameA(lpBuffer, pcbBuffer) +if res == 0 +then + printLastError("GetUserNameA failed") + terminate_this_custom_script +end + +trace "current username is %s" lpBuffer +free_dynamic_library advapi32 +terminate_this_custom_script + +function printLastError(msg: string) + if DynamicLibrary kernel32 = load_dynamic_library "kernel32.dll" + then + if GetLastError pGetLastError = get_dynamic_library_procedure "GetLastError" kernel32 + then + int errorCode = pGetLastError() + print_help_formatted {text} "%s. Error code %d" msg errorCode + end + free_dynamic_library kernel32 + end + + function GetLastError: int // + +end + +:name_buf +hex + 00(256) +end +:size_buf +hex + FF 00 00 00 +end + + +function GetUserNameA(lpBuffer: int, pcbBuffer: int): int diff --git a/src/sanny_update/ffi.rs b/src/sanny_update/ffi.rs new file mode 100644 index 0000000..36a5eab --- /dev/null +++ b/src/sanny_update/ffi.rs @@ -0,0 +1,36 @@ +use crate::common_ffi::{pchar_to_str, PChar}; + +#[no_mangle] +pub unsafe extern "C" fn sb_download_update(channel: u8, local_version: PChar) -> bool { + boolclosure!({ + super::download_update(channel.into(), pchar_to_str(local_version)?).then_some(()) + }) +} + +#[no_mangle] +pub unsafe extern "C" fn sb_update_exists(channel: u8, local_version: PChar) -> bool { + boolclosure!({ super::has_update(channel.into(), pchar_to_str(local_version)?).then_some(()) }) +} + +#[no_mangle] +pub unsafe extern "C" fn sb_update_cleanup() { + super::cleanup(); +} + +#[no_mangle] +pub unsafe extern "C" fn sb_update_get_remote_version(channel: u8, out: *mut PChar) -> bool { + boolclosure!({ + let remote_version = super::get_latest_version_from_github(channel.into())?; + *out = std::ffi::CString::new(remote_version).unwrap().into_raw(); + Some(()) + }) +} + +#[no_mangle] +pub unsafe extern "C" fn sb_update_get_release_notes(channel: u8, out: *mut PChar) -> bool { + boolclosure!({ + let release_notes = super::get_release_notes_from_github(channel.into())?; + *out = std::ffi::CString::new(release_notes).unwrap().into_raw(); + Some(()) + }) +} diff --git a/src/sanny_update/http_client.rs b/src/sanny_update/http_client.rs new file mode 100644 index 0000000..116281d --- /dev/null +++ b/src/sanny_update/http_client.rs @@ -0,0 +1,22 @@ +pub fn get_string(url: &str) -> Option { + get(url)?.into_string().ok() +} + +pub fn get_json(url: &str) -> Option { + get(url)?.into_json::().ok() +} + +pub fn get_binary(url: &str) -> Option> { + let mut reader = get(url)?.into_reader(); + let mut buf: Vec = Vec::new(); + reader.read_to_end(&mut buf).ok(); + Some(buf) +} + +fn get(url: &str) -> Option { + let agent = ureq::builder() + .timeout_connect(std::time::Duration::from_secs(5)) + .build(); + + agent.get(url).call().ok() +} diff --git a/src/sanny_update/mod.rs b/src/sanny_update/mod.rs new file mode 100644 index 0000000..0a86b99 --- /dev/null +++ b/src/sanny_update/mod.rs @@ -0,0 +1,380 @@ +#![feature(io_error_more)] +use std::{ + collections::HashMap, + path::{Path, PathBuf}, + sync::atomic::{AtomicI64, Ordering}, +}; +use cached::proc_macro::cached; +use ctor::ctor; +use serde::Deserialize; +use simplelog::*; +mod ffi; +mod http_client; +use const_format::concatcp; + +use crate::utils::version::compare_versions; + +const GITHUB_ORG_NAME: &str = "sannybuilder"; +const GITHUB_REPO_NAME: &str = "dev"; +const GITHUB_BRANCH_NAME: &str = "updates"; +const UPDATE_DIR_NAME: &str = "updates"; +const DELETE_EXTENSION: &str = "deleteonlaunch"; + +const UPDATE_REPO: &str = concatcp!( + "https://api.github.com/repos/", + GITHUB_ORG_NAME, + "/", + GITHUB_REPO_NAME, + "/commits/", + GITHUB_BRANCH_NAME +); +const CONTENT_URL: &str = concatcp!( + "https://raw.githubusercontent.com/", + GITHUB_ORG_NAME, + "/", + GITHUB_REPO_NAME, + "/", + GITHUB_BRANCH_NAME +); + +#[derive(Eq, PartialEq, Hash, Clone, Copy)] +pub enum Channel { + Stable, + Beta, + Nightly, +} + +impl From for Channel { + fn from(val: u8) -> Self { + match val { + 1 => Channel::Beta, + 2 => Channel::Nightly, + _ => Channel::Stable, + } + } +} + +#[derive(Deserialize, Debug, Default)] +struct UpdateInfo { + min: String, + url: String, +} + +#[derive(Deserialize, Debug, Default)] +struct Manifest { + version: String, + + #[serde(rename = "_")] + info: Vec, + + release_notes: String, +} + +impl std::fmt::Display for Channel { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Channel::Stable => write!(f, "stable"), + Channel::Beta => write!(f, "beta"), + Channel::Nightly => write!(f, "nightly"), + } + } +} + +fn get_exe_folder() -> std::io::Result { + let path = std::env::current_exe()?; + match path.parent() { + Some(p) => Ok(p.to_path_buf()), + None => Err(std::io::Error::from(std::io::ErrorKind::Other)), + } +} + +fn get_update_dir() -> std::io::Result { + Ok(get_exe_folder()?.join(UPDATE_DIR_NAME)) +} + +// fn store_update_check_timestamp() { +// let ts = chrono::Utc::now().timestamp(); + +// match get_exe_folder() { +// Ok(folder) => { +// let ini_file = folder.join(SETTINGS_FILE); +// ini::Ini::load_from_file_noescape(&ini_file).and_then(|mut map| { +// map.set_to(Some("Main"), UPDATE_TIME_KEY.into(), ts.to_string()); +// map.write_to_file_opt( +// ini_file, +// ini::WriteOption { +// escape_policy: ini::EscapePolicy::Nothing, // don't escape backslashes +// ..Default::default() +// }, +// )?; +// log::debug!("update check timestamp updated to {}", ts); +// Ok(()) +// }); +// } +// _ => { +// log::error!("failed to get exe folder"); +// } +// } +// } + +// fn get_update_check_timestamp() -> i64 { +// match get_exe_folder() { +// Ok(folder) => { +// let ini_file = folder.join(SETTINGS_FILE); +// match ini::Ini::load_from_file_noescape(&ini_file) { +// Ok(map) => match map.get_from(Some("Main"), UPDATE_TIME_KEY.into()) { +// Some(t) => t.parse::().unwrap_or_else(|e| { +// log::error!("parse error while reading settings.ini {e}"); +// 0 +// }), +// _ => { +// log::debug!( +// "LastUpdateChecklast check value is not found in ini file {}", +// ini_file.display() +// ); +// 0 +// } +// }, +// Err(e) => { +// log::debug!("can't load ini file {}: {e}", ini_file.display()); +// 0 +// } +// } +// } +// _ => { +// log::error!("failed to get exe folder"); +// 0 +// } +// } +// } + +// pub fn has_passed_check_cooldown() -> bool { +// let cooldown = chrono::Utc::now() - chrono::Duration::hours(COOLDOWN_HOURS); +// let last = get_update_check_timestamp(); +// cooldown.timestamp() >= last +// } + +#[cached(time=60)] +fn get_from_channel(channel: Channel) -> Option { + http_client::get_json(&format!("{}/{}", CONTENT_URL, channel)) +} + +fn get_latest_version_from_github(channel: Channel) -> Option { + get_from_channel(channel).and_then(|json| match serde_json::from_value::(json) { + Ok(manifest) => Some(manifest.version), + Err(e) => { + log::error!("{e}"); + None + } + }) +} + +fn get_release_notes_from_github(channel: Channel) -> Option { + get_from_channel(channel).and_then(|json| match serde_json::from_value::(json) { + Ok(manifest) => Some(manifest.release_notes), + Err(e) => { + log::error!("{e}"); + None + } + }) +} + +fn get_download_link(channel: Channel, local_version: &str) -> Option { + get_from_channel(channel).and_then(|json| { + match serde_json::from_value::(json) { + Ok(manifest) => { + for i in manifest.info { + // fallback url + if (i.min.eq("*")) { + return Some(i.url); + } + let min = version_compare::Version::from(&i.min); + let current = version_compare::Version::from(local_version); + + if current >= min { + return Some(i.url); + } + } + log::error!("no suitable update found for {local_version}"); + None + } + Err(e) => { + log::error!("{e}"); + None + } + } + }) +} + +fn zip_unpack(reader: impl std::io::Read + std::io::Seek, base: &Path) -> std::io::Result<()> { + let mut archive = zip::ZipArchive::new(reader)?; + archive.extract(base)?; + Ok(()) +} + +fn move_files(src_dir: impl AsRef, dst_dir: impl AsRef) -> std::io::Result<()> { + use std::fs; + for entry in fs::read_dir(src_dir)? { + let entry = entry?; + let ty = entry.file_type()?; + let dst = dst_dir.as_ref().join(entry.file_name()); + if ty.is_dir() { + // entry is a folder in source + move_files(entry.path(), dst)?; + } else { + // entry is a file in source + log::debug!("copying {} to {}", entry.path().display(), dst.display()); + match fs::rename(entry.path(), &dst) { + Ok(_) => {} + Err(e) => { + // try renaming original file + match dst.extension().and_then(std::ffi::OsStr::to_str) { + Some(ext) + if ["exe", "dll"].contains(&ext.to_ascii_lowercase().as_str()) => + { + // + match fs::rename( + &dst, + &dst.with_extension(format!("{ext}.{DELETE_EXTENSION}")), + ) { + Ok(_) => { + // try again + if let Ok(_) = fs::rename(entry.path(), dst) { + continue; + } + } + Err(e) => { + log::error!("Failed on renaming used file {}", dst.display()); + } + } + } + _ => {} + }; + + return Err(e); + } + } + } + } + Ok(()) +} + +fn delete_files(src_dir: impl AsRef, extension: &str) -> std::io::Result<()> { + use std::fs; + for entry in fs::read_dir(src_dir)? { + let entry = entry?; + let ty = entry.file_type()?; + if ty.is_dir() { + // entry is a folder in source + delete_files(entry.path(), extension)?; + } else { + // entry is a file in source + let file = entry.path(); + + match file.extension().and_then(std::ffi::OsStr::to_str) { + Some(ext) if ext.eq_ignore_ascii_case(extension) => { + log::debug!("deleting {}", file.display()); + match fs::remove_file(file) { + Ok(_) => {} + Err(e) => { + log::error!("{e}"); + } + } + } + _ => {} + } + } + } + Ok(()) +} + +pub fn download_update(channel: Channel, local_version: &str) -> bool { + log::info!("downloading update from channel {channel}"); + + let Some(url) = get_download_link(channel, local_version) else { + log::error!("failed to get download link"); + return false; + }; + + log::debug!("downloading archive from {}", url); + + let Some(buf) = http_client::get_binary(&url) else { + log::error!("failed to download archive"); + return false; + }; + // save to local file + let Ok(cwd) = get_exe_folder() else { + log::error!("failed to get exe folder"); + return false; + }; + + let Ok(temp) = get_update_dir() else { + log::error!("failed to get update dir"); + return false; + }; + log::info!("unpacking update to {}", temp.display()); + + match zip_unpack(&mut std::io::Cursor::new(buf), &temp) { + Ok(_) => {} + Err(e) => { + log::error!("unpack failed: {e}"); + return false; + } + } + + // // copy all files from pending to cwd + log::info!("copying files to {}", cwd.display()); + match move_files(temp, cwd) { + Ok(_) => { + log::debug!("copied all files"); + } + Err(e) => { + log::error!("copy failed: {e}"); + return false; + } + } + + log::info!("update complete."); + + true +} + +pub fn has_update(channel: Channel, local_version: &str) -> bool { + use version_compare::Version; + + // if (!has_passed_check_cooldown()) { + // log::info!("update check cooldown has not passed yet."); + // return false; + // } + // store_update_check_timestamp(); + + log::info!("checking for updates from channel {channel}"); + let Some(remote_version) = get_latest_version_from_github(channel) else { + log::error!("failed to get remote snapshot"); + return false; + }; + let remote_version = Version::from(&remote_version); + log::debug!("remote version: {:?}", remote_version); + + let local_version = Version::from(local_version); + log::debug!("local version: {:?}", local_version); + + if local_version >= remote_version { + log::info!("already on latest version. no update needed"); + return false; + } + + log::info!("new version available: {:?}", remote_version); + + return true; +} + +pub fn cleanup() { + log::info!("Performing cleanup on current directory"); + let Ok(cwd) = get_exe_folder() else { + log::error!("failed to get exe folder"); + return; + }; + + delete_files(cwd, DELETE_EXTENSION); +} diff --git a/src/source_map/ffi.rs b/src/source_map/ffi.rs new file mode 100644 index 0000000..31bca0a --- /dev/null +++ b/src/source_map/ffi.rs @@ -0,0 +1,138 @@ +use super::source_map::SourceMap; +use crate::common_ffi::*; + +#[no_mangle] +pub extern "C" fn source_map_new() -> *mut SourceMap { + ptr_new(SourceMap::new()) +} + +#[no_mangle] +pub unsafe extern "C" fn source_map_free(map: *mut SourceMap) { + ptr_free(map) +} + +#[no_mangle] +pub unsafe extern "C" fn source_map_clear(map: *mut SourceMap) -> bool { + boolclosure!({ + let map = map.as_mut()?; + (*map).clear(); + Some(()) + }) +} + +#[no_mangle] +pub unsafe extern "C" fn source_map_add( + map: *mut SourceMap, + path: PChar, + line: u32, + offset: u32, +) -> bool { + boolclosure!({ + let path = pchar_to_str(path)?; + (*map).add(path, line, offset); + Some(()) + }) +} + +#[no_mangle] +pub unsafe extern "C" fn source_map_get_offset( + map: *mut SourceMap, + path: PChar, + line: u32, + out: *mut u32, +) -> bool { + boolclosure!({ + let path = pchar_to_str(path)?; + *out = (*map).get_offset(path, line)?; + Some(()) + }) +} + +#[no_mangle] +pub unsafe extern "C" fn source_map_get_line( + map: *mut SourceMap, + path: PChar, + offset: u32, + out: *mut u32, +) -> bool { + boolclosure!({ + let path = pchar_to_str(path)?; + *out = (*map).get_line(path, offset)?; + Some(()) + }) +} + + +#[no_mangle] +pub unsafe extern "C" fn source_map_adjust_offset_by(map: *mut SourceMap, delta: u32) { + (*map).adjust_offset_by(delta); +} + +#[no_mangle] +pub unsafe extern "C" fn source_map_dump(map: *mut SourceMap, path: PChar) -> bool { + boolclosure!({ + let path = pchar_to_str(path)?; + let mut file = std::fs::File::create(path).ok()?; + let map = map.as_ref()?; + serde_json::to_writer(&mut file, map).ok()?; + Some(()) + }) +} + +#[no_mangle] +pub unsafe extern "C" fn source_map_new_local_variable_scope( + map: *mut SourceMap, + file_name: PChar, + line: u32, + var_name: PChar, + var_index: i32, +) -> bool { + boolclosure!({ + let file_name = pchar_to_str(file_name)?; + let name = pchar_to_str(var_name)?; + (*map).new_local_variable_scope(file_name, line, name, var_index); + Some(()) + }) +} + +#[no_mangle] +pub unsafe extern "C" fn source_map_find_local_variable_index( + map: *mut SourceMap, + file_name: PChar, + line: u32, + var_name: PChar, + out: *mut i32, +) -> bool { + boolclosure!({ + let file_name = pchar_to_str(file_name)?; + let name = pchar_to_str(var_name)?; + *out = (*map).find_local_variable_index(file_name, line, name)?; + Some(()) + }) +} + +#[no_mangle] +pub unsafe extern "C" fn source_map_new_global_variable( + map: *mut SourceMap, + name: PChar, + var_index: i32, +) -> bool { + boolclosure!({ + let name = pchar_to_str(name)?; + (*map).new_global_variable(name, var_index); + Some(()) + }) +} + +#[no_mangle] +pub unsafe extern "C" fn source_map_find_global_variable_index( + map: *mut SourceMap, + name: PChar, + out: *mut i32, +) -> bool { + boolclosure!({ + let name = pchar_to_str(name)?; + *out = (*map).find_global_variable_index(name)?; + Some(()) + }) +} diff --git a/src/source_map/mod.rs b/src/source_map/mod.rs new file mode 100644 index 0000000..4ddd584 --- /dev/null +++ b/src/source_map/mod.rs @@ -0,0 +1,2 @@ +pub mod ffi; +pub mod source_map; \ No newline at end of file diff --git a/src/source_map/source_map.rs b/src/source_map/source_map.rs new file mode 100644 index 0000000..4a7279d --- /dev/null +++ b/src/source_map/source_map.rs @@ -0,0 +1,164 @@ +use std::collections::HashMap; + +use serde::Serialize; + +#[derive(Default, Serialize)] +pub struct SourceMap { + files: HashMap>, + local_variables: + HashMap>>, + global_variables: HashMap, +} + +impl SourceMap { + pub fn new() -> Self { + Self::default() + } + + pub fn clear(&mut self) { + self.files.clear(); + } + + pub fn add(&mut self, path: &str, line: u32, offset: u32) { + self.files + .entry(path.to_string()) + .or_insert_with(HashMap::new) + .insert(line, offset); + } + + pub fn get_offset(&self, path: &str, line: u32) -> Option { + let file = self.files.get(path)?; + file.get(&line).cloned() + } + + pub fn get_line(&self, path: &str, offset: u32) -> Option { + let file = self.files.get(path)?; + for (line, off) in file.iter() { + if *off == offset { + return Some(*line); + } + } + None + } + + pub fn adjust_offset_by(&mut self, delta: u32) { + for file in self.files.values_mut() { + for offset in file.values_mut() { + *offset += delta; + } + } + } + + pub fn new_local_variable_scope( + &mut self, + file_name: &str, + line: u32, + name: &str, + var_index: i32, + ) { + self.local_variables + .entry(file_name.to_lowercase()) + .or_insert_with(HashMap::new) + .entry(name.to_lowercase()) + .or_insert_with(Vec::new) + .push((line, var_index)); + } + + pub fn find_local_variable_index( + &self, + file_name: &str, + line: u32, + var_name: &str, + ) -> Option { + let file_map = self.local_variables.get(&file_name.to_lowercase())?; + let scopes = file_map.get(&var_name.to_lowercase())?; + + for (i, scope) in scopes.iter().enumerate() { + if (*scope).0 > line { + if i == 0 { + // our line is before the first declaration + return None; + } + // our variable is not in this scope, which means it was declared in the previous one + // so we return the index from the previous scope + return Some(scopes[i - 1].1); + } + + if i == scopes.len() - 1 { + // our line is after the last declaration + return Some(scope.1); + } + } + + None + } + + pub fn new_global_variable(&mut self, name: &str, var_index: i32) { + self.global_variables.insert(name.to_lowercase(), var_index); + } + + pub fn find_global_variable_index(&self, name: &str) -> Option { + self.global_variables.get(&name.to_lowercase()).cloned() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_source_map() { + let mut map = SourceMap::new(); + map.add("file1", 1, 10); + map.add("file1", 2, 20); + map.add("file1", 3, 30); + map.add("file2", 1, 40); + map.add("file2", 2, 50); + map.add("file2", 3, 60); + + assert_eq!(map.get_offset("file1", 1), Some(10)); + assert_eq!(map.get_offset("file1", 2), Some(20)); + assert_eq!(map.get_offset("file1", 3), Some(30)); + assert_eq!(map.get_offset("file2", 1), Some(40)); + assert_eq!(map.get_offset("file2", 2), Some(50)); + assert_eq!(map.get_offset("file2", 3), Some(60)); + + map.adjust_offset_by(100); + + assert_eq!(map.get_offset("file1", 1), Some(110)); + assert_eq!(map.get_offset("file1", 2), Some(120)); + assert_eq!(map.get_offset("file1", 3), Some(130)); + assert_eq!(map.get_offset("file2", 1), Some(140)); + assert_eq!(map.get_offset("file2", 2), Some(150)); + assert_eq!(map.get_offset("file2", 3), Some(160)); + } + + #[test] + fn test_local_variables() { + let mut map = SourceMap::new(); + map.new_local_variable_scope("file1", 1, "var1", 10); + map.new_local_variable_scope("file1", 3, "var1", 20); + map.new_local_variable_scope("file1", 5, "var1", 30); + + assert_eq!(map.find_local_variable_index("file1", 0, "var1"), None); + assert_eq!(map.find_local_variable_index("file1", 1, "var1"), Some(10)); + assert_eq!(map.find_local_variable_index("file1", 2, "var1"), Some(10)); + assert_eq!(map.find_local_variable_index("file1", 3, "var1"), Some(20)); + assert_eq!(map.find_local_variable_index("file1", 4, "var1"), Some(20)); + assert_eq!(map.find_local_variable_index("file1", 6, "var1"), Some(30)); + } + + #[test] + fn test_local_variables2() { + let mut map = SourceMap::new(); + map.new_local_variable_scope("file1", 5, "var1", 10); + + assert_eq!(map.find_local_variable_index("file1", 0, "var1"), None); + assert_eq!(map.find_local_variable_index("file1", 1, "var1"), None); + assert_eq!(map.find_local_variable_index("file1", 2, "var1"), None); + assert_eq!(map.find_local_variable_index("file1", 3, "var1"), None); + assert_eq!(map.find_local_variable_index("file1", 4, "var1"), None); + assert_eq!(map.find_local_variable_index("file1", 5, "var1"), Some(10)); + assert_eq!(map.find_local_variable_index("file1", 6, "var1"), Some(10)); + } +} diff --git a/src/update_service/service.rs b/src/update_service/service.rs index 5b01567..82d8951 100644 --- a/src/update_service/service.rs +++ b/src/update_service/service.rs @@ -4,7 +4,7 @@ use std::error::Error; use version_compare::Version; const API_URL: &str = "https://api.github.com/repos/sannybuilder/library/contents"; -const RAW_CONTENT: &str = "https://raw.githubusercontent.com/sannybuilder/library/master"; +const REPO: &str = "https://raw.githubusercontent.com/sannybuilder/library"; pub struct UpdateService { pub status_change: StatusChangeCallback, @@ -20,9 +20,12 @@ impl UpdateService { let status_change = self.status_change; let game = params.to_string(); std::thread::spawn(move || { - let decoded = match get_file_gracefully(game, "version.txt") { + let decoded = match download_file_gracefully(&game, "version.txt", "master") { Ok(v) => std::ffi::CString::new(v).unwrap(), - Err(_) => std::ffi::CString::new("").unwrap(), + Err(e) => { + log::error!("{}", e); + std::ffi::CString::new("").unwrap() + } }; (status_change)(decoded.as_ptr()); }); @@ -30,20 +33,15 @@ impl UpdateService { } pub fn auto_update(&self, params: &str) -> Option<()> { - let mut v = Vec::new(); - let mut x = params.split_terminator('|').map(|x| x.to_string()); - while let Some(game) = x.next() { - if let Some(version) = x.next() { - v.push((game, version, x.next())); - } else { - return None; - } - } let status_change = self.status_change; + let params = params.to_owned(); std::thread::spawn(move || { - let result = match auto_update(v) { + let result = match auto_update(params) { Ok(v) => std::ffi::CString::new(v).unwrap(), - Err(_) => std::ffi::CString::new("").unwrap(), + Err(e) => { + log::error!("{}", e); + std::ffi::CString::new("").unwrap() + } }; (status_change)(result.as_ptr()); }); @@ -51,59 +49,88 @@ impl UpdateService { } pub fn download(&self, params: &str) -> Option<()> { - let mut x = params.split_terminator('|').map(|x| x.to_string()); - if let Some(game) = x.next() { - let status_change = self.status_change; - let destination = x.next(); - std::thread::spawn(move || { - let result = match download(game, destination) { - Ok(v) => std::ffi::CString::new(v).unwrap(), - Err(_) => std::ffi::CString::new("").unwrap(), - }; - (status_change)(result.as_ptr()); - }); - } else { - return None; - } + static EMPTY_STRING: i8 = '\0' as i8; + + let params = params.to_owned(); + let status_change = self.status_change; + std::thread::spawn(move || { + let mut x = params.split_terminator('|'); //.map(|x| x.to_string()); + let game = match x.next() { + Some(x) if !x.is_empty() => x, + _ => { + log::error!("Error during update: no game specified"); + (status_change)(&EMPTY_STRING as _); + return; + } + }; + let library_path = match x.next() { + Some(x) if !x.is_empty() => x, + _ => { + log::error!("Error during update: no library path specified"); + (status_change)(&EMPTY_STRING as _); + return; + } + }; + + let classes_path = x.next(); + let enums_path = x.next(); + let examples_path = x.next(); + + let result = match download_library(game, library_path) { + Ok(v) => { + download_config_files(classes_path, enums_path, examples_path, game); + std::ffi::CString::new(v).unwrap() + } + Err(e) => { + log::error!("{}", e); + std::ffi::CString::new("").unwrap() + } + }; + + (status_change)(result.as_ptr()); + }); + Some(()) } } -fn get_file_gracefully(game: String, file_name: &str) -> Result> { - let url = format!("{}/{}/{}", RAW_CONTENT, game, file_name); +fn download_file_gracefully( + folder: &str, + file_name: &str, + branch: &str, +) -> Result> { + let url = format!("{}/{}/{}/{}", REPO, branch, folder, file_name); + log::info!("Downloading {}", url.as_str()); match super::http_client::get_string(url.as_str()) { Some(s) => Ok(s), - None => get_file_with_api(game, file_name), + None => download_file_using_api(folder, file_name, branch), } } -fn get_file_with_api(game: String, file_name: &str) -> Result> { - let path = &format!("{}/{}", game, file_name); +fn download_file_using_api( + folder: &str, + file_name: &str, + branch: &str, +) -> Result> { + let path = &format!("{}/{}", folder, file_name); - let contents = super::http_client::get_json(format!("{}/{}", API_URL, game).as_str()) - .ok_or("Request failed")?; + let contents = + super::http_client::get_json(format!("{}/{}?ref={}", API_URL, folder, branch).as_str()) + .ok_or("Request failed")?; let contents = contents.as_array().ok_or("Response is not an array")?; let git_url = contents .iter() .find_map(|x| { - let x = x.as_object()?; - if x.get("path")?.eq(path) { - return x.get("git_url")?.as_str(); + if x["path"].eq(path) { + return x["git_url"].as_str(); } None }) .ok_or(format!("Can't find git_url for {}", path))?; let body = super::http_client::get_json(git_url).ok_or("Request failed")?; - - let content = body - .as_object() - .ok_or("Response is not an object")? - .get("content") - .ok_or("Key 'content' is not found")? - .as_str() - .ok_or("Content is not a string")?; + let content = body["content"].as_str().ok_or("Content is not a string")?; let content = content.replace('\n', ""); let decoded = decode(content)?; @@ -111,42 +138,82 @@ fn get_file_with_api(game: String, file_name: &str) -> Result(game: &str) -> Option<&'a str> { +fn get_library_json_name<'a>(game: &str) -> Option<&'a str> { match game { "gta3" => Some("gta3.json"), "vc" => Some("vc.json"), "sa" => Some("sa.json"), "sa_mobile" => Some("sa_mobile.json"), "vc_mobile" => Some("vc_mobile.json"), + // "lcs" => Some("lcs.json"), + // "vcs" => Some("vcs.json"), _ => None, } } -fn download(game: String, destination: Option) -> Result> { - let file_name = get_file_name(&game).ok_or("Unsupported game")?; - let destination = destination.unwrap_or(file_name.to_owned()); - let decoded = get_file_gracefully(game, file_name)?; +fn download_library(game: &str, destination: &str) -> Result> { + let file_name = get_library_json_name(&game).ok_or("Unsupported game")?; + let decoded = download_file_gracefully(&game, file_name, "master")?; + log::info!("Saving new file {}", destination); std::fs::write(destination, &decoded)?; let lib = serde_json::from_str::(decoded.as_str())?; Ok(lib.meta.version) } -fn auto_update(options: Vec<(String, String, Option)>) -> Result> { +fn download_text_file( + folder: &str, + file_name: &str, + destination: &str, +) -> Result<(), Box> { + if folder.is_empty() || destination.is_empty() { + return Ok(()); + } + let decoded = download_file_gracefully(&folder, file_name, "gh-pages/assets")?; + log::info!("Saving new file {}", destination); + std::fs::write(destination, &decoded)?; + Ok(()) +} + +fn auto_update(options: String) -> Result> { let mut versions: String = String::new(); - for (game, version, destination) in options { - let v = match get_file_gracefully(game.clone(), "version.txt") { - Ok(x) => x, - Err(_) => continue, + let mut x = options.split_terminator('|'); + while let Some(game) = x.next() { + let version = x.next(); + let library_path = x.next(); + let classes_path = x.next(); + let enums_path = x.next(); + let examples_path = x.next(); + + let Some(file_name) = get_library_json_name(&game) else { + log::error!("Unsupported game {game}"); + continue; + }; + let Ok(v) = download_file_gracefully(&game, "version.txt", "master") else { + log::error!("Can't get version for {game}"); + continue; + }; + + let Some(version) = version else { + log::error!("No version for {game}"); + continue; }; - let file_name = get_file_name(&game).ok_or("Unsupported game")?; - let destination = destination.unwrap_or(file_name.to_owned()); + let destination = library_path.unwrap_or(file_name); let remote_version = Version::from(v.as_str()); - let local_version = Version::from(version.as_str()); + let local_version = Version::from(version); + + log::debug!( + "Remote version: {:?}, Local version: {:?}", + remote_version, + local_version + ); if remote_version > local_version { - let decoded = get_file_gracefully(game.clone(), file_name)?; + let decoded = download_file_gracefully(&game, file_name, "master")?; + log::info!("Saving new file {}", destination); std::fs::write(destination, &decoded)?; + + download_config_files(classes_path, enums_path, examples_path, game); let lib = serde_json::from_str::(decoded.as_str())?; versions.push_str(format!("{} {}", game, lib.meta.version).as_str()); @@ -154,3 +221,22 @@ fn auto_update(options: Vec<(String, String, Option)>) -> Result, + enums_path: Option<&str>, + examples_path: Option<&str>, + game: &str, +) { + ["classes.db", "enums.txt", "opcodes.txt"] + .iter() + .zip([classes_path, enums_path, examples_path].iter()) + .for_each(|(file, path)| match path { + Some(path) if !path.is_empty() => { + if let Err(e) = download_text_file(game, file, path) { + log::error!("{}", e) + } + } + _ => (), + }); +} diff --git a/src/utils/compiler_const.rs b/src/utils/compiler_const.rs index 01111ef..b50a208 100644 --- a/src/utils/compiler_const.rs +++ b/src/utils/compiler_const.rs @@ -1,10 +1,25 @@ // from compiler.ini pub const TOKEN_INCLUDE: i32 = 103; -pub const TOKEN_CONST: i32 = 65; -pub const TOKEN_END: i32 = 255; +pub const TOKEN_INCLUDE_ONCE: i32 = 109; + pub const TOKEN_INT: i32 = 1; pub const TOKEN_FLOAT: i32 = 2; pub const TOKEN_STRING: i32 = 3; pub const TOKEN_LONGSTRING: i32 = 4; pub const TOKEN_HANDLE: i32 = 5; -pub const TOKEN_BOOL: i32 = 6; \ No newline at end of file +pub const TOKEN_BOOL: i32 = 6; + +pub const TOKEN_DEFINE: i32 = 43; +pub const TOKEN_EXPORT: i32 = 44; + +// blocks +pub const TOKEN_HEX: i32 = 45; +pub const TOKEN_IF: i32 = 50; +pub const TOKEN_VAR: i32 = 60; // can be inlined +pub const TOKEN_CONST: i32 = 65; // can be inlined +pub const TOKEN_FUNCTION: i32 = 69; +pub const TOKEN_FOR: i32 = 70; +pub const TOKEN_WHILE: i32 = 71; +pub const TOKEN_SWITCH: i32 = 75; + +pub const TOKEN_END: i32 = 255; diff --git a/src/utils/mod.rs b/src/utils/mod.rs index 101246a..1de21e9 100644 --- a/src/utils/mod.rs +++ b/src/utils/mod.rs @@ -1,3 +1,5 @@ pub mod version; pub mod ffi; pub mod compiler_const; +pub mod path; +pub mod visibility_zone; \ No newline at end of file diff --git a/src/utils/path.rs b/src/utils/path.rs new file mode 100644 index 0000000..c66ffbb --- /dev/null +++ b/src/utils/path.rs @@ -0,0 +1,34 @@ +use std::path::{Path, PathBuf}; + +pub fn resolve_path(p: &str, parent_file: Option<&PathBuf>) -> Option { + let path = Path::new(p); + + if path.is_absolute() { + // return Some(p.to_string()); + return normalize_file_name(path); + } + + match parent_file { + Some(x) => { + // todo: + // If the file path is relative, the compiler scans directories in the following order to find the file: + // 1. directory of the file with the directive + // 2. data folder for the current edit mode + // 3. Sanny Builder root directory + // 4. the game directory + let dir_name = Path::new(&x).parent()?; + let abs_name = dir_name.join(path); + + // Some(String::from(abs_name.to_str()?)) + normalize_file_name(&abs_name) + } + None => None, + } +} + + + +pub fn normalize_file_name(file_name: &Path) -> Option { + use normpath::PathExt; + Some(file_name.normalize_virtually().ok()?.into_path_buf()) +} diff --git a/src/utils/visibility_zone.rs b/src/utils/visibility_zone.rs new file mode 100644 index 0000000..7942f36 --- /dev/null +++ b/src/utils/visibility_zone.rs @@ -0,0 +1,12 @@ + +#[derive(Clone, Debug, Default)] +pub struct VisibilityZone { + pub start: usize, // line number where the symbol is defined + pub end: usize, // line number where the symbol can no longer be seen (start of a new function or end of the file) +} + +impl VisibilityZone { + pub fn is_visible_at(&self, line_number: usize) -> bool { + line_number >= self.start && (self.end == 0 || line_number < self.end) + } +} diff --git a/src/v4/ffi.rs b/src/v4/ffi.rs index 4553f07..ecfb32a 100644 --- a/src/v4/ffi.rs +++ b/src/v4/ffi.rs @@ -1,5 +1,4 @@ use crate::common_ffi::{pchar_to_string, PChar}; -use crate::dictionary::dictionary_num_by_str::DictNumByStr; use crate::dictionary::dictionary_str_by_str::DictStrByStr; use crate::legacy_ini::OpcodeTable; use crate::namespaces::namespaces::Namespaces; @@ -9,13 +8,13 @@ pub unsafe extern "C" fn v4_try_transform( input: PChar, ns: *const Namespaces, legacy_ini: *const OpcodeTable, - var_types: *const DictNumByStr, const_lookup: *const DictStrByStr, + compile_callback: extern "C" fn(u32, PChar), out: *mut PChar, ) -> bool { boolclosure! {{ let input = pchar_to_string(input)?; - let result = super::transform(&input, ns.as_ref()?, legacy_ini.as_ref()?, var_types.as_ref()?, const_lookup.as_ref()?)?; + let result = super::transform(&input, ns.as_ref()?, legacy_ini.as_ref()?, const_lookup.as_ref()?)?; *out = std::ffi::CString::new(result).unwrap().into_raw(); Some(()) }} diff --git a/src/v4/helpers.rs b/src/v4/helpers.rs index 01dcd44..ad272bf 100644 --- a/src/v4/helpers.rs +++ b/src/v4/helpers.rs @@ -47,6 +47,8 @@ pub fn as_token(node: &Node) -> Option<&Token> { Variable::ArrayElement(v) => Some(&v.token), Variable::Indexed(v) => Some(&v.token), Variable::Adma(v) => Some(&v.token), + Variable::Pop(v) => Some(&v), + Variable::Push(v) => Some(&v), }, _ => None, } @@ -75,7 +77,7 @@ pub fn as_number(node: &Node) -> Option<&Token> { Node::Literal(t) if matches!( t.syntax_kind, - SyntaxKind::IntegerLiteral | SyntaxKind::FloatLiteral + SyntaxKind::IntegerLiteral | SyntaxKind::FloatLiteral | SyntaxKind::LabelLiteral ) => { Some(t) diff --git a/src/v4/mod.rs b/src/v4/mod.rs index eea63e5..ecb0528 100644 --- a/src/v4/mod.rs +++ b/src/v4/mod.rs @@ -1,6 +1,5 @@ use crate::{ - dictionary::{dictionary_num_by_str::DictNumByStr, dictionary_str_by_str::DictStrByStr}, - legacy_ini::OpcodeTable, + dictionary::dictionary_str_by_str::DictStrByStr, legacy_ini::OpcodeTable, namespaces::namespaces::Namespaces, }; @@ -12,11 +11,10 @@ pub fn transform( expr: &str, ns: &Namespaces, legacy_ini: &OpcodeTable, - var_types: &DictNumByStr, const_lookup: &DictStrByStr, ) -> Option { let body = crate::parser::parse(expr).ok()?.1; - transform::try_tranform(&body, expr, ns, legacy_ini, var_types, const_lookup) + transform::try_tranform(&body, expr, ns, legacy_ini, const_lookup) } #[cfg(test)] @@ -32,12 +30,11 @@ mod tests { table.load_from_file("SASCM.ini"); let mut ns = Namespaces::new(); ns.load_library("sa.json"); - let dict = DictNumByStr::default(); let mut const_lookup = DictStrByStr::default(); const_lookup.add(CString::new("x").unwrap(), CString::new("3@").unwrap()); let t = |input: &str| -> String { - transform(input, &ns, &table, &dict, &const_lookup).unwrap_or_default() + transform(input, &ns, &table, &const_lookup).unwrap_or_default() }; assert_eq!(t("~0@"), "0B1A: 0@"); @@ -46,6 +43,10 @@ mod tests { assert_eq!(t("~10@($_,1i)"), "0B1A: 10@($_,1i)"); assert_eq!(t("~$0101(1000@,12f)"), "0B1A: $0101(1000@,12f)"); assert_eq!(t("~x"), "0B1A: 3@"); + assert_eq!(t("~x[0]"), "0B1A: x[0]"); + assert_eq!(t("~x[1@]"), "0B1A: x[1@]"); + assert_eq!(t("~1@[x]"), "0B1A: 1@[x]"); + assert_eq!(t("~x(1@,1i)"), "0B1A: x(1@,1i)"); } #[test] @@ -54,7 +55,6 @@ mod tests { table.load_from_file("SASCM.ini"); let mut ns = Namespaces::new(); ns.load_library("sa.json"); - let dict = DictNumByStr::default(); let mut const_lookup = DictStrByStr::default(); const_lookup.add(CString::new("x").unwrap(), CString::new("3@").unwrap()); const_lookup.add(CString::new("y").unwrap(), CString::new("4@").unwrap()); @@ -66,7 +66,7 @@ mod tests { ); let t = |input: &str| -> String { - transform(input, &ns, &table, &dict, &const_lookup).unwrap_or_default() + transform(input, &ns, &table, &const_lookup).unwrap_or_default() }; assert_eq!(t("0@ &= 1@"), "0B17: 0@ 1@"); assert_eq!(t("0@ &= 100"), "0B17: 0@ 100"); @@ -78,6 +78,12 @@ mod tests { assert_eq!(t("0@ >>= 1@"), "0B1C: 0@ 1@"); assert_eq!(t("0@ <<= 1@"), "0B1D: 0@ 1@"); assert_eq!(t("&101 <<= &123"), "0B1D: &101 &123"); + assert_eq!(t("0@[0] &= 1@(10@,1i)"), "0B17: 0@[0] 1@(10@,1i)"); + assert_eq!(t("0@[0] |= 1@(10@,1i)"), "0B18: 0@[0] 1@(10@,1i)"); + assert_eq!(t("0@[0] ^= 1@(10@,1i)"), "0B19: 0@[0] 1@(10@,1i)"); + assert_eq!(t("0@[0] %= 1@(10@,1i)"), "0B1B: 0@[0] 1@(10@,1i)"); + assert_eq!(t("0@[0] >>= 1@(10@,1i)"), "0B1C: 0@[0] 1@(10@,1i)"); + assert_eq!(t("0@[0] <<= 1@(10@,1i)"), "0B1D: 0@[0] 1@(10@,1i)"); assert_eq!(t("$var = 5"), "0004: $var 5"); assert_eq!(t("$var = -5"), "0004: $var -5"); @@ -115,6 +121,12 @@ mod tests { assert_eq!(t("x = 5.0"), "0007: 3@ 5.0"); assert_ne!(t("0@ ^= ~1@"), "0B19: 0@ ~1@"); + + assert_eq!(t("|< = 5"), "0006: |< 5"); + assert_eq!(t("|< = -5"), "0006: |< -5"); + assert_eq!(t("|< = n"), "0006: |< 100"); + assert_eq!(t("|< = -n"), "0006: |< -100"); + } #[test] @@ -123,14 +135,13 @@ mod tests { table.load_from_file("SASCM.ini"); let mut ns = Namespaces::new(); ns.load_library("sa.json"); - let dict = DictNumByStr::default(); let mut const_lookup = DictStrByStr::default(); const_lookup.add(CString::new("x").unwrap(), CString::new("3@").unwrap()); const_lookup.add(CString::new("y").unwrap(), CString::new("4@").unwrap()); const_lookup.add(CString::new("z").unwrap(), CString::new("5@").unwrap()); let t = |input: &str| -> String { - transform(input, &ns, &table, &dict, &const_lookup).unwrap_or_default() + transform(input, &ns, &table, &const_lookup).unwrap_or_default() }; assert_eq!(t("0@ = -1 & 1@"), "0B10: 0@ -1 1@"); assert_eq!(t("0@ = 1 | 1@"), "0B11: 0@ 1 1@"); @@ -153,8 +164,62 @@ mod tests { assert_eq!(t("x = y - z"), "0A8F: 3@ 4@ 5@"); assert_eq!(t("x = y * z"), "0A90: 3@ 4@ 5@"); assert_eq!(t("x = y / z"), "0A91: 3@ 4@ 5@"); - + assert_eq!(t("x[0] = y[0] & y[1]"), "0B10: x[0] y[0] y[1]"); + assert_eq!(t("x[0] = y[0] | y[1]"), "0B11: x[0] y[0] y[1]"); + assert_eq!(t("x[0] = y[0] ^ y[1]"), "0B12: x[0] y[0] y[1]"); + assert_eq!(t("x[0] = y[0] % y[1]"), "0B14: x[0] y[0] y[1]"); + assert_eq!(t("x[0] = y[0] >> y[1]"), "0B15: x[0] y[0] y[1]"); + assert_eq!(t("x[0] = y[0] << y[1]"), "0B16: x[0] y[0] y[1]"); + assert_eq!(t("x[0] = y[0] + y[1]"), "0A8E: x[0] y[0] y[1]"); + assert_eq!(t("x[0] = y[0] - y[1]"), "0A8F: x[0] y[0] y[1]"); + assert_eq!(t("x[0] = y[0] * y[1]"), "0A90: x[0] y[0] y[1]"); + assert_eq!(t("x[0] = y[0] / y[1]"), "0A91: x[0] y[0] y[1]"); + assert_eq!( + t("x(0@,1i) = y(n,2i) & y(m,2i)"), + "0B10: x(0@,1i) y(n,2i) y(m,2i)" + ); + assert_eq!( + t("x(0@,1i) = y(n,2i) | y(m,2i)"), + "0B11: x(0@,1i) y(n,2i) y(m,2i)" + ); + assert_eq!( + t("x(0@,1i) = y(n,2i) ^ y(m,2i)"), + "0B12: x(0@,1i) y(n,2i) y(m,2i)" + ); + assert_eq!( + t("x(0@,1i) = y(n,2i) % y(m,2i)"), + "0B14: x(0@,1i) y(n,2i) y(m,2i)" + ); + assert_eq!( + t("x(0@,1i) = y(n,2i) >> y(m,2i)"), + "0B15: x(0@,1i) y(n,2i) y(m,2i)" + ); + assert_eq!( + t("x(0@,1i) = y(n,2i) << y(m,2i)"), + "0B16: x(0@,1i) y(n,2i) y(m,2i)" + ); + assert_eq!( + t("x(0@,1i) = y(n,2i) + y(m,2i)"), + "0A8E: x(0@,1i) y(n,2i) y(m,2i)" + ); + assert_eq!( + t("x(0@,1i) = y(n,2i) - y(m,2i)"), + "0A8F: x(0@,1i) y(n,2i) y(m,2i)" + ); + assert_eq!( + t("x(0@,1i) = y(n,2i) * y(m,2i)"), + "0A90: x(0@,1i) y(n,2i) y(m,2i)" + ); + assert_eq!( + t("x(0@,1i) = y(n,2i) / y(m,2i)"), + "0A91: x(0@,1i) y(n,2i) y(m,2i)" + ); assert_ne!(t("0@ += 1 | 1@"), "0B11: 0@ 1 1@"); + + assert_eq!(t("|< = |> + |>"), "0A8E: |< |> |>"); + assert_eq!(t("|< = |> - |>"), "0A8F: |< |> |>"); + assert_eq!(t("|< = |> * |>"), "0A90: |< |> |>"); + assert_eq!(t("|< = |> / |>"), "0A91: |< |> |>"); } #[test] @@ -163,16 +228,19 @@ mod tests { table.load_from_file("SASCM.ini"); let mut ns = Namespaces::new(); ns.load_library("sa.json"); - let dict = DictNumByStr::default(); let mut const_lookup = DictStrByStr::default(); const_lookup.add(CString::new("x").unwrap(), CString::new("3@").unwrap()); let t = |input: &str| -> String { - transform(input, &ns, &table, &dict, &const_lookup).unwrap_or_default() + transform(input, &ns, &table, &const_lookup).unwrap_or_default() }; assert_eq!(t("0@ = ~1@"), "0B13: 0@ 1@"); assert_eq!(t("~0@"), "0B1A: 0@"); assert_eq!(t("~x"), "0B1A: 3@"); + assert_eq!(t("~x[0]"), "0B1A: x[0]"); + assert_eq!(t("~x[1@]"), "0B1A: x[1@]"); + assert_eq!(t("~1@[x]"), "0B1A: 1@[x]"); + assert_eq!(t("~x(1@,1i)"), "0B1A: x(1@,1i)"); } #[test] @@ -181,14 +249,13 @@ mod tests { table.load_from_file("SASCM.ini"); let mut ns = Namespaces::new(); ns.load_library("sa.json"); - let dict = DictNumByStr::default(); let mut const_lookup = DictStrByStr::default(); const_lookup.add(CString::new("x").unwrap(), CString::new("3@").unwrap()); const_lookup.add(CString::new("y").unwrap(), CString::new("4@").unwrap()); const_lookup.add(CString::new("n").unwrap(), CString::new("100").unwrap()); const_lookup.add(CString::new("f").unwrap(), CString::new("100.0").unwrap()); let t = |input: &str| -> String { - transform(input, &ns, &table, &dict, &const_lookup).unwrap_or_default() + transform(input, &ns, &table, &const_lookup).unwrap_or_default() }; // +=@ assert_eq!(t("$var +=@ 5.0"), "0078: $var 5.0"); @@ -201,6 +268,10 @@ mod tests { assert_eq!(t("0@ +=@ $var"), "007C: 0@ $var"); assert_eq!(t("$var +=@ 1@"), "007D: $var 1@"); assert_eq!(t("x +=@ y"), "007B: 3@ 4@"); + assert_eq!(t("x[0] +=@ y"), "007B: x[0] 4@"); + assert_eq!(t("x[1@] +=@ y"), "007B: x[1@] 4@"); + assert_eq!(t("x[n] +=@ y"), "007B: x[n] 4@"); + assert_eq!(t("1@[x] +=@ y"), "007B: 1@[x] 4@"); //-=@ assert_eq!(t("$var -=@ 5.0"), "007E: $var 5.0"); @@ -212,38 +283,50 @@ mod tests { assert_eq!(t("0@ -=@ $var"), "0082: 0@ $var"); assert_eq!(t("$var -=@ 1@"), "0083: $var 1@"); assert_eq!(t("x -=@ y"), "0081: 3@ 4@"); + assert_eq!(t("x[0] -=@ y"), "0081: x[0] 4@"); + assert_eq!(t("x[1@] -=@ y"), "0081: x[1@] 4@"); + assert_eq!(t("x[n] -=@ y"), "0081: x[n] 4@"); + assert_eq!(t("1@[x] -=@ y"), "0081: 1@[x] 4@"); } - #[test] - fn test_cast_assignment() { - use crate::utils::compiler_const::{TOKEN_FLOAT, TOKEN_INT}; + // #[test] + // fn test_cast_assignment() { + // use crate::utils::compiler_const::{TOKEN_FLOAT, TOKEN_INT}; - let mut table = OpcodeTable::new(Game::SA); - table.load_from_file("SASCM.ini"); - let mut ns = Namespaces::new(); - ns.load_library("sa.json"); - let mut dict = DictNumByStr::default(); - let mut const_lookup = DictStrByStr::default(); - const_lookup.add(CString::new("x").unwrap(), CString::new("3@").unwrap()); - const_lookup.add(CString::new("y").unwrap(), CString::new("4@").unwrap()); + // let mut table = OpcodeTable::new(Game::SA); + // table.load_from_file("SASCM.ini"); + // let mut ns = Namespaces::new(); + // ns.load_library("sa.json"); + // let mut dict = DictNumByStr::default(); + // let mut const_lookup = DictStrByStr::default(); + // const_lookup.add(CString::new("x").unwrap(), CString::new("3@").unwrap()); + // const_lookup.add(CString::new("y").unwrap(), CString::new("4@").unwrap()); - dict.add("$i".to_string(), TOKEN_INT); - dict.add("0@".to_string(), TOKEN_INT); - dict.add("3@".to_string(), TOKEN_INT); - dict.add("$f".to_string(), TOKEN_FLOAT); - dict.add("1@".to_string(), TOKEN_FLOAT); - dict.add("4@".to_string(), TOKEN_FLOAT); - let t = |input: &str| -> String { - transform(input, &ns, &table, &dict, &const_lookup).unwrap_or_default() - }; - assert_eq!(t("$i =# $f"), "008C: $i $f"); - assert_eq!(t("$f =# $i"), "008D: $f $i"); - assert_eq!(t("0@ =# $f"), "008E: 0@ $f"); - assert_eq!(t("1@ =# $i"), "008F: 1@ $i"); - assert_eq!(t("$i =# 1@"), "0090: $i 1@"); - assert_eq!(t("$f =# 0@"), "0091: $f 0@"); - assert_eq!(t("0@ =# 1@"), "0092: 0@ 1@"); - assert_eq!(t("1@ =# 0@"), "0093: 1@ 0@"); - assert_eq!(t("x =# y"), "0092: 3@ 4@"); - } + // dict.add(CString::new("$i").unwrap(), TOKEN_INT); + // dict.add(CString::new("0@").unwrap(), TOKEN_INT); + // dict.add(CString::new("3@").unwrap(), TOKEN_INT); + // dict.add(CString::new("$f").unwrap(), TOKEN_FLOAT); + // dict.add(CString::new("1@").unwrap(), TOKEN_FLOAT); + // dict.add(CString::new("4@").unwrap(), TOKEN_FLOAT); + // let t = |input: &str| -> String { + // transform(input, &ns, &table, &dict, &const_lookup).unwrap_or_default() + // }; + // assert_eq!(t("$i =# $f"), "008C: $i $f"); + // assert_eq!(t("$f =# $i"), "008D: $f $i"); + // assert_eq!(t("0@ =# $f"), "008E: 0@ $f"); + // assert_eq!(t("1@ =# $i"), "008F: 1@ $i"); + // assert_eq!(t("$i =# 1@"), "0090: $i 1@"); + // assert_eq!(t("$f =# 0@"), "0091: $f 0@"); + // assert_eq!(t("0@ =# 1@"), "0092: 0@ 1@"); + // assert_eq!(t("1@ =# 0@"), "0093: 1@ 0@"); + + // assert_eq!(t("$i(0@,1i) =# $f(1@,1f)"), "008C: $i(0@,1i) $f(1@,1f)"); + // assert_eq!(t("$i[0] =# $f[0]"), "008C: $i[0] $f[0]"); + // todo: + // assert_eq!(t("$m(0@,1i) =# $n(0@,1f)"), "008C: $m(0@,1i) $n(0@,1f)"); // should not require decl + // assert_eq!(t("x =# y"), "0092: 3@ 4@"); // should resolve const name + // assert_eq!(t("x[0] =# y"), "0092: x[0] 4@"); + // assert_eq!(t("x[1@] =# y"), "0092: x[1@] 4@"); + // assert_eq!(t("x[n] =# y"), "0092: x[n] 4@"); + // } } diff --git a/src/v4/transform.rs b/src/v4/transform.rs index 24886fb..0e871c3 100644 --- a/src/v4/transform.rs +++ b/src/v4/transform.rs @@ -1,6 +1,6 @@ use super::helpers::*; use crate::{ - dictionary::{dictionary_num_by_str::DictNumByStr, dictionary_str_by_str::DictStrByStr}, + dictionary::dictionary_str_by_str::DictStrByStr, legacy_ini::OpcodeTable, namespaces::namespaces::Namespaces, parser::{ @@ -50,22 +50,21 @@ static OP_SUB_TIMED_FLOAT_LVAR_FROM_FLOAT_LVAR: &'static str = static OP_SUB_TIMED_FLOAT_VAR_FROM_FLOAT_LVAR: &'static str = "SUB_TIMED_FLOAT_VAR_FROM_FLOAT_LVAR"; static OP_SUB_TIMED_FLOAT_LVAR_FROM_FLOAT_VAR: &'static str = "SUB_TIMED_FLOAT_LVAR_FROM_FLOAT_VAR"; -static OP_CSET_VAR_INT_TO_VAR_FLOAT: &'static str = "CSET_VAR_INT_TO_VAR_FLOAT"; -static OP_CSET_VAR_FLOAT_TO_VAR_INT: &'static str = "CSET_VAR_FLOAT_TO_VAR_INT"; -static OP_CSET_LVAR_INT_TO_VAR_FLOAT: &'static str = "CSET_LVAR_INT_TO_VAR_FLOAT"; -static OP_CSET_LVAR_FLOAT_TO_VAR_INT: &'static str = "CSET_LVAR_FLOAT_TO_VAR_INT"; +// static OP_CSET_VAR_INT_TO_VAR_FLOAT: &'static str = "CSET_VAR_INT_TO_VAR_FLOAT"; +// static OP_CSET_VAR_FLOAT_TO_VAR_INT: &'static str = "CSET_VAR_FLOAT_TO_VAR_INT"; +// static OP_CSET_LVAR_INT_TO_VAR_FLOAT: &'static str = "CSET_LVAR_INT_TO_VAR_FLOAT"; +// static OP_CSET_LVAR_FLOAT_TO_VAR_INT: &'static str = "CSET_LVAR_FLOAT_TO_VAR_INT"; -static OP_CSET_VAR_INT_TO_LVAR_FLOAT: &'static str = "CSET_VAR_INT_TO_LVAR_FLOAT"; -static OP_CSET_VAR_FLOAT_TO_LVAR_INT: &'static str = "CSET_VAR_FLOAT_TO_LVAR_INT"; -static OP_CSET_LVAR_INT_TO_LVAR_FLOAT: &'static str = "CSET_LVAR_INT_TO_LVAR_FLOAT"; -static OP_CSET_LVAR_FLOAT_TO_LVAR_INT: &'static str = "CSET_LVAR_FLOAT_TO_LVAR_INT"; +// static OP_CSET_VAR_INT_TO_LVAR_FLOAT: &'static str = "CSET_VAR_INT_TO_LVAR_FLOAT"; +// static OP_CSET_VAR_FLOAT_TO_LVAR_INT: &'static str = "CSET_VAR_FLOAT_TO_LVAR_INT"; +// static OP_CSET_LVAR_INT_TO_LVAR_FLOAT: &'static str = "CSET_LVAR_INT_TO_LVAR_FLOAT"; +// static OP_CSET_LVAR_FLOAT_TO_LVAR_INT: &'static str = "CSET_LVAR_FLOAT_TO_LVAR_INT"; pub fn try_tranform( ast: &AST, expr: &str, ns: &Namespaces, legacy_ini: &OpcodeTable, - var_types: &DictNumByStr, const_lookup: &DictStrByStr, ) -> Option { let e = ast.body.get(0)?; @@ -73,7 +72,9 @@ pub fn try_tranform( macro_rules! resolve { ($node: expr) => {{ let x = |token| -> Option<(Node, String)> { - let name = token_str(expr, token); + // keys in DictStrByStr are lower-case + let name = token_str(expr, token).to_ascii_lowercase(); + let const_value = const_lookup .map .get(&CString::new(name).unwrap()) @@ -145,8 +146,8 @@ pub fn try_tranform( return None; } let (var, var_name) = resolve!(left); - let (operand, operand_name) = resolve!(&unary.operand); - if !is_variable(&var) || !is_variable(&operand) { + let (_, operand_name) = resolve!(&unary.operand); + if !is_variable(&var) { return None; } // var = ~var @@ -306,98 +307,89 @@ pub fn try_tranform( } } } - SyntaxKind::OperatorCastEqual => { - // requires type info - if !is_variable(&right_operand) { - return None; - } - let t1 = *var_types.map.get(&var_name)?; - let t2 = *var_types.map.get(&right_operand_name)?; + // SyntaxKind::OperatorCastEqual => { + // // requires type info + // if !is_variable(&right_operand) { + // return None; + // } + // let left_var = as_variable(&var)?; + // let right_var = as_variable(&right_operand)?; - use crate::utils::compiler_const::*; - let left_var = as_variable(&var)?; - match right_token.syntax_kind { - SyntaxKind::GlobalVariable - if left_var.is_global() - && t1 == TOKEN_INT - && t2 == TOKEN_FLOAT => - { - // var =# var // int = float - return op(OP_CSET_VAR_INT_TO_VAR_FLOAT); - } - SyntaxKind::GlobalVariable - if left_var.is_global() - && t1 == TOKEN_FLOAT - && t2 == TOKEN_INT => - { - op(OP_CSET_VAR_FLOAT_TO_VAR_INT) - } - SyntaxKind::LocalVariable - if left_var.is_local() - && t1 == TOKEN_INT - && t2 == TOKEN_FLOAT => - { - // lvar =# lvar // int = float - return op(OP_CSET_LVAR_INT_TO_LVAR_FLOAT) - .or(op(OP_CSET_VAR_INT_TO_VAR_FLOAT)); // vcs - } - SyntaxKind::LocalVariable - if left_var.is_local() - && t1 == TOKEN_FLOAT - && t2 == TOKEN_INT => - { - // lvar =# lvar // float = int - return op(OP_CSET_LVAR_FLOAT_TO_LVAR_INT) - .or(op(OP_CSET_VAR_FLOAT_TO_VAR_INT)); // vcs - } - SyntaxKind::GlobalVariable - if left_var.is_local() - && t1 == TOKEN_INT - && t2 == TOKEN_FLOAT => - { - // lvar =# var // int = float - return op(OP_CSET_LVAR_INT_TO_VAR_FLOAT) - .or(op(OP_CSET_VAR_INT_TO_VAR_FLOAT)); // vcs - } - SyntaxKind::GlobalVariable - if left_var.is_local() - && t1 == TOKEN_FLOAT - && t2 == TOKEN_INT => - { - // lvar =# var // float = int - return op(OP_CSET_LVAR_FLOAT_TO_VAR_INT) - .or(op(OP_CSET_VAR_FLOAT_TO_VAR_INT)); // vcs - } - SyntaxKind::LocalVariable - if left_var.is_global() - && t1 == TOKEN_INT - && t2 == TOKEN_FLOAT => - { - // var =# lvar // int = float - return op(OP_CSET_VAR_INT_TO_LVAR_FLOAT) - .or(op(OP_CSET_VAR_INT_TO_VAR_FLOAT)); // vcs - } - SyntaxKind::LocalVariable - if left_var.is_global() - && t1 == TOKEN_FLOAT - && t2 == TOKEN_INT => - { - // var =# lvar // float = int - return op(OP_CSET_VAR_FLOAT_TO_LVAR_INT) - .or(op(OP_CSET_VAR_FLOAT_TO_VAR_INT)); // vcs - } + // let l1 = + // &CString::new(token_str(expr, left_var.get_var_name())).unwrap(); + // let l2 = + // &CString::new(token_str(expr, right_var.get_var_name())).unwrap(); + // let t1 = *var_types.map.get(l1)?; + // let t2 = *var_types.map.get(l2)?; - _ => None, - } - } + // use crate::utils::compiler_const::*; + // if right_var.is_global() { + // if left_var.is_global() { + // if t1 == TOKEN_INT && t2 == TOKEN_FLOAT { + // // var =# var // int = float + // return op(OP_CSET_VAR_INT_TO_VAR_FLOAT); + // }; + // if t1 == TOKEN_FLOAT && t2 == TOKEN_INT { + // // var =# var // float = int + // return op(OP_CSET_VAR_FLOAT_TO_VAR_INT); + // }; + // } + + // if left_var.is_local() { + // if t1 == TOKEN_INT && t2 == TOKEN_FLOAT { + // // lvar =# var // int = float + // return op(OP_CSET_LVAR_INT_TO_VAR_FLOAT) + // .or(op(OP_CSET_VAR_INT_TO_VAR_FLOAT)); + // } + // if t1 == TOKEN_FLOAT && t2 == TOKEN_INT { + // // lvar =# var // float = int + // return op(OP_CSET_LVAR_FLOAT_TO_VAR_INT) + // .or(op(OP_CSET_VAR_FLOAT_TO_VAR_INT)); + // } + // } + // } + + // if right_var.is_local() { + // if left_var.is_local() { + // if t1 == TOKEN_INT && t2 == TOKEN_FLOAT { + // // lvar =# lvar // int = float + // return op(OP_CSET_LVAR_INT_TO_LVAR_FLOAT) + // .or(op(OP_CSET_VAR_INT_TO_VAR_FLOAT)); + // } + + // if t1 == TOKEN_FLOAT && t2 == TOKEN_INT { + // // lvar =# lvar // float = int + // return op(OP_CSET_LVAR_FLOAT_TO_LVAR_INT) + // .or(op(OP_CSET_VAR_FLOAT_TO_VAR_INT)); + // } + // } + + // if left_var.is_global() { + // if t1 == TOKEN_INT && t2 == TOKEN_FLOAT { + // // var =# lvar // int = float + // return op(OP_CSET_VAR_INT_TO_LVAR_FLOAT) + // .or(op(OP_CSET_VAR_INT_TO_VAR_FLOAT)); + // } + + // if t1 == TOKEN_FLOAT && t2 == TOKEN_INT { + // // var =# lvar // float = int + // return op(OP_CSET_VAR_FLOAT_TO_LVAR_INT) + // .or(op(OP_CSET_VAR_FLOAT_TO_VAR_INT)); + // } + // } + // } + + // return None; + // } SyntaxKind::OperatorEqual => { let left_var = as_variable(&var)?; let right_number = as_number(&right_operand)?; match right_number.syntax_kind { - // var = int - SyntaxKind::IntegerLiteral if left_var.is_global() => { + SyntaxKind::IntegerLiteral | SyntaxKind::LabelLiteral + if left_var.is_global() => + { op(OP_SET_VAR_INT) } // var = float @@ -405,7 +397,9 @@ pub fn try_tranform( op(OP_SET_VAR_FLOAT) } // lvar = int - SyntaxKind::IntegerLiteral if left_var.is_local() => { + SyntaxKind::IntegerLiteral | SyntaxKind::LabelLiteral + if left_var.is_local() => + { op(OP_SET_LVAR_INT).or(op(OP_SET_VAR_INT)) } // lvar = float