From 6d407eb5b3aa5f7ef4ec6d2d44e53e34e89537f0 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Thu, 19 Oct 2023 09:24:21 +1100 Subject: [PATCH 001/129] Reduced verbosity on a warning --- Cargo.lock | 280 ++++++++++++++++++------------------- lib/wasix/src/state/env.rs | 18 ++- 2 files changed, 150 insertions(+), 148 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 99d8d138184..58ae60f2b27 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -47,9 +47,9 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "1.1.1" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea5d730647d4fadd988536d06fecce94b7b4f2a7efdae548f1cf4b63205518ab" +checksum = "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0" dependencies = [ "memchr", ] @@ -152,9 +152,9 @@ dependencies = [ [[package]] name = "arbitrary" -version = "1.3.0" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2d098ff73c1ca148721f37baad5ea6a465a13f9573aba8641fbbbae8164a54e" +checksum = "a2e1373abdaa212b704512ec2bd8b26bd0b7d5c3f70117411a5d9a451383c859" dependencies = [ "derive_arbitrary", ] @@ -204,7 +204,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "88903cb14723e4d4003335bb7f8a14f27691649105346a0f0957466c096adfe6" dependencies = [ "anstyle", - "bstr 1.6.2", + "bstr 1.7.0", "doc-comment", "predicates 3.0.4", "predicates-core", @@ -214,9 +214,9 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.4.3" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb42b2197bf15ccb092b62c74515dbd8b86d0effd934795f6687c93b6e679a2c" +checksum = "f658e2baef915ba0f26f1f7c42bfb8e12f532a01f449a090ded75ae7a07e9ba2" dependencies = [ "flate2", "futures-core", @@ -227,13 +227,13 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.73" +version = "0.1.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0" +checksum = "a66537f1bb974b254c98ed142ff995236e81b9d0fe4db0575f46612cb15eb0f9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.38", ] [[package]] @@ -298,9 +298,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.4.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635" +checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" [[package]] name = "bitvec" @@ -349,12 +349,12 @@ dependencies = [ [[package]] name = "bstr" -version = "1.6.2" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c2f7349907b712260e64b0afe2f84692af14a454be26187d9df565c7f69266a" +checksum = "c79ad7fb2dd38f3dabd76b09c6a5a20c038fc0213ef1e9afd30eb777f120f019" dependencies = [ "memchr", - "regex-automata 0.3.9", + "regex-automata 0.4.2", "serde", ] @@ -403,9 +403,9 @@ dependencies = [ [[package]] name = "byteorder" -version = "1.4.3" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" @@ -436,9 +436,9 @@ dependencies = [ [[package]] name = "cargo-platform" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cfa25e60aea747ec7e1124f238816749faa93759c6ff5b31f1ccdda137f4479" +checksum = "12024c4645c97566567129c204f65d5815a8c9aecf30fcbe682b2fe034996d36" dependencies = [ "serde", ] @@ -451,7 +451,7 @@ checksum = "eee4243f1f26fc7a42710e7439c149e2b10b05472f88090acce52632f231a73a" dependencies = [ "camino", "cargo-platform", - "semver 1.0.19", + "semver 1.0.20", "serde", "serde_json", "thiserror", @@ -589,7 +589,7 @@ dependencies = [ "heck 0.4.1", "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.38", ] [[package]] @@ -1112,7 +1112,7 @@ dependencies = [ "ident_case", "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.38", ] [[package]] @@ -1134,7 +1134,7 @@ checksum = "836a9bbc7ad63342d6d6e7b815ccab164bc77a2d95d84bc3117a8c0d5c98e2d5" dependencies = [ "darling_core 0.20.3", "quote", - "syn 2.0.37", + "syn 2.0.38", ] [[package]] @@ -1158,10 +1158,11 @@ checksum = "c2e66c9d817f1720209181c316d28635c050fa304f9c79e47a520882661b7308" [[package]] name = "deranged" -version = "0.3.8" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2696e8a945f658fd14dc3b87242e6b80cd0f36ff04ea560fa39082368847946" +checksum = "0f32d04922c60427da6f9fef14d042d9edddef64cb9d4ce0d64d0685fbeb1fd3" dependencies = [ + "powerfmt", "serde", ] @@ -1184,7 +1185,7 @@ checksum = "53e0efad4403bfc52dc201159c4b842a246a14b98c64b55dfd0f2d89729dfeb8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.38", ] [[package]] @@ -1422,14 +1423,14 @@ dependencies = [ "num-traits", "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.38", ] [[package]] name = "enumset" -version = "1.1.2" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e875f1719c16de097dee81ed675e2d9bb63096823ed3f0ca827b7dea3028bbbb" +checksum = "226c0da7462c13fb57e5cc9e0dc8f0635e7d27f276a3a7fd30054647f669007d" dependencies = [ "enumset_derive", ] @@ -1443,7 +1444,7 @@ dependencies = [ "darling 0.20.3", "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.38", ] [[package]] @@ -1463,25 +1464,14 @@ dependencies = [ [[package]] name = "errno" -version = "0.3.4" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "add4f07d43996f76ef320709726a556a9d4f965d9410d8d0271132d2f8293480" +checksum = "ac3e13f66a2f95e32a39eaa81f6b95d42878ca0e1db0c7543723dfe12557e860" dependencies = [ - "errno-dragonfly", "libc", "windows-sys 0.48.0", ] -[[package]] -name = "errno-dragonfly" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" -dependencies = [ - "cc", - "libc", -] - [[package]] name = "fallible-iterator" version = "0.2.0" @@ -1540,9 +1530,9 @@ checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" [[package]] name = "flate2" -version = "1.0.27" +version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6c98ee8095e9d1dcbf2fcc6d95acccb90d1c81db1e44725c6a984b1dbdfb010" +checksum = "46303f565772937ffe1d394a4fac6f411c6013172fadde9dcdb1e147a086940e" dependencies = [ "crc32fast", "miniz_oxide", @@ -1655,7 +1645,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.38", ] [[package]] @@ -1737,7 +1727,7 @@ checksum = "ba330b70a5341d3bc730b8e205aaee97ddab5d9c448c4f51a7c2d924266fa8f9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.38", ] [[package]] @@ -2236,9 +2226,9 @@ dependencies = [ [[package]] name = "insta" -version = "1.33.0" +version = "1.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1aa511b2e298cd49b1856746f6bb73e17036bcd66b25f5e92cdcdbec9bd75686" +checksum = "5d64600be34b2fcfc267740a243fa7744441bb4947a619ac4e5bb6507f35fbfc" dependencies = [ "console", "lazy_static", @@ -2313,9 +2303,9 @@ checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" [[package]] name = "jobserver" -version = "0.1.26" +version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "936cfd212a0155903bcbc060e316fb6cc7cbf2e1907329391ebadc1fe0ce77c2" +checksum = "8c37f63953c4c63420ed5fd3d6d398c719489b9f872b9fa683262f8edd363c7d" dependencies = [ "libc", ] @@ -2365,9 +2355,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.148" +version = "0.2.149" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cdc71e17332e86d2e1d38c1f99edcb6288ee11b815fb1a4b049eaa2114d369b" +checksum = "a08173bc88b7955d1b3145aa561539096c421ac8debde8cbc3612ec635fee29b" [[package]] name = "libfuzzer-sys" @@ -2418,9 +2408,9 @@ dependencies = [ [[package]] name = "linux-raw-sys" -version = "0.4.8" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3852614a3bd9ca9804678ba6be5e3b8ce76dfc902cae004e3e0c44051b6e88db" +checksum = "da2479e8c062e40bf0066ffa0bc823de0a9368974af99c9f6df941d2c231e03f" [[package]] name = "llvm-sys" @@ -2432,7 +2422,7 @@ dependencies = [ "lazy_static", "libc", "regex", - "semver 1.0.19", + "semver 1.0.20", ] [[package]] @@ -2766,9 +2756,9 @@ dependencies = [ [[package]] name = "num-traits" -version = "0.2.16" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f30b0abd723be7e2ffca1272140fac1a2f084c77ec3e123c192b66af1ee9e6c2" +checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c" dependencies = [ "autocfg", ] @@ -2859,7 +2849,7 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c62dcb6174f9cb326eac248f07e955d5d559c272730b6c03e396b443b562788" dependencies = [ - "bstr 1.6.2", + "bstr 1.7.0", "normpath", "winapi", ] @@ -2870,7 +2860,7 @@ version = "0.10.57" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bac25ee399abb46215765b1cb35bc0212377e58a061560d8b29b024fd0430e7c" dependencies = [ - "bitflags 2.4.0", + "bitflags 2.4.1", "cfg-if", "foreign-types", "libc", @@ -2887,7 +2877,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.38", ] [[package]] @@ -3069,7 +3059,7 @@ checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.38", ] [[package]] @@ -3096,6 +3086,12 @@ version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "31114a898e107c51bb1609ffaf55a0e011cf6a4d7f1170d0015a165082c0338b" +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "ppv-lite86" version = "0.2.17" @@ -3213,9 +3209,9 @@ checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" [[package]] name = "proc-macro2" -version = "1.0.67" +version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d433d9f1a3e8c1263d9456598b16fec66f4acc9a74dacffd35c7bb09b3a1328" +checksum = "134c189feb4956b20f6f547d2cf727d4c0fe06722b20a0eec87ed445a97f92da" dependencies = [ "unicode-ident", ] @@ -3395,7 +3391,7 @@ checksum = "7f7473c2cfcf90008193dd0e3e16599455cb601a9fce322b5bb55de799664925" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.38", ] [[package]] @@ -3418,14 +3414,14 @@ dependencies = [ [[package]] name = "regex" -version = "1.9.6" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebee201405406dbf528b8b672104ae6d6d63e6d118cb10e4d51abbc7b58044ff" +checksum = "aaac441002f822bc9705a681810a4dd2963094b9ca0ddc41cb963a4c189189ea" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.3.9", - "regex-syntax 0.7.5", + "regex-automata 0.4.2", + "regex-syntax 0.8.2", ] [[package]] @@ -3439,13 +3435,13 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.3.9" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59b23e92ee4318893fa3fe3e6fb365258efbfe6ac6ab30f090cdcbb7aa37efa9" +checksum = "5011c7e263a695dc8ca064cddb722af1be54e517a280b12a5356f98366899e5d" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.7.5", + "regex-syntax 0.8.2", ] [[package]] @@ -3456,9 +3452,9 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "regex-syntax" -version = "0.7.5" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" +checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" [[package]] name = "region" @@ -3656,7 +3652,7 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" dependencies = [ - "semver 1.0.19", + "semver 1.0.20", ] [[package]] @@ -3673,11 +3669,11 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.15" +version = "0.38.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2f9da0cbd88f9f09e7814e388301c8414c51c62aa6ce1e4b5c551d49d96e531" +checksum = "745ecfa778e66b2b63c88a61cb36e0eea109e803b0b86bf9879fbc77c70e86ed" dependencies = [ - "bitflags 2.4.0", + "bitflags 2.4.1", "errno", "libc", "linux-raw-sys", @@ -3912,9 +3908,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.19" +version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad977052201c6de01a8ef2aa3378c4bd23217a056337d1d6da40468d267a4fb0" +checksum = "836fa6a3e1e547f9a2c4040802ec865b5d85f4014efe00555d7090a3dcaa1090" dependencies = [ "serde", ] @@ -3930,9 +3926,9 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.188" +version = "1.0.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf9e0fcba69a370eed61bcf2b728575f726b50b55cba78064753d708ddc7549e" +checksum = "8e422a44e74ad4001bdc8eede9a4570ab52f71190e9c076d14369f38b9200537" dependencies = [ "serde_derive", ] @@ -3969,13 +3965,13 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.188" +version = "1.0.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" +checksum = "1e48d1f918009ce3145511378cf68d613e3b3d9137d67272562080d68a2b32d5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.38", ] [[package]] @@ -4102,9 +4098,9 @@ dependencies = [ [[package]] name = "sharded-slab" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1b21f559e07218024e7e9f90f96f601825397de0e25420135f7f952453fed0b" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" dependencies = [ "lazy_static", ] @@ -4163,9 +4159,9 @@ checksum = "f27f6278552951f1f2b8cf9da965d10969b2efdea95a6ec47987ab46edfe263a" [[package]] name = "similar" -version = "2.2.1" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "420acb44afdae038210c99e69aae24109f32f15500aa708e81d46c9f29d55fcf" +checksum = "2aeaf503862c419d66959f5d7ca015337d864e9c49485d771b732e2a20453597" [[package]] name = "slab" @@ -4303,9 +4299,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.37" +version = "2.0.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7303ef2c05cd654186cb250d29049a24840ca25d2747c25c0381c8d9e2f582e8" +checksum = "e96b79aaa137db8f61e26363a0c9b47d8b4ec75da28b7d1d614c2303e232408b" dependencies = [ "proc-macro2", "quote", @@ -4430,13 +4426,13 @@ dependencies = [ [[package]] name = "test-log" -version = "0.2.12" +version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9601d162c1d77e62c1ea0bc8116cd1caf143ce3af947536c3c9052a1677fe0c" +checksum = "f66edd6b6cd810743c0c71e1d085e92b01ce6a72782032e3f794c8284fe4bcdd" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.38", ] [[package]] @@ -4469,7 +4465,7 @@ checksum = "10712f02019e9288794769fba95cd6847df9874d49d871d062172f9dd41bc4cc" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.38", ] [[package]] @@ -4484,12 +4480,13 @@ dependencies = [ [[package]] name = "time" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "426f806f4089c493dcac0d24c29c01e2c38baf8e30f1b716ee37e83d200b18fe" +checksum = "c4a34ab300f2dee6e562c10a046fc05e358b29f9bf92277f30c3c8d82275f6f5" dependencies = [ "deranged", "itoa", + "powerfmt", "serde", "time-core", "time-macros", @@ -4551,9 +4548,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.32.0" +version = "1.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17ed6077ed6cd6c74735e21f37eb16dc3935f96878b1fe961074089cc80893f9" +checksum = "4f38200e3ef7995e5ef13baec2f432a6da0aa9ac495b2c0e8f3b7eec2c92d653" dependencies = [ "backtrace", "bytes", @@ -4574,7 +4571,7 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.38", ] [[package]] @@ -4745,7 +4742,7 @@ version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61c5bb1d698276a2443e5ecfabc1008bf15a36c12e6a7176e7bf089ea9131140" dependencies = [ - "bitflags 2.4.0", + "bitflags 2.4.1", "bytes", "futures-core", "futures-util", @@ -4773,11 +4770,10 @@ checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" [[package]] name = "tracing" -version = "0.1.37" +version = "0.1.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" +checksum = "ee2ef2af84856a50c1d430afce2fdded0a4ec7eda868db86409b4543df0797f9" dependencies = [ - "cfg-if", "log", "pin-project-lite", "tracing-attributes", @@ -4786,20 +4782,20 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.26" +version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" +checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.38", ] [[package]] name = "tracing-core" -version = "0.1.31" +version = "0.1.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0955b8137a1df6f1a2e9a37d8a6656291ff0297c1a97c24e0d8425fe2312f79a" +checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" dependencies = [ "once_cell", "valuable", @@ -5391,7 +5387,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.38", "wasm-bindgen-shared", ] @@ -5448,7 +5444,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.38", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -5530,9 +5526,9 @@ dependencies = [ [[package]] name = "wasm-encoder" -version = "0.32.0" +version = "0.35.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ba64e81215916eaeb48fee292f29401d69235d62d8b8fd92a7b2844ec5ae5f7" +checksum = "9ca90ba1b5b0a70d3d49473c5579951f3bddc78d47b59256d2f9d4922b150aca" dependencies = [ "leb128", ] @@ -5735,7 +5731,7 @@ dependencies = [ "prettytable-rs", "regex", "reqwest", - "semver 1.0.19", + "semver 1.0.20", "serde", "serde_json", "sha2", @@ -5857,7 +5853,7 @@ dependencies = [ "rayon", "regex", "rustc_version 0.4.0", - "semver 1.0.19", + "semver 1.0.20", "smallvec", "target-lexicon 0.12.11", "wasmer-compiler", @@ -5902,7 +5898,7 @@ dependencies = [ "once_cell", "regex", "reqwest", - "semver 1.0.19", + "semver 1.0.20", "serde", "serde_json", "serde_yaml 0.8.26", @@ -5919,7 +5915,7 @@ dependencies = [ "wasmer-deploy-schema", "wasmer-deploy-util", "wasmer-registry 5.8.0 (registry+https://github.com/rust-lang/crates.io-index)", - "wasmer-toml 0.9.0", + "wasmer-toml 0.9.2", "webc", ] @@ -6087,7 +6083,7 @@ dependencies = [ "reqwest", "rpassword", "rusqlite", - "semver 1.0.19", + "semver 1.0.20", "serde", "serde_json", "tar", @@ -6128,7 +6124,7 @@ dependencies = [ "reqwest", "rpassword", "rusqlite", - "semver 1.0.19", + "semver 1.0.20", "serde", "serde_json", "tar", @@ -6168,7 +6164,7 @@ dependencies = [ "anyhow", "derive_builder", "indexmap 1.9.3", - "semver 1.0.19", + "semver 1.0.20", "serde", "serde_cbor", "serde_json", @@ -6179,14 +6175,14 @@ dependencies = [ [[package]] name = "wasmer-toml" -version = "0.9.0" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64a1f578659a6f88c6943f7d6575e1be6b1b58b1dcdcfb4c9b8412f22328249a" +checksum = "d21472954ee9443235ca32522b17fc8f0fe58e2174556266a0d9766db055cc52" dependencies = [ "anyhow", "derive_builder", "indexmap 2.0.2", - "semver 1.0.19", + "semver 1.0.20", "serde", "serde_cbor", "serde_json", @@ -6270,7 +6266,7 @@ dependencies = [ "rand", "rayon", "reqwest", - "semver 1.0.19", + "semver 1.0.20", "serde", "serde_cbor", "serde_derive", @@ -6446,22 +6442,22 @@ dependencies = [ [[package]] name = "wasmparser" -version = "0.113.3" +version = "0.115.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "286049849b5a5bd09a8773171be96824afabffc7cc3df6caaf33a38db6cd07ae" +checksum = "e06c0641a4add879ba71ccb3a1e4278fd546f76f1eafb21d8f7b07733b547cd5" dependencies = [ "indexmap 2.0.2", - "semver 1.0.19", + "semver 1.0.20", ] [[package]] name = "wasmprinter" -version = "0.2.68" +version = "0.2.70" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "537030718ce76e985896e91fe2cac77c1913c8dccd46eaf8ab1a4cd56d824cc3" +checksum = "e74458a9bc5cc9c7108abfa0fe4dc88d5abf1f3baf194df3264985f17d559b5e" dependencies = [ "anyhow", - "wasmparser 0.113.3", + "wasmparser 0.115.0", ] [[package]] @@ -6484,23 +6480,23 @@ dependencies = [ [[package]] name = "wast" -version = "64.0.0" +version = "66.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a259b226fd6910225aa7baeba82f9d9933b6d00f2ce1b49b80fa4214328237cc" +checksum = "93cb43b0ac6dd156f2c375735ccfd72b012a7c0a6e6d09503499b8d3cb6e6072" dependencies = [ "leb128", "memchr", "unicode-width", - "wasm-encoder 0.32.0", + "wasm-encoder 0.35.0", ] [[package]] name = "wat" -version = "1.0.71" +version = "1.0.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53253d920ab413fca1c7dc2161d601c79b4fdf631d0ba51dd4343bf9b556c3f6" +checksum = "e367582095d2903caeeea9acbb140e1db9c7677001efa4347c3687fd34fe7072" dependencies = [ - "wast 64.0.0", + "wast 66.0.2", ] [[package]] @@ -6615,9 +6611,9 @@ dependencies = [ [[package]] name = "webc" -version = "5.5.1" +version = "5.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b56acc943f6b80cc2842231f34f99a02cd406896a23f3c6dacd8130c24ab3d1" +checksum = "d56e44a162b95647aef18b6b37b870836a0ada3e67124ef60022e0445e2734f5" dependencies = [ "anyhow", "base64", @@ -6630,7 +6626,7 @@ dependencies = [ "once_cell", "path-clean", "rand", - "semver 1.0.19", + "semver 1.0.20", "serde", "serde_cbor", "serde_json", @@ -6642,7 +6638,7 @@ dependencies = [ "toml 0.7.8", "url", "walkdir", - "wasmer-toml 0.8.1", + "wasmer-toml 0.9.2", ] [[package]] @@ -6884,9 +6880,9 @@ checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" [[package]] name = "winnow" -version = "0.5.15" +version = "0.5.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c2e3184b9c4e92ad5167ca73039d0c42476302ab603e2fec4487511f38ccefc" +checksum = "a3b801d0e0a6726477cc207f60162da452f3a95adb368399bef20a946e06f65c" dependencies = [ "memchr", ] diff --git a/lib/wasix/src/state/env.rs b/lib/wasix/src/state/env.rs index 1b6d3eed282..ea9dcb2b2df 100644 --- a/lib/wasix/src/state/env.rs +++ b/lib/wasix/src/state/env.rs @@ -668,18 +668,24 @@ impl WasiEnv { match err.downcast::() { Ok(wasi_err) => { warn!( - "wasi[{}]::signal handler wasi error - {}", + "wasi[{}]::signal handler wasi error (sig={:?}) - {}", ctx.data().pid(), + signal, wasi_err ); return Err(wasi_err); } Err(runtime_err) => { - warn!( - "wasi[{}]::signal handler runtime error - {}", - ctx.data().pid(), - runtime_err - ); + // anything other than a kill command should report + // the error, killed things may not gracefully close properly + if signal != Signal::Sigkill { + warn!( + "wasi[{}]::signal handler runtime error (sig={:?}) - {}", + ctx.data().pid(), + signal, + runtime_err + ); + } return Err(WasiError::Exit(Errno::Intr.into())); } } From 523add3856e38fa60e9f611edd47a5625d9af960 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Fri, 20 Oct 2023 11:45:26 +1100 Subject: [PATCH 002/129] Added the draft API and CLI changes needed for the snapshot functionality --- lib/cli/src/commands/run/wasi.rs | 48 +++++ lib/wasix/src/lib.rs | 1 + lib/wasix/src/os/task/thread.rs | 3 +- lib/wasix/src/runtime/mod.rs | 9 +- lib/wasix/src/snapshot/log_file.rs | 243 ++++++++++++++++++++++++++ lib/wasix/src/snapshot/mod.rs | 7 + lib/wasix/src/snapshot/shooter.rs | 81 +++++++++ lib/wasix/src/snapshot/unsupported.rs | 18 ++ 8 files changed, 408 insertions(+), 2 deletions(-) create mode 100644 lib/wasix/src/snapshot/log_file.rs create mode 100644 lib/wasix/src/snapshot/mod.rs create mode 100644 lib/wasix/src/snapshot/shooter.rs create mode 100644 lib/wasix/src/snapshot/unsupported.rs diff --git a/lib/cli/src/commands/run/wasi.rs b/lib/cli/src/commands/run/wasi.rs index ada07096471..92f96b4b456 100644 --- a/lib/cli/src/commands/run/wasi.rs +++ b/lib/cli/src/commands/run/wasi.rs @@ -1,6 +1,7 @@ use std::{ collections::{BTreeSet, HashMap}, path::{Path, PathBuf}, + str::FromStr, sync::{mpsc::Sender, Arc}, time::Duration, }; @@ -105,6 +106,21 @@ pub struct Wasi { #[clap(long = "enable-async-threads")] pub enable_async_threads: bool, + /// Specifies the snapshot file that Wasmer will use to store + /// the state of the WASM process so that it can be later restored + #[clap(long = "snapshot-to")] + pub snapshot_to: Option, + + /// Indicates what events will cause a snapshot to be taken + /// and written to the snapshot file. + #[clap(long = "snapshot-on")] + pub snapshot_on: Vec, + + /// When specified, the runtime will restore a previous snapshot + /// using the supplied file. + #[clap(long = "resume-from")] + pub resume_from: Option, + /// Allow instances to send http requests. /// /// Access to domains is granted by default. @@ -116,6 +132,38 @@ pub struct Wasi { pub deny_multiple_wasi_versions: bool, } +/// Various triggers that will cause the runtime to take snapshot +/// of the WASM state and store it in the snapshot file. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum SnapshotTrigger { + /// Triggered when all the threads in the process goes idle + OnIdle, + /// Issued if the user sends an interrupt signal (Ctrl + C). + Sigint, + /// Alarm clock signal (used for timers) + Sigalrm, + /// The SIGTSTP signal is sent to a process by its controlling terminal to request it to stop temporarily. It is commonly initiated by the user pressing Ctrl-Z. + Sigtstp, + /// The SIGSTOP signal instructs the operating system to stop a process for later resumption. + Sigstop, +} + +impl FromStr for SnapshotTrigger { + type Err = anyhow::Error; + + fn from_str(s: &str) -> std::result::Result { + let s = s.trim().to_lowercase(); + Ok(match s.as_str() { + "onidle" | "on-idle" => Self::OnIdle, + "sigint" | "ctrlc" | "ctrl-c" => Self::Sigint, + "sigalrm" => Self::Sigalrm, + "sigtstp" | "ctrlz" | "ctrl-z" => Self::Sigtstp, + "sigstop" => Self::Sigstop, + a => return Err(anyhow::format_err!("invalid or unknown trigger ({a})")), + }) + } +} + pub struct RunProperties { pub ctx: WasiFunctionEnv, pub path: PathBuf, diff --git a/lib/wasix/src/lib.rs b/lib/wasix/src/lib.rs index b273a4f5e89..1ff8642bbf1 100644 --- a/lib/wasix/src/lib.rs +++ b/lib/wasix/src/lib.rs @@ -50,6 +50,7 @@ pub mod http; mod rewind; pub mod runners; pub mod runtime; +pub mod snapshot; mod state; mod syscalls; mod utils; diff --git a/lib/wasix/src/os/task/thread.rs b/lib/wasix/src/os/task/thread.rs index 894d3ae74a9..8dbd83b6fb8 100644 --- a/lib/wasix/src/os/task/thread.rs +++ b/lib/wasix/src/os/task/thread.rs @@ -1,3 +1,4 @@ +use serde::{Deserialize, Serialize}; use std::{ collections::HashMap, ops::{Deref, DerefMut}, @@ -23,7 +24,7 @@ use super::{ }; /// Represents the ID of a WASI thread -#[derive(Default, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[derive(Default, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] pub struct WasiThreadId(u32); impl WasiThreadId { diff --git a/lib/wasix/src/runtime/mod.rs b/lib/wasix/src/runtime/mod.rs index 3cb1b552ffc..b9583a5ef92 100644 --- a/lib/wasix/src/runtime/mod.rs +++ b/lib/wasix/src/runtime/mod.rs @@ -27,6 +27,7 @@ use crate::{ package_loader::{PackageLoader, UnsupportedPackageLoader}, resolver::{MultiSource, Source, WapmSource}, }, + snapshot::{DynSnapShooter, UnsupportedSnapShooter}, WasiTtyState, }; @@ -90,7 +91,7 @@ where } /// Load a a Webassembly module, trying to use a pre-compiled version if possible. - fn load_module<'a>(&'a self, wasm: &'a [u8]) -> BoxFuture<'a, Result> { + fn load_module<'a>(&'a self, wasm: &'a [u8]) -> BoxFuture<'a, anyhow::Result> { let engine = self.engine(); let module_cache = self.module_cache(); @@ -105,6 +106,12 @@ where fn load_module_sync(&self, wasm: &[u8]) -> Result { InlineWaker::block_on(self.load_module(wasm)) } + + /// The snap shooter takes and restores snapshots of the WASM process at specific + /// points in time by reading and writing log entries + fn snap_shooter<'a>(&'a self) -> Arc { + Arc::new(UnsupportedSnapShooter::default()) as Arc + } } /// Load a a Webassembly module, trying to use a pre-compiled version if possible. diff --git a/lib/wasix/src/snapshot/log_file.rs b/lib/wasix/src/snapshot/log_file.rs new file mode 100644 index 00000000000..30694760989 --- /dev/null +++ b/lib/wasix/src/snapshot/log_file.rs @@ -0,0 +1,243 @@ +use serde::{Deserialize, Serialize}; +use std::{ + io::{self, ErrorKind, SeekFrom}, + mem::MaybeUninit, + path::Path, +}; + +use futures::future::BoxFuture; +use virtual_fs::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt, Fd}; + +use crate::WasiThreadId; + +use super::*; + +/// The snapshot log entries are serializable which +/// allows them to be written directly to a file +/// +/// Note: This structure is versioned which allows for +/// changes to the log entry types without having to +/// worry about backward and forward compatibility +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum SnapshotLogEntry { + TerminalDataV1 { + data: Vec, + }, + UpdateMemoryRegionV1 { + start: u64, + end: u64, + data: Vec, + }, + CloseThreadV1 { + id: WasiThreadId, + }, + SetThreadV1 { + id: WasiThreadId, + call_stack: Vec, + memory_stack: Vec, + }, + CloseFileDescriptorV1 { + fd: Fd, + }, + OpenFileDescriptorV1 { + fd: Fd, + state: FdSnapshot, + }, + RemoveFileSystemEntryV1 { + path: String, + }, + UpdateFileSystemEntryV1 { + path: String, + ft: FileEntryType, + accessed: u64, + created: u64, + modified: u64, + len: u64, + data: Vec, + }, +} + +impl<'a> From> for SnapshotLogEntry { + fn from(value: SnapshotLog<'a>) -> Self { + match value { + SnapshotLog::TerminalData { data } => Self::TerminalDataV1 { + data: data.into_owned(), + }, + SnapshotLog::UpdateMemoryRegion { region, data } => Self::UpdateMemoryRegionV1 { + start: region.start, + end: region.end, + data: data.into_owned(), + }, + SnapshotLog::CloseThread { id } => Self::CloseThreadV1 { id }, + SnapshotLog::SetThread { + id, + call_stack, + memory_stack, + } => Self::SetThreadV1 { + id, + call_stack: call_stack.into_owned(), + memory_stack: memory_stack.into_owned(), + }, + SnapshotLog::CloseFileDescriptor { fd } => Self::CloseFileDescriptorV1 { fd }, + SnapshotLog::OpenFileDescriptor { fd, state } => { + Self::OpenFileDescriptorV1 { fd, state } + } + SnapshotLog::RemoveFileSystemEntry { path } => Self::RemoveFileSystemEntryV1 { + path: path.into_owned(), + }, + SnapshotLog::UpdateFileSystemEntry { + path, + ft, + accessed, + created, + modified, + len, + data, + } => Self::UpdateFileSystemEntryV1 { + path: path.into_owned(), + ft, + accessed, + created, + modified, + len, + data: data.into_owned(), + }, + } + } +} + +impl<'a> From for SnapshotLog<'a> { + fn from(value: SnapshotLogEntry) -> Self { + match value { + SnapshotLogEntry::TerminalDataV1 { data } => Self::TerminalData { data: data.into() }, + SnapshotLogEntry::UpdateMemoryRegionV1 { start, end, data } => { + Self::UpdateMemoryRegion { + region: start..end, + data: data.into(), + } + } + SnapshotLogEntry::CloseThreadV1 { id } => Self::CloseThread { id }, + SnapshotLogEntry::SetThreadV1 { + id, + call_stack, + memory_stack, + } => Self::SetThread { + id: id, + call_stack: call_stack.into(), + memory_stack: memory_stack.into(), + }, + SnapshotLogEntry::CloseFileDescriptorV1 { fd } => Self::CloseFileDescriptor { fd }, + SnapshotLogEntry::OpenFileDescriptorV1 { fd, state } => Self::OpenFileDescriptor { + fd, + state: state.clone(), + }, + SnapshotLogEntry::RemoveFileSystemEntryV1 { path } => { + Self::RemoveFileSystemEntry { path: path.into() } + } + SnapshotLogEntry::UpdateFileSystemEntryV1 { + path, + ft, + accessed, + created, + modified, + len, + data, + } => Self::UpdateFileSystemEntry { + path: path.into(), + ft: ft.clone(), + accessed, + created, + modified, + len, + data: data.into(), + }, + } + } +} + +struct State { + file: tokio::fs::File, + at_end: bool, +} + +/// The LogFile snap shooter will write its snapshots to a linear journal +/// and read them when restoring. It uses the `bincode` serializer which +/// means that forwards and backwards compatibility must be dealt with +/// carefully. +/// +/// When opening an existing journal file that was previously saved +/// then new entries will be added to the end regardless of if +/// its been read. +/// +/// The logfile snapshooter uses a 64bit number as a entry encoding +/// delimiter. +pub struct LogFileSnapShooter { + state: tokio::sync::Mutex, +} + +impl LogFileSnapShooter { + pub async fn new(path: impl AsRef) -> io::Result { + let state = State { + file: tokio::fs::File::options() + .read(true) + .write(true) + .create(true) + .open(path) + .await?, + at_end: false, + }; + Ok(Self { + state: tokio::sync::Mutex::new(state), + }) + } +} + +#[async_trait::async_trait] +impl SnapShooter for LogFileSnapShooter { + fn write<'a>(&'a self, entry: SnapshotLog<'a>) -> BoxFuture<'a, anyhow::Result<()>> { + Box::pin(async { + let entry: SnapshotLogEntry = entry.into(); + + let mut state = self.state.lock().await; + if state.at_end == false { + state.file.seek(SeekFrom::End(0)).await?; + state.at_end = true; + } + + let data = bincode::serialize(&entry)?; + let data_len = data.len() as u64; + let data_len = data_len.to_ne_bytes(); + state.file.write_all(&data_len).await?; + state.file.write_all(&data).await?; + + Ok(()) + }) + } + + /// UNSAFE: This method uses unsafe operations to remove the need to zero + /// the buffer before its read the log entries into it + fn read<'a>(&'a self) -> BoxFuture<'a, anyhow::Result>>> { + Box::pin(async { + let mut state = self.state.lock().await; + + let mut data_len = [0u8; 8]; + match state.file.read_exact(&mut data_len).await { + Err(err) if err.kind() == ErrorKind::UnexpectedEof => return Ok(None), + Err(err) => return Err(err.into()), + Ok(_) => {} + } + + let data_len = u64::from_ne_bytes(data_len); + let mut data = Vec::with_capacity(data_len as usize); + let data_unsafe: &mut [MaybeUninit] = data.spare_capacity_mut(); + let data_unsafe: &mut [u8] = unsafe { std::mem::transmute(data_unsafe) }; + state.file.read_exact(data_unsafe).await?; + unsafe { + data.set_len(data_len as usize); + } + + let entry: SnapshotLogEntry = bincode::deserialize(&data)?; + Ok(Some(entry.into())) + }) + } +} diff --git a/lib/wasix/src/snapshot/mod.rs b/lib/wasix/src/snapshot/mod.rs new file mode 100644 index 00000000000..6c7ab7a969b --- /dev/null +++ b/lib/wasix/src/snapshot/mod.rs @@ -0,0 +1,7 @@ +mod log_file; +mod shooter; +mod unsupported; + +pub use log_file::*; +pub use shooter::*; +pub use unsupported::*; diff --git a/lib/wasix/src/snapshot/shooter.rs b/lib/wasix/src/snapshot/shooter.rs new file mode 100644 index 00000000000..c7ed1b9bca1 --- /dev/null +++ b/lib/wasix/src/snapshot/shooter.rs @@ -0,0 +1,81 @@ +use serde::{Deserialize, Serialize}; +use std::{borrow::Cow, ops::Range}; + +use futures::future::BoxFuture; +use virtual_fs::Fd; + +use crate::WasiThreadId; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum FdSnapshot { + Stdin, + Stdout, + OpenFile, + Socket, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum FileEntryType { + Directory, + File, + Symlink, + CharDevice, + BlockDevice, + Socket, + Fifo, +} + +/// Represents a log entry in a snapshot log stream that represents the total +/// state of a WASM process at a point in time. +pub enum SnapshotLog<'a> { + TerminalData { + data: Cow<'a, [u8]>, + }, + UpdateMemoryRegion { + region: Range, + data: Cow<'a, [u8]>, + }, + CloseThread { + id: WasiThreadId, + }, + SetThread { + id: WasiThreadId, + call_stack: Cow<'a, [u8]>, + memory_stack: Cow<'a, [u8]>, + }, + CloseFileDescriptor { + fd: Fd, + }, + OpenFileDescriptor { + fd: Fd, + state: FdSnapshot, + }, + RemoveFileSystemEntry { + path: Cow<'a, str>, + }, + UpdateFileSystemEntry { + path: Cow<'a, str>, + ft: FileEntryType, + accessed: u64, + created: u64, + modified: u64, + len: u64, + data: Cow<'a, [u8]>, + }, +} + +/// The snap shooter will take a series of objects that represents the state of +/// a WASM process at a point in time and saves it so that it can be restored. +/// It also allows for the restoration of that state at a later moment +#[allow(unused_variables)] +pub trait SnapShooter { + /// Takes in a stream of snapshot log entries and saves them so that they + /// may be restored at a later moment + fn write<'a>(&'a self, entry: SnapshotLog<'a>) -> BoxFuture<'a, anyhow::Result<()>>; + + /// Returns a stream of snapshot objects that the runtime will use + /// to restore the state of a WASM process to a previous moment in time + fn read<'a>(&'a self) -> BoxFuture<'a, anyhow::Result>>>; +} + +pub type DynSnapShooter = dyn SnapShooter + Send + Sync; diff --git a/lib/wasix/src/snapshot/unsupported.rs b/lib/wasix/src/snapshot/unsupported.rs new file mode 100644 index 00000000000..6364fd97149 --- /dev/null +++ b/lib/wasix/src/snapshot/unsupported.rs @@ -0,0 +1,18 @@ +use futures::future::BoxFuture; + +use super::*; + +/// The default for runtime is to use the unsupported snap-shooter +/// which will fail to snapshot if one attempts to do so. +#[derive(Debug, Default)] +pub struct UnsupportedSnapShooter {} + +impl SnapShooter for UnsupportedSnapShooter { + fn write<'a>(&'a self, _entry: SnapshotLog<'a>) -> BoxFuture<'a, anyhow::Result<()>> { + Box::pin(async { Err(anyhow::format_err!("unsupported")) }) + } + + fn read<'a>(&'a self) -> BoxFuture<'a, anyhow::Result>>> { + Box::pin(async { Ok(None) }) + } +} From 6940ba844f1cd27843151bf6578005a2ee25dc32 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Fri, 20 Oct 2023 11:55:36 +1100 Subject: [PATCH 003/129] Added more snapshot trigger types --- lib/cli/src/commands/run/wasi.rs | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/lib/cli/src/commands/run/wasi.rs b/lib/cli/src/commands/run/wasi.rs index 92f96b4b456..d78a06ebf21 100644 --- a/lib/cli/src/commands/run/wasi.rs +++ b/lib/cli/src/commands/run/wasi.rs @@ -113,9 +113,18 @@ pub struct Wasi { /// Indicates what events will cause a snapshot to be taken /// and written to the snapshot file. + /// + /// If not specified, the default is to snapshot on idle plus if a + /// snapshot period is provided it will also default to periodic snapshots + /// as well. #[clap(long = "snapshot-on")] pub snapshot_on: Vec, + /// Time in seconds between taking snapshots of the process and dumping + /// them to the snapshot file. + #[clap(long = "snapshot-period")] + pub snapshot_period: Option, + /// When specified, the runtime will restore a previous snapshot /// using the supplied file. #[clap(long = "resume-from")] @@ -137,7 +146,13 @@ pub struct Wasi { #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum SnapshotTrigger { /// Triggered when all the threads in the process goes idle - OnIdle, + Idle, + /// Triggered when a listen syscall is invoked on a socket + Listen, + /// Triggered when the process reads stdin for the first time + Stdin, + /// Triggered periodically (default 10 seconds) which can be specified using the `snapshot-period` option + Periodic, /// Issued if the user sends an interrupt signal (Ctrl + C). Sigint, /// Alarm clock signal (used for timers) @@ -154,11 +169,14 @@ impl FromStr for SnapshotTrigger { fn from_str(s: &str) -> std::result::Result { let s = s.trim().to_lowercase(); Ok(match s.as_str() { - "onidle" | "on-idle" => Self::OnIdle, - "sigint" | "ctrlc" | "ctrl-c" => Self::Sigint, - "sigalrm" => Self::Sigalrm, + "idle" => Self::Idle, + "listen" => Self::Listen, + "stdin" => Self::Stdin, + "periodic" => Self::Periodic, + "intr" | "sigint" | "ctrlc" | "ctrl-c" => Self::Sigint, + "alarm" | "timer" | "sigalrm" => Self::Sigalrm, "sigtstp" | "ctrlz" | "ctrl-z" => Self::Sigtstp, - "sigstop" => Self::Sigstop, + "stop" | "sigstop" => Self::Sigstop, a => return Err(anyhow::format_err!("invalid or unknown trigger ({a})")), }) } From 6d048487e431b57a03ebce5cf2bda3c963a5990e Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Fri, 20 Oct 2023 23:18:52 +1100 Subject: [PATCH 004/129] Puts the snapshot functionality behind a feature flag --- lib/cli/Cargo.toml | 2 ++ lib/cli/src/commands/run/wasi.rs | 10 ++++++++++ lib/wasix/Cargo.toml | 1 + lib/wasix/src/lib.rs | 1 + lib/wasix/src/runtime/mod.rs | 18 +++++++++++++++++- lib/wasix/src/snapshot/log_file.rs | 3 +++ lib/wasix/src/snapshot/shooter.rs | 2 ++ 7 files changed, 36 insertions(+), 1 deletion(-) diff --git a/lib/cli/Cargo.toml b/lib/cli/Cargo.toml index f8cee396a60..38cbd26c80c 100644 --- a/lib/cli/Cargo.toml +++ b/lib/cli/Cargo.toml @@ -160,9 +160,11 @@ default = [ "wat", "wast", "compiler", + "snapshooter", "wasmer-artifact-create", "static-artifact-create", ] +snapshooter = ["wasmer-wasix/snapshooter"] backend = [] coredump = ["wasm-coredump-builder"] sys = ["compiler", "wasmer-vm"] diff --git a/lib/cli/src/commands/run/wasi.rs b/lib/cli/src/commands/run/wasi.rs index d78a06ebf21..69d71fe70d2 100644 --- a/lib/cli/src/commands/run/wasi.rs +++ b/lib/cli/src/commands/run/wasi.rs @@ -34,6 +34,7 @@ use wasmer_wasix::{ VirtualTaskManagerExt, }, }, + snapshot, types::__WASI_STDIN_FILENO, wasmer_wasix_types::wasi::Errno, PluggableRuntime, RewindState, Runtime, WasiEnv, WasiEnvBuilder, WasiError, WasiFunctionEnv, @@ -108,6 +109,7 @@ pub struct Wasi { /// Specifies the snapshot file that Wasmer will use to store /// the state of the WASM process so that it can be later restored + #[cfg(feature = "snapshooter")] #[clap(long = "snapshot-to")] pub snapshot_to: Option, @@ -117,16 +119,19 @@ pub struct Wasi { /// If not specified, the default is to snapshot on idle plus if a /// snapshot period is provided it will also default to periodic snapshots /// as well. + #[cfg(feature = "snapshooter")] #[clap(long = "snapshot-on")] pub snapshot_on: Vec, /// Time in seconds between taking snapshots of the process and dumping /// them to the snapshot file. + #[cfg(feature = "snapshooter")] #[clap(long = "snapshot-period")] pub snapshot_period: Option, /// When specified, the runtime will restore a previous snapshot /// using the supplied file. + #[cfg(feature = "snapshooter")] #[clap(long = "resume-from")] pub resume_from: Option, @@ -333,6 +338,11 @@ impl Wasi { rt.set_networking_implementation(virtual_net::UnsupportedVirtualNetworking::default()); } + #[cfg(feature = "snapshooter")] + if let Some(path) = &self.resume_from { + rt.set_snapshooter(snapshot::LogFileSnapShooter::new(path).await?); + } + if !self.no_tty { let tty = Arc::new(SysTty::default()); tty.reset(); diff --git a/lib/wasix/Cargo.toml b/lib/wasix/Cargo.toml index 98554d3797f..1f1bf65ee0b 100644 --- a/lib/wasix/Cargo.toml +++ b/lib/wasix/Cargo.toml @@ -119,6 +119,7 @@ sys = ["webc/mmap", "time", "virtual-mio/sys"] sys-default = ["sys", "logging", "host-fs", "sys-poll", "sys-thread", "host-vnet", "host-threads", "host-reqwest"] sys-poll = [] sys-thread = ["tokio/rt", "tokio/time", "tokio/rt-multi-thread", "rayon"] +snapshooter = [] # Deprecated. Kept it for compatibility compiler = [] diff --git a/lib/wasix/src/lib.rs b/lib/wasix/src/lib.rs index 1ff8642bbf1..11453f8b5c9 100644 --- a/lib/wasix/src/lib.rs +++ b/lib/wasix/src/lib.rs @@ -50,6 +50,7 @@ pub mod http; mod rewind; pub mod runners; pub mod runtime; +#[cfg(feature = "snapshooter")] pub mod snapshot; mod state; mod syscalls; diff --git a/lib/wasix/src/runtime/mod.rs b/lib/wasix/src/runtime/mod.rs index b9583a5ef92..13259c34a6d 100644 --- a/lib/wasix/src/runtime/mod.rs +++ b/lib/wasix/src/runtime/mod.rs @@ -19,6 +19,8 @@ use futures::future::BoxFuture; use virtual_net::{DynVirtualNetworking, VirtualNetworking}; use wasmer::Module; +#[cfg(feature = "snapshooter")] +use crate::snapshot::{DynSnapShooter, UnsupportedSnapShooter}; use crate::{ http::{DynHttpClient, HttpClient}, os::TtyBridge, @@ -27,7 +29,6 @@ use crate::{ package_loader::{PackageLoader, UnsupportedPackageLoader}, resolver::{MultiSource, Source, WapmSource}, }, - snapshot::{DynSnapShooter, UnsupportedSnapShooter}, WasiTtyState, }; @@ -109,6 +110,7 @@ where /// The snap shooter takes and restores snapshots of the WASM process at specific /// points in time by reading and writing log entries + #[cfg(feature = "snapshooter")] fn snap_shooter<'a>(&'a self) -> Arc { Arc::new(UnsupportedSnapShooter::default()) as Arc } @@ -187,6 +189,9 @@ pub struct PluggableRuntime { pub module_cache: Arc, #[derivative(Debug = "ignore")] pub tty: Option>, + #[cfg(feature = "snapshooter")] + #[derivative(Debug = "ignore")] + pub snapshooter: Arc, } impl PluggableRuntime { @@ -221,6 +226,7 @@ impl PluggableRuntime { source: Arc::new(source), package_loader: Arc::new(loader), module_cache: Arc::new(module_cache::in_memory()), + snapshooter: Arc::new(UnsupportedSnapShooter::default()) as Arc, } } @@ -270,6 +276,11 @@ impl PluggableRuntime { self.http_client = Some(Arc::new(client)); self } + + pub fn set_snapshooter(&mut self, snapshooter: Arc) -> &mut Self { + self.snapshooter = snapshooter; + self + } } impl Runtime for PluggableRuntime { @@ -315,4 +326,9 @@ impl Runtime for PluggableRuntime { fn module_cache(&self) -> Arc { self.module_cache.clone() } + + #[cfg(feature = "snapshooter")] + fn snap_shooter<'a>(&'a self) -> Arc { + self.snapshooter.clone() + } } diff --git a/lib/wasix/src/snapshot/log_file.rs b/lib/wasix/src/snapshot/log_file.rs index 30694760989..cbb2a1db86e 100644 --- a/lib/wasix/src/snapshot/log_file.rs +++ b/lib/wasix/src/snapshot/log_file.rs @@ -55,6 +55,7 @@ pub enum SnapshotLogEntry { len: u64, data: Vec, }, + SnapshotV1, } impl<'a> From> for SnapshotLogEntry { @@ -102,6 +103,7 @@ impl<'a> From> for SnapshotLogEntry { len, data: data.into_owned(), }, + SnapshotLog::SnapshotV1 => Self::SnapshotV1, } } } @@ -151,6 +153,7 @@ impl<'a> From for SnapshotLog<'a> { len, data: data.into(), }, + SnapshotLogEntry::SnapshotV1 => Self::SnapshotV1, } } } diff --git a/lib/wasix/src/snapshot/shooter.rs b/lib/wasix/src/snapshot/shooter.rs index c7ed1b9bca1..f81c44f8d04 100644 --- a/lib/wasix/src/snapshot/shooter.rs +++ b/lib/wasix/src/snapshot/shooter.rs @@ -62,6 +62,8 @@ pub enum SnapshotLog<'a> { len: u64, data: Cow<'a, [u8]>, }, + /// Represents the marker for the end of a snapshot + SnapshotV1, } /// The snap shooter will take a series of objects that represents the state of From c1872fd48a9cbd0bdaf9ebb61dd331c0d9c206c4 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Fri, 20 Oct 2023 23:48:35 +1100 Subject: [PATCH 005/129] Added an example on how certain events will be loggin to the snap shooter --- lib/cli/src/commands/run/wasi.rs | 2 +- lib/wasix/src/runtime/mod.rs | 9 +++++---- lib/wasix/src/snapshot/log_file.rs | 19 +++++++++++++++-- lib/wasix/src/snapshot/shooter.rs | 2 +- lib/wasix/src/snapshot/unsupported.rs | 2 ++ lib/wasix/src/state/env.rs | 6 ++++++ lib/wasix/src/syscalls/wasi/fd_write.rs | 27 ++++++++++++++++++++++++- lib/wasix/src/utils/mod.rs | 5 +++++ 8 files changed, 63 insertions(+), 9 deletions(-) diff --git a/lib/cli/src/commands/run/wasi.rs b/lib/cli/src/commands/run/wasi.rs index 69d71fe70d2..c20166a5440 100644 --- a/lib/cli/src/commands/run/wasi.rs +++ b/lib/cli/src/commands/run/wasi.rs @@ -340,7 +340,7 @@ impl Wasi { #[cfg(feature = "snapshooter")] if let Some(path) = &self.resume_from { - rt.set_snapshooter(snapshot::LogFileSnapShooter::new(path).await?); + rt.set_snapshooter(Arc::new(snapshot::LogFileSnapShooter::new_std(path)?)); } if !self.no_tty { diff --git a/lib/wasix/src/runtime/mod.rs b/lib/wasix/src/runtime/mod.rs index 13259c34a6d..74604ad1d53 100644 --- a/lib/wasix/src/runtime/mod.rs +++ b/lib/wasix/src/runtime/mod.rs @@ -8,6 +8,7 @@ use self::{ module_cache::{CacheError, ModuleHash}, task_manager::InlineWaker, }; +use crate::snapshot::UNSUPPORTED_SNAP_SHOOTER; use std::{ fmt, @@ -111,8 +112,8 @@ where /// The snap shooter takes and restores snapshots of the WASM process at specific /// points in time by reading and writing log entries #[cfg(feature = "snapshooter")] - fn snap_shooter<'a>(&'a self) -> Arc { - Arc::new(UnsupportedSnapShooter::default()) as Arc + fn snap_shooter<'a>(&'a self) -> &'_ DynSnapShooter { + &UNSUPPORTED_SNAP_SHOOTER } } @@ -328,7 +329,7 @@ impl Runtime for PluggableRuntime { } #[cfg(feature = "snapshooter")] - fn snap_shooter<'a>(&'a self) -> Arc { - self.snapshooter.clone() + fn snap_shooter<'a>(&'a self) -> &DynSnapShooter { + self.snapshooter.as_ref() } } diff --git a/lib/wasix/src/snapshot/log_file.rs b/lib/wasix/src/snapshot/log_file.rs index cbb2a1db86e..0379c457775 100644 --- a/lib/wasix/src/snapshot/log_file.rs +++ b/lib/wasix/src/snapshot/log_file.rs @@ -103,7 +103,7 @@ impl<'a> From> for SnapshotLogEntry { len, data: data.into_owned(), }, - SnapshotLog::SnapshotV1 => Self::SnapshotV1, + SnapshotLog::Snapshot => Self::SnapshotV1, } } } @@ -153,7 +153,7 @@ impl<'a> From for SnapshotLog<'a> { len, data: data.into(), }, - SnapshotLogEntry::SnapshotV1 => Self::SnapshotV1, + SnapshotLogEntry::SnapshotV1 => Self::Snapshot, } } } @@ -193,6 +193,21 @@ impl LogFileSnapShooter { state: tokio::sync::Mutex::new(state), }) } + + pub fn new_std(path: impl AsRef) -> io::Result { + let file = std::fs::File::options() + .read(true) + .write(true) + .create(true) + .open(path)?; + let state = State { + file: tokio::fs::File::from_std(file), + at_end: false, + }; + Ok(Self { + state: tokio::sync::Mutex::new(state), + }) + } } #[async_trait::async_trait] diff --git a/lib/wasix/src/snapshot/shooter.rs b/lib/wasix/src/snapshot/shooter.rs index f81c44f8d04..edad217d17d 100644 --- a/lib/wasix/src/snapshot/shooter.rs +++ b/lib/wasix/src/snapshot/shooter.rs @@ -63,7 +63,7 @@ pub enum SnapshotLog<'a> { data: Cow<'a, [u8]>, }, /// Represents the marker for the end of a snapshot - SnapshotV1, + Snapshot, } /// The snap shooter will take a series of objects that represents the state of diff --git a/lib/wasix/src/snapshot/unsupported.rs b/lib/wasix/src/snapshot/unsupported.rs index 6364fd97149..04febd99d7a 100644 --- a/lib/wasix/src/snapshot/unsupported.rs +++ b/lib/wasix/src/snapshot/unsupported.rs @@ -2,6 +2,8 @@ use futures::future::BoxFuture; use super::*; +pub static UNSUPPORTED_SNAP_SHOOTER: UnsupportedSnapShooter = UnsupportedSnapShooter {}; + /// The default for runtime is to use the unsupported snap-shooter /// which will fail to snapshot if one attempts to do so. #[derive(Debug, Default)] diff --git a/lib/wasix/src/state/env.rs b/lib/wasix/src/state/env.rs index ea9dcb2b2df..534aaefcdd6 100644 --- a/lib/wasix/src/state/env.rs +++ b/lib/wasix/src/state/env.rs @@ -299,6 +299,9 @@ pub struct WasiEnv { /// Is this environment capable and setup for deep sleeping pub enable_deep_sleep: bool, + /// Indicates if the feed to the snapshot system is enabled or not + pub enable_snapshot_feed: bool, + /// Inner functions and references that are loaded before the environment starts /// (inner is not safe to send between threads and so it is private and will /// not be cloned when `WasiEnv` is cloned) @@ -328,6 +331,7 @@ impl Clone for WasiEnv { runtime: self.runtime.clone(), capabilities: self.capabilities.clone(), enable_deep_sleep: self.enable_deep_sleep, + enable_snapshot_feed: self.enable_snapshot_feed, } } } @@ -364,6 +368,7 @@ impl WasiEnv { runtime: self.runtime.clone(), capabilities: self.capabilities.clone(), enable_deep_sleep: self.enable_deep_sleep, + enable_snapshot_feed: self.enable_snapshot_feed, }; Ok((new_env, handle)) } @@ -424,6 +429,7 @@ impl WasiEnv { runtime: init.runtime, bin_factory: init.bin_factory, enable_deep_sleep: init.capabilities.threading.enable_asynchronous_threading, + enable_snapshot_feed: false, capabilities: init.capabilities, }; env.owned_handles.push(thread); diff --git a/lib/wasix/src/syscalls/wasi/fd_write.rs b/lib/wasix/src/syscalls/wasi/fd_write.rs index e60ba61ae25..896e1209066 100644 --- a/lib/wasix/src/syscalls/wasi/fd_write.rs +++ b/lib/wasix/src/syscalls/wasi/fd_write.rs @@ -1,7 +1,7 @@ use std::task::Waker; use super::*; -use crate::{net::socket::TimeType, syscalls::*}; +use crate::{net::socket::TimeType, snapshot::SnapshotLog, syscalls::*, utils::map_snapshot_err}; /// ### `fd_write()` /// Write data to the file descriptor @@ -101,6 +101,31 @@ pub(crate) fn fd_write_internal( return Ok(Errno::Access); } + // If snap-shooting is enabled and this is to stdio then we + // will record a terminal event. + #[cfg(feature = "snapshooter")] + if is_stdio && ctx.data().enable_snapshot_feed { + __asyncify_light(env, None, async { + let iovs_arr = iovs_arr.access().map_err(mem_error_to_wasi)?; + for iovs in iovs_arr.iter() { + let buf = WasmPtr::::new(iovs.buf) + .slice(&memory, iovs.buf_len) + .map_err(mem_error_to_wasi)? + .access() + .map_err(mem_error_to_wasi)?; + ctx.data() + .runtime() + .snap_shooter() + .write(SnapshotLog::TerminalData { + data: Cow::Borrowed(buf.as_ref()), + }) + .await + .map_err(map_snapshot_err)?; + } + Ok(()) + }); + } + let fd_flags = fd_entry.flags; let (bytes_written, can_update_cursor) = { diff --git a/lib/wasix/src/utils/mod.rs b/lib/wasix/src/utils/mod.rs index 6cf471da1d8..a36fd6d4651 100644 --- a/lib/wasix/src/utils/mod.rs +++ b/lib/wasix/src/utils/mod.rs @@ -38,6 +38,11 @@ pub fn map_io_err(err: std::io::Error) -> Errno { From::::from(err) } +pub fn map_snapshot_err(err: anyhow::Error) -> Errno { + tracing::warn!("unknown snapshot error: {}", err); + Errno::Unknown +} + /// The version of WASI. This is determined by the imports namespace /// string. #[derive(Debug, Clone, Copy, Eq)] From ec74b83f255c383999b5c1bbe2358e05adebfcb0 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Mon, 23 Oct 2023 06:16:36 +1100 Subject: [PATCH 006/129] Added the basics of the snapshot mechanism --- lib/api/src/externals/memory_view.rs | 13 +- lib/api/src/js/externals/memory_view.rs | 15 +- lib/api/src/jsc/externals/memory_view.rs | 15 +- lib/api/src/sys/externals/memory_view.rs | 15 +- lib/wasix/src/os/task/control_plane.rs | 12 +- lib/wasix/src/os/task/process.rs | 185 +++++++++++++++---- lib/wasix/src/os/task/thread.rs | 38 +++- lib/wasix/src/snapshot/effector.rs | 143 ++++++++++++++ lib/wasix/src/snapshot/mod.rs | 2 + lib/wasix/src/state/env.rs | 33 ++-- lib/wasix/src/state/func_env.rs | 14 ++ lib/wasix/src/syscalls/wasi/fd_write.rs | 32 +--- lib/wasix/src/syscalls/wasix/proc_fork.rs | 2 +- lib/wasix/src/syscalls/wasix/proc_join.rs | 2 +- lib/wasix/src/syscalls/wasix/proc_spawn.rs | 2 +- lib/wasix/src/syscalls/wasix/thread_spawn.rs | 2 +- 16 files changed, 423 insertions(+), 102 deletions(-) create mode 100644 lib/wasix/src/snapshot/effector.rs diff --git a/lib/api/src/externals/memory_view.rs b/lib/api/src/externals/memory_view.rs index 7a7dc9a3192..1fbda5c9afd 100644 --- a/lib/api/src/externals/memory_view.rs +++ b/lib/api/src/externals/memory_view.rs @@ -2,6 +2,7 @@ use super::memory::{Memory, MemoryBuffer}; use crate::store::AsStoreRef; use crate::MemoryAccessError; use std::mem::MaybeUninit; +use std::ops::Range; use wasmer_types::Pages; #[cfg(feature = "js")] @@ -145,11 +146,17 @@ impl<'a> MemoryView<'a> { /// Copies the memory and returns it as a vector of bytes pub fn copy_to_vec(&self) -> Result, MemoryAccessError> { + self.copy_range_to_vec(0..self.data_size()) + } + + /// Copies a range of the memory and returns it as a vector of bytes + pub fn copy_range_to_vec(&self, range: Range) -> Result, MemoryAccessError> { let mut new_memory = Vec::new(); - let mut offset = 0; + let mut offset = range.start; + let end = range.end.min(self.data_size()); let mut chunk = [0u8; 40960]; - while offset < self.data_size() { - let remaining = self.data_size() - offset; + while offset < end { + let remaining = end - offset; let sublen = remaining.min(chunk.len() as u64) as usize; self.read(offset, &mut chunk[..sublen])?; new_memory.extend_from_slice(&chunk[..sublen]); diff --git a/lib/api/src/js/externals/memory_view.rs b/lib/api/src/js/externals/memory_view.rs index fccd385ba88..938371b6acc 100644 --- a/lib/api/src/js/externals/memory_view.rs +++ b/lib/api/src/js/externals/memory_view.rs @@ -1,9 +1,9 @@ use crate::mem_access::MemoryAccessError; use crate::store::AsStoreRef; -use std::convert::TryInto; use std::marker::PhantomData; use std::mem::MaybeUninit; use std::slice; +use std::{convert::TryInto, ops::Range}; #[cfg(feature = "tracing")] use tracing::warn; use wasm_bindgen::JsCast; @@ -256,11 +256,18 @@ impl<'a> MemoryView<'a> { /// Copies the memory and returns it as a vector of bytes #[allow(unused)] pub fn copy_to_vec(&self) -> Result, MemoryAccessError> { + self.copy_range_to_vec(0..self.data_size()) + } + + /// Copies a range of the memory and returns it as a vector of bytes + #[allow(unused)] + pub fn copy_range_to_vec(&self, range: Range) -> Result, MemoryAccessError> { let mut new_memory = Vec::new(); - let mut offset = 0; + let mut offset = range.start; + let end = range.end.min(self.data_size()); let mut chunk = [0u8; 40960]; - while offset < self.data_size() { - let remaining = self.data_size() - offset; + while offset < end { + let remaining = end - offset; let sublen = remaining.min(chunk.len() as u64) as usize; self.read(offset, &mut chunk[..sublen])?; new_memory.extend_from_slice(&chunk[..sublen]); diff --git a/lib/api/src/jsc/externals/memory_view.rs b/lib/api/src/jsc/externals/memory_view.rs index eb3e3515477..bb891293f24 100644 --- a/lib/api/src/jsc/externals/memory_view.rs +++ b/lib/api/src/jsc/externals/memory_view.rs @@ -2,10 +2,10 @@ use crate::store::AsStoreRef; use crate::MemoryAccessError; use rusty_jsc::JSObject; use std::convert::TryFrom; -use std::convert::TryInto; use std::marker::PhantomData; use std::mem::MaybeUninit; use std::slice; +use std::{convert::TryInto, ops::Range}; use wasmer_types::{Bytes, Pages}; use super::memory::{Memory, MemoryBuffer}; @@ -178,11 +178,18 @@ impl<'a> MemoryView<'a> { #[allow(unused)] /// Copies the memory and returns it as a vector of bytes pub fn copy_to_vec(&self) -> Result, MemoryAccessError> { + self.copy_range_to_vec(0..self.data_size()) + } + + #[allow(unused)] + /// Copies a range of the memory and returns it as a vector of bytes + pub fn copy_range_to_vec(&self, range: Range) -> Result, MemoryAccessError> { let mut new_memory = Vec::new(); - let mut offset = 0; + let mut offset = range.start; + let end = range.end.min(self.data_size()); let mut chunk = [0u8; 40960]; - while offset < self.data_size() { - let remaining = self.data_size() - offset; + while offset < end { + let remaining = end - offset; let sublen = remaining.min(chunk.len() as u64) as usize; self.read(offset, &mut chunk[..sublen])?; new_memory.extend_from_slice(&chunk[..sublen]); diff --git a/lib/api/src/sys/externals/memory_view.rs b/lib/api/src/sys/externals/memory_view.rs index a506abd9dc2..b1dc5e1f8dc 100644 --- a/lib/api/src/sys/externals/memory_view.rs +++ b/lib/api/src/sys/externals/memory_view.rs @@ -1,9 +1,9 @@ use crate::store::AsStoreRef; use crate::MemoryAccessError; -use std::convert::TryInto; use std::marker::PhantomData; use std::mem::MaybeUninit; use std::slice; +use std::{convert::TryInto, ops::Range}; use wasmer_types::Pages; use wasmer_vm::LinearMemory; @@ -163,11 +163,18 @@ impl<'a> MemoryView<'a> { #[allow(unused)] /// Copies the memory and returns it as a vector of bytes pub fn copy_to_vec(&self) -> Result, MemoryAccessError> { + self.copy_range_to_vec(0..self.data_size()) + } + + #[allow(unused)] + /// Copies a range of the memory and returns it as a vector of bytes + pub fn copy_range_to_vec(&self, range: Range) -> Result, MemoryAccessError> { let mut new_memory = Vec::new(); - let mut offset = 0; + let mut offset = range.start; + let end = range.end.min(self.data_size()); let mut chunk = [0u8; 40960]; - while offset < self.data_size() { - let remaining = self.data_size() - offset; + while offset < end { + let remaining = end - offset; let sublen = remaining.min(chunk.len() as u64) as usize; self.read(offset, &mut chunk[..sublen])?; new_memory.extend_from_slice(&chunk[..sublen]); diff --git a/lib/wasix/src/os/task/control_plane.rs b/lib/wasix/src/os/task/control_plane.rs index be064ac4c80..af432d5e0d4 100644 --- a/lib/wasix/src/os/task/control_plane.rs +++ b/lib/wasix/src/os/task/control_plane.rs @@ -203,6 +203,8 @@ pub enum ControlPlaneError { #[cfg(test)] mod tests { + use crate::os::task::thread::WasiMemoryLayout; + use super::*; /// Simple test to ensure task limits are respected. @@ -214,8 +216,8 @@ mod tests { }); let p1 = p.new_process().unwrap(); - let _t1 = p1.new_thread().unwrap(); - let _t2 = p1.new_thread().unwrap(); + let _t1 = p1.new_thread(WasiMemoryLayout::default()).unwrap(); + let _t2 = p1.new_thread(WasiMemoryLayout::default()).unwrap(); assert_eq!( p.new_process().unwrap_err(), @@ -234,11 +236,11 @@ mod tests { let p1 = p.new_process().unwrap(); for _ in 0..10 { - let _thread = p1.new_thread().unwrap(); + let _thread = p1.new_thread(WasiMemoryLayout::default()).unwrap(); } - let _t1 = p1.new_thread().unwrap(); - let _t2 = p1.new_thread().unwrap(); + let _t1 = p1.new_thread(WasiMemoryLayout::default()).unwrap(); + let _t2 = p1.new_thread(WasiMemoryLayout::default()).unwrap(); assert_eq!( p.new_process().unwrap_err(), diff --git a/lib/wasix/src/os/task/process.rs b/lib/wasix/src/os/task/process.rs index d26b95561aa..c7277944f9e 100644 --- a/lib/wasix/src/os/task/process.rs +++ b/lib/wasix/src/os/task/process.rs @@ -1,15 +1,17 @@ -use crate::WasiRuntimeError; +use crate::{snapshot::SnapshotEffector, unwind, WasiEnv, WasiError, WasiRuntimeError}; use serde::{Deserialize, Serialize}; use std::{ collections::HashMap, convert::TryInto, sync::{ atomic::{AtomicU32, Ordering}, - Arc, RwLock, RwLockReadGuard, RwLockWriteGuard, Weak, + Arc, Condvar, Mutex, MutexGuard, RwLock, Weak, }, time::Duration, }; use tracing::trace; +use wasmer::FunctionEnvMut; +use wasmer_types::MemorySize; use wasmer_wasix_types::{ types::Signal, wasi::{Errno, ExitCode, Snapshot0Clockid}, @@ -24,6 +26,7 @@ use super::{ control_plane::{ControlPlaneError, WasiControlPlaneHandle}, signal::{SignalDeliveryError, SignalHandlerAbi}, task_join_handle::OwnedTaskStatus, + thread::WasiMemoryLayout, }; /// Represents the ID of a sub-process @@ -73,15 +76,16 @@ impl std::fmt::Debug for WasiProcessId { } /// Represents a process running within the compute state -// TODO: fields should be private and only accessed via methods. +/// TODO: fields should be private and only accessed via methods. #[derive(Debug, Clone)] pub struct WasiProcess { /// Unique ID of this process pub(crate) pid: WasiProcessId, /// List of all the children spawned from this thread pub(crate) parent: Option>>, - /// The inner protected region of the process - pub(crate) inner: Arc>, + /// The inner protected region of the process with a conditional + /// variable that is used for coordination such as checksums. + pub(crate) inner: Arc<(Mutex, Condvar)>, /// Reference back to the compute engine // TODO: remove this reference, access should happen via separate state instead // (we don't want cyclical references) @@ -92,6 +96,20 @@ pub struct WasiProcess { pub(crate) waiting: Arc, } +/// Represents a freeze of all threads to perform some action +/// on the total state-machine. This is normally done for +/// things like snapshots which require the memory to remain +/// stable while it performs a diff. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub enum WasiProcessCheckpoint { + /// No checkpoint will take place and the process + /// should just execute as per normal + Execute, + /// The process needs to take a snapshot of the + /// memory and state-machine + Snapshot, +} + // TODO: fields should be private and only accessed via methods. #[derive(Debug)] pub struct WasiProcessInner { @@ -105,6 +123,9 @@ pub struct WasiProcessInner { pub signal_intervals: HashMap, /// List of all the children spawned from this thread pub children: Vec, + /// Represents a checkpoint which blocks all the threads + /// and then executes some maintenance action + pub checkpoint: WasiProcessCheckpoint, } // TODO: why do we need this, how is it used? @@ -133,13 +154,17 @@ impl WasiProcess { pid, parent: None, compute: plane, - inner: Arc::new(RwLock::new(WasiProcessInner { - pid, - threads: Default::default(), - thread_count: Default::default(), - signal_intervals: Default::default(), - children: Default::default(), - })), + inner: Arc::new(( + Mutex::new(WasiProcessInner { + pid, + threads: Default::default(), + thread_count: Default::default(), + signal_intervals: Default::default(), + children: Default::default(), + checkpoint: WasiProcessCheckpoint::Execute, + }), + Condvar::new(), + )), finished: Arc::new(OwnedTaskStatus::default()), waiting: Arc::new(AtomicU32::new(0)), } @@ -164,26 +189,23 @@ impl WasiProcess { .unwrap_or(WasiProcessId(0)) } - /// Gains write access to the process internals - // TODO: Make this private, all inner access should be exposed with methods. - pub fn write(&self) -> RwLockWriteGuard { - self.inner.write().unwrap() - } - - /// Gains read access to the process internals + /// Gains access to the process internals // TODO: Make this private, all inner access should be exposed with methods. - pub fn read(&self) -> RwLockReadGuard { - self.inner.read().unwrap() + pub fn lock(&self) -> MutexGuard<'_, WasiProcessInner> { + self.inner.0.lock().unwrap() } /// Creates a a thread and returns it - pub fn new_thread(&self) -> Result { + pub fn new_thread( + &self, + layout: WasiMemoryLayout, + ) -> Result { let control_plane = self.compute.must_upgrade(); let task_count_guard = control_plane.register_task()?; // Determine if its the main thread or not let is_main = { - let inner = self.inner.read().unwrap(); + let inner = self.inner.0.lock().unwrap(); inner.thread_count == 0 }; @@ -197,7 +219,7 @@ impl WasiProcess { }; // The wait finished should be the process version if its the main thread - let mut inner = self.inner.write().unwrap(); + let mut inner = self.inner.0.lock().unwrap(); let finished = if is_main { self.finished.clone() } else { @@ -205,7 +227,7 @@ impl WasiProcess { }; // Insert the thread into the pool - let ctrl = WasiThread::new(self.pid(), tid, is_main, finished, task_count_guard); + let ctrl = WasiThread::new(self.pid(), tid, is_main, finished, task_count_guard, layout); inner.threads.insert(tid, ctrl.clone()); inner.thread_count += 1; @@ -214,7 +236,7 @@ impl WasiProcess { /// Gets a reference to a particular thread pub fn get_thread(&self, tid: &WasiThreadId) -> Option { - let inner = self.inner.read().unwrap(); + let inner = self.inner.0.lock().unwrap(); inner.threads.get(tid).cloned() } @@ -230,7 +252,7 @@ impl WasiProcess { let pid = self.pid(); tracing::trace!(%pid, %tid, "signal-thread({:?})", signal); - let inner = self.inner.read().unwrap(); + let inner = self.inner.0.lock().unwrap(); if let Some(thread) = inner.threads.get(&tid) { thread.signal(signal); } else { @@ -249,7 +271,7 @@ impl WasiProcess { tracing::trace!(%pid, "signal-process({:?})", signal); { - let inner = self.inner.read().unwrap(); + let inner = self.inner.0.lock().unwrap(); if self.waiting.load(Ordering::Acquire) > 0 { let mut triggered = false; for child in inner.children.iter() { @@ -261,7 +283,7 @@ impl WasiProcess { } } } - let inner = self.inner.read().unwrap(); + let inner = self.inner.0.lock().unwrap(); for thread in inner.threads.values() { thread.signal(signal); } @@ -269,7 +291,7 @@ impl WasiProcess { /// Signals one of the threads every interval pub fn signal_interval(&self, signal: Signal, interval: Option, repeat: bool) { - let mut inner = self.inner.write().unwrap(); + let mut inner = self.inner.0.lock().unwrap(); let interval = match interval { None => { @@ -293,10 +315,101 @@ impl WasiProcess { /// Returns the number of active threads for this process pub fn active_threads(&self) -> u32 { - let inner = self.inner.read().unwrap(); + let inner = self.inner.0.lock().unwrap(); inner.thread_count } + /// Checkpoints the process which will cause all other threads to + /// pause and for the thread and memory state to be saved + pub fn checkpoint( + &self, + ctx: FunctionEnvMut<'_, WasiEnv>, + for_what: WasiProcessCheckpoint, + ) -> Result, WasiError> { + // Set the checkpoint flag and then enter the normal processing loop + { + let mut inner = self.inner.0.lock().unwrap(); + inner.checkpoint = for_what; + } + + self.maybe_checkpoint::(ctx) + } + + /// If a checkpoint has been started this will block the current process + /// until the checkpoint operation has completed + pub fn maybe_checkpoint( + &self, + ctx: FunctionEnvMut<'_, WasiEnv>, + ) -> Result, WasiError> { + // Enter the lock which will determine if we are in a checkpoint or not + let inner = self.inner.clone(); + let guard = inner.0.lock().unwrap(); + if guard.checkpoint == WasiProcessCheckpoint::Execute { + // No checkpoint so just carry on + return Ok(Ok(())); + } + trace!("checkpoint capture"); + drop(guard); + + // Perform the unwind action + unwind::(ctx, move |mut ctx, memory_stack, rewind_stack| { + // Write our thread state to the snapshot + let tid = ctx.data().thread.tid(); + if let Err(err) = SnapshotEffector::write_thread_state( + &mut ctx, + tid, + memory_stack.freeze(), + rewind_stack.freeze(), + ) { + return wasmer_types::OnCalledAction::Trap(Box::new(err)); + } + + let mut guard = inner.0.lock().unwrap(); + + // Wait for the checkpoint to finish (or if we are the last thread + // to freeze then we have to execute the checksum operation) + loop { + if WasiProcessCheckpoint::Execute == guard.checkpoint { + ctx.data().thread.set_check_pointing(false); + trace!("checkpoint finished"); + break; + } + + ctx.data().thread.set_check_pointing(true); + + match guard.checkpoint { + WasiProcessCheckpoint::Execute => {} + WasiProcessCheckpoint::Snapshot => { + // Now if we are the last thread we also write the memory + let is_last_thread = + guard.threads.values().all(WasiThread::is_check_pointing); + if is_last_thread { + if let Err(err) = + SnapshotEffector::write_memory_and_snapshot(&mut ctx, &mut guard) + { + inner.1.notify_all(); + return wasmer_types::OnCalledAction::Trap(Box::new(err)); + } + + // Clear the checkpointing flag and notify everyone to wake up + ctx.data().thread.set_check_pointing(false); + guard.checkpoint = WasiProcessCheckpoint::Execute; + trace!("checkpoint complete"); + inner.1.notify_all(); + } else { + guard = inner.1.wait(guard).unwrap(); + } + } + } + } + + // Resume execution + wasmer_types::OnCalledAction::InvokeAgain + })?; + + Ok(Ok(())) + } + /// Waits until the process is finished. pub async fn join(&self) -> Result> { let _guard = WasiProcessWait::new(self); @@ -312,7 +425,7 @@ impl WasiProcess { pub async fn join_children(&mut self) -> Option>> { let _guard = WasiProcessWait::new(self); let children: Vec<_> = { - let inner = self.inner.read().unwrap(); + let inner = self.inner.0.lock().unwrap(); inner.children.clone() }; if children.is_empty() { @@ -324,7 +437,7 @@ impl WasiProcess { let inner = self.inner.clone(); waits.push(async move { let join = process.join().await; - let mut inner = inner.write().unwrap(); + let mut inner = inner.0.lock().unwrap(); inner.children.retain(|a| a.pid != child.pid); join }) @@ -340,7 +453,7 @@ impl WasiProcess { pub async fn join_any_child(&mut self) -> Result, Errno> { let _guard = WasiProcessWait::new(self); let children: Vec<_> = { - let inner = self.inner.read().unwrap(); + let inner = self.inner.0.lock().unwrap(); inner.children.clone() }; if children.is_empty() { @@ -353,7 +466,7 @@ impl WasiProcess { let inner = self.inner.clone(); waits.push(async move { let join = process.join().await; - let mut inner = inner.write().unwrap(); + let mut inner = inner.0.lock().unwrap(); inner.children.retain(|a| a.pid != child.pid); (child, join) }) @@ -373,7 +486,7 @@ impl WasiProcess { pub fn terminate(&self, exit_code: ExitCode) { // FIXME: this is wrong, threads might still be running! // Need special logic for the main thread. - let guard = self.inner.read().unwrap(); + let guard = self.inner.0.lock().unwrap(); for thread in guard.threads.values() { thread.set_status_finished(Ok(exit_code)) } diff --git a/lib/wasix/src/os/task/thread.rs b/lib/wasix/src/os/task/thread.rs index 8dbd83b6fb8..ce36f02021e 100644 --- a/lib/wasix/src/os/task/thread.rs +++ b/lib/wasix/src/os/task/thread.rs @@ -2,7 +2,10 @@ use serde::{Deserialize, Serialize}; use std::{ collections::HashMap, ops::{Deref, DerefMut}, - sync::{Arc, Mutex, RwLock, Weak}, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, Condvar, Mutex, Weak, + }, task::Waker, }; @@ -96,6 +99,7 @@ pub struct ThreadStack { #[derive(Clone, Debug)] pub struct WasiThread { state: Arc, + layout: WasiMemoryLayout, // This is used for stack rewinds rewind: Option, @@ -111,6 +115,28 @@ impl WasiThread { pub(crate) fn take_rewind(&mut self) -> Option { self.rewind.take() } + + /// Sets a flag that tells others that this thread is currently + /// check pointing itself + pub(crate) fn set_check_pointing(&self, val: bool) { + self.state.check_pointing.store(val, Ordering::SeqCst); + } + + /// Reads a flag that determines if this thread is currently + /// check pointing itself or not + pub(crate) fn is_check_pointing(&self) -> bool { + self.state.check_pointing.load(Ordering::SeqCst) + } + + /// Gets the memory layout for this thread + pub(crate) fn memory_layout(&self) -> &WasiMemoryLayout { + &self.layout + } + + /// Gets the memory layout for this thread + pub(crate) fn set_memory_layout(&mut self, layout: WasiMemoryLayout) { + self.layout = layout; + } } /// A guard that ensures a thread is marked as terminated when dropped. @@ -171,6 +197,7 @@ struct WasiThreadState { signals: Mutex<(Vec, Vec)>, stack: Mutex, status: Arc, + check_pointing: AtomicBool, // Registers the task termination with the ControlPlane on drop. // Never accessed, since it's a drop guard. @@ -186,6 +213,7 @@ impl WasiThread { is_main: bool, status: Arc, guard: TaskCountGuard, + layout: WasiMemoryLayout, ) -> Self { Self { state: Arc::new(WasiThreadState { @@ -195,8 +223,10 @@ impl WasiThread { status, signals: Mutex::new((Vec::new(), Vec::new())), stack: Mutex::new(ThreadStack::default()), + check_pointing: AtomicBool::new(false), _task_count_guard: guard, }), + layout, rewind: None, } } @@ -446,7 +476,7 @@ impl WasiThread { #[derive(Debug)] pub struct WasiThreadHandleProtected { thread: WasiThread, - inner: Weak>, + inner: Weak<(Mutex, Condvar)>, } #[derive(Debug, Clone)] @@ -457,7 +487,7 @@ pub struct WasiThreadHandle { impl WasiThreadHandle { pub(crate) fn new( thread: WasiThread, - inner: &Arc>, + inner: &Arc<(Mutex, Condvar)>, ) -> WasiThreadHandle { Self { protected: Arc::new(WasiThreadHandleProtected { @@ -480,7 +510,7 @@ impl Drop for WasiThreadHandleProtected { fn drop(&mut self) { let id = self.thread.tid(); if let Some(inner) = Weak::upgrade(&self.inner) { - let mut inner = inner.write().unwrap(); + let mut inner = inner.0.lock().unwrap(); if let Some(ctrl) = inner.threads.remove(&id) { ctrl.set_status_finished(Ok(Errno::Success.into())); } diff --git a/lib/wasix/src/snapshot/effector.rs b/lib/wasix/src/snapshot/effector.rs new file mode 100644 index 00000000000..95f8eb622ae --- /dev/null +++ b/lib/wasix/src/snapshot/effector.rs @@ -0,0 +1,143 @@ +use std::{borrow::Cow, collections::LinkedList, ops::Range, sync::MutexGuard}; + +use bytes::Bytes; +use wasmer::{FunctionEnvMut, WasmPtr}; +use wasmer_types::MemorySize; +use wasmer_wasix_types::{types::__wasi_ciovec_t, wasi::Errno}; + +use crate::{ + mem_error_to_wasi, os::task::process::WasiProcessInner, syscalls::__asyncify_light, + utils::map_snapshot_err, WasiEnv, WasiError, WasiThreadId, +}; + +use super::*; + +#[derive(Debug, Clone)] +pub struct SnapshotEffector {} + +impl SnapshotEffector { + pub fn write_terminal_data( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + iovs: WasmPtr<__wasi_ciovec_t, M>, + iovs_len: M::Offset, + ) -> Result, WasiError> { + let env = ctx.data(); + let memory = unsafe { env.memory_view(&ctx) }; + let iovs_arr = wasi_try_mem_ok_ok!(iovs.slice(&memory, iovs_len)); + + wasi_try_ok_ok!(__asyncify_light(env, None, async { + let iovs_arr = iovs_arr.access().map_err(mem_error_to_wasi)?; + for iovs in iovs_arr.iter() { + let buf = WasmPtr::::new(iovs.buf) + .slice(&memory, iovs.buf_len) + .map_err(mem_error_to_wasi)? + .access() + .map_err(mem_error_to_wasi)?; + ctx.data() + .runtime() + .snap_shooter() + .write(SnapshotLog::TerminalData { + data: Cow::Borrowed(buf.as_ref()), + }) + .await + .map_err(map_snapshot_err)?; + } + Ok(()) + })?); + Ok(Ok(())) + } + + pub fn write_thread_state( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + id: WasiThreadId, + memory_stack: Bytes, + rewind_stack: Bytes, + ) -> Result, WasiError> { + let env = ctx.data(); + wasi_try_ok_ok!(__asyncify_light(env, None, async { + ctx.data() + .runtime() + .snap_shooter() + .write(SnapshotLog::SetThread { + id, + call_stack: Cow::Owned(rewind_stack.into()), + memory_stack: Cow::Owned(memory_stack.into()), + }) + .await + .map_err(map_snapshot_err)?; + Ok(()) + })?); + Ok(Ok(())) + } + + pub fn write_memory_and_snapshot( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + process: &mut MutexGuard<'_, WasiProcessInner>, + ) -> Result, WasiError> { + let env = ctx.data(); + let memory = unsafe { env.memory_view(ctx) }; + + // Compute all the regions that we need to save which is basically + // everything in the memory except for the memory stacks. + // + // We do not want the regions to be greater than 128KB as this will + // otherwise create too much inefficiency. + let mut cur = 0u64; + let mut regions = LinkedList::>::new(); + while cur < memory.data_size() { + let mut again = false; + let mut end = memory.data_size().min(cur + 131_072); + for (_, thread) in process.threads.iter() { + let layout = thread.memory_layout(); + if cur >= layout.stack_lower && cur < layout.stack_upper { + cur = layout.stack_upper; + again = true; + break; + } + if end > layout.stack_lower { + end = end.min(layout.stack_lower); + } + } + if again { + continue; + } + regions.push_back(cur..end); + cur = end; + } + + // Now that we known all the regions that need to be saved we + // enter a processing loop that dumps all the data to the log + // file in an orderly manner. + wasi_try_ok_ok!(__asyncify_light(env, None, async { + let memory = unsafe { env.memory_view(ctx) }; + let shooter = ctx.data().runtime().snap_shooter(); + + for region in regions { + // We grab this region of memory as a vector and hash + // it, which allows us to make some logging efficiency + // gains. + let data = memory + .copy_range_to_vec(region.clone()) + .map_err(mem_error_to_wasi)?; + + // Now we write it to the snap shooter + shooter + .write(SnapshotLog::UpdateMemoryRegion { + region, + data: data.into(), + }) + .await + .map_err(map_snapshot_err)?; + } + + // Finally we mark the end of the snapshot so that + // it can act as a restoration point + shooter + .write(SnapshotLog::Snapshot) + .await + .map_err(map_snapshot_err)?; + Ok(()) + })?); + Ok(Ok(())) + } +} diff --git a/lib/wasix/src/snapshot/mod.rs b/lib/wasix/src/snapshot/mod.rs index 6c7ab7a969b..302a3fb1002 100644 --- a/lib/wasix/src/snapshot/mod.rs +++ b/lib/wasix/src/snapshot/mod.rs @@ -1,7 +1,9 @@ +mod effector; mod log_file; mod shooter; mod unsupported; +pub use effector::*; pub use log_file::*; pub use shooter::*; pub use unsupported::*; diff --git a/lib/wasix/src/state/env.rs b/lib/wasix/src/state/env.rs index 534aaefcdd6..28b18169124 100644 --- a/lib/wasix/src/state/env.rs +++ b/lib/wasix/src/state/env.rs @@ -345,7 +345,7 @@ impl WasiEnv { /// Forking the WasiState is used when either fork or vfork is called pub fn fork(&self) -> Result<(Self, WasiThreadHandle), ControlPlaneError> { let process = self.control_plane.new_process()?; - let handle = process.new_thread()?; + let handle = process.new_thread(self.layout.clone())?; let thread = handle.as_thread(); thread.copy_stack_from(&self.thread); @@ -410,17 +410,19 @@ impl WasiEnv { } else { init.control_plane.new_process()? }; + + let layout = WasiMemoryLayout::default(); let thread = if let Some(t) = init.thread { t } else { - process.new_thread()? + process.new_thread(layout.clone())? }; let mut env = Self { control_plane: init.control_plane, process, thread: thread.as_thread(), - layout: WasiMemoryLayout::default(), + layout, vfork: None, poll_seed: 0, state: Arc::new(init.state), @@ -637,29 +639,28 @@ impl WasiEnv { if let Some(handler) = inner.signal.clone() { // We might also have signals that trigger on timers let mut now = 0; - let has_signal_interval = { - let mut any = false; - let inner = env.process.inner.read().unwrap(); + { + let mut has_signal_interval = false; + let inner = env.process.inner.0.lock().unwrap(); if !inner.signal_intervals.is_empty() { now = platform_clock_time_get(Snapshot0Clockid::Monotonic, 1_000_000).unwrap() as u128; for signal in inner.signal_intervals.values() { let elapsed = now - signal.last_signal; if elapsed >= signal.interval.as_nanos() { - any = true; + has_signal_interval = true; break; } } } - any - }; - if has_signal_interval { - let mut inner = env.process.inner.write().unwrap(); - for signal in inner.signal_intervals.values_mut() { - let elapsed = now - signal.last_signal; - if elapsed >= signal.interval.as_nanos() { - signal.last_signal = now; - signals.push(signal.signal); + if has_signal_interval { + let mut inner = env.process.inner.0.lock().unwrap(); + for signal in inner.signal_intervals.values_mut() { + let elapsed = now - signal.last_signal; + if elapsed >= signal.interval.as_nanos() { + signal.last_signal = now; + signals.push(signal.signal); + } } } } diff --git a/lib/wasix/src/state/func_env.rs b/lib/wasix/src/state/func_env.rs index 02cee353b42..9a92732db6f 100644 --- a/lib/wasix/src/state/func_env.rs +++ b/lib/wasix/src/state/func_env.rs @@ -170,9 +170,23 @@ impl WasiFunctionEnv { // Update the stack layout which is need for asyncify let env = self.data_mut(store); + let tid = env.tid(); let layout = &mut env.layout; layout.stack_upper = stack_base; layout.stack_size = layout.stack_upper - layout.stack_lower; + + // Replace the thread object itself + env.thread.set_memory_layout(layout.clone()); + + // Replace the thread object with this new layout + { + let mut guard = env.process.lock(); + guard + .threads + .values_mut() + .filter(|t| t.tid() == tid) + .for_each(|t| t.set_memory_layout(layout.clone())) + } } tracing::trace!("initializing with layout {:?}", self.data(store).layout); diff --git a/lib/wasix/src/syscalls/wasi/fd_write.rs b/lib/wasix/src/syscalls/wasi/fd_write.rs index 896e1209066..410e48372f8 100644 --- a/lib/wasix/src/syscalls/wasi/fd_write.rs +++ b/lib/wasix/src/syscalls/wasi/fd_write.rs @@ -1,7 +1,12 @@ use std::task::Waker; use super::*; -use crate::{net::socket::TimeType, snapshot::SnapshotLog, syscalls::*, utils::map_snapshot_err}; +use crate::{ + net::socket::TimeType, + snapshot::{SnapshotEffector, SnapshotLog}, + syscalls::*, + utils::map_snapshot_err, +}; /// ### `fd_write()` /// Write data to the file descriptor @@ -90,8 +95,6 @@ pub(crate) fn fd_write_internal( let mut env = ctx.data(); let state = env.state.clone(); - let mut memory = unsafe { env.memory_view(&ctx) }; - let iovs_arr = wasi_try_mem_ok!(iovs.slice(&memory, iovs_len)); let fd_entry = wasi_try_ok!(state.fs.get_fd(fd)); let is_stdio = fd_entry.is_stdio; @@ -105,28 +108,13 @@ pub(crate) fn fd_write_internal( // will record a terminal event. #[cfg(feature = "snapshooter")] if is_stdio && ctx.data().enable_snapshot_feed { - __asyncify_light(env, None, async { - let iovs_arr = iovs_arr.access().map_err(mem_error_to_wasi)?; - for iovs in iovs_arr.iter() { - let buf = WasmPtr::::new(iovs.buf) - .slice(&memory, iovs.buf_len) - .map_err(mem_error_to_wasi)? - .access() - .map_err(mem_error_to_wasi)?; - ctx.data() - .runtime() - .snap_shooter() - .write(SnapshotLog::TerminalData { - data: Cow::Borrowed(buf.as_ref()), - }) - .await - .map_err(map_snapshot_err)?; - } - Ok(()) - }); + SnapshotEffector::write_terminal_data(&mut ctx, iovs, iovs_len)?; + env = ctx.data(); } let fd_flags = fd_entry.flags; + let mut memory = unsafe { env.memory_view(&ctx) }; + let iovs_arr = wasi_try_mem_ok!(iovs.slice(&memory, iovs_len)); let (bytes_written, can_update_cursor) = { let iovs_arr = wasi_try_mem_ok!(iovs_arr.access()); diff --git a/lib/wasix/src/syscalls/wasix/proc_fork.rs b/lib/wasix/src/syscalls/wasix/proc_fork.rs index 8a2571f73ac..3a660dcccda 100644 --- a/lib/wasix/src/syscalls/wasix/proc_fork.rs +++ b/lib/wasix/src/syscalls/wasix/proc_fork.rs @@ -62,7 +62,7 @@ pub fn proc_fork( // We write a zero to the PID before we capture the stack // so that this is what will be returned to the child { - let mut inner = ctx.data().process.inner.write().unwrap(); + let mut inner = ctx.data().process.lock(); inner.children.push(child_env.process.clone()); } let env = ctx.data(); diff --git a/lib/wasix/src/syscalls/wasix/proc_join.rs b/lib/wasix/src/syscalls/wasix/proc_join.rs index b544afab3ca..bb56f2d4710 100644 --- a/lib/wasix/src/syscalls/wasix/proc_join.rs +++ b/lib/wasix/src/syscalls/wasix/proc_join.rs @@ -155,7 +155,7 @@ pub(super) fn proc_join_internal( // Waiting for a process that is an explicit child will join it // meaning it will no longer be a sub-process of the main process let mut process = { - let mut inner = ctx.data().process.inner.write().unwrap(); + let mut inner = ctx.data().process.lock(); let process = inner .children .iter() diff --git a/lib/wasix/src/syscalls/wasix/proc_spawn.rs b/lib/wasix/src/syscalls/wasix/proc_spawn.rs index 2646878bf01..a98775fa396 100644 --- a/lib/wasix/src/syscalls/wasix/proc_spawn.rs +++ b/lib/wasix/src/syscalls/wasix/proc_spawn.rs @@ -245,7 +245,7 @@ pub fn proc_spawn_internal( // Add the process to the environment state { - let mut inner = ctx.data().process.inner.write().unwrap(); + let mut inner = ctx.data().process.lock(); inner.children.push(child_process); } let env = ctx.data(); diff --git a/lib/wasix/src/syscalls/wasix/thread_spawn.rs b/lib/wasix/src/syscalls/wasix/thread_spawn.rs index 67e9f30f6e2..0cd4a00d689 100644 --- a/lib/wasix/src/syscalls/wasix/thread_spawn.rs +++ b/lib/wasix/src/syscalls/wasix/thread_spawn.rs @@ -73,7 +73,7 @@ pub(crate) fn thread_spawn_internal( tracing::trace!("spawn with layout {:?}", layout); // Create the handle that represents this thread - let mut thread_handle = match env.process.new_thread() { + let mut thread_handle = match env.process.new_thread(layout.clone()) { Ok(h) => Arc::new(h), Err(err) => { error!( From 8f11ac78b00d3f07b774e776d20e54221dc9e33f Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Mon, 23 Oct 2023 07:27:12 +1100 Subject: [PATCH 007/129] Addressed comments and added a basic doc --- docs/snapshots.md | 78 ++++++++++++++ lib/cli/Cargo.toml | 4 +- lib/cli/src/commands/run/wasi.rs | 29 ++--- lib/wasix/Cargo.toml | 101 ++++++++++++++---- lib/wasix/src/lib.rs | 2 +- lib/wasix/src/runtime/mod.rs | 31 +++--- .../src/snapshot/{shooter.rs => capturer.rs} | 6 +- lib/wasix/src/snapshot/effector.rs | 12 +-- lib/wasix/src/snapshot/log_file.rs | 10 +- lib/wasix/src/snapshot/mod.rs | 4 +- lib/wasix/src/snapshot/unsupported.rs | 9 +- lib/wasix/src/syscalls/wasi/fd_write.rs | 2 +- 12 files changed, 218 insertions(+), 70 deletions(-) create mode 100644 docs/snapshots.md rename lib/wasix/src/snapshot/{shooter.rs => capturer.rs} (91%) diff --git a/docs/snapshots.md b/docs/snapshots.md new file mode 100644 index 00000000000..15669370035 --- /dev/null +++ b/docs/snapshots.md @@ -0,0 +1,78 @@ +# WASM Snapshot Functionality + +Wasmer now supports snapshots of the current running process into a journal +log file which allows for the resumption from an earlier point in time. + +# Triggers + +Various triggers are possible that will cause a snapshot to be taken at +a specific point in time, these are: + +## On Idle + +Triggered when all the threads in the process goes idle. + +## On Listen + +Triggered when a listen syscall is invoked on a socket. + +## On Stdin + +Triggered when the process reads stdin for the first time + +## On Timer + +Triggered periodically based on a timer (default 10 seconds) which can be specified using the `snapshot-timer` option + +## On Sigint (Ctrl+C) + +Issued if the user sends an interrupt signal (Ctrl + C). + +## On Sigalrm + +Alarm clock signal (used for timers) +(see `man alarm`) + +## On Sigtstp + +The SIGTSTP signal is sent to a process by its controlling terminal to request it to stop temporarily. It is commonly initiated by the user pressing Ctrl-Z. + +# On Sigstop + +The SIGSTOP signal instructs the operating system to stop a process for later resumption + +# On Non Deterministic Call + +When a non-determinstic call is made from WASM + +# Limitations + +- The WASM process must have had the `asyncify` post processing step applied to the binary. +- Taking a snapshot can consume large amounts of memory while its processing. +- Snapshots are not instant and have overhead when generating. +- The layout of the memory must be known by the runtime in order to take snapshots. + +# Design + +On startup if the restore snapshot file is specified then the runtime will restore the +state of the WASM process by reading and processing the log entries in the snapshot +journal. This restoration will bring the memory and the thread stacks back to a previous +point in time and then resume all the threads. + +When a trigger occurs a new snapshot will be taken of the WASM process which will +take the following steps: + +1. Pause all threads +2. Capture the stack of each thread +3. Write the thread state to the journal +4. Write the memory (excluding stacks) to the journal +5. Resume execution. + +The implementation is currently able to save and restore the following: + +- WASM Memory +- Stack memory +- Call stack +- Open sockets +- Open files +- Terminal text diff --git a/lib/cli/Cargo.toml b/lib/cli/Cargo.toml index 38cbd26c80c..9a1d1d97385 100644 --- a/lib/cli/Cargo.toml +++ b/lib/cli/Cargo.toml @@ -160,11 +160,11 @@ default = [ "wat", "wast", "compiler", - "snapshooter", + "snapshot", "wasmer-artifact-create", "static-artifact-create", ] -snapshooter = ["wasmer-wasix/snapshooter"] +snapshot = ["wasmer-wasix/snapshot"] backend = [] coredump = ["wasm-coredump-builder"] sys = ["compiler", "wasmer-vm"] diff --git a/lib/cli/src/commands/run/wasi.rs b/lib/cli/src/commands/run/wasi.rs index c20166a5440..769b892e54a 100644 --- a/lib/cli/src/commands/run/wasi.rs +++ b/lib/cli/src/commands/run/wasi.rs @@ -109,7 +109,7 @@ pub struct Wasi { /// Specifies the snapshot file that Wasmer will use to store /// the state of the WASM process so that it can be later restored - #[cfg(feature = "snapshooter")] + #[cfg(feature = "snapshot")] #[clap(long = "snapshot-to")] pub snapshot_to: Option, @@ -119,19 +119,20 @@ pub struct Wasi { /// If not specified, the default is to snapshot on idle plus if a /// snapshot period is provided it will also default to periodic snapshots /// as well. - #[cfg(feature = "snapshooter")] + #[cfg(feature = "snapshot")] #[clap(long = "snapshot-on")] pub snapshot_on: Vec, - /// Time in seconds between taking snapshots of the process and dumping - /// them to the snapshot file. - #[cfg(feature = "snapshooter")] - #[clap(long = "snapshot-period")] - pub snapshot_period: Option, + /// Adds a timer (measured in seconds) that takes snapshots of the process and dumps the + /// journal of events to the snapshot file. When specifying this parameter it implies + /// that `--snapshot-on timer` has also been specified. + #[cfg(feature = "snapshot")] + #[clap(long = "snapshot-timer")] + pub snapshot_timer: Option, /// When specified, the runtime will restore a previous snapshot /// using the supplied file. - #[cfg(feature = "snapshooter")] + #[cfg(feature = "snapshot")] #[clap(long = "resume-from")] pub resume_from: Option, @@ -156,8 +157,8 @@ pub enum SnapshotTrigger { Listen, /// Triggered when the process reads stdin for the first time Stdin, - /// Triggered periodically (default 10 seconds) which can be specified using the `snapshot-period` option - Periodic, + /// Triggered periodically based on a timer (default 10 seconds) which can be specified using the `snapshot-timer` option + Timer, /// Issued if the user sends an interrupt signal (Ctrl + C). Sigint, /// Alarm clock signal (used for timers) @@ -166,6 +167,8 @@ pub enum SnapshotTrigger { Sigtstp, /// The SIGSTOP signal instructs the operating system to stop a process for later resumption. Sigstop, + /// When a non-determinstic call is made + NonDeterministicCall, } impl FromStr for SnapshotTrigger { @@ -177,7 +180,7 @@ impl FromStr for SnapshotTrigger { "idle" => Self::Idle, "listen" => Self::Listen, "stdin" => Self::Stdin, - "periodic" => Self::Periodic, + "periodic" => Self::Timer, "intr" | "sigint" | "ctrlc" | "ctrl-c" => Self::Sigint, "alarm" | "timer" | "sigalrm" => Self::Sigalrm, "sigtstp" | "ctrlz" | "ctrl-z" => Self::Sigtstp, @@ -338,9 +341,9 @@ impl Wasi { rt.set_networking_implementation(virtual_net::UnsupportedVirtualNetworking::default()); } - #[cfg(feature = "snapshooter")] + #[cfg(feature = "snapshot")] if let Some(path) = &self.resume_from { - rt.set_snapshooter(Arc::new(snapshot::LogFileSnapShooter::new_std(path)?)); + rt.set_snapshot_capturer(Arc::new(snapshot::LogFileSnapshotCapturer::new_std(path)?)); } if !self.no_tty { diff --git a/lib/wasix/Cargo.toml b/lib/wasix/Cargo.toml index 1f1bf65ee0b..3a69a4b7733 100644 --- a/lib/wasix/Cargo.toml +++ b/lib/wasix/Cargo.toml @@ -17,17 +17,28 @@ cfg-if = "1.0" thiserror = "1" tracing = { version = "0.1.37" } getrandom = "0.2" -wasmer-wasix-types = { path = "../wasi-types", version = "0.15.0", features = [ "enable-serde" ] } +wasmer-wasix-types = { path = "../wasi-types", version = "0.15.0", features = [ + "enable-serde", +] } wasmer-types = { path = "../types", version = "=4.2.2", default-features = false } -wasmer = { path = "../api", version = "=4.2.2", default-features = false, features = ["wat", "js-serializable-module"] } -virtual-mio = { path = "../virtual-io", version = "0.3.0", default-features = false } -virtual-fs = { path = "../virtual-fs", version = "0.9.0", default-features = false, features = ["webc-fs"] } +wasmer = { path = "../api", version = "=4.2.2", default-features = false, features = [ + "wat", + "js-serializable-module", +] } +virtual-mio = { path = "../virtual-io", version = "0.3.0", default-features = false } +virtual-fs = { path = "../virtual-fs", version = "0.9.0", default-features = false, features = [ + "webc-fs", +] } virtual-net = { path = "../virtual-net", version = "0.6.1", default-features = false } wasmer-emscripten = { path = "../emscripten", version = "=4.2.2", optional = true } typetag = { version = "0.1", optional = true } serde = { version = "1.0", default-features = false, features = ["derive"] } bincode = { version = "1.3" } -chrono = { version = "^0.4", default-features = false, features = [ "wasmbind", "std", "clock" ], optional = true } +chrono = { version = "^0.4", default-features = false, features = [ + "wasmbind", + "std", + "clock", +], optional = true } derivative = { version = "^2" } bytes = "1" webc = { workspace = true } @@ -38,7 +49,12 @@ sha2 = { version = "0.10" } waker-fn = { version = "1.1" } cooked-waker = "^5" rand = "0.8" -tokio = { version = "1", features = ["sync", "macros", "time", "rt"], default_features = false } +tokio = { version = "1", features = [ + "sync", + "macros", + "time", + "rt", +], default_features = false } futures = { version = "0.3" } # used by feature='os' async-trait = { version = "^0.1" } @@ -51,7 +67,9 @@ hex = { version = "^0.4" } term_size = { version = "0.3" } linked_hash_set = { version = "0.1" } http = "0.2.8" -wai-bindgen-wasmer = { path = "../wai-bindgen-wasmer", version = "0.15.0", features = ["tracing"] } +wai-bindgen-wasmer = { path = "../wai-bindgen-wasmer", version = "0.15.0", features = [ + "tracing", +] } heapless = "0.7.16" once_cell = "1.17.0" pin-project = "1.0.12" @@ -62,7 +80,12 @@ tempfile = "3.6.0" hyper = { version = "0.14", features = ["server", "stream"], optional = true } wcgi = { version = "0.1.2", optional = true } wcgi-host = { version = "0.1.2", optional = true } -tower-http = { version = "0.4.0", features = ["trace", "util", "catch-panic", "cors"], optional = true } +tower-http = { version = "0.4.0", features = [ + "trace", + "util", + "catch-panic", + "cors", +], optional = true } tower = { version = "0.4.13", features = ["make", "util"], optional = true } url = "2.3.1" petgraph = "0.6.3" @@ -70,7 +93,15 @@ rayon = { version = "1.7.0", optional = true } wasm-bindgen = { version = "0.2.87", optional = true } js-sys = { version = "0.3.64", optional = true } wasm-bindgen-futures = { version = "0.4.37", optional = true } -web-sys = { version = "0.3.64", features = ["Request", "RequestInit", "Window", "WorkerGlobalScope", "RequestMode", "Response", "Headers"], optional = true } +web-sys = { version = "0.3.64", features = [ + "Request", + "RequestInit", + "Window", + "WorkerGlobalScope", + "RequestMode", + "Response", + "Headers", +], optional = true } [target.'cfg(not(target_arch = "riscv64"))'.dependencies.reqwest] version = "0.11" @@ -94,8 +125,15 @@ termios = { version = "0.3" } winapi = "0.3" [dev-dependencies] -wasmer = { path = "../api", version = "=4.2.2", default-features = false, features = ["wat", "js-serializable-module"] } -tokio = { version = "1", features = [ "sync", "macros", "rt" ], default_features = false } +wasmer = { path = "../api", version = "=4.2.2", default-features = false, features = [ + "wat", + "js-serializable-module", +] } +tokio = { version = "1", features = [ + "sync", + "macros", + "rt", +], default_features = false } pretty_assertions = "1.3.0" wasm-bindgen-test = "0.3.0" @@ -105,7 +143,11 @@ tracing-wasm = "0.2" [target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies] tracing-subscriber = { version = "^0.3" } -wasmer = { path = "../api", version = "=4.2.2", default-features = false, features = ["wat", "js-serializable-module", "cranelift"] } +wasmer = { path = "../api", version = "=4.2.2", default-features = false, features = [ + "wat", + "js-serializable-module", + "cranelift", +] } [features] default = ["sys-default"] @@ -116,15 +158,32 @@ webc_runner_rt_wcgi = ["hyper", "wcgi", "wcgi-host", "tower", "tower-http"] webc_runner_rt_emscripten = ["wasmer-emscripten"] sys = ["webc/mmap", "time", "virtual-mio/sys"] -sys-default = ["sys", "logging", "host-fs", "sys-poll", "sys-thread", "host-vnet", "host-threads", "host-reqwest"] +sys-default = [ + "sys", + "logging", + "host-fs", + "sys-poll", + "sys-thread", + "host-vnet", + "host-threads", + "host-reqwest", +] sys-poll = [] sys-thread = ["tokio/rt", "tokio/time", "tokio/rt-multi-thread", "rayon"] -snapshooter = [] +snapshot = [] # Deprecated. Kept it for compatibility compiler = [] -js = ["virtual-fs/no-time", "getrandom/js", "chrono", "js-sys", "wasm-bindgen", "wasm-bindgen-futures", "web-sys"] +js = [ + "virtual-fs/no-time", + "getrandom/js", + "chrono", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] js-default = ["js"] test-js = ["js", "wasmer/wat"] @@ -136,11 +195,17 @@ remote-vnet = ["virtual-net/remote"] logging = ["tracing/log"] disable-all-logging = ["tracing/release_max_level_off", "tracing/max_level_off"] -enable-serde = ["typetag", "virtual-fs/enable-serde", "wasmer-wasix-types/enable-serde"] +enable-serde = [ + "typetag", + "virtual-fs/enable-serde", + "wasmer-wasix-types/enable-serde", +] [package.metadata.docs.rs] features = [ - "wasmer/sys", "webc_runner_rt_wcgi", - "webc_runner_rt_emscripten", "sys-default", + "wasmer/sys", + "webc_runner_rt_wcgi", + "webc_runner_rt_emscripten", + "sys-default", ] rustc-args = ["--cfg", "docsrs"] diff --git a/lib/wasix/src/lib.rs b/lib/wasix/src/lib.rs index 11453f8b5c9..6c2cec5bc94 100644 --- a/lib/wasix/src/lib.rs +++ b/lib/wasix/src/lib.rs @@ -50,7 +50,7 @@ pub mod http; mod rewind; pub mod runners; pub mod runtime; -#[cfg(feature = "snapshooter")] +#[cfg(feature = "snapshot")] pub mod snapshot; mod state; mod syscalls; diff --git a/lib/wasix/src/runtime/mod.rs b/lib/wasix/src/runtime/mod.rs index 74604ad1d53..fdfacabc787 100644 --- a/lib/wasix/src/runtime/mod.rs +++ b/lib/wasix/src/runtime/mod.rs @@ -8,7 +8,7 @@ use self::{ module_cache::{CacheError, ModuleHash}, task_manager::InlineWaker, }; -use crate::snapshot::UNSUPPORTED_SNAP_SHOOTER; +use crate::snapshot::UNSUPPORTED_SNAPSHOT_CAPTURER; use std::{ fmt, @@ -20,8 +20,8 @@ use futures::future::BoxFuture; use virtual_net::{DynVirtualNetworking, VirtualNetworking}; use wasmer::Module; -#[cfg(feature = "snapshooter")] -use crate::snapshot::{DynSnapShooter, UnsupportedSnapShooter}; +#[cfg(feature = "snapshot")] +use crate::snapshot::{DynSnapshotCapturer, UnsupportedSnapshotCapturer}; use crate::{ http::{DynHttpClient, HttpClient}, os::TtyBridge, @@ -109,11 +109,11 @@ where InlineWaker::block_on(self.load_module(wasm)) } - /// The snap shooter takes and restores snapshots of the WASM process at specific + /// The snapshot capturer takes and restores snapshots of the WASM process at specific /// points in time by reading and writing log entries - #[cfg(feature = "snapshooter")] - fn snap_shooter<'a>(&'a self) -> &'_ DynSnapShooter { - &UNSUPPORTED_SNAP_SHOOTER + #[cfg(feature = "snapshot")] + fn snapshot_capturer<'a>(&'a self) -> &'_ DynSnapshotCapturer { + &UNSUPPORTED_SNAPSHOT_CAPTURER } } @@ -190,9 +190,9 @@ pub struct PluggableRuntime { pub module_cache: Arc, #[derivative(Debug = "ignore")] pub tty: Option>, - #[cfg(feature = "snapshooter")] + #[cfg(feature = "snapshot")] #[derivative(Debug = "ignore")] - pub snapshooter: Arc, + pub snapshot_capturer: Arc, } impl PluggableRuntime { @@ -227,7 +227,8 @@ impl PluggableRuntime { source: Arc::new(source), package_loader: Arc::new(loader), module_cache: Arc::new(module_cache::in_memory()), - snapshooter: Arc::new(UnsupportedSnapShooter::default()) as Arc, + snapshot_capturer: Arc::new(UnsupportedSnapshotCapturer::default()) + as Arc, } } @@ -278,8 +279,8 @@ impl PluggableRuntime { self } - pub fn set_snapshooter(&mut self, snapshooter: Arc) -> &mut Self { - self.snapshooter = snapshooter; + pub fn set_snapshot_capturer(&mut self, capturer: Arc) -> &mut Self { + self.snapshot_capturer = capturer; self } } @@ -328,8 +329,8 @@ impl Runtime for PluggableRuntime { self.module_cache.clone() } - #[cfg(feature = "snapshooter")] - fn snap_shooter<'a>(&'a self) -> &DynSnapShooter { - self.snapshooter.as_ref() + #[cfg(feature = "snapshot")] + fn snapshot_capturer<'a>(&'a self) -> &DynSnapshotCapturer { + self.snapshot_capturer.as_ref() } } diff --git a/lib/wasix/src/snapshot/shooter.rs b/lib/wasix/src/snapshot/capturer.rs similarity index 91% rename from lib/wasix/src/snapshot/shooter.rs rename to lib/wasix/src/snapshot/capturer.rs index edad217d17d..2e4780590aa 100644 --- a/lib/wasix/src/snapshot/shooter.rs +++ b/lib/wasix/src/snapshot/capturer.rs @@ -66,11 +66,11 @@ pub enum SnapshotLog<'a> { Snapshot, } -/// The snap shooter will take a series of objects that represents the state of +/// The snapshot capturer will take a series of objects that represents the state of /// a WASM process at a point in time and saves it so that it can be restored. /// It also allows for the restoration of that state at a later moment #[allow(unused_variables)] -pub trait SnapShooter { +pub trait SnapshotCapturer { /// Takes in a stream of snapshot log entries and saves them so that they /// may be restored at a later moment fn write<'a>(&'a self, entry: SnapshotLog<'a>) -> BoxFuture<'a, anyhow::Result<()>>; @@ -80,4 +80,4 @@ pub trait SnapShooter { fn read<'a>(&'a self) -> BoxFuture<'a, anyhow::Result>>>; } -pub type DynSnapShooter = dyn SnapShooter + Send + Sync; +pub type DynSnapshotCapturer = dyn SnapshotCapturer + Send + Sync; diff --git a/lib/wasix/src/snapshot/effector.rs b/lib/wasix/src/snapshot/effector.rs index 95f8eb622ae..921bae12a6e 100644 --- a/lib/wasix/src/snapshot/effector.rs +++ b/lib/wasix/src/snapshot/effector.rs @@ -35,7 +35,7 @@ impl SnapshotEffector { .map_err(mem_error_to_wasi)?; ctx.data() .runtime() - .snap_shooter() + .snapshot_capturer() .write(SnapshotLog::TerminalData { data: Cow::Borrowed(buf.as_ref()), }) @@ -57,7 +57,7 @@ impl SnapshotEffector { wasi_try_ok_ok!(__asyncify_light(env, None, async { ctx.data() .runtime() - .snap_shooter() + .snapshot_capturer() .write(SnapshotLog::SetThread { id, call_stack: Cow::Owned(rewind_stack.into()), @@ -110,7 +110,7 @@ impl SnapshotEffector { // file in an orderly manner. wasi_try_ok_ok!(__asyncify_light(env, None, async { let memory = unsafe { env.memory_view(ctx) }; - let shooter = ctx.data().runtime().snap_shooter(); + let capturer = ctx.data().runtime().snapshot_capturer(); for region in regions { // We grab this region of memory as a vector and hash @@ -120,8 +120,8 @@ impl SnapshotEffector { .copy_range_to_vec(region.clone()) .map_err(mem_error_to_wasi)?; - // Now we write it to the snap shooter - shooter + // Now we write it to the snap snapshot capturer + capturer .write(SnapshotLog::UpdateMemoryRegion { region, data: data.into(), @@ -132,7 +132,7 @@ impl SnapshotEffector { // Finally we mark the end of the snapshot so that // it can act as a restoration point - shooter + capturer .write(SnapshotLog::Snapshot) .await .map_err(map_snapshot_err)?; diff --git a/lib/wasix/src/snapshot/log_file.rs b/lib/wasix/src/snapshot/log_file.rs index 0379c457775..96ea17d53fd 100644 --- a/lib/wasix/src/snapshot/log_file.rs +++ b/lib/wasix/src/snapshot/log_file.rs @@ -163,7 +163,7 @@ struct State { at_end: bool, } -/// The LogFile snap shooter will write its snapshots to a linear journal +/// The LogFile snapshot capturer will write its snapshots to a linear journal /// and read them when restoring. It uses the `bincode` serializer which /// means that forwards and backwards compatibility must be dealt with /// carefully. @@ -172,13 +172,13 @@ struct State { /// then new entries will be added to the end regardless of if /// its been read. /// -/// The logfile snapshooter uses a 64bit number as a entry encoding +/// The logfile snapshot capturer uses a 64bit number as a entry encoding /// delimiter. -pub struct LogFileSnapShooter { +pub struct LogFileSnapshotCapturer { state: tokio::sync::Mutex, } -impl LogFileSnapShooter { +impl LogFileSnapshotCapturer { pub async fn new(path: impl AsRef) -> io::Result { let state = State { file: tokio::fs::File::options() @@ -211,7 +211,7 @@ impl LogFileSnapShooter { } #[async_trait::async_trait] -impl SnapShooter for LogFileSnapShooter { +impl SnapshotCapturer for LogFileSnapshotCapturer { fn write<'a>(&'a self, entry: SnapshotLog<'a>) -> BoxFuture<'a, anyhow::Result<()>> { Box::pin(async { let entry: SnapshotLogEntry = entry.into(); diff --git a/lib/wasix/src/snapshot/mod.rs b/lib/wasix/src/snapshot/mod.rs index 302a3fb1002..c4a4b07cffa 100644 --- a/lib/wasix/src/snapshot/mod.rs +++ b/lib/wasix/src/snapshot/mod.rs @@ -1,9 +1,9 @@ +mod capturer; mod effector; mod log_file; -mod shooter; mod unsupported; +pub use capturer::*; pub use effector::*; pub use log_file::*; -pub use shooter::*; pub use unsupported::*; diff --git a/lib/wasix/src/snapshot/unsupported.rs b/lib/wasix/src/snapshot/unsupported.rs index 04febd99d7a..b11b4cdb15e 100644 --- a/lib/wasix/src/snapshot/unsupported.rs +++ b/lib/wasix/src/snapshot/unsupported.rs @@ -2,14 +2,15 @@ use futures::future::BoxFuture; use super::*; -pub static UNSUPPORTED_SNAP_SHOOTER: UnsupportedSnapShooter = UnsupportedSnapShooter {}; +pub static UNSUPPORTED_SNAPSHOT_CAPTURER: UnsupportedSnapshotCapturer = + UnsupportedSnapshotCapturer {}; -/// The default for runtime is to use the unsupported snap-shooter +/// The default for runtime is to use the unsupported snapshot capturer /// which will fail to snapshot if one attempts to do so. #[derive(Debug, Default)] -pub struct UnsupportedSnapShooter {} +pub struct UnsupportedSnapshotCapturer {} -impl SnapShooter for UnsupportedSnapShooter { +impl SnapshotCapturer for UnsupportedSnapshotCapturer { fn write<'a>(&'a self, _entry: SnapshotLog<'a>) -> BoxFuture<'a, anyhow::Result<()>> { Box::pin(async { Err(anyhow::format_err!("unsupported")) }) } diff --git a/lib/wasix/src/syscalls/wasi/fd_write.rs b/lib/wasix/src/syscalls/wasi/fd_write.rs index 410e48372f8..fac29a96fe0 100644 --- a/lib/wasix/src/syscalls/wasi/fd_write.rs +++ b/lib/wasix/src/syscalls/wasi/fd_write.rs @@ -106,7 +106,7 @@ pub(crate) fn fd_write_internal( // If snap-shooting is enabled and this is to stdio then we // will record a terminal event. - #[cfg(feature = "snapshooter")] + #[cfg(feature = "snapshot")] if is_stdio && ctx.data().enable_snapshot_feed { SnapshotEffector::write_terminal_data(&mut ctx, iovs, iovs_len)?; env = ctx.data(); From 99ecf2b46e8e515c88f2a8f9faf57089a06b4c25 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Mon, 23 Oct 2023 16:15:57 +1100 Subject: [PATCH 008/129] Added compactor and filter snapshot capturer --- docs/snapshots.md | 27 +++++ lib/wasix/src/snapshot/compactor.rs | 154 ++++++++++++++++++++++++++++ lib/wasix/src/snapshot/filter.rs | 152 +++++++++++++++++++++++++++ lib/wasix/src/snapshot/mod.rs | 4 + 4 files changed, 337 insertions(+) create mode 100644 lib/wasix/src/snapshot/compactor.rs create mode 100644 lib/wasix/src/snapshot/filter.rs diff --git a/docs/snapshots.md b/docs/snapshots.md index 15669370035..1df9566d479 100644 --- a/docs/snapshots.md +++ b/docs/snapshots.md @@ -76,3 +76,30 @@ The implementation is currently able to save and restore the following: - Open sockets - Open files - Terminal text + +# Capturer Implementations + +## Log File Capturer + +Writes the log events to a linear log file on the local file system +as they are received by the trait. Log files can be concatenated +together to make larger log files. + +## Unsupported Capturer + +The default implementation does not support snapshots and will error +out if an attempt is made to send it events. Using the unsupported +capturer as a restoration point will restore nothing but will not +error out. + +## Compacting Capturer + +Deduplicates memory and stacks to reduce the number of volume of +log events sent to its inner capturer. Compacting the events occurs +in line as the events are generated + +## Filtered Capturer + +Filters out a specific set of log events and drops the rest, this +capturer can be useful for restoring to a previous call point but +retaining the memory changes (e.g. WCGI runner). diff --git a/lib/wasix/src/snapshot/compactor.rs b/lib/wasix/src/snapshot/compactor.rs new file mode 100644 index 00000000000..9c5a3a640d1 --- /dev/null +++ b/lib/wasix/src/snapshot/compactor.rs @@ -0,0 +1,154 @@ +use std::{ + collections::{HashMap, HashSet}, + ops::Range, + sync::Mutex, +}; + +use futures::future::BoxFuture; +use sha2::{Digest, Sha256}; +use virtual_fs::Fd; + +use super::*; + +struct State { + memory_map: HashMap, [u8; 32]>, + open_file: HashMap, + close_file: HashSet, +} + +/// Deduplicates memory and stacks to reduce the number of volume of +/// log events sent to its inner capturer. Compacting the events occurs +/// in line as the events are generated +pub struct CompactingSnapshotCapturer { + inner: Box, + state: Mutex, +} + +impl CompactingSnapshotCapturer { + pub fn new(inner: Box) -> Self { + Self { + inner, + state: Mutex::new(State { + memory_map: Default::default(), + open_file: Default::default(), + close_file: Default::default(), + }), + } + } +} + +impl SnapshotCapturer for CompactingSnapshotCapturer { + fn write<'a>(&'a self, entry: SnapshotLog<'a>) -> BoxFuture<'a, anyhow::Result<()>> { + Box::pin(async { + match entry { + SnapshotLog::UpdateMemoryRegion { region, data } => { + let mut hasher = Sha256::default(); + hasher.update(data.as_ref()); + let hash: [u8; 32] = hasher.finalize().try_into().unwrap(); + + { + let mut state = self.state.lock().unwrap(); + if let Some(other) = state.memory_map.get_mut(®ion) { + if *other == hash { + return Ok(()); + } else { + *other = hash; + } + } else { + let to_remove = state + .memory_map + .keys() + .filter(|r| { + // Covers the whole range + ( + region.start <= r.start && + region.end >= r.end + ) || + // Clips the left side + ( + region.start <= r.start && + region.end > r.start + ) || + // Clips the right side + ( + region.start < r.end && + region.end >= r.end + ) + }) + .cloned() + .collect::>(); + for r in to_remove { + state.memory_map.remove(&r); + } + + state.memory_map.insert(region.clone(), hash); + } + } + return self + .inner + .write(SnapshotLog::UpdateMemoryRegion { region, data }) + .await; + } + SnapshotLog::CloseFileDescriptor { fd } => { + let mut state = self.state.lock().unwrap(); + state.open_file.remove(&fd); + state.close_file.insert(fd); + } + SnapshotLog::OpenFileDescriptor { + fd, + state: fd_state, + } => { + let mut state = self.state.lock().unwrap(); + state.close_file.remove(&fd); + state.open_file.insert(fd, fd_state); + } + SnapshotLog::Snapshot => { + let (to_close, to_open) = { + let mut state = self.state.lock().unwrap(); + ( + state.close_file.drain().collect::>(), + state.open_file.drain().collect::>(), + ) + }; + for fd in to_close { + self.inner + .write(SnapshotLog::CloseFileDescriptor { fd }) + .await?; + } + for (fd, fd_state) in to_open { + self.inner + .write(SnapshotLog::OpenFileDescriptor { + fd, + state: fd_state, + }) + .await?; + } + return self.inner.write(entry).await; + } + entry => { + return self.inner.write(entry).await; + } + } + Ok(()) + }) + } + + fn read<'a>(&'a self) -> BoxFuture<'a, anyhow::Result>>> { + Box::pin(async { + Ok(match self.inner.read().await? { + Some(SnapshotLog::UpdateMemoryRegion { region, data }) => { + let mut hasher = Sha256::default(); + hasher.update(data.as_ref()); + let hash: [u8; 32] = hasher.finalize().try_into().unwrap(); + + let mut state = self.state.lock().unwrap(); + state.memory_map.insert(region.clone(), hash); + + Some(SnapshotLog::UpdateMemoryRegion { region, data }) + } + Some(entry) => Some(entry), + None => None, + }) + }) + } +} diff --git a/lib/wasix/src/snapshot/filter.rs b/lib/wasix/src/snapshot/filter.rs new file mode 100644 index 00000000000..345c9bb37ed --- /dev/null +++ b/lib/wasix/src/snapshot/filter.rs @@ -0,0 +1,152 @@ +use futures::future::BoxFuture; + +use super::*; + +/// Filters out a specific set of log events and drops the rest, this +/// capturer can be useful for restoring to a previous call point but +/// retaining the memory changes (e.g. WCGI runner). +pub struct FilteredSnapshotCapturer { + inner: Box, + filter_memory: bool, + filter_threads: bool, + filter_files: bool, + filter_terminal: bool, + filter_snapshots: bool, + filter_descriptors: bool, +} + +impl FilteredSnapshotCapturer { + pub fn new(inner: Box) -> Self { + Self { + inner, + filter_memory: false, + filter_threads: false, + filter_files: false, + filter_terminal: false, + filter_snapshots: false, + filter_descriptors: false, + } + } + + pub fn with_ignore_memory(mut self, val: bool) -> Self { + self.filter_memory = val; + self + } + + pub fn with_ignore_threads(mut self, val: bool) -> Self { + self.filter_threads = val; + self + } + + pub fn with_ignore_files(mut self, val: bool) -> Self { + self.filter_files = val; + self + } + + pub fn with_ignore_terminal(mut self, val: bool) -> Self { + self.filter_terminal = val; + self + } + + pub fn with_ignore_snapshots(mut self, val: bool) -> Self { + self.filter_snapshots = val; + self + } + + pub fn with_ignore_descriptors(mut self, val: bool) -> Self { + self.filter_descriptors = val; + self + } +} + +impl SnapshotCapturer for FilteredSnapshotCapturer { + fn write<'a>(&'a self, entry: SnapshotLog<'a>) -> BoxFuture<'a, anyhow::Result<()>> { + Box::pin(async { + let evt = match entry { + SnapshotLog::TerminalData { data } => { + if self.filter_terminal { + return Ok(()); + } + SnapshotLog::TerminalData { data } + } + SnapshotLog::UpdateMemoryRegion { region, data } => { + if self.filter_memory { + return Ok(()); + } + SnapshotLog::UpdateMemoryRegion { region, data } + } + SnapshotLog::CloseThread { id } => { + if self.filter_threads { + return Ok(()); + } + SnapshotLog::CloseThread { id } + } + SnapshotLog::SetThread { + id, + call_stack, + memory_stack, + } => { + if self.filter_threads { + return Ok(()); + } + SnapshotLog::SetThread { + id, + call_stack, + memory_stack, + } + } + SnapshotLog::CloseFileDescriptor { fd } => { + if self.filter_descriptors { + return Ok(()); + } + SnapshotLog::CloseFileDescriptor { fd } + } + SnapshotLog::OpenFileDescriptor { fd, state } => { + if self.filter_descriptors { + return Ok(()); + } + SnapshotLog::OpenFileDescriptor { fd, state } + } + SnapshotLog::RemoveFileSystemEntry { path } => { + if self.filter_files { + return Ok(()); + } + SnapshotLog::RemoveFileSystemEntry { path } + } + SnapshotLog::UpdateFileSystemEntry { + path, + ft, + accessed, + created, + modified, + len, + data, + } => { + if self.filter_files { + return Ok(()); + } + SnapshotLog::UpdateFileSystemEntry { + path, + ft, + accessed, + created, + modified, + len, + data, + } + } + SnapshotLog::Snapshot => { + if self.filter_snapshots { + return Ok(()); + } + SnapshotLog::Snapshot + } + }; + self.inner.write(evt).await + }) + } + + fn read<'a>(&'a self) -> BoxFuture<'a, anyhow::Result>>> { + Box::pin(async { self.inner.read().await }) + } +} diff --git a/lib/wasix/src/snapshot/mod.rs b/lib/wasix/src/snapshot/mod.rs index c4a4b07cffa..4f09554e16f 100644 --- a/lib/wasix/src/snapshot/mod.rs +++ b/lib/wasix/src/snapshot/mod.rs @@ -1,9 +1,13 @@ mod capturer; +mod compactor; mod effector; +mod filter; mod log_file; mod unsupported; pub use capturer::*; +pub use compactor::*; pub use effector::*; +pub use filter::*; pub use log_file::*; pub use unsupported::*; From 2a9e1473f530631522d258c51e69193cd730e3b9 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Mon, 23 Oct 2023 16:25:03 +1100 Subject: [PATCH 009/129] Added missing snapshot-on type --- lib/cli/src/commands/run/wasi.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/cli/src/commands/run/wasi.rs b/lib/cli/src/commands/run/wasi.rs index 769b892e54a..f5ed9848f4d 100644 --- a/lib/cli/src/commands/run/wasi.rs +++ b/lib/cli/src/commands/run/wasi.rs @@ -185,6 +185,7 @@ impl FromStr for SnapshotTrigger { "alarm" | "timer" | "sigalrm" => Self::Sigalrm, "sigtstp" | "ctrlz" | "ctrl-z" => Self::Sigtstp, "stop" | "sigstop" => Self::Sigstop, + "non-deterministic-call" => Self::NonDeterministicCall, a => return Err(anyhow::format_err!("invalid or unknown trigger ({a})")), }) } From 85b12b1b26bd51d0df971f9cf3d13fcc1ecd7bce Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Tue, 24 Oct 2023 05:13:16 +1100 Subject: [PATCH 010/129] Added an init log entry which has the WASM hash --- lib/wasix/src/snapshot/capturer.rs | 3 +++ lib/wasix/src/snapshot/filter.rs | 1 + lib/wasix/src/snapshot/log_file.rs | 5 +++++ 3 files changed, 9 insertions(+) diff --git a/lib/wasix/src/snapshot/capturer.rs b/lib/wasix/src/snapshot/capturer.rs index 2e4780590aa..7a088132cc3 100644 --- a/lib/wasix/src/snapshot/capturer.rs +++ b/lib/wasix/src/snapshot/capturer.rs @@ -28,6 +28,9 @@ pub enum FileEntryType { /// Represents a log entry in a snapshot log stream that represents the total /// state of a WASM process at a point in time. pub enum SnapshotLog<'a> { + Init { + wasm_hash: [u8; 32], + }, TerminalData { data: Cow<'a, [u8]>, }, diff --git a/lib/wasix/src/snapshot/filter.rs b/lib/wasix/src/snapshot/filter.rs index 345c9bb37ed..2bc8a888fb8 100644 --- a/lib/wasix/src/snapshot/filter.rs +++ b/lib/wasix/src/snapshot/filter.rs @@ -63,6 +63,7 @@ impl SnapshotCapturer for FilteredSnapshotCapturer { fn write<'a>(&'a self, entry: SnapshotLog<'a>) -> BoxFuture<'a, anyhow::Result<()>> { Box::pin(async { let evt = match entry { + SnapshotLog::Init { wasm_hash } => SnapshotLog::Init { wasm_hash }, SnapshotLog::TerminalData { data } => { if self.filter_terminal { return Ok(()); diff --git a/lib/wasix/src/snapshot/log_file.rs b/lib/wasix/src/snapshot/log_file.rs index 96ea17d53fd..b62632d7ef3 100644 --- a/lib/wasix/src/snapshot/log_file.rs +++ b/lib/wasix/src/snapshot/log_file.rs @@ -20,6 +20,9 @@ use super::*; /// worry about backward and forward compatibility #[derive(Debug, Clone, Serialize, Deserialize)] pub enum SnapshotLogEntry { + Init { + wasm_hash: [u8; 32], + }, TerminalDataV1 { data: Vec, }, @@ -61,6 +64,7 @@ pub enum SnapshotLogEntry { impl<'a> From> for SnapshotLogEntry { fn from(value: SnapshotLog<'a>) -> Self { match value { + SnapshotLog::Init { wasm_hash } => Self::Init { wasm_hash }, SnapshotLog::TerminalData { data } => Self::TerminalDataV1 { data: data.into_owned(), }, @@ -111,6 +115,7 @@ impl<'a> From> for SnapshotLogEntry { impl<'a> From for SnapshotLog<'a> { fn from(value: SnapshotLogEntry) -> Self { match value { + SnapshotLogEntry::Init { wasm_hash } => Self::Init { wasm_hash }, SnapshotLogEntry::TerminalDataV1 { data } => Self::TerminalData { data: data.into() }, SnapshotLogEntry::UpdateMemoryRegionV1 { start, end, data } => { Self::UpdateMemoryRegion { From 1fe43038412019cead42d25397171d99edcd9e4a Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Tue, 24 Oct 2023 10:12:22 +1100 Subject: [PATCH 011/129] Added the remaining data structures for the snapshot journal --- lib/wasix/src/snapshot/capturer.rs | 86 ++++++++++++++++++++++++++--- lib/wasix/src/snapshot/compactor.rs | 6 +- lib/wasix/src/snapshot/effector.rs | 22 +++++++- lib/wasix/src/snapshot/filter.rs | 4 +- lib/wasix/src/snapshot/log_file.rs | 18 +++--- 5 files changed, 115 insertions(+), 21 deletions(-) diff --git a/lib/wasix/src/snapshot/capturer.rs b/lib/wasix/src/snapshot/capturer.rs index 7a088132cc3..06a9b11066a 100644 --- a/lib/wasix/src/snapshot/capturer.rs +++ b/lib/wasix/src/snapshot/capturer.rs @@ -1,4 +1,6 @@ use serde::{Deserialize, Serialize}; +use std::net::SocketAddr; +use std::time::SystemTime; use std::{borrow::Cow, ops::Range}; use futures::future::BoxFuture; @@ -7,11 +9,79 @@ use virtual_fs::Fd; use crate::WasiThreadId; #[derive(Debug, Clone, Serialize, Deserialize)] -pub enum FdSnapshot { - Stdin, - Stdout, - OpenFile, - Socket, +pub enum SocketSnapshot { + TcpListen { + listen_addr: SocketAddr, + reuse_port: bool, + reuse_addr: bool, + }, + TcpStream { + local_addr: SocketAddr, + peer_addr: SocketAddr, + }, + UdpSocket { + local_addr: SocketAddr, + peer_addr: SocketAddr, + reuse_port: bool, + reuse_addr: bool, + }, + Icmp { + addr: SocketAddr, + }, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum FdSnapshot<'a> { + Stdin { + non_blocking: bool, + }, + Stdout { + non_blocking: bool, + }, + Stderr { + non_blocking: bool, + }, + OpenFile { + path: Cow<'a, str>, + offset: u64, + read: bool, + write: bool, + non_blocking: bool, + }, + Socket { + state: SocketSnapshot, + non_blocking: bool, + }, +} + +impl<'a> FdSnapshot<'a> { + pub fn into_owned(self) -> FdSnapshot<'static> { + match self { + FdSnapshot::Stdin { non_blocking } => FdSnapshot::Stdin { non_blocking }, + FdSnapshot::Stdout { non_blocking } => FdSnapshot::Stdout { non_blocking }, + FdSnapshot::Stderr { non_blocking } => FdSnapshot::Stderr { non_blocking }, + FdSnapshot::OpenFile { + path, + offset, + read, + write, + non_blocking, + } => FdSnapshot::OpenFile { + path: Cow::Owned(path.into_owned()), + offset, + read, + write, + non_blocking, + }, + FdSnapshot::Socket { + state, + non_blocking, + } => FdSnapshot::Socket { + state, + non_blocking, + }, + } + } } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -51,7 +121,7 @@ pub enum SnapshotLog<'a> { }, OpenFileDescriptor { fd: Fd, - state: FdSnapshot, + state: FdSnapshot<'a>, }, RemoveFileSystemEntry { path: Cow<'a, str>, @@ -66,7 +136,9 @@ pub enum SnapshotLog<'a> { data: Cow<'a, [u8]>, }, /// Represents the marker for the end of a snapshot - Snapshot, + Snapshot { + when: SystemTime, + }, } /// The snapshot capturer will take a series of objects that represents the state of diff --git a/lib/wasix/src/snapshot/compactor.rs b/lib/wasix/src/snapshot/compactor.rs index 9c5a3a640d1..61ae74628b2 100644 --- a/lib/wasix/src/snapshot/compactor.rs +++ b/lib/wasix/src/snapshot/compactor.rs @@ -12,7 +12,7 @@ use super::*; struct State { memory_map: HashMap, [u8; 32]>, - open_file: HashMap, + open_file: HashMap>, close_file: HashSet, } @@ -100,9 +100,9 @@ impl SnapshotCapturer for CompactingSnapshotCapturer { } => { let mut state = self.state.lock().unwrap(); state.close_file.remove(&fd); - state.open_file.insert(fd, fd_state); + state.open_file.insert(fd, fd_state.into_owned()); } - SnapshotLog::Snapshot => { + SnapshotLog::Snapshot { .. } => { let (to_close, to_open) = { let mut state = self.state.lock().unwrap(); ( diff --git a/lib/wasix/src/snapshot/effector.rs b/lib/wasix/src/snapshot/effector.rs index 921bae12a6e..0e9eb4c70ff 100644 --- a/lib/wasix/src/snapshot/effector.rs +++ b/lib/wasix/src/snapshot/effector.rs @@ -1,4 +1,4 @@ -use std::{borrow::Cow, collections::LinkedList, ops::Range, sync::MutexGuard}; +use std::{borrow::Cow, collections::LinkedList, ops::Range, sync::MutexGuard, time::SystemTime}; use bytes::Bytes; use wasmer::{FunctionEnvMut, WasmPtr}; @@ -47,6 +47,23 @@ impl SnapshotEffector { Ok(Ok(())) } + pub fn write_thread_exit( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + id: WasiThreadId, + ) -> Result, WasiError> { + let env = ctx.data(); + wasi_try_ok_ok!(__asyncify_light(env, None, async { + ctx.data() + .runtime() + .snapshot_capturer() + .write(SnapshotLog::CloseThread { id }) + .await + .map_err(map_snapshot_err)?; + Ok(()) + })?); + Ok(Ok(())) + } + pub fn write_thread_state( ctx: &mut FunctionEnvMut<'_, WasiEnv>, id: WasiThreadId, @@ -132,8 +149,9 @@ impl SnapshotEffector { // Finally we mark the end of the snapshot so that // it can act as a restoration point + let when = SystemTime::now(); capturer - .write(SnapshotLog::Snapshot) + .write(SnapshotLog::Snapshot { when }) .await .map_err(map_snapshot_err)?; Ok(()) diff --git a/lib/wasix/src/snapshot/filter.rs b/lib/wasix/src/snapshot/filter.rs index 2bc8a888fb8..b3290aa7e72 100644 --- a/lib/wasix/src/snapshot/filter.rs +++ b/lib/wasix/src/snapshot/filter.rs @@ -136,11 +136,11 @@ impl SnapshotCapturer for FilteredSnapshotCapturer { data, } } - SnapshotLog::Snapshot => { + SnapshotLog::Snapshot { when } => { if self.filter_snapshots { return Ok(()); } - SnapshotLog::Snapshot + SnapshotLog::Snapshot { when } } }; self.inner.write(evt).await diff --git a/lib/wasix/src/snapshot/log_file.rs b/lib/wasix/src/snapshot/log_file.rs index b62632d7ef3..a4e8bce91a7 100644 --- a/lib/wasix/src/snapshot/log_file.rs +++ b/lib/wasix/src/snapshot/log_file.rs @@ -3,6 +3,7 @@ use std::{ io::{self, ErrorKind, SeekFrom}, mem::MaybeUninit, path::Path, + time::SystemTime, }; use futures::future::BoxFuture; @@ -44,7 +45,7 @@ pub enum SnapshotLogEntry { }, OpenFileDescriptorV1 { fd: Fd, - state: FdSnapshot, + state: FdSnapshot<'static>, }, RemoveFileSystemEntryV1 { path: String, @@ -58,7 +59,9 @@ pub enum SnapshotLogEntry { len: u64, data: Vec, }, - SnapshotV1, + SnapshotV1 { + when: SystemTime, + }, } impl<'a> From> for SnapshotLogEntry { @@ -84,9 +87,10 @@ impl<'a> From> for SnapshotLogEntry { memory_stack: memory_stack.into_owned(), }, SnapshotLog::CloseFileDescriptor { fd } => Self::CloseFileDescriptorV1 { fd }, - SnapshotLog::OpenFileDescriptor { fd, state } => { - Self::OpenFileDescriptorV1 { fd, state } - } + SnapshotLog::OpenFileDescriptor { fd, state } => Self::OpenFileDescriptorV1 { + fd, + state: state.into_owned(), + }, SnapshotLog::RemoveFileSystemEntry { path } => Self::RemoveFileSystemEntryV1 { path: path.into_owned(), }, @@ -107,7 +111,7 @@ impl<'a> From> for SnapshotLogEntry { len, data: data.into_owned(), }, - SnapshotLog::Snapshot => Self::SnapshotV1, + SnapshotLog::Snapshot { when } => Self::SnapshotV1 { when }, } } } @@ -158,7 +162,7 @@ impl<'a> From for SnapshotLog<'a> { len, data: data.into(), }, - SnapshotLogEntry::SnapshotV1 => Self::Snapshot, + SnapshotLogEntry::SnapshotV1 { when } => Self::Snapshot { when }, } } } From ebfe824f63533564267b660a1d51ad2450849cd7 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Tue, 24 Oct 2023 11:09:16 +1100 Subject: [PATCH 012/129] Cleaned up type complexity using WasiResult --- lib/wasix/src/lib.rs | 2 ++ lib/wasix/src/os/task/process.rs | 10 +++--- lib/wasix/src/snapshot/effector.rs | 41 ++++++++++++++++------ lib/wasix/src/state/env.rs | 10 ++---- lib/wasix/src/syscalls/mod.rs | 10 +++--- lib/wasix/src/syscalls/wasi/fd_read.rs | 2 +- lib/wasix/src/syscalls/wasi/fd_write.rs | 2 +- lib/wasix/src/syscalls/wasix/proc_spawn.rs | 2 +- lib/wasix/src/syscalls/wasix/sock_recv.rs | 2 +- 9 files changed, 49 insertions(+), 32 deletions(-) diff --git a/lib/wasix/src/lib.rs b/lib/wasix/src/lib.rs index 6c2cec5bc94..e5b9ac3bc49 100644 --- a/lib/wasix/src/lib.rs +++ b/lib/wasix/src/lib.rs @@ -122,6 +122,8 @@ pub enum WasiError { UnknownWasiVersion, } +pub type WasiResult = Result, WasiError>; + #[deny(unused, dead_code)] #[derive(Error, Debug)] pub enum SpawnError { diff --git a/lib/wasix/src/os/task/process.rs b/lib/wasix/src/os/task/process.rs index c7277944f9e..cfaf07f6b38 100644 --- a/lib/wasix/src/os/task/process.rs +++ b/lib/wasix/src/os/task/process.rs @@ -1,4 +1,4 @@ -use crate::{snapshot::SnapshotEffector, unwind, WasiEnv, WasiError, WasiRuntimeError}; +use crate::{snapshot::SnapshotEffector, unwind, WasiEnv, WasiResult, WasiRuntimeError}; use serde::{Deserialize, Serialize}; use std::{ collections::HashMap, @@ -325,7 +325,7 @@ impl WasiProcess { &self, ctx: FunctionEnvMut<'_, WasiEnv>, for_what: WasiProcessCheckpoint, - ) -> Result, WasiError> { + ) -> WasiResult<()> { // Set the checkpoint flag and then enter the normal processing loop { let mut inner = self.inner.0.lock().unwrap(); @@ -340,7 +340,7 @@ impl WasiProcess { pub fn maybe_checkpoint( &self, ctx: FunctionEnvMut<'_, WasiEnv>, - ) -> Result, WasiError> { + ) -> WasiResult<()> { // Enter the lock which will determine if we are in a checkpoint or not let inner = self.inner.clone(); let guard = inner.0.lock().unwrap(); @@ -355,7 +355,7 @@ impl WasiProcess { unwind::(ctx, move |mut ctx, memory_stack, rewind_stack| { // Write our thread state to the snapshot let tid = ctx.data().thread.tid(); - if let Err(err) = SnapshotEffector::write_thread_state( + if let Err(err) = SnapshotEffector::save_thread_state( &mut ctx, tid, memory_stack.freeze(), @@ -385,7 +385,7 @@ impl WasiProcess { guard.threads.values().all(WasiThread::is_check_pointing); if is_last_thread { if let Err(err) = - SnapshotEffector::write_memory_and_snapshot(&mut ctx, &mut guard) + SnapshotEffector::save_memory_and_snapshot(&mut ctx, &mut guard) { inner.1.notify_all(); return wasmer_types::OnCalledAction::Trap(Box::new(err)); diff --git a/lib/wasix/src/snapshot/effector.rs b/lib/wasix/src/snapshot/effector.rs index 0e9eb4c70ff..23815c54813 100644 --- a/lib/wasix/src/snapshot/effector.rs +++ b/lib/wasix/src/snapshot/effector.rs @@ -1,13 +1,18 @@ use std::{borrow::Cow, collections::LinkedList, ops::Range, sync::MutexGuard, time::SystemTime}; use bytes::Bytes; +use virtual_fs::AsyncWriteExt; use wasmer::{FunctionEnvMut, WasmPtr}; use wasmer_types::MemorySize; -use wasmer_wasix_types::{types::__wasi_ciovec_t, wasi::Errno}; +use wasmer_wasix_types::types::__wasi_ciovec_t; use crate::{ - mem_error_to_wasi, os::task::process::WasiProcessInner, syscalls::__asyncify_light, - utils::map_snapshot_err, WasiEnv, WasiError, WasiThreadId, + fs::fs_error_into_wasi_err, + mem_error_to_wasi, + os::task::process::WasiProcessInner, + syscalls::__asyncify_light, + utils::{map_io_err, map_snapshot_err}, + WasiEnv, WasiResult, WasiThreadId, }; use super::*; @@ -16,11 +21,11 @@ use super::*; pub struct SnapshotEffector {} impl SnapshotEffector { - pub fn write_terminal_data( + pub fn save_terminal_data( ctx: &mut FunctionEnvMut<'_, WasiEnv>, iovs: WasmPtr<__wasi_ciovec_t, M>, iovs_len: M::Offset, - ) -> Result, WasiError> { + ) -> WasiResult<()> { let env = ctx.data(); let memory = unsafe { env.memory_view(&ctx) }; let iovs_arr = wasi_try_mem_ok_ok!(iovs.slice(&memory, iovs_len)); @@ -47,10 +52,24 @@ impl SnapshotEffector { Ok(Ok(())) } - pub fn write_thread_exit( + pub fn apply_terminal_data( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + data: &[u8], + ) -> WasiResult<()> { + let env = ctx.data(); + wasi_try_ok_ok!(__asyncify_light(env, None, async { + if let Some(mut stdout) = ctx.data().stdout().map_err(fs_error_into_wasi_err)? { + stdout.write_all(data).await.map_err(map_io_err)?; + } + Ok(()) + })?); + Ok(Ok(())) + } + + pub fn save_thread_exit( ctx: &mut FunctionEnvMut<'_, WasiEnv>, id: WasiThreadId, - ) -> Result, WasiError> { + ) -> WasiResult<()> { let env = ctx.data(); wasi_try_ok_ok!(__asyncify_light(env, None, async { ctx.data() @@ -64,12 +83,12 @@ impl SnapshotEffector { Ok(Ok(())) } - pub fn write_thread_state( + pub fn save_thread_state( ctx: &mut FunctionEnvMut<'_, WasiEnv>, id: WasiThreadId, memory_stack: Bytes, rewind_stack: Bytes, - ) -> Result, WasiError> { + ) -> WasiResult<()> { let env = ctx.data(); wasi_try_ok_ok!(__asyncify_light(env, None, async { ctx.data() @@ -87,10 +106,10 @@ impl SnapshotEffector { Ok(Ok(())) } - pub fn write_memory_and_snapshot( + pub fn save_memory_and_snapshot( ctx: &mut FunctionEnvMut<'_, WasiEnv>, process: &mut MutexGuard<'_, WasiProcessInner>, - ) -> Result, WasiError> { + ) -> WasiResult<()> { let env = ctx.data(); let memory = unsafe { env.memory_view(ctx) }; diff --git a/lib/wasix/src/state/env.rs b/lib/wasix/src/state/env.rs index 28b18169124..c3c0f95ceef 100644 --- a/lib/wasix/src/state/env.rs +++ b/lib/wasix/src/state/env.rs @@ -34,7 +34,7 @@ use crate::{ runtime::{resolver::PackageSpecifier, task_manager::InlineWaker, SpawnMemoryType}, syscalls::platform_clock_time_get, Runtime, VirtualTaskManager, WasiControlPlane, WasiEnvBuilder, WasiError, WasiFunctionEnv, - WasiRuntimeError, WasiStateCreationError, WasiVFork, + WasiResult, WasiRuntimeError, WasiStateCreationError, WasiVFork, }; pub(crate) use super::handles::*; @@ -566,9 +566,7 @@ impl WasiEnv { } /// Porcesses any signals that are batched up or any forced exit codes - pub(crate) fn process_signals_and_exit( - ctx: &mut FunctionEnvMut<'_, Self>, - ) -> Result, WasiError> { + pub(crate) fn process_signals_and_exit(ctx: &mut FunctionEnvMut<'_, Self>) -> WasiResult { // If a signal handler has never been set then we need to handle signals // differently let env = ctx.data(); @@ -603,9 +601,7 @@ impl WasiEnv { } /// Porcesses any signals that are batched up - pub(crate) fn process_signals( - ctx: &mut FunctionEnvMut<'_, Self>, - ) -> Result, WasiError> { + pub(crate) fn process_signals(ctx: &mut FunctionEnvMut<'_, Self>) -> WasiResult { // If a signal handler has never been set then we need to handle signals // differently let env = ctx.data(); diff --git a/lib/wasix/src/syscalls/mod.rs b/lib/wasix/src/syscalls/mod.rs index ef24583bafd..2b688bc1265 100644 --- a/lib/wasix/src/syscalls/mod.rs +++ b/lib/wasix/src/syscalls/mod.rs @@ -120,7 +120,7 @@ use crate::{ os::task::thread::RewindResult, runtime::task_manager::InlineWaker, utils::store::InstanceSnapshot, - DeepSleepWork, RewindPostProcess, RewindState, SpawnError, WasiInodes, + DeepSleepWork, RewindPostProcess, RewindState, SpawnError, WasiInodes, WasiResult, }; pub(crate) use crate::{net::net_error_into_wasi_err, utils::WasiParkingLot}; @@ -241,9 +241,9 @@ fn block_on_with_timeout( tasks: &Arc, timeout: Option, work: Fut, -) -> Result, WasiError> +) -> WasiResult where - Fut: Future, WasiError>>, + Fut: Future>, { let mut nonblocking = false; if timeout == Some(Duration::ZERO) { @@ -293,7 +293,7 @@ pub(crate) fn __asyncify( ctx: &mut FunctionEnvMut<'_, WasiEnv>, timeout: Option, work: Fut, -) -> Result, WasiError> +) -> WasiResult where T: 'static, Fut: std::future::Future>, @@ -481,7 +481,7 @@ pub(crate) fn __asyncify_light( env: &WasiEnv, timeout: Option, work: Fut, -) -> Result, WasiError> +) -> WasiResult where T: 'static, Fut: Future>, diff --git a/lib/wasix/src/syscalls/wasi/fd_read.rs b/lib/wasix/src/syscalls/wasi/fd_read.rs index 220d1af7cb2..838167e46c0 100644 --- a/lib/wasix/src/syscalls/wasi/fd_read.rs +++ b/lib/wasix/src/syscalls/wasi/fd_read.rs @@ -109,7 +109,7 @@ pub(crate) fn fd_read_internal( offset: usize, nread: WasmPtr, should_update_cursor: bool, -) -> Result, WasiError> { +) -> WasiResult { wasi_try_ok_ok!(WasiEnv::process_signals_and_exit(ctx)?); let env = ctx.data(); diff --git a/lib/wasix/src/syscalls/wasi/fd_write.rs b/lib/wasix/src/syscalls/wasi/fd_write.rs index fac29a96fe0..e941713240a 100644 --- a/lib/wasix/src/syscalls/wasi/fd_write.rs +++ b/lib/wasix/src/syscalls/wasi/fd_write.rs @@ -108,7 +108,7 @@ pub(crate) fn fd_write_internal( // will record a terminal event. #[cfg(feature = "snapshot")] if is_stdio && ctx.data().enable_snapshot_feed { - SnapshotEffector::write_terminal_data(&mut ctx, iovs, iovs_len)?; + SnapshotEffector::save_terminal_data(&mut ctx, iovs, iovs_len)?; env = ctx.data(); } diff --git a/lib/wasix/src/syscalls/wasix/proc_spawn.rs b/lib/wasix/src/syscalls/wasix/proc_spawn.rs index a98775fa396..bbec1b3aa40 100644 --- a/lib/wasix/src/syscalls/wasix/proc_spawn.rs +++ b/lib/wasix/src/syscalls/wasix/proc_spawn.rs @@ -100,7 +100,7 @@ pub fn proc_spawn_internal( stdin: WasiStdioMode, stdout: WasiStdioMode, stderr: WasiStdioMode, -) -> Result), Errno>, WasiError> { +) -> WasiResult<(ProcessHandles, FunctionEnvMut<'_, WasiEnv>)> { let env = ctx.data(); // Build a new store that will be passed to the thread diff --git a/lib/wasix/src/syscalls/wasix/sock_recv.rs b/lib/wasix/src/syscalls/wasix/sock_recv.rs index 8444c675957..0dfafd09c2c 100644 --- a/lib/wasix/src/syscalls/wasix/sock_recv.rs +++ b/lib/wasix/src/syscalls/wasix/sock_recv.rs @@ -107,7 +107,7 @@ pub(super) fn sock_recv_internal( ri_flags: RiFlags, ro_data_len: WasmPtr, ro_flags: WasmPtr, -) -> Result, WasiError> { +) -> WasiResult { wasi_try_ok_ok!(WasiEnv::process_signals_and_exit(ctx)?); let mut env = ctx.data(); From 34f233b2364cd13b2e225bc0132312239b0c3436 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Tue, 24 Oct 2023 12:07:14 +1100 Subject: [PATCH 013/129] Wired up the snapshot triggers --- lib/cli/src/commands/run.rs | 5 ++- lib/cli/src/commands/run/wasi.rs | 46 +------------------- lib/wasix/src/os/task/process.rs | 16 ++++++- lib/wasix/src/runners/wasi.rs | 12 ++++- lib/wasix/src/runners/wasi_common.rs | 3 +- lib/wasix/src/snapshot/capturer.rs | 2 + lib/wasix/src/snapshot/effector.rs | 11 +++-- lib/wasix/src/snapshot/filter.rs | 4 +- lib/wasix/src/snapshot/log_file.rs | 8 +++- lib/wasix/src/snapshot/mod.rs | 46 ++++++++++++++++++++ lib/wasix/src/state/builder.rs | 10 +++++ lib/wasix/src/state/env.rs | 31 ++++++++++--- lib/wasix/src/syscalls/wasi/fd_write.rs | 2 +- lib/wasix/src/syscalls/wasix/thread_spawn.rs | 1 + 14 files changed, 129 insertions(+), 68 deletions(-) diff --git a/lib/cli/src/commands/run.rs b/lib/cli/src/commands/run.rs index 6e26340a30f..251a346784c 100644 --- a/lib/cli/src/commands/run.rs +++ b/lib/cli/src/commands/run.rs @@ -37,6 +37,7 @@ use wasmer_wasix::{ resolver::{PackageSpecifier, QueryError}, task_manager::VirtualTaskManagerExt, }, + snapshot::SnapshotTrigger, WasiError, }; use wasmer_wasix::{ @@ -223,7 +224,9 @@ impl Run { if self.wasi.forward_host_env { runner.set_forward_host_env(); } - + for trigger in self.wasi.snapshot_on.iter().cloned() { + runner.add_snapshot_trigger(trigger); + } *runner.capabilities() = self.wasi.capabilities(); runner.run_command(command_name, pkg, runtime) diff --git a/lib/cli/src/commands/run/wasi.rs b/lib/cli/src/commands/run/wasi.rs index f5ed9848f4d..d1ce9215e30 100644 --- a/lib/cli/src/commands/run/wasi.rs +++ b/lib/cli/src/commands/run/wasi.rs @@ -34,7 +34,7 @@ use wasmer_wasix::{ VirtualTaskManagerExt, }, }, - snapshot, + snapshot::{self, SnapshotTrigger}, types::__WASI_STDIN_FILENO, wasmer_wasix_types::wasi::Errno, PluggableRuntime, RewindState, Runtime, WasiEnv, WasiEnvBuilder, WasiError, WasiFunctionEnv, @@ -147,50 +147,6 @@ pub struct Wasi { pub deny_multiple_wasi_versions: bool, } -/// Various triggers that will cause the runtime to take snapshot -/// of the WASM state and store it in the snapshot file. -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub enum SnapshotTrigger { - /// Triggered when all the threads in the process goes idle - Idle, - /// Triggered when a listen syscall is invoked on a socket - Listen, - /// Triggered when the process reads stdin for the first time - Stdin, - /// Triggered periodically based on a timer (default 10 seconds) which can be specified using the `snapshot-timer` option - Timer, - /// Issued if the user sends an interrupt signal (Ctrl + C). - Sigint, - /// Alarm clock signal (used for timers) - Sigalrm, - /// The SIGTSTP signal is sent to a process by its controlling terminal to request it to stop temporarily. It is commonly initiated by the user pressing Ctrl-Z. - Sigtstp, - /// The SIGSTOP signal instructs the operating system to stop a process for later resumption. - Sigstop, - /// When a non-determinstic call is made - NonDeterministicCall, -} - -impl FromStr for SnapshotTrigger { - type Err = anyhow::Error; - - fn from_str(s: &str) -> std::result::Result { - let s = s.trim().to_lowercase(); - Ok(match s.as_str() { - "idle" => Self::Idle, - "listen" => Self::Listen, - "stdin" => Self::Stdin, - "periodic" => Self::Timer, - "intr" | "sigint" | "ctrlc" | "ctrl-c" => Self::Sigint, - "alarm" | "timer" | "sigalrm" => Self::Sigalrm, - "sigtstp" | "ctrlz" | "ctrl-z" => Self::Sigtstp, - "stop" | "sigstop" => Self::Sigstop, - "non-deterministic-call" => Self::NonDeterministicCall, - a => return Err(anyhow::format_err!("invalid or unknown trigger ({a})")), - }) - } -} - pub struct RunProperties { pub ctx: WasiFunctionEnv, pub path: PathBuf, diff --git a/lib/wasix/src/os/task/process.rs b/lib/wasix/src/os/task/process.rs index cfaf07f6b38..13d7ec4f0bc 100644 --- a/lib/wasix/src/os/task/process.rs +++ b/lib/wasix/src/os/task/process.rs @@ -1,7 +1,10 @@ -use crate::{snapshot::SnapshotEffector, unwind, WasiEnv, WasiResult, WasiRuntimeError}; +use crate::{ + snapshot::{SnapshotEffector, SnapshotTrigger}, + unwind, WasiEnv, WasiResult, WasiRuntimeError, +}; use serde::{Deserialize, Serialize}; use std::{ - collections::HashMap, + collections::{HashMap, HashSet}, convert::TryInto, sync::{ atomic::{AtomicU32, Ordering}, @@ -94,6 +97,9 @@ pub struct WasiProcess { pub(crate) finished: Arc, /// Number of threads waiting for children to exit pub(crate) waiting: Arc, + /// List of situations that the process will checkpoint on + #[cfg(feature = "snapshot")] + pub(crate) checkpoint_on: HashSet, } /// Represents a freeze of all threads to perform some action @@ -167,6 +173,7 @@ impl WasiProcess { )), finished: Arc::new(OwnedTaskStatus::default()), waiting: Arc::new(AtomicU32::new(0)), + checkpoint_on: Default::default(), } } @@ -174,6 +181,11 @@ impl WasiProcess { self.pid = pid; } + /// Adds another trigger that will cause a snapshot to be taken + pub fn add_snapshot_trigger(&mut self, on: SnapshotTrigger) { + self.checkpoint_on.insert(on); + } + /// Gets the process ID of this process pub fn pid(&self) -> WasiProcessId { self.pid diff --git a/lib/wasix/src/runners/wasi.rs b/lib/wasix/src/runners/wasi.rs index 6a46a125c11..eb81ddb2d81 100644 --- a/lib/wasix/src/runners/wasi.rs +++ b/lib/wasix/src/runners/wasi.rs @@ -10,6 +10,7 @@ use crate::{ bin_factory::BinaryPackage, capabilities::Capabilities, runners::{wasi_common::CommonWasiOptions, MappedDirectory}, + snapshot::SnapshotTrigger, Runtime, WasiEnvBuilder, }; @@ -146,6 +147,11 @@ impl WasiRunner { self.wasi.capabilities = capabilities; } + pub fn add_snapshot_trigger(&mut self, on: SnapshotTrigger) -> &mut Self { + self.wasi.snapshot_on.push(on); + self + } + pub fn with_stdin(mut self, stdin: Box) -> Self { self.set_stdin(stdin); self @@ -231,10 +237,14 @@ impl crate::runners::Runner for WasiRunner { let module = runtime.load_module_sync(cmd.atom())?; let mut store = runtime.new_store(); - let env = self + let mut env = self .prepare_webc_env(command_name, &wasi, pkg, runtime, None) .context("Unable to prepare the WASI environment")?; + for snapshot_trigger in self.wasi.snapshot_on.iter().cloned() { + env.add_snapshot_trigger(snapshot_trigger); + } + if self .wasi .capabilities diff --git a/lib/wasix/src/runners/wasi_common.rs b/lib/wasix/src/runners/wasi_common.rs index e76b26efde2..16ad0ed8462 100644 --- a/lib/wasix/src/runners/wasi_common.rs +++ b/lib/wasix/src/runners/wasi_common.rs @@ -11,7 +11,7 @@ use webc::metadata::annotations::Wasi as WasiAnnotation; use crate::{ bin_factory::BinaryPackage, capabilities::Capabilities, runners::MappedDirectory, - WasiEnvBuilder, + snapshot::SnapshotTrigger, WasiEnvBuilder, }; #[derive(Debug, Default, Clone)] @@ -22,6 +22,7 @@ pub(crate) struct CommonWasiOptions { pub(crate) mapped_dirs: Vec, pub(crate) injected_packages: Vec, pub(crate) capabilities: Capabilities, + pub(crate) snapshot_on: Vec, } impl CommonWasiOptions { diff --git a/lib/wasix/src/snapshot/capturer.rs b/lib/wasix/src/snapshot/capturer.rs index 06a9b11066a..09f3482008c 100644 --- a/lib/wasix/src/snapshot/capturer.rs +++ b/lib/wasix/src/snapshot/capturer.rs @@ -2,6 +2,7 @@ use serde::{Deserialize, Serialize}; use std::net::SocketAddr; use std::time::SystemTime; use std::{borrow::Cow, ops::Range}; +use wasmer_wasix_types::wasi::ExitCode; use futures::future::BoxFuture; use virtual_fs::Fd; @@ -110,6 +111,7 @@ pub enum SnapshotLog<'a> { }, CloseThread { id: WasiThreadId, + exit_code: Option, }, SetThread { id: WasiThreadId, diff --git a/lib/wasix/src/snapshot/effector.rs b/lib/wasix/src/snapshot/effector.rs index 23815c54813..9e27a08afb4 100644 --- a/lib/wasix/src/snapshot/effector.rs +++ b/lib/wasix/src/snapshot/effector.rs @@ -4,7 +4,7 @@ use bytes::Bytes; use virtual_fs::AsyncWriteExt; use wasmer::{FunctionEnvMut, WasmPtr}; use wasmer_types::MemorySize; -use wasmer_wasix_types::types::__wasi_ciovec_t; +use wasmer_wasix_types::{types::__wasi_ciovec_t, wasi::ExitCode}; use crate::{ fs::fs_error_into_wasi_err, @@ -67,15 +67,14 @@ impl SnapshotEffector { } pub fn save_thread_exit( - ctx: &mut FunctionEnvMut<'_, WasiEnv>, + env: &WasiEnv, id: WasiThreadId, + exit_code: Option, ) -> WasiResult<()> { - let env = ctx.data(); wasi_try_ok_ok!(__asyncify_light(env, None, async { - ctx.data() - .runtime() + env.runtime() .snapshot_capturer() - .write(SnapshotLog::CloseThread { id }) + .write(SnapshotLog::CloseThread { id, exit_code }) .await .map_err(map_snapshot_err)?; Ok(()) diff --git a/lib/wasix/src/snapshot/filter.rs b/lib/wasix/src/snapshot/filter.rs index b3290aa7e72..8518c78c438 100644 --- a/lib/wasix/src/snapshot/filter.rs +++ b/lib/wasix/src/snapshot/filter.rs @@ -76,11 +76,11 @@ impl SnapshotCapturer for FilteredSnapshotCapturer { } SnapshotLog::UpdateMemoryRegion { region, data } } - SnapshotLog::CloseThread { id } => { + SnapshotLog::CloseThread { id, exit_code } => { if self.filter_threads { return Ok(()); } - SnapshotLog::CloseThread { id } + SnapshotLog::CloseThread { id, exit_code } } SnapshotLog::SetThread { id, diff --git a/lib/wasix/src/snapshot/log_file.rs b/lib/wasix/src/snapshot/log_file.rs index a4e8bce91a7..609fa301311 100644 --- a/lib/wasix/src/snapshot/log_file.rs +++ b/lib/wasix/src/snapshot/log_file.rs @@ -5,6 +5,7 @@ use std::{ path::Path, time::SystemTime, }; +use wasmer_wasix_types::wasi::ExitCode; use futures::future::BoxFuture; use virtual_fs::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt, Fd}; @@ -34,6 +35,7 @@ pub enum SnapshotLogEntry { }, CloseThreadV1 { id: WasiThreadId, + exit_code: Option, }, SetThreadV1 { id: WasiThreadId, @@ -76,7 +78,7 @@ impl<'a> From> for SnapshotLogEntry { end: region.end, data: data.into_owned(), }, - SnapshotLog::CloseThread { id } => Self::CloseThreadV1 { id }, + SnapshotLog::CloseThread { id, exit_code } => Self::CloseThreadV1 { id, exit_code }, SnapshotLog::SetThread { id, call_stack, @@ -127,7 +129,9 @@ impl<'a> From for SnapshotLog<'a> { data: data.into(), } } - SnapshotLogEntry::CloseThreadV1 { id } => Self::CloseThread { id }, + SnapshotLogEntry::CloseThreadV1 { id, exit_code } => { + Self::CloseThread { id, exit_code } + } SnapshotLogEntry::SetThreadV1 { id, call_stack, diff --git a/lib/wasix/src/snapshot/mod.rs b/lib/wasix/src/snapshot/mod.rs index 4f09554e16f..25fce9af2e1 100644 --- a/lib/wasix/src/snapshot/mod.rs +++ b/lib/wasix/src/snapshot/mod.rs @@ -5,9 +5,55 @@ mod filter; mod log_file; mod unsupported; +use std::str::FromStr; + pub use capturer::*; pub use compactor::*; pub use effector::*; pub use filter::*; pub use log_file::*; pub use unsupported::*; + +/// Various triggers that will cause the runtime to take snapshot +/// of the WASM state and store it in the snapshot file. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum SnapshotTrigger { + /// Triggered when all the threads in the process goes idle + Idle, + /// Triggered when a listen syscall is invoked on a socket + Listen, + /// Triggered when the process reads stdin for the first time + Stdin, + /// Triggered periodically based on a timer (default 10 seconds) which can be specified using the `snapshot-timer` option + Timer, + /// Issued if the user sends an interrupt signal (Ctrl + C). + Sigint, + /// Alarm clock signal (used for timers) + Sigalrm, + /// The SIGTSTP signal is sent to a process by its controlling terminal to request it to stop temporarily. It is commonly initiated by the user pressing Ctrl-Z. + Sigtstp, + /// The SIGSTOP signal instructs the operating system to stop a process for later resumption. + Sigstop, + /// When a non-determinstic call is made + NonDeterministicCall, +} + +impl FromStr for SnapshotTrigger { + type Err = anyhow::Error; + + fn from_str(s: &str) -> std::result::Result { + let s = s.trim().to_lowercase(); + Ok(match s.as_str() { + "idle" => Self::Idle, + "listen" => Self::Listen, + "stdin" => Self::Stdin, + "periodic" => Self::Timer, + "intr" | "sigint" | "ctrlc" | "ctrl-c" => Self::Sigint, + "alarm" | "timer" | "sigalrm" => Self::Sigalrm, + "sigtstp" | "ctrlz" | "ctrl-z" => Self::Sigtstp, + "stop" | "sigstop" => Self::Sigstop, + "non-deterministic-call" => Self::NonDeterministicCall, + a => return Err(anyhow::format_err!("invalid or unknown trigger ({a})")), + }) + } +} diff --git a/lib/wasix/src/state/builder.rs b/lib/wasix/src/state/builder.rs index cbe113ee8cb..3c71cfc06ee 100644 --- a/lib/wasix/src/state/builder.rs +++ b/lib/wasix/src/state/builder.rs @@ -21,6 +21,7 @@ use crate::{ fs::{WasiFs, WasiFsRoot, WasiInodes}, os::task::control_plane::{ControlPlaneConfig, ControlPlaneError, WasiControlPlane}, runtime::task_manager::InlineWaker, + snapshot::SnapshotTrigger, state::WasiState, syscalls::types::{__WASI_STDERR_FILENO, __WASI_STDIN_FILENO, __WASI_STDOUT_FILENO}, RewindState, Runtime, WasiEnv, WasiError, WasiFunctionEnv, WasiRuntimeError, @@ -70,6 +71,8 @@ pub struct WasiEnvBuilder { pub(super) map_commands: HashMap, pub(super) capabilites: Capabilities, + + pub(super) snapshot_on: Vec, } impl std::fmt::Debug for WasiEnvBuilder { @@ -573,6 +576,10 @@ impl WasiEnvBuilder { self.capabilites = capabilities; } + pub fn add_snapshot_trigger(&mut self, on: SnapshotTrigger) { + self.snapshot_on.push(on); + } + /// Consumes the [`WasiEnvBuilder`] and produces a [`WasiEnvInit`], which /// can be used to construct a new [`WasiEnv`]. /// @@ -731,6 +738,8 @@ impl WasiEnvBuilder { }; let control_plane = WasiControlPlane::new(plane_config); + let snapshot_on = self.snapshot_on; + let init = WasiEnvInit { state, runtime, @@ -745,6 +754,7 @@ impl WasiEnvBuilder { call_initialize: true, can_deep_sleep: false, extra_tracing: true, + snapshot_on, }; Ok(init) diff --git a/lib/wasix/src/state/env.rs b/lib/wasix/src/state/env.rs index c3c0f95ceef..c8d83475735 100644 --- a/lib/wasix/src/state/env.rs +++ b/lib/wasix/src/state/env.rs @@ -32,6 +32,7 @@ use crate::{ thread::{WasiMemoryLayout, WasiThread, WasiThreadHandle, WasiThreadId}, }, runtime::{resolver::PackageSpecifier, task_manager::InlineWaker, SpawnMemoryType}, + snapshot::{SnapshotEffector, SnapshotTrigger}, syscalls::platform_clock_time_get, Runtime, VirtualTaskManager, WasiControlPlane, WasiEnvBuilder, WasiError, WasiFunctionEnv, WasiResult, WasiRuntimeError, WasiStateCreationError, WasiVFork, @@ -230,6 +231,9 @@ pub struct WasiEnvInit { /// Indicates if extra tracing should be output pub extra_tracing: bool, + + /// Indicates triggers that will cause a snapshot to be taken + pub snapshot_on: Vec, } impl WasiEnvInit { @@ -266,6 +270,7 @@ impl WasiEnvInit { call_initialize: self.call_initialize, can_deep_sleep: self.can_deep_sleep, extra_tracing: false, + snapshot_on: self.snapshot_on.clone(), } } } @@ -299,9 +304,6 @@ pub struct WasiEnv { /// Is this environment capable and setup for deep sleeping pub enable_deep_sleep: bool, - /// Indicates if the feed to the snapshot system is enabled or not - pub enable_snapshot_feed: bool, - /// Inner functions and references that are loaded before the environment starts /// (inner is not safe to send between threads and so it is private and will /// not be cloned when `WasiEnv` is cloned) @@ -331,7 +333,6 @@ impl Clone for WasiEnv { runtime: self.runtime.clone(), capabilities: self.capabilities.clone(), enable_deep_sleep: self.enable_deep_sleep, - enable_snapshot_feed: self.enable_snapshot_feed, } } } @@ -368,7 +369,6 @@ impl WasiEnv { runtime: self.runtime.clone(), capabilities: self.capabilities.clone(), enable_deep_sleep: self.enable_deep_sleep, - enable_snapshot_feed: self.enable_snapshot_feed, }; Ok((new_env, handle)) } @@ -405,12 +405,16 @@ impl WasiEnv { #[allow(clippy::result_large_err)] pub(crate) fn from_init(init: WasiEnvInit) -> Result { - let process = if let Some(p) = init.process { + let mut process = if let Some(p) = init.process { p } else { init.control_plane.new_process()? }; + for snapshot_on in init.snapshot_on { + process.add_snapshot_trigger(snapshot_on) + } + let layout = WasiMemoryLayout::default(); let thread = if let Some(t) = init.thread { t @@ -431,7 +435,6 @@ impl WasiEnv { runtime: init.runtime, bin_factory: init.bin_factory, enable_deep_sleep: init.capabilities.threading.enable_asynchronous_threading, - enable_snapshot_feed: false, capabilities: init.capabilities, }; env.owned_handles.push(thread); @@ -820,6 +823,12 @@ impl WasiEnv { self.state.stdin() } + /// Returns true if the process should perform snapshots or not + #[cfg(feature = "snapshot")] + pub fn should_feed_snapshot(&self) -> bool { + !self.process.checkpoint_on.is_empty() + } + /// Internal helper function to get a standard device handle. /// Expects one of `__WASI_STDIN_FILENO`, `__WASI_STDOUT_FILENO`, `__WASI_STDERR_FILENO`. pub fn std_dev_get( @@ -1036,6 +1045,14 @@ impl WasiEnv { pub fn cleanup(&self, exit_code: Option) -> BoxFuture<'static, ()> { const CLEANUP_TIMEOUT: Duration = Duration::from_secs(10); + // If snap-shooting is enabled then we should record an event that the thread has exited. + #[cfg(feature = "snapshot")] + if self.should_feed_snapshot() { + if let Err(err) = SnapshotEffector::save_thread_exit(self, self.tid(), exit_code) { + tracing::warn!("failed to save snapshot event for thread exit - {}", err); + } + } + // If this is the main thread then also close all the files if self.thread.is_main() { trace!("wasi[{}]:: cleaning up open file handles", self.pid()); diff --git a/lib/wasix/src/syscalls/wasi/fd_write.rs b/lib/wasix/src/syscalls/wasi/fd_write.rs index e941713240a..e36ec92c5b9 100644 --- a/lib/wasix/src/syscalls/wasi/fd_write.rs +++ b/lib/wasix/src/syscalls/wasi/fd_write.rs @@ -107,7 +107,7 @@ pub(crate) fn fd_write_internal( // If snap-shooting is enabled and this is to stdio then we // will record a terminal event. #[cfg(feature = "snapshot")] - if is_stdio && ctx.data().enable_snapshot_feed { + if is_stdio && ctx.data().should_feed_snapshot() { SnapshotEffector::save_terminal_data(&mut ctx, iovs, iovs_len)?; env = ctx.data(); } diff --git a/lib/wasix/src/syscalls/wasix/thread_spawn.rs b/lib/wasix/src/syscalls/wasix/thread_spawn.rs index 0cd4a00d689..7ddb78a46e2 100644 --- a/lib/wasix/src/syscalls/wasix/thread_spawn.rs +++ b/lib/wasix/src/syscalls/wasix/thread_spawn.rs @@ -5,6 +5,7 @@ use crate::{ capture_snapshot, os::task::thread::WasiMemoryLayout, runtime::task_manager::{TaskWasm, TaskWasmRunProperties}, + snapshot::SnapshotEffector, syscalls::*, WasiThreadHandle, }; From b829407b1d6e9f1b8f2c1dd6c7508977fae34959 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Tue, 24 Oct 2023 12:54:33 +1100 Subject: [PATCH 014/129] Runtime will now snapshot when the STDIN trigger is used --- lib/wasix/src/os/task/process.rs | 195 +++++++++++++------------ lib/wasix/src/snapshot/capturer.rs | 3 + lib/wasix/src/snapshot/effector.rs | 3 +- lib/wasix/src/snapshot/filter.rs | 4 +- lib/wasix/src/snapshot/log_file.rs | 5 +- lib/wasix/src/snapshot/mod.rs | 7 +- lib/wasix/src/state/env.rs | 12 ++ lib/wasix/src/syscalls/wasi/fd_read.rs | 30 +++- 8 files changed, 156 insertions(+), 103 deletions(-) diff --git a/lib/wasix/src/os/task/process.rs b/lib/wasix/src/os/task/process.rs index 13d7ec4f0bc..eb8e23e599f 100644 --- a/lib/wasix/src/os/task/process.rs +++ b/lib/wasix/src/os/task/process.rs @@ -78,6 +78,8 @@ impl std::fmt::Debug for WasiProcessId { } } +pub type LockableWasiProcessInner = Arc<(Mutex, Condvar)>; + /// Represents a process running within the compute state /// TODO: fields should be private and only accessed via methods. #[derive(Debug, Clone)] @@ -88,7 +90,7 @@ pub struct WasiProcess { pub(crate) parent: Option>>, /// The inner protected region of the process with a conditional /// variable that is used for coordination such as checksums. - pub(crate) inner: Arc<(Mutex, Condvar)>, + pub(crate) inner: LockableWasiProcessInner, /// Reference back to the compute engine // TODO: remove this reference, access should happen via separate state instead // (we don't want cyclical references) @@ -113,7 +115,7 @@ pub enum WasiProcessCheckpoint { Execute, /// The process needs to take a snapshot of the /// memory and state-machine - Snapshot, + Snapshot { trigger: SnapshotTrigger }, } // TODO: fields should be private and only accessed via methods. @@ -134,6 +136,104 @@ pub struct WasiProcessInner { pub checkpoint: WasiProcessCheckpoint, } +pub enum MaybeCheckpointResult<'a> { + NotThisTime(FunctionEnvMut<'a, WasiEnv>), + Unwinding, +} + +impl WasiProcessInner { + /// Checkpoints the process which will cause all other threads to + /// pause and for the thread and memory state to be saved + pub fn checkpoint( + inner: LockableWasiProcessInner, + ctx: FunctionEnvMut<'_, WasiEnv>, + for_what: WasiProcessCheckpoint, + ) -> WasiResult> { + // Set the checkpoint flag and then enter the normal processing loop + { + let mut inner = inner.0.lock().unwrap(); + inner.checkpoint = for_what; + } + + Self::maybe_checkpoint::(inner, ctx) + } + + /// If a checkpoint has been started this will block the current process + /// until the checkpoint operation has completed + pub fn maybe_checkpoint( + inner: LockableWasiProcessInner, + ctx: FunctionEnvMut<'_, WasiEnv>, + ) -> WasiResult> { + // Enter the lock which will determine if we are in a checkpoint or not + let guard = inner.0.lock().unwrap(); + if guard.checkpoint == WasiProcessCheckpoint::Execute { + // No checkpoint so just carry on + return Ok(Ok(MaybeCheckpointResult::NotThisTime(ctx))); + } + trace!("checkpoint capture"); + drop(guard); + + // Perform the unwind action + unwind::(ctx, move |mut ctx, memory_stack, rewind_stack| { + // Write our thread state to the snapshot + let tid = ctx.data().thread.tid(); + if let Err(err) = SnapshotEffector::save_thread_state( + &mut ctx, + tid, + memory_stack.freeze(), + rewind_stack.freeze(), + ) { + return wasmer_types::OnCalledAction::Trap(Box::new(err)); + } + + let mut guard = inner.0.lock().unwrap(); + + // Wait for the checkpoint to finish (or if we are the last thread + // to freeze then we have to execute the checksum operation) + loop { + if let WasiProcessCheckpoint::Snapshot { .. } = guard.checkpoint { + } else { + ctx.data().thread.set_check_pointing(false); + trace!("checkpoint finished"); + break; + } + + ctx.data().thread.set_check_pointing(true); + + match guard.checkpoint { + WasiProcessCheckpoint::Snapshot { trigger } => { + // Now if we are the last thread we also write the memory + let is_last_thread = + guard.threads.values().all(WasiThread::is_check_pointing); + if is_last_thread { + if let Err(err) = SnapshotEffector::save_memory_and_snapshot( + &mut ctx, &mut guard, trigger, + ) { + inner.1.notify_all(); + return wasmer_types::OnCalledAction::Trap(Box::new(err)); + } + + // Clear the checkpointing flag and notify everyone to wake up + ctx.data().thread.set_check_pointing(false); + guard.checkpoint = WasiProcessCheckpoint::Execute; + trace!("checkpoint complete"); + inner.1.notify_all(); + } else { + guard = inner.1.wait(guard).unwrap(); + } + } + _ => {} + } + } + + // Resume execution + wasmer_types::OnCalledAction::InvokeAgain + })?; + + Ok(Ok(MaybeCheckpointResult::Unwinding)) + } +} + // TODO: why do we need this, how is it used? pub(crate) struct WasiProcessWait { waiting: Arc, @@ -331,97 +431,6 @@ impl WasiProcess { inner.thread_count } - /// Checkpoints the process which will cause all other threads to - /// pause and for the thread and memory state to be saved - pub fn checkpoint( - &self, - ctx: FunctionEnvMut<'_, WasiEnv>, - for_what: WasiProcessCheckpoint, - ) -> WasiResult<()> { - // Set the checkpoint flag and then enter the normal processing loop - { - let mut inner = self.inner.0.lock().unwrap(); - inner.checkpoint = for_what; - } - - self.maybe_checkpoint::(ctx) - } - - /// If a checkpoint has been started this will block the current process - /// until the checkpoint operation has completed - pub fn maybe_checkpoint( - &self, - ctx: FunctionEnvMut<'_, WasiEnv>, - ) -> WasiResult<()> { - // Enter the lock which will determine if we are in a checkpoint or not - let inner = self.inner.clone(); - let guard = inner.0.lock().unwrap(); - if guard.checkpoint == WasiProcessCheckpoint::Execute { - // No checkpoint so just carry on - return Ok(Ok(())); - } - trace!("checkpoint capture"); - drop(guard); - - // Perform the unwind action - unwind::(ctx, move |mut ctx, memory_stack, rewind_stack| { - // Write our thread state to the snapshot - let tid = ctx.data().thread.tid(); - if let Err(err) = SnapshotEffector::save_thread_state( - &mut ctx, - tid, - memory_stack.freeze(), - rewind_stack.freeze(), - ) { - return wasmer_types::OnCalledAction::Trap(Box::new(err)); - } - - let mut guard = inner.0.lock().unwrap(); - - // Wait for the checkpoint to finish (or if we are the last thread - // to freeze then we have to execute the checksum operation) - loop { - if WasiProcessCheckpoint::Execute == guard.checkpoint { - ctx.data().thread.set_check_pointing(false); - trace!("checkpoint finished"); - break; - } - - ctx.data().thread.set_check_pointing(true); - - match guard.checkpoint { - WasiProcessCheckpoint::Execute => {} - WasiProcessCheckpoint::Snapshot => { - // Now if we are the last thread we also write the memory - let is_last_thread = - guard.threads.values().all(WasiThread::is_check_pointing); - if is_last_thread { - if let Err(err) = - SnapshotEffector::save_memory_and_snapshot(&mut ctx, &mut guard) - { - inner.1.notify_all(); - return wasmer_types::OnCalledAction::Trap(Box::new(err)); - } - - // Clear the checkpointing flag and notify everyone to wake up - ctx.data().thread.set_check_pointing(false); - guard.checkpoint = WasiProcessCheckpoint::Execute; - trace!("checkpoint complete"); - inner.1.notify_all(); - } else { - guard = inner.1.wait(guard).unwrap(); - } - } - } - } - - // Resume execution - wasmer_types::OnCalledAction::InvokeAgain - })?; - - Ok(Ok(())) - } - /// Waits until the process is finished. pub async fn join(&self) -> Result> { let _guard = WasiProcessWait::new(self); diff --git a/lib/wasix/src/snapshot/capturer.rs b/lib/wasix/src/snapshot/capturer.rs index 09f3482008c..55eda1ee719 100644 --- a/lib/wasix/src/snapshot/capturer.rs +++ b/lib/wasix/src/snapshot/capturer.rs @@ -9,6 +9,8 @@ use virtual_fs::Fd; use crate::WasiThreadId; +use super::SnapshotTrigger; + #[derive(Debug, Clone, Serialize, Deserialize)] pub enum SocketSnapshot { TcpListen { @@ -140,6 +142,7 @@ pub enum SnapshotLog<'a> { /// Represents the marker for the end of a snapshot Snapshot { when: SystemTime, + trigger: SnapshotTrigger, }, } diff --git a/lib/wasix/src/snapshot/effector.rs b/lib/wasix/src/snapshot/effector.rs index 9e27a08afb4..56c555dbedf 100644 --- a/lib/wasix/src/snapshot/effector.rs +++ b/lib/wasix/src/snapshot/effector.rs @@ -108,6 +108,7 @@ impl SnapshotEffector { pub fn save_memory_and_snapshot( ctx: &mut FunctionEnvMut<'_, WasiEnv>, process: &mut MutexGuard<'_, WasiProcessInner>, + trigger: SnapshotTrigger, ) -> WasiResult<()> { let env = ctx.data(); let memory = unsafe { env.memory_view(ctx) }; @@ -169,7 +170,7 @@ impl SnapshotEffector { // it can act as a restoration point let when = SystemTime::now(); capturer - .write(SnapshotLog::Snapshot { when }) + .write(SnapshotLog::Snapshot { when, trigger }) .await .map_err(map_snapshot_err)?; Ok(()) diff --git a/lib/wasix/src/snapshot/filter.rs b/lib/wasix/src/snapshot/filter.rs index 8518c78c438..c443697857f 100644 --- a/lib/wasix/src/snapshot/filter.rs +++ b/lib/wasix/src/snapshot/filter.rs @@ -136,11 +136,11 @@ impl SnapshotCapturer for FilteredSnapshotCapturer { data, } } - SnapshotLog::Snapshot { when } => { + SnapshotLog::Snapshot { when, trigger } => { if self.filter_snapshots { return Ok(()); } - SnapshotLog::Snapshot { when } + SnapshotLog::Snapshot { when, trigger } } }; self.inner.write(evt).await diff --git a/lib/wasix/src/snapshot/log_file.rs b/lib/wasix/src/snapshot/log_file.rs index 609fa301311..0ae2c6710ba 100644 --- a/lib/wasix/src/snapshot/log_file.rs +++ b/lib/wasix/src/snapshot/log_file.rs @@ -63,6 +63,7 @@ pub enum SnapshotLogEntry { }, SnapshotV1 { when: SystemTime, + trigger: SnapshotTrigger, }, } @@ -113,7 +114,7 @@ impl<'a> From> for SnapshotLogEntry { len, data: data.into_owned(), }, - SnapshotLog::Snapshot { when } => Self::SnapshotV1 { when }, + SnapshotLog::Snapshot { when, trigger } => Self::SnapshotV1 { when, trigger }, } } } @@ -166,7 +167,7 @@ impl<'a> From for SnapshotLog<'a> { len, data: data.into(), }, - SnapshotLogEntry::SnapshotV1 { when } => Self::Snapshot { when }, + SnapshotLogEntry::SnapshotV1 { when, trigger } => Self::Snapshot { when, trigger }, } } } diff --git a/lib/wasix/src/snapshot/mod.rs b/lib/wasix/src/snapshot/mod.rs index 25fce9af2e1..51e3f560891 100644 --- a/lib/wasix/src/snapshot/mod.rs +++ b/lib/wasix/src/snapshot/mod.rs @@ -5,8 +5,6 @@ mod filter; mod log_file; mod unsupported; -use std::str::FromStr; - pub use capturer::*; pub use compactor::*; pub use effector::*; @@ -14,9 +12,12 @@ pub use filter::*; pub use log_file::*; pub use unsupported::*; +use serde::{Deserialize, Serialize}; +use std::str::FromStr; + /// Various triggers that will cause the runtime to take snapshot /// of the WASM state and store it in the snapshot file. -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] pub enum SnapshotTrigger { /// Triggered when all the threads in the process goes idle Idle, diff --git a/lib/wasix/src/state/env.rs b/lib/wasix/src/state/env.rs index c8d83475735..8254f3ba2be 100644 --- a/lib/wasix/src/state/env.rs +++ b/lib/wasix/src/state/env.rs @@ -829,6 +829,18 @@ impl WasiEnv { !self.process.checkpoint_on.is_empty() } + /// Returns true if a particular snapshot trigger is enabled + #[cfg(feature = "snapshot")] + pub fn has_snapshot_trigger(&self, trigger: SnapshotTrigger) -> bool { + self.process.checkpoint_on.contains(&trigger) + } + + /// Returns true if a particular snapshot trigger is enabled + #[cfg(feature = "snapshot")] + pub fn pop_snapshot_trigger(&mut self, trigger: SnapshotTrigger) -> bool { + self.process.checkpoint_on.remove(&trigger) + } + /// Internal helper function to get a standard device handle. /// Expects one of `__WASI_STDIN_FILENO`, `__WASI_STDOUT_FILENO`, `__WASI_STDERR_FILENO`. pub fn std_dev_get( diff --git a/lib/wasix/src/syscalls/wasi/fd_read.rs b/lib/wasix/src/syscalls/wasi/fd_read.rs index 838167e46c0..036bee058ff 100644 --- a/lib/wasix/src/syscalls/wasi/fd_read.rs +++ b/lib/wasix/src/syscalls/wasi/fd_read.rs @@ -1,9 +1,15 @@ use std::{collections::VecDeque, task::Waker}; -use virtual_fs::{AsyncReadExt, ReadBuf}; +use virtual_fs::{AsyncReadExt, DeviceFile, ReadBuf}; use super::*; -use crate::{fs::NotificationInner, net::socket::TimeType, syscalls::*}; +use crate::{ + fs::NotificationInner, + net::socket::TimeType, + os::task::process::{MaybeCheckpointResult, WasiProcessCheckpoint, WasiProcessInner}, + snapshot::SnapshotTrigger, + syscalls::*, +}; /// ### `fd_read()` /// Read data from file descriptor @@ -69,6 +75,26 @@ pub fn fd_pread( let pid = ctx.data().pid(); let tid = ctx.data().tid(); + // If snap-shooting is enabled and this is to stdio then we + // we may actually trigger a snapshot event heere. + #[cfg(feature = "snapshot")] + if fd == DeviceFile::STDIN && ctx.data_mut().pop_snapshot_trigger(SnapshotTrigger::Stdin) { + let inner = ctx.data().process.inner.clone(); + let res = wasi_try_ok!(WasiProcessInner::checkpoint::( + inner, + ctx, + WasiProcessCheckpoint::Snapshot { + trigger: SnapshotTrigger::Stdin, + }, + )?); + match res { + MaybeCheckpointResult::Unwinding => return Ok(Errno::Success), + MaybeCheckpointResult::NotThisTime(c) => { + ctx = c; + } + } + } + let res = fd_read_internal::(&mut ctx, fd, iovs, iovs_len, offset as usize, nread, false)?; fd_read_internal_handler::(ctx, res, nread) } From 19c39469ee58851fab5ecb53dbdb95fddbb25d21 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Tue, 24 Oct 2023 13:03:51 +1100 Subject: [PATCH 015/129] Moved the snapshot trigger list from the process to WasiEnv which is a better home --- lib/wasix/src/os/task/process.rs | 11 +---------- lib/wasix/src/state/env.rs | 25 ++++++++++++++++--------- 2 files changed, 17 insertions(+), 19 deletions(-) diff --git a/lib/wasix/src/os/task/process.rs b/lib/wasix/src/os/task/process.rs index eb8e23e599f..ef3207eecd5 100644 --- a/lib/wasix/src/os/task/process.rs +++ b/lib/wasix/src/os/task/process.rs @@ -4,7 +4,7 @@ use crate::{ }; use serde::{Deserialize, Serialize}; use std::{ - collections::{HashMap, HashSet}, + collections::HashMap, convert::TryInto, sync::{ atomic::{AtomicU32, Ordering}, @@ -99,9 +99,6 @@ pub struct WasiProcess { pub(crate) finished: Arc, /// Number of threads waiting for children to exit pub(crate) waiting: Arc, - /// List of situations that the process will checkpoint on - #[cfg(feature = "snapshot")] - pub(crate) checkpoint_on: HashSet, } /// Represents a freeze of all threads to perform some action @@ -273,7 +270,6 @@ impl WasiProcess { )), finished: Arc::new(OwnedTaskStatus::default()), waiting: Arc::new(AtomicU32::new(0)), - checkpoint_on: Default::default(), } } @@ -281,11 +277,6 @@ impl WasiProcess { self.pid = pid; } - /// Adds another trigger that will cause a snapshot to be taken - pub fn add_snapshot_trigger(&mut self, on: SnapshotTrigger) { - self.checkpoint_on.insert(on); - } - /// Gets the process ID of this process pub fn pid(&self) -> WasiProcessId { self.pid diff --git a/lib/wasix/src/state/env.rs b/lib/wasix/src/state/env.rs index 8254f3ba2be..67119908dd9 100644 --- a/lib/wasix/src/state/env.rs +++ b/lib/wasix/src/state/env.rs @@ -1,5 +1,5 @@ use std::{ - collections::HashMap, + collections::{HashMap, HashSet}, ops::Deref, path::{Path, PathBuf}, sync::Arc, @@ -304,6 +304,10 @@ pub struct WasiEnv { /// Is this environment capable and setup for deep sleeping pub enable_deep_sleep: bool, + /// List of situations that the process will checkpoint on + #[cfg(feature = "snapshot")] + pub(crate) snapshot_on: HashSet, + /// Inner functions and references that are loaded before the environment starts /// (inner is not safe to send between threads and so it is private and will /// not be cloned when `WasiEnv` is cloned) @@ -333,6 +337,7 @@ impl Clone for WasiEnv { runtime: self.runtime.clone(), capabilities: self.capabilities.clone(), enable_deep_sleep: self.enable_deep_sleep, + snapshot_on: self.snapshot_on.clone(), } } } @@ -369,6 +374,7 @@ impl WasiEnv { runtime: self.runtime.clone(), capabilities: self.capabilities.clone(), enable_deep_sleep: self.enable_deep_sleep, + snapshot_on: self.snapshot_on.clone(), }; Ok((new_env, handle)) } @@ -405,16 +411,12 @@ impl WasiEnv { #[allow(clippy::result_large_err)] pub(crate) fn from_init(init: WasiEnvInit) -> Result { - let mut process = if let Some(p) = init.process { + let process = if let Some(p) = init.process { p } else { init.control_plane.new_process()? }; - for snapshot_on in init.snapshot_on { - process.add_snapshot_trigger(snapshot_on) - } - let layout = WasiMemoryLayout::default(); let thread = if let Some(t) = init.thread { t @@ -436,9 +438,14 @@ impl WasiEnv { bin_factory: init.bin_factory, enable_deep_sleep: init.capabilities.threading.enable_asynchronous_threading, capabilities: init.capabilities, + snapshot_on: Default::default(), }; env.owned_handles.push(thread); + for snapshot_on in init.snapshot_on { + env.snapshot_on.insert(snapshot_on); + } + // TODO: should not be here - should be callers responsibility! for pkg in &init.webc_dependencies { env.use_package(pkg)?; @@ -826,19 +833,19 @@ impl WasiEnv { /// Returns true if the process should perform snapshots or not #[cfg(feature = "snapshot")] pub fn should_feed_snapshot(&self) -> bool { - !self.process.checkpoint_on.is_empty() + !self.snapshot_on.is_empty() } /// Returns true if a particular snapshot trigger is enabled #[cfg(feature = "snapshot")] pub fn has_snapshot_trigger(&self, trigger: SnapshotTrigger) -> bool { - self.process.checkpoint_on.contains(&trigger) + self.snapshot_on.contains(&trigger) } /// Returns true if a particular snapshot trigger is enabled #[cfg(feature = "snapshot")] pub fn pop_snapshot_trigger(&mut self, trigger: SnapshotTrigger) -> bool { - self.process.checkpoint_on.remove(&trigger) + self.snapshot_on.remove(&trigger) } /// Internal helper function to get a standard device handle. From 31c4a61f7d3975d4020bb9d631c577cf38aca244 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Tue, 24 Oct 2023 15:30:17 +1100 Subject: [PATCH 016/129] Refactored the snapshot feature toggle --- lib/cli/src/commands/run.rs | 1 + lib/wasix/src/lib.rs | 3 +- lib/wasix/src/os/task/process.rs | 14 +++--- lib/wasix/src/os/task/thread.rs | 12 +++-- lib/wasix/src/runners/wasi.rs | 2 + lib/wasix/src/runtime/mod.rs | 3 ++ lib/wasix/src/runtime/task_manager/mod.rs | 4 +- lib/wasix/src/snapshot/effector.rs | 1 + lib/wasix/src/snapshot/mod.rs | 2 + lib/wasix/src/state/builder.rs | 7 ++- lib/wasix/src/state/env.rs | 13 ++++- lib/wasix/src/state/func_env.rs | 4 +- lib/wasix/src/syscalls/mod.rs | 47 +++++++++++++++++-- lib/wasix/src/syscalls/wasi/fd_read.rs | 22 ++------- lib/wasix/src/syscalls/wasi/fd_write.rs | 4 +- lib/wasix/src/syscalls/wasix/proc_fork.rs | 11 +++-- .../src/syscalls/wasix/stack_checkpoint.rs | 2 +- lib/wasix/src/syscalls/wasix/thread_spawn.rs | 7 +-- lib/wasix/src/utils/mod.rs | 1 + lib/wasix/src/utils/store.rs | 4 +- 20 files changed, 109 insertions(+), 55 deletions(-) diff --git a/lib/cli/src/commands/run.rs b/lib/cli/src/commands/run.rs index 251a346784c..f3b39c9409e 100644 --- a/lib/cli/src/commands/run.rs +++ b/lib/cli/src/commands/run.rs @@ -224,6 +224,7 @@ impl Run { if self.wasi.forward_host_env { runner.set_forward_host_env(); } + #[cfg(feature = "snapshot")] for trigger in self.wasi.snapshot_on.iter().cloned() { runner.add_snapshot_trigger(trigger); } diff --git a/lib/wasix/src/lib.rs b/lib/wasix/src/lib.rs index e5b9ac3bc49..10469926cb7 100644 --- a/lib/wasix/src/lib.rs +++ b/lib/wasix/src/lib.rs @@ -50,7 +50,6 @@ pub mod http; mod rewind; pub mod runners; pub mod runtime; -#[cfg(feature = "snapshot")] pub mod snapshot; mod state; mod syscalls; @@ -105,7 +104,7 @@ pub use crate::{ utils::is_wasix_module, utils::{ get_wasi_version, get_wasi_versions, is_wasi_module, - store::{capture_snapshot, restore_snapshot, InstanceSnapshot}, + store::{capture_instance_snapshot, restore_instance_snapshot, InstanceSnapshot}, WasiVersion, }, }; diff --git a/lib/wasix/src/os/task/process.rs b/lib/wasix/src/os/task/process.rs index ef3207eecd5..787bdde385b 100644 --- a/lib/wasix/src/os/task/process.rs +++ b/lib/wasix/src/os/task/process.rs @@ -1,7 +1,6 @@ -use crate::{ - snapshot::{SnapshotEffector, SnapshotTrigger}, - unwind, WasiEnv, WasiResult, WasiRuntimeError, -}; +#[cfg(feature = "snapshot")] +use crate::{snapshot::SnapshotEffector, unwind, WasiResult}; +use crate::{snapshot::SnapshotTrigger, WasiEnv, WasiRuntimeError}; use serde::{Deserialize, Serialize}; use std::{ collections::HashMap, @@ -14,7 +13,6 @@ use std::{ }; use tracing::trace; use wasmer::FunctionEnvMut; -use wasmer_types::MemorySize; use wasmer_wasix_types::{ types::Signal, wasi::{Errno, ExitCode, Snapshot0Clockid}, @@ -141,7 +139,8 @@ pub enum MaybeCheckpointResult<'a> { impl WasiProcessInner { /// Checkpoints the process which will cause all other threads to /// pause and for the thread and memory state to be saved - pub fn checkpoint( + #[cfg(feature = "snapshot")] + pub fn checkpoint( inner: LockableWasiProcessInner, ctx: FunctionEnvMut<'_, WasiEnv>, for_what: WasiProcessCheckpoint, @@ -157,7 +156,8 @@ impl WasiProcessInner { /// If a checkpoint has been started this will block the current process /// until the checkpoint operation has completed - pub fn maybe_checkpoint( + #[cfg(feature = "snapshot")] + pub fn maybe_checkpoint( inner: LockableWasiProcessInner, ctx: FunctionEnvMut<'_, WasiEnv>, ) -> WasiResult> { diff --git a/lib/wasix/src/os/task/thread.rs b/lib/wasix/src/os/task/thread.rs index ce36f02021e..9e8a4117fa9 100644 --- a/lib/wasix/src/os/task/thread.rs +++ b/lib/wasix/src/os/task/thread.rs @@ -1,11 +1,10 @@ use serde::{Deserialize, Serialize}; +#[cfg(feature = "snapshot")] +use std::sync::atomic::{AtomicBool, Ordering}; use std::{ collections::HashMap, ops::{Deref, DerefMut}, - sync::{ - atomic::{AtomicBool, Ordering}, - Arc, Condvar, Mutex, Weak, - }, + sync::{Arc, Condvar, Mutex, Weak}, task::Waker, }; @@ -118,17 +117,20 @@ impl WasiThread { /// Sets a flag that tells others that this thread is currently /// check pointing itself + #[cfg(feature = "snapshot")] pub(crate) fn set_check_pointing(&self, val: bool) { self.state.check_pointing.store(val, Ordering::SeqCst); } /// Reads a flag that determines if this thread is currently /// check pointing itself or not + #[cfg(feature = "snapshot")] pub(crate) fn is_check_pointing(&self) -> bool { self.state.check_pointing.load(Ordering::SeqCst) } /// Gets the memory layout for this thread + #[allow(dead_code)] pub(crate) fn memory_layout(&self) -> &WasiMemoryLayout { &self.layout } @@ -197,6 +199,7 @@ struct WasiThreadState { signals: Mutex<(Vec, Vec)>, stack: Mutex, status: Arc, + #[cfg(feature = "snapshot")] check_pointing: AtomicBool, // Registers the task termination with the ControlPlane on drop. @@ -223,6 +226,7 @@ impl WasiThread { status, signals: Mutex::new((Vec::new(), Vec::new())), stack: Mutex::new(ThreadStack::default()), + #[cfg(feature = "snapshot")] check_pointing: AtomicBool::new(false), _task_count_guard: guard, }), diff --git a/lib/wasix/src/runners/wasi.rs b/lib/wasix/src/runners/wasi.rs index eb81ddb2d81..333f189b106 100644 --- a/lib/wasix/src/runners/wasi.rs +++ b/lib/wasix/src/runners/wasi.rs @@ -237,10 +237,12 @@ impl crate::runners::Runner for WasiRunner { let module = runtime.load_module_sync(cmd.atom())?; let mut store = runtime.new_store(); + #[allow(unused_mut)] let mut env = self .prepare_webc_env(command_name, &wasi, pkg, runtime, None) .context("Unable to prepare the WASI environment")?; + #[cfg(feature = "snapshot")] for snapshot_trigger in self.wasi.snapshot_on.iter().cloned() { env.add_snapshot_trigger(snapshot_trigger); } diff --git a/lib/wasix/src/runtime/mod.rs b/lib/wasix/src/runtime/mod.rs index fdfacabc787..19203fb2db6 100644 --- a/lib/wasix/src/runtime/mod.rs +++ b/lib/wasix/src/runtime/mod.rs @@ -8,6 +8,7 @@ use self::{ module_cache::{CacheError, ModuleHash}, task_manager::InlineWaker, }; +#[cfg(feature = "snapshot")] use crate::snapshot::UNSUPPORTED_SNAPSHOT_CAPTURER; use std::{ @@ -227,6 +228,7 @@ impl PluggableRuntime { source: Arc::new(source), package_loader: Arc::new(loader), module_cache: Arc::new(module_cache::in_memory()), + #[cfg(feature = "snapshot")] snapshot_capturer: Arc::new(UnsupportedSnapshotCapturer::default()) as Arc, } @@ -279,6 +281,7 @@ impl PluggableRuntime { self } + #[cfg(feature = "snapshot")] pub fn set_snapshot_capturer(&mut self, capturer: Arc) -> &mut Self { self.snapshot_capturer = capturer; self diff --git a/lib/wasix/src/runtime/task_manager/mod.rs b/lib/wasix/src/runtime/task_manager/mod.rs index e9902d07aa8..c1c53e32eac 100644 --- a/lib/wasix/src/runtime/task_manager/mod.rs +++ b/lib/wasix/src/runtime/task_manager/mod.rs @@ -14,7 +14,7 @@ use wasmer_wasix_types::wasi::{Errno, ExitCode}; use crate::os::task::thread::WasiThreadError; use crate::syscalls::AsyncifyFuture; -use crate::{capture_snapshot, InstanceSnapshot, WasiEnv, WasiFunctionEnv, WasiThread}; +use crate::{capture_instance_snapshot, InstanceSnapshot, WasiEnv, WasiFunctionEnv, WasiThread}; pub use virtual_mio::waker::*; @@ -295,7 +295,7 @@ impl dyn VirtualTaskManager { } } - let snapshot = capture_snapshot(&mut store.as_store_mut()); + let snapshot = capture_instance_snapshot(&mut store.as_store_mut()); let env = ctx.data(&store); let module = env.inner().module_clone(); let memory = env.inner().memory_clone(); diff --git a/lib/wasix/src/snapshot/effector.rs b/lib/wasix/src/snapshot/effector.rs index 56c555dbedf..a7780efcfbd 100644 --- a/lib/wasix/src/snapshot/effector.rs +++ b/lib/wasix/src/snapshot/effector.rs @@ -20,6 +20,7 @@ use super::*; #[derive(Debug, Clone)] pub struct SnapshotEffector {} +#[cfg(feature = "snapshot")] impl SnapshotEffector { pub fn save_terminal_data( ctx: &mut FunctionEnvMut<'_, WasiEnv>, diff --git a/lib/wasix/src/snapshot/mod.rs b/lib/wasix/src/snapshot/mod.rs index 51e3f560891..b05bc5343ec 100644 --- a/lib/wasix/src/snapshot/mod.rs +++ b/lib/wasix/src/snapshot/mod.rs @@ -1,5 +1,6 @@ mod capturer; mod compactor; +#[cfg(feature = "snapshot")] mod effector; mod filter; mod log_file; @@ -7,6 +8,7 @@ mod unsupported; pub use capturer::*; pub use compactor::*; +#[cfg(feature = "snapshot")] pub use effector::*; pub use filter::*; pub use log_file::*; diff --git a/lib/wasix/src/state/builder.rs b/lib/wasix/src/state/builder.rs index 3c71cfc06ee..a457f9f9fab 100644 --- a/lib/wasix/src/state/builder.rs +++ b/lib/wasix/src/state/builder.rs @@ -13,6 +13,8 @@ use virtual_fs::{ArcFile, FsError, TmpFileSystem, VirtualFile}; use wasmer::{AsStoreMut, Instance, Module, RuntimeError, Store}; use wasmer_wasix_types::wasi::{Errno, ExitCode}; +#[cfg(feature = "snapshot")] +use crate::snapshot::SnapshotTrigger; #[cfg(feature = "sys")] use crate::PluggableRuntime; use crate::{ @@ -21,7 +23,6 @@ use crate::{ fs::{WasiFs, WasiFsRoot, WasiInodes}, os::task::control_plane::{ControlPlaneConfig, ControlPlaneError, WasiControlPlane}, runtime::task_manager::InlineWaker, - snapshot::SnapshotTrigger, state::WasiState, syscalls::types::{__WASI_STDERR_FILENO, __WASI_STDIN_FILENO, __WASI_STDOUT_FILENO}, RewindState, Runtime, WasiEnv, WasiError, WasiFunctionEnv, WasiRuntimeError, @@ -72,6 +73,7 @@ pub struct WasiEnvBuilder { pub(super) capabilites: Capabilities, + #[cfg(feature = "snapshot")] pub(super) snapshot_on: Vec, } @@ -576,6 +578,7 @@ impl WasiEnvBuilder { self.capabilites = capabilities; } + #[cfg(feature = "snapshot")] pub fn add_snapshot_trigger(&mut self, on: SnapshotTrigger) { self.snapshot_on.push(on); } @@ -738,6 +741,7 @@ impl WasiEnvBuilder { }; let control_plane = WasiControlPlane::new(plane_config); + #[cfg(feature = "snapshot")] let snapshot_on = self.snapshot_on; let init = WasiEnvInit { @@ -754,6 +758,7 @@ impl WasiEnvBuilder { call_initialize: true, can_deep_sleep: false, extra_tracing: true, + #[cfg(feature = "snapshot")] snapshot_on, }; diff --git a/lib/wasix/src/state/env.rs b/lib/wasix/src/state/env.rs index 67119908dd9..21b29678d26 100644 --- a/lib/wasix/src/state/env.rs +++ b/lib/wasix/src/state/env.rs @@ -1,5 +1,7 @@ +#[cfg(feature = "snapshot")] +use std::collections::HashSet; use std::{ - collections::{HashMap, HashSet}, + collections::HashMap, ops::Deref, path::{Path, PathBuf}, sync::Arc, @@ -21,6 +23,8 @@ use wasmer_wasix_types::{ wasi::{Errno, ExitCode, Snapshot0Clockid}, }; +#[cfg(feature = "snapshot")] +use crate::snapshot::{SnapshotEffector, SnapshotTrigger}; use crate::{ bin_factory::{BinFactory, BinaryPackage}, capabilities::Capabilities, @@ -32,7 +36,6 @@ use crate::{ thread::{WasiMemoryLayout, WasiThread, WasiThreadHandle, WasiThreadId}, }, runtime::{resolver::PackageSpecifier, task_manager::InlineWaker, SpawnMemoryType}, - snapshot::{SnapshotEffector, SnapshotTrigger}, syscalls::platform_clock_time_get, Runtime, VirtualTaskManager, WasiControlPlane, WasiEnvBuilder, WasiError, WasiFunctionEnv, WasiResult, WasiRuntimeError, WasiStateCreationError, WasiVFork, @@ -233,6 +236,7 @@ pub struct WasiEnvInit { pub extra_tracing: bool, /// Indicates triggers that will cause a snapshot to be taken + #[cfg(feature = "snapshot")] pub snapshot_on: Vec, } @@ -270,6 +274,7 @@ impl WasiEnvInit { call_initialize: self.call_initialize, can_deep_sleep: self.can_deep_sleep, extra_tracing: false, + #[cfg(feature = "snapshot")] snapshot_on: self.snapshot_on.clone(), } } @@ -337,6 +342,7 @@ impl Clone for WasiEnv { runtime: self.runtime.clone(), capabilities: self.capabilities.clone(), enable_deep_sleep: self.enable_deep_sleep, + #[cfg(feature = "snapshot")] snapshot_on: self.snapshot_on.clone(), } } @@ -374,6 +380,7 @@ impl WasiEnv { runtime: self.runtime.clone(), capabilities: self.capabilities.clone(), enable_deep_sleep: self.enable_deep_sleep, + #[cfg(feature = "snapshot")] snapshot_on: self.snapshot_on.clone(), }; Ok((new_env, handle)) @@ -438,10 +445,12 @@ impl WasiEnv { bin_factory: init.bin_factory, enable_deep_sleep: init.capabilities.threading.enable_asynchronous_threading, capabilities: init.capabilities, + #[cfg(feature = "snapshot")] snapshot_on: Default::default(), }; env.owned_handles.push(thread); + #[cfg(feature = "snapshot")] for snapshot_on in init.snapshot_on { env.snapshot_on.insert(snapshot_on); } diff --git a/lib/wasix/src/state/func_env.rs b/lib/wasix/src/state/func_env.rs index 9a92732db6f..69b8001bd74 100644 --- a/lib/wasix/src/state/func_env.rs +++ b/lib/wasix/src/state/func_env.rs @@ -8,7 +8,7 @@ use crate::{ import_object_for_all_wasi_versions, runtime::SpawnMemoryType, state::WasiInstanceHandles, - utils::{get_wasi_version, get_wasi_versions, store::restore_snapshot}, + utils::{get_wasi_version, get_wasi_versions, store::restore_instance_snapshot}, InstanceSnapshot, WasiEnv, WasiError, WasiThreadError, }; @@ -74,7 +74,7 @@ impl WasiFunctionEnv { // Set all the globals if let Some(snapshot) = snapshot { tracing::trace!("restoring snapshot for new thread"); - restore_snapshot(&mut store, snapshot); + restore_instance_snapshot(&mut store, snapshot); } Ok((ctx, store)) diff --git a/lib/wasix/src/syscalls/mod.rs b/lib/wasix/src/syscalls/mod.rs index 2b688bc1265..9131a007aa1 100644 --- a/lib/wasix/src/syscalls/mod.rs +++ b/lib/wasix/src/syscalls/mod.rs @@ -117,7 +117,7 @@ use crate::{ fs_error_into_wasi_err, virtual_file_type_to_wasi_file_type, Fd, InodeVal, Kind, MAX_SYMLINKS, }, - os::task::thread::RewindResult, + os::task::{process::MaybeCheckpointResult, thread::RewindResult}, runtime::task_manager::InlineWaker, utils::store::InstanceSnapshot, DeepSleepWork, RewindPostProcess, RewindState, SpawnError, WasiInodes, WasiResult, @@ -949,7 +949,7 @@ pub(crate) fn deep_sleep( trigger: Pin>, ) -> Result<(), WasiError> { // Grab all the globals and serialize them - let store_data = crate::utils::store::capture_snapshot(&mut ctx.as_store_mut()) + let store_data = crate::utils::store::capture_instance_snapshot(&mut ctx.as_store_mut()) .serialize() .unwrap(); let store_data = Bytes::from(store_data); @@ -1148,7 +1148,7 @@ pub fn rewind_ext( return Errno::Unknown; } }; - crate::utils::store::restore_snapshot(&mut ctx, &store_snapshot); + crate::utils::store::restore_instance_snapshot(&mut ctx, &store_snapshot); let env = ctx.data(); let memory = match env.try_memory_view(&ctx) { Some(v) => v, @@ -1212,6 +1212,47 @@ pub fn rewind_ext( Errno::Success } +#[cfg(not(feature = "snapshot"))] +pub fn maybe_snapshot( + ctx: FunctionEnvMut<'_, WasiEnv>, + _trigger: crate::snapshot::SnapshotTrigger, +) -> WasiResult> { + Ok(Ok(ctx)) +} + +#[cfg(feature = "snapshot")] +pub fn maybe_snapshot( + mut ctx: FunctionEnvMut<'_, WasiEnv>, + trigger: crate::snapshot::SnapshotTrigger, +) -> WasiResult> { + use crate::os::task::process::{WasiProcessCheckpoint, WasiProcessInner}; + + if ctx.data_mut().pop_snapshot_trigger(trigger) { + let inner = ctx.data().process.inner.clone(); + let res = wasi_try_ok_ok!(WasiProcessInner::checkpoint::( + inner, + ctx, + WasiProcessCheckpoint::Snapshot { trigger: trigger }, + )?); + match res { + MaybeCheckpointResult::Unwinding => return Ok(Err(Errno::Success)), + MaybeCheckpointResult::NotThisTime(c) => { + ctx = c; + } + } + } else if ctx.data().should_feed_snapshot() { + let inner = ctx.data().process.inner.clone(); + let res = wasi_try_ok_ok!(WasiProcessInner::maybe_checkpoint::(inner, ctx)?); + match res { + MaybeCheckpointResult::Unwinding => return Ok(Err(Errno::Success)), + MaybeCheckpointResult::NotThisTime(c) => { + ctx = c; + } + } + } + Ok(Ok(ctx)) +} + pub(crate) unsafe fn handle_rewind( ctx: &mut FunctionEnvMut<'_, WasiEnv>, ) -> Option diff --git a/lib/wasix/src/syscalls/wasi/fd_read.rs b/lib/wasix/src/syscalls/wasi/fd_read.rs index 036bee058ff..15cbeb1ba2c 100644 --- a/lib/wasix/src/syscalls/wasi/fd_read.rs +++ b/lib/wasix/src/syscalls/wasi/fd_read.rs @@ -44,6 +44,8 @@ pub fn fd_read( fd_entry.offset.load(Ordering::Acquire) as usize }; + ctx = wasi_try_ok!(maybe_snapshot::(ctx, SnapshotTrigger::Stdin)?); + let res = fd_read_internal::(&mut ctx, fd, iovs, iovs_len, offset, nread, true)?; fd_read_internal_handler(ctx, res, nread) } @@ -75,25 +77,7 @@ pub fn fd_pread( let pid = ctx.data().pid(); let tid = ctx.data().tid(); - // If snap-shooting is enabled and this is to stdio then we - // we may actually trigger a snapshot event heere. - #[cfg(feature = "snapshot")] - if fd == DeviceFile::STDIN && ctx.data_mut().pop_snapshot_trigger(SnapshotTrigger::Stdin) { - let inner = ctx.data().process.inner.clone(); - let res = wasi_try_ok!(WasiProcessInner::checkpoint::( - inner, - ctx, - WasiProcessCheckpoint::Snapshot { - trigger: SnapshotTrigger::Stdin, - }, - )?); - match res { - MaybeCheckpointResult::Unwinding => return Ok(Errno::Success), - MaybeCheckpointResult::NotThisTime(c) => { - ctx = c; - } - } - } + ctx = wasi_try_ok!(maybe_snapshot::(ctx, SnapshotTrigger::Stdin)?); let res = fd_read_internal::(&mut ctx, fd, iovs, iovs_len, offset as usize, nread, false)?; fd_read_internal_handler::(ctx, res, nread) diff --git a/lib/wasix/src/syscalls/wasi/fd_write.rs b/lib/wasix/src/syscalls/wasi/fd_write.rs index e36ec92c5b9..1cc912a3459 100644 --- a/lib/wasix/src/syscalls/wasi/fd_write.rs +++ b/lib/wasix/src/syscalls/wasi/fd_write.rs @@ -1,10 +1,10 @@ use std::task::Waker; use super::*; +use crate::{net::socket::TimeType, syscalls::*}; +#[cfg(feature = "snapshot")] use crate::{ - net::socket::TimeType, snapshot::{SnapshotEffector, SnapshotLog}, - syscalls::*, utils::map_snapshot_err, }; diff --git a/lib/wasix/src/syscalls/wasix/proc_fork.rs b/lib/wasix/src/syscalls/wasix/proc_fork.rs index 3a660dcccda..d55bc864225 100644 --- a/lib/wasix/src/syscalls/wasix/proc_fork.rs +++ b/lib/wasix/src/syscalls/wasix/proc_fork.rs @@ -1,6 +1,6 @@ use super::*; use crate::{ - capture_snapshot, + capture_instance_snapshot, os::task::OwnedTaskStatus, runtime::task_manager::{TaskWasm, TaskWasmRunProperties}, syscalls::*, @@ -84,9 +84,10 @@ pub fn proc_fork( // Perform the unwind action return unwind::(ctx, move |mut ctx, mut memory_stack, rewind_stack| { // Grab all the globals and serialize them - let store_data = crate::utils::store::capture_snapshot(&mut ctx.as_store_mut()) - .serialize() - .unwrap(); + let store_data = + crate::utils::store::capture_instance_snapshot(&mut ctx.as_store_mut()) + .serialize() + .unwrap(); let store_data = Bytes::from(store_data); // We first fork the environment and replace the current environment @@ -129,7 +130,7 @@ pub fn proc_fork( let bin_factory = env.bin_factory.clone(); // Perform the unwind action - let snapshot = capture_snapshot(&mut ctx.as_store_mut()); + let snapshot = capture_instance_snapshot(&mut ctx.as_store_mut()); unwind::(ctx, move |mut ctx, mut memory_stack, rewind_stack| { let tasks = ctx.data().tasks().clone(); let span = debug_span!( diff --git a/lib/wasix/src/syscalls/wasix/stack_checkpoint.rs b/lib/wasix/src/syscalls/wasix/stack_checkpoint.rs index 59b8d3a430d..84fefad10ba 100644 --- a/lib/wasix/src/syscalls/wasix/stack_checkpoint.rs +++ b/lib/wasix/src/syscalls/wasix/stack_checkpoint.rs @@ -45,7 +45,7 @@ pub fn stack_checkpoint( // Perform the unwind action unwind::(ctx, move |mut ctx, mut memory_stack, rewind_stack| { // Grab all the globals and serialize them - let store_data = crate::utils::store::capture_snapshot(&mut ctx.as_store_mut()) + let store_data = crate::utils::store::capture_instance_snapshot(&mut ctx.as_store_mut()) .serialize() .unwrap(); let env = ctx.data(); diff --git a/lib/wasix/src/syscalls/wasix/thread_spawn.rs b/lib/wasix/src/syscalls/wasix/thread_spawn.rs index 7ddb78a46e2..e1f0dbdb50d 100644 --- a/lib/wasix/src/syscalls/wasix/thread_spawn.rs +++ b/lib/wasix/src/syscalls/wasix/thread_spawn.rs @@ -1,11 +1,12 @@ use std::f32::consts::E; use super::*; +#[cfg(feature = "snapshot")] +use crate::snapshot::SnapshotEffector; use crate::{ - capture_snapshot, + capture_instance_snapshot, os::task::thread::WasiMemoryLayout, runtime::task_manager::{TaskWasm, TaskWasmRunProperties}, - snapshot::SnapshotEffector, syscalls::*, WasiThreadHandle, }; @@ -120,7 +121,7 @@ pub(crate) fn thread_spawn_internal( return Err(Errno::Notcapable); } let thread_module = unsafe { env.inner() }.module_clone(); - let snapshot = capture_snapshot(&mut ctx.as_store_mut()); + let snapshot = capture_instance_snapshot(&mut ctx.as_store_mut()); let spawn_type = crate::runtime::SpawnMemoryType::ShareMemory(thread_memory, ctx.as_store_ref()); diff --git a/lib/wasix/src/utils/mod.rs b/lib/wasix/src/utils/mod.rs index a36fd6d4651..52fa38e1d5a 100644 --- a/lib/wasix/src/utils/mod.rs +++ b/lib/wasix/src/utils/mod.rs @@ -38,6 +38,7 @@ pub fn map_io_err(err: std::io::Error) -> Errno { From::::from(err) } +#[cfg(feature = "snapshot")] pub fn map_snapshot_err(err: anyhow::Error) -> Errno { tracing::warn!("unknown snapshot error: {}", err); Errno::Unknown diff --git a/lib/wasix/src/utils/store.rs b/lib/wasix/src/utils/store.rs index 047b09d9aef..1fac581ba90 100644 --- a/lib/wasix/src/utils/store.rs +++ b/lib/wasix/src/utils/store.rs @@ -15,13 +15,13 @@ impl InstanceSnapshot { } } -pub fn capture_snapshot(store: &mut impl wasmer::AsStoreMut) -> InstanceSnapshot { +pub fn capture_instance_snapshot(store: &mut impl wasmer::AsStoreMut) -> InstanceSnapshot { let objs = store.objects_mut(); let globals = objs.as_u128_globals(); InstanceSnapshot { globals } } -pub fn restore_snapshot(store: &mut impl wasmer::AsStoreMut, snapshot: &InstanceSnapshot) { +pub fn restore_instance_snapshot(store: &mut impl wasmer::AsStoreMut, snapshot: &InstanceSnapshot) { let objs = store.objects_mut(); for (index, value) in snapshot.globals.iter().enumerate() { From 24368b504ec43dc9a1cbc053b2c944969bb45ae5 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Tue, 24 Oct 2023 16:29:13 +1100 Subject: [PATCH 017/129] More wiring of the snapshot options and the intercepts --- lib/cli/src/commands/run.rs | 9 +++- lib/cli/src/commands/run/wasi.rs | 5 +++ lib/wasix/src/os/task/process.rs | 6 +++ lib/wasix/src/runners/wcgi/runner.rs | 5 +++ lib/wasix/src/snapshot/capturer.rs | 1 + lib/wasix/src/snapshot/unsupported.rs | 3 +- lib/wasix/src/state/env.rs | 18 ++++---- lib/wasix/src/syscalls/mod.rs | 43 +++++++++++++++----- lib/wasix/src/syscalls/wasi/fd_read.rs | 8 +++- lib/wasix/src/syscalls/wasi/fd_write.rs | 2 +- lib/wasix/src/syscalls/wasi/poll_oneoff.rs | 2 + lib/wasix/src/syscalls/wasix/epoll_wait.rs | 2 + lib/wasix/src/syscalls/wasix/futex_wait.rs | 2 + lib/wasix/src/syscalls/wasix/proc_join.rs | 2 + lib/wasix/src/syscalls/wasix/sock_accept.rs | 2 + lib/wasix/src/syscalls/wasix/sock_listen.rs | 12 +++--- lib/wasix/src/syscalls/wasix/thread_join.rs | 2 + lib/wasix/src/syscalls/wasix/thread_sleep.rs | 2 + 18 files changed, 99 insertions(+), 27 deletions(-) diff --git a/lib/cli/src/commands/run.rs b/lib/cli/src/commands/run.rs index f3b39c9409e..5c3afc40d91 100644 --- a/lib/cli/src/commands/run.rs +++ b/lib/cli/src/commands/run.rs @@ -224,6 +224,7 @@ impl Run { if self.wasi.forward_host_env { runner.set_forward_host_env(); } + #[cfg(feature = "snapshot")] for trigger in self.wasi.snapshot_on.iter().cloned() { runner.add_snapshot_trigger(trigger); @@ -255,6 +256,11 @@ impl Run { runner.config().forward_host_env(); } + #[cfg(feature = "snapshot")] + for trigger in self.wasi.snapshot_on.iter().cloned() { + runner.config().add_snapshot_trigger(trigger); + } + runner.run_command(command_name, pkg, runtime) } @@ -312,7 +318,8 @@ impl Run { ) -> Result<(), Error> { let program_name = wasm_path.display().to_string(); - let builder = self + #[allow(unused_mut)] + let mut builder = self .wasi .prepare(module, program_name, self.args.clone(), runtime)?; diff --git a/lib/cli/src/commands/run/wasi.rs b/lib/cli/src/commands/run/wasi.rs index d1ce9215e30..eb81f892828 100644 --- a/lib/cli/src/commands/run/wasi.rs +++ b/lib/cli/src/commands/run/wasi.rs @@ -258,6 +258,11 @@ impl Wasi { *builder.capabilities_mut() = self.capabilities(); + #[cfg(feature = "snapshot")] + for trigger in self.snapshot_on.iter().cloned() { + builder.add_snapshot_trigger(trigger); + } + #[cfg(feature = "experimental-io-devices")] { if self.enable_experimental_io_devices { diff --git a/lib/wasix/src/os/task/process.rs b/lib/wasix/src/os/task/process.rs index 787bdde385b..ac1ef0de08b 100644 --- a/lib/wasix/src/os/task/process.rs +++ b/lib/wasix/src/os/task/process.rs @@ -172,6 +172,12 @@ impl WasiProcessInner { // Perform the unwind action unwind::(ctx, move |mut ctx, memory_stack, rewind_stack| { + tracing::debug!( + "stack snapshot unwind (memory_stack={}, rewind_stack={})", + memory_stack.len(), + rewind_stack.len() + ); + // Write our thread state to the snapshot let tid = ctx.data().thread.tid(); if let Err(err) = SnapshotEffector::save_thread_state( diff --git a/lib/wasix/src/runners/wcgi/runner.rs b/lib/wasix/src/runners/wcgi/runner.rs index 66bbf40b77c..c765fc04c2c 100644 --- a/lib/wasix/src/runners/wcgi/runner.rs +++ b/lib/wasix/src/runners/wcgi/runner.rs @@ -241,6 +241,11 @@ impl Config { pub fn capabilities(&mut self) -> &mut Capabilities { &mut self.wasi.capabilities } + + #[cfg(feature = "snapshot")] + pub fn add_snapshot_trigger(&mut self, on: crate::snapshot::SnapshotTrigger) { + self.wasi.snapshot_on.push(on); + } } impl Default for Config { diff --git a/lib/wasix/src/snapshot/capturer.rs b/lib/wasix/src/snapshot/capturer.rs index 55eda1ee719..2e24020fd0b 100644 --- a/lib/wasix/src/snapshot/capturer.rs +++ b/lib/wasix/src/snapshot/capturer.rs @@ -100,6 +100,7 @@ pub enum FileEntryType { /// Represents a log entry in a snapshot log stream that represents the total /// state of a WASM process at a point in time. +#[derive(Debug)] pub enum SnapshotLog<'a> { Init { wasm_hash: [u8; 32], diff --git a/lib/wasix/src/snapshot/unsupported.rs b/lib/wasix/src/snapshot/unsupported.rs index b11b4cdb15e..4729b222c06 100644 --- a/lib/wasix/src/snapshot/unsupported.rs +++ b/lib/wasix/src/snapshot/unsupported.rs @@ -11,7 +11,8 @@ pub static UNSUPPORTED_SNAPSHOT_CAPTURER: UnsupportedSnapshotCapturer = pub struct UnsupportedSnapshotCapturer {} impl SnapshotCapturer for UnsupportedSnapshotCapturer { - fn write<'a>(&'a self, _entry: SnapshotLog<'a>) -> BoxFuture<'a, anyhow::Result<()>> { + fn write<'a>(&'a self, entry: SnapshotLog<'a>) -> BoxFuture<'a, anyhow::Result<()>> { + tracing::debug!("snapshot event: {:?}", entry); Box::pin(async { Err(anyhow::format_err!("unsupported")) }) } diff --git a/lib/wasix/src/state/env.rs b/lib/wasix/src/state/env.rs index 21b29678d26..a06b61980f3 100644 --- a/lib/wasix/src/state/env.rs +++ b/lib/wasix/src/state/env.rs @@ -309,9 +309,12 @@ pub struct WasiEnv { /// Is this environment capable and setup for deep sleeping pub enable_deep_sleep: bool, + /// Enables the snap shotting functionality + pub enable_snapshot_capture: bool, + /// List of situations that the process will checkpoint on #[cfg(feature = "snapshot")] - pub(crate) snapshot_on: HashSet, + snapshot_on: HashSet, /// Inner functions and references that are loaded before the environment starts /// (inner is not safe to send between threads and so it is private and will @@ -342,6 +345,7 @@ impl Clone for WasiEnv { runtime: self.runtime.clone(), capabilities: self.capabilities.clone(), enable_deep_sleep: self.enable_deep_sleep, + enable_snapshot_capture: self.enable_snapshot_capture, #[cfg(feature = "snapshot")] snapshot_on: self.snapshot_on.clone(), } @@ -380,6 +384,7 @@ impl WasiEnv { runtime: self.runtime.clone(), capabilities: self.capabilities.clone(), enable_deep_sleep: self.enable_deep_sleep, + enable_snapshot_capture: self.enable_snapshot_capture, #[cfg(feature = "snapshot")] snapshot_on: self.snapshot_on.clone(), }; @@ -446,15 +451,14 @@ impl WasiEnv { enable_deep_sleep: init.capabilities.threading.enable_asynchronous_threading, capabilities: init.capabilities, #[cfg(feature = "snapshot")] - snapshot_on: Default::default(), + enable_snapshot_capture: !init.snapshot_on.is_empty(), + #[cfg(not(feature = "snapshot"))] + enable_snapshot_capture: false, + #[cfg(feature = "snapshot")] + snapshot_on: init.snapshot_on.into_iter().collect(), }; env.owned_handles.push(thread); - #[cfg(feature = "snapshot")] - for snapshot_on in init.snapshot_on { - env.snapshot_on.insert(snapshot_on); - } - // TODO: should not be here - should be callers responsibility! for pkg in &init.webc_dependencies { env.use_package(pkg)?; diff --git a/lib/wasix/src/syscalls/mod.rs b/lib/wasix/src/syscalls/mod.rs index 9131a007aa1..abade710157 100644 --- a/lib/wasix/src/syscalls/mod.rs +++ b/lib/wasix/src/syscalls/mod.rs @@ -1213,7 +1213,7 @@ pub fn rewind_ext( } #[cfg(not(feature = "snapshot"))] -pub fn maybe_snapshot( +pub fn maybe_snapshot_once( ctx: FunctionEnvMut<'_, WasiEnv>, _trigger: crate::snapshot::SnapshotTrigger, ) -> WasiResult> { @@ -1221,12 +1221,16 @@ pub fn maybe_snapshot( } #[cfg(feature = "snapshot")] -pub fn maybe_snapshot( +pub fn maybe_snapshot_once( mut ctx: FunctionEnvMut<'_, WasiEnv>, trigger: crate::snapshot::SnapshotTrigger, ) -> WasiResult> { use crate::os::task::process::{WasiProcessCheckpoint, WasiProcessInner}; + if ctx.data().enable_snapshot_capture == false { + return Ok(Ok(ctx)); + } + if ctx.data_mut().pop_snapshot_trigger(trigger) { let inner = ctx.data().process.inner.clone(); let res = wasi_try_ok_ok!(WasiProcessInner::checkpoint::( @@ -1240,14 +1244,33 @@ pub fn maybe_snapshot( ctx = c; } } - } else if ctx.data().should_feed_snapshot() { - let inner = ctx.data().process.inner.clone(); - let res = wasi_try_ok_ok!(WasiProcessInner::maybe_checkpoint::(inner, ctx)?); - match res { - MaybeCheckpointResult::Unwinding => return Ok(Err(Errno::Success)), - MaybeCheckpointResult::NotThisTime(c) => { - ctx = c; - } + } + Ok(Ok(ctx)) +} + +#[cfg(not(feature = "snapshot"))] +pub fn maybe_snapshot( + ctx: FunctionEnvMut<'_, WasiEnv>, +) -> WasiResult> { + Ok(Ok(ctx)) +} + +#[cfg(feature = "snapshot")] +pub fn maybe_snapshot( + mut ctx: FunctionEnvMut<'_, WasiEnv>, +) -> WasiResult> { + use crate::os::task::process::{WasiProcessCheckpoint, WasiProcessInner}; + + if ctx.data().enable_snapshot_capture == false { + return Ok(Ok(ctx)); + } + + let inner = ctx.data().process.inner.clone(); + let res = wasi_try_ok_ok!(WasiProcessInner::maybe_checkpoint::(inner, ctx)?); + match res { + MaybeCheckpointResult::Unwinding => return Ok(Err(Errno::Success)), + MaybeCheckpointResult::NotThisTime(c) => { + ctx = c; } } Ok(Ok(ctx)) diff --git a/lib/wasix/src/syscalls/wasi/fd_read.rs b/lib/wasix/src/syscalls/wasi/fd_read.rs index 15cbeb1ba2c..16cbae4c549 100644 --- a/lib/wasix/src/syscalls/wasi/fd_read.rs +++ b/lib/wasix/src/syscalls/wasi/fd_read.rs @@ -44,7 +44,9 @@ pub fn fd_read( fd_entry.offset.load(Ordering::Acquire) as usize }; - ctx = wasi_try_ok!(maybe_snapshot::(ctx, SnapshotTrigger::Stdin)?); + if fd == DeviceFile::STDIN { + ctx = wasi_try_ok!(maybe_snapshot_once::(ctx, SnapshotTrigger::Stdin)?); + } let res = fd_read_internal::(&mut ctx, fd, iovs, iovs_len, offset, nread, true)?; fd_read_internal_handler(ctx, res, nread) @@ -77,7 +79,9 @@ pub fn fd_pread( let pid = ctx.data().pid(); let tid = ctx.data().tid(); - ctx = wasi_try_ok!(maybe_snapshot::(ctx, SnapshotTrigger::Stdin)?); + if fd == DeviceFile::STDIN { + ctx = wasi_try_ok!(maybe_snapshot_once::(ctx, SnapshotTrigger::Stdin)?); + } let res = fd_read_internal::(&mut ctx, fd, iovs, iovs_len, offset as usize, nread, false)?; fd_read_internal_handler::(ctx, res, nread) diff --git a/lib/wasix/src/syscalls/wasi/fd_write.rs b/lib/wasix/src/syscalls/wasi/fd_write.rs index 1cc912a3459..b184aa0dd73 100644 --- a/lib/wasix/src/syscalls/wasi/fd_write.rs +++ b/lib/wasix/src/syscalls/wasi/fd_write.rs @@ -107,7 +107,7 @@ pub(crate) fn fd_write_internal( // If snap-shooting is enabled and this is to stdio then we // will record a terminal event. #[cfg(feature = "snapshot")] - if is_stdio && ctx.data().should_feed_snapshot() { + if is_stdio && env.enable_snapshot_capture { SnapshotEffector::save_terminal_data(&mut ctx, iovs, iovs_len)?; env = ctx.data(); } diff --git a/lib/wasix/src/syscalls/wasi/poll_oneoff.rs b/lib/wasix/src/syscalls/wasi/poll_oneoff.rs index 2c9a743afb0..a4957a49d0e 100644 --- a/lib/wasix/src/syscalls/wasi/poll_oneoff.rs +++ b/lib/wasix/src/syscalls/wasi/poll_oneoff.rs @@ -64,6 +64,8 @@ pub fn poll_oneoff( ) -> Result { wasi_try_ok!(WasiEnv::process_signals_and_exit(&mut ctx)?); + ctx = wasi_try_ok!(maybe_snapshot::(ctx)?); + ctx.data_mut().poll_seed += 1; let mut env = ctx.data(); let mut memory = unsafe { env.memory_view(&ctx) }; diff --git a/lib/wasix/src/syscalls/wasix/epoll_wait.rs b/lib/wasix/src/syscalls/wasix/epoll_wait.rs index 37c7dc75671..c3ea56dea51 100644 --- a/lib/wasix/src/syscalls/wasix/epoll_wait.rs +++ b/lib/wasix/src/syscalls/wasix/epoll_wait.rs @@ -26,6 +26,8 @@ pub fn epoll_wait<'a, M: MemorySize + 'static>( ) -> Result { wasi_try_ok!(WasiEnv::process_signals_and_exit(&mut ctx)?); + ctx = wasi_try_ok!(maybe_snapshot::(ctx)?); + if timeout == TIMEOUT_FOREVER { tracing::trace!(maxevents, epfd, "waiting forever on wakers"); } else { diff --git a/lib/wasix/src/syscalls/wasix/futex_wait.rs b/lib/wasix/src/syscalls/wasix/futex_wait.rs index a66889289e9..241d026d8dc 100644 --- a/lib/wasix/src/syscalls/wasix/futex_wait.rs +++ b/lib/wasix/src/syscalls/wasix/futex_wait.rs @@ -90,6 +90,8 @@ pub(super) fn futex_wait_internal( ) -> Result { wasi_try_ok!(WasiEnv::process_signals_and_exit(&mut ctx)?); + ctx = wasi_try_ok!(maybe_snapshot::(ctx)?); + // If we were just restored then we were woken after a deep sleep // and thus we repeat all the checks again, we do not immediately // exit here as it could be the case that we were woken but the diff --git a/lib/wasix/src/syscalls/wasix/proc_join.rs b/lib/wasix/src/syscalls/wasix/proc_join.rs index bb56f2d4710..a8b29b53344 100644 --- a/lib/wasix/src/syscalls/wasix/proc_join.rs +++ b/lib/wasix/src/syscalls/wasix/proc_join.rs @@ -38,6 +38,8 @@ pub(super) fn proc_join_internal( ) -> Result { wasi_try_ok!(WasiEnv::process_signals_and_exit(&mut ctx)?); + ctx = wasi_try_ok!(maybe_snapshot::(ctx)?); + // This lambda will look at what we wrote in the status variable // and use this to determine the return code sent back to the caller let ret_result = { diff --git a/lib/wasix/src/syscalls/wasix/sock_accept.rs b/lib/wasix/src/syscalls/wasix/sock_accept.rs index ce17a0f6c54..1768f4e4d8e 100644 --- a/lib/wasix/src/syscalls/wasix/sock_accept.rs +++ b/lib/wasix/src/syscalls/wasix/sock_accept.rs @@ -24,6 +24,8 @@ pub fn sock_accept( ) -> Result { wasi_try_ok!(WasiEnv::process_signals_and_exit(&mut ctx)?); + ctx = wasi_try_ok!(maybe_snapshot::(ctx)?); + let env = ctx.data(); let (memory, state, _) = unsafe { env.get_memory_and_wasi_state_and_inodes(&ctx, 0) }; diff --git a/lib/wasix/src/syscalls/wasix/sock_listen.rs b/lib/wasix/src/syscalls/wasix/sock_listen.rs index 50126ce6dce..0fe336db047 100644 --- a/lib/wasix/src/syscalls/wasix/sock_listen.rs +++ b/lib/wasix/src/syscalls/wasix/sock_listen.rs @@ -1,5 +1,5 @@ use super::*; -use crate::syscalls::*; +use crate::{snapshot::SnapshotTrigger, syscalls::*}; /// ### `sock_listen()` /// Listen for connections on a socket @@ -18,18 +18,20 @@ pub fn sock_listen( mut ctx: FunctionEnvMut<'_, WasiEnv>, sock: WasiFd, backlog: M::Offset, -) -> Errno { +) -> Result { + ctx = wasi_try_ok!(maybe_snapshot_once::(ctx, SnapshotTrigger::Listen)?); + let env = ctx.data(); let net = env.net().clone(); - let backlog: usize = wasi_try!(backlog.try_into().map_err(|_| Errno::Inval)); + let backlog: usize = wasi_try_ok!(backlog.try_into().map_err(|_| Errno::Inval)); let tasks = ctx.data().tasks().clone(); - wasi_try!(__sock_upgrade( + wasi_try_ok!(__sock_upgrade( &mut ctx, sock, Rights::SOCK_LISTEN, |socket| async move { socket.listen(tasks.deref(), net.deref(), backlog).await } )); - Errno::Success + Ok(Errno::Success) } diff --git a/lib/wasix/src/syscalls/wasix/thread_join.rs b/lib/wasix/src/syscalls/wasix/thread_join.rs index 20d4a13d48d..2be67a229a3 100644 --- a/lib/wasix/src/syscalls/wasix/thread_join.rs +++ b/lib/wasix/src/syscalls/wasix/thread_join.rs @@ -27,6 +27,8 @@ pub(super) fn thread_join_internal( return Ok(Errno::Success); } + ctx = wasi_try_ok!(maybe_snapshot::(ctx)?); + let env = ctx.data(); let tid: WasiThreadId = join_tid.into(); let other_thread = env.process.get_thread(&tid); diff --git a/lib/wasix/src/syscalls/wasix/thread_sleep.rs b/lib/wasix/src/syscalls/wasix/thread_sleep.rs index 5157a8befd5..1322fc640f2 100644 --- a/lib/wasix/src/syscalls/wasix/thread_sleep.rs +++ b/lib/wasix/src/syscalls/wasix/thread_sleep.rs @@ -27,6 +27,8 @@ pub(crate) fn thread_sleep_internal( return Ok(Errno::Success); } + ctx = wasi_try_ok!(maybe_snapshot::(ctx)?); + let env = ctx.data(); #[cfg(feature = "sys-thread")] From e29917032982ecfc52fd60bf89a23f0e0df48c1a Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Wed, 25 Oct 2023 07:19:15 +1100 Subject: [PATCH 018/129] Added a snapshot trigger on reading the environment variables --- lib/wasix/src/snapshot/mod.rs | 3 +++ lib/wasix/src/syscalls/wasi/environ_get.rs | 15 +++++++++++---- .../src/syscalls/wasi/environ_sizes_get.rs | 19 +++++++++++-------- 3 files changed, 25 insertions(+), 12 deletions(-) diff --git a/lib/wasix/src/snapshot/mod.rs b/lib/wasix/src/snapshot/mod.rs index b05bc5343ec..0338567f95d 100644 --- a/lib/wasix/src/snapshot/mod.rs +++ b/lib/wasix/src/snapshot/mod.rs @@ -25,6 +25,8 @@ pub enum SnapshotTrigger { Idle, /// Triggered when a listen syscall is invoked on a socket Listen, + /// Triggered on reading the environment variables for the first time + Environ, /// Triggered when the process reads stdin for the first time Stdin, /// Triggered periodically based on a timer (default 10 seconds) which can be specified using the `snapshot-timer` option @@ -50,6 +52,7 @@ impl FromStr for SnapshotTrigger { "idle" => Self::Idle, "listen" => Self::Listen, "stdin" => Self::Stdin, + "environ" => Self::Environ, "periodic" => Self::Timer, "intr" | "sigint" | "ctrlc" | "ctrl-c" => Self::Sigint, "alarm" | "timer" | "sigalrm" => Self::Sigalrm, diff --git a/lib/wasix/src/syscalls/wasi/environ_get.rs b/lib/wasix/src/syscalls/wasi/environ_get.rs index fabd186be92..3c2443e13b4 100644 --- a/lib/wasix/src/syscalls/wasi/environ_get.rs +++ b/lib/wasix/src/syscalls/wasi/environ_get.rs @@ -1,5 +1,5 @@ use super::*; -use crate::syscalls::*; +use crate::{snapshot::SnapshotTrigger, syscalls::*}; /// ### `environ_get()` /// Read environment variable data. @@ -11,12 +11,19 @@ use crate::syscalls::*; /// A pointer to a buffer to write the environment variable string data. #[instrument(level = "debug", skip_all, ret)] pub fn environ_get( - ctx: FunctionEnvMut<'_, WasiEnv>, + mut ctx: FunctionEnvMut<'_, WasiEnv>, environ: WasmPtr, M>, environ_buf: WasmPtr, -) -> Errno { +) -> Result { + ctx = wasi_try_ok!(maybe_snapshot_once::(ctx, SnapshotTrigger::Environ)?); + let env = ctx.data(); let (memory, mut state) = unsafe { env.get_memory_and_wasi_state(&ctx, 0) }; - write_buffer_array(&memory, &state.envs, environ, environ_buf) + Ok(write_buffer_array( + &memory, + &state.envs, + environ, + environ_buf, + )) } diff --git a/lib/wasix/src/syscalls/wasi/environ_sizes_get.rs b/lib/wasix/src/syscalls/wasi/environ_sizes_get.rs index eb3b3f08d98..0e6a9bf8d03 100644 --- a/lib/wasix/src/syscalls/wasi/environ_sizes_get.rs +++ b/lib/wasix/src/syscalls/wasi/environ_sizes_get.rs @@ -1,5 +1,5 @@ use super::*; -use crate::syscalls::*; +use crate::{snapshot::SnapshotTrigger, syscalls::*}; /// ### `environ_sizes_get()` /// Return command-line argument data sizes. @@ -10,10 +10,12 @@ use crate::syscalls::*; /// The size of the environment variable string data. #[instrument(level = "trace", skip_all, ret)] pub fn environ_sizes_get( - ctx: FunctionEnvMut<'_, WasiEnv>, + mut ctx: FunctionEnvMut<'_, WasiEnv>, environ_count: WasmPtr, environ_buf_size: WasmPtr, -) -> Errno { +) -> Result { + ctx = wasi_try_ok!(maybe_snapshot_once::(ctx, SnapshotTrigger::Environ)?); + let env = ctx.data(); let (memory, mut state) = unsafe { env.get_memory_and_wasi_state(&ctx, 0) }; @@ -21,16 +23,17 @@ pub fn environ_sizes_get( let environ_buf_size = environ_buf_size.deref(&memory); let env_var_count: M::Offset = - wasi_try!(state.envs.len().try_into().map_err(|_| Errno::Overflow)); + wasi_try_ok!(state.envs.len().try_into().map_err(|_| Errno::Overflow)); let env_buf_size: usize = state.envs.iter().map(|v| v.len() + 1).sum(); - let env_buf_size: M::Offset = wasi_try!(env_buf_size.try_into().map_err(|_| Errno::Overflow)); - wasi_try_mem!(environ_count.write(env_var_count)); - wasi_try_mem!(environ_buf_size.write(env_buf_size)); + let env_buf_size: M::Offset = + wasi_try_ok!(env_buf_size.try_into().map_err(|_| Errno::Overflow)); + wasi_try_mem_ok!(environ_count.write(env_var_count)); + wasi_try_mem_ok!(environ_buf_size.write(env_buf_size)); trace!( %env_var_count, %env_buf_size ); - Errno::Success + Ok(Errno::Success) } From 63980d8c4e9a190e1b8ac2a8d9990b353f2c7544 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Wed, 25 Oct 2023 09:29:00 +1100 Subject: [PATCH 019/129] Minor document updates --- docs/snapshots.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/snapshots.md b/docs/snapshots.md index 1df9566d479..7bba60be9ee 100644 --- a/docs/snapshots.md +++ b/docs/snapshots.md @@ -20,6 +20,10 @@ Triggered when a listen syscall is invoked on a socket. Triggered when the process reads stdin for the first time +## On Environ + +Triggered when the process reads an environment variable for the first time + ## On Timer Triggered periodically based on a timer (default 10 seconds) which can be specified using the `snapshot-timer` option From ffeeb33d01dea7680871a9b0249ebdb50f46c002 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Wed, 25 Oct 2023 09:56:40 +1100 Subject: [PATCH 020/129] Process will now properly terminate when snapshots fail --- lib/api/src/sys/externals/function.rs | 2 +- lib/wasix/src/os/task/process.rs | 4 +- lib/wasix/src/snapshot/capturer.rs | 73 ++++++++++++++++++++++++- lib/wasix/src/snapshot/effector.rs | 51 +++++++++-------- lib/wasix/src/syscalls/mod.rs | 20 +++---- lib/wasix/src/syscalls/wasi/fd_write.rs | 8 ++- 6 files changed, 120 insertions(+), 38 deletions(-) diff --git a/lib/api/src/sys/externals/function.rs b/lib/api/src/sys/externals/function.rs index 1ce6fe64007..41c292f90cb 100644 --- a/lib/api/src/sys/externals/function.rs +++ b/lib/api/src/sys/externals/function.rs @@ -295,7 +295,7 @@ impl Function { break; } Ok(wasmer_types::OnCalledAction::Trap(trap)) => { - return Err(RuntimeError::user(trap)) + return Err(RuntimeError::user(trap)); } Err(trap) => return Err(RuntimeError::user(trap)), } diff --git a/lib/wasix/src/os/task/process.rs b/lib/wasix/src/os/task/process.rs index ac1ef0de08b..0e1aabd257e 100644 --- a/lib/wasix/src/os/task/process.rs +++ b/lib/wasix/src/os/task/process.rs @@ -186,7 +186,7 @@ impl WasiProcessInner { memory_stack.freeze(), rewind_stack.freeze(), ) { - return wasmer_types::OnCalledAction::Trap(Box::new(err)); + return wasmer_types::OnCalledAction::Trap(err.into()); } let mut guard = inner.0.lock().unwrap(); @@ -213,7 +213,7 @@ impl WasiProcessInner { &mut ctx, &mut guard, trigger, ) { inner.1.notify_all(); - return wasmer_types::OnCalledAction::Trap(Box::new(err)); + return wasmer_types::OnCalledAction::Trap(err.into()); } // Clear the checkpointing flag and notify everyone to wake up diff --git a/lib/wasix/src/snapshot/capturer.rs b/lib/wasix/src/snapshot/capturer.rs index 2e24020fd0b..2406dd9fa3b 100644 --- a/lib/wasix/src/snapshot/capturer.rs +++ b/lib/wasix/src/snapshot/capturer.rs @@ -1,4 +1,5 @@ use serde::{Deserialize, Serialize}; +use std::fmt; use std::net::SocketAddr; use std::time::SystemTime; use std::{borrow::Cow, ops::Range}; @@ -100,7 +101,6 @@ pub enum FileEntryType { /// Represents a log entry in a snapshot log stream that represents the total /// state of a WASM process at a point in time. -#[derive(Debug)] pub enum SnapshotLog<'a> { Init { wasm_hash: [u8; 32], @@ -147,6 +147,77 @@ pub enum SnapshotLog<'a> { }, } +impl fmt::Debug for SnapshotLog<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Init { wasm_hash } => f + .debug_struct("Init") + .field("wasm_hash.len", &wasm_hash.len()) + .finish(), + Self::TerminalData { data } => f + .debug_struct("TerminalData") + .field("data.len", &data.len()) + .finish(), + Self::UpdateMemoryRegion { region, data } => f + .debug_struct("UpdateMemoryRegion") + .field("region", region) + .field("data.len", &data.len()) + .finish(), + Self::CloseThread { id, exit_code } => f + .debug_struct("CloseThread") + .field("id", id) + .field("exit_code", exit_code) + .finish(), + Self::SetThread { + id, + call_stack, + memory_stack, + } => f + .debug_struct("SetThread") + .field("id", id) + .field("call_stack.len", &call_stack.len()) + .field("memory_stack.len", &memory_stack.len()) + .finish(), + Self::CloseFileDescriptor { fd } => f + .debug_struct("CloseFileDescriptor") + .field("fd", fd) + .finish(), + Self::OpenFileDescriptor { fd, state } => f + .debug_struct("OpenFileDescriptor") + .field("fd", fd) + .field("state", state) + .finish(), + Self::RemoveFileSystemEntry { path } => f + .debug_struct("RemoveFileSystemEntry") + .field("path", path) + .finish(), + Self::UpdateFileSystemEntry { + path, + ft, + accessed, + created, + modified, + len, + data, + } => f + .debug_struct("UpdateFileSystemEntry") + .field("path", path) + .field("ft", ft) + .field("accessed", accessed) + .field("created", created) + .field("modified", modified) + .field("len", len) + .field("data.len", &data.len()) + .finish(), + Self::Snapshot { when, trigger } => f + .debug_struct("Snapshot") + .field("when", when) + .field("trigger", trigger) + .finish(), + } + } +} + /// The snapshot capturer will take a series of objects that represents the state of /// a WASM process at a point in time and saves it so that it can be restored. /// It also allows for the restoration of that state at a later moment diff --git a/lib/wasix/src/snapshot/effector.rs b/lib/wasix/src/snapshot/effector.rs index a7780efcfbd..3ddf7fefe71 100644 --- a/lib/wasix/src/snapshot/effector.rs +++ b/lib/wasix/src/snapshot/effector.rs @@ -12,7 +12,7 @@ use crate::{ os::task::process::WasiProcessInner, syscalls::__asyncify_light, utils::{map_io_err, map_snapshot_err}, - WasiEnv, WasiResult, WasiThreadId, + WasiEnv, WasiError, WasiThreadId, }; use super::*; @@ -26,12 +26,12 @@ impl SnapshotEffector { ctx: &mut FunctionEnvMut<'_, WasiEnv>, iovs: WasmPtr<__wasi_ciovec_t, M>, iovs_len: M::Offset, - ) -> WasiResult<()> { + ) -> anyhow::Result<()> { let env = ctx.data(); let memory = unsafe { env.memory_view(&ctx) }; - let iovs_arr = wasi_try_mem_ok_ok!(iovs.slice(&memory, iovs_len)); + let iovs_arr = iovs.slice(&memory, iovs_len)?; - wasi_try_ok_ok!(__asyncify_light(env, None, async { + __asyncify_light(env, None, async { let iovs_arr = iovs_arr.access().map_err(mem_error_to_wasi)?; for iovs in iovs_arr.iter() { let buf = WasmPtr::::new(iovs.buf) @@ -49,38 +49,41 @@ impl SnapshotEffector { .map_err(map_snapshot_err)?; } Ok(()) - })?); - Ok(Ok(())) + })? + .map_err(|err| WasiError::Exit(ExitCode::Errno(err)))?; + Ok(()) } pub fn apply_terminal_data( ctx: &mut FunctionEnvMut<'_, WasiEnv>, data: &[u8], - ) -> WasiResult<()> { + ) -> anyhow::Result<()> { let env = ctx.data(); - wasi_try_ok_ok!(__asyncify_light(env, None, async { + __asyncify_light(env, None, async { if let Some(mut stdout) = ctx.data().stdout().map_err(fs_error_into_wasi_err)? { stdout.write_all(data).await.map_err(map_io_err)?; } Ok(()) - })?); - Ok(Ok(())) + })? + .map_err(|err| WasiError::Exit(ExitCode::Errno(err)))?; + Ok(()) } pub fn save_thread_exit( env: &WasiEnv, id: WasiThreadId, exit_code: Option, - ) -> WasiResult<()> { - wasi_try_ok_ok!(__asyncify_light(env, None, async { + ) -> anyhow::Result<()> { + __asyncify_light(env, None, async { env.runtime() .snapshot_capturer() .write(SnapshotLog::CloseThread { id, exit_code }) .await .map_err(map_snapshot_err)?; Ok(()) - })?); - Ok(Ok(())) + })? + .map_err(|err| WasiError::Exit(ExitCode::Errno(err)))?; + Ok(()) } pub fn save_thread_state( @@ -88,9 +91,10 @@ impl SnapshotEffector { id: WasiThreadId, memory_stack: Bytes, rewind_stack: Bytes, - ) -> WasiResult<()> { + ) -> anyhow::Result<()> { let env = ctx.data(); - wasi_try_ok_ok!(__asyncify_light(env, None, async { + + __asyncify_light(env, None, async { ctx.data() .runtime() .snapshot_capturer() @@ -102,15 +106,17 @@ impl SnapshotEffector { .await .map_err(map_snapshot_err)?; Ok(()) - })?); - Ok(Ok(())) + })? + .map_err(|err| WasiError::Exit(ExitCode::Errno(err)))?; + + Ok(()) } pub fn save_memory_and_snapshot( ctx: &mut FunctionEnvMut<'_, WasiEnv>, process: &mut MutexGuard<'_, WasiProcessInner>, trigger: SnapshotTrigger, - ) -> WasiResult<()> { + ) -> anyhow::Result<()> { let env = ctx.data(); let memory = unsafe { env.memory_view(ctx) }; @@ -145,7 +151,7 @@ impl SnapshotEffector { // Now that we known all the regions that need to be saved we // enter a processing loop that dumps all the data to the log // file in an orderly manner. - wasi_try_ok_ok!(__asyncify_light(env, None, async { + __asyncify_light(env, None, async { let memory = unsafe { env.memory_view(ctx) }; let capturer = ctx.data().runtime().snapshot_capturer(); @@ -175,7 +181,8 @@ impl SnapshotEffector { .await .map_err(map_snapshot_err)?; Ok(()) - })?); - Ok(Ok(())) + })? + .map_err(|err| WasiError::Exit(ExitCode::Errno(err)))?; + Ok(()) } } diff --git a/lib/wasix/src/syscalls/mod.rs b/lib/wasix/src/syscalls/mod.rs index abade710157..aed4f460a2f 100644 --- a/lib/wasix/src/syscalls/mod.rs +++ b/lib/wasix/src/syscalls/mod.rs @@ -958,17 +958,15 @@ pub(crate) fn deep_sleep( let tasks = ctx.data().tasks().clone(); let res = unwind::(ctx, move |_ctx, memory_stack, rewind_stack| { // Schedule the process on the stack so that it can be resumed - OnCalledAction::Trap(Box::new(RuntimeError::user(Box::new( - WasiError::DeepSleep(DeepSleepWork { - trigger, - rewind: RewindState { - memory_stack: memory_stack.freeze(), - rewind_stack: rewind_stack.freeze(), - store_data, - is_64bit: M::is_64bit(), - }, - }), - )))) + OnCalledAction::Trap(Box::new(WasiError::DeepSleep(DeepSleepWork { + trigger, + rewind: RewindState { + memory_stack: memory_stack.freeze(), + rewind_stack: rewind_stack.freeze(), + store_data, + is_64bit: M::is_64bit(), + }, + }))) })?; // If there is an error then exit the process, otherwise we are done diff --git a/lib/wasix/src/syscalls/wasi/fd_write.rs b/lib/wasix/src/syscalls/wasi/fd_write.rs index b184aa0dd73..aa5b5f964b7 100644 --- a/lib/wasix/src/syscalls/wasi/fd_write.rs +++ b/lib/wasix/src/syscalls/wasi/fd_write.rs @@ -108,7 +108,13 @@ pub(crate) fn fd_write_internal( // will record a terminal event. #[cfg(feature = "snapshot")] if is_stdio && env.enable_snapshot_capture { - SnapshotEffector::save_terminal_data(&mut ctx, iovs, iovs_len)?; + SnapshotEffector::save_terminal_data(&mut ctx, iovs, iovs_len).map_err(|err| { + tracing::warn!( + "failed to save terminal data to snapshot capturer - {}", + err + ); + WasiError::Exit(ExitCode::Errno(Errno::Fault)) + })?; env = ctx.data(); } From 34fa6eafbff536d81431342c8ee4fc0151d78fd2 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Wed, 25 Oct 2023 09:56:57 +1100 Subject: [PATCH 021/129] Process will now properly terminate when snapshots fail --- lib/wasix/src/syscalls/wasi/fd_write.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/wasix/src/syscalls/wasi/fd_write.rs b/lib/wasix/src/syscalls/wasi/fd_write.rs index aa5b5f964b7..a62703f90b5 100644 --- a/lib/wasix/src/syscalls/wasi/fd_write.rs +++ b/lib/wasix/src/syscalls/wasi/fd_write.rs @@ -109,7 +109,7 @@ pub(crate) fn fd_write_internal( #[cfg(feature = "snapshot")] if is_stdio && env.enable_snapshot_capture { SnapshotEffector::save_terminal_data(&mut ctx, iovs, iovs_len).map_err(|err| { - tracing::warn!( + tracing::error!( "failed to save terminal data to snapshot capturer - {}", err ); From 485176b213fdd3aa89f944d0d8db4f1334990466 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Wed, 25 Oct 2023 10:02:17 +1100 Subject: [PATCH 022/129] Simplified the checkpoint loop --- lib/wasix/src/os/task/process.rs | 49 ++++++++++++++------------------ 1 file changed, 21 insertions(+), 28 deletions(-) diff --git a/lib/wasix/src/os/task/process.rs b/lib/wasix/src/os/task/process.rs index 0e1aabd257e..2e2826fae05 100644 --- a/lib/wasix/src/os/task/process.rs +++ b/lib/wasix/src/os/task/process.rs @@ -194,39 +194,32 @@ impl WasiProcessInner { // Wait for the checkpoint to finish (or if we are the last thread // to freeze then we have to execute the checksum operation) loop { - if let WasiProcessCheckpoint::Snapshot { .. } = guard.checkpoint { + if let WasiProcessCheckpoint::Snapshot { trigger } = guard.checkpoint { + ctx.data().thread.set_check_pointing(true); + + // Now if we are the last thread we also write the memory + let is_last_thread = guard.threads.values().all(WasiThread::is_check_pointing); + if is_last_thread { + if let Err(err) = SnapshotEffector::save_memory_and_snapshot( + &mut ctx, &mut guard, trigger, + ) { + inner.1.notify_all(); + return wasmer_types::OnCalledAction::Trap(err.into()); + } + + // Clear the checkpointing flag and notify everyone to wake up + ctx.data().thread.set_check_pointing(false); + guard.checkpoint = WasiProcessCheckpoint::Execute; + trace!("checkpoint complete"); + inner.1.notify_all(); + } else { + guard = inner.1.wait(guard).unwrap(); + } } else { ctx.data().thread.set_check_pointing(false); trace!("checkpoint finished"); break; } - - ctx.data().thread.set_check_pointing(true); - - match guard.checkpoint { - WasiProcessCheckpoint::Snapshot { trigger } => { - // Now if we are the last thread we also write the memory - let is_last_thread = - guard.threads.values().all(WasiThread::is_check_pointing); - if is_last_thread { - if let Err(err) = SnapshotEffector::save_memory_and_snapshot( - &mut ctx, &mut guard, trigger, - ) { - inner.1.notify_all(); - return wasmer_types::OnCalledAction::Trap(err.into()); - } - - // Clear the checkpointing flag and notify everyone to wake up - ctx.data().thread.set_check_pointing(false); - guard.checkpoint = WasiProcessCheckpoint::Execute; - trace!("checkpoint complete"); - inner.1.notify_all(); - } else { - guard = inner.1.wait(guard).unwrap(); - } - } - _ => {} - } } // Resume execution From 6e557bad6447f135fee1ba94033eb816c3ef34b2 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Wed, 25 Oct 2023 11:05:49 +1100 Subject: [PATCH 023/129] The runtime now properly saves the journal events to a snapshot file --- lib/cli/src/commands/run.rs | 41 +++++++++++++++++++++++++- lib/cli/src/commands/run/wasi.rs | 23 +++++++++++++-- lib/wasix/src/lib.rs | 14 ++++++++- lib/wasix/src/runners/wasi.rs | 22 +++++++++++++- lib/wasix/src/runners/wasi_common.rs | 15 ++++++++-- lib/wasix/src/runners/wcgi/runner.rs | 18 ++++++++++++ lib/wasix/src/snapshot/capturer.rs | 4 +-- lib/wasix/src/snapshot/compactor.rs | 4 +-- lib/wasix/src/snapshot/filter.rs | 4 +-- lib/wasix/src/snapshot/log_file.rs | 11 +++++-- lib/wasix/src/snapshot/unsupported.rs | 4 +-- lib/wasix/src/state/builder.rs | 42 +++++++++++++++++++++++++-- lib/wasix/src/state/env.rs | 2 +- 13 files changed, 182 insertions(+), 22 deletions(-) diff --git a/lib/cli/src/commands/run.rs b/lib/cli/src/commands/run.rs index 5c3afc40d91..ed119417916 100644 --- a/lib/cli/src/commands/run.rs +++ b/lib/cli/src/commands/run.rs @@ -37,7 +37,7 @@ use wasmer_wasix::{ resolver::{PackageSpecifier, QueryError}, task_manager::VirtualTaskManagerExt, }, - snapshot::SnapshotTrigger, + snapshot::{LogFileSnapshotCapturer, SnapshotTrigger}, WasiError, }; use wasmer_wasix::{ @@ -229,6 +229,24 @@ impl Run { for trigger in self.wasi.snapshot_on.iter().cloned() { runner.add_snapshot_trigger(trigger); } + + #[cfg(feature = "snapshot")] + match (self.wasi.snapshot_to.clone(), self.wasi.resume_from.clone()) { + (Some(save), Some(restore)) if save == restore => { + return Err(anyhow::format_err!( + "The snapshot save path and snapshot restore path can not be the same" + )); + } + (_, _) => { + if let Some(path) = self.wasi.snapshot_to.clone() { + runner.with_snapshot_save(Arc::new(LogFileSnapshotCapturer::new_std(path)?)); + } + if let Some(path) = self.wasi.resume_from.clone() { + runner.with_snapshot_restore(Arc::new(LogFileSnapshotCapturer::new_std(path)?)); + } + } + } + *runner.capabilities() = self.wasi.capabilities(); runner.run_command(command_name, pkg, runtime) @@ -261,6 +279,27 @@ impl Run { runner.config().add_snapshot_trigger(trigger); } + #[cfg(feature = "snapshot")] + match (self.wasi.snapshot_to.clone(), self.wasi.resume_from.clone()) { + (Some(save), Some(restore)) if save == restore => { + return Err(anyhow::format_err!( + "The snapshot save path and snapshot restore path can not be the same" + )); + } + (_, _) => { + if let Some(path) = self.wasi.snapshot_to.clone() { + runner + .config() + .with_snapshot_save(Arc::new(LogFileSnapshotCapturer::new_std(path)?)); + } + if let Some(path) = self.wasi.resume_from.clone() { + runner + .config() + .with_snapshot_restore(Arc::new(LogFileSnapshotCapturer::new_std(path)?)); + } + } + } + runner.run_command(command_name, pkg, runtime) } diff --git a/lib/cli/src/commands/run/wasi.rs b/lib/cli/src/commands/run/wasi.rs index eb81f892828..555358780d9 100644 --- a/lib/cli/src/commands/run/wasi.rs +++ b/lib/cli/src/commands/run/wasi.rs @@ -34,7 +34,7 @@ use wasmer_wasix::{ VirtualTaskManagerExt, }, }, - snapshot::{self, SnapshotTrigger}, + snapshot::{self, LogFileSnapshotCapturer, SnapshotTrigger}, types::__WASI_STDIN_FILENO, wasmer_wasix_types::wasi::Errno, PluggableRuntime, RewindState, Runtime, WasiEnv, WasiEnvBuilder, WasiError, WasiFunctionEnv, @@ -263,6 +263,25 @@ impl Wasi { builder.add_snapshot_trigger(trigger); } + #[cfg(feature = "snapshot")] + match (self.snapshot_to.clone(), self.resume_from.clone()) { + (Some(save), Some(restore)) if save == restore => { + return Err(anyhow::format_err!( + "The snapshot save path and snapshot restore path can not be the same" + )); + } + (_, _) => { + if let Some(path) = self.snapshot_to.clone() { + builder = builder + .with_snapshot_save(Arc::new(LogFileSnapshotCapturer::new_std(path)?)); + } + if let Some(path) = self.resume_from.clone() { + builder = builder + .with_snapshot_restore(Arc::new(LogFileSnapshotCapturer::new_std(path)?)); + } + } + } + #[cfg(feature = "experimental-io-devices")] { if self.enable_experimental_io_devices { @@ -304,7 +323,7 @@ impl Wasi { } #[cfg(feature = "snapshot")] - if let Some(path) = &self.resume_from { + if let Some(path) = &self.snapshot_to { rt.set_snapshot_capturer(Arc::new(snapshot::LogFileSnapshotCapturer::new_std(path)?)); } diff --git a/lib/wasix/src/lib.rs b/lib/wasix/src/lib.rs index 10469926cb7..a8b1c5903ca 100644 --- a/lib/wasix/src/lib.rs +++ b/lib/wasix/src/lib.rs @@ -58,9 +58,12 @@ mod utils; /// WAI based bindings. mod bindings; +use std::sync::Arc; + #[allow(unused_imports)] use bytes::{Bytes, BytesMut}; use os::task::control_plane::ControlPlaneError; +use snapshot::DynSnapshotCapturer; use thiserror::Error; use tracing::error; // re-exports needed for OS @@ -231,8 +234,16 @@ impl WasiRuntimeError { pub(crate) fn run_wasi_func( func: &wasmer::Function, store: &mut impl AsStoreMut, + restorer: Option>, params: &[wasmer::Value], ) -> Result, WasiRuntimeError> { + // TODO - do the snapshot restoration here + if restorer.is_some() { + return Err(WasiRuntimeError::Runtime(RuntimeError::user( + anyhow::format_err!("snapshot restoration is not currently supported").into(), + ))); + } + func.call(store, params).map_err(|err| { if let Some(_werr) = err.downcast_ref::() { let werr = err.downcast::().unwrap(); @@ -253,8 +264,9 @@ pub(crate) fn run_wasi_func( pub(crate) fn run_wasi_func_start( func: &wasmer::Function, store: &mut impl AsStoreMut, + restorer: Option>, ) -> Result<(), WasiRuntimeError> { - run_wasi_func(func, store, &[])?; + run_wasi_func(func, store, restorer, &[])?; Ok(()) } diff --git a/lib/wasix/src/runners/wasi.rs b/lib/wasix/src/runners/wasi.rs index 333f189b106..f340e150e18 100644 --- a/lib/wasix/src/runners/wasi.rs +++ b/lib/wasix/src/runners/wasi.rs @@ -10,7 +10,7 @@ use crate::{ bin_factory::BinaryPackage, capabilities::Capabilities, runners::{wasi_common::CommonWasiOptions, MappedDirectory}, - snapshot::SnapshotTrigger, + snapshot::{DynSnapshotCapturer, SnapshotTrigger}, Runtime, WasiEnvBuilder, }; @@ -152,6 +152,16 @@ impl WasiRunner { self } + pub fn with_snapshot_restore(&mut self, capturer: Arc) -> &mut Self { + self.wasi.snapshot_restore.replace(capturer); + self + } + + pub fn with_snapshot_save(&mut self, capturer: Arc) -> &mut Self { + self.wasi.snapshot_save.replace(capturer); + self + } + pub fn with_stdin(mut self, stdin: Box) -> Self { self.set_stdin(stdin); self @@ -247,6 +257,16 @@ impl crate::runners::Runner for WasiRunner { env.add_snapshot_trigger(snapshot_trigger); } + #[cfg(feature = "snapshot")] + if let Some(capturer) = self.wasi.snapshot_save.clone() { + env = env.with_snapshot_save(capturer); + } + + #[cfg(feature = "snapshot")] + if let Some(capturer) = self.wasi.snapshot_restore.clone() { + env = env.with_snapshot_restore(capturer); + } + if self .wasi .capabilities diff --git a/lib/wasix/src/runners/wasi_common.rs b/lib/wasix/src/runners/wasi_common.rs index 16ad0ed8462..7a7ed2fdc35 100644 --- a/lib/wasix/src/runners/wasi_common.rs +++ b/lib/wasix/src/runners/wasi_common.rs @@ -5,16 +5,21 @@ use std::{ }; use anyhow::{Context, Error}; +use derivative::Derivative; use futures::future::BoxFuture; use virtual_fs::{FileSystem, FsError, OverlayFileSystem, RootFileSystemBuilder, TmpFileSystem}; use webc::metadata::annotations::Wasi as WasiAnnotation; use crate::{ - bin_factory::BinaryPackage, capabilities::Capabilities, runners::MappedDirectory, - snapshot::SnapshotTrigger, WasiEnvBuilder, + bin_factory::BinaryPackage, + capabilities::Capabilities, + runners::MappedDirectory, + snapshot::{DynSnapshotCapturer, SnapshotTrigger}, + WasiEnvBuilder, }; -#[derive(Debug, Default, Clone)] +#[derive(Derivative, Default, Clone)] +#[derivative(Debug)] pub(crate) struct CommonWasiOptions { pub(crate) args: Vec, pub(crate) env: HashMap, @@ -22,6 +27,10 @@ pub(crate) struct CommonWasiOptions { pub(crate) mapped_dirs: Vec, pub(crate) injected_packages: Vec, pub(crate) capabilities: Capabilities, + #[derivative(Debug = "ignore")] + pub(crate) snapshot_save: Option>, + #[derivative(Debug = "ignore")] + pub(crate) snapshot_restore: Option>, pub(crate) snapshot_on: Vec, } diff --git a/lib/wasix/src/runners/wcgi/runner.rs b/lib/wasix/src/runners/wcgi/runner.rs index c765fc04c2c..8a51022ee12 100644 --- a/lib/wasix/src/runners/wcgi/runner.rs +++ b/lib/wasix/src/runners/wcgi/runner.rs @@ -246,6 +246,24 @@ impl Config { pub fn add_snapshot_trigger(&mut self, on: crate::snapshot::SnapshotTrigger) { self.wasi.snapshot_on.push(on); } + + #[cfg(feature = "snapshot")] + pub fn with_snapshot_restore( + &mut self, + capturer: Arc, + ) -> &mut Self { + self.wasi.snapshot_restore.replace(capturer); + self + } + + #[cfg(feature = "snapshot")] + pub fn with_snapshot_save( + &mut self, + capturer: Arc, + ) -> &mut Self { + self.wasi.snapshot_save.replace(capturer); + self + } } impl Default for Config { diff --git a/lib/wasix/src/snapshot/capturer.rs b/lib/wasix/src/snapshot/capturer.rs index 2406dd9fa3b..fa6acb2c8e2 100644 --- a/lib/wasix/src/snapshot/capturer.rs +++ b/lib/wasix/src/snapshot/capturer.rs @@ -5,7 +5,7 @@ use std::time::SystemTime; use std::{borrow::Cow, ops::Range}; use wasmer_wasix_types::wasi::ExitCode; -use futures::future::BoxFuture; +use futures::future::{BoxFuture, LocalBoxFuture}; use virtual_fs::Fd; use crate::WasiThreadId; @@ -225,7 +225,7 @@ impl fmt::Debug for SnapshotLog<'_> { pub trait SnapshotCapturer { /// Takes in a stream of snapshot log entries and saves them so that they /// may be restored at a later moment - fn write<'a>(&'a self, entry: SnapshotLog<'a>) -> BoxFuture<'a, anyhow::Result<()>>; + fn write<'a>(&'a self, entry: SnapshotLog<'a>) -> LocalBoxFuture<'a, anyhow::Result<()>>; /// Returns a stream of snapshot objects that the runtime will use /// to restore the state of a WASM process to a previous moment in time diff --git a/lib/wasix/src/snapshot/compactor.rs b/lib/wasix/src/snapshot/compactor.rs index 61ae74628b2..c223aa3c8a6 100644 --- a/lib/wasix/src/snapshot/compactor.rs +++ b/lib/wasix/src/snapshot/compactor.rs @@ -4,7 +4,7 @@ use std::{ sync::Mutex, }; -use futures::future::BoxFuture; +use futures::future::{BoxFuture, LocalBoxFuture}; use sha2::{Digest, Sha256}; use virtual_fs::Fd; @@ -38,7 +38,7 @@ impl CompactingSnapshotCapturer { } impl SnapshotCapturer for CompactingSnapshotCapturer { - fn write<'a>(&'a self, entry: SnapshotLog<'a>) -> BoxFuture<'a, anyhow::Result<()>> { + fn write<'a>(&'a self, entry: SnapshotLog<'a>) -> LocalBoxFuture<'a, anyhow::Result<()>> { Box::pin(async { match entry { SnapshotLog::UpdateMemoryRegion { region, data } => { diff --git a/lib/wasix/src/snapshot/filter.rs b/lib/wasix/src/snapshot/filter.rs index c443697857f..a288ab0bb83 100644 --- a/lib/wasix/src/snapshot/filter.rs +++ b/lib/wasix/src/snapshot/filter.rs @@ -1,4 +1,4 @@ -use futures::future::BoxFuture; +use futures::future::{BoxFuture, LocalBoxFuture}; use super::*; @@ -60,7 +60,7 @@ impl FilteredSnapshotCapturer { } impl SnapshotCapturer for FilteredSnapshotCapturer { - fn write<'a>(&'a self, entry: SnapshotLog<'a>) -> BoxFuture<'a, anyhow::Result<()>> { + fn write<'a>(&'a self, entry: SnapshotLog<'a>) -> LocalBoxFuture<'a, anyhow::Result<()>> { Box::pin(async { let evt = match entry { SnapshotLog::Init { wasm_hash } => SnapshotLog::Init { wasm_hash }, diff --git a/lib/wasix/src/snapshot/log_file.rs b/lib/wasix/src/snapshot/log_file.rs index 0ae2c6710ba..f4bc9004f8d 100644 --- a/lib/wasix/src/snapshot/log_file.rs +++ b/lib/wasix/src/snapshot/log_file.rs @@ -5,9 +5,10 @@ use std::{ path::Path, time::SystemTime, }; +use tokio::runtime::Handle; use wasmer_wasix_types::wasi::ExitCode; -use futures::future::BoxFuture; +use futures::future::{BoxFuture, LocalBoxFuture}; use virtual_fs::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt, Fd}; use crate::WasiThreadId; @@ -190,6 +191,7 @@ struct State { /// delimiter. pub struct LogFileSnapshotCapturer { state: tokio::sync::Mutex, + handle: Handle, } impl LogFileSnapshotCapturer { @@ -205,6 +207,7 @@ impl LogFileSnapshotCapturer { }; Ok(Self { state: tokio::sync::Mutex::new(state), + handle: Handle::current(), }) } @@ -220,16 +223,18 @@ impl LogFileSnapshotCapturer { }; Ok(Self { state: tokio::sync::Mutex::new(state), + handle: Handle::current(), }) } } #[async_trait::async_trait] impl SnapshotCapturer for LogFileSnapshotCapturer { - fn write<'a>(&'a self, entry: SnapshotLog<'a>) -> BoxFuture<'a, anyhow::Result<()>> { + fn write<'a>(&'a self, entry: SnapshotLog<'a>) -> LocalBoxFuture<'a, anyhow::Result<()>> { Box::pin(async { let entry: SnapshotLogEntry = entry.into(); + let _guard = Handle::try_current().map_err(|_| self.handle.enter()); let mut state = self.state.lock().await; if state.at_end == false { state.file.seek(SeekFrom::End(0)).await?; @@ -239,9 +244,9 @@ impl SnapshotCapturer for LogFileSnapshotCapturer { let data = bincode::serialize(&entry)?; let data_len = data.len() as u64; let data_len = data_len.to_ne_bytes(); + state.file.write_all(&data_len).await?; state.file.write_all(&data).await?; - Ok(()) }) } diff --git a/lib/wasix/src/snapshot/unsupported.rs b/lib/wasix/src/snapshot/unsupported.rs index 4729b222c06..e8040b48430 100644 --- a/lib/wasix/src/snapshot/unsupported.rs +++ b/lib/wasix/src/snapshot/unsupported.rs @@ -1,4 +1,4 @@ -use futures::future::BoxFuture; +use futures::future::{BoxFuture, LocalBoxFuture}; use super::*; @@ -11,7 +11,7 @@ pub static UNSUPPORTED_SNAPSHOT_CAPTURER: UnsupportedSnapshotCapturer = pub struct UnsupportedSnapshotCapturer {} impl SnapshotCapturer for UnsupportedSnapshotCapturer { - fn write<'a>(&'a self, entry: SnapshotLog<'a>) -> BoxFuture<'a, anyhow::Result<()>> { + fn write<'a>(&'a self, entry: SnapshotLog<'a>) -> LocalBoxFuture<'a, anyhow::Result<()>> { tracing::debug!("snapshot event: {:?}", entry); Box::pin(async { Err(anyhow::format_err!("unsupported")) }) } diff --git a/lib/wasix/src/state/builder.rs b/lib/wasix/src/state/builder.rs index a457f9f9fab..093d5bb1e4f 100644 --- a/lib/wasix/src/state/builder.rs +++ b/lib/wasix/src/state/builder.rs @@ -23,6 +23,7 @@ use crate::{ fs::{WasiFs, WasiFsRoot, WasiInodes}, os::task::control_plane::{ControlPlaneConfig, ControlPlaneError, WasiControlPlane}, runtime::task_manager::InlineWaker, + snapshot::DynSnapshotCapturer, state::WasiState, syscalls::types::{__WASI_STDERR_FILENO, __WASI_STDIN_FILENO, __WASI_STDOUT_FILENO}, RewindState, Runtime, WasiEnv, WasiError, WasiFunctionEnv, WasiRuntimeError, @@ -75,6 +76,12 @@ pub struct WasiEnvBuilder { #[cfg(feature = "snapshot")] pub(super) snapshot_on: Vec, + + #[cfg(feature = "snapshot")] + pub(super) snapshot_restore: Option>, + + #[cfg(feature = "snapshot")] + pub(super) snapshot_save: Option>, } impl std::fmt::Debug for WasiEnvBuilder { @@ -485,6 +492,21 @@ impl WasiEnvBuilder { Ok(self) } + /// Supplies a snapshot capturer which will be used to read the + /// snapshot journal events and replay them into the WasiEnv + /// rather than starting a new instance from scratch + pub fn with_snapshot_restore(mut self, snapshot_capturer: Arc) -> Self { + self.snapshot_restore.replace(snapshot_capturer); + self + } + + /// Supplies a snapshot capturer where the snapshot journal events + /// will be sent to as they are generated + pub fn with_snapshot_save(mut self, snapshot_capturer: Arc) -> Self { + self.snapshot_save.replace(snapshot_capturer); + self + } + /// Overwrite the default WASI `stdout`, if you want to hold on to the /// original `stdout` use [`WasiFs::swap_file`] after building. pub fn stdout(mut self, new_file: Box) -> Self { @@ -719,7 +741,11 @@ impl WasiEnvBuilder { let runtime = self.runtime.unwrap_or_else(|| { #[cfg(feature = "sys-thread")] { - Arc::new(PluggableRuntime::new(Arc::new(crate::runtime::task_manager::tokio::TokioTaskManager::default()))) + let mut runtime = PluggableRuntime::new(Arc::new(crate::runtime::task_manager::tokio::TokioTaskManager::default())); + if let Some(capturer) = self.snapshot_save.clone() { + runtime.set_snapshot_capturer(capturer); + } + Arc::new(runtime) } #[cfg(not(feature = "sys-thread"))] @@ -755,6 +781,9 @@ impl WasiEnvBuilder { memory_ty: None, process: None, thread: None, + #[cfg(feature = "snapshot")] + call_initialize: self.snapshot_restore.is_none(), + #[cfg(not(feature = "snapshot"))] call_initialize: true, can_deep_sleep: false, extra_tracing: true, @@ -830,12 +859,14 @@ impl WasiEnvBuilder { ); } + let snapshot_restore = self.snapshot_restore.clone(); + let (instance, env) = self.instantiate(module, store)?; let start = instance.exports.get_function("_start")?; env.data(&store).thread.set_status_running(); - let result = crate::run_wasi_func_start(start, store); + let result = crate::run_wasi_func_start(start, store, snapshot_restore); let (result, exit_code) = wasi_exit_code(result); let pid = env.data(&store).pid(); @@ -874,6 +905,13 @@ impl WasiEnvBuilder { #[cfg(feature = "sys-thread")] let _guard = _guard.as_ref().map(|r| r.enter()); + let snapshot_restore = self.snapshot_restore.clone(); + + // TODO - need to restore the memory and pass the rewind_state to the `run_with_deep_sleep` emthod + if snapshot_restore.is_some() { + panic!("Snapshot restoration is not currently supported."); + } + let (_, env) = self.instantiate(module, &mut store)?; env.data(&store).thread.set_status_running(); diff --git a/lib/wasix/src/state/env.rs b/lib/wasix/src/state/env.rs index a06b61980f3..d763f96f56b 100644 --- a/lib/wasix/src/state/env.rs +++ b/lib/wasix/src/state/env.rs @@ -549,7 +549,7 @@ impl WasiEnv { // If this module exports an _initialize function, run that first. if call_initialize { if let Ok(initialize) = instance.exports.get_function("_initialize") { - if let Err(err) = crate::run_wasi_func_start(initialize, &mut store) { + if let Err(err) = crate::run_wasi_func_start(initialize, &mut store, None) { func_env .data(&store) .blocking_cleanup(Some(Errno::Noexec.into())); From c3f0e52a56cc515f120ba26be277be23bac60fe4 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Wed, 25 Oct 2023 11:47:35 +1100 Subject: [PATCH 024/129] Now currently saving the snapshot state --- lib/wasix/src/bin_factory/exec.rs | 4 +- lib/wasix/src/os/task/process.rs | 48 +++++++++++++++----- lib/wasix/src/snapshot/capturer.rs | 3 ++ lib/wasix/src/snapshot/effector.rs | 11 +++-- lib/wasix/src/snapshot/filter.rs | 2 + lib/wasix/src/snapshot/log_file.rs | 12 +++-- lib/wasix/src/state/builder.rs | 4 +- lib/wasix/src/syscalls/mod.rs | 20 +++++--- lib/wasix/src/syscalls/wasix/proc_fork.rs | 2 +- lib/wasix/src/syscalls/wasix/thread_spawn.rs | 2 +- 10 files changed, 78 insertions(+), 30 deletions(-) diff --git a/lib/wasix/src/bin_factory/exec.rs b/lib/wasix/src/bin_factory/exec.rs index 044a8477d33..dcc5a1cb00d 100644 --- a/lib/wasix/src/bin_factory/exec.rs +++ b/lib/wasix/src/bin_factory/exec.rs @@ -167,7 +167,7 @@ fn call_module( rewind_state.memory_stack, rewind_state.rewind_stack, rewind_state.store_data, - rewind_result, + Some(rewind_result), ); if res != Errno::Success { ctx.data(&store).blocking_cleanup(Some(res.into())); @@ -179,7 +179,7 @@ fn call_module( rewind_state.memory_stack, rewind_state.rewind_stack, rewind_state.store_data, - rewind_result, + Some(rewind_result), ); if res != Errno::Success { ctx.data(&store).blocking_cleanup(Some(res.into())); diff --git a/lib/wasix/src/os/task/process.rs b/lib/wasix/src/os/task/process.rs index 2e2826fae05..f9b6134815f 100644 --- a/lib/wasix/src/os/task/process.rs +++ b/lib/wasix/src/os/task/process.rs @@ -162,6 +162,12 @@ impl WasiProcessInner { ctx: FunctionEnvMut<'_, WasiEnv>, ) -> WasiResult> { // Enter the lock which will determine if we are in a checkpoint or not + + use bytes::Bytes; + use wasmer::AsStoreMut; + use wasmer_types::OnCalledAction; + + use crate::{rewind_ext, WasiError}; let guard = inner.0.lock().unwrap(); if guard.checkpoint == WasiProcessCheckpoint::Execute { // No checkpoint so just carry on @@ -172,10 +178,20 @@ impl WasiProcessInner { // Perform the unwind action unwind::(ctx, move |mut ctx, memory_stack, rewind_stack| { + // Grab all the globals and serialize them + let store_data = + crate::utils::store::capture_instance_snapshot(&mut ctx.as_store_mut()) + .serialize() + .unwrap(); + let memory_stack = memory_stack.freeze(); + let rewind_stack = rewind_stack.freeze(); + let store_data = Bytes::from(store_data); + tracing::debug!( - "stack snapshot unwind (memory_stack={}, rewind_stack={})", + "stack snapshot unwind (memory_stack={}, rewind_stack={}, store_data={})", memory_stack.len(), - rewind_stack.len() + rewind_stack.len(), + store_data.len(), ); // Write our thread state to the snapshot @@ -183,8 +199,9 @@ impl WasiProcessInner { if let Err(err) = SnapshotEffector::save_thread_state( &mut ctx, tid, - memory_stack.freeze(), - rewind_stack.freeze(), + memory_stack.clone(), + rewind_stack.clone(), + store_data.clone(), ) { return wasmer_types::OnCalledAction::Trap(err.into()); } @@ -215,15 +232,24 @@ impl WasiProcessInner { } else { guard = inner.1.wait(guard).unwrap(); } - } else { - ctx.data().thread.set_check_pointing(false); - trace!("checkpoint finished"); - break; + continue; } - } - // Resume execution - wasmer_types::OnCalledAction::InvokeAgain + ctx.data().thread.set_check_pointing(false); + trace!("checkpoint finished"); + + // Rewind the stack and carry on + return match rewind_ext::(ctx, memory_stack, rewind_stack, store_data, None) { + Errno::Success => OnCalledAction::InvokeAgain, + err => { + tracing::warn!( + "snapshot resumption failed - could not rewind the stack - errno={}", + err + ); + OnCalledAction::Trap(Box::new(WasiError::Exit(err.into()))) + } + }; + } })?; Ok(Ok(MaybeCheckpointResult::Unwinding)) diff --git a/lib/wasix/src/snapshot/capturer.rs b/lib/wasix/src/snapshot/capturer.rs index fa6acb2c8e2..abf05757162 100644 --- a/lib/wasix/src/snapshot/capturer.rs +++ b/lib/wasix/src/snapshot/capturer.rs @@ -120,6 +120,7 @@ pub enum SnapshotLog<'a> { id: WasiThreadId, call_stack: Cow<'a, [u8]>, memory_stack: Cow<'a, [u8]>, + store_data: Cow<'a, [u8]>, }, CloseFileDescriptor { fd: Fd, @@ -172,11 +173,13 @@ impl fmt::Debug for SnapshotLog<'_> { id, call_stack, memory_stack, + store_data, } => f .debug_struct("SetThread") .field("id", id) .field("call_stack.len", &call_stack.len()) .field("memory_stack.len", &memory_stack.len()) + .field("store_data.len", &store_data.len()) .finish(), Self::CloseFileDescriptor { fd } => f .debug_struct("CloseFileDescriptor") diff --git a/lib/wasix/src/snapshot/effector.rs b/lib/wasix/src/snapshot/effector.rs index 3ddf7fefe71..149ec197601 100644 --- a/lib/wasix/src/snapshot/effector.rs +++ b/lib/wasix/src/snapshot/effector.rs @@ -91,6 +91,7 @@ impl SnapshotEffector { id: WasiThreadId, memory_stack: Bytes, rewind_stack: Bytes, + store_data: Bytes, ) -> anyhow::Result<()> { let env = ctx.data(); @@ -102,6 +103,7 @@ impl SnapshotEffector { id, call_stack: Cow::Owned(rewind_stack.into()), memory_stack: Cow::Owned(memory_stack.into()), + store_data: Cow::Owned(store_data.into()), }) .await .map_err(map_snapshot_err)?; @@ -123,13 +125,14 @@ impl SnapshotEffector { // Compute all the regions that we need to save which is basically // everything in the memory except for the memory stacks. // - // We do not want the regions to be greater than 128KB as this will - // otherwise create too much inefficiency. + // We do not want the regions to be greater than 64KB as this will + // otherwise create too much inefficiency. We choose 64KB as its + // aligned with the standard WASM page size. let mut cur = 0u64; let mut regions = LinkedList::>::new(); while cur < memory.data_size() { let mut again = false; - let mut end = memory.data_size().min(cur + 131_072); + let mut end = memory.data_size().min(cur + 65536); for (_, thread) in process.threads.iter() { let layout = thread.memory_layout(); if cur >= layout.stack_lower && cur < layout.stack_upper { @@ -137,7 +140,7 @@ impl SnapshotEffector { again = true; break; } - if end > layout.stack_lower { + if end > layout.stack_lower && end < layout.stack_upper { end = end.min(layout.stack_lower); } } diff --git a/lib/wasix/src/snapshot/filter.rs b/lib/wasix/src/snapshot/filter.rs index a288ab0bb83..64216f97626 100644 --- a/lib/wasix/src/snapshot/filter.rs +++ b/lib/wasix/src/snapshot/filter.rs @@ -86,6 +86,7 @@ impl SnapshotCapturer for FilteredSnapshotCapturer { id, call_stack, memory_stack, + store_data, } => { if self.filter_threads { return Ok(()); @@ -94,6 +95,7 @@ impl SnapshotCapturer for FilteredSnapshotCapturer { id, call_stack, memory_stack, + store_data, } } SnapshotLog::CloseFileDescriptor { fd } => { diff --git a/lib/wasix/src/snapshot/log_file.rs b/lib/wasix/src/snapshot/log_file.rs index f4bc9004f8d..6c8fba3a6f8 100644 --- a/lib/wasix/src/snapshot/log_file.rs +++ b/lib/wasix/src/snapshot/log_file.rs @@ -23,7 +23,7 @@ use super::*; /// worry about backward and forward compatibility #[derive(Debug, Clone, Serialize, Deserialize)] pub enum SnapshotLogEntry { - Init { + InitV1 { wasm_hash: [u8; 32], }, TerminalDataV1 { @@ -42,6 +42,7 @@ pub enum SnapshotLogEntry { id: WasiThreadId, call_stack: Vec, memory_stack: Vec, + store_data: Vec, }, CloseFileDescriptorV1 { fd: Fd, @@ -71,7 +72,7 @@ pub enum SnapshotLogEntry { impl<'a> From> for SnapshotLogEntry { fn from(value: SnapshotLog<'a>) -> Self { match value { - SnapshotLog::Init { wasm_hash } => Self::Init { wasm_hash }, + SnapshotLog::Init { wasm_hash } => Self::InitV1 { wasm_hash }, SnapshotLog::TerminalData { data } => Self::TerminalDataV1 { data: data.into_owned(), }, @@ -85,10 +86,12 @@ impl<'a> From> for SnapshotLogEntry { id, call_stack, memory_stack, + store_data, } => Self::SetThreadV1 { id, call_stack: call_stack.into_owned(), memory_stack: memory_stack.into_owned(), + store_data: store_data.into_owned(), }, SnapshotLog::CloseFileDescriptor { fd } => Self::CloseFileDescriptorV1 { fd }, SnapshotLog::OpenFileDescriptor { fd, state } => Self::OpenFileDescriptorV1 { @@ -123,7 +126,7 @@ impl<'a> From> for SnapshotLogEntry { impl<'a> From for SnapshotLog<'a> { fn from(value: SnapshotLogEntry) -> Self { match value { - SnapshotLogEntry::Init { wasm_hash } => Self::Init { wasm_hash }, + SnapshotLogEntry::InitV1 { wasm_hash } => Self::Init { wasm_hash }, SnapshotLogEntry::TerminalDataV1 { data } => Self::TerminalData { data: data.into() }, SnapshotLogEntry::UpdateMemoryRegionV1 { start, end, data } => { Self::UpdateMemoryRegion { @@ -138,10 +141,12 @@ impl<'a> From for SnapshotLog<'a> { id, call_stack, memory_stack, + store_data, } => Self::SetThread { id: id, call_stack: call_stack.into(), memory_stack: memory_stack.into(), + store_data: store_data.into(), }, SnapshotLogEntry::CloseFileDescriptorV1 { fd } => Self::CloseFileDescriptor { fd }, SnapshotLogEntry::OpenFileDescriptorV1 { fd, state } => Self::OpenFileDescriptor { @@ -231,6 +236,7 @@ impl LogFileSnapshotCapturer { #[async_trait::async_trait] impl SnapshotCapturer for LogFileSnapshotCapturer { fn write<'a>(&'a self, entry: SnapshotLog<'a>) -> LocalBoxFuture<'a, anyhow::Result<()>> { + tracing::debug!("snapshot event: {:?}", entry); Box::pin(async { let entry: SnapshotLogEntry = entry.into(); diff --git a/lib/wasix/src/state/builder.rs b/lib/wasix/src/state/builder.rs index 093d5bb1e4f..a577fd19bac 100644 --- a/lib/wasix/src/state/builder.rs +++ b/lib/wasix/src/state/builder.rs @@ -988,7 +988,7 @@ fn run_with_deep_sleep( rewind_state.memory_stack, rewind_state.rewind_stack, rewind_state.store_data, - rewind_result, + Some(rewind_result), ) } else { crate::rewind_ext::( @@ -996,7 +996,7 @@ fn run_with_deep_sleep( rewind_state.memory_stack, rewind_state.rewind_stack, rewind_state.store_data, - rewind_result, + Some(rewind_result), ) }; diff --git a/lib/wasix/src/syscalls/mod.rs b/lib/wasix/src/syscalls/mod.rs index aed4f460a2f..c8ecd33bb1d 100644 --- a/lib/wasix/src/syscalls/mod.rs +++ b/lib/wasix/src/syscalls/mod.rs @@ -1120,7 +1120,13 @@ where T: serde::Serialize, { let rewind_result = bincode::serialize(&result).unwrap().into(); - rewind_ext::(ctx, memory_stack, rewind_stack, store_data, rewind_result) + rewind_ext::( + ctx, + memory_stack, + rewind_stack, + store_data, + Some(rewind_result), + ) } #[instrument(level = "debug", skip_all, fields(memory_stack_len = memory_stack.len(), rewind_stack_len = rewind_stack.len(), store_data_len = store_data.len()))] @@ -1130,13 +1136,15 @@ pub fn rewind_ext( memory_stack: Bytes, rewind_stack: Bytes, store_data: Bytes, - rewind_result: Bytes, + rewind_result: Option, ) -> Errno { // Store the memory stack so that it can be restored later - ctx.data_mut().thread.set_rewind(RewindResult { - memory_stack, - rewind_result, - }); + if let Some(rewind_result) = rewind_result { + ctx.data_mut().thread.set_rewind(RewindResult { + memory_stack, + rewind_result, + }); + } // Deserialize the store data back into a snapshot let store_snapshot = match InstanceSnapshot::deserialize(&store_data[..]) { diff --git a/lib/wasix/src/syscalls/wasix/proc_fork.rs b/lib/wasix/src/syscalls/wasix/proc_fork.rs index d55bc864225..32a03599423 100644 --- a/lib/wasix/src/syscalls/wasix/proc_fork.rs +++ b/lib/wasix/src/syscalls/wasix/proc_fork.rs @@ -252,7 +252,7 @@ fn run( rewind_state.memory_stack, rewind_state.rewind_stack, rewind_state.store_data, - rewind_result, + Some(rewind_result), ); if res != Errno::Success { return res.into(); diff --git a/lib/wasix/src/syscalls/wasix/thread_spawn.rs b/lib/wasix/src/syscalls/wasix/thread_spawn.rs index e1f0dbdb50d..2a4387b6572 100644 --- a/lib/wasix/src/syscalls/wasix/thread_spawn.rs +++ b/lib/wasix/src/syscalls/wasix/thread_spawn.rs @@ -210,7 +210,7 @@ fn call_module( rewind_state.memory_stack, rewind_state.rewind_stack, rewind_state.store_data, - rewind_result, + Some(rewind_result), ); if res != Errno::Success { return Err(res); From 27dcb1dfa9eecf1723b3c06ce898a9fb08188753 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Wed, 25 Oct 2023 11:48:46 +1100 Subject: [PATCH 025/129] Now currently saving the snapshot state --- lib/wasix/src/os/task/process.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/wasix/src/os/task/process.rs b/lib/wasix/src/os/task/process.rs index f9b6134815f..acd80c02358 100644 --- a/lib/wasix/src/os/task/process.rs +++ b/lib/wasix/src/os/task/process.rs @@ -238,6 +238,8 @@ impl WasiProcessInner { ctx.data().thread.set_check_pointing(false); trace!("checkpoint finished"); + return OnCalledAction::InvokeAgain; + /* // Rewind the stack and carry on return match rewind_ext::(ctx, memory_stack, rewind_stack, store_data, None) { Errno::Success => OnCalledAction::InvokeAgain, @@ -249,6 +251,7 @@ impl WasiProcessInner { OnCalledAction::Trap(Box::new(WasiError::Exit(err.into()))) } }; + */ } })?; From a99b84036546fc09246402bff21bf32756950d97 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Wed, 25 Oct 2023 11:49:59 +1100 Subject: [PATCH 026/129] Now currently saving the snapshot state --- lib/wasix/src/os/task/process.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/lib/wasix/src/os/task/process.rs b/lib/wasix/src/os/task/process.rs index acd80c02358..f9b6134815f 100644 --- a/lib/wasix/src/os/task/process.rs +++ b/lib/wasix/src/os/task/process.rs @@ -238,8 +238,6 @@ impl WasiProcessInner { ctx.data().thread.set_check_pointing(false); trace!("checkpoint finished"); - return OnCalledAction::InvokeAgain; - /* // Rewind the stack and carry on return match rewind_ext::(ctx, memory_stack, rewind_stack, store_data, None) { Errno::Success => OnCalledAction::InvokeAgain, @@ -251,7 +249,6 @@ impl WasiProcessInner { OnCalledAction::Trap(Box::new(WasiError::Exit(err.into()))) } }; - */ } })?; From e320aee899aa9242bd99f27e93d161527c31f8ee Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Wed, 25 Oct 2023 13:09:54 +1100 Subject: [PATCH 027/129] The rewinding is now fixed again for snapshots --- lib/wasix/src/os/task/thread.rs | 16 +++++++++++- lib/wasix/src/syscalls/mod.rs | 45 ++++++++++++++++++++++++++------- 2 files changed, 51 insertions(+), 10 deletions(-) diff --git a/lib/wasix/src/os/task/thread.rs b/lib/wasix/src/os/task/thread.rs index 9e8a4117fa9..6e1aa8a9443 100644 --- a/lib/wasix/src/os/task/thread.rs +++ b/lib/wasix/src/os/task/thread.rs @@ -17,6 +17,7 @@ use wasmer_wasix_types::{ use crate::{ os::task::process::{WasiProcessId, WasiProcessInner}, + syscalls::HandleRewindType, WasiRuntimeError, }; @@ -115,6 +116,19 @@ impl WasiThread { self.rewind.take() } + pub(crate) fn has_rewind_of_type(&self, _type: HandleRewindType) -> bool { + match _type { + HandleRewindType::ResultDriven => match &self.rewind { + Some(rewind) => rewind.rewind_result.is_some(), + None => false, + }, + HandleRewindType::Resultless => match &self.rewind { + Some(rewind) => rewind.rewind_result.is_none(), + None => false, + }, + } + } + /// Sets a flag that tells others that this thread is currently /// check pointing itself #[cfg(feature = "snapshot")] @@ -188,7 +202,7 @@ pub(crate) struct RewindResult { pub memory_stack: Bytes, /// Generic serialized object passed back to the rewind resumption code /// (uses the bincode serializer) - pub rewind_result: Bytes, + pub rewind_result: Option, } #[derive(Debug)] diff --git a/lib/wasix/src/syscalls/mod.rs b/lib/wasix/src/syscalls/mod.rs index c8ecd33bb1d..712c2070ca8 100644 --- a/lib/wasix/src/syscalls/mod.rs +++ b/lib/wasix/src/syscalls/mod.rs @@ -1139,12 +1139,10 @@ pub fn rewind_ext( rewind_result: Option, ) -> Errno { // Store the memory stack so that it can be restored later - if let Some(rewind_result) = rewind_result { - ctx.data_mut().thread.set_rewind(RewindResult { - memory_stack, - rewind_result, - }); - } + ctx.data_mut().thread.set_rewind(RewindResult { + memory_stack, + rewind_result, + }); // Deserialize the store data back into a snapshot let store_snapshot = match InstanceSnapshot::deserialize(&store_data[..]) { @@ -1233,6 +1231,8 @@ pub fn maybe_snapshot_once( ) -> WasiResult> { use crate::os::task::process::{WasiProcessCheckpoint, WasiProcessInner}; + unsafe { handle_rewind_ext::(&mut ctx, HandleRewindType::Resultless) }; + if ctx.data().enable_snapshot_capture == false { return Ok(Ok(ctx)); } @@ -1288,12 +1288,32 @@ pub(crate) unsafe fn handle_rewind( where T: serde::de::DeserializeOwned, { + handle_rewind_ext::(ctx, HandleRewindType::ResultDriven) +} + +pub(crate) enum HandleRewindType { + /// Handle rewind types that have a result to be processed + ResultDriven, + /// Handle rewind types that are resultless (generally these + /// are caused by snapshot events) + Resultless, +} + +pub(crate) unsafe fn handle_rewind_ext( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + _type: HandleRewindType, +) -> Option +where + T: serde::de::DeserializeOwned, +{ + if !ctx.data().thread.has_rewind_of_type(_type) { + return None; + }; + // If the stack has been restored if let Some(result) = ctx.data_mut().thread.take_rewind() { // Deserialize the result let memory_stack = result.memory_stack; - let ret = bincode::deserialize(&result.rewind_result) - .expect("failed to deserialize the rewind result"); // Notify asyncify that we are no longer rewinding let env = ctx.data(); @@ -1307,7 +1327,14 @@ where // Restore the memory stack let (env, mut store) = ctx.data_and_store_mut(); set_memory_stack::(env, &mut store, memory_stack); - Some(ret) + + if let Some(rewind_result) = result.rewind_result { + let ret = bincode::deserialize(&rewind_result) + .expect("failed to deserialize the rewind result"); + Some(ret) + } else { + None + } } else { None } From 46aa87330331a07634cc48cc1f9ac786f34e48c7 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Wed, 25 Oct 2023 18:19:18 +1100 Subject: [PATCH 028/129] Finished a basic version of the snapshot restoration --- lib/cli/src/commands/run.rs | 14 ++- lib/cli/src/commands/run/wasi.rs | 13 ++- lib/wasix/src/lib.rs | 16 +-- lib/wasix/src/os/task/process.rs | 2 +- lib/wasix/src/runners/wasi.rs | 16 ++- lib/wasix/src/runners/wasi_common.rs | 4 +- lib/wasix/src/runners/wcgi/runner.rs | 8 +- lib/wasix/src/snapshot/capturer.rs | 7 +- lib/wasix/src/snapshot/compactor.rs | 4 +- lib/wasix/src/snapshot/effector.rs | 3 +- lib/wasix/src/snapshot/filter.rs | 6 +- lib/wasix/src/snapshot/log_file.rs | 9 +- lib/wasix/src/snapshot/unsupported.rs | 4 +- lib/wasix/src/state/builder.rs | 55 +++++++---- lib/wasix/src/syscalls/mod.rs | 134 +++++++++++++++++++++++++- 15 files changed, 244 insertions(+), 51 deletions(-) diff --git a/lib/cli/src/commands/run.rs b/lib/cli/src/commands/run.rs index ed119417916..2b54ffff5c7 100644 --- a/lib/cli/src/commands/run.rs +++ b/lib/cli/src/commands/run.rs @@ -242,7 +242,11 @@ impl Run { runner.with_snapshot_save(Arc::new(LogFileSnapshotCapturer::new_std(path)?)); } if let Some(path) = self.wasi.resume_from.clone() { - runner.with_snapshot_restore(Arc::new(LogFileSnapshotCapturer::new_std(path)?)); + let n_snapshots = self.wasi.resume_num_snapshots; + runner.with_snapshot_restore( + Arc::new(LogFileSnapshotCapturer::new_std(path)?), + n_snapshots, + ); } } } @@ -293,9 +297,11 @@ impl Run { .with_snapshot_save(Arc::new(LogFileSnapshotCapturer::new_std(path)?)); } if let Some(path) = self.wasi.resume_from.clone() { - runner - .config() - .with_snapshot_restore(Arc::new(LogFileSnapshotCapturer::new_std(path)?)); + let n_snapshots = self.wasi.resume_num_snapshots; + runner.config().with_snapshot_restore( + Arc::new(LogFileSnapshotCapturer::new_std(path)?), + n_snapshots, + ); } } } diff --git a/lib/cli/src/commands/run/wasi.rs b/lib/cli/src/commands/run/wasi.rs index 555358780d9..8fb785602e4 100644 --- a/lib/cli/src/commands/run/wasi.rs +++ b/lib/cli/src/commands/run/wasi.rs @@ -136,6 +136,12 @@ pub struct Wasi { #[clap(long = "resume-from")] pub resume_from: Option, + /// When specified this limits the number of restoration points + /// that the resume will take before it resumes execution. + #[cfg(feature = "snapshot")] + #[clap(long = "resume-snapshot-limit")] + pub resume_num_snapshots: Option, + /// Allow instances to send http requests. /// /// Access to domains is granted by default. @@ -276,8 +282,11 @@ impl Wasi { .with_snapshot_save(Arc::new(LogFileSnapshotCapturer::new_std(path)?)); } if let Some(path) = self.resume_from.clone() { - builder = builder - .with_snapshot_restore(Arc::new(LogFileSnapshotCapturer::new_std(path)?)); + let n_snapshots = self.resume_num_snapshots; + builder = builder.with_snapshot_restore( + Arc::new(LogFileSnapshotCapturer::new_std(path)?), + n_snapshots, + ); } } } diff --git a/lib/wasix/src/lib.rs b/lib/wasix/src/lib.rs index a8b1c5903ca..72f8d89397e 100644 --- a/lib/wasix/src/lib.rs +++ b/lib/wasix/src/lib.rs @@ -58,12 +58,10 @@ mod utils; /// WAI based bindings. mod bindings; -use std::sync::Arc; - #[allow(unused_imports)] use bytes::{Bytes, BytesMut}; use os::task::control_plane::ControlPlaneError; -use snapshot::DynSnapshotCapturer; +use syscalls::state::SnapshotRestore; use thiserror::Error; use tracing::error; // re-exports needed for OS @@ -234,13 +232,15 @@ impl WasiRuntimeError { pub(crate) fn run_wasi_func( func: &wasmer::Function, store: &mut impl AsStoreMut, - restorer: Option>, + restorer: Option, params: &[wasmer::Value], ) -> Result, WasiRuntimeError> { - // TODO - do the snapshot restoration here if restorer.is_some() { return Err(WasiRuntimeError::Runtime(RuntimeError::user( - anyhow::format_err!("snapshot restoration is not currently supported").into(), + anyhow::format_err!( + "snapshot restoration is not currently supported when running specific functions" + ) + .into(), ))); } @@ -264,9 +264,9 @@ pub(crate) fn run_wasi_func( pub(crate) fn run_wasi_func_start( func: &wasmer::Function, store: &mut impl AsStoreMut, - restorer: Option>, + restore: Option, ) -> Result<(), WasiRuntimeError> { - run_wasi_func(func, store, restorer, &[])?; + run_wasi_func(func, store, restore, &[])?; Ok(()) } diff --git a/lib/wasix/src/os/task/process.rs b/lib/wasix/src/os/task/process.rs index f9b6134815f..307ba465a86 100644 --- a/lib/wasix/src/os/task/process.rs +++ b/lib/wasix/src/os/task/process.rs @@ -196,7 +196,7 @@ impl WasiProcessInner { // Write our thread state to the snapshot let tid = ctx.data().thread.tid(); - if let Err(err) = SnapshotEffector::save_thread_state( + if let Err(err) = SnapshotEffector::save_thread_state::( &mut ctx, tid, memory_stack.clone(), diff --git a/lib/wasix/src/runners/wasi.rs b/lib/wasix/src/runners/wasi.rs index f340e150e18..41b37d3d7eb 100644 --- a/lib/wasix/src/runners/wasi.rs +++ b/lib/wasix/src/runners/wasi.rs @@ -11,6 +11,7 @@ use crate::{ capabilities::Capabilities, runners::{wasi_common::CommonWasiOptions, MappedDirectory}, snapshot::{DynSnapshotCapturer, SnapshotTrigger}, + state::SnapshotRestore, Runtime, WasiEnvBuilder, }; @@ -152,8 +153,15 @@ impl WasiRunner { self } - pub fn with_snapshot_restore(&mut self, capturer: Arc) -> &mut Self { - self.wasi.snapshot_restore.replace(capturer); + pub fn with_snapshot_restore( + &mut self, + restorer: Arc, + n_snapshots: Option, + ) -> &mut Self { + self.wasi.snapshot_restore.replace(SnapshotRestore { + restorer, + n_snapshots, + }); self } @@ -263,8 +271,8 @@ impl crate::runners::Runner for WasiRunner { } #[cfg(feature = "snapshot")] - if let Some(capturer) = self.wasi.snapshot_restore.clone() { - env = env.with_snapshot_restore(capturer); + if let Some(restore) = self.wasi.snapshot_restore.clone() { + env = env.with_snapshot_restore(restore.restorer, restore.n_snapshots); } if self diff --git a/lib/wasix/src/runners/wasi_common.rs b/lib/wasix/src/runners/wasi_common.rs index 7a7ed2fdc35..39a644573ef 100644 --- a/lib/wasix/src/runners/wasi_common.rs +++ b/lib/wasix/src/runners/wasi_common.rs @@ -15,7 +15,7 @@ use crate::{ capabilities::Capabilities, runners::MappedDirectory, snapshot::{DynSnapshotCapturer, SnapshotTrigger}, - WasiEnvBuilder, + WasiEnvBuilder, state::SnapshotRestore, }; #[derive(Derivative, Default, Clone)] @@ -30,7 +30,7 @@ pub(crate) struct CommonWasiOptions { #[derivative(Debug = "ignore")] pub(crate) snapshot_save: Option>, #[derivative(Debug = "ignore")] - pub(crate) snapshot_restore: Option>, + pub(crate) snapshot_restore: Option, pub(crate) snapshot_on: Vec, } diff --git a/lib/wasix/src/runners/wcgi/runner.rs b/lib/wasix/src/runners/wcgi/runner.rs index 8a51022ee12..ea53562ac13 100644 --- a/lib/wasix/src/runners/wcgi/runner.rs +++ b/lib/wasix/src/runners/wcgi/runner.rs @@ -251,8 +251,14 @@ impl Config { pub fn with_snapshot_restore( &mut self, capturer: Arc, + n_snapshots: Option, ) -> &mut Self { - self.wasi.snapshot_restore.replace(capturer); + use crate::state::SnapshotRestore; + + self.wasi.snapshot_restore.replace(SnapshotRestore { + restorer: capturer, + n_snapshots, + }); self } diff --git a/lib/wasix/src/snapshot/capturer.rs b/lib/wasix/src/snapshot/capturer.rs index abf05757162..1b1e662c27a 100644 --- a/lib/wasix/src/snapshot/capturer.rs +++ b/lib/wasix/src/snapshot/capturer.rs @@ -5,7 +5,7 @@ use std::time::SystemTime; use std::{borrow::Cow, ops::Range}; use wasmer_wasix_types::wasi::ExitCode; -use futures::future::{BoxFuture, LocalBoxFuture}; +use futures::future::LocalBoxFuture; use virtual_fs::Fd; use crate::WasiThreadId; @@ -121,6 +121,7 @@ pub enum SnapshotLog<'a> { call_stack: Cow<'a, [u8]>, memory_stack: Cow<'a, [u8]>, store_data: Cow<'a, [u8]>, + is_64bit: bool, }, CloseFileDescriptor { fd: Fd, @@ -174,12 +175,14 @@ impl fmt::Debug for SnapshotLog<'_> { call_stack, memory_stack, store_data, + is_64bit, } => f .debug_struct("SetThread") .field("id", id) .field("call_stack.len", &call_stack.len()) .field("memory_stack.len", &memory_stack.len()) .field("store_data.len", &store_data.len()) + .field("is_64bit", is_64bit) .finish(), Self::CloseFileDescriptor { fd } => f .debug_struct("CloseFileDescriptor") @@ -232,7 +235,7 @@ pub trait SnapshotCapturer { /// Returns a stream of snapshot objects that the runtime will use /// to restore the state of a WASM process to a previous moment in time - fn read<'a>(&'a self) -> BoxFuture<'a, anyhow::Result>>>; + fn read<'a>(&'a self) -> LocalBoxFuture<'a, anyhow::Result>>>; } pub type DynSnapshotCapturer = dyn SnapshotCapturer + Send + Sync; diff --git a/lib/wasix/src/snapshot/compactor.rs b/lib/wasix/src/snapshot/compactor.rs index c223aa3c8a6..461916fe2ab 100644 --- a/lib/wasix/src/snapshot/compactor.rs +++ b/lib/wasix/src/snapshot/compactor.rs @@ -4,7 +4,7 @@ use std::{ sync::Mutex, }; -use futures::future::{BoxFuture, LocalBoxFuture}; +use futures::future::LocalBoxFuture; use sha2::{Digest, Sha256}; use virtual_fs::Fd; @@ -133,7 +133,7 @@ impl SnapshotCapturer for CompactingSnapshotCapturer { }) } - fn read<'a>(&'a self) -> BoxFuture<'a, anyhow::Result>>> { + fn read<'a>(&'a self) -> LocalBoxFuture<'a, anyhow::Result>>> { Box::pin(async { Ok(match self.inner.read().await? { Some(SnapshotLog::UpdateMemoryRegion { region, data }) => { diff --git a/lib/wasix/src/snapshot/effector.rs b/lib/wasix/src/snapshot/effector.rs index 149ec197601..fa8d2fe2b5b 100644 --- a/lib/wasix/src/snapshot/effector.rs +++ b/lib/wasix/src/snapshot/effector.rs @@ -86,7 +86,7 @@ impl SnapshotEffector { Ok(()) } - pub fn save_thread_state( + pub fn save_thread_state( ctx: &mut FunctionEnvMut<'_, WasiEnv>, id: WasiThreadId, memory_stack: Bytes, @@ -104,6 +104,7 @@ impl SnapshotEffector { call_stack: Cow::Owned(rewind_stack.into()), memory_stack: Cow::Owned(memory_stack.into()), store_data: Cow::Owned(store_data.into()), + is_64bit: M::is_64bit(), }) .await .map_err(map_snapshot_err)?; diff --git a/lib/wasix/src/snapshot/filter.rs b/lib/wasix/src/snapshot/filter.rs index 64216f97626..30f2b928b10 100644 --- a/lib/wasix/src/snapshot/filter.rs +++ b/lib/wasix/src/snapshot/filter.rs @@ -1,4 +1,4 @@ -use futures::future::{BoxFuture, LocalBoxFuture}; +use futures::future::LocalBoxFuture; use super::*; @@ -87,6 +87,7 @@ impl SnapshotCapturer for FilteredSnapshotCapturer { call_stack, memory_stack, store_data, + is_64bit, } => { if self.filter_threads { return Ok(()); @@ -96,6 +97,7 @@ impl SnapshotCapturer for FilteredSnapshotCapturer { call_stack, memory_stack, store_data, + is_64bit, } } SnapshotLog::CloseFileDescriptor { fd } => { @@ -149,7 +151,7 @@ impl SnapshotCapturer for FilteredSnapshotCapturer { }) } - fn read<'a>(&'a self) -> BoxFuture<'a, anyhow::Result>>> { + fn read<'a>(&'a self) -> LocalBoxFuture<'a, anyhow::Result>>> { Box::pin(async { self.inner.read().await }) } } diff --git a/lib/wasix/src/snapshot/log_file.rs b/lib/wasix/src/snapshot/log_file.rs index 6c8fba3a6f8..52d97bb1fc6 100644 --- a/lib/wasix/src/snapshot/log_file.rs +++ b/lib/wasix/src/snapshot/log_file.rs @@ -8,7 +8,7 @@ use std::{ use tokio::runtime::Handle; use wasmer_wasix_types::wasi::ExitCode; -use futures::future::{BoxFuture, LocalBoxFuture}; +use futures::future::LocalBoxFuture; use virtual_fs::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt, Fd}; use crate::WasiThreadId; @@ -43,6 +43,7 @@ pub enum SnapshotLogEntry { call_stack: Vec, memory_stack: Vec, store_data: Vec, + is_64bit: bool, }, CloseFileDescriptorV1 { fd: Fd, @@ -87,11 +88,13 @@ impl<'a> From> for SnapshotLogEntry { call_stack, memory_stack, store_data, + is_64bit, } => Self::SetThreadV1 { id, call_stack: call_stack.into_owned(), memory_stack: memory_stack.into_owned(), store_data: store_data.into_owned(), + is_64bit, }, SnapshotLog::CloseFileDescriptor { fd } => Self::CloseFileDescriptorV1 { fd }, SnapshotLog::OpenFileDescriptor { fd, state } => Self::OpenFileDescriptorV1 { @@ -142,11 +145,13 @@ impl<'a> From for SnapshotLog<'a> { call_stack, memory_stack, store_data, + is_64bit, } => Self::SetThread { id: id, call_stack: call_stack.into(), memory_stack: memory_stack.into(), store_data: store_data.into(), + is_64bit, }, SnapshotLogEntry::CloseFileDescriptorV1 { fd } => Self::CloseFileDescriptor { fd }, SnapshotLogEntry::OpenFileDescriptorV1 { fd, state } => Self::OpenFileDescriptor { @@ -259,7 +264,7 @@ impl SnapshotCapturer for LogFileSnapshotCapturer { /// UNSAFE: This method uses unsafe operations to remove the need to zero /// the buffer before its read the log entries into it - fn read<'a>(&'a self) -> BoxFuture<'a, anyhow::Result>>> { + fn read<'a>(&'a self) -> LocalBoxFuture<'a, anyhow::Result>>> { Box::pin(async { let mut state = self.state.lock().await; diff --git a/lib/wasix/src/snapshot/unsupported.rs b/lib/wasix/src/snapshot/unsupported.rs index e8040b48430..51d3cb227ba 100644 --- a/lib/wasix/src/snapshot/unsupported.rs +++ b/lib/wasix/src/snapshot/unsupported.rs @@ -1,4 +1,4 @@ -use futures::future::{BoxFuture, LocalBoxFuture}; +use futures::future::LocalBoxFuture; use super::*; @@ -16,7 +16,7 @@ impl SnapshotCapturer for UnsupportedSnapshotCapturer { Box::pin(async { Err(anyhow::format_err!("unsupported")) }) } - fn read<'a>(&'a self) -> BoxFuture<'a, anyhow::Result>>> { + fn read<'a>(&'a self) -> LocalBoxFuture<'a, anyhow::Result>>> { Box::pin(async { Ok(None) }) } } diff --git a/lib/wasix/src/state/builder.rs b/lib/wasix/src/state/builder.rs index a577fd19bac..bc4e7d1fe44 100644 --- a/lib/wasix/src/state/builder.rs +++ b/lib/wasix/src/state/builder.rs @@ -25,7 +25,10 @@ use crate::{ runtime::task_manager::InlineWaker, snapshot::DynSnapshotCapturer, state::WasiState, - syscalls::types::{__WASI_STDERR_FILENO, __WASI_STDIN_FILENO, __WASI_STDOUT_FILENO}, + syscalls::{ + restore_snapshot, + types::{__WASI_STDERR_FILENO, __WASI_STDIN_FILENO, __WASI_STDOUT_FILENO}, + }, RewindState, Runtime, WasiEnv, WasiError, WasiFunctionEnv, WasiRuntimeError, }; @@ -78,12 +81,20 @@ pub struct WasiEnvBuilder { pub(super) snapshot_on: Vec, #[cfg(feature = "snapshot")] - pub(super) snapshot_restore: Option>, + pub(super) snapshot_restore: Option, #[cfg(feature = "snapshot")] pub(super) snapshot_save: Option>, } +#[derive(Clone)] +pub struct SnapshotRestore { + /// Snapshot capturer that will be used to restore state + pub restorer: Arc, + /// Maximum number of snapshots taken before execution resumes + pub n_snapshots: Option, +} + impl std::fmt::Debug for WasiEnvBuilder { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { // TODO: update this when stable @@ -495,8 +506,15 @@ impl WasiEnvBuilder { /// Supplies a snapshot capturer which will be used to read the /// snapshot journal events and replay them into the WasiEnv /// rather than starting a new instance from scratch - pub fn with_snapshot_restore(mut self, snapshot_capturer: Arc) -> Self { - self.snapshot_restore.replace(snapshot_capturer); + pub fn with_snapshot_restore( + mut self, + restorer: Arc, + n_snapshots: Option, + ) -> Self { + self.snapshot_restore.replace(SnapshotRestore { + restorer, + n_snapshots, + }); self } @@ -859,14 +877,14 @@ impl WasiEnvBuilder { ); } - let snapshot_restore = self.snapshot_restore.clone(); + let restore = self.snapshot_restore.clone(); let (instance, env) = self.instantiate(module, store)?; let start = instance.exports.get_function("_start")?; env.data(&store).thread.set_status_running(); - let result = crate::run_wasi_func_start(start, store, snapshot_restore); + let result = crate::run_wasi_func_start(start, store, restore); let (result, exit_code) = wasi_exit_code(result); let pid = env.data(&store).pid(); @@ -907,15 +925,17 @@ impl WasiEnvBuilder { let snapshot_restore = self.snapshot_restore.clone(); - // TODO - need to restore the memory and pass the rewind_state to the `run_with_deep_sleep` emthod - if snapshot_restore.is_some() { - panic!("Snapshot restoration is not currently supported."); - } - let (_, env) = self.instantiate(module, &mut store)?; env.data(&store).thread.set_status_running(); + let mut rewind_state = None; + if let Some(restore) = snapshot_restore { + let ctx = env.env.clone().into_mut(&mut store); + let rewind = restore_snapshot(ctx, restore)?; + rewind_state = Some((rewind, None)); + } + let tasks = env.data(&store).tasks().clone(); let pid = env.data(&store).pid(); let tid = env.data(&store).tid(); @@ -926,7 +946,7 @@ impl WasiEnvBuilder { let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel(); tasks.task_dedicated(Box::new(move || { - run_with_deep_sleep(store, None, env, tx); + run_with_deep_sleep(store, rewind_state, env, tx); }))?; let result = InlineWaker::block_on(rx.recv()); @@ -976,7 +996,7 @@ fn wasi_exit_code( fn run_with_deep_sleep( mut store: Store, - rewind_state: Option<(RewindState, Bytes)>, + rewind_state: Option<(RewindState, Option)>, env: WasiFunctionEnv, sender: tokio::sync::mpsc::UnboundedSender>, ) { @@ -988,7 +1008,7 @@ fn run_with_deep_sleep( rewind_state.memory_stack, rewind_state.rewind_stack, rewind_state.store_data, - Some(rewind_result), + rewind_result, ) } else { crate::rewind_ext::( @@ -996,7 +1016,7 @@ fn run_with_deep_sleep( rewind_state.memory_stack, rewind_state.rewind_stack, rewind_state.store_data, - Some(rewind_result), + rewind_result, ) }; @@ -1048,8 +1068,9 @@ fn handle_result( let tasks = env.data(&store).tasks().clone(); let rewind = work.rewind; - let respawn = - move |ctx, store, res| run_with_deep_sleep(store, Some((rewind, res)), ctx, sender); + let respawn = move |ctx, store, res| { + run_with_deep_sleep(store, Some((rewind, Some(res))), ctx, sender) + }; // Spawns the WASM process after a trigger unsafe { diff --git a/lib/wasix/src/syscalls/mod.rs b/lib/wasix/src/syscalls/mod.rs index 712c2070ca8..41b32c6fa8f 100644 --- a/lib/wasix/src/syscalls/mod.rs +++ b/lib/wasix/src/syscalls/mod.rs @@ -90,7 +90,10 @@ pub(crate) use self::types::{ }, *, }; -use self::{state::WasiInstanceGuardMemory, utils::WasiDummyWaker}; +use self::{ + state::{SnapshotRestore, WasiInstanceGuardMemory}, + utils::WasiDummyWaker, +}; pub(crate) use crate::os::task::{ process::{WasiProcessId, WasiProcessWait}, thread::{WasiThread, WasiThreadId}, @@ -119,8 +122,10 @@ use crate::{ }, os::task::{process::MaybeCheckpointResult, thread::RewindResult}, runtime::task_manager::InlineWaker, + snapshot::DynSnapshotCapturer, utils::store::InstanceSnapshot, DeepSleepWork, RewindPostProcess, RewindState, SpawnError, WasiInodes, WasiResult, + WasiRuntimeError, }; pub(crate) use crate::{net::net_error_into_wasi_err, utils::WasiParkingLot}; @@ -1282,6 +1287,133 @@ pub fn maybe_snapshot( Ok(Ok(ctx)) } +pub fn restore_snapshot( + mut ctx: FunctionEnvMut<'_, WasiEnv>, + restore: SnapshotRestore, +) -> Result { + let restorer = restore.restorer; + let mut n_snapshots = restore.n_snapshots; + + InlineWaker::block_on(async { + let mut rewind = None; + while let Some(next) = restorer + .read() + .await + .map_err(|err| WasiRuntimeError::Runtime(RuntimeError::user(err.into())))? + { + tracing::trace!("Restoring snapshot event - {next:?}"); + match next { + crate::snapshot::SnapshotLog::Init { .. } => { + // TODO: Here we need to check the hash matches the binary + } + crate::snapshot::SnapshotLog::TerminalData { data } => { + if let Some(mut stdout) = ctx + .data() + .stdout() + .map_err(|err| WasiRuntimeError::Runtime(RuntimeError::user(err.into())))? + { + stdout.write_all(data.as_ref()).await.map_err(|err| { + WasiRuntimeError::Runtime(RuntimeError::user(err.into())) + })?; + } + } + crate::snapshot::SnapshotLog::UpdateMemoryRegion { region, data } => { + let (env, mut store) = ctx.data_and_store_mut(); + let memory = unsafe { env.memory_view(&mut store) }; + memory + .write(region.start, data.as_ref()) + .map_err(|err| WasiRuntimeError::Runtime(RuntimeError::user(err.into())))? + } + crate::snapshot::SnapshotLog::CloseThread { id, exit_code } => { + if id == ctx.data().tid() { + return Err(WasiRuntimeError::Runtime(RuntimeError::user( + anyhow::format_err!( + "Snapshot restoration has already closed the main thread." + ) + .into(), + ))); + } + } + crate::snapshot::SnapshotLog::SetThread { + id, + call_stack, + memory_stack, + store_data, + is_64bit, + } => { + if id == ctx.data().tid() { + rewind.replace(RewindState { + memory_stack: memory_stack.to_vec().into(), + rewind_stack: call_stack.to_vec().into(), + store_data: store_data.to_vec().into(), + is_64bit: is_64bit, + }); + } else { + return Err(WasiRuntimeError::Runtime(RuntimeError::user( + anyhow::format_err!( + "Snapshot restoration does not currently support multiple threads." + ) + .into(), + ))); + } + } + crate::snapshot::SnapshotLog::CloseFileDescriptor { fd: _ } => { + return Err(WasiRuntimeError::Runtime(RuntimeError::user( + anyhow::format_err!( + "Snapshot restoration does not currently support file descriptors." + ) + .into(), + ))); + } + crate::snapshot::SnapshotLog::OpenFileDescriptor { fd: _, state: _ } => { + return Err(WasiRuntimeError::Runtime(RuntimeError::user( + anyhow::format_err!( + "Snapshot restoration does not currently support file descriptors." + ) + .into(), + ))); + } + crate::snapshot::SnapshotLog::RemoveFileSystemEntry { path: _ } => { + return Err(WasiRuntimeError::Runtime(RuntimeError::user( + anyhow::format_err!( + "Snapshot restoration does not currently support local file changes." + ) + .into(), + ))); + } + crate::snapshot::SnapshotLog::UpdateFileSystemEntry { + path: _, + ft: _, + accessed: _, + created: _, + modified: _, + len: _, + data: _, + } => { + return Err(WasiRuntimeError::Runtime(RuntimeError::user( + anyhow::format_err!( + "Snapshot restoration does not currently support local file changes." + ) + .into(), + ))); + } + crate::snapshot::SnapshotLog::Snapshot { + when: _, + trigger: _, + } => { + if let Some(n_snapshots) = &mut n_snapshots { + if *n_snapshots <= 1 { + break; + } + *n_snapshots -= 1; + } + } + } + } + rewind.ok_or(WasiRuntimeError::Runtime(RuntimeError::user(anyhow::format_err!("The restored snapshot journal does not have a thread stack events and hence we can not restore the state of the process.").into()))) + }) +} + pub(crate) unsafe fn handle_rewind( ctx: &mut FunctionEnvMut<'_, WasiEnv>, ) -> Option From c51902296afd67603ffe91b4f8e28eabccd46d7f Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Fri, 10 Nov 2023 15:59:48 +1100 Subject: [PATCH 029/129] Added more snapshot events for the file system --- lib/types/src/memory.rs | 2 + lib/wasix/src/snapshot/capturer.rs | 149 ++++++-- lib/wasix/src/snapshot/compactor.rs | 2 +- lib/wasix/src/snapshot/effector.rs | 198 ++++++++-- lib/wasix/src/snapshot/filter.rs | 74 ++-- lib/wasix/src/snapshot/log_file.rs | 82 ++++- lib/wasix/src/syscalls/mod.rs | 62 ++-- lib/wasix/src/syscalls/wasi/fd_write.rs | 339 ++++++++++++------ .../syscalls/wasi/path_remove_directory.rs | 57 ++- lib/wasix/src/syscalls/wasi/path_rename.rs | 68 ++-- .../src/syscalls/wasi/path_unlink_file.rs | 32 +- 11 files changed, 783 insertions(+), 282 deletions(-) diff --git a/lib/types/src/memory.rs b/lib/types/src/memory.rs index 66c09654317..5999f0c11c3 100644 --- a/lib/types/src/memory.rs +++ b/lib/types/src/memory.rs @@ -1,4 +1,5 @@ use crate::{Pages, ValueType}; +use core::ops::SubAssign; use rkyv::{Archive, Deserialize as RkyvDeserialize, Serialize as RkyvSerialize}; #[cfg(feature = "enable-serde")] use serde::{Deserialize, Serialize}; @@ -93,6 +94,7 @@ pub unsafe trait MemorySize: Copy { + Add + Sum + AddAssign + + SubAssign + 'static; /// Type used to pass this value as an argument or return value for a Wasm function. diff --git a/lib/wasix/src/snapshot/capturer.rs b/lib/wasix/src/snapshot/capturer.rs index 1b1e662c27a..7eb5cfefa52 100644 --- a/lib/wasix/src/snapshot/capturer.rs +++ b/lib/wasix/src/snapshot/capturer.rs @@ -3,7 +3,9 @@ use std::fmt; use std::net::SocketAddr; use std::time::SystemTime; use std::{borrow::Cow, ops::Range}; -use wasmer_wasix_types::wasi::ExitCode; +use wasmer_wasix_types::wasi::{ + Advice, EpollCtl, ExitCode, Filesize, Fstflags, LookupFlags, Snapshot0Clockid, Timestamp, Tty, +}; use futures::future::LocalBoxFuture; use virtual_fs::Fd; @@ -35,7 +37,7 @@ pub enum SocketSnapshot { } #[derive(Debug, Clone, Serialize, Deserialize)] -pub enum FdSnapshot<'a> { +pub enum FdOpenSnapshot<'a> { Stdin { non_blocking: bool, }, @@ -58,29 +60,29 @@ pub enum FdSnapshot<'a> { }, } -impl<'a> FdSnapshot<'a> { - pub fn into_owned(self) -> FdSnapshot<'static> { +impl<'a> FdOpenSnapshot<'a> { + pub fn into_owned(self) -> FdOpenSnapshot<'static> { match self { - FdSnapshot::Stdin { non_blocking } => FdSnapshot::Stdin { non_blocking }, - FdSnapshot::Stdout { non_blocking } => FdSnapshot::Stdout { non_blocking }, - FdSnapshot::Stderr { non_blocking } => FdSnapshot::Stderr { non_blocking }, - FdSnapshot::OpenFile { + FdOpenSnapshot::Stdin { non_blocking } => FdOpenSnapshot::Stdin { non_blocking }, + FdOpenSnapshot::Stdout { non_blocking } => FdOpenSnapshot::Stdout { non_blocking }, + FdOpenSnapshot::Stderr { non_blocking } => FdOpenSnapshot::Stderr { non_blocking }, + FdOpenSnapshot::OpenFile { path, offset, read, write, non_blocking, - } => FdSnapshot::OpenFile { + } => FdOpenSnapshot::OpenFile { path: Cow::Owned(path.into_owned()), offset, read, write, non_blocking, }, - FdSnapshot::Socket { + FdOpenSnapshot::Socket { state, non_blocking, - } => FdSnapshot::Socket { + } => FdOpenSnapshot::Socket { state, non_blocking, }, @@ -105,13 +107,20 @@ pub enum SnapshotLog<'a> { Init { wasm_hash: [u8; 32], }, - TerminalData { + FileDescriptorWrite { + fd: Fd, + offset: u64, data: Cow<'a, [u8]>, + is_64bit: bool, }, UpdateMemoryRegion { region: Range, data: Cow<'a, [u8]>, }, + SetClockTime { + clock_id: Snapshot0Clockid, + time: Timestamp, + }, CloseThread { id: WasiThreadId, exit_code: Option, @@ -128,19 +137,85 @@ pub enum SnapshotLog<'a> { }, OpenFileDescriptor { fd: Fd, - state: FdSnapshot<'a>, + state: FdOpenSnapshot<'a>, + }, + RenumberFileDescriptor { + old_fd: Fd, + new_fd: Fd, + }, + DuplicateFileDescriptor { + old_fd: Fd, + new_fd: Fd, }, - RemoveFileSystemEntry { + CreateDirectory { + fd: Fd, path: Cow<'a, str>, }, - UpdateFileSystemEntry { + RemoveDirectory { + fd: Fd, path: Cow<'a, str>, - ft: FileEntryType, - accessed: u64, - created: u64, - modified: u64, - len: u64, - data: Cow<'a, [u8]>, + }, + FileDescriptorSetTimes { + fd: Fd, + st_atim: Timestamp, + st_mtim: Timestamp, + fst_flags: Fstflags, + }, + FileDescriptorSetSize { + fd: Fd, + size: Filesize, + }, + FileDescriptorAdvise { + fd: Fd, + offset: Filesize, + len: Filesize, + advice: Advice, + }, + FileDescriptorAllocate { + fd: Fd, + offset: Filesize, + len: Filesize, + }, + CreateHardLink { + old_fd: Fd, + old_path: Cow<'a, str>, + old_flags: LookupFlags, + new_fd: Fd, + new_path: Cow<'a, str>, + }, + CreateSymbolicLink { + old_fd: Fd, + old_path: Cow<'a, str>, + new_fd: Fd, + new_path: Cow<'a, str>, + }, + UnlinkFile { + fd: Fd, + path: Cow<'a, str>, + }, + PathRename { + old_fd: Fd, + old_path: Cow<'a, str>, + new_fd: Fd, + new_path: Cow<'a, str>, + }, + ChangeDirectory { + path: Cow<'a, str>, + }, + EpollCreate { + fd: Fd, + }, + EpollCtl { + epfd: Fd, + op: EpollCtl, + fd: Fd, + }, + TtySet { + tty: Tty, + }, + CreatePipe { + fd1: Fd, + fd2: Fd, }, /// Represents the marker for the end of a snapshot Snapshot { @@ -156,9 +231,17 @@ impl fmt::Debug for SnapshotLog<'_> { .debug_struct("Init") .field("wasm_hash.len", &wasm_hash.len()) .finish(), - Self::TerminalData { data } => f + Self::FileDescriptorWrite { + fd, + offset, + data, + is_64bit, + } => f .debug_struct("TerminalData") + .field("fd", &fd) + .field("offset", &offset) .field("data.len", &data.len()) + .field("is_64bit", &is_64bit) .finish(), Self::UpdateMemoryRegion { region, data } => f .debug_struct("UpdateMemoryRegion") @@ -193,10 +276,28 @@ impl fmt::Debug for SnapshotLog<'_> { .field("fd", fd) .field("state", state) .finish(), - Self::RemoveFileSystemEntry { path } => f - .debug_struct("RemoveFileSystemEntry") + Self::RemoveDirectory { fd, path } => f + .debug_struct("RemoveDirectory") + .field("fd", fd) + .field("path", path) + .finish(), + Self::UnlinkFile { fd, path } => f + .debug_struct("UnlinkFile") + .field("fd", fd) .field("path", path) .finish(), + Self::PathRename { + old_fd, + old_path, + new_fd, + new_path, + } => f + .debug_struct("UnlinkFile") + .field("old_fd", old_fd) + .field("old_path", old_path) + .field("new_fd", new_fd) + .field("new_path", new_path) + .finish(), Self::UpdateFileSystemEntry { path, ft, diff --git a/lib/wasix/src/snapshot/compactor.rs b/lib/wasix/src/snapshot/compactor.rs index 461916fe2ab..a31ed9dfb3c 100644 --- a/lib/wasix/src/snapshot/compactor.rs +++ b/lib/wasix/src/snapshot/compactor.rs @@ -12,7 +12,7 @@ use super::*; struct State { memory_map: HashMap, [u8; 32]>, - open_file: HashMap>, + open_file: HashMap>, close_file: HashSet, } diff --git a/lib/wasix/src/snapshot/effector.rs b/lib/wasix/src/snapshot/effector.rs index fa8d2fe2b5b..80088c74bb1 100644 --- a/lib/wasix/src/snapshot/effector.rs +++ b/lib/wasix/src/snapshot/effector.rs @@ -1,18 +1,19 @@ use std::{borrow::Cow, collections::LinkedList, ops::Range, sync::MutexGuard, time::SystemTime}; use bytes::Bytes; -use virtual_fs::AsyncWriteExt; -use wasmer::{FunctionEnvMut, WasmPtr}; +use wasmer::{FunctionEnvMut, RuntimeError, WasmPtr}; use wasmer_types::MemorySize; -use wasmer_wasix_types::{types::__wasi_ciovec_t, wasi::ExitCode}; +use wasmer_wasix_types::{ + types::__wasi_ciovec_t, + wasi::{Errno, ExitCode, Fd}, +}; use crate::{ - fs::fs_error_into_wasi_err, mem_error_to_wasi, os::task::process::WasiProcessInner, - syscalls::__asyncify_light, - utils::{map_io_err, map_snapshot_err}, - WasiEnv, WasiError, WasiThreadId, + syscalls::{__asyncify_light, fd_write_internal, FdWriteSource}, + utils::map_snapshot_err, + WasiEnv, WasiError, WasiRuntimeError, WasiThreadId, }; use super::*; @@ -22,8 +23,11 @@ pub struct SnapshotEffector {} #[cfg(feature = "snapshot")] impl SnapshotEffector { - pub fn save_terminal_data( + pub fn save_write( ctx: &mut FunctionEnvMut<'_, WasiEnv>, + fd: Fd, + offset: u64, + written: usize, iovs: WasmPtr<__wasi_ciovec_t, M>, iovs_len: M::Offset, ) -> anyhow::Result<()> { @@ -33,17 +37,27 @@ impl SnapshotEffector { __asyncify_light(env, None, async { let iovs_arr = iovs_arr.access().map_err(mem_error_to_wasi)?; + let mut remaining: M::Offset = TryFrom::::try_from(written).unwrap_or_default(); for iovs in iovs_arr.iter() { + let sub = iovs.buf_len.min(remaining); + if sub == M::ZERO { + continue; + } + remaining -= sub; + let buf = WasmPtr::::new(iovs.buf) - .slice(&memory, iovs.buf_len) + .slice(&memory, sub) .map_err(mem_error_to_wasi)? .access() .map_err(mem_error_to_wasi)?; ctx.data() .runtime() .snapshot_capturer() - .write(SnapshotLog::TerminalData { + .write(SnapshotLog::FileDescriptorWrite { + fd, + offset, data: Cow::Borrowed(buf.as_ref()), + is_64bit: M::is_64bit(), }) .await .map_err(map_snapshot_err)?; @@ -54,18 +68,29 @@ impl SnapshotEffector { Ok(()) } - pub fn apply_terminal_data( + pub async fn apply_write( ctx: &mut FunctionEnvMut<'_, WasiEnv>, - data: &[u8], + fd: Fd, + offset: u64, + data: Cow<'_, [u8]>, ) -> anyhow::Result<()> { - let env = ctx.data(); - __asyncify_light(env, None, async { - if let Some(mut stdout) = ctx.data().stdout().map_err(fs_error_into_wasi_err)? { - stdout.write_all(data).await.map_err(map_io_err)?; - } - Ok(()) - })? - .map_err(|err| WasiError::Exit(ExitCode::Errno(err)))?; + let ret = fd_write_internal( + ctx, + fd, + FdWriteSource::<'_, M>::Buffer(data), + offset, + None, + true, + false, + )?; + if ret != Errno::Success { + tracing::debug!( + fd, + offset, + "restore error: failed to write to descriptor - {}", + ret + ); + } Ok(()) } @@ -189,4 +214,137 @@ impl SnapshotEffector { .map_err(|err| WasiError::Exit(ExitCode::Errno(err)))?; Ok(()) } + + pub fn apply_memory( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + region: Range, + data: &[u8], + ) -> anyhow::Result<()> { + let (env, mut store) = ctx.data_and_store_mut(); + let memory = unsafe { env.memory_view(&mut store) }; + memory + .write(region.start, data.as_ref()) + .map_err(|err| WasiRuntimeError::Runtime(RuntimeError::user(err.into())))?; + Ok(()) + } + + pub fn save_remove_directory( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + fd: Fd, + path: String, + ) -> anyhow::Result<()> { + let env = ctx.data(); + + __asyncify_light(env, None, async { + ctx.data() + .runtime() + .snapshot_capturer() + .write(SnapshotLog::RemoveDirectory { + fd, + path: Cow::Owned(path), + }) + .await + .map_err(map_snapshot_err)?; + Ok(()) + })? + .map_err(|err| WasiError::Exit(ExitCode::Errno(err)))?; + + Ok(()) + } + + pub fn apply_remove_directory( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + fd: Fd, + path: &str, + ) -> anyhow::Result<()> { + if let Err(err) = crate::syscalls::path_remove_directory_internal(ctx, fd, path) { + tracing::debug!("restore error: failed to remove directory - {}", err); + } + Ok(()) + } + + pub fn save_unlink_file( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + fd: Fd, + path: String, + ) -> anyhow::Result<()> { + let env = ctx.data(); + + __asyncify_light(env, None, async { + ctx.data() + .runtime() + .snapshot_capturer() + .write(SnapshotLog::UnlinkFile { + fd, + path: Cow::Owned(path), + }) + .await + .map_err(map_snapshot_err)?; + Ok(()) + })? + .map_err(|err| WasiError::Exit(ExitCode::Errno(err)))?; + + Ok(()) + } + + pub fn apply_unlink_file( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + fd: Fd, + path: &str, + ) -> anyhow::Result<()> { + let ret = crate::syscalls::path_unlink_file_internal(ctx, fd, path)?; + if ret != Errno::Success { + tracing::debug!(fd, path, "restore error: failed to remove file - {}", ret); + } + Ok(()) + } + + pub fn save_rename( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + old_fd: Fd, + old_path: String, + new_fd: Fd, + new_path: String, + ) -> anyhow::Result<()> { + let env = ctx.data(); + + __asyncify_light(env, None, async { + ctx.data() + .runtime() + .snapshot_capturer() + .write(SnapshotLog::PathRename { + old_fd, + old_path: Cow::Owned(old_path), + new_fd, + new_path: Cow::Owned(new_path), + }) + .await + .map_err(map_snapshot_err)?; + Ok(()) + })? + .map_err(|err| WasiError::Exit(ExitCode::Errno(err)))?; + + Ok(()) + } + + pub fn apply_rename( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + old_fd: Fd, + old_path: &str, + new_fd: Fd, + new_path: &str, + ) -> anyhow::Result<()> { + let ret = crate::syscalls::path_rename_internal(ctx, old_fd, old_path, new_fd, new_path)?; + if ret != Errno::Success { + tracing::debug!( + old_fd, + old_path, + new_fd, + new_path, + "restore error: failed to rename path - {}", + ret + ); + } + Ok(()) + } } diff --git a/lib/wasix/src/snapshot/filter.rs b/lib/wasix/src/snapshot/filter.rs index 30f2b928b10..740414f527e 100644 --- a/lib/wasix/src/snapshot/filter.rs +++ b/lib/wasix/src/snapshot/filter.rs @@ -64,87 +64,71 @@ impl SnapshotCapturer for FilteredSnapshotCapturer { Box::pin(async { let evt = match entry { SnapshotLog::Init { wasm_hash } => SnapshotLog::Init { wasm_hash }, - SnapshotLog::TerminalData { data } => { + SnapshotLog::FileDescriptorWrite { .. } => { if self.filter_terminal { return Ok(()); } - SnapshotLog::TerminalData { data } + entry } - SnapshotLog::UpdateMemoryRegion { region, data } => { + SnapshotLog::UpdateMemoryRegion { .. } => { if self.filter_memory { return Ok(()); } - SnapshotLog::UpdateMemoryRegion { region, data } + entry } - SnapshotLog::CloseThread { id, exit_code } => { + SnapshotLog::CloseThread { .. } => { if self.filter_threads { return Ok(()); } - SnapshotLog::CloseThread { id, exit_code } + entry } - SnapshotLog::SetThread { - id, - call_stack, - memory_stack, - store_data, - is_64bit, - } => { + SnapshotLog::SetThread { .. } => { if self.filter_threads { return Ok(()); } - SnapshotLog::SetThread { - id, - call_stack, - memory_stack, - store_data, - is_64bit, - } + entry } - SnapshotLog::CloseFileDescriptor { fd } => { + SnapshotLog::CloseFileDescriptor { .. } => { if self.filter_descriptors { return Ok(()); } - SnapshotLog::CloseFileDescriptor { fd } + entry } - SnapshotLog::OpenFileDescriptor { fd, state } => { + SnapshotLog::OpenFileDescriptor { .. } => { if self.filter_descriptors { return Ok(()); } - SnapshotLog::OpenFileDescriptor { fd, state } + entry + } + SnapshotLog::RemoveDirectory { .. } => { + if self.filter_files { + return Ok(()); + } + entry } - SnapshotLog::RemoveFileSystemEntry { path } => { + SnapshotLog::UnlinkFile { .. } => { if self.filter_files { return Ok(()); } - SnapshotLog::RemoveFileSystemEntry { path } + entry } - SnapshotLog::UpdateFileSystemEntry { - path, - ft, - accessed, - created, - modified, - len, - data, - } => { + SnapshotLog::PathRename { .. } => { if self.filter_files { return Ok(()); } - SnapshotLog::UpdateFileSystemEntry { - path, - ft, - accessed, - created, - modified, - len, - data, + entry + } + SnapshotLog::UpdateFileSystemEntry { .. } => { + if self.filter_files { + return Ok(()); } + entry } - SnapshotLog::Snapshot { when, trigger } => { + SnapshotLog::Snapshot { .. } => { if self.filter_snapshots { return Ok(()); } - SnapshotLog::Snapshot { when, trigger } + entry } }; self.inner.write(evt).await diff --git a/lib/wasix/src/snapshot/log_file.rs b/lib/wasix/src/snapshot/log_file.rs index 52d97bb1fc6..d22ab39c263 100644 --- a/lib/wasix/src/snapshot/log_file.rs +++ b/lib/wasix/src/snapshot/log_file.rs @@ -26,8 +26,11 @@ pub enum SnapshotLogEntry { InitV1 { wasm_hash: [u8; 32], }, - TerminalDataV1 { + Write { + fd: Fd, + offset: Option, data: Vec, + is_64bit: bool, }, UpdateMemoryRegionV1 { start: u64, @@ -50,11 +53,22 @@ pub enum SnapshotLogEntry { }, OpenFileDescriptorV1 { fd: Fd, - state: FdSnapshot<'static>, + state: FdOpenSnapshot<'static>, + }, + RemoveDirectoryV1 { + fd: Fd, + path: String, }, - RemoveFileSystemEntryV1 { + UnlinkFileV1 { + fd: Fd, path: String, }, + PathRenameV1 { + old_fd: Fd, + old_path: String, + new_fd: Fd, + new_path: String, + }, UpdateFileSystemEntryV1 { path: String, ft: FileEntryType, @@ -74,8 +88,16 @@ impl<'a> From> for SnapshotLogEntry { fn from(value: SnapshotLog<'a>) -> Self { match value { SnapshotLog::Init { wasm_hash } => Self::InitV1 { wasm_hash }, - SnapshotLog::TerminalData { data } => Self::TerminalDataV1 { + SnapshotLog::FileDescriptorWrite { + fd, + offset, + data, + is_64bit, + } => Self::Write { + fd, + offset, data: data.into_owned(), + is_64bit, }, SnapshotLog::UpdateMemoryRegion { region, data } => Self::UpdateMemoryRegionV1 { start: region.start, @@ -101,9 +123,25 @@ impl<'a> From> for SnapshotLogEntry { fd, state: state.into_owned(), }, - SnapshotLog::RemoveFileSystemEntry { path } => Self::RemoveFileSystemEntryV1 { + SnapshotLog::RemoveDirectory { fd, path } => Self::RemoveDirectoryV1 { + fd, + path: path.into_owned(), + }, + SnapshotLog::UnlinkFile { fd, path } => Self::UnlinkFileV1 { + fd, path: path.into_owned(), }, + SnapshotLog::PathRename { + old_fd, + old_path, + new_fd, + new_path, + } => Self::PathRenameV1 { + old_fd, + old_path: old_path.into_owned(), + new_fd, + new_path: new_path.into_owned(), + }, SnapshotLog::UpdateFileSystemEntry { path, ft, @@ -130,7 +168,17 @@ impl<'a> From for SnapshotLog<'a> { fn from(value: SnapshotLogEntry) -> Self { match value { SnapshotLogEntry::InitV1 { wasm_hash } => Self::Init { wasm_hash }, - SnapshotLogEntry::TerminalDataV1 { data } => Self::TerminalData { data: data.into() }, + SnapshotLogEntry::Write { + data, + fd, + offset, + is_64bit, + } => Self::FileDescriptorWrite { + data: data.into(), + fd, + offset, + is_64bit, + }, SnapshotLogEntry::UpdateMemoryRegionV1 { start, end, data } => { Self::UpdateMemoryRegion { region: start..end, @@ -158,9 +206,25 @@ impl<'a> From for SnapshotLog<'a> { fd, state: state.clone(), }, - SnapshotLogEntry::RemoveFileSystemEntryV1 { path } => { - Self::RemoveFileSystemEntry { path: path.into() } - } + SnapshotLogEntry::RemoveDirectoryV1 { fd, path } => Self::RemoveDirectory { + fd, + path: path.into(), + }, + SnapshotLogEntry::UnlinkFileV1 { fd, path } => Self::UnlinkFile { + fd, + path: path.into(), + }, + SnapshotLogEntry::PathRenameV1 { + old_fd, + old_path, + new_fd, + new_path, + } => Self::PathRename { + old_fd, + old_path: old_path.into(), + new_fd, + new_path: new_path.into(), + }, SnapshotLogEntry::UpdateFileSystemEntryV1 { path, ft, diff --git a/lib/wasix/src/syscalls/mod.rs b/lib/wasix/src/syscalls/mod.rs index 41b32c6fa8f..f0d682f6ba8 100644 --- a/lib/wasix/src/syscalls/mod.rs +++ b/lib/wasix/src/syscalls/mod.rs @@ -122,7 +122,7 @@ use crate::{ }, os::task::{process::MaybeCheckpointResult, thread::RewindResult}, runtime::task_manager::InlineWaker, - snapshot::DynSnapshotCapturer, + snapshot::{DynSnapshotCapturer, SnapshotEffector}, utils::store::InstanceSnapshot, DeepSleepWork, RewindPostProcess, RewindState, SpawnError, WasiInodes, WasiResult, WasiRuntimeError, @@ -1287,6 +1287,10 @@ pub fn maybe_snapshot( Ok(Ok(ctx)) } +pub fn anyhow_err_to_runtime_err(err: anyhow::Error) -> WasiRuntimeError { + WasiRuntimeError::Runtime(RuntimeError::user(err.into())) +} + pub fn restore_snapshot( mut ctx: FunctionEnvMut<'_, WasiEnv>, restore: SnapshotRestore, @@ -1296,33 +1300,28 @@ pub fn restore_snapshot( InlineWaker::block_on(async { let mut rewind = None; - while let Some(next) = restorer - .read() - .await - .map_err(|err| WasiRuntimeError::Runtime(RuntimeError::user(err.into())))? - { + while let Some(next) = restorer.read().await.map_err(anyhow_err_to_runtime_err)? { tracing::trace!("Restoring snapshot event - {next:?}"); match next { crate::snapshot::SnapshotLog::Init { .. } => { // TODO: Here we need to check the hash matches the binary } - crate::snapshot::SnapshotLog::TerminalData { data } => { - if let Some(mut stdout) = ctx - .data() - .stdout() - .map_err(|err| WasiRuntimeError::Runtime(RuntimeError::user(err.into())))? - { - stdout.write_all(data.as_ref()).await.map_err(|err| { - WasiRuntimeError::Runtime(RuntimeError::user(err.into())) - })?; + crate::snapshot::SnapshotLog::FileDescriptorWrite { + fd, + offset, + data, + is_64bit, + } => { + if is_64bit { + SnapshotEffector::apply_write::(&mut ctx, fd, offset, data).await + } else { + SnapshotEffector::apply_write::(&mut ctx, fd, offset, data).await } + .map_err(anyhow_err_to_runtime_err)?; } crate::snapshot::SnapshotLog::UpdateMemoryRegion { region, data } => { - let (env, mut store) = ctx.data_and_store_mut(); - let memory = unsafe { env.memory_view(&mut store) }; - memory - .write(region.start, data.as_ref()) - .map_err(|err| WasiRuntimeError::Runtime(RuntimeError::user(err.into())))? + SnapshotEffector::apply_memory(&mut ctx, region, &data) + .map_err(anyhow_err_to_runtime_err)?; } crate::snapshot::SnapshotLog::CloseThread { id, exit_code } => { if id == ctx.data().tid() { @@ -1373,13 +1372,22 @@ pub fn restore_snapshot( .into(), ))); } - crate::snapshot::SnapshotLog::RemoveFileSystemEntry { path: _ } => { - return Err(WasiRuntimeError::Runtime(RuntimeError::user( - anyhow::format_err!( - "Snapshot restoration does not currently support local file changes." - ) - .into(), - ))); + crate::snapshot::SnapshotLog::RemoveDirectory { fd, path } => { + SnapshotEffector::apply_remove_directory(&mut ctx, fd, &path) + .map_err(anyhow_err_to_runtime_err)?; + } + crate::snapshot::SnapshotLog::UnlinkFile { fd, path } => { + SnapshotEffector::apply_unlink_file(&mut ctx, fd, &path) + .map_err(anyhow_err_to_runtime_err)?; + } + crate::snapshot::SnapshotLog::PathRename { + old_fd, + old_path, + new_fd, + new_path, + } => { + SnapshotEffector::apply_rename(&mut ctx, old_fd, &old_path, new_fd, &new_path) + .map_err(anyhow_err_to_runtime_err)?; } crate::snapshot::SnapshotLog::UpdateFileSystemEntry { path: _, diff --git a/lib/wasix/src/syscalls/wasi/fd_write.rs b/lib/wasix/src/syscalls/wasi/fd_write.rs index a62703f90b5..0c5c7c99fe6 100644 --- a/lib/wasix/src/syscalls/wasi/fd_write.rs +++ b/lib/wasix/src/syscalls/wasi/fd_write.rs @@ -30,16 +30,25 @@ pub fn fd_write( iovs_len: M::Offset, nwritten: WasmPtr, ) -> Result { + let env = ctx.data(); let offset = { - let mut env = ctx.data(); let state = env.state.clone(); let inodes = state.inodes.clone(); let fd_entry = wasi_try_ok!(state.fs.get_fd(fd)); fd_entry.offset.load(Ordering::Acquire) as usize }; + let enable_snapshot_capture = env.enable_snapshot_capture; - fd_write_internal::(ctx, fd, iovs, iovs_len, offset, nwritten, true) + fd_write_internal::( + &mut ctx, + fd, + FdWriteSource::Iovs { iovs, iovs_len }, + offset as u64, + Some(nwritten), + true, + enable_snapshot_capture, + ) } /// ### `fd_pwrite()` @@ -65,31 +74,34 @@ pub fn fd_pwrite( offset: Filesize, nwritten: WasmPtr, ) -> Result { - fd_write_internal::(ctx, fd, iovs, iovs_len, offset as usize, nwritten, false) + let enable_snapshot_capture = ctx.data().enable_snapshot_capture; + fd_write_internal::( + &mut ctx, + fd, + FdWriteSource::Iovs { iovs, iovs_len }, + offset as u64, + Some(nwritten), + false, + enable_snapshot_capture, + ) +} + +pub(crate) enum FdWriteSource<'a, M: MemorySize> { + Iovs { + iovs: WasmPtr<__wasi_ciovec_t, M>, + iovs_len: M::Offset, + }, + Buffer(Cow<'a, [u8]>), } -/// ### `fd_pwrite()` -/// Write to a file without adjusting its offset -/// Inputs: -/// - `Fd` -/// File descriptor (opened with writing) to write to -/// - `const __wasi_ciovec_t *iovs` -/// List of vectors to read data from -/// - `u32 iovs_len` -/// Length of data in `iovs` -/// - `Filesize offset` -/// The offset to write at -/// Output: -/// - `u32 *nwritten` -/// Number of bytes written pub(crate) fn fd_write_internal( - mut ctx: FunctionEnvMut<'_, WasiEnv>, + ctx: &mut FunctionEnvMut<'_, WasiEnv>, fd: WasiFd, - iovs: WasmPtr<__wasi_ciovec_t, M>, - iovs_len: M::Offset, - offset: usize, - nwritten: WasmPtr, + data: FdWriteSource<'_, M>, + offset: u64, + nwritten: Option>, should_update_cursor: bool, + should_snapshot: bool, ) -> Result { wasi_try_ok!(WasiEnv::process_signals_and_exit(&mut ctx)?); @@ -104,27 +116,10 @@ pub(crate) fn fd_write_internal( return Ok(Errno::Access); } - // If snap-shooting is enabled and this is to stdio then we - // will record a terminal event. - #[cfg(feature = "snapshot")] - if is_stdio && env.enable_snapshot_capture { - SnapshotEffector::save_terminal_data(&mut ctx, iovs, iovs_len).map_err(|err| { - tracing::error!( - "failed to save terminal data to snapshot capturer - {}", - err - ); - WasiError::Exit(ExitCode::Errno(Errno::Fault)) - })?; - env = ctx.data(); - } - let fd_flags = fd_entry.flags; let mut memory = unsafe { env.memory_view(&ctx) }; - let iovs_arr = wasi_try_mem_ok!(iovs.slice(&memory, iovs_len)); - - let (bytes_written, can_update_cursor) = { - let iovs_arr = wasi_try_mem_ok!(iovs_arr.access()); + let (bytes_written, can_update_cursor, can_snapshot) = { let (mut memory, _) = unsafe { env.get_memory_and_wasi_state(&ctx, 0) }; let mut guard = fd_entry.inode.write(); match guard.deref_mut() { @@ -150,22 +145,38 @@ pub(crate) fn fd_write_internal( } let mut written = 0usize; - for iovs in iovs_arr.iter() { - let buf = WasmPtr::::new(iovs.buf) - .slice(&memory, iovs.buf_len) - .map_err(mem_error_to_wasi)? - .access() - .map_err(mem_error_to_wasi)?; - let local_written = match handle.write(buf.as_ref()).await { - Ok(s) => s, - Err(_) if written > 0 => break, - Err(err) => return Err(map_io_err(err)), - }; - written += local_written; - if local_written != buf.len() { - break; + + match &data { + FdWriteSource::Iovs { iovs, iovs_len } => { + let iovs_arr = iovs + .slice(&memory, iovs_len.clone()) + .map_err(mem_error_to_wasi)?; + let iovs_arr = + iovs_arr.access().map_err(mem_error_to_wasi)?; + for iovs in iovs_arr.iter() { + let buf = WasmPtr::::new(iovs.buf) + .slice(&memory, iovs.buf_len) + .map_err(mem_error_to_wasi)? + .access() + .map_err(mem_error_to_wasi)?; + let local_written = + match handle.write(buf.as_ref()).await { + Ok(s) => s, + Err(_) if written > 0 => break, + Err(err) => return Err(map_io_err(err)), + }; + written += local_written; + if local_written != buf.len() { + break; + } + } + } + FdWriteSource::Buffer(data) => { + handle.write_all(data).await?; + written += data.len(); } } + if is_stdio { handle.flush().await.map_err(map_io_err)?; } @@ -177,7 +188,7 @@ pub(crate) fn fd_write_internal( a => a, })); - (written, true) + (written, true, true) } else { return Ok(Errno::Inval); } @@ -195,43 +206,78 @@ pub(crate) fn fd_write_internal( let tasks = env.tasks().clone(); - let res = __asyncify_light(env, None, async move { + let res = __asyncify_light(env, None, async { let mut sent = 0usize; - for iovs in iovs_arr.iter() { - let buf = WasmPtr::::new(iovs.buf) - .slice(&memory, iovs.buf_len) - .map_err(mem_error_to_wasi)? - .access() - .map_err(mem_error_to_wasi)?; - let local_sent = socket - .send(tasks.deref(), buf.as_ref(), Some(timeout), nonblocking) - .await?; - sent += local_sent; - if local_sent != buf.len() { - break; + + match &data { + FdWriteSource::Iovs { iovs, iovs_len } => { + let iovs_arr = iovs + .slice(&memory, iovs_len.clone()) + .map_err(mem_error_to_wasi)?; + let iovs_arr = iovs_arr.access().map_err(mem_error_to_wasi)?; + for iovs in iovs_arr.iter() { + let buf = WasmPtr::::new(iovs.buf) + .slice(&memory, iovs.buf_len) + .map_err(mem_error_to_wasi)? + .access() + .map_err(mem_error_to_wasi)?; + let local_sent = socket + .send( + tasks.deref(), + buf.as_ref(), + Some(timeout), + nonblocking, + ) + .await?; + sent += local_sent; + if local_sent != buf.len() { + break; + } + } + } + FdWriteSource::Buffer(data) => { + sent += socket + .send(tasks.deref(), data.as_ref(), Some(timeout), nonblocking) + .await?; } } Ok(sent) }); let written = wasi_try_ok!(res?); - (written, false) + (written, false, false) } Kind::Pipe { pipe } => { let mut written = 0usize; - for iovs in iovs_arr.iter() { - let buf = wasi_try_ok!(WasmPtr::::new(iovs.buf) - .slice(&memory, iovs.buf_len) - .map_err(mem_error_to_wasi)); - let buf = wasi_try_ok!(buf.access().map_err(mem_error_to_wasi)); - let local_written = wasi_try_ok!( - std::io::Write::write(pipe, buf.as_ref()).map_err(map_io_err) - ); - written += local_written; - if local_written != buf.len() { - break; + + match &data { + FdWriteSource::Iovs { iovs, iovs_len } => { + let iovs_arr = wasi_try_ok!(iovs + .slice(&memory, iovs_len.clone()) + .map_err(mem_error_to_wasi)); + let iovs_arr = + wasi_try_ok!(iovs_arr.access().map_err(mem_error_to_wasi)); + for iovs in iovs_arr.iter() { + let buf = wasi_try_ok!(WasmPtr::::new(iovs.buf) + .slice(&memory, iovs.buf_len) + .map_err(mem_error_to_wasi)); + let buf = wasi_try_ok!(buf.access().map_err(mem_error_to_wasi)); + let local_written = + wasi_try_ok!(std::io::Write::write(pipe, buf.as_ref()) + .map_err(map_io_err)); + + written += local_written; + if local_written != buf.len() { + break; + } + } + } + FdWriteSource::Buffer(data) => { + wasi_try_ok!(std::io::Write::write_all(pipe, data).map_err(map_io_err)); + written += data.len(); } } - (written, false) + + (written, false, true) } Kind::Dir { .. } | Kind::Root { .. } => { // TODO: verify @@ -239,48 +285,107 @@ pub(crate) fn fd_write_internal( } Kind::EventNotifications { inner } => { let mut written = 0usize; - for iovs in iovs_arr.iter() { - let buf_len: usize = - wasi_try_ok!(iovs.buf_len.try_into().map_err(|_| Errno::Inval)); - let will_be_written = buf_len; - - let val_cnt = buf_len / std::mem::size_of::(); - let val_cnt: M::Offset = - wasi_try_ok!(val_cnt.try_into().map_err(|_| Errno::Inval)); - - let vals = wasi_try_ok!(WasmPtr::::new(iovs.buf) - .slice(&memory, val_cnt as M::Offset) - .map_err(mem_error_to_wasi)); - let vals = wasi_try_ok!(vals.access().map_err(mem_error_to_wasi)); - for val in vals.iter() { - inner.write(*val); - } - written += will_be_written; + match &data { + FdWriteSource::Iovs { iovs, iovs_len } => { + let iovs_arr = wasi_try_ok!(iovs + .slice(&memory, iovs_len.clone()) + .map_err(mem_error_to_wasi)); + let iovs_arr = + wasi_try_ok!(iovs_arr.access().map_err(mem_error_to_wasi)); + for iovs in iovs_arr.iter() { + let buf_len: usize = + wasi_try_ok!(iovs.buf_len.try_into().map_err(|_| Errno::Inval)); + let will_be_written = buf_len; + + let val_cnt = buf_len / std::mem::size_of::(); + let val_cnt: M::Offset = + wasi_try_ok!(val_cnt.try_into().map_err(|_| Errno::Inval)); + + let vals = wasi_try_ok!(WasmPtr::::new(iovs.buf) + .slice(&memory, val_cnt as M::Offset) + .map_err(mem_error_to_wasi)); + let vals = wasi_try_ok!(vals.access().map_err(mem_error_to_wasi)); + for val in vals.iter() { + inner.write(*val); + } + + written += will_be_written; + } + } + FdWriteSource::Buffer(data) => { + let cnt = data.len() / std::mem::size_of::(); + for n in 0..cnt { + let start = n * std::mem::size_of::(); + let data = [ + data[start], + data[start + 1], + data[start + 2], + data[start + 3], + data[start + 4], + data[start + 5], + data[start + 6], + data[start + 7], + ]; + inner.write(u64::from_ne_bytes(data)); + } + } } - (written, false) + + (written, false, true) } Kind::Symlink { .. } | Kind::Epoll { .. } => return Ok(Errno::Inval), Kind::Buffer { buffer } => { let mut written = 0usize; - for iovs in iovs_arr.iter() { - let buf = wasi_try_ok!(WasmPtr::::new(iovs.buf) - .slice(&memory, iovs.buf_len) - .map_err(mem_error_to_wasi)); - let buf = wasi_try_ok!(buf.access().map_err(mem_error_to_wasi)); - let local_written = + + match &data { + FdWriteSource::Iovs { iovs, iovs_len } => { + let iovs_arr = wasi_try_ok!(iovs + .slice(&memory, iovs_len.clone()) + .map_err(mem_error_to_wasi)); + let iovs_arr = + wasi_try_ok!(iovs_arr.access().map_err(mem_error_to_wasi)); + for iovs in iovs_arr.iter() { + let buf = wasi_try_ok!(WasmPtr::::new(iovs.buf) + .slice(&memory, iovs.buf_len) + .map_err(mem_error_to_wasi)); + let buf = wasi_try_ok!(buf.access().map_err(mem_error_to_wasi)); + let local_written = + wasi_try_ok!(std::io::Write::write(buffer, buf.as_ref()) + .map_err(map_io_err)); + written += local_written; + if local_written != buf.len() { + break; + } + } + } + FdWriteSource::Buffer(data) => { wasi_try_ok!( - std::io::Write::write(buffer, buf.as_ref()).map_err(map_io_err) + std::io::Write::write_all(buffer, data).map_err(map_io_err) ); - written += local_written; - if local_written != buf.len() { - break; + written += data.len(); } } - (written, false) + + (written, false, true) } } }; + + #[cfg(feature = "snapshot")] + if should_snapshot && can_snapshot && bytes_written > 0 { + if let FdWriteSource::Iovs { iovs, iovs_len } = data { + SnapshotEffector::save_write(&mut ctx, fd, offset, bytes_written, iovs, iovs_len) + .map_err(|err| { + tracing::error!( + "failed to save terminal data to snapshot capturer - {}", + err + ); + WasiError::Exit(ExitCode::Errno(Errno::Fault)) + })?; + } + } + env = ctx.data(); memory = unsafe { env.memory_view(&ctx) }; @@ -303,13 +408,15 @@ pub(crate) fn fd_write_internal( } bytes_written }; - Span::current().record("nwritten", bytes_written); + if let Some(nwritten) = nwritten { + Span::current().record("nwritten", bytes_written); - let memory = unsafe { env.memory_view(&ctx) }; - let nwritten_ref = nwritten.deref(&memory); - let bytes_written: M::Offset = - wasi_try_ok!(bytes_written.try_into().map_err(|_| Errno::Overflow)); - wasi_try_mem_ok!(nwritten_ref.write(bytes_written)); + let memory = unsafe { env.memory_view(&ctx) }; + let nwritten_ref = nwritten.deref(&memory); + let bytes_written: M::Offset = + wasi_try_ok!(bytes_written.try_into().map_err(|_| Errno::Overflow)); + wasi_try_mem_ok!(nwritten_ref.write(bytes_written)); + } Ok(Errno::Success) } diff --git a/lib/wasix/src/syscalls/wasi/path_remove_directory.rs b/lib/wasix/src/syscalls/wasi/path_remove_directory.rs index 6eb72b1786c..328bd63016f 100644 --- a/lib/wasix/src/syscalls/wasi/path_remove_directory.rs +++ b/lib/wasix/src/syscalls/wasi/path_remove_directory.rs @@ -4,7 +4,7 @@ use crate::syscalls::*; /// Returns Errno::Notemtpy if directory is not empty #[instrument(level = "debug", skip_all, fields(%fd, path = field::Empty), ret)] pub fn path_remove_directory( - ctx: FunctionEnvMut<'_, WasiEnv>, + mut ctx: FunctionEnvMut<'_, WasiEnv>, fd: WasiFd, path: WasmPtr, path_len: M::Offset, @@ -25,25 +25,47 @@ pub fn path_remove_directory( ); } - let inode = wasi_try!(state.fs.get_inode_at_path(inodes, fd, &path_str, false)); - let (parent_inode, childs_name) = wasi_try!(state.fs.get_parent_inode_at_path( - inodes, - fd, - std::path::Path::new(&path_str), - false - )); + wasi_try!(path_remove_directory_internal(&mut ctx, fd, &path_str)); + let env = ctx.data(); + + #[cfg(feature = "snapshot")] + if env.enable_snapshot_capture { + wasi_try!( + SnapshotEffector::save_remove_directory(&mut ctx, fd, path_str).map_err(|err| { + tracing::error!("failed to save unlink event to snapshot capturer - {}", err); + Errno::Fault + }) + ) + } + + Errno::Success +} + +pub(crate) fn path_remove_directory_internal( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + fd: WasiFd, + path: &str, +) -> Result<(), Errno> { + let env = ctx.data(); + let (memory, mut state, inodes) = unsafe { env.get_memory_and_wasi_state_and_inodes(&ctx, 0) }; + + let inode = state.fs.get_inode_at_path(inodes, fd, &path, false)?; + let (parent_inode, childs_name) = + state + .fs + .get_parent_inode_at_path(inodes, fd, std::path::Path::new(&path), false)?; let host_path_to_remove = { let guard = inode.read(); match guard.deref() { Kind::Dir { entries, path, .. } => { - if !entries.is_empty() || wasi_try!(state.fs_read_dir(path)).count() != 0 { - return Errno::Notempty; + if !entries.is_empty() || state.fs_read_dir(path)?.count() != 0 { + return Err(Errno::Notempty); } path.clone() } - Kind::Root { .. } => return Errno::Access, - _ => return Errno::Notdir, + Kind::Root { .. } => return Err(Errno::Access), + _ => return Err(Errno::Notdir), } }; @@ -53,17 +75,20 @@ pub fn path_remove_directory( Kind::Dir { ref mut entries, .. } => { - let removed_inode = wasi_try!(entries.remove(&childs_name).ok_or(Errno::Inval)); + let removed_inode = entries.remove(&childs_name).ok_or(Errno::Inval)?; + // TODO: make this a debug assert in the future assert!(inode.ino() == removed_inode.ino()); } - Kind::Root { .. } => return Errno::Access, + Kind::Root { .. } => return Err(Errno::Access), _ => unreachable!( "Internal logic error in wasi::path_remove_directory, parent is not a directory" ), } } + let env = ctx.data(); + let (memory, mut state, inodes) = unsafe { env.get_memory_and_wasi_state_and_inodes(&ctx, 0) }; if let Err(err) = state.fs_remove_dir(host_path_to_remove) { // reinsert to prevent FS from being in bad state let mut guard = parent_inode.write(); @@ -73,8 +98,8 @@ pub fn path_remove_directory( { entries.insert(childs_name, inode); } - return err; + return Err(err); } - Errno::Success + Ok(()) } diff --git a/lib/wasix/src/syscalls/wasi/path_rename.rs b/lib/wasix/src/syscalls/wasi/path_rename.rs index 50e86a90bf5..e2e9b26e4b5 100644 --- a/lib/wasix/src/syscalls/wasi/path_rename.rs +++ b/lib/wasix/src/syscalls/wasi/path_rename.rs @@ -18,7 +18,7 @@ use crate::syscalls::*; /// The number of bytes to read from `new_path` #[instrument(level = "debug", skip_all, fields(%old_fd, %new_fd, old_path = field::Empty, new_path = field::Empty), ret)] pub fn path_rename( - ctx: FunctionEnvMut<'_, WasiEnv>, + mut ctx: FunctionEnvMut<'_, WasiEnv>, old_fd: WasiFd, old_path: WasmPtr, old_path_len: M::Offset, @@ -31,43 +31,69 @@ pub fn path_rename( let mut source_str = unsafe { get_input_str_ok!(&memory, old_path, old_path_len) }; Span::current().record("old_path", source_str.as_str()); source_str = ctx.data().state.fs.relative_path_to_absolute(source_str); - let source_path = std::path::Path::new(&source_str); let mut target_str = unsafe { get_input_str_ok!(&memory, new_path, new_path_len) }; Span::current().record("new_path", target_str.as_str()); target_str = ctx.data().state.fs.relative_path_to_absolute(target_str); - let target_path = std::path::Path::new(&target_str); + + let ret = path_rename_internal(&mut ctx, old_fd, &source_str, new_fd, &target_str)?; + let env = ctx.data(); + + if ret == Errno::Success { + #[cfg(feature = "snapshot")] + if env.enable_snapshot_capture { + wasi_try_ok!(SnapshotEffector::save_rename( + &mut ctx, old_fd, source_str, new_fd, target_str + ) + .map_err(|err| { + tracing::error!("failed to save unlink event to snapshot capturer - {}", err); + Errno::Fault + })) + } + } + Ok(ret) +} + +pub fn path_rename_internal( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + source_fd: WasiFd, + source_path: &str, + target_fd: WasiFd, + target_path: &str, +) -> Result { + let env = ctx.data(); + let (memory, mut state, inodes) = unsafe { env.get_memory_and_wasi_state_and_inodes(&ctx, 0) }; { - let source_fd = wasi_try_ok!(state.fs.get_fd(old_fd)); + let source_fd = wasi_try_ok!(state.fs.get_fd(source_fd)); if !source_fd.rights.contains(Rights::PATH_RENAME_SOURCE) { return Ok(Errno::Access); } - let target_fd = wasi_try_ok!(state.fs.get_fd(new_fd)); + let target_fd = wasi_try_ok!(state.fs.get_fd(target_fd)); if !target_fd.rights.contains(Rights::PATH_RENAME_TARGET) { return Ok(Errno::Access); } } // this is to be sure the source file is fetch from filesystem if needed - wasi_try_ok!(state.fs.get_inode_at_path( + wasi_try_ok!(state + .fs + .get_inode_at_path(inodes, source_fd, source_path, true)); + // Create the destination inode if the file exists. + let _ = state + .fs + .get_inode_at_path(inodes, target_fd, target_path, true); + let (source_parent_inode, source_entry_name) = wasi_try_ok!(state.fs.get_parent_inode_at_path( inodes, - old_fd, - source_path.to_str().as_ref().unwrap(), + source_fd, + Path::new(source_path), + true + )); + let (target_parent_inode, target_entry_name) = wasi_try_ok!(state.fs.get_parent_inode_at_path( + inodes, + target_fd, + Path::new(target_path), true )); - // Create the destination inode if the file exists. - let _ = - state - .fs - .get_inode_at_path(inodes, new_fd, target_path.to_str().as_ref().unwrap(), true); - let (source_parent_inode, source_entry_name) = - wasi_try_ok!(state - .fs - .get_parent_inode_at_path(inodes, old_fd, source_path, true)); - let (target_parent_inode, target_entry_name) = - wasi_try_ok!(state - .fs - .get_parent_inode_at_path(inodes, new_fd, target_path, true)); let mut need_create = true; let host_adjusted_target_path = { let guard = target_parent_inode.read(); diff --git a/lib/wasix/src/syscalls/wasi/path_unlink_file.rs b/lib/wasix/src/syscalls/wasi/path_unlink_file.rs index c9711c70728..2e9a44682ed 100644 --- a/lib/wasix/src/syscalls/wasi/path_unlink_file.rs +++ b/lib/wasix/src/syscalls/wasi/path_unlink_file.rs @@ -12,7 +12,7 @@ use crate::syscalls::*; /// The number of bytes in the `path` array #[instrument(level = "debug", skip_all, fields(%fd, path = field::Empty), ret)] pub fn path_unlink_file( - ctx: FunctionEnvMut<'_, WasiEnv>, + mut ctx: FunctionEnvMut<'_, WasiEnv>, fd: WasiFd, path: WasmPtr, path_len: M::Offset, @@ -32,11 +32,37 @@ pub fn path_unlink_file( path_str = ctx.data().state.fs.relative_path_to_absolute(path_str); } - let inode = wasi_try_ok!(state.fs.get_inode_at_path(inodes, fd, &path_str, false)); + let ret = path_unlink_file_internal(&mut ctx, fd, &path_str)?; + let env = ctx.data(); + + if ret == Errno::Success { + #[cfg(feature = "snapshot")] + if env.enable_snapshot_capture { + wasi_try_ok!( + SnapshotEffector::save_unlink_file(&mut ctx, fd, path_str).map_err(|err| { + tracing::error!("failed to save unlink event to snapshot capturer - {}", err); + Errno::Fault + }) + ) + } + } + + Ok(ret) +} + +pub(crate) fn path_unlink_file_internal( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + fd: WasiFd, + path: &str, +) -> Result { + let env = ctx.data(); + let (memory, mut state, inodes) = unsafe { env.get_memory_and_wasi_state_and_inodes(&ctx, 0) }; + + let inode = wasi_try_ok!(state.fs.get_inode_at_path(inodes, fd, path, false)); let (parent_inode, childs_name) = wasi_try_ok!(state.fs.get_parent_inode_at_path( inodes, fd, - std::path::Path::new(&path_str), + std::path::Path::new(path), false )); From 3efa30e2708ba36bc9d4099e8d79e38492821251 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Sun, 12 Nov 2023 04:14:43 +1100 Subject: [PATCH 030/129] Added intercept and replay calls for all the major file system operations --- lib/wasi-types/src/wasi/bindings.rs | 7 +- lib/wasi-types/src/wasi/wasix_manual.rs | 15 +- lib/wasix/src/fs/mod.rs | 16 + lib/wasix/src/snapshot/capturer.rs | 213 +----- lib/wasix/src/snapshot/compactor.rs | 35 +- lib/wasix/src/snapshot/effector.rs | 350 --------- .../snapshot/effector/memory_and_snapshot.rs | 91 +++ lib/wasix/src/snapshot/effector/mod.rs | 66 ++ lib/wasix/src/snapshot/effector/save_event.rs | 23 + .../src/snapshot/effector/syscalls/chdir.rs | 18 + .../snapshot/effector/syscalls/clock_time.rs | 28 + .../effector/syscalls/epoll_create.rs | 29 + .../snapshot/effector/syscalls/epoll_ctl.rs | 51 ++ .../snapshot/effector/syscalls/fd_advise.rs | 41 + .../snapshot/effector/syscalls/fd_allocate.rs | 34 + .../snapshot/effector/syscalls/fd_close.rs | 20 + .../effector/syscalls/fd_duplicate.rs | 44 ++ .../src/snapshot/effector/syscalls/fd_pipe.rs | 43 ++ .../snapshot/effector/syscalls/fd_renumber.rs | 34 + .../src/snapshot/effector/syscalls/fd_seek.rs | 31 + .../effector/syscalls/fd_set_flags.rs | 34 + .../effector/syscalls/fd_set_rights.rs | 38 + .../snapshot/effector/syscalls/fd_set_size.rs | 34 + .../effector/syscalls/fd_set_times.rs | 42 ++ .../snapshot/effector/syscalls/fd_write.rs | 74 ++ .../syscalls/path_create_directory.rs | 33 + .../snapshot/effector/syscalls/path_link.rs | 46 ++ .../snapshot/effector/syscalls/path_open.rs | 91 +++ .../syscalls/path_remove_directory.rs | 31 + .../snapshot/effector/syscalls/path_rename.rs | 42 ++ .../effector/syscalls/path_set_times.rs | 50 ++ .../effector/syscalls/path_symlink.rs | 38 + .../snapshot/effector/syscalls/path_unlink.rs | 34 + .../src/snapshot/effector/syscalls/tty_set.rs | 38 + .../src/snapshot/effector/thread_exit.rs | 20 + .../src/snapshot/effector/thread_state.rs | 22 + lib/wasix/src/snapshot/filter.rs | 138 +++- lib/wasix/src/snapshot/log_file.rs | 706 ++++++++++++++++-- lib/wasix/src/syscalls/mod.rs | 212 +++++- lib/wasix/src/syscalls/wasi/clock_time_set.rs | 22 +- lib/wasix/src/syscalls/wasi/fd_advise.rs | 27 +- lib/wasix/src/syscalls/wasi/fd_allocate.rs | 46 +- lib/wasix/src/syscalls/wasi/fd_close.rs | 11 + lib/wasix/src/syscalls/wasi/fd_dup.rs | 31 +- .../src/syscalls/wasi/fd_fdstat_set_flags.rs | 21 + .../src/syscalls/wasi/fd_fdstat_set_rights.rs | 36 +- .../src/syscalls/wasi/fd_filestat_set_size.rs | 43 +- .../syscalls/wasi/fd_filestat_set_times.rs | 41 +- lib/wasix/src/syscalls/wasi/fd_renumber.rs | 27 +- lib/wasix/src/syscalls/wasi/fd_seek.rs | 61 +- lib/wasix/src/syscalls/wasi/fd_write.rs | 16 +- .../syscalls/wasi/path_create_directory.rs | 74 +- .../syscalls/wasi/path_filestat_set_times.rs | 81 +- lib/wasix/src/syscalls/wasi/path_link.rs | 84 ++- lib/wasix/src/syscalls/wasi/path_open.rs | 138 ++-- .../syscalls/wasi/path_remove_directory.rs | 2 +- lib/wasix/src/syscalls/wasi/path_rename.rs | 12 +- lib/wasix/src/syscalls/wasi/path_symlink.rs | 70 +- .../src/syscalls/wasi/path_unlink_file.rs | 2 +- lib/wasix/src/syscalls/wasix/chdir.rs | 32 +- lib/wasix/src/syscalls/wasix/epoll_create.rs | 32 +- lib/wasix/src/syscalls/wasix/epoll_ctl.rs | 68 +- lib/wasix/src/syscalls/wasix/fd_pipe.rs | 40 +- lib/wasix/src/syscalls/wasix/tty_set.rs | 39 +- 64 files changed, 3069 insertions(+), 899 deletions(-) delete mode 100644 lib/wasix/src/snapshot/effector.rs create mode 100644 lib/wasix/src/snapshot/effector/memory_and_snapshot.rs create mode 100644 lib/wasix/src/snapshot/effector/mod.rs create mode 100644 lib/wasix/src/snapshot/effector/save_event.rs create mode 100644 lib/wasix/src/snapshot/effector/syscalls/chdir.rs create mode 100644 lib/wasix/src/snapshot/effector/syscalls/clock_time.rs create mode 100644 lib/wasix/src/snapshot/effector/syscalls/epoll_create.rs create mode 100644 lib/wasix/src/snapshot/effector/syscalls/epoll_ctl.rs create mode 100644 lib/wasix/src/snapshot/effector/syscalls/fd_advise.rs create mode 100644 lib/wasix/src/snapshot/effector/syscalls/fd_allocate.rs create mode 100644 lib/wasix/src/snapshot/effector/syscalls/fd_close.rs create mode 100644 lib/wasix/src/snapshot/effector/syscalls/fd_duplicate.rs create mode 100644 lib/wasix/src/snapshot/effector/syscalls/fd_pipe.rs create mode 100644 lib/wasix/src/snapshot/effector/syscalls/fd_renumber.rs create mode 100644 lib/wasix/src/snapshot/effector/syscalls/fd_seek.rs create mode 100644 lib/wasix/src/snapshot/effector/syscalls/fd_set_flags.rs create mode 100644 lib/wasix/src/snapshot/effector/syscalls/fd_set_rights.rs create mode 100644 lib/wasix/src/snapshot/effector/syscalls/fd_set_size.rs create mode 100644 lib/wasix/src/snapshot/effector/syscalls/fd_set_times.rs create mode 100644 lib/wasix/src/snapshot/effector/syscalls/fd_write.rs create mode 100644 lib/wasix/src/snapshot/effector/syscalls/path_create_directory.rs create mode 100644 lib/wasix/src/snapshot/effector/syscalls/path_link.rs create mode 100644 lib/wasix/src/snapshot/effector/syscalls/path_open.rs create mode 100644 lib/wasix/src/snapshot/effector/syscalls/path_remove_directory.rs create mode 100644 lib/wasix/src/snapshot/effector/syscalls/path_rename.rs create mode 100644 lib/wasix/src/snapshot/effector/syscalls/path_set_times.rs create mode 100644 lib/wasix/src/snapshot/effector/syscalls/path_symlink.rs create mode 100644 lib/wasix/src/snapshot/effector/syscalls/path_unlink.rs create mode 100644 lib/wasix/src/snapshot/effector/syscalls/tty_set.rs create mode 100644 lib/wasix/src/snapshot/effector/thread_exit.rs create mode 100644 lib/wasix/src/snapshot/effector/thread_state.rs diff --git a/lib/wasi-types/src/wasi/bindings.rs b/lib/wasi-types/src/wasi/bindings.rs index 68204e07d9d..41fa23c65eb 100644 --- a/lib/wasi-types/src/wasi/bindings.rs +++ b/lib/wasi-types/src/wasi/bindings.rs @@ -1,3 +1,4 @@ +use num_enum::{IntoPrimitive, TryFromPrimitive}; #[cfg(feature = "enable-serde")] use serde::{Deserialize, Serialize}; use std::mem::MaybeUninit; @@ -100,7 +101,7 @@ impl core::fmt::Debug for Clockid { #[doc = " API; some are used in higher-level library layers, and others are provided"] #[doc = " merely for alignment with POSIX."] #[repr(u16)] -#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, IntoPrimitive, TryFromPrimitive)] #[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] pub enum Errno { #[doc = " No error occurred. System call completed successfully."] @@ -639,7 +640,7 @@ impl core::fmt::Debug for Dirent { } #[doc = " File or memory access pattern advisory information."] #[repr(u8)] -#[derive(Clone, Copy, PartialEq, Eq)] +#[derive(Clone, Serialize, Deserialize, Copy, PartialEq, Eq)] pub enum Advice { #[doc = " The application has no advice to give on its behavior with respect to the specified data."] Normal, @@ -1186,7 +1187,7 @@ impl core::fmt::Debug for Whence { } } #[repr(C)] -#[derive(Copy, Clone)] +#[derive(Copy, Clone, Serialize, Deserialize)] pub struct Tty { pub cols: u32, pub rows: u32, diff --git a/lib/wasi-types/src/wasi/wasix_manual.rs b/lib/wasi-types/src/wasi/wasix_manual.rs index b160ade1e14..13c1d2b0547 100644 --- a/lib/wasi-types/src/wasi/wasix_manual.rs +++ b/lib/wasi-types/src/wasi/wasix_manual.rs @@ -376,7 +376,9 @@ wai_bindgen_rust::bitflags::bitflags! { #[doc = " Epoll operation."] #[repr(u32)] -#[derive(Clone, Copy, PartialEq, Eq, num_enum :: TryFromPrimitive, Hash)] +#[derive( + Clone, Copy, Serialize, Deserialize, PartialEq, Eq, num_enum :: TryFromPrimitive, Hash, +)] pub enum EpollCtl { #[doc = " Add an entry to the interest list of the epoll file descriptor, epfd."] Add, @@ -428,6 +430,15 @@ unsafe impl wasmer::FromToNativeWasmType for EpollCtl { } } +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +pub struct EpollEventCtl { + pub events: EpollType, + pub ptr: u64, + pub fd: Fd, + pub data1: u32, + pub data2: u64, +} + /// An event that can be triggered #[repr(C)] #[derive(Copy, Clone)] @@ -459,7 +470,7 @@ where #[repr(C)] #[derive(Copy, Clone)] pub struct EpollEvent { - /// Pointer to the dataa + /// Pointer to the data pub events: EpollType, /// File descriptor pub data: EpollData, diff --git a/lib/wasix/src/fs/mod.rs b/lib/wasix/src/fs/mod.rs index 671bcfd65b7..e1c97d71469 100644 --- a/lib/wasix/src/fs/mod.rs +++ b/lib/wasix/src/fs/mod.rs @@ -1671,6 +1671,22 @@ impl WasiFs { Ok(idx) } + pub fn make_max_fd(&self, fd: u32) { + loop { + let existing = self.next_fd.load(Ordering::SeqCst); + if existing >= fd { + return; + } + if self + .next_fd + .compare_exchange(existing, fd, Ordering::SeqCst, Ordering::Relaxed) + .is_ok() + { + break; + } + } + } + pub fn create_fd_ext( &self, rights: Rights, diff --git a/lib/wasix/src/snapshot/capturer.rs b/lib/wasix/src/snapshot/capturer.rs index 7eb5cfefa52..777cbaa6864 100644 --- a/lib/wasix/src/snapshot/capturer.rs +++ b/lib/wasix/src/snapshot/capturer.rs @@ -1,10 +1,10 @@ use serde::{Deserialize, Serialize}; -use std::fmt; use std::net::SocketAddr; use std::time::SystemTime; use std::{borrow::Cow, ops::Range}; use wasmer_wasix_types::wasi::{ - Advice, EpollCtl, ExitCode, Filesize, Fstflags, LookupFlags, Snapshot0Clockid, Timestamp, Tty, + Advice, EpollCtl, EpollEventCtl, ExitCode, Fdflags, FileDelta, Filesize, Fstflags, LookupFlags, + Oflags, Rights, Snapshot0Clockid, Timestamp, Tty, Whence, }; use futures::future::LocalBoxFuture; @@ -36,77 +36,18 @@ pub enum SocketSnapshot { }, } -#[derive(Debug, Clone, Serialize, Deserialize)] -pub enum FdOpenSnapshot<'a> { - Stdin { - non_blocking: bool, - }, - Stdout { - non_blocking: bool, - }, - Stderr { - non_blocking: bool, - }, - OpenFile { - path: Cow<'a, str>, - offset: u64, - read: bool, - write: bool, - non_blocking: bool, - }, - Socket { - state: SocketSnapshot, - non_blocking: bool, - }, -} - -impl<'a> FdOpenSnapshot<'a> { - pub fn into_owned(self) -> FdOpenSnapshot<'static> { - match self { - FdOpenSnapshot::Stdin { non_blocking } => FdOpenSnapshot::Stdin { non_blocking }, - FdOpenSnapshot::Stdout { non_blocking } => FdOpenSnapshot::Stdout { non_blocking }, - FdOpenSnapshot::Stderr { non_blocking } => FdOpenSnapshot::Stderr { non_blocking }, - FdOpenSnapshot::OpenFile { - path, - offset, - read, - write, - non_blocking, - } => FdOpenSnapshot::OpenFile { - path: Cow::Owned(path.into_owned()), - offset, - read, - write, - non_blocking, - }, - FdOpenSnapshot::Socket { - state, - non_blocking, - } => FdOpenSnapshot::Socket { - state, - non_blocking, - }, - } - } -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub enum FileEntryType { - Directory, - File, - Symlink, - CharDevice, - BlockDevice, - Socket, - Fifo, -} - /// Represents a log entry in a snapshot log stream that represents the total /// state of a WASM process at a point in time. +#[derive(Debug)] pub enum SnapshotLog<'a> { Init { wasm_hash: [u8; 32], }, + FileDescriptorSeek { + fd: Fd, + offset: FileDelta, + whence: Whence, + }, FileDescriptorWrite { fd: Fd, offset: u64, @@ -137,15 +78,22 @@ pub enum SnapshotLog<'a> { }, OpenFileDescriptor { fd: Fd, - state: FdOpenSnapshot<'a>, + dirfd: Fd, + dirflags: LookupFlags, + path: Cow<'a, str>, + o_flags: Oflags, + fs_rights_base: Rights, + fs_rights_inheriting: Rights, + fs_flags: Fdflags, + is_64bit: bool, }, RenumberFileDescriptor { old_fd: Fd, new_fd: Fd, }, DuplicateFileDescriptor { - old_fd: Fd, - new_fd: Fd, + original_fd: Fd, + copied_fd: Fd, }, CreateDirectory { fd: Fd, @@ -155,15 +103,32 @@ pub enum SnapshotLog<'a> { fd: Fd, path: Cow<'a, str>, }, + PathSetTimes { + fd: Fd, + flags: LookupFlags, + path: Cow<'a, str>, + st_atim: Timestamp, + st_mtim: Timestamp, + fst_flags: Fstflags, + }, FileDescriptorSetTimes { fd: Fd, st_atim: Timestamp, st_mtim: Timestamp, fst_flags: Fstflags, }, + FileDescriptorSetFlags { + fd: Fd, + flags: Fdflags, + }, + FileDescriptorSetRights { + fd: Fd, + fs_rights_base: Rights, + fs_rights_inheriting: Rights, + }, FileDescriptorSetSize { fd: Fd, - size: Filesize, + st_size: Filesize, }, FileDescriptorAdvise { fd: Fd, @@ -184,9 +149,8 @@ pub enum SnapshotLog<'a> { new_path: Cow<'a, str>, }, CreateSymbolicLink { - old_fd: Fd, old_path: Cow<'a, str>, - new_fd: Fd, + fd: Fd, new_path: Cow<'a, str>, }, UnlinkFile { @@ -209,9 +173,11 @@ pub enum SnapshotLog<'a> { epfd: Fd, op: EpollCtl, fd: Fd, + event: Option, }, TtySet { tty: Tty, + line_feeds: bool, }, CreatePipe { fd1: Fd, @@ -224,107 +190,6 @@ pub enum SnapshotLog<'a> { }, } -impl fmt::Debug for SnapshotLog<'_> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::Init { wasm_hash } => f - .debug_struct("Init") - .field("wasm_hash.len", &wasm_hash.len()) - .finish(), - Self::FileDescriptorWrite { - fd, - offset, - data, - is_64bit, - } => f - .debug_struct("TerminalData") - .field("fd", &fd) - .field("offset", &offset) - .field("data.len", &data.len()) - .field("is_64bit", &is_64bit) - .finish(), - Self::UpdateMemoryRegion { region, data } => f - .debug_struct("UpdateMemoryRegion") - .field("region", region) - .field("data.len", &data.len()) - .finish(), - Self::CloseThread { id, exit_code } => f - .debug_struct("CloseThread") - .field("id", id) - .field("exit_code", exit_code) - .finish(), - Self::SetThread { - id, - call_stack, - memory_stack, - store_data, - is_64bit, - } => f - .debug_struct("SetThread") - .field("id", id) - .field("call_stack.len", &call_stack.len()) - .field("memory_stack.len", &memory_stack.len()) - .field("store_data.len", &store_data.len()) - .field("is_64bit", is_64bit) - .finish(), - Self::CloseFileDescriptor { fd } => f - .debug_struct("CloseFileDescriptor") - .field("fd", fd) - .finish(), - Self::OpenFileDescriptor { fd, state } => f - .debug_struct("OpenFileDescriptor") - .field("fd", fd) - .field("state", state) - .finish(), - Self::RemoveDirectory { fd, path } => f - .debug_struct("RemoveDirectory") - .field("fd", fd) - .field("path", path) - .finish(), - Self::UnlinkFile { fd, path } => f - .debug_struct("UnlinkFile") - .field("fd", fd) - .field("path", path) - .finish(), - Self::PathRename { - old_fd, - old_path, - new_fd, - new_path, - } => f - .debug_struct("UnlinkFile") - .field("old_fd", old_fd) - .field("old_path", old_path) - .field("new_fd", new_fd) - .field("new_path", new_path) - .finish(), - Self::UpdateFileSystemEntry { - path, - ft, - accessed, - created, - modified, - len, - data, - } => f - .debug_struct("UpdateFileSystemEntry") - .field("path", path) - .field("ft", ft) - .field("accessed", accessed) - .field("created", created) - .field("modified", modified) - .field("len", len) - .field("data.len", &data.len()) - .finish(), - Self::Snapshot { when, trigger } => f - .debug_struct("Snapshot") - .field("when", when) - .field("trigger", trigger) - .finish(), - } - } -} - /// The snapshot capturer will take a series of objects that represents the state of /// a WASM process at a point in time and saves it so that it can be restored. /// It also allows for the restoration of that state at a later moment diff --git a/lib/wasix/src/snapshot/compactor.rs b/lib/wasix/src/snapshot/compactor.rs index a31ed9dfb3c..dc4b8a53b94 100644 --- a/lib/wasix/src/snapshot/compactor.rs +++ b/lib/wasix/src/snapshot/compactor.rs @@ -12,7 +12,7 @@ use super::*; struct State { memory_map: HashMap, [u8; 32]>, - open_file: HashMap>, + open_file: HashMap>, close_file: HashSet, } @@ -96,11 +96,31 @@ impl SnapshotCapturer for CompactingSnapshotCapturer { } SnapshotLog::OpenFileDescriptor { fd, - state: fd_state, + dirfd, + dirflags, + path, + o_flags, + fs_rights_base, + fs_rights_inheriting, + fs_flags, + is_64bit, } => { let mut state = self.state.lock().unwrap(); state.close_file.remove(&fd); - state.open_file.insert(fd, fd_state.into_owned()); + state.open_file.insert( + fd, + SnapshotLog::OpenFileDescriptor { + fd, + dirfd, + dirflags, + path: path.into_owned().into(), + o_flags, + fs_rights_base, + fs_rights_inheriting, + fs_flags, + is_64bit, + }, + ); } SnapshotLog::Snapshot { .. } => { let (to_close, to_open) = { @@ -115,13 +135,8 @@ impl SnapshotCapturer for CompactingSnapshotCapturer { .write(SnapshotLog::CloseFileDescriptor { fd }) .await?; } - for (fd, fd_state) in to_open { - self.inner - .write(SnapshotLog::OpenFileDescriptor { - fd, - state: fd_state, - }) - .await?; + for (_, entry) in to_open { + self.inner.write(entry).await?; } return self.inner.write(entry).await; } diff --git a/lib/wasix/src/snapshot/effector.rs b/lib/wasix/src/snapshot/effector.rs deleted file mode 100644 index 80088c74bb1..00000000000 --- a/lib/wasix/src/snapshot/effector.rs +++ /dev/null @@ -1,350 +0,0 @@ -use std::{borrow::Cow, collections::LinkedList, ops::Range, sync::MutexGuard, time::SystemTime}; - -use bytes::Bytes; -use wasmer::{FunctionEnvMut, RuntimeError, WasmPtr}; -use wasmer_types::MemorySize; -use wasmer_wasix_types::{ - types::__wasi_ciovec_t, - wasi::{Errno, ExitCode, Fd}, -}; - -use crate::{ - mem_error_to_wasi, - os::task::process::WasiProcessInner, - syscalls::{__asyncify_light, fd_write_internal, FdWriteSource}, - utils::map_snapshot_err, - WasiEnv, WasiError, WasiRuntimeError, WasiThreadId, -}; - -use super::*; - -#[derive(Debug, Clone)] -pub struct SnapshotEffector {} - -#[cfg(feature = "snapshot")] -impl SnapshotEffector { - pub fn save_write( - ctx: &mut FunctionEnvMut<'_, WasiEnv>, - fd: Fd, - offset: u64, - written: usize, - iovs: WasmPtr<__wasi_ciovec_t, M>, - iovs_len: M::Offset, - ) -> anyhow::Result<()> { - let env = ctx.data(); - let memory = unsafe { env.memory_view(&ctx) }; - let iovs_arr = iovs.slice(&memory, iovs_len)?; - - __asyncify_light(env, None, async { - let iovs_arr = iovs_arr.access().map_err(mem_error_to_wasi)?; - let mut remaining: M::Offset = TryFrom::::try_from(written).unwrap_or_default(); - for iovs in iovs_arr.iter() { - let sub = iovs.buf_len.min(remaining); - if sub == M::ZERO { - continue; - } - remaining -= sub; - - let buf = WasmPtr::::new(iovs.buf) - .slice(&memory, sub) - .map_err(mem_error_to_wasi)? - .access() - .map_err(mem_error_to_wasi)?; - ctx.data() - .runtime() - .snapshot_capturer() - .write(SnapshotLog::FileDescriptorWrite { - fd, - offset, - data: Cow::Borrowed(buf.as_ref()), - is_64bit: M::is_64bit(), - }) - .await - .map_err(map_snapshot_err)?; - } - Ok(()) - })? - .map_err(|err| WasiError::Exit(ExitCode::Errno(err)))?; - Ok(()) - } - - pub async fn apply_write( - ctx: &mut FunctionEnvMut<'_, WasiEnv>, - fd: Fd, - offset: u64, - data: Cow<'_, [u8]>, - ) -> anyhow::Result<()> { - let ret = fd_write_internal( - ctx, - fd, - FdWriteSource::<'_, M>::Buffer(data), - offset, - None, - true, - false, - )?; - if ret != Errno::Success { - tracing::debug!( - fd, - offset, - "restore error: failed to write to descriptor - {}", - ret - ); - } - Ok(()) - } - - pub fn save_thread_exit( - env: &WasiEnv, - id: WasiThreadId, - exit_code: Option, - ) -> anyhow::Result<()> { - __asyncify_light(env, None, async { - env.runtime() - .snapshot_capturer() - .write(SnapshotLog::CloseThread { id, exit_code }) - .await - .map_err(map_snapshot_err)?; - Ok(()) - })? - .map_err(|err| WasiError::Exit(ExitCode::Errno(err)))?; - Ok(()) - } - - pub fn save_thread_state( - ctx: &mut FunctionEnvMut<'_, WasiEnv>, - id: WasiThreadId, - memory_stack: Bytes, - rewind_stack: Bytes, - store_data: Bytes, - ) -> anyhow::Result<()> { - let env = ctx.data(); - - __asyncify_light(env, None, async { - ctx.data() - .runtime() - .snapshot_capturer() - .write(SnapshotLog::SetThread { - id, - call_stack: Cow::Owned(rewind_stack.into()), - memory_stack: Cow::Owned(memory_stack.into()), - store_data: Cow::Owned(store_data.into()), - is_64bit: M::is_64bit(), - }) - .await - .map_err(map_snapshot_err)?; - Ok(()) - })? - .map_err(|err| WasiError::Exit(ExitCode::Errno(err)))?; - - Ok(()) - } - - pub fn save_memory_and_snapshot( - ctx: &mut FunctionEnvMut<'_, WasiEnv>, - process: &mut MutexGuard<'_, WasiProcessInner>, - trigger: SnapshotTrigger, - ) -> anyhow::Result<()> { - let env = ctx.data(); - let memory = unsafe { env.memory_view(ctx) }; - - // Compute all the regions that we need to save which is basically - // everything in the memory except for the memory stacks. - // - // We do not want the regions to be greater than 64KB as this will - // otherwise create too much inefficiency. We choose 64KB as its - // aligned with the standard WASM page size. - let mut cur = 0u64; - let mut regions = LinkedList::>::new(); - while cur < memory.data_size() { - let mut again = false; - let mut end = memory.data_size().min(cur + 65536); - for (_, thread) in process.threads.iter() { - let layout = thread.memory_layout(); - if cur >= layout.stack_lower && cur < layout.stack_upper { - cur = layout.stack_upper; - again = true; - break; - } - if end > layout.stack_lower && end < layout.stack_upper { - end = end.min(layout.stack_lower); - } - } - if again { - continue; - } - regions.push_back(cur..end); - cur = end; - } - - // Now that we known all the regions that need to be saved we - // enter a processing loop that dumps all the data to the log - // file in an orderly manner. - __asyncify_light(env, None, async { - let memory = unsafe { env.memory_view(ctx) }; - let capturer = ctx.data().runtime().snapshot_capturer(); - - for region in regions { - // We grab this region of memory as a vector and hash - // it, which allows us to make some logging efficiency - // gains. - let data = memory - .copy_range_to_vec(region.clone()) - .map_err(mem_error_to_wasi)?; - - // Now we write it to the snap snapshot capturer - capturer - .write(SnapshotLog::UpdateMemoryRegion { - region, - data: data.into(), - }) - .await - .map_err(map_snapshot_err)?; - } - - // Finally we mark the end of the snapshot so that - // it can act as a restoration point - let when = SystemTime::now(); - capturer - .write(SnapshotLog::Snapshot { when, trigger }) - .await - .map_err(map_snapshot_err)?; - Ok(()) - })? - .map_err(|err| WasiError::Exit(ExitCode::Errno(err)))?; - Ok(()) - } - - pub fn apply_memory( - ctx: &mut FunctionEnvMut<'_, WasiEnv>, - region: Range, - data: &[u8], - ) -> anyhow::Result<()> { - let (env, mut store) = ctx.data_and_store_mut(); - let memory = unsafe { env.memory_view(&mut store) }; - memory - .write(region.start, data.as_ref()) - .map_err(|err| WasiRuntimeError::Runtime(RuntimeError::user(err.into())))?; - Ok(()) - } - - pub fn save_remove_directory( - ctx: &mut FunctionEnvMut<'_, WasiEnv>, - fd: Fd, - path: String, - ) -> anyhow::Result<()> { - let env = ctx.data(); - - __asyncify_light(env, None, async { - ctx.data() - .runtime() - .snapshot_capturer() - .write(SnapshotLog::RemoveDirectory { - fd, - path: Cow::Owned(path), - }) - .await - .map_err(map_snapshot_err)?; - Ok(()) - })? - .map_err(|err| WasiError::Exit(ExitCode::Errno(err)))?; - - Ok(()) - } - - pub fn apply_remove_directory( - ctx: &mut FunctionEnvMut<'_, WasiEnv>, - fd: Fd, - path: &str, - ) -> anyhow::Result<()> { - if let Err(err) = crate::syscalls::path_remove_directory_internal(ctx, fd, path) { - tracing::debug!("restore error: failed to remove directory - {}", err); - } - Ok(()) - } - - pub fn save_unlink_file( - ctx: &mut FunctionEnvMut<'_, WasiEnv>, - fd: Fd, - path: String, - ) -> anyhow::Result<()> { - let env = ctx.data(); - - __asyncify_light(env, None, async { - ctx.data() - .runtime() - .snapshot_capturer() - .write(SnapshotLog::UnlinkFile { - fd, - path: Cow::Owned(path), - }) - .await - .map_err(map_snapshot_err)?; - Ok(()) - })? - .map_err(|err| WasiError::Exit(ExitCode::Errno(err)))?; - - Ok(()) - } - - pub fn apply_unlink_file( - ctx: &mut FunctionEnvMut<'_, WasiEnv>, - fd: Fd, - path: &str, - ) -> anyhow::Result<()> { - let ret = crate::syscalls::path_unlink_file_internal(ctx, fd, path)?; - if ret != Errno::Success { - tracing::debug!(fd, path, "restore error: failed to remove file - {}", ret); - } - Ok(()) - } - - pub fn save_rename( - ctx: &mut FunctionEnvMut<'_, WasiEnv>, - old_fd: Fd, - old_path: String, - new_fd: Fd, - new_path: String, - ) -> anyhow::Result<()> { - let env = ctx.data(); - - __asyncify_light(env, None, async { - ctx.data() - .runtime() - .snapshot_capturer() - .write(SnapshotLog::PathRename { - old_fd, - old_path: Cow::Owned(old_path), - new_fd, - new_path: Cow::Owned(new_path), - }) - .await - .map_err(map_snapshot_err)?; - Ok(()) - })? - .map_err(|err| WasiError::Exit(ExitCode::Errno(err)))?; - - Ok(()) - } - - pub fn apply_rename( - ctx: &mut FunctionEnvMut<'_, WasiEnv>, - old_fd: Fd, - old_path: &str, - new_fd: Fd, - new_path: &str, - ) -> anyhow::Result<()> { - let ret = crate::syscalls::path_rename_internal(ctx, old_fd, old_path, new_fd, new_path)?; - if ret != Errno::Success { - tracing::debug!( - old_fd, - old_path, - new_fd, - new_path, - "restore error: failed to rename path - {}", - ret - ); - } - Ok(()) - } -} diff --git a/lib/wasix/src/snapshot/effector/memory_and_snapshot.rs b/lib/wasix/src/snapshot/effector/memory_and_snapshot.rs new file mode 100644 index 00000000000..59569e6e446 --- /dev/null +++ b/lib/wasix/src/snapshot/effector/memory_and_snapshot.rs @@ -0,0 +1,91 @@ +use super::*; + +impl SnapshotEffector { + pub fn save_memory_and_snapshot( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + process: &mut MutexGuard<'_, WasiProcessInner>, + trigger: SnapshotTrigger, + ) -> anyhow::Result<()> { + let env = ctx.data(); + let memory = unsafe { env.memory_view(ctx) }; + + // Compute all the regions that we need to save which is basically + // everything in the memory except for the memory stacks. + // + // We do not want the regions to be greater than 64KB as this will + // otherwise create too much inefficiency. We choose 64KB as its + // aligned with the standard WASM page size. + let mut cur = 0u64; + let mut regions = LinkedList::>::new(); + while cur < memory.data_size() { + let mut again = false; + let mut end = memory.data_size().min(cur + 65536); + for (_, thread) in process.threads.iter() { + let layout = thread.memory_layout(); + if cur >= layout.stack_lower && cur < layout.stack_upper { + cur = layout.stack_upper; + again = true; + break; + } + if end > layout.stack_lower && end < layout.stack_upper { + end = end.min(layout.stack_lower); + } + } + if again { + continue; + } + regions.push_back(cur..end); + cur = end; + } + + // Now that we known all the regions that need to be saved we + // enter a processing loop that dumps all the data to the log + // file in an orderly manner. + __asyncify_light(env, None, async { + let memory = unsafe { env.memory_view(ctx) }; + let capturer = ctx.data().runtime().snapshot_capturer(); + + for region in regions { + // We grab this region of memory as a vector and hash + // it, which allows us to make some logging efficiency + // gains. + let data = memory + .copy_range_to_vec(region.clone()) + .map_err(mem_error_to_wasi)?; + + // Now we write it to the snap snapshot capturer + capturer + .write(SnapshotLog::UpdateMemoryRegion { + region, + data: data.into(), + }) + .await + .map_err(map_snapshot_err)?; + } + + // Finally we mark the end of the snapshot so that + // it can act as a restoration point + let when = SystemTime::now(); + capturer + .write(SnapshotLog::Snapshot { when, trigger }) + .await + .map_err(map_snapshot_err)?; + Ok(()) + })? + .map_err(|err| WasiError::Exit(ExitCode::Errno(err)))?; + Ok(()) + } + + pub fn apply_memory( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + region: Range, + data: &[u8], + ) -> anyhow::Result<()> { + let (env, mut store) = ctx.data_and_store_mut(); + let memory = unsafe { env.memory_view(&mut store) }; + memory + .write(region.start, data.as_ref()) + .map_err(|err| WasiRuntimeError::Runtime(RuntimeError::user(err.into())))?; + Ok(()) + } +} diff --git a/lib/wasix/src/snapshot/effector/mod.rs b/lib/wasix/src/snapshot/effector/mod.rs new file mode 100644 index 00000000000..37a77482923 --- /dev/null +++ b/lib/wasix/src/snapshot/effector/mod.rs @@ -0,0 +1,66 @@ +pub(super) use std::{ + borrow::Cow, collections::LinkedList, ops::Range, sync::MutexGuard, time::SystemTime, +}; + +pub(super) use anyhow::bail; +pub(super) use bytes::Bytes; +pub(super) use wasmer::{FunctionEnvMut, RuntimeError, WasmPtr}; +pub(super) use wasmer_types::{Memory32, Memory64, MemorySize}; +pub(super) use wasmer_wasix_types::{ + types::__wasi_ciovec_t, + wasi::{ + Advice, EpollCtl, EpollEventCtl, Errno, ExitCode, Fd, Fdflags, Filesize, Fstflags, + LookupFlags, Oflags, Rights, Snapshot0Clockid, Timestamp, Whence, + }, +}; + +pub(super) use crate::{ + mem_error_to_wasi, + os::task::process::WasiProcessInner, + syscalls::{__asyncify_light, fd_write_internal, FdWriteSource}, + utils::map_snapshot_err, + WasiEnv, WasiError, WasiRuntimeError, WasiThreadId, +}; + +use super::*; + +#[cfg(feature = "snapshot")] +mod syscalls { + pub(super) use super::*; + mod chdir; + mod clock_time; + mod epoll_create; + mod epoll_ctl; + mod fd_advise; + mod fd_allocate; + mod fd_close; + mod fd_duplicate; + mod fd_pipe; + mod fd_renumber; + mod fd_seek; + mod fd_set_flags; + mod fd_set_rights; + mod fd_set_size; + mod fd_set_times; + mod fd_write; + mod path_create_directory; + mod path_link; + mod path_open; + mod path_remove_directory; + mod path_rename; + mod path_set_times; + mod path_symlink; + mod path_unlink; + mod tty_set; +} +#[cfg(feature = "snapshot")] +mod memory_and_snapshot; +#[cfg(feature = "snapshot")] +mod save_event; +#[cfg(feature = "snapshot")] +mod thread_exit; +#[cfg(feature = "snapshot")] +mod thread_state; + +#[derive(Debug, Clone)] +pub struct SnapshotEffector {} diff --git a/lib/wasix/src/snapshot/effector/save_event.rs b/lib/wasix/src/snapshot/effector/save_event.rs new file mode 100644 index 00000000000..7f213a0d11e --- /dev/null +++ b/lib/wasix/src/snapshot/effector/save_event.rs @@ -0,0 +1,23 @@ +use super::*; + +impl SnapshotEffector { + pub(super) fn save_event( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + event: SnapshotLog, + ) -> anyhow::Result<()> { + let env = ctx.data(); + + __asyncify_light(env, None, async { + ctx.data() + .runtime() + .snapshot_capturer() + .write(event) + .await + .map_err(map_snapshot_err)?; + Ok(()) + })? + .map_err(|err| WasiError::Exit(ExitCode::Errno(err)))?; + + Ok(()) + } +} diff --git a/lib/wasix/src/snapshot/effector/syscalls/chdir.rs b/lib/wasix/src/snapshot/effector/syscalls/chdir.rs new file mode 100644 index 00000000000..b62346592d0 --- /dev/null +++ b/lib/wasix/src/snapshot/effector/syscalls/chdir.rs @@ -0,0 +1,18 @@ +use super::*; + +impl SnapshotEffector { + pub fn save_chdir(ctx: &mut FunctionEnvMut<'_, WasiEnv>, path: String) -> anyhow::Result<()> { + Self::save_event(ctx, SnapshotLog::ChangeDirectory { path: path.into() }) + } + + pub fn apply_chdir(ctx: &mut FunctionEnvMut<'_, WasiEnv>, path: &str) -> anyhow::Result<()> { + crate::syscalls::chdir_internal(ctx, path).map_err(|err| { + anyhow::format_err!( + "snapshot restore error: failed to change directory (path={}) - {}", + path, + err + ) + })?; + Ok(()) + } +} diff --git a/lib/wasix/src/snapshot/effector/syscalls/clock_time.rs b/lib/wasix/src/snapshot/effector/syscalls/clock_time.rs new file mode 100644 index 00000000000..c5bbf67abe8 --- /dev/null +++ b/lib/wasix/src/snapshot/effector/syscalls/clock_time.rs @@ -0,0 +1,28 @@ +use super::*; + +impl SnapshotEffector { + pub fn save_clock_time_set( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + clock_id: Snapshot0Clockid, + time: Timestamp, + ) -> anyhow::Result<()> { + Self::save_event(ctx, SnapshotLog::SetClockTime { clock_id, time }) + } + + pub fn apply_clock_time_set( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + clock_id: Snapshot0Clockid, + time: Timestamp, + ) -> anyhow::Result<()> { + let ret = crate::syscalls::clock_time_set_internal(ctx, clock_id, time); + if ret != Errno::Success { + bail!( + "snapshot restore error: failed to set clock time (clock_id={:?}, time={}) - {}", + clock_id, + time, + ret + ); + } + Ok(()) + } +} diff --git a/lib/wasix/src/snapshot/effector/syscalls/epoll_create.rs b/lib/wasix/src/snapshot/effector/syscalls/epoll_create.rs new file mode 100644 index 00000000000..d98d9977bfc --- /dev/null +++ b/lib/wasix/src/snapshot/effector/syscalls/epoll_create.rs @@ -0,0 +1,29 @@ +use super::*; + +impl SnapshotEffector { + pub fn save_epoll_create(ctx: &mut FunctionEnvMut<'_, WasiEnv>, fd: Fd) -> anyhow::Result<()> { + Self::save_event(ctx, SnapshotLog::EpollCreate { fd }) + } + + pub fn apply_epoll_create(ctx: &mut FunctionEnvMut<'_, WasiEnv>, fd: Fd) -> anyhow::Result<()> { + let ret_fd = crate::syscalls::epoll_create_internal(ctx) + .map_err(|err| { + anyhow::format_err!("snapshot restore error: failed to create epoll - {}", err) + })? + .map_err(|err| { + anyhow::format_err!("snapshot restore error: failed to create epoll - {}", err) + })?; + + let ret = crate::syscalls::fd_renumber_internal(ctx, ret_fd, fd); + if ret != Errno::Success { + bail!( + "snapshot restore error: failed renumber file descriptor after epoll create (from={}, to={}) - {}", + ret_fd, + fd, + ret + ); + } + + Ok(()) + } +} diff --git a/lib/wasix/src/snapshot/effector/syscalls/epoll_ctl.rs b/lib/wasix/src/snapshot/effector/syscalls/epoll_ctl.rs new file mode 100644 index 00000000000..4d9c7817c29 --- /dev/null +++ b/lib/wasix/src/snapshot/effector/syscalls/epoll_ctl.rs @@ -0,0 +1,51 @@ +use super::*; + +impl SnapshotEffector { + pub fn save_epoll_ctl( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + epfd: Fd, + op: EpollCtl, + fd: Fd, + event: Option, + ) -> anyhow::Result<()> { + Self::save_event( + ctx, + SnapshotLog::EpollCtl { + epfd, + op: op.into(), + fd, + event, + }, + ) + } + + pub fn apply_epoll_ctl( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + pfd: Fd, + op: EpollCtl, + fd: Fd, + event: Option, + ) -> anyhow::Result<()> { + crate::syscalls::epoll_ctl_internal(ctx, pfd, op, fd, event.as_ref()) + .map_err(|err| { + anyhow::format_err!( + "snapshot restore error: failed to epoll ctl (pfd={}, op={:?}, fd={}) - {}", + pfd, + op, + fd, + err + ) + })? + .map_err(|err| { + anyhow::format_err!( + "snapshot restore error: failed to epoll ctl (pfd={}, op={:?}, fd={}) - {}", + pfd, + op, + fd, + err + ) + })?; + + Ok(()) + } +} diff --git a/lib/wasix/src/snapshot/effector/syscalls/fd_advise.rs b/lib/wasix/src/snapshot/effector/syscalls/fd_advise.rs new file mode 100644 index 00000000000..005ca3918ab --- /dev/null +++ b/lib/wasix/src/snapshot/effector/syscalls/fd_advise.rs @@ -0,0 +1,41 @@ +use super::*; + +impl SnapshotEffector { + pub fn save_fd_advise( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + fd: Fd, + offset: Filesize, + len: Filesize, + advice: Advice, + ) -> anyhow::Result<()> { + Self::save_event( + ctx, + SnapshotLog::FileDescriptorAdvise { + fd, + offset, + len, + advice: advice.into(), + }, + ) + } + + pub fn apply_fd_advise( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + fd: Fd, + offset: Filesize, + len: Filesize, + advice: Advice, + ) -> anyhow::Result<()> { + crate::syscalls::fd_advise_internal(ctx, fd, offset, len, advice).map_err(|err| { + anyhow::format_err!( + "snapshot restore error: failed to advise file descriptor (fd={}, offset={}, len={}, advice={:?}) - {}", + fd, + offset, + len, + advice, + err + ) + })?; + Ok(()) + } +} diff --git a/lib/wasix/src/snapshot/effector/syscalls/fd_allocate.rs b/lib/wasix/src/snapshot/effector/syscalls/fd_allocate.rs new file mode 100644 index 00000000000..430a24098c1 --- /dev/null +++ b/lib/wasix/src/snapshot/effector/syscalls/fd_allocate.rs @@ -0,0 +1,34 @@ +use super::*; + +impl SnapshotEffector { + pub fn save_fd_allocate( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + fd: Fd, + offset: Filesize, + len: Filesize, + ) -> anyhow::Result<()> { + Self::save_event( + ctx, + SnapshotLog::FileDescriptorAllocate { fd, offset, len }, + ) + } + + pub fn apply_fd_allocate( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + fd: Fd, + offset: Filesize, + len: Filesize, + ) -> anyhow::Result<()> { + crate::syscalls::fd_allocate_internal(ctx, fd, offset, len) + .map_err(|err| { + anyhow::format_err!( + "snapshot restore error: failed to allocate on file descriptor (fd={}, offset={}, len={}) - {}", + fd, + offset, + len, + err + ) + })?; + Ok(()) + } +} diff --git a/lib/wasix/src/snapshot/effector/syscalls/fd_close.rs b/lib/wasix/src/snapshot/effector/syscalls/fd_close.rs new file mode 100644 index 00000000000..1b07a8b0a33 --- /dev/null +++ b/lib/wasix/src/snapshot/effector/syscalls/fd_close.rs @@ -0,0 +1,20 @@ +use super::*; + +impl SnapshotEffector { + pub fn save_fd_close(ctx: &mut FunctionEnvMut<'_, WasiEnv>, fd: Fd) -> anyhow::Result<()> { + Self::save_event(ctx, SnapshotLog::CloseFileDescriptor { fd }) + } + + pub fn apply_fd_close(ctx: &mut FunctionEnvMut<'_, WasiEnv>, fd: Fd) -> anyhow::Result<()> { + let env = ctx.data(); + let (_, state) = unsafe { env.get_memory_and_wasi_state(&ctx, 0) }; + if let Err(err) = state.fs.close_fd(fd) { + bail!( + "snapshot restore error: failed to close descriptor (fd={}) - {}", + fd, + err + ); + } + Ok(()) + } +} diff --git a/lib/wasix/src/snapshot/effector/syscalls/fd_duplicate.rs b/lib/wasix/src/snapshot/effector/syscalls/fd_duplicate.rs new file mode 100644 index 00000000000..0c225426184 --- /dev/null +++ b/lib/wasix/src/snapshot/effector/syscalls/fd_duplicate.rs @@ -0,0 +1,44 @@ +use super::*; + +impl SnapshotEffector { + pub fn save_fd_duplicate( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + original_fd: Fd, + copied_fd: Fd, + ) -> anyhow::Result<()> { + Self::save_event( + ctx, + SnapshotLog::DuplicateFileDescriptor { + original_fd, + copied_fd, + }, + ) + } + + pub fn apply_fd_duplicate( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + original_fd: Fd, + copied_fd: Fd, + ) -> anyhow::Result<()> { + let ret_fd = crate::syscalls::fd_dup_internal(ctx, original_fd) + .map_err(|err| { + anyhow::format_err!( + "snapshot restore error: failed to duplicate file descriptor (original={}, copied={}) - {}", + original_fd, + copied_fd, + err + ) + })?; + + let ret = crate::syscalls::fd_renumber_internal(ctx, ret_fd, copied_fd); + if ret != Errno::Success { + bail!( + "snapshot restore error: failed renumber file descriptor after duplicate (from={}, to={}) - {}", + ret_fd, + copied_fd, + ret + ); + } + Ok(()) + } +} \ No newline at end of file diff --git a/lib/wasix/src/snapshot/effector/syscalls/fd_pipe.rs b/lib/wasix/src/snapshot/effector/syscalls/fd_pipe.rs new file mode 100644 index 00000000000..100338cf835 --- /dev/null +++ b/lib/wasix/src/snapshot/effector/syscalls/fd_pipe.rs @@ -0,0 +1,43 @@ +use super::*; + +impl SnapshotEffector { + pub fn save_fd_pipe( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + fd1: Fd, + fd2: Fd, + ) -> anyhow::Result<()> { + Self::save_event(ctx, SnapshotLog::CreatePipe { fd1, fd2 }) + } + + pub fn apply_fd_pipe( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + fd1: Fd, + fd2: Fd, + ) -> anyhow::Result<()> { + let (ret_fd1, ret_fd2) = crate::syscalls::fd_pipe_internal(ctx).map_err(|err| { + anyhow::format_err!("snapshot restore error: failed to create pipe - {}", err) + })?; + + let ret = crate::syscalls::fd_renumber_internal(ctx, ret_fd1, fd1); + if ret != Errno::Success { + bail!( + "snapshot restore error: failed renumber file descriptor after create pipe (from={}, to={}) - {}", + ret_fd1, + fd1, + ret + ); + } + + let ret = crate::syscalls::fd_renumber_internal(ctx, ret_fd2, fd2); + if ret != Errno::Success { + bail!( + "snapshot restore error: failed renumber file descriptor after create pipe (from={}, to={}) - {}", + ret_fd2, + fd2, + ret + ); + } + + Ok(()) + } +} diff --git a/lib/wasix/src/snapshot/effector/syscalls/fd_renumber.rs b/lib/wasix/src/snapshot/effector/syscalls/fd_renumber.rs new file mode 100644 index 00000000000..1423f7e3a96 --- /dev/null +++ b/lib/wasix/src/snapshot/effector/syscalls/fd_renumber.rs @@ -0,0 +1,34 @@ +use super::*; + +impl SnapshotEffector { + pub fn save_fd_renumber( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + from: Fd, + to: Fd, + ) -> anyhow::Result<()> { + Self::save_event( + ctx, + SnapshotLog::RenumberFileDescriptor { + old_fd: from, + new_fd: to, + }, + ) + } + + pub fn apply_fd_renumber( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + from: Fd, + to: Fd, + ) -> anyhow::Result<()> { + let ret = crate::syscalls::fd_renumber_internal(ctx, from, to); + if ret != Errno::Success { + bail!( + "snapshot restore error: failed to renumber descriptor (from={}, to={}) - {}", + from, + to, + ret + ); + } + Ok(()) + } +} diff --git a/lib/wasix/src/snapshot/effector/syscalls/fd_seek.rs b/lib/wasix/src/snapshot/effector/syscalls/fd_seek.rs new file mode 100644 index 00000000000..10de379ff39 --- /dev/null +++ b/lib/wasix/src/snapshot/effector/syscalls/fd_seek.rs @@ -0,0 +1,31 @@ +use super::*; + +impl SnapshotEffector { + pub fn save_fd_seek( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + fd: Fd, + offset: i64, + whence: Whence, + ) -> anyhow::Result<()> { + Self::save_event(ctx, SnapshotLog::FileDescriptorSeek { fd, offset, whence }) + } + + pub async fn apply_fd_seek( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + fd: Fd, + offset: i64, + whence: Whence, + ) -> anyhow::Result<()> { + crate::syscalls::fd_seek_internal(ctx, fd, offset, whence)? + .map_err(|err| { + anyhow::format_err!( + "snapshot restore error: failed to seek descriptor (fd={}, offset={}, whence={:?}) - {}", + fd, + offset, + whence, + err + ) + })?; + Ok(()) + } +} diff --git a/lib/wasix/src/snapshot/effector/syscalls/fd_set_flags.rs b/lib/wasix/src/snapshot/effector/syscalls/fd_set_flags.rs new file mode 100644 index 00000000000..bc4939ebc75 --- /dev/null +++ b/lib/wasix/src/snapshot/effector/syscalls/fd_set_flags.rs @@ -0,0 +1,34 @@ +use super::*; + +impl SnapshotEffector { + pub fn save_fd_set_flags( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + fd: Fd, + flags: Fdflags + ) -> anyhow::Result<()> { + Self::save_event( + ctx, + SnapshotLog::FileDescriptorSetFlags { + fd, + flags, + }, + ) + } + + pub fn apply_fd_set_flags( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + fd: Fd, + flags: Fdflags + ) -> anyhow::Result<()> { + crate::syscalls::fd_fdstat_set_flags_internal(ctx, fd, flags) + .map_err(|err| { + anyhow::format_err!( + "snapshot restore error: failed to duplicate file descriptor (fd={}, flags={:?}) - {}", + fd, + flags, + err + ) + })?; + Ok(()) + } +} diff --git a/lib/wasix/src/snapshot/effector/syscalls/fd_set_rights.rs b/lib/wasix/src/snapshot/effector/syscalls/fd_set_rights.rs new file mode 100644 index 00000000000..7c8a59ccd11 --- /dev/null +++ b/lib/wasix/src/snapshot/effector/syscalls/fd_set_rights.rs @@ -0,0 +1,38 @@ +use super::*; + +impl SnapshotEffector { + pub fn save_fd_set_rights( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + fd: Fd, + fs_rights_base: Rights, + fs_rights_inheriting: Rights, + ) -> anyhow::Result<()> { + Self::save_event( + ctx, + SnapshotLog::FileDescriptorSetRights { + fd, + fs_rights_base, + fs_rights_inheriting, + }, + ) + } + + pub fn apply_fd_set_rights( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + fd: Fd, + fs_rights_base: Rights, + fs_rights_inheriting: Rights, + ) -> anyhow::Result<()> { + crate::syscalls::fd_fdstat_set_rights_internal(ctx, fd, fs_rights_base, fs_rights_inheriting) + .map_err(|err| { + anyhow::format_err!( + "snapshot restore error: failed to duplicate file descriptor (fd={}, fs_rights_base={:?}, fs_rights_inheriting={:?}) - {}", + fd, + fs_rights_base, + fs_rights_inheriting, + err + ) + })?; + Ok(()) + } +} diff --git a/lib/wasix/src/snapshot/effector/syscalls/fd_set_size.rs b/lib/wasix/src/snapshot/effector/syscalls/fd_set_size.rs new file mode 100644 index 00000000000..cfbc3dd25fd --- /dev/null +++ b/lib/wasix/src/snapshot/effector/syscalls/fd_set_size.rs @@ -0,0 +1,34 @@ +use super::*; + +impl SnapshotEffector { + pub fn save_fd_set_size( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + fd: Fd, + st_size: Filesize + ) -> anyhow::Result<()> { + Self::save_event( + ctx, + SnapshotLog::FileDescriptorSetSize { + fd, + st_size, + }, + ) + } + + pub fn apply_fd_set_size( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + fd: Fd, + st_size: Filesize + ) -> anyhow::Result<()> { + crate::syscalls::fd_filestat_set_size_internal(ctx, fd, st_size) + .map_err(|err| { + anyhow::format_err!( + "snapshot restore error: failed to duplicate file descriptor (fd={}, st_size={}) - {}", + fd, + st_size, + err + ) + })?; + Ok(()) + } +} diff --git a/lib/wasix/src/snapshot/effector/syscalls/fd_set_times.rs b/lib/wasix/src/snapshot/effector/syscalls/fd_set_times.rs new file mode 100644 index 00000000000..b419bb6c4ca --- /dev/null +++ b/lib/wasix/src/snapshot/effector/syscalls/fd_set_times.rs @@ -0,0 +1,42 @@ +use super::*; + +impl SnapshotEffector { + pub fn save_fd_set_times( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + fd: Fd, + st_atim: Timestamp, + st_mtim: Timestamp, + fst_flags: Fstflags, + ) -> anyhow::Result<()> { + Self::save_event( + ctx, + SnapshotLog::FileDescriptorSetTimes { + fd, + st_atim, + st_mtim, + fst_flags, + }, + ) + } + + pub fn apply_fd_set_times( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + fd: Fd, + st_atim: Timestamp, + st_mtim: Timestamp, + fst_flags: Fstflags, + ) -> anyhow::Result<()> { + crate::syscalls::fd_filestat_set_times_internal(ctx, fd, st_atim, st_mtim, fst_flags) + .map_err(|err| { + anyhow::format_err!( + "snapshot restore error: failed to duplicate file descriptor (fd={}, st_atim={}, st_mtim={}, fst_flags={:?}) - {}", + fd, + st_atim, + st_mtim, + fst_flags, + err + ) + })?; + Ok(()) + } +} diff --git a/lib/wasix/src/snapshot/effector/syscalls/fd_write.rs b/lib/wasix/src/snapshot/effector/syscalls/fd_write.rs new file mode 100644 index 00000000000..c9b844207f1 --- /dev/null +++ b/lib/wasix/src/snapshot/effector/syscalls/fd_write.rs @@ -0,0 +1,74 @@ +use super::*; + +impl SnapshotEffector { + pub fn save_fd_write( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + fd: Fd, + offset: u64, + written: usize, + iovs: WasmPtr<__wasi_ciovec_t, M>, + iovs_len: M::Offset, + ) -> anyhow::Result<()> { + let env = ctx.data(); + let memory = unsafe { env.memory_view(&ctx) }; + let iovs_arr = iovs.slice(&memory, iovs_len)?; + + __asyncify_light(env, None, async { + let iovs_arr = iovs_arr.access().map_err(mem_error_to_wasi)?; + let mut remaining: M::Offset = TryFrom::::try_from(written).unwrap_or_default(); + for iovs in iovs_arr.iter() { + let sub = iovs.buf_len.min(remaining); + if sub == M::ZERO { + continue; + } + remaining -= sub; + + let buf = WasmPtr::::new(iovs.buf) + .slice(&memory, sub) + .map_err(mem_error_to_wasi)? + .access() + .map_err(mem_error_to_wasi)?; + ctx.data() + .runtime() + .snapshot_capturer() + .write(SnapshotLog::FileDescriptorWrite { + fd, + offset, + data: Cow::Borrowed(buf.as_ref()), + is_64bit: M::is_64bit(), + }) + .await + .map_err(map_snapshot_err)?; + } + Ok(()) + })? + .map_err(|err| WasiError::Exit(ExitCode::Errno(err)))?; + Ok(()) + } + + pub async fn apply_fd_write( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + fd: Fd, + offset: u64, + data: Cow<'_, [u8]>, + ) -> anyhow::Result<()> { + let ret = fd_write_internal( + ctx, + fd, + FdWriteSource::<'_, M>::Buffer(data), + offset, + None, + true, + false, + )?; + if ret != Errno::Success { + bail!( + "snapshot restore error: failed to write to descriptor (fd={}, offset={}) - {}", + fd, + offset, + ret + ); + } + Ok(()) + } +} diff --git a/lib/wasix/src/snapshot/effector/syscalls/path_create_directory.rs b/lib/wasix/src/snapshot/effector/syscalls/path_create_directory.rs new file mode 100644 index 00000000000..11ce32a96af --- /dev/null +++ b/lib/wasix/src/snapshot/effector/syscalls/path_create_directory.rs @@ -0,0 +1,33 @@ +use super::*; + +impl SnapshotEffector { + pub fn save_path_create_directory( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + fd: Fd, + path: String, + ) -> anyhow::Result<()> { + Self::save_event( + ctx, + SnapshotLog::CreateDirectory { + fd, + path: path.into(), + }, + ) + } + + pub fn apply_path_create_directory( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + fd: Fd, + path: &str, + ) -> anyhow::Result<()> { + crate::syscalls::path_create_directory_internal(ctx, fd, path).map_err(|err| { + anyhow::format_err!( + "snapshot restore error: failed to create directory path (fd={}, path={}) - {}", + fd, + path, + err + ) + })?; + Ok(()) + } +} diff --git a/lib/wasix/src/snapshot/effector/syscalls/path_link.rs b/lib/wasix/src/snapshot/effector/syscalls/path_link.rs new file mode 100644 index 00000000000..b0b9251ca1a --- /dev/null +++ b/lib/wasix/src/snapshot/effector/syscalls/path_link.rs @@ -0,0 +1,46 @@ +use super::*; + +impl SnapshotEffector { + pub fn save_path_link( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + old_fd: Fd, + old_flags: LookupFlags, + old_path: String, + new_fd: Fd, + new_path: String, + ) -> anyhow::Result<()> { + Self::save_event( + ctx, + SnapshotLog::CreateHardLink { + old_fd, + old_flags, + old_path: old_path.into(), + new_fd, + new_path: new_path.into() + }, + ) + } + + pub fn apply_path_link( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + old_fd: Fd, + old_flags: LookupFlags, + old_path: &str, + new_fd: Fd, + new_path: &str, + ) -> anyhow::Result<()> { + crate::syscalls::path_link_internal(ctx, old_fd, old_flags, old_path, new_fd, new_path) + .map_err(|err| { + anyhow::format_err!( + "snapshot restore error: failed to create hard link (old_fd={}, old_flags={}, old_path={}, new_fd={}, new_path={}) - {}", + old_fd, + old_flags, + old_path, + new_fd, + new_path, + err + ) + })?; + Ok(()) + } +} diff --git a/lib/wasix/src/snapshot/effector/syscalls/path_open.rs b/lib/wasix/src/snapshot/effector/syscalls/path_open.rs new file mode 100644 index 00000000000..b82223b88e5 --- /dev/null +++ b/lib/wasix/src/snapshot/effector/syscalls/path_open.rs @@ -0,0 +1,91 @@ +use super::*; + +impl SnapshotEffector { + pub fn save_path_open( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + fd: Fd, + dirfd: Fd, + dirflags: LookupFlags, + path: String, + o_flags: Oflags, + fs_rights_base: Rights, + fs_rights_inheriting: Rights, + fs_flags: Fdflags, + is_64bit: bool, + ) -> anyhow::Result<()> { + Self::save_event( + ctx, + SnapshotLog::OpenFileDescriptor { + fd, + dirfd, + dirflags, + path: path.into(), + o_flags, + fs_rights_base, + fs_rights_inheriting, + fs_flags, + is_64bit, + }, + ) + } + + pub fn apply_path_open( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + fd: Fd, + dirfd: Fd, + dirflags: LookupFlags, + path: &str, + o_flags: Oflags, + fs_rights_base: Rights, + fs_rights_inheriting: Rights, + fs_flags: Fdflags, + is_64bit: bool, + ) -> anyhow::Result<()> { + let res = if is_64bit { + crate::syscalls::path_open_internal::( + ctx, + dirfd, + dirflags, + path, + o_flags, + fs_rights_base, + fs_rights_inheriting, + fs_flags, + ) + } else { + crate::syscalls::path_open_internal::( + ctx, + dirfd, + dirflags, + path, + o_flags, + fs_rights_base, + fs_rights_inheriting, + fs_flags, + ) + }; + let ret_fd = match res? { + Ok(fd) => fd, + Err(err) => { + bail!( + "snapshot restore error: failed to open descriptor (fd={}, path={}) - {}", + fd, + path, + err + ); + } + }; + + let ret = crate::syscalls::fd_renumber_internal(ctx, ret_fd, fd); + if ret != Errno::Success { + bail!( + "snapshot restore error: failed renumber file descriptor after open (from={}, to={}) - {}", + ret_fd, + fd, + ret + ); + } + + Ok(()) + } +} diff --git a/lib/wasix/src/snapshot/effector/syscalls/path_remove_directory.rs b/lib/wasix/src/snapshot/effector/syscalls/path_remove_directory.rs new file mode 100644 index 00000000000..18174fb3885 --- /dev/null +++ b/lib/wasix/src/snapshot/effector/syscalls/path_remove_directory.rs @@ -0,0 +1,31 @@ +use super::*; + +impl SnapshotEffector { + pub fn save_path_remove_directory( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + fd: Fd, + path: String, + ) -> anyhow::Result<()> { + Self::save_event( + ctx, + SnapshotLog::RemoveDirectory { + fd, + path: Cow::Owned(path), + }, + ) + } + + pub fn apply_path_remove_directory( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + fd: Fd, + path: &str, + ) -> anyhow::Result<()> { + if let Err(err) = crate::syscalls::path_remove_directory_internal(ctx, fd, path) { + bail!( + "snapshot restore error: failed to remove directory - {}", + err + ); + } + Ok(()) + } +} diff --git a/lib/wasix/src/snapshot/effector/syscalls/path_rename.rs b/lib/wasix/src/snapshot/effector/syscalls/path_rename.rs new file mode 100644 index 00000000000..13f8f868809 --- /dev/null +++ b/lib/wasix/src/snapshot/effector/syscalls/path_rename.rs @@ -0,0 +1,42 @@ +use super::*; + +impl SnapshotEffector { + pub fn save_path_rename( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + old_fd: Fd, + old_path: String, + new_fd: Fd, + new_path: String, + ) -> anyhow::Result<()> { + Self::save_event( + ctx, + SnapshotLog::PathRename { + old_fd, + old_path: Cow::Owned(old_path), + new_fd, + new_path: Cow::Owned(new_path), + }, + ) + } + + pub fn apply_path_rename( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + old_fd: Fd, + old_path: &str, + new_fd: Fd, + new_path: &str, + ) -> anyhow::Result<()> { + let ret = crate::syscalls::path_rename_internal(ctx, old_fd, old_path, new_fd, new_path)?; + if ret != Errno::Success { + bail!( + "snapshot restore error: failed to rename path (old_fd={}, old_path={}, new_fd={}, new_path={}) - {}", + old_fd, + old_path, + new_fd, + new_path, + ret + ); + } + Ok(()) + } +} diff --git a/lib/wasix/src/snapshot/effector/syscalls/path_set_times.rs b/lib/wasix/src/snapshot/effector/syscalls/path_set_times.rs new file mode 100644 index 00000000000..3ff1886df56 --- /dev/null +++ b/lib/wasix/src/snapshot/effector/syscalls/path_set_times.rs @@ -0,0 +1,50 @@ +use super::*; + +impl SnapshotEffector { + pub fn save_path_set_times( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + fd: Fd, + flags: LookupFlags, + path: String, + st_atim: Timestamp, + st_mtim: Timestamp, + fst_flags: Fstflags, + ) -> anyhow::Result<()> { + Self::save_event( + ctx, + SnapshotLog::PathSetTimes { + fd, + flags, + path: path.into(), + st_atim, + st_mtim, + fst_flags, + }, + ) + } + + pub fn apply_path_set_times( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + fd: Fd, + flags: LookupFlags, + path: &str, + st_atim: Timestamp, + st_mtim: Timestamp, + fst_flags: Fstflags, + ) -> anyhow::Result<()> { + crate::syscalls::path_filestat_set_times_internal(ctx, fd, flags, path, st_atim, st_mtim, fst_flags) + .map_err(|err| { + anyhow::format_err!( + "snapshot restore error: failed to set path times (fd={}, flags={}, path={}, st_atim={}, st_mtim={}, fst_flags={:?}) - {}", + fd, + flags, + path, + st_atim, + st_mtim, + fst_flags, + err + ) + })?; + Ok(()) + } +} diff --git a/lib/wasix/src/snapshot/effector/syscalls/path_symlink.rs b/lib/wasix/src/snapshot/effector/syscalls/path_symlink.rs new file mode 100644 index 00000000000..cd18218bbf3 --- /dev/null +++ b/lib/wasix/src/snapshot/effector/syscalls/path_symlink.rs @@ -0,0 +1,38 @@ +use super::*; + +impl SnapshotEffector { + pub fn save_path_symlink( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + old_path: String, + fd: Fd, + new_path: String, + ) -> anyhow::Result<()> { + Self::save_event( + ctx, + SnapshotLog::CreateSymbolicLink { + old_path: old_path.into(), + fd, + new_path: new_path.into() + }, + ) + } + + pub fn apply_path_symlink( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + old_path: &str, + fd: Fd, + new_path: &str, + ) -> anyhow::Result<()> { + crate::syscalls::path_symlink_internal(ctx, old_path, fd, new_path) + .map_err(|err| { + anyhow::format_err!( + "snapshot restore error: failed to create symlink (old_path={}, fd={}, new_path={}) - {}", + old_path, + fd, + new_path, + err + ) + })?; + Ok(()) + } +} diff --git a/lib/wasix/src/snapshot/effector/syscalls/path_unlink.rs b/lib/wasix/src/snapshot/effector/syscalls/path_unlink.rs new file mode 100644 index 00000000000..dda892cfdad --- /dev/null +++ b/lib/wasix/src/snapshot/effector/syscalls/path_unlink.rs @@ -0,0 +1,34 @@ +use super::*; + +impl SnapshotEffector { + pub fn save_path_unlink( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + fd: Fd, + path: String, + ) -> anyhow::Result<()> { + Self::save_event( + ctx, + SnapshotLog::UnlinkFile { + fd, + path: Cow::Owned(path), + }, + ) + } + + pub fn apply_path_unlink( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + fd: Fd, + path: &str, + ) -> anyhow::Result<()> { + let ret = crate::syscalls::path_unlink_file_internal(ctx, fd, path)?; + if ret != Errno::Success { + bail!( + "snapshot restore error: failed to remove file (fd={}, path={}) - {}", + fd, + path, + ret + ); + } + Ok(()) + } +} diff --git a/lib/wasix/src/snapshot/effector/syscalls/tty_set.rs b/lib/wasix/src/snapshot/effector/syscalls/tty_set.rs new file mode 100644 index 00000000000..4650751107f --- /dev/null +++ b/lib/wasix/src/snapshot/effector/syscalls/tty_set.rs @@ -0,0 +1,38 @@ +use crate::WasiTtyState; + +use super::*; + +impl SnapshotEffector { + pub fn save_tty_set( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + state: WasiTtyState, + ) -> anyhow::Result<()> { + Self::save_event( + ctx, + SnapshotLog::TtySet { + tty: wasmer_wasix_types::wasi::Tty { + cols: state.cols, + rows: state.rows, + width: state.width, + height: state.height, + stdin_tty: state.stdin_tty, + stdout_tty: state.stdout_tty, + stderr_tty: state.stderr_tty, + echo: state.echo, + line_buffered: state.line_buffered, + }, + line_feeds: state.line_feeds, + }, + ) + } + + pub fn apply_tty_set( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + state: WasiTtyState, + ) -> anyhow::Result<()> { + crate::syscalls::tty_set_internal(ctx, state).map_err(|err| { + anyhow::format_err!("snapshot restore error: failed to set tty - {}", err) + })?; + Ok(()) + } +} diff --git a/lib/wasix/src/snapshot/effector/thread_exit.rs b/lib/wasix/src/snapshot/effector/thread_exit.rs new file mode 100644 index 00000000000..d496eb8c272 --- /dev/null +++ b/lib/wasix/src/snapshot/effector/thread_exit.rs @@ -0,0 +1,20 @@ +use super::*; + +impl SnapshotEffector { + pub fn save_thread_exit( + env: &WasiEnv, + id: WasiThreadId, + exit_code: Option, + ) -> anyhow::Result<()> { + __asyncify_light(env, None, async { + env.runtime() + .snapshot_capturer() + .write(SnapshotLog::CloseThread { id, exit_code }) + .await + .map_err(map_snapshot_err)?; + Ok(()) + })? + .map_err(|err| WasiError::Exit(ExitCode::Errno(err)))?; + Ok(()) + } +} diff --git a/lib/wasix/src/snapshot/effector/thread_state.rs b/lib/wasix/src/snapshot/effector/thread_state.rs new file mode 100644 index 00000000000..925d67a5996 --- /dev/null +++ b/lib/wasix/src/snapshot/effector/thread_state.rs @@ -0,0 +1,22 @@ +use super::*; + +impl SnapshotEffector { + pub fn save_thread_state( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + id: WasiThreadId, + memory_stack: Bytes, + rewind_stack: Bytes, + store_data: Bytes, + ) -> anyhow::Result<()> { + Self::save_event( + ctx, + SnapshotLog::SetThread { + id, + call_stack: Cow::Owned(rewind_stack.into()), + memory_stack: Cow::Owned(memory_stack.into()), + store_data: Cow::Owned(store_data.into()), + is_64bit: M::is_64bit(), + }, + ) + } +} diff --git a/lib/wasix/src/snapshot/filter.rs b/lib/wasix/src/snapshot/filter.rs index 740414f527e..65b8dca395f 100644 --- a/lib/wasix/src/snapshot/filter.rs +++ b/lib/wasix/src/snapshot/filter.rs @@ -10,9 +10,12 @@ pub struct FilteredSnapshotCapturer { filter_memory: bool, filter_threads: bool, filter_files: bool, + filter_chdir: bool, + filter_clock: bool, filter_terminal: bool, filter_snapshots: bool, filter_descriptors: bool, + filter_epoll: bool, } impl FilteredSnapshotCapturer { @@ -22,9 +25,12 @@ impl FilteredSnapshotCapturer { filter_memory: false, filter_threads: false, filter_files: false, + filter_chdir: false, + filter_clock: false, filter_terminal: false, filter_snapshots: false, filter_descriptors: false, + filter_epoll: false, } } @@ -43,6 +49,21 @@ impl FilteredSnapshotCapturer { self } + pub fn with_ignore_chdir(mut self, val: bool) -> Self { + self.filter_chdir = val; + self + } + + pub fn with_ignore_clock(mut self, val: bool) -> Self { + self.filter_clock = val; + self + } + + pub fn with_ignore_epoll(mut self, val: bool) -> Self { + self.filter_epoll = val; + self + } + pub fn with_ignore_terminal(mut self, val: bool) -> Self { self.filter_terminal = val; self @@ -64,8 +85,14 @@ impl SnapshotCapturer for FilteredSnapshotCapturer { Box::pin(async { let evt = match entry { SnapshotLog::Init { wasm_hash } => SnapshotLog::Init { wasm_hash }, + SnapshotLog::FileDescriptorSeek { .. } => { + if self.filter_descriptors { + return Ok(()); + } + entry + } SnapshotLog::FileDescriptorWrite { .. } => { - if self.filter_terminal { + if self.filter_descriptors { return Ok(()); } entry @@ -118,14 +145,117 @@ impl SnapshotCapturer for FilteredSnapshotCapturer { } entry } - SnapshotLog::UpdateFileSystemEntry { .. } => { + SnapshotLog::Snapshot { .. } => { + if self.filter_snapshots { + return Ok(()); + } + entry + } + SnapshotLog::SetClockTime { .. } => { + if self.filter_clock { + return Ok(()); + } + entry + } + SnapshotLog::RenumberFileDescriptor { .. } => { + if self.filter_terminal { + return Ok(()); + } + entry + } + SnapshotLog::DuplicateFileDescriptor { .. } => { + if self.filter_terminal { + return Ok(()); + } + entry + } + SnapshotLog::CreateDirectory { .. } => { + if self.filter_files { + return Ok(()); + } + entry + } + SnapshotLog::PathSetTimes { .. } => { if self.filter_files { return Ok(()); } entry } - SnapshotLog::Snapshot { .. } => { - if self.filter_snapshots { + SnapshotLog::FileDescriptorSetFlags { .. } => { + if self.filter_descriptors { + return Ok(()); + } + entry + } + SnapshotLog::FileDescriptorAdvise { .. } => { + if self.filter_descriptors { + return Ok(()); + } + entry + } + SnapshotLog::FileDescriptorAllocate { .. } => { + if self.filter_descriptors { + return Ok(()); + } + entry + } + + SnapshotLog::FileDescriptorSetRights { .. } => { + if self.filter_descriptors { + return Ok(()); + } + entry + } + SnapshotLog::FileDescriptorSetTimes { .. } => { + if self.filter_descriptors { + return Ok(()); + } + entry + } + SnapshotLog::FileDescriptorSetSize { .. } => { + if self.filter_descriptors { + return Ok(()); + } + entry + } + SnapshotLog::CreateHardLink { .. } => { + if self.filter_files { + return Ok(()); + } + entry + } + SnapshotLog::CreateSymbolicLink { .. } => { + if self.filter_files { + return Ok(()); + } + entry + } + SnapshotLog::ChangeDirectory { .. } => { + if self.filter_chdir { + return Ok(()); + } + entry + } + SnapshotLog::EpollCreate { .. } => { + if self.filter_memory { + return Ok(()); + } + entry + } + SnapshotLog::EpollCtl { .. } => { + if self.filter_memory { + return Ok(()); + } + entry + } + SnapshotLog::TtySet { .. } => { + if self.filter_terminal { + return Ok(()); + } + entry + } + SnapshotLog::CreatePipe { .. } => { + if self.filter_files { return Ok(()); } entry diff --git a/lib/wasix/src/snapshot/log_file.rs b/lib/wasix/src/snapshot/log_file.rs index d22ab39c263..787c95355a3 100644 --- a/lib/wasix/src/snapshot/log_file.rs +++ b/lib/wasix/src/snapshot/log_file.rs @@ -6,10 +6,10 @@ use std::{ time::SystemTime, }; use tokio::runtime::Handle; -use wasmer_wasix_types::wasi::ExitCode; +use wasmer_wasix_types::wasi::{self, EpollEventCtl}; use futures::future::LocalBoxFuture; -use virtual_fs::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt, Fd}; +use virtual_fs::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt}; use crate::WasiThreadId; @@ -26,9 +26,14 @@ pub enum SnapshotLogEntry { InitV1 { wasm_hash: [u8; 32], }, - Write { + FileDescriptorSeekV1 { fd: Fd, - offset: Option, + offset: i64, + whence: WhenceV1, + }, + FileDescriptorWriteV1 { + fd: u32, + offset: u64, data: Vec, is_64bit: bool, }, @@ -37,9 +42,9 @@ pub enum SnapshotLogEntry { end: u64, data: Vec, }, - CloseThreadV1 { - id: WasiThreadId, - exit_code: Option, + SetClockTimeV1 { + clock_id: Snapshot0ClockidV1, + time: u64, }, SetThreadV1 { id: WasiThreadId, @@ -48,42 +53,340 @@ pub enum SnapshotLogEntry { store_data: Vec, is_64bit: bool, }, + CloseThreadV1 { + id: WasiThreadId, + exit_code: Option, + }, CloseFileDescriptorV1 { - fd: Fd, + fd: u32, }, OpenFileDescriptorV1 { - fd: Fd, - state: FdOpenSnapshot<'static>, + fd: u32, + dirfd: u32, + dirflags: u32, + path: String, + o_flags: u16, + fs_rights_base: u64, + fs_rights_inheriting: u64, + fs_flags: u16, + is_64bit: bool, + }, + RenumberFileDescriptorV1 { + old_fd: u32, + new_fd: u32, + }, + DuplicateFileDescriptorV1 { + original_fd: u32, + copied_fd: u32, + }, + CreateDirectoryV1 { + fd: u32, + path: String, }, RemoveDirectoryV1 { + fd: u32, + path: String, + }, + PathSetTimesV1 { fd: Fd, + flags: u32, path: String, + st_atim: u64, + st_mtim: u64, + fst_flags: u16, + }, + FileDescriptorSetTimesV1 { + fd: u32, + st_atim: u64, + st_mtim: u64, + fst_flags: u16, + }, + FileDescriptorSetSizeV1 { + fd: u32, + st_size: u64, + }, + FileDescriptorSetFlagsV1 { + fd: u32, + flags: u16, + }, + FileDescriptorSetRightsV1 { + fd: u32, + fs_rights_base: u64, + fs_rights_inheriting: u64, + }, + FileDescriptorAdviseV1 { + fd: u32, + offset: u64, + len: u64, + advice: AdviceV1, + }, + FileDescriptorAllocateV1 { + fd: u32, + offset: u64, + len: u64, + }, + CreateHardLinkV1 { + old_fd: u32, + old_path: String, + old_flags: u32, + new_fd: u32, + new_path: String, + }, + CreateSymbolicLinkV1 { + old_path: String, + fd: u32, + new_path: String, }, UnlinkFileV1 { - fd: Fd, + fd: u32, path: String, }, PathRenameV1 { - old_fd: Fd, + old_fd: u32, old_path: String, - new_fd: Fd, + new_fd: u32, new_path: String, }, - UpdateFileSystemEntryV1 { + ChangeDirectoryV1 { path: String, - ft: FileEntryType, - accessed: u64, - created: u64, - modified: u64, - len: u64, - data: Vec, + }, + EpollCreateV1 { + fd: u32, + }, + EpollCtlV1 { + epfd: u32, + op: EpollCtlV1, + fd: u32, + event: Option, + }, + TtySetV1 { + cols: u32, + rows: u32, + width: u32, + height: u32, + stdin_tty: bool, + stdout_tty: bool, + stderr_tty: bool, + echo: bool, + line_buffered: bool, + line_feeds: bool, + }, + CreatePipeV1 { + fd1: u32, + fd2: u32, }, SnapshotV1 { when: SystemTime, - trigger: SnapshotTrigger, + trigger: SnapshotTriggerV1, }, } +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum Snapshot0ClockidV1 { + Realtime, + Monotonic, + ProcessCputimeId, + ThreadCputimeId, + Unknown = 255, +} + +impl Into for wasi::Snapshot0Clockid { + fn into(self) -> Snapshot0ClockidV1 { + match self { + Snapshot0Clockid::Realtime => Snapshot0ClockidV1::Realtime, + Snapshot0Clockid::Monotonic => Snapshot0ClockidV1::Monotonic, + Snapshot0Clockid::ProcessCputimeId => Snapshot0ClockidV1::ProcessCputimeId, + Snapshot0Clockid::ThreadCputimeId => Snapshot0ClockidV1::ThreadCputimeId, + Snapshot0Clockid::Unknown => Snapshot0ClockidV1::Unknown, + } + } +} + +impl Into for Snapshot0ClockidV1 { + fn into(self) -> wasi::Snapshot0Clockid { + match self { + Snapshot0ClockidV1::Realtime => Snapshot0Clockid::Realtime, + Snapshot0ClockidV1::Monotonic => Snapshot0Clockid::Monotonic, + Snapshot0ClockidV1::ProcessCputimeId => Snapshot0Clockid::ProcessCputimeId, + Snapshot0ClockidV1::ThreadCputimeId => Snapshot0Clockid::ThreadCputimeId, + Snapshot0ClockidV1::Unknown => Snapshot0Clockid::Unknown, + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum WhenceV1 { + Set, + Cur, + End, + Unknown = 255, +} + +impl Into for wasi::Whence { + fn into(self) -> WhenceV1 { + match self { + wasi::Whence::Set => WhenceV1::Set, + wasi::Whence::Cur => WhenceV1::Cur, + wasi::Whence::End => WhenceV1::End, + wasi::Whence::Unknown => WhenceV1::Unknown, + } + } +} + +impl Into for WhenceV1 { + fn into(self) -> wasi::Whence { + match self { + WhenceV1::Set => Whence::Set, + WhenceV1::Cur => Whence::Cur, + WhenceV1::End => Whence::End, + WhenceV1::Unknown => Whence::Unknown, + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum AdviceV1 { + Normal, + Sequential, + Random, + Willneed, + Dontneed, + Noreuse, + Unknown = 255, +} + +impl Into for wasi::Advice { + fn into(self) -> AdviceV1 { + match self { + Advice::Normal => AdviceV1::Normal, + Advice::Sequential => AdviceV1::Sequential, + Advice::Random => AdviceV1::Random, + Advice::Willneed => AdviceV1::Willneed, + Advice::Dontneed => AdviceV1::Dontneed, + Advice::Noreuse => AdviceV1::Noreuse, + Advice::Unknown => AdviceV1::Unknown, + } + } +} + +impl Into for AdviceV1 { + fn into(self) -> wasi::Advice { + match self { + AdviceV1::Normal => Advice::Normal, + AdviceV1::Sequential => Advice::Sequential, + AdviceV1::Random => Advice::Random, + AdviceV1::Willneed => Advice::Willneed, + AdviceV1::Dontneed => Advice::Dontneed, + AdviceV1::Noreuse => Advice::Noreuse, + AdviceV1::Unknown => Advice::Unknown, + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum ExitCodeV1 { + Errno(u16), + Other(i32), +} + +impl Into for wasi::ExitCode { + fn into(self) -> ExitCodeV1 { + match self { + wasi::ExitCode::Errno(errno) => ExitCodeV1::Errno(errno as u16), + wasi::ExitCode::Other(id) => ExitCodeV1::Other(id), + } + } +} + +impl Into for ExitCodeV1 { + fn into(self) -> wasi::ExitCode { + match self { + ExitCodeV1::Errno(errno) => { + wasi::ExitCode::Errno(errno.try_into().unwrap_or(wasi::Errno::Unknown)) + } + ExitCodeV1::Other(id) => wasi::ExitCode::Other(id), + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +pub enum SnapshotTriggerV1 { + Idle, + Listen, + Environ, + Stdin, + Timer, + Sigint, + Sigalrm, + Sigtstp, + Sigstop, + NonDeterministicCall, +} + +impl Into for SnapshotTrigger { + fn into(self) -> SnapshotTriggerV1 { + match self { + SnapshotTrigger::Idle => SnapshotTriggerV1::Idle, + SnapshotTrigger::Listen => SnapshotTriggerV1::Listen, + SnapshotTrigger::Environ => SnapshotTriggerV1::Environ, + SnapshotTrigger::Stdin => SnapshotTriggerV1::Stdin, + SnapshotTrigger::Timer => SnapshotTriggerV1::Timer, + SnapshotTrigger::Sigint => SnapshotTriggerV1::Sigint, + SnapshotTrigger::Sigalrm => SnapshotTriggerV1::Sigalrm, + SnapshotTrigger::Sigtstp => SnapshotTriggerV1::Sigtstp, + SnapshotTrigger::Sigstop => SnapshotTriggerV1::Sigstop, + SnapshotTrigger::NonDeterministicCall => SnapshotTriggerV1::NonDeterministicCall, + } + } +} + +impl Into for SnapshotTriggerV1 { + fn into(self) -> SnapshotTrigger { + match self { + SnapshotTriggerV1::Idle => SnapshotTrigger::Idle, + SnapshotTriggerV1::Listen => SnapshotTrigger::Listen, + SnapshotTriggerV1::Environ => SnapshotTrigger::Environ, + SnapshotTriggerV1::Stdin => SnapshotTrigger::Stdin, + SnapshotTriggerV1::Timer => SnapshotTrigger::Timer, + SnapshotTriggerV1::Sigint => SnapshotTrigger::Sigint, + SnapshotTriggerV1::Sigalrm => SnapshotTrigger::Sigalrm, + SnapshotTriggerV1::Sigtstp => SnapshotTrigger::Sigtstp, + SnapshotTriggerV1::Sigstop => SnapshotTrigger::Sigstop, + SnapshotTriggerV1::NonDeterministicCall => SnapshotTrigger::NonDeterministicCall, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +pub enum EpollCtlV1 { + Add, + Mod, + Del, + Unknown, +} + +impl Into for wasi::EpollCtl { + fn into(self) -> EpollCtlV1 { + match self { + wasi::EpollCtl::Add => EpollCtlV1::Add, + wasi::EpollCtl::Mod => EpollCtlV1::Mod, + wasi::EpollCtl::Del => EpollCtlV1::Del, + wasi::EpollCtl::Unknown => EpollCtlV1::Unknown, + } + } +} + +impl Into for EpollCtlV1 { + fn into(self) -> wasi::EpollCtl { + match self { + EpollCtlV1::Add => EpollCtl::Add, + EpollCtlV1::Mod => EpollCtl::Mod, + EpollCtlV1::Del => EpollCtl::Del, + EpollCtlV1::Unknown => EpollCtl::Unknown, + } + } +} + impl<'a> From> for SnapshotLogEntry { fn from(value: SnapshotLog<'a>) -> Self { match value { @@ -93,18 +396,26 @@ impl<'a> From> for SnapshotLogEntry { offset, data, is_64bit, - } => Self::Write { + } => Self::FileDescriptorWriteV1 { fd, offset, data: data.into_owned(), is_64bit, }, + SnapshotLog::FileDescriptorSeek { fd, offset, whence } => Self::FileDescriptorSeekV1 { + fd, + offset, + whence: whence.into(), + }, SnapshotLog::UpdateMemoryRegion { region, data } => Self::UpdateMemoryRegionV1 { start: region.start, end: region.end, data: data.into_owned(), }, - SnapshotLog::CloseThread { id, exit_code } => Self::CloseThreadV1 { id, exit_code }, + SnapshotLog::CloseThread { id, exit_code } => Self::CloseThreadV1 { + id, + exit_code: exit_code.map(|code| code.into()), + }, SnapshotLog::SetThread { id, call_stack, @@ -119,9 +430,26 @@ impl<'a> From> for SnapshotLogEntry { is_64bit, }, SnapshotLog::CloseFileDescriptor { fd } => Self::CloseFileDescriptorV1 { fd }, - SnapshotLog::OpenFileDescriptor { fd, state } => Self::OpenFileDescriptorV1 { + SnapshotLog::OpenFileDescriptor { + fd, + dirfd, + dirflags, + path, + o_flags, + fs_rights_base, + fs_rights_inheriting, + fs_flags, + is_64bit, + } => Self::OpenFileDescriptorV1 { fd, - state: state.into_owned(), + dirfd, + dirflags, + path: path.into_owned(), + o_flags: o_flags.bits(), + fs_rights_base: fs_rights_base.bits(), + fs_rights_inheriting: fs_rights_inheriting.bits(), + fs_flags: fs_flags.bits(), + is_64bit, }, SnapshotLog::RemoveDirectory { fd, path } => Self::RemoveDirectoryV1 { fd, @@ -142,24 +470,134 @@ impl<'a> From> for SnapshotLogEntry { new_fd, new_path: new_path.into_owned(), }, - SnapshotLog::UpdateFileSystemEntry { + SnapshotLog::Snapshot { when, trigger } => Self::SnapshotV1 { + when, + trigger: trigger.into(), + }, + SnapshotLog::SetClockTime { clock_id, time } => Self::SetClockTimeV1 { + clock_id: clock_id.into(), + time, + }, + SnapshotLog::RenumberFileDescriptor { old_fd, new_fd } => { + Self::RenumberFileDescriptorV1 { old_fd, new_fd } + } + SnapshotLog::DuplicateFileDescriptor { + original_fd, + copied_fd, + } => Self::DuplicateFileDescriptorV1 { + original_fd, + copied_fd, + }, + SnapshotLog::CreateDirectory { fd, path } => Self::CreateDirectoryV1 { + fd, + path: path.into_owned(), + }, + SnapshotLog::PathSetTimes { + fd, path, - ft, - accessed, - created, - modified, - len, - data, - } => Self::UpdateFileSystemEntryV1 { + flags, + st_atim, + st_mtim, + fst_flags, + } => Self::PathSetTimesV1 { + fd, path: path.into_owned(), - ft, - accessed, - created, - modified, + flags, + st_atim, + st_mtim, + fst_flags: fst_flags.bits(), + }, + SnapshotLog::FileDescriptorSetTimes { + fd, + st_atim, + st_mtim, + fst_flags, + } => Self::FileDescriptorSetTimesV1 { + fd, + st_atim, + st_mtim, + fst_flags: fst_flags.bits(), + }, + SnapshotLog::FileDescriptorSetSize { fd, st_size } => { + Self::FileDescriptorSetSizeV1 { fd, st_size } + } + SnapshotLog::FileDescriptorSetFlags { fd, flags } => Self::FileDescriptorSetFlagsV1 { + fd, + flags: flags.bits(), + }, + SnapshotLog::FileDescriptorSetRights { + fd, + fs_rights_base, + fs_rights_inheriting, + } => Self::FileDescriptorSetRightsV1 { + fd, + fs_rights_base: fs_rights_base.bits(), + fs_rights_inheriting: fs_rights_inheriting.bits(), + }, + SnapshotLog::FileDescriptorAdvise { + fd, + offset, len, - data: data.into_owned(), + advice, + } => Self::FileDescriptorAdviseV1 { + fd, + offset, + len, + advice: advice.into(), }, - SnapshotLog::Snapshot { when, trigger } => Self::SnapshotV1 { when, trigger }, + SnapshotLog::FileDescriptorAllocate { fd, offset, len } => { + Self::FileDescriptorAllocateV1 { fd, offset, len } + } + SnapshotLog::CreateHardLink { + old_fd, + old_path, + old_flags, + new_fd, + new_path, + } => Self::CreateHardLinkV1 { + old_fd, + old_path: old_path.into_owned(), + old_flags, + new_fd, + new_path: new_path.into_owned(), + }, + SnapshotLog::CreateSymbolicLink { + old_path, + fd, + new_path, + } => Self::CreateSymbolicLinkV1 { + old_path: old_path.into_owned(), + fd, + new_path: new_path.into_owned(), + }, + SnapshotLog::ChangeDirectory { path } => Self::ChangeDirectoryV1 { + path: path.into_owned(), + }, + SnapshotLog::EpollCreate { fd } => Self::EpollCreateV1 { fd }, + SnapshotLog::EpollCtl { + epfd, + op, + fd, + event, + } => Self::EpollCtlV1 { + epfd, + op: op.into(), + fd, + event, + }, + SnapshotLog::TtySet { tty, line_feeds } => Self::TtySetV1 { + cols: tty.cols, + rows: tty.rows, + width: tty.width, + height: tty.height, + stdin_tty: tty.stdin_tty, + stdout_tty: tty.stdout_tty, + stderr_tty: tty.stderr_tty, + echo: tty.echo, + line_buffered: tty.line_buffered, + line_feeds, + }, + SnapshotLog::CreatePipe { fd1, fd2 } => Self::CreatePipeV1 { fd1, fd2 }, } } } @@ -168,7 +606,7 @@ impl<'a> From for SnapshotLog<'a> { fn from(value: SnapshotLogEntry) -> Self { match value { SnapshotLogEntry::InitV1 { wasm_hash } => Self::Init { wasm_hash }, - SnapshotLogEntry::Write { + SnapshotLogEntry::FileDescriptorWriteV1 { data, fd, offset, @@ -179,15 +617,23 @@ impl<'a> From for SnapshotLog<'a> { offset, is_64bit, }, + SnapshotLogEntry::FileDescriptorSeekV1 { fd, offset, whence } => { + Self::FileDescriptorSeek { + fd, + offset, + whence: whence.into(), + } + } SnapshotLogEntry::UpdateMemoryRegionV1 { start, end, data } => { Self::UpdateMemoryRegion { region: start..end, data: data.into(), } } - SnapshotLogEntry::CloseThreadV1 { id, exit_code } => { - Self::CloseThread { id, exit_code } - } + SnapshotLogEntry::CloseThreadV1 { id, exit_code } => Self::CloseThread { + id, + exit_code: exit_code.map(|code| code.into()), + }, SnapshotLogEntry::SetThreadV1 { id, call_stack, @@ -202,9 +648,26 @@ impl<'a> From for SnapshotLog<'a> { is_64bit, }, SnapshotLogEntry::CloseFileDescriptorV1 { fd } => Self::CloseFileDescriptor { fd }, - SnapshotLogEntry::OpenFileDescriptorV1 { fd, state } => Self::OpenFileDescriptor { + SnapshotLogEntry::OpenFileDescriptorV1 { + fd, + dirfd, + dirflags, + path, + o_flags, + fs_rights_base, + fs_rights_inheriting, + fs_flags, + is_64bit, + } => Self::OpenFileDescriptor { fd, - state: state.clone(), + dirfd, + dirflags, + path: path.into(), + o_flags: wasi::Oflags::from_bits_truncate(o_flags), + fs_rights_base: wasi::Rights::from_bits_truncate(fs_rights_base), + fs_rights_inheriting: wasi::Rights::from_bits_truncate(fs_rights_inheriting), + fs_flags: wasi::Fdflags::from_bits_truncate(fs_flags), + is_64bit, }, SnapshotLogEntry::RemoveDirectoryV1 { fd, path } => Self::RemoveDirectory { fd, @@ -225,24 +688,149 @@ impl<'a> From for SnapshotLog<'a> { new_fd, new_path: new_path.into(), }, - SnapshotLogEntry::UpdateFileSystemEntryV1 { + SnapshotLogEntry::SnapshotV1 { when, trigger } => Self::Snapshot { + when, + trigger: trigger.into(), + }, + SnapshotLogEntry::SetClockTimeV1 { clock_id, time } => Self::SetClockTime { + clock_id: clock_id.into(), + time, + }, + SnapshotLogEntry::RenumberFileDescriptorV1 { old_fd, new_fd } => { + Self::RenumberFileDescriptor { old_fd, new_fd } + } + SnapshotLogEntry::DuplicateFileDescriptorV1 { + original_fd: old_fd, + copied_fd: new_fd, + } => Self::DuplicateFileDescriptor { + original_fd: old_fd, + copied_fd: new_fd, + }, + SnapshotLogEntry::CreateDirectoryV1 { fd, path } => Self::CreateDirectory { + fd, + path: path.into(), + }, + SnapshotLogEntry::PathSetTimesV1 { + fd, path, - ft, - accessed, - created, - modified, - len, - data, - } => Self::UpdateFileSystemEntry { + flags, + st_atim, + st_mtim, + fst_flags, + } => Self::PathSetTimes { + fd, path: path.into(), - ft: ft.clone(), - accessed, - created, - modified, + flags, + st_atim, + st_mtim, + fst_flags: wasi::Fstflags::from_bits_truncate(fst_flags), + }, + SnapshotLogEntry::FileDescriptorSetTimesV1 { + fd, + st_atim, + st_mtim, + fst_flags, + } => Self::FileDescriptorSetTimes { + fd, + st_atim, + st_mtim, + fst_flags: wasi::Fstflags::from_bits_truncate(fst_flags), + }, + SnapshotLogEntry::FileDescriptorSetSizeV1 { fd, st_size } => { + Self::FileDescriptorSetSize { fd, st_size } + } + SnapshotLogEntry::FileDescriptorSetFlagsV1 { fd, flags } => { + Self::FileDescriptorSetFlags { + fd, + flags: Fdflags::from_bits_truncate(flags), + } + } + SnapshotLogEntry::FileDescriptorSetRightsV1 { + fd, + fs_rights_base, + fs_rights_inheriting, + } => Self::FileDescriptorSetRights { + fd, + fs_rights_base: Rights::from_bits_truncate(fs_rights_base), + fs_rights_inheriting: Rights::from_bits_truncate(fs_rights_inheriting), + }, + SnapshotLogEntry::FileDescriptorAdviseV1 { + fd, + offset, len, - data: data.into(), + advice, + } => Self::FileDescriptorAdvise { + fd, + offset, + len, + advice: advice.into(), + }, + SnapshotLogEntry::FileDescriptorAllocateV1 { fd, offset, len } => { + Self::FileDescriptorAllocate { fd, offset, len } + } + SnapshotLogEntry::CreateHardLinkV1 { + old_fd, + old_path, + old_flags, + new_fd, + new_path, + } => Self::CreateHardLink { + old_fd, + old_path: old_path.into(), + old_flags, + new_fd, + new_path: new_path.into(), + }, + SnapshotLogEntry::CreateSymbolicLinkV1 { + old_path, + fd, + new_path, + } => Self::CreateSymbolicLink { + old_path: old_path.into(), + fd, + new_path: new_path.into(), + }, + SnapshotLogEntry::ChangeDirectoryV1 { path } => { + Self::ChangeDirectory { path: path.into() } + } + SnapshotLogEntry::EpollCreateV1 { fd } => Self::EpollCreate { fd }, + SnapshotLogEntry::EpollCtlV1 { + epfd, + op, + fd, + event, + } => Self::EpollCtl { + epfd, + op: op.into(), + fd, + event, + }, + SnapshotLogEntry::TtySetV1 { + cols, + rows, + width, + height, + stdin_tty, + stdout_tty, + stderr_tty, + echo, + line_buffered, + line_feeds, + } => Self::TtySet { + tty: wasi::Tty { + cols, + rows, + width, + height, + stdin_tty, + stdout_tty, + stderr_tty, + echo, + line_buffered, + }, + line_feeds, }, - SnapshotLogEntry::SnapshotV1 { when, trigger } => Self::Snapshot { when, trigger }, + SnapshotLogEntry::CreatePipeV1 { fd1, fd2 } => Self::CreatePipe { fd1, fd2 }, } } } diff --git a/lib/wasix/src/syscalls/mod.rs b/lib/wasix/src/syscalls/mod.rs index f0d682f6ba8..66bfb208a81 100644 --- a/lib/wasix/src/syscalls/mod.rs +++ b/lib/wasix/src/syscalls/mod.rs @@ -1313,12 +1313,19 @@ pub fn restore_snapshot( is_64bit, } => { if is_64bit { - SnapshotEffector::apply_write::(&mut ctx, fd, offset, data).await + SnapshotEffector::apply_fd_write::(&mut ctx, fd, offset, data) + .await } else { - SnapshotEffector::apply_write::(&mut ctx, fd, offset, data).await + SnapshotEffector::apply_fd_write::(&mut ctx, fd, offset, data) + .await } .map_err(anyhow_err_to_runtime_err)?; } + crate::snapshot::SnapshotLog::FileDescriptorSeek { fd, offset, whence } => { + SnapshotEffector::apply_fd_seek(&mut ctx, fd, offset, whence) + .await + .map_err(anyhow_err_to_runtime_err)?; + } crate::snapshot::SnapshotLog::UpdateMemoryRegion { region, data } => { SnapshotEffector::apply_memory(&mut ctx, region, &data) .map_err(anyhow_err_to_runtime_err)?; @@ -1356,28 +1363,41 @@ pub fn restore_snapshot( ))); } } - crate::snapshot::SnapshotLog::CloseFileDescriptor { fd: _ } => { - return Err(WasiRuntimeError::Runtime(RuntimeError::user( - anyhow::format_err!( - "Snapshot restoration does not currently support file descriptors." - ) - .into(), - ))); + crate::snapshot::SnapshotLog::CloseFileDescriptor { fd } => { + SnapshotEffector::apply_fd_close(&mut ctx, fd) + .map_err(anyhow_err_to_runtime_err)?; } - crate::snapshot::SnapshotLog::OpenFileDescriptor { fd: _, state: _ } => { - return Err(WasiRuntimeError::Runtime(RuntimeError::user( - anyhow::format_err!( - "Snapshot restoration does not currently support file descriptors." - ) - .into(), - ))); + crate::snapshot::SnapshotLog::OpenFileDescriptor { + fd, + dirfd, + dirflags, + path, + o_flags, + fs_rights_base, + fs_rights_inheriting, + fs_flags, + is_64bit, + } => { + SnapshotEffector::apply_path_open( + &mut ctx, + fd, + dirfd, + dirflags, + &path, + o_flags, + fs_rights_base, + fs_rights_inheriting, + fs_flags, + is_64bit, + ) + .map_err(anyhow_err_to_runtime_err)?; } crate::snapshot::SnapshotLog::RemoveDirectory { fd, path } => { - SnapshotEffector::apply_remove_directory(&mut ctx, fd, &path) + SnapshotEffector::apply_path_remove_directory(&mut ctx, fd, &path) .map_err(anyhow_err_to_runtime_err)?; } crate::snapshot::SnapshotLog::UnlinkFile { fd, path } => { - SnapshotEffector::apply_unlink_file(&mut ctx, fd, &path) + SnapshotEffector::apply_path_unlink(&mut ctx, fd, &path) .map_err(anyhow_err_to_runtime_err)?; } crate::snapshot::SnapshotLog::PathRename { @@ -1386,24 +1406,10 @@ pub fn restore_snapshot( new_fd, new_path, } => { - SnapshotEffector::apply_rename(&mut ctx, old_fd, &old_path, new_fd, &new_path) - .map_err(anyhow_err_to_runtime_err)?; - } - crate::snapshot::SnapshotLog::UpdateFileSystemEntry { - path: _, - ft: _, - accessed: _, - created: _, - modified: _, - len: _, - data: _, - } => { - return Err(WasiRuntimeError::Runtime(RuntimeError::user( - anyhow::format_err!( - "Snapshot restoration does not currently support local file changes." - ) - .into(), - ))); + SnapshotEffector::apply_path_rename( + &mut ctx, old_fd, &old_path, new_fd, &new_path, + ) + .map_err(anyhow_err_to_runtime_err)?; } crate::snapshot::SnapshotLog::Snapshot { when: _, @@ -1416,6 +1422,140 @@ pub fn restore_snapshot( *n_snapshots -= 1; } } + crate::snapshot::SnapshotLog::SetClockTime { clock_id, time } => { + SnapshotEffector::apply_clock_time_set(&mut ctx, clock_id, time) + .map_err(anyhow_err_to_runtime_err)?; + } + crate::snapshot::SnapshotLog::RenumberFileDescriptor { old_fd, new_fd } => { + SnapshotEffector::apply_fd_renumber(&mut ctx, old_fd, new_fd) + .map_err(anyhow_err_to_runtime_err)?; + } + crate::snapshot::SnapshotLog::DuplicateFileDescriptor { + original_fd, + copied_fd, + } => { + SnapshotEffector::apply_fd_duplicate(&mut ctx, original_fd, copied_fd) + .map_err(anyhow_err_to_runtime_err)?; + } + crate::snapshot::SnapshotLog::CreateDirectory { fd, path } => { + SnapshotEffector::apply_path_create_directory(&mut ctx, fd, &path) + .map_err(anyhow_err_to_runtime_err)?; + } + crate::snapshot::SnapshotLog::PathSetTimes { + fd, + flags, + path, + st_atim, + st_mtim, + fst_flags, + } => { + SnapshotEffector::apply_path_set_times( + &mut ctx, fd, flags, &path, st_atim, st_mtim, fst_flags, + ) + .map_err(anyhow_err_to_runtime_err)?; + } + crate::snapshot::SnapshotLog::FileDescriptorSetTimes { + fd, + st_atim, + st_mtim, + fst_flags, + } => { + SnapshotEffector::apply_fd_set_times(&mut ctx, fd, st_atim, st_mtim, fst_flags) + .map_err(anyhow_err_to_runtime_err)?; + } + crate::snapshot::SnapshotLog::FileDescriptorSetSize { fd, st_size } => { + SnapshotEffector::apply_fd_set_size(&mut ctx, fd, st_size) + .map_err(anyhow_err_to_runtime_err)?; + } + crate::snapshot::SnapshotLog::FileDescriptorSetFlags { fd, flags } => { + SnapshotEffector::apply_fd_set_flags(&mut ctx, fd, flags) + .map_err(anyhow_err_to_runtime_err)?; + } + crate::snapshot::SnapshotLog::FileDescriptorSetRights { + fd, + fs_rights_base, + fs_rights_inheriting, + } => { + SnapshotEffector::apply_fd_set_rights( + &mut ctx, + fd, + fs_rights_base, + fs_rights_inheriting, + ) + .map_err(anyhow_err_to_runtime_err)?; + } + crate::snapshot::SnapshotLog::FileDescriptorAdvise { + fd, + offset, + len, + advice, + } => { + SnapshotEffector::apply_fd_advise(&mut ctx, fd, offset, len, advice) + .map_err(anyhow_err_to_runtime_err)?; + } + crate::snapshot::SnapshotLog::FileDescriptorAllocate { fd, offset, len } => { + SnapshotEffector::apply_fd_allocate(&mut ctx, fd, offset, len) + .map_err(anyhow_err_to_runtime_err)?; + } + crate::snapshot::SnapshotLog::CreateHardLink { + old_fd, + old_path, + old_flags, + new_fd, + new_path, + } => { + SnapshotEffector::apply_path_link( + &mut ctx, old_fd, old_flags, &old_path, new_fd, &new_path, + ) + .map_err(anyhow_err_to_runtime_err)?; + } + crate::snapshot::SnapshotLog::CreateSymbolicLink { + old_path, + fd, + new_path, + } => { + SnapshotEffector::apply_path_symlink(&mut ctx, &old_path, fd, &new_path) + .map_err(anyhow_err_to_runtime_err)?; + } + crate::snapshot::SnapshotLog::ChangeDirectory { path } => { + SnapshotEffector::apply_chdir(&mut ctx, &path) + .map_err(anyhow_err_to_runtime_err)?; + } + crate::snapshot::SnapshotLog::CreatePipe { fd1, fd2 } => { + SnapshotEffector::apply_fd_pipe(&mut ctx, fd1, fd2) + .map_err(anyhow_err_to_runtime_err)?; + } + crate::snapshot::SnapshotLog::EpollCreate { fd } => { + SnapshotEffector::apply_epoll_create(&mut ctx, fd) + .map_err(anyhow_err_to_runtime_err)?; + } + crate::snapshot::SnapshotLog::EpollCtl { + epfd, + op, + fd, + event, + } => { + SnapshotEffector::apply_epoll_ctl(&mut ctx, epfd, op, fd, event) + .map_err(anyhow_err_to_runtime_err)?; + } + crate::snapshot::SnapshotLog::TtySet { tty, line_feeds } => { + SnapshotEffector::apply_tty_set( + &mut ctx, + crate::WasiTtyState { + cols: tty.cols, + rows: tty.rows, + width: tty.width, + height: tty.height, + stdin_tty: tty.stdin_tty, + stdout_tty: tty.stdout_tty, + stderr_tty: tty.stderr_tty, + echo: tty.echo, + line_buffered: tty.line_buffered, + line_feeds: line_feeds, + }, + ) + .map_err(anyhow_err_to_runtime_err)?; + } } } rewind.ok_or(WasiRuntimeError::Runtime(RuntimeError::user(anyhow::format_err!("The restored snapshot journal does not have a thread stack events and hence we can not restore the state of the process.").into()))) diff --git a/lib/wasix/src/syscalls/wasi/clock_time_set.rs b/lib/wasix/src/syscalls/wasi/clock_time_set.rs index 60b2ccbbe6c..02ff2c231d9 100644 --- a/lib/wasix/src/syscalls/wasi/clock_time_set.rs +++ b/lib/wasix/src/syscalls/wasi/clock_time_set.rs @@ -10,7 +10,27 @@ use crate::syscalls::*; /// The value of the clock in nanoseconds #[instrument(level = "trace", skip_all, fields(?clock_id, %time), ret)] pub fn clock_time_set( - ctx: FunctionEnvMut<'_, WasiEnv>, + mut ctx: FunctionEnvMut<'_, WasiEnv>, + clock_id: Snapshot0Clockid, + time: Timestamp, +) -> Result { + let ret = clock_time_set_internal(&mut ctx, clock_id, time); + let env = ctx.data(); + + if ret == Errno::Success { + #[cfg(feature = "snapshot")] + if env.enable_snapshot_capture { + SnapshotEffector::save_clock_time_set(&mut ctx, clock_id, time).map_err(|err| { + tracing::error!("failed to save clock time set event - {}", err); + WasiError::Exit(ExitCode::Errno(Errno::Fault)) + })?; + } + } + Ok(ret) +} + +pub fn clock_time_set_internal( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, clock_id: Snapshot0Clockid, time: Timestamp, ) -> Errno { diff --git a/lib/wasix/src/syscalls/wasi/fd_advise.rs b/lib/wasix/src/syscalls/wasi/fd_advise.rs index 0426d6da07c..38ef1586a94 100644 --- a/lib/wasix/src/syscalls/wasi/fd_advise.rs +++ b/lib/wasix/src/syscalls/wasi/fd_advise.rs @@ -14,13 +14,34 @@ use crate::syscalls::*; /// The advice to give #[instrument(level = "debug", skip_all, fields(%fd, %offset, %len, ?advice), ret)] pub fn fd_advise( - ctx: FunctionEnvMut<'_, WasiEnv>, + mut ctx: FunctionEnvMut<'_, WasiEnv>, fd: WasiFd, offset: Filesize, len: Filesize, advice: Advice, -) -> Errno { +) -> Result { + wasi_try_ok!(fd_advise_internal(&mut ctx, fd, offset, len, advice)); + let env = ctx.data(); + + #[cfg(feature = "snapshot")] + if env.enable_snapshot_capture { + SnapshotEffector::save_fd_advise(&mut ctx, fd, offset, len, advice).map_err(|err| { + tracing::error!("failed to save file descriptor advise event - {}", err); + WasiError::Exit(ExitCode::Errno(Errno::Fault)) + })?; + } + + Ok(Errno::Success) +} + +pub(crate) fn fd_advise_internal( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + fd: WasiFd, + offset: Filesize, + len: Filesize, + advice: Advice, +) -> Result<(), Errno> { // this is used for our own benefit, so just returning success is a valid // implementation for now - Errno::Success + Ok(()) } diff --git a/lib/wasix/src/syscalls/wasi/fd_allocate.rs b/lib/wasix/src/syscalls/wasi/fd_allocate.rs index d3450298bcc..f2a1ab542a7 100644 --- a/lib/wasix/src/syscalls/wasi/fd_allocate.rs +++ b/lib/wasix/src/syscalls/wasi/fd_allocate.rs @@ -12,43 +12,63 @@ use crate::syscalls::*; /// The length from the offset marking the end of the allocation #[instrument(level = "debug", skip_all, fields(%fd, %offset, %len), ret)] pub fn fd_allocate( - ctx: FunctionEnvMut<'_, WasiEnv>, + mut ctx: FunctionEnvMut<'_, WasiEnv>, fd: WasiFd, offset: Filesize, len: Filesize, -) -> Errno { +) -> Result { + wasi_try_ok!(fd_allocate_internal(&mut ctx, fd, offset, len)); + let env = ctx.data(); + + #[cfg(feature = "snapshot")] + if env.enable_snapshot_capture { + SnapshotEffector::save_fd_allocate(&mut ctx, fd, offset, len).map_err(|err| { + tracing::error!("failed to save file descriptor allocate event - {}", err); + WasiError::Exit(ExitCode::Errno(Errno::Fault)) + })?; + } + + Ok(Errno::Success) +} + +pub(crate) fn fd_allocate_internal( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + fd: WasiFd, + offset: Filesize, + len: Filesize, +) -> Result<(), Errno> { let env = ctx.data(); let (_, mut state) = unsafe { env.get_memory_and_wasi_state(&ctx, 0) }; - let fd_entry = wasi_try!(state.fs.get_fd(fd)); + let fd_entry = state.fs.get_fd(fd)?; let inode = fd_entry.inode; if !fd_entry.rights.contains(Rights::FD_ALLOCATE) { - return Errno::Access; + return Err(Errno::Access); } - let new_size = wasi_try!(offset.checked_add(len).ok_or(Errno::Inval)); + let new_size = offset.checked_add(len).ok_or(Errno::Inval)?; { let mut guard = inode.write(); match guard.deref_mut() { Kind::File { handle, .. } => { if let Some(handle) = handle { let mut handle = handle.write().unwrap(); - wasi_try!(handle.set_len(new_size).map_err(fs_error_into_wasi_err)); + handle.set_len(new_size).map_err(fs_error_into_wasi_err)?; } else { - return Errno::Badf; + return Err(Errno::Badf); } } - Kind::Socket { .. } => return Errno::Badf, - Kind::Pipe { .. } => return Errno::Badf, + Kind::Socket { .. } => return Err(Errno::Badf), + Kind::Pipe { .. } => return Err(Errno::Badf), Kind::Buffer { buffer } => { buffer.resize(new_size as usize, 0); } - Kind::Symlink { .. } => return Errno::Badf, - Kind::EventNotifications { .. } | Kind::Epoll { .. } => return Errno::Badf, - Kind::Dir { .. } | Kind::Root { .. } => return Errno::Isdir, + Kind::Symlink { .. } => return Err(Errno::Badf), + Kind::EventNotifications { .. } | Kind::Epoll { .. } => return Err(Errno::Badf), + Kind::Dir { .. } | Kind::Root { .. } => return Err(Errno::Isdir), } } inode.stat.write().unwrap().st_size = new_size; debug!(%new_size); - Errno::Success + Ok(()) } diff --git a/lib/wasix/src/syscalls/wasi/fd_close.rs b/lib/wasix/src/syscalls/wasi/fd_close.rs index 1d38b4391c6..4d75cc508bd 100644 --- a/lib/wasix/src/syscalls/wasi/fd_close.rs +++ b/lib/wasix/src/syscalls/wasi/fd_close.rs @@ -32,5 +32,16 @@ pub fn fd_close(mut ctx: FunctionEnvMut<'_, WasiEnv>, fd: WasiFd) -> Result( - ctx: FunctionEnvMut<'_, WasiEnv>, + mut ctx: FunctionEnvMut<'_, WasiEnv>, fd: WasiFd, ret_fd: WasmPtr, -) -> Errno { +) -> Result { + let copied_fd = wasi_try_ok!(fd_dup_internal(&mut ctx, fd)); + let env = ctx.data(); + + #[cfg(feature = "snapshot")] + if env.enable_snapshot_capture { + SnapshotEffector::save_fd_duplicate(&mut ctx, fd, copied_fd).map_err(|err| { + tracing::error!("failed to save file descriptor renumber event - {}", err); + WasiError::Exit(ExitCode::Errno(Errno::Fault)) + })?; + } + + Span::current().record("ret_fd", copied_fd); let env = ctx.data(); let (memory, state) = unsafe { env.get_memory_and_wasi_state(&ctx, 0) }; - let fd = wasi_try!(state.fs.clone_fd(fd)); + wasi_try_mem_ok!(ret_fd.write(&memory, copied_fd)); - Span::current().record("ret_fd", fd); - wasi_try_mem!(ret_fd.write(&memory, fd)); + Ok(Errno::Success) +} - Errno::Success +pub(crate) fn fd_dup_internal( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + fd: WasiFd, +) -> Result { + let env = ctx.data(); + let (memory, state) = unsafe { env.get_memory_and_wasi_state(&ctx, 0) }; + let fd = state.fs.clone_fd(fd)?; + Ok(fd) } diff --git a/lib/wasix/src/syscalls/wasi/fd_fdstat_set_flags.rs b/lib/wasix/src/syscalls/wasi/fd_fdstat_set_flags.rs index 7ee8790a721..aed536619da 100644 --- a/lib/wasix/src/syscalls/wasi/fd_fdstat_set_flags.rs +++ b/lib/wasix/src/syscalls/wasi/fd_fdstat_set_flags.rs @@ -13,6 +13,27 @@ pub fn fd_fdstat_set_flags( mut ctx: FunctionEnvMut<'_, WasiEnv>, fd: WasiFd, flags: Fdflags, +) -> Result { + let ret = fd_fdstat_set_flags_internal(&mut ctx, fd, flags)?; + let env = ctx.data(); + + if ret == Errno::Success { + #[cfg(feature = "snapshot")] + if env.enable_snapshot_capture { + SnapshotEffector::save_fd_set_flags(&mut ctx, fd, flags).map_err(|err| { + tracing::error!("failed to save file set flags event - {}", err); + WasiError::Exit(ExitCode::Errno(Errno::Fault)) + })?; + } + } + + Ok(ret) +} + +pub(crate) fn fd_fdstat_set_flags_internal( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + fd: WasiFd, + flags: Fdflags, ) -> Result { { let env = ctx.data(); diff --git a/lib/wasix/src/syscalls/wasi/fd_fdstat_set_rights.rs b/lib/wasix/src/syscalls/wasi/fd_fdstat_set_rights.rs index ec724f9d536..c4933444230 100644 --- a/lib/wasix/src/syscalls/wasi/fd_fdstat_set_rights.rs +++ b/lib/wasix/src/syscalls/wasi/fd_fdstat_set_rights.rs @@ -12,25 +12,51 @@ use crate::syscalls::*; /// The inheriting rights to apply to `fd` #[instrument(level = "debug", skip_all, fields(%fd), ret)] pub fn fd_fdstat_set_rights( - ctx: FunctionEnvMut<'_, WasiEnv>, + mut ctx: FunctionEnvMut<'_, WasiEnv>, fd: WasiFd, fs_rights_base: Rights, fs_rights_inheriting: Rights, -) -> Errno { +) -> Result { + wasi_try_ok!(fd_fdstat_set_rights_internal( + &mut ctx, + fd, + fs_rights_base, + fs_rights_inheriting + )); + let env = ctx.data(); + + #[cfg(feature = "snapshot")] + if env.enable_snapshot_capture { + SnapshotEffector::save_fd_set_rights(&mut ctx, fd, fs_rights_base, fs_rights_inheriting) + .map_err(|err| { + tracing::error!("failed to save file set rights event - {}", err); + WasiError::Exit(ExitCode::Errno(Errno::Fault)) + })?; + } + + Ok(Errno::Success) +} + +pub(crate) fn fd_fdstat_set_rights_internal( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + fd: WasiFd, + fs_rights_base: Rights, + fs_rights_inheriting: Rights, +) -> Result<(), Errno> { let env = ctx.data(); let (_, mut state) = unsafe { env.get_memory_and_wasi_state(&ctx, 0) }; let mut fd_map = state.fs.fd_map.write().unwrap(); - let fd_entry = wasi_try!(fd_map.get_mut(&fd).ok_or(Errno::Badf)); + let fd_entry = fd_map.get_mut(&fd).ok_or(Errno::Badf)?; // ensure new rights are a subset of current rights if fd_entry.rights | fs_rights_base != fd_entry.rights || fd_entry.rights_inheriting | fs_rights_inheriting != fd_entry.rights_inheriting { - return Errno::Notcapable; + return Err(Errno::Notcapable); } fd_entry.rights = fs_rights_base; fd_entry.rights_inheriting = fs_rights_inheriting; - Errno::Success + Ok(()) } diff --git a/lib/wasix/src/syscalls/wasi/fd_filestat_set_size.rs b/lib/wasix/src/syscalls/wasi/fd_filestat_set_size.rs index 40979d8d8fb..2ae923302d3 100644 --- a/lib/wasix/src/syscalls/wasi/fd_filestat_set_size.rs +++ b/lib/wasix/src/syscalls/wasi/fd_filestat_set_size.rs @@ -10,17 +10,36 @@ use crate::syscalls::*; /// New size that `fd` will be set to #[instrument(level = "debug", skip_all, fields(%fd, %st_size), ret)] pub fn fd_filestat_set_size( - ctx: FunctionEnvMut<'_, WasiEnv>, + mut ctx: FunctionEnvMut<'_, WasiEnv>, fd: WasiFd, st_size: Filesize, -) -> Errno { +) -> Result { + wasi_try_ok!(fd_filestat_set_size_internal(&mut ctx, fd, st_size)); + let env = ctx.data(); + + #[cfg(feature = "snapshot")] + if env.enable_snapshot_capture { + SnapshotEffector::save_fd_set_size(&mut ctx, fd, st_size).map_err(|err| { + tracing::error!("failed to save file set size event - {}", err); + WasiError::Exit(ExitCode::Errno(Errno::Fault)) + })?; + } + + Ok(Errno::Success) +} + +pub(crate) fn fd_filestat_set_size_internal( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + fd: WasiFd, + st_size: Filesize, +) -> Result<(), Errno> { let env = ctx.data(); let (_, mut state) = unsafe { env.get_memory_and_wasi_state(&ctx, 0) }; - let fd_entry = wasi_try!(state.fs.get_fd(fd)); + let fd_entry = state.fs.get_fd(fd)?; let inode = fd_entry.inode; if !fd_entry.rights.contains(Rights::FD_FILESTAT_SET_SIZE) { - return Errno::Access; + return Err(Errno::Access); } { @@ -29,22 +48,22 @@ pub fn fd_filestat_set_size( Kind::File { handle, .. } => { if let Some(handle) = handle { let mut handle = handle.write().unwrap(); - wasi_try!(handle.set_len(st_size).map_err(fs_error_into_wasi_err)); + handle.set_len(st_size).map_err(fs_error_into_wasi_err)?; } else { - return Errno::Badf; + return Err(Errno::Badf); } } Kind::Buffer { buffer } => { buffer.resize(st_size as usize, 0); } - Kind::Socket { .. } => return Errno::Badf, - Kind::Pipe { .. } => return Errno::Badf, - Kind::Symlink { .. } => return Errno::Badf, - Kind::EventNotifications { .. } | Kind::Epoll { .. } => return Errno::Badf, - Kind::Dir { .. } | Kind::Root { .. } => return Errno::Isdir, + Kind::Socket { .. } => return Err(Errno::Badf), + Kind::Pipe { .. } => return Err(Errno::Badf), + Kind::Symlink { .. } => return Err(Errno::Badf), + Kind::EventNotifications { .. } | Kind::Epoll { .. } => return Err(Errno::Badf), + Kind::Dir { .. } | Kind::Root { .. } => return Err(Errno::Isdir), } } inode.stat.write().unwrap().st_size = st_size; - Errno::Success + Ok(()) } diff --git a/lib/wasix/src/syscalls/wasi/fd_filestat_set_times.rs b/lib/wasix/src/syscalls/wasi/fd_filestat_set_times.rs index 5eea1e19ad6..0998286368a 100644 --- a/lib/wasix/src/syscalls/wasi/fd_filestat_set_times.rs +++ b/lib/wasix/src/syscalls/wasi/fd_filestat_set_times.rs @@ -12,24 +12,49 @@ use crate::syscalls::*; /// Bit-vector for controlling which times get set #[instrument(level = "debug", skip_all, fields(%fd, %st_atim, %st_mtim), ret)] pub fn fd_filestat_set_times( - ctx: FunctionEnvMut<'_, WasiEnv>, + mut ctx: FunctionEnvMut<'_, WasiEnv>, fd: WasiFd, st_atim: Timestamp, st_mtim: Timestamp, fst_flags: Fstflags, -) -> Errno { +) -> Result { + wasi_try_ok!(fd_filestat_set_times_internal( + &mut ctx, fd, st_atim, st_mtim, fst_flags + )); + let env = ctx.data(); + + #[cfg(feature = "snapshot")] + if env.enable_snapshot_capture { + SnapshotEffector::save_fd_set_times(&mut ctx, fd, st_atim, st_mtim, fst_flags).map_err( + |err| { + tracing::error!("failed to save file set times event - {}", err); + WasiError::Exit(ExitCode::Errno(Errno::Fault)) + }, + )?; + } + + Ok(Errno::Success) +} + +pub(crate) fn fd_filestat_set_times_internal( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + fd: WasiFd, + st_atim: Timestamp, + st_mtim: Timestamp, + fst_flags: Fstflags, +) -> Result<(), Errno> { let env = ctx.data(); let (_, mut state) = unsafe { env.get_memory_and_wasi_state(&ctx, 0) }; - let fd_entry = wasi_try!(state.fs.get_fd(fd)); + let fd_entry = state.fs.get_fd(fd)?; if !fd_entry.rights.contains(Rights::FD_FILESTAT_SET_TIMES) { - return Errno::Access; + return Err(Errno::Access); } if (fst_flags.contains(Fstflags::SET_ATIM) && fst_flags.contains(Fstflags::SET_ATIM_NOW)) || (fst_flags.contains(Fstflags::SET_MTIM) && fst_flags.contains(Fstflags::SET_MTIM_NOW)) { - return Errno::Inval; + return Err(Errno::Inval); } let inode = fd_entry.inode; @@ -38,7 +63,7 @@ pub fn fd_filestat_set_times( let time_to_set = if fst_flags.contains(Fstflags::SET_ATIM) { st_atim } else { - wasi_try!(get_current_time_in_nanos()) + get_current_time_in_nanos()? }; inode.stat.write().unwrap().st_atim = time_to_set; } @@ -47,10 +72,10 @@ pub fn fd_filestat_set_times( let time_to_set = if fst_flags.contains(Fstflags::SET_MTIM) { st_mtim } else { - wasi_try!(get_current_time_in_nanos()) + get_current_time_in_nanos()? }; inode.stat.write().unwrap().st_mtim = time_to_set; } - Errno::Success + Ok(()) } diff --git a/lib/wasix/src/syscalls/wasi/fd_renumber.rs b/lib/wasix/src/syscalls/wasi/fd_renumber.rs index 1305003f344..e15cc64d267 100644 --- a/lib/wasix/src/syscalls/wasi/fd_renumber.rs +++ b/lib/wasix/src/syscalls/wasi/fd_renumber.rs @@ -9,7 +9,31 @@ use crate::syscalls::*; /// - `Fd to` /// Location to copy file descriptor to #[instrument(level = "debug", skip_all, fields(%from, %to), ret)] -pub fn fd_renumber(ctx: FunctionEnvMut<'_, WasiEnv>, from: WasiFd, to: WasiFd) -> Errno { +pub fn fd_renumber( + mut ctx: FunctionEnvMut<'_, WasiEnv>, + from: WasiFd, + to: WasiFd, +) -> Result { + let ret = fd_renumber_internal(&mut ctx, from, to); + let env = ctx.data(); + + if ret == Errno::Success { + #[cfg(feature = "snapshot")] + if env.enable_snapshot_capture { + SnapshotEffector::save_fd_renumber(&mut ctx, from, to).map_err(|err| { + tracing::error!("failed to save file descriptor renumber event - {}", err); + WasiError::Exit(ExitCode::Errno(Errno::Fault)) + })?; + } + } + Ok(ret) +} + +pub(crate) fn fd_renumber_internal( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + from: WasiFd, + to: WasiFd, +) -> Errno { if from == to { return Errno::Success; } @@ -27,6 +51,7 @@ pub fn fd_renumber(ctx: FunctionEnvMut<'_, WasiEnv>, from: WasiFd, to: WasiFd) - ..*fd_entry }; fd_map.insert(to, new_fd_entry); + state.fs.make_max_fd(to + 1); Errno::Success } diff --git a/lib/wasix/src/syscalls/wasi/fd_seek.rs b/lib/wasix/src/syscalls/wasi/fd_seek.rs index 50391f9c5db..83d575b1a74 100644 --- a/lib/wasix/src/syscalls/wasi/fd_seek.rs +++ b/lib/wasix/src/syscalls/wasi/fd_seek.rs @@ -23,20 +23,51 @@ pub fn fd_seek( ) -> Result { wasi_try_ok!(WasiEnv::process_signals_and_exit(&mut ctx)?); + let new_offset = wasi_try_ok!(fd_seek_internal(&mut ctx, fd, offset, whence)?); + let env = ctx.data(); + + #[cfg(feature = "snapshot")] + if env.enable_snapshot_capture { + SnapshotEffector::save_fd_seek(&mut ctx, fd, offset, whence).map_err(|err| { + tracing::error!("failed to save file descriptor seek event - {}", err); + WasiError::Exit(ExitCode::Errno(Errno::Fault)) + })?; + } + + // reborrow + let env = ctx.data(); + let memory = unsafe { env.memory_view(&ctx) }; + let new_offset_ref = newoffset.deref(&memory); + let fd_entry = wasi_try_ok!(env.state.fs.get_fd(fd)); + wasi_try_mem_ok!(new_offset_ref.write(new_offset)); + + trace!( + %new_offset, + ); + + Ok(Errno::Success) +} + +pub(crate) fn fd_seek_internal( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + fd: WasiFd, + offset: FileDelta, + whence: Whence, +) -> Result, WasiError> { let env = ctx.data(); let state = env.state.clone(); let (memory, _) = unsafe { env.get_memory_and_wasi_state(&ctx, 0) }; - let fd_entry = wasi_try_ok!(state.fs.get_fd(fd)); + let fd_entry = wasi_try_ok_ok!(state.fs.get_fd(fd)); if !fd_entry.rights.contains(Rights::FD_SEEK) { - return Ok(Errno::Access); + return Ok(Err(Errno::Access)); } // TODO: handle case if fd is a dir? let new_offset = match whence { Whence::Cur => { let mut fd_map = state.fs.fd_map.write().unwrap(); - let fd_entry = wasi_try_ok!(fd_map.get_mut(&fd).ok_or(Errno::Badf)); + let fd_entry = wasi_try_ok_ok!(fd_map.get_mut(&fd).ok_or(Errno::Badf)); #[allow(clippy::comparison_chain)] if offset > 0 { @@ -62,7 +93,7 @@ pub fn fd_seek( let handle = handle.clone(); drop(guard); - wasi_try_ok!(__asyncify(&mut ctx, None, async move { + wasi_try_ok_ok!(__asyncify(ctx, None, async move { let mut handle = handle.write().unwrap(); let end = handle .seek(SeekFrom::End(offset)) @@ -77,7 +108,7 @@ pub fn fd_seek( Ok(()) })?); } else { - return Ok(Errno::Inval); + return Ok(Err(Errno::Inval)); } } Kind::Symlink { .. } => { @@ -90,34 +121,24 @@ pub fn fd_seek( | Kind::EventNotifications { .. } | Kind::Epoll { .. } => { // TODO: check this - return Ok(Errno::Inval); + return Ok(Err(Errno::Inval)); } Kind::Buffer { .. } => { // seeking buffers probably makes sense // FIXME: implement this - return Ok(Errno::Inval); + return Ok(Err(Errno::Inval)); } } fd_entry.offset.load(Ordering::Acquire) } Whence::Set => { let mut fd_map = state.fs.fd_map.write().unwrap(); - let fd_entry = wasi_try_ok!(fd_map.get_mut(&fd).ok_or(Errno::Badf)); + let fd_entry = wasi_try_ok_ok!(fd_map.get_mut(&fd).ok_or(Errno::Badf)); fd_entry.offset.store(offset as u64, Ordering::Release); offset as u64 } - _ => return Ok(Errno::Inval), + _ => return Ok(Err(Errno::Inval)), }; - // reborrow - let env = ctx.data(); - let memory = unsafe { env.memory_view(&ctx) }; - let new_offset_ref = newoffset.deref(&memory); - let fd_entry = wasi_try_ok!(env.state.fs.get_fd(fd)); - wasi_try_mem_ok!(new_offset_ref.write(new_offset)); - - trace!( - %new_offset, - ); - Ok(Errno::Success) + Ok(Ok(new_offset)) } diff --git a/lib/wasix/src/syscalls/wasi/fd_write.rs b/lib/wasix/src/syscalls/wasi/fd_write.rs index 0c5c7c99fe6..04f28380184 100644 --- a/lib/wasix/src/syscalls/wasi/fd_write.rs +++ b/lib/wasix/src/syscalls/wasi/fd_write.rs @@ -103,7 +103,7 @@ pub(crate) fn fd_write_internal( should_update_cursor: bool, should_snapshot: bool, ) -> Result { - wasi_try_ok!(WasiEnv::process_signals_and_exit(&mut ctx)?); + wasi_try_ok!(WasiEnv::process_signals_and_exit(ctx)?); let mut env = ctx.data(); let state = env.state.clone(); @@ -375,14 +375,14 @@ pub(crate) fn fd_write_internal( #[cfg(feature = "snapshot")] if should_snapshot && can_snapshot && bytes_written > 0 { if let FdWriteSource::Iovs { iovs, iovs_len } = data { - SnapshotEffector::save_write(&mut ctx, fd, offset, bytes_written, iovs, iovs_len) + SnapshotEffector::save_fd_write(ctx, fd, offset, bytes_written, iovs, iovs_len) .map_err(|err| { - tracing::error!( - "failed to save terminal data to snapshot capturer - {}", - err - ); - WasiError::Exit(ExitCode::Errno(Errno::Fault)) - })?; + tracing::error!( + "failed to save terminal data to snapshot capturer - {}", + err + ); + WasiError::Exit(ExitCode::Errno(Errno::Fault)) + })?; } } diff --git a/lib/wasix/src/syscalls/wasi/path_create_directory.rs b/lib/wasix/src/syscalls/wasi/path_create_directory.rs index 473bc0e1f72..401ba363e6b 100644 --- a/lib/wasix/src/syscalls/wasi/path_create_directory.rs +++ b/lib/wasix/src/syscalls/wasi/path_create_directory.rs @@ -16,25 +16,15 @@ use crate::syscalls::*; /// This right must be set on the directory that the file is created in (TODO: verify that this is true) #[instrument(level = "trace", skip_all, fields(%fd, path = field::Empty), ret)] pub fn path_create_directory( - ctx: FunctionEnvMut<'_, WasiEnv>, + mut ctx: FunctionEnvMut<'_, WasiEnv>, fd: WasiFd, path: WasmPtr, path_len: M::Offset, -) -> Errno { +) -> Result { let env = ctx.data(); let (memory, state, inodes) = unsafe { env.get_memory_and_wasi_state_and_inodes(&ctx, 0) }; - let working_dir = wasi_try!(state.fs.get_fd(fd)); - { - let guard = working_dir.inode.read(); - if let Kind::Root { .. } = guard.deref() { - return Errno::Access; - } - } - if !working_dir.rights.contains(Rights::PATH_CREATE_DIRECTORY) { - return Errno::Access; - } - let mut path_string = unsafe { get_input_str!(&memory, path, path_len) }; + let mut path_string = unsafe { get_input_str_ok!(&memory, path, path_len) }; Span::current().record("path", path_string.as_str()); // Convert relative paths into absolute paths @@ -45,8 +35,43 @@ pub fn path_create_directory( ); } - let path = std::path::PathBuf::from(&path_string); - let path_vec = wasi_try!(path + wasi_try_ok!(path_create_directory_internal(&mut ctx, fd, &path_string)); + let env = ctx.data(); + + #[cfg(feature = "snapshot")] + if env.enable_snapshot_capture { + SnapshotEffector::save_path_create_directory(&mut ctx, fd, path_string).map_err(|err| { + tracing::error!( + "failed to save create directory event to snapshot capturer - {}", + err + ); + WasiError::Exit(ExitCode::Errno(Errno::Fault)) + })?; + } + + Ok(Errno::Success) +} + +pub(crate) fn path_create_directory_internal( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + fd: WasiFd, + path: &str, +) -> Result<(), Errno> { + let env = ctx.data(); + let (memory, state, inodes) = unsafe { env.get_memory_and_wasi_state_and_inodes(&ctx, 0) }; + let working_dir = state.fs.get_fd(fd)?; + { + let guard = working_dir.inode.read(); + if let Kind::Root { .. } = guard.deref() { + return Err(Errno::Access); + } + } + if !working_dir.rights.contains(Rights::PATH_CREATE_DIRECTORY) { + return Err(Errno::Access); + } + + let path = std::path::PathBuf::from(path); + let path_vec = path .components() .map(|comp| { comp.as_os_str() @@ -54,9 +79,9 @@ pub fn path_create_directory( .map(|inner_str| inner_str.to_string()) .ok_or(Errno::Inval) }) - .collect::, Errno>>()); + .collect::, Errno>>()?; if path_vec.is_empty() { - return Errno::Inval; + return Err(Errno::Inval); } let mut cur_dir_inode = working_dir.inode; @@ -96,18 +121,19 @@ pub fn path_create_directory( &adjusted_path.to_string_lossy(), ) { if adjusted_path_stat.st_filetype != Filetype::Directory { - return Errno::Notdir; + return Err(Errno::Notdir); } } else { - wasi_try!(state.fs_create_dir(&adjusted_path)); + state.fs_create_dir(&adjusted_path)?; } let kind = Kind::Dir { parent: cur_dir_inode.downgrade(), path: adjusted_path, entries: Default::default(), }; - let new_inode = - wasi_try!(state.fs.create_inode(inodes, kind, false, comp.to_string())); + let new_inode = state + .fs + .create_inode(inodes, kind, false, comp.to_string())?; // reborrow to insert { @@ -122,10 +148,10 @@ pub fn path_create_directory( cur_dir_inode = new_inode; } } - Kind::Root { .. } => return Errno::Access, - _ => return Errno::Notdir, + Kind::Root { .. } => return Err(Errno::Access), + _ => return Err(Errno::Notdir), } } - Errno::Success + Ok(()) } diff --git a/lib/wasix/src/syscalls/wasi/path_filestat_set_times.rs b/lib/wasix/src/syscalls/wasi/path_filestat_set_times.rs index 93e3accc57a..ca119fa66d7 100644 --- a/lib/wasix/src/syscalls/wasi/path_filestat_set_times.rs +++ b/lib/wasix/src/syscalls/wasi/path_filestat_set_times.rs @@ -20,7 +20,7 @@ use crate::syscalls::*; /// A bitmask controlling which attributes are set #[instrument(level = "debug", skip_all, fields(%fd, path = field::Empty, %st_atim, %st_mtim), ret)] pub fn path_filestat_set_times( - ctx: FunctionEnvMut<'_, WasiEnv>, + mut ctx: FunctionEnvMut<'_, WasiEnv>, fd: WasiFd, flags: LookupFlags, path: WasmPtr, @@ -28,21 +28,11 @@ pub fn path_filestat_set_times( st_atim: Timestamp, st_mtim: Timestamp, fst_flags: Fstflags, -) -> Errno { +) -> Result { let env = ctx.data(); let (memory, mut state, inodes) = unsafe { env.get_memory_and_wasi_state_and_inodes(&ctx, 0) }; - let fd_entry = wasi_try!(state.fs.get_fd(fd)); - let fd_inode = fd_entry.inode; - if !fd_entry.rights.contains(Rights::PATH_FILESTAT_SET_TIMES) { - return Errno::Access; - } - if (fst_flags.contains(Fstflags::SET_ATIM) && fst_flags.contains(Fstflags::SET_ATIM_NOW)) - || (fst_flags.contains(Fstflags::SET_MTIM) && fst_flags.contains(Fstflags::SET_MTIM_NOW)) - { - return Errno::Inval; - } - let mut path_string = unsafe { get_input_str!(&memory, path, path_len) }; + let mut path_string = unsafe { get_input_str_ok!(&memory, path, path_len) }; Span::current().record("path", path_string.as_str()); // Convert relative paths into absolute paths @@ -53,22 +43,73 @@ pub fn path_filestat_set_times( ); } - let file_inode = wasi_try!(state.fs.get_inode_at_path( - inodes, + wasi_try_ok!(path_filestat_set_times_internal( + &mut ctx, fd, + flags, &path_string, - flags & __WASI_LOOKUP_SYMLINK_FOLLOW != 0, + st_atim, + st_mtim, + fst_flags )); + let env = ctx.data(); + + #[cfg(feature = "snapshot")] + if env.enable_snapshot_capture { + SnapshotEffector::save_path_set_times( + &mut ctx, + fd, + flags, + path_string, + st_atim, + st_mtim, + fst_flags, + ) + .map_err(|err| { + tracing::error!("failed to save file set times event - {}", err); + WasiError::Exit(ExitCode::Errno(Errno::Fault)) + })?; + } + + Ok(Errno::Success) +} + +pub(crate) fn path_filestat_set_times_internal( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + fd: WasiFd, + flags: LookupFlags, + path: &str, + st_atim: Timestamp, + st_mtim: Timestamp, + fst_flags: Fstflags, +) -> Result<(), Errno> { + let env = ctx.data(); + let (memory, mut state, inodes) = unsafe { env.get_memory_and_wasi_state_and_inodes(&ctx, 0) }; + let fd_entry = state.fs.get_fd(fd)?; + let fd_inode = fd_entry.inode; + if !fd_entry.rights.contains(Rights::PATH_FILESTAT_SET_TIMES) { + return Err(Errno::Access); + } + if (fst_flags.contains(Fstflags::SET_ATIM) && fst_flags.contains(Fstflags::SET_ATIM_NOW)) + || (fst_flags.contains(Fstflags::SET_MTIM) && fst_flags.contains(Fstflags::SET_MTIM_NOW)) + { + return Err(Errno::Inval); + } + + let file_inode = + state + .fs + .get_inode_at_path(inodes, fd, path, flags & __WASI_LOOKUP_SYMLINK_FOLLOW != 0)?; let stat = { let guard = file_inode.read(); - wasi_try!(state.fs.get_stat_for_kind(guard.deref())) + state.fs.get_stat_for_kind(guard.deref())? }; if fst_flags.contains(Fstflags::SET_ATIM) || fst_flags.contains(Fstflags::SET_ATIM_NOW) { let time_to_set = if fst_flags.contains(Fstflags::SET_ATIM) { st_atim } else { - wasi_try!(get_current_time_in_nanos()) + get_current_time_in_nanos()? }; fd_inode.stat.write().unwrap().st_atim = time_to_set; } @@ -76,10 +117,10 @@ pub fn path_filestat_set_times( let time_to_set = if fst_flags.contains(Fstflags::SET_MTIM) { st_mtim } else { - wasi_try!(get_current_time_in_nanos()) + get_current_time_in_nanos()? }; fd_inode.stat.write().unwrap().st_mtim = time_to_set; } - Errno::Success + Ok(()) } diff --git a/lib/wasix/src/syscalls/wasi/path_link.rs b/lib/wasix/src/syscalls/wasi/path_link.rs index 56ce45eafa1..06dea90a68a 100644 --- a/lib/wasix/src/syscalls/wasi/path_link.rs +++ b/lib/wasix/src/syscalls/wasi/path_link.rs @@ -20,7 +20,7 @@ use crate::syscalls::*; /// Length of the `new_path` string #[instrument(level = "debug", skip_all, fields(%old_fd, %new_fd, old_path = field::Empty, new_path = field::Empty, follow_symlinks = false), ret)] pub fn path_link( - ctx: FunctionEnvMut<'_, WasiEnv>, + mut ctx: FunctionEnvMut<'_, WasiEnv>, old_fd: WasiFd, old_flags: LookupFlags, old_path: WasmPtr, @@ -28,64 +28,112 @@ pub fn path_link( new_fd: WasiFd, new_path: WasmPtr, new_path_len: M::Offset, -) -> Errno { +) -> Result { if old_flags & __WASI_LOOKUP_SYMLINK_FOLLOW != 0 { Span::current().record("follow_symlinks", true); } let env = ctx.data(); let (memory, mut state, inodes) = unsafe { env.get_memory_and_wasi_state_and_inodes(&ctx, 0) }; - let mut old_path_str = unsafe { get_input_str!(&memory, old_path, old_path_len) }; + let mut old_path_str = unsafe { get_input_str_ok!(&memory, old_path, old_path_len) }; Span::current().record("old_path", old_path_str.as_str()); - let mut new_path_str = unsafe { get_input_str!(&memory, new_path, new_path_len) }; + let mut new_path_str = unsafe { get_input_str_ok!(&memory, new_path, new_path_len) }; Span::current().record("new_path", new_path_str.as_str()); - let source_fd = wasi_try!(state.fs.get_fd(old_fd)); - let target_fd = wasi_try!(state.fs.get_fd(new_fd)); + + wasi_try_ok!(path_link_internal( + &mut ctx, + old_fd, + old_flags, + &old_path_str, + new_fd, + &new_path_str + )); + let env = ctx.data(); + + #[cfg(feature = "snapshot")] + if env.enable_snapshot_capture { + SnapshotEffector::save_path_link( + &mut ctx, + old_fd, + old_flags, + old_path_str, + new_fd, + new_path_str, + ) + .map_err(|err| { + tracing::error!("failed to save path hard link event - {}", err); + WasiError::Exit(ExitCode::Errno(Errno::Fault)) + })?; + } + + Ok(Errno::Success) +} + +pub(crate) fn path_link_internal( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + old_fd: WasiFd, + old_flags: LookupFlags, + old_path: &str, + new_fd: WasiFd, + new_path: &str, +) -> Result<(), Errno> { + let env = ctx.data(); + let (memory, mut state, inodes) = unsafe { env.get_memory_and_wasi_state_and_inodes(&ctx, 0) }; + let source_fd = state.fs.get_fd(old_fd)?; + let target_fd = state.fs.get_fd(new_fd)?; if !source_fd.rights.contains(Rights::PATH_LINK_SOURCE) || !target_fd.rights.contains(Rights::PATH_LINK_TARGET) { - return Errno::Access; + return Err(Errno::Access); } // Convert relative paths into absolute paths - old_path_str = ctx.data().state.fs.relative_path_to_absolute(old_path_str); - new_path_str = ctx.data().state.fs.relative_path_to_absolute(new_path_str); + let old_path_str = ctx + .data() + .state + .fs + .relative_path_to_absolute(old_path.to_string()); + let new_path_str = ctx + .data() + .state + .fs + .relative_path_to_absolute(new_path.to_string()); - let source_inode = wasi_try!(state.fs.get_inode_at_path( + let source_inode = state.fs.get_inode_at_path( inodes, old_fd, &old_path_str, old_flags & __WASI_LOOKUP_SYMLINK_FOLLOW != 0, - )); + )?; let target_path_arg = std::path::PathBuf::from(&new_path_str); let (target_parent_inode, new_entry_name) = - wasi_try!(state + state .fs - .get_parent_inode_at_path(inodes, new_fd, &target_path_arg, false)); + .get_parent_inode_at_path(inodes, new_fd, &target_path_arg, false)?; if source_inode.stat.write().unwrap().st_nlink == Linkcount::max_value() { - return Errno::Mlink; + return Err(Errno::Mlink); } { let mut guard = target_parent_inode.write(); match guard.deref_mut() { Kind::Dir { entries, .. } => { if entries.contains_key(&new_entry_name) { - return Errno::Exist; + return Err(Errno::Exist); } entries.insert(new_entry_name, source_inode.clone()); } - Kind::Root { .. } => return Errno::Inval, + Kind::Root { .. } => return Err(Errno::Inval), Kind::File { .. } | Kind::Symlink { .. } | Kind::Buffer { .. } | Kind::Socket { .. } | Kind::Pipe { .. } | Kind::EventNotifications { .. } - | Kind::Epoll { .. } => return Errno::Notdir, + | Kind::Epoll { .. } => return Err(Errno::Notdir), } } source_inode.stat.write().unwrap().st_nlink += 1; - Errno::Success + Ok(()) } diff --git a/lib/wasix/src/syscalls/wasi/path_open.rs b/lib/wasix/src/syscalls/wasi/path_open.rs index cb314fd0939..fa88aecc6ab 100644 --- a/lib/wasix/src/syscalls/wasi/path_open.rs +++ b/lib/wasix/src/syscalls/wasi/path_open.rs @@ -27,7 +27,7 @@ use crate::syscalls::*; /// - `Errno::Access`, `Errno::Badf`, `Errno::Fault`, `Errno::Fbig?`, `Errno::Inval`, `Errno::Io`, `Errno::Loop`, `Errno::Mfile`, `Errno::Nametoolong?`, `Errno::Nfile`, `Errno::Noent`, `Errno::Notdir`, `Errno::Rofs`, and `Errno::Notcapable` #[instrument(level = "debug", skip_all, fields(%dirfd, path = field::Empty, follow_symlinks = field::Empty, ret_fd = field::Empty), ret)] pub fn path_open( - ctx: FunctionEnvMut<'_, WasiEnv>, + mut ctx: FunctionEnvMut<'_, WasiEnv>, dirfd: WasiFd, dirflags: LookupFlags, path: WasmPtr, @@ -37,7 +37,7 @@ pub fn path_open( fs_rights_inheriting: Rights, fs_flags: Fdflags, fd: WasmPtr, -) -> Errno { +) -> Result { if dirflags & __WASI_LOOKUP_SYMLINK_FOLLOW != 0 { Span::current().record("follow_symlinks", true); } @@ -47,26 +47,16 @@ pub fn path_open( /* TODO: find actual upper bound on name size (also this is a path, not a name :think-fish:) */ let path_len64: u64 = path_len.into(); if path_len64 > 1024u64 * 1024u64 { - return Errno::Nametoolong; + return Ok(Errno::Nametoolong); } - let fd_ref = fd.deref(&memory); - // o_flags: // - __WASI_O_CREAT (create if it does not exist) // - __WASI_O_DIRECTORY (fail if not dir) // - __WASI_O_EXCL (fail if file exists) // - __WASI_O_TRUNC (truncate size to 0) - let working_dir = wasi_try!(state.fs.get_fd(dirfd)); - let working_dir_rights_inheriting = working_dir.rights_inheriting; - - // ASSUMPTION: open rights apply recursively - if !working_dir.rights.contains(Rights::PATH_OPEN) { - return Errno::Access; - } - - let mut path_string = unsafe { get_input_str!(&memory, path, path_len) }; + let mut path_string = unsafe { get_input_str_ok!(&memory, path, path_len) }; Span::current().record("path", path_string.as_str()); // Convert relative paths into absolute paths @@ -77,14 +67,80 @@ pub fn path_open( ); } - let path_arg = std::path::PathBuf::from(&path_string); + let out_fd = wasi_try_ok!(path_open_internal::( + &mut ctx, + dirfd, + dirflags, + &path_string, + o_flags, + fs_rights_base, + fs_rights_inheriting, + fs_flags, + )?); + let env = ctx.data(); + + #[cfg(feature = "snapshot")] + if env.enable_snapshot_capture { + SnapshotEffector::save_path_open( + &mut ctx, + out_fd, + dirfd, + dirflags, + path_string, + o_flags, + fs_rights_base, + fs_rights_inheriting, + fs_flags, + M::is_64bit(), + ) + .map_err(|err| { + tracing::error!("failed to save unlink event to snapshot capturer - {}", err); + WasiError::Exit(ExitCode::Errno(Errno::Fault)) + })?; + } + + let env = ctx.data(); + let (memory, mut state, mut inodes) = + unsafe { env.get_memory_and_wasi_state_and_inodes(&ctx, 0) }; + + Span::current().record("ret_fd", out_fd); + + let fd_ref = fd.deref(&memory); + wasi_try_mem_ok!(fd_ref.write(out_fd)); + + Ok(Errno::Success) +} + +pub(crate) fn path_open_internal( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + dirfd: WasiFd, + dirflags: LookupFlags, + path: &str, + o_flags: Oflags, + fs_rights_base: Rights, + fs_rights_inheriting: Rights, + fs_flags: Fdflags, +) -> Result, WasiError> { + let env = ctx.data(); + let (memory, mut state, mut inodes) = + unsafe { env.get_memory_and_wasi_state_and_inodes(&ctx, 0) }; + + let path_arg = std::path::PathBuf::from(&path); let maybe_inode = state.fs.get_inode_at_path( inodes, dirfd, - &path_string, + &path, dirflags & __WASI_LOOKUP_SYMLINK_FOLLOW != 0, ); + let working_dir = wasi_try_ok_ok!(state.fs.get_fd(dirfd)); + let working_dir_rights_inheriting = working_dir.rights_inheriting; + + // ASSUMPTION: open rights apply recursively + if !working_dir.rights.contains(Rights::PATH_OPEN) { + return Ok(Err(Errno::Access)); + } + let mut open_flags = 0; // TODO: traverse rights of dirs properly // COMMENTED OUT: WASI isn't giving appropriate rights here when opening @@ -160,14 +216,13 @@ pub fn path_open( if let Some(special_fd) = fd { // short circuit if we're dealing with a special file assert!(handle.is_some()); - wasi_try_mem!(fd_ref.write(*special_fd)); - return Errno::Success; + return Ok(Ok(*special_fd)); } if o_flags.contains(Oflags::DIRECTORY) { - return Errno::Notdir; + return Ok(Err(Errno::Notdir)); } if o_flags.contains(Oflags::EXCL) { - return Errno::Exist; + return Ok(Err(Errno::Exist)); } let open_options = open_options @@ -188,31 +243,30 @@ pub fn path_open( if minimum_rights.truncate { open_flags |= Fd::TRUNCATE; } - *handle = Some(Arc::new(std::sync::RwLock::new(wasi_try!(open_options - .open(&path) - .map_err(fs_error_into_wasi_err))))); + *handle = Some(Arc::new(std::sync::RwLock::new(wasi_try_ok_ok!( + open_options.open(&path).map_err(fs_error_into_wasi_err) + )))); if let Some(handle) = handle { let handle = handle.read().unwrap(); if let Some(fd) = handle.get_special_fd() { // We clone the file descriptor so that when its closed // nothing bad happens - let dup_fd = wasi_try!(state.fs.clone_fd(fd)); + let dup_fd = wasi_try_ok_ok!(state.fs.clone_fd(fd)); trace!( %dup_fd ); // some special files will return a constant FD rather than // actually open the file (/dev/stdin, /dev/stdout, /dev/stderr) - wasi_try_mem!(fd_ref.write(dup_fd)); - return Errno::Success; + return Ok(Ok(dup_fd)); } } } Kind::Buffer { .. } => unimplemented!("wasi::path_open for Buffer type files"), Kind::Root { .. } => { if !o_flags.contains(Oflags::DIRECTORY) { - return Errno::Notcapable; + return Ok(Err(Errno::Notcapable)); } } Kind::Dir { .. } @@ -235,16 +289,17 @@ pub fn path_open( // less-happy path, we have to try to create the file if o_flags.contains(Oflags::CREATE) { if o_flags.contains(Oflags::DIRECTORY) { - return Errno::Notdir; + return Ok(Err(Errno::Notdir)); } // strip end file name - let (parent_inode, new_entity_name) = wasi_try!(state.fs.get_parent_inode_at_path( - inodes, - dirfd, - &path_arg, - dirflags & __WASI_LOOKUP_SYMLINK_FOLLOW != 0 - )); + let (parent_inode, new_entity_name) = + wasi_try_ok_ok!(state.fs.get_parent_inode_at_path( + inodes, + dirfd, + &path_arg, + dirflags & __WASI_LOOKUP_SYMLINK_FOLLOW != 0 + )); let new_file_host_path = { let guard = parent_inode.read(); match guard.deref() { @@ -258,7 +313,7 @@ pub fn path_open( new_path.push(&new_entity_name); new_path } - _ => return Errno::Inval, + _ => return Ok(Err(Errno::Inval)), } }; // once we got the data we need from the parent, we lookup the host file @@ -283,7 +338,7 @@ pub fn path_open( open_flags |= Fd::TRUNCATE; } - Some(wasi_try!(open_options + Some(wasi_try_ok_ok!(open_options .open(&new_file_host_path) .map_err(|e| { fs_error_into_wasi_err(e) }))) }; @@ -294,7 +349,7 @@ pub fn path_open( path: new_file_host_path, fd: None, }; - wasi_try!(state + wasi_try_ok_ok!(state .fs .create_inode(inodes, kind, false, new_entity_name.clone())) }; @@ -311,13 +366,13 @@ pub fn path_open( new_inode } else { - return maybe_inode.unwrap_err(); + return Ok(Err(maybe_inode.unwrap_err())); } }; // TODO: check and reduce these // TODO: ensure a mutable fd to root can never be opened - let out_fd = wasi_try!(state.fs.create_fd( + let out_fd = wasi_try_ok_ok!(state.fs.create_fd( adjusted_rights, fs_rights_inheriting, fs_flags, @@ -325,8 +380,5 @@ pub fn path_open( inode )); - Span::current().record("ret_fd", out_fd); - - wasi_try_mem!(fd_ref.write(out_fd)); - Errno::Success + Ok(Ok(out_fd)) } diff --git a/lib/wasix/src/syscalls/wasi/path_remove_directory.rs b/lib/wasix/src/syscalls/wasi/path_remove_directory.rs index 328bd63016f..d03e5a55e94 100644 --- a/lib/wasix/src/syscalls/wasi/path_remove_directory.rs +++ b/lib/wasix/src/syscalls/wasi/path_remove_directory.rs @@ -31,7 +31,7 @@ pub fn path_remove_directory( #[cfg(feature = "snapshot")] if env.enable_snapshot_capture { wasi_try!( - SnapshotEffector::save_remove_directory(&mut ctx, fd, path_str).map_err(|err| { + SnapshotEffector::save_path_remove_directory(&mut ctx, fd, path_str).map_err(|err| { tracing::error!("failed to save unlink event to snapshot capturer - {}", err); Errno::Fault }) diff --git a/lib/wasix/src/syscalls/wasi/path_rename.rs b/lib/wasix/src/syscalls/wasi/path_rename.rs index e2e9b26e4b5..0b1954c25b7 100644 --- a/lib/wasix/src/syscalls/wasi/path_rename.rs +++ b/lib/wasix/src/syscalls/wasi/path_rename.rs @@ -41,13 +41,11 @@ pub fn path_rename( if ret == Errno::Success { #[cfg(feature = "snapshot")] if env.enable_snapshot_capture { - wasi_try_ok!(SnapshotEffector::save_rename( - &mut ctx, old_fd, source_str, new_fd, target_str - ) - .map_err(|err| { - tracing::error!("failed to save unlink event to snapshot capturer - {}", err); - Errno::Fault - })) + SnapshotEffector::save_path_rename(&mut ctx, old_fd, source_str, new_fd, target_str) + .map_err(|err| { + tracing::error!("failed to save unlink event to snapshot capturer - {}", err); + WasiError::Exit(ExitCode::Errno(Errno::Fault)) + })?; } } Ok(ret) diff --git a/lib/wasix/src/syscalls/wasi/path_symlink.rs b/lib/wasix/src/syscalls/wasi/path_symlink.rs index e39619d57a4..b91d2beb9bc 100644 --- a/lib/wasix/src/syscalls/wasi/path_symlink.rs +++ b/lib/wasix/src/syscalls/wasi/path_symlink.rs @@ -16,32 +16,62 @@ use crate::syscalls::*; /// The number of bytes to read from `new_path` #[instrument(level = "debug", skip_all, fields(%fd, old_path = field::Empty, new_path = field::Empty), ret)] pub fn path_symlink( - ctx: FunctionEnvMut<'_, WasiEnv>, + mut ctx: FunctionEnvMut<'_, WasiEnv>, old_path: WasmPtr, old_path_len: M::Offset, fd: WasiFd, new_path: WasmPtr, new_path_len: M::Offset, -) -> Errno { +) -> Result { let env = ctx.data(); let (memory, mut state, inodes) = unsafe { env.get_memory_and_wasi_state_and_inodes(&ctx, 0) }; - let mut old_path_str = unsafe { get_input_str!(&memory, old_path, old_path_len) }; + let mut old_path_str = unsafe { get_input_str_ok!(&memory, old_path, old_path_len) }; Span::current().record("old_path", old_path_str.as_str()); - let mut new_path_str = unsafe { get_input_str!(&memory, new_path, new_path_len) }; + let mut new_path_str = unsafe { get_input_str_ok!(&memory, new_path, new_path_len) }; Span::current().record("new_path", new_path_str.as_str()); old_path_str = ctx.data().state.fs.relative_path_to_absolute(old_path_str); new_path_str = ctx.data().state.fs.relative_path_to_absolute(new_path_str); - let base_fd = wasi_try!(state.fs.get_fd(fd)); + + wasi_try_ok!(path_symlink_internal( + &mut ctx, + &old_path_str, + fd, + &new_path_str + )); + let env = ctx.data(); + + #[cfg(feature = "snapshot")] + if env.enable_snapshot_capture { + SnapshotEffector::save_path_symlink(&mut ctx, old_path_str, fd, new_path_str).map_err( + |err| { + tracing::error!("failed to save path symbolic link event - {}", err); + WasiError::Exit(ExitCode::Errno(Errno::Fault)) + }, + )?; + } + + Ok(Errno::Success) +} + +pub fn path_symlink_internal( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + old_path: &str, + fd: WasiFd, + new_path: &str, +) -> Result<(), Errno> { + let env = ctx.data(); + let (memory, mut state, inodes) = unsafe { env.get_memory_and_wasi_state_and_inodes(&ctx, 0) }; + + let base_fd = state.fs.get_fd(fd)?; if !base_fd.rights.contains(Rights::PATH_SYMLINK) { - return Errno::Access; + return Err(Errno::Access); } // get the depth of the parent + 1 (UNDER INVESTIGATION HMMMMMMMM THINK FISH ^ THINK FISH) - let old_path_path = std::path::Path::new(&old_path_str); - let (source_inode, _) = - wasi_try!(state - .fs - .get_parent_inode_at_path(inodes, fd, old_path_path, true)); + let old_path_path = std::path::Path::new(old_path); + let (source_inode, _) = state + .fs + .get_parent_inode_at_path(inodes, fd, old_path_path, true)?; let depth = state.fs.path_depth_from_fd(fd, source_inode); // depth == -1 means folder is not relative. See issue #3233. @@ -50,11 +80,11 @@ pub fn path_symlink( Err(_) => -1, }; - let new_path_path = std::path::Path::new(&new_path_str); + let new_path_path = std::path::Path::new(new_path); let (target_parent_inode, entry_name) = - wasi_try!(state + state .fs - .get_parent_inode_at_path(inodes, fd, new_path_path, true)); + .get_parent_inode_at_path(inodes, fd, new_path_path, true)?; // short circuit if anything is wrong, before we create an inode { @@ -62,21 +92,21 @@ pub fn path_symlink( match guard.deref() { Kind::Dir { entries, .. } => { if entries.contains_key(&entry_name) { - return Errno::Exist; + return Err(Errno::Exist); } } - Kind::Root { .. } => return Errno::Notcapable, + Kind::Root { .. } => return Err(Errno::Notcapable), Kind::Socket { .. } | Kind::Pipe { .. } | Kind::EventNotifications { .. } - | Kind::Epoll { .. } => return Errno::Inval, + | Kind::Epoll { .. } => return Err(Errno::Inval), Kind::File { .. } | Kind::Symlink { .. } | Kind::Buffer { .. } => { unreachable!("get_parent_inode_at_path returned something other than a Dir or Root") } } } - let mut source_path = std::path::Path::new(&old_path_str); + let mut source_path = std::path::Path::new(old_path); let mut relative_path = std::path::PathBuf::new(); for _ in 0..depth { relative_path.push(".."); @@ -85,7 +115,7 @@ pub fn path_symlink( let kind = Kind::Symlink { base_po_dir: fd, - path_to_symlink: std::path::PathBuf::from(new_path_str), + path_to_symlink: std::path::PathBuf::from(new_path), relative_path, }; let new_inode = @@ -103,5 +133,5 @@ pub fn path_symlink( } } - Errno::Success + Ok(()) } diff --git a/lib/wasix/src/syscalls/wasi/path_unlink_file.rs b/lib/wasix/src/syscalls/wasi/path_unlink_file.rs index 2e9a44682ed..ad05a3fbd00 100644 --- a/lib/wasix/src/syscalls/wasi/path_unlink_file.rs +++ b/lib/wasix/src/syscalls/wasi/path_unlink_file.rs @@ -39,7 +39,7 @@ pub fn path_unlink_file( #[cfg(feature = "snapshot")] if env.enable_snapshot_capture { wasi_try_ok!( - SnapshotEffector::save_unlink_file(&mut ctx, fd, path_str).map_err(|err| { + SnapshotEffector::save_path_unlink(&mut ctx, fd, path_str).map_err(|err| { tracing::error!("failed to save unlink event to snapshot capturer - {}", err); Errno::Fault }) diff --git a/lib/wasix/src/syscalls/wasix/chdir.rs b/lib/wasix/src/syscalls/wasix/chdir.rs index 2816c8c7ee8..b842434f8f9 100644 --- a/lib/wasix/src/syscalls/wasix/chdir.rs +++ b/lib/wasix/src/syscalls/wasix/chdir.rs @@ -5,20 +5,38 @@ use crate::syscalls::*; /// Sets the current working directory #[instrument(level = "debug", skip_all, fields(name = field::Empty), ret)] pub fn chdir( - ctx: FunctionEnvMut<'_, WasiEnv>, + mut ctx: FunctionEnvMut<'_, WasiEnv>, path: WasmPtr, path_len: M::Offset, -) -> Errno { +) -> Result { let env = ctx.data(); let (memory, mut state) = unsafe { env.get_memory_and_wasi_state(&ctx, 0) }; - let path = unsafe { get_input_str!(&memory, path, path_len) }; + let path = unsafe { get_input_str_ok!(&memory, path, path_len) }; Span::current().record("path", path.as_str()); + wasi_try_ok!(chdir_internal(&mut ctx, &path,)); + let env = ctx.data(); + + #[cfg(feature = "snapshot")] + if env.enable_snapshot_capture { + SnapshotEffector::save_chdir(&mut ctx, path).map_err(|err| { + tracing::error!("failed to chdir event - {}", err); + WasiError::Exit(ExitCode::Errno(Errno::Fault)) + })?; + } + + Ok(Errno::Success) +} + +pub fn chdir_internal(ctx: &mut FunctionEnvMut<'_, WasiEnv>, path: &str) -> Result<(), Errno> { + let env = ctx.data(); + let (memory, mut state) = unsafe { env.get_memory_and_wasi_state(ctx, 0) }; + // Check if the directory exists - if state.fs.root_fs.read_dir(Path::new(path.as_str())).is_err() { - return Errno::Noent; + if state.fs.root_fs.read_dir(Path::new(path)).is_err() { + return Err(Errno::Noent); } - state.fs.set_current_dir(path.as_str()); - Errno::Success + state.fs.set_current_dir(path); + Ok(()) } diff --git a/lib/wasix/src/syscalls/wasix/epoll_create.rs b/lib/wasix/src/syscalls/wasix/epoll_create.rs index 9296f41792e..92e6ee55261 100644 --- a/lib/wasix/src/syscalls/wasix/epoll_create.rs +++ b/lib/wasix/src/syscalls/wasix/epoll_create.rs @@ -18,7 +18,30 @@ pub fn epoll_create( mut ctx: FunctionEnvMut<'_, WasiEnv>, ret_fd: WasmPtr, ) -> Result { - wasi_try_ok!(WasiEnv::process_signals_and_exit(&mut ctx)?); + let fd = wasi_try_ok!(epoll_create_internal(&mut ctx)?); + let env = ctx.data(); + + #[cfg(feature = "snapshot")] + if env.enable_snapshot_capture { + SnapshotEffector::save_epoll_create(&mut ctx, fd).map_err(|err| { + tracing::error!("failed to save epoll_create event - {}", err); + WasiError::Exit(ExitCode::Errno(Errno::Fault)) + })?; + } + + Span::current().record("fd", fd); + + let env = ctx.data(); + let (memory, state, inodes) = unsafe { env.get_memory_and_wasi_state_and_inodes(&ctx, 0) }; + wasi_try_mem_ok!(ret_fd.write(&memory, fd)); + + Ok(Errno::Success) +} + +pub fn epoll_create_internal( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, +) -> Result, WasiError> { + wasi_try_ok_ok!(WasiEnv::process_signals_and_exit(ctx)?); let env = ctx.data(); let (memory, state, inodes) = unsafe { env.get_memory_and_wasi_state_and_inodes(&ctx, 0) }; @@ -37,12 +60,9 @@ pub fn epoll_create( ); let rights = Rights::POLL_FD_READWRITE | Rights::FD_FDSTAT_SET_FLAGS; - let fd = wasi_try_ok!(state + let fd = wasi_try_ok_ok!(state .fs .create_fd(rights, rights, Fdflags::empty(), 0, inode)); - Span::current().record("fd", fd); - - wasi_try_mem_ok!(ret_fd.write(&memory, fd)); - Ok(Errno::Success) + Ok(Ok(fd)) } diff --git a/lib/wasix/src/syscalls/wasix/epoll_ctl.rs b/lib/wasix/src/syscalls/wasix/epoll_ctl.rs index 16946c5c5cc..7aee40c2fde 100644 --- a/lib/wasix/src/syscalls/wasix/epoll_ctl.rs +++ b/lib/wasix/src/syscalls/wasix/epoll_ctl.rs @@ -3,7 +3,7 @@ use tokio::sync::{mpsc::UnboundedSender, watch}; use virtual_mio::{InterestHandler, InterestType}; use virtual_net::net_error_into_io_err; use wasmer_wasix_types::wasi::{ - EpollCtl, EpollEvent, EpollType, SubscriptionClock, SubscriptionUnion, Userdata, + EpollCtl, EpollEvent, EpollEventCtl, EpollType, SubscriptionClock, SubscriptionUnion, Userdata, }; use std::task::{Context, Poll, RawWaker, RawWakerVTable, Waker}; @@ -43,7 +43,43 @@ pub fn epoll_ctl( None }; - let fd_entry = wasi_try_ok!(env.state.fs.get_fd(epfd)); + let event_ctl = event.map(|evt| EpollEventCtl { + events: evt.events, + ptr: evt.data.ptr.into(), + fd: evt.data.fd, + data1: evt.data.data1, + data2: evt.data.data2, + }); + + wasi_try_ok!(epoll_ctl_internal( + &mut ctx, + epfd, + op, + fd, + event_ctl.as_ref() + )?); + let env = ctx.data(); + + #[cfg(feature = "snapshot")] + if env.enable_snapshot_capture { + SnapshotEffector::save_epoll_ctl(&mut ctx, epfd, op, fd, event_ctl).map_err(|err| { + tracing::error!("failed to save epoll_create event - {}", err); + WasiError::Exit(ExitCode::Errno(Errno::Fault)) + })?; + } + + Ok(Errno::Success) +} + +pub(crate) fn epoll_ctl_internal( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + epfd: WasiFd, + op: EpollCtl, + fd: WasiFd, + event_ctl: Option<&EpollEventCtl>, +) -> Result, WasiError> { + let env = ctx.data(); + let fd_entry = wasi_try_ok_ok!(env.state.fs.get_fd(epfd)); let tasks = env.tasks().clone(); let mut inode_guard = fd_entry.inode.read(); @@ -58,22 +94,22 @@ pub fn epoll_ctl( tracing::trace!(fd, "unregistering waker"); } if let EpollCtl::Add | EpollCtl::Mod = op { - if let Some(event) = event { + if let Some(event) = event_ctl { let epoll_fd = EpollFd { events: event.events, - ptr: wasi_try_ok!(event.data.ptr.try_into().map_err(|_| Errno::Overflow)), - fd: event.data.fd, - data1: event.data.data1, - data2: event.data.data2, + ptr: wasi_try_ok_ok!(event.ptr.try_into().map_err(|_| Errno::Overflow)), + fd: event.fd, + data1: event.data1, + data2: event.data2, }; // Output debug tracing::trace!( peb = ?event.events, - ptr = ?event.data.ptr, - data1 = event.data.data1, - data2 = event.data.data2, - fd = event.data.fd, + ptr = ?event.ptr, + data1 = event.data1, + data2 = event.data2, + fd = event.fd, "registering waker" ); @@ -81,24 +117,24 @@ pub fn epoll_ctl( // We have to register the subscription before we register the waker // as otherwise there is a race condition let mut guard = subscriptions.lock().unwrap(); - guard.insert(event.data.fd, (epoll_fd.clone(), Vec::new())); + guard.insert(event.fd, (epoll_fd.clone(), Vec::new())); } // Now we register the epoll waker let tx = tx.clone(); let mut fd_guards = - wasi_try_ok!(register_epoll_waker(&env.state, &epoll_fd, tx)); + wasi_try_ok_ok!(register_epoll_waker(&env.state, &epoll_fd, tx)); // After the guards are created we need to attach them to the subscription let mut guard = subscriptions.lock().unwrap(); - if let Some(subs) = guard.get_mut(&event.data.fd) { + if let Some(subs) = guard.get_mut(&event.fd) { subs.1.append(&mut fd_guards); } } } - Ok(Errno::Success) + Ok(Ok(())) } - _ => Ok(Errno::Inval), + _ => Ok(Err(Errno::Inval)), } } diff --git a/lib/wasix/src/syscalls/wasix/fd_pipe.rs b/lib/wasix/src/syscalls/wasix/fd_pipe.rs index 6e74b37a6d2..bf80ffc4165 100644 --- a/lib/wasix/src/syscalls/wasix/fd_pipe.rs +++ b/lib/wasix/src/syscalls/wasix/fd_pipe.rs @@ -12,13 +12,35 @@ use crate::syscalls::*; /// Second file handle that represents the other end of the pipe #[instrument(level = "trace", skip_all, fields(fd1 = field::Empty, fd2 = field::Empty), ret)] pub fn fd_pipe( - ctx: FunctionEnvMut<'_, WasiEnv>, + mut ctx: FunctionEnvMut<'_, WasiEnv>, ro_fd1: WasmPtr, ro_fd2: WasmPtr, -) -> Errno { +) -> Result { + let (fd1, fd2) = wasi_try_ok!(fd_pipe_internal(&mut ctx)); + let env = ctx.data(); + + #[cfg(feature = "snapshot")] + if env.enable_snapshot_capture { + SnapshotEffector::save_fd_pipe(&mut ctx, fd1, fd2).map_err(|err| { + tracing::error!("failed to save create pipe event - {}", err); + WasiError::Exit(ExitCode::Errno(Errno::Fault)) + })?; + } + let env = ctx.data(); let (memory, state, inodes) = unsafe { env.get_memory_and_wasi_state_and_inodes(&ctx, 0) }; + Span::current().record("fd1", fd1).record("fd2", fd2); + + wasi_try_mem_ok!(ro_fd1.write(&memory, fd1)); + wasi_try_mem_ok!(ro_fd2.write(&memory, fd2)); + + Ok(Errno::Success) +} + +pub fn fd_pipe_internal(ctx: &mut FunctionEnvMut<'_, WasiEnv>) -> Result<(WasiFd, WasiFd), Errno> { + let env = ctx.data(); + let (memory, state, inodes) = unsafe { env.get_memory_and_wasi_state_and_inodes(&ctx, 0) }; let (pipe1, pipe2) = Pipe::channel(); let inode1 = state.fs.create_inode_with_default_stat( @@ -41,16 +63,12 @@ pub fn fd_pipe( | Rights::POLL_FD_READWRITE | Rights::SOCK_SEND | Rights::FD_FDSTAT_SET_FLAGS; - let fd1 = wasi_try!(state + let fd1 = state .fs - .create_fd(rights, rights, Fdflags::empty(), 0, inode1)); - let fd2 = wasi_try!(state + .create_fd(rights, rights, Fdflags::empty(), 0, inode1)?; + let fd2 = state .fs - .create_fd(rights, rights, Fdflags::empty(), 0, inode2)); - Span::current().record("fd1", fd1).record("fd2", fd2); - - wasi_try_mem!(ro_fd1.write(&memory, fd1)); - wasi_try_mem!(ro_fd2.write(&memory, fd2)); + .create_fd(rights, rights, Fdflags::empty(), 0, inode2)?; - Errno::Success + Ok((fd1, fd2)) } diff --git a/lib/wasix/src/syscalls/wasix/tty_set.rs b/lib/wasix/src/syscalls/wasix/tty_set.rs index 0f46e1e25ab..9ded32a0132 100644 --- a/lib/wasix/src/syscalls/wasix/tty_set.rs +++ b/lib/wasix/src/syscalls/wasix/tty_set.rs @@ -1,22 +1,17 @@ use super::*; -use crate::syscalls::*; +use crate::{syscalls::*, WasiTtyState}; /// ### `tty_set()` /// Updates the properties of the rect #[instrument(level = "debug", skip_all, ret)] pub fn tty_set( - ctx: FunctionEnvMut<'_, WasiEnv>, + mut ctx: FunctionEnvMut<'_, WasiEnv>, tty_state: WasmPtr, -) -> Errno { +) -> Result { let env = ctx.data(); - let bridge = if let Some(t) = env.runtime.tty() { - t - } else { - return Errno::Notsup; - }; let memory = unsafe { env.memory_view(&ctx) }; - let state = wasi_try_mem!(tty_state.read(&memory)); + let state = wasi_try_mem_ok!(tty_state.read(&memory)); let echo = state.echo; let line_buffered = state.line_buffered; let line_feeds = true; @@ -39,7 +34,31 @@ pub fn tty_set( line_feeds, }; + wasi_try_ok!(tty_set_internal(&mut ctx, state.clone())); + let env = ctx.data(); + + #[cfg(feature = "snapshot")] + if env.enable_snapshot_capture { + SnapshotEffector::save_tty_set(&mut ctx, state).map_err(|err| { + tracing::error!("failed to save path symbolic link event - {}", err); + WasiError::Exit(ExitCode::Errno(Errno::Fault)) + })?; + } + + Ok(Errno::Success) +} + +pub fn tty_set_internal( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + state: WasiTtyState, +) -> Result<(), Errno> { + let env = ctx.data(); + let bridge = if let Some(t) = env.runtime.tty() { + t + } else { + return Err(Errno::Notsup); + }; bridge.tty_set(state); - Errno::Success + Ok(()) } From f167eb984aa373b3e7459d8c8717df1e19cf4cd7 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Mon, 20 Nov 2023 09:18:30 +1100 Subject: [PATCH 031/129] cargo fmt --- .../src/snapshot/effector/syscalls/fd_allocate.rs | 5 +---- .../src/snapshot/effector/syscalls/fd_duplicate.rs | 2 +- .../src/snapshot/effector/syscalls/fd_set_flags.rs | 12 +++--------- .../src/snapshot/effector/syscalls/fd_set_size.rs | 12 +++--------- .../src/snapshot/effector/syscalls/path_link.rs | 2 +- .../src/snapshot/effector/syscalls/path_symlink.rs | 2 +- 6 files changed, 10 insertions(+), 25 deletions(-) diff --git a/lib/wasix/src/snapshot/effector/syscalls/fd_allocate.rs b/lib/wasix/src/snapshot/effector/syscalls/fd_allocate.rs index 430a24098c1..ee20fbfad1a 100644 --- a/lib/wasix/src/snapshot/effector/syscalls/fd_allocate.rs +++ b/lib/wasix/src/snapshot/effector/syscalls/fd_allocate.rs @@ -7,10 +7,7 @@ impl SnapshotEffector { offset: Filesize, len: Filesize, ) -> anyhow::Result<()> { - Self::save_event( - ctx, - SnapshotLog::FileDescriptorAllocate { fd, offset, len }, - ) + Self::save_event(ctx, SnapshotLog::FileDescriptorAllocate { fd, offset, len }) } pub fn apply_fd_allocate( diff --git a/lib/wasix/src/snapshot/effector/syscalls/fd_duplicate.rs b/lib/wasix/src/snapshot/effector/syscalls/fd_duplicate.rs index 0c225426184..52083e10d74 100644 --- a/lib/wasix/src/snapshot/effector/syscalls/fd_duplicate.rs +++ b/lib/wasix/src/snapshot/effector/syscalls/fd_duplicate.rs @@ -41,4 +41,4 @@ impl SnapshotEffector { } Ok(()) } -} \ No newline at end of file +} diff --git a/lib/wasix/src/snapshot/effector/syscalls/fd_set_flags.rs b/lib/wasix/src/snapshot/effector/syscalls/fd_set_flags.rs index bc4939ebc75..995de16b3aa 100644 --- a/lib/wasix/src/snapshot/effector/syscalls/fd_set_flags.rs +++ b/lib/wasix/src/snapshot/effector/syscalls/fd_set_flags.rs @@ -4,21 +4,15 @@ impl SnapshotEffector { pub fn save_fd_set_flags( ctx: &mut FunctionEnvMut<'_, WasiEnv>, fd: Fd, - flags: Fdflags + flags: Fdflags, ) -> anyhow::Result<()> { - Self::save_event( - ctx, - SnapshotLog::FileDescriptorSetFlags { - fd, - flags, - }, - ) + Self::save_event(ctx, SnapshotLog::FileDescriptorSetFlags { fd, flags }) } pub fn apply_fd_set_flags( ctx: &mut FunctionEnvMut<'_, WasiEnv>, fd: Fd, - flags: Fdflags + flags: Fdflags, ) -> anyhow::Result<()> { crate::syscalls::fd_fdstat_set_flags_internal(ctx, fd, flags) .map_err(|err| { diff --git a/lib/wasix/src/snapshot/effector/syscalls/fd_set_size.rs b/lib/wasix/src/snapshot/effector/syscalls/fd_set_size.rs index cfbc3dd25fd..f22b5033089 100644 --- a/lib/wasix/src/snapshot/effector/syscalls/fd_set_size.rs +++ b/lib/wasix/src/snapshot/effector/syscalls/fd_set_size.rs @@ -4,21 +4,15 @@ impl SnapshotEffector { pub fn save_fd_set_size( ctx: &mut FunctionEnvMut<'_, WasiEnv>, fd: Fd, - st_size: Filesize + st_size: Filesize, ) -> anyhow::Result<()> { - Self::save_event( - ctx, - SnapshotLog::FileDescriptorSetSize { - fd, - st_size, - }, - ) + Self::save_event(ctx, SnapshotLog::FileDescriptorSetSize { fd, st_size }) } pub fn apply_fd_set_size( ctx: &mut FunctionEnvMut<'_, WasiEnv>, fd: Fd, - st_size: Filesize + st_size: Filesize, ) -> anyhow::Result<()> { crate::syscalls::fd_filestat_set_size_internal(ctx, fd, st_size) .map_err(|err| { diff --git a/lib/wasix/src/snapshot/effector/syscalls/path_link.rs b/lib/wasix/src/snapshot/effector/syscalls/path_link.rs index b0b9251ca1a..b9e2a4dee83 100644 --- a/lib/wasix/src/snapshot/effector/syscalls/path_link.rs +++ b/lib/wasix/src/snapshot/effector/syscalls/path_link.rs @@ -16,7 +16,7 @@ impl SnapshotEffector { old_flags, old_path: old_path.into(), new_fd, - new_path: new_path.into() + new_path: new_path.into(), }, ) } diff --git a/lib/wasix/src/snapshot/effector/syscalls/path_symlink.rs b/lib/wasix/src/snapshot/effector/syscalls/path_symlink.rs index cd18218bbf3..685947c9350 100644 --- a/lib/wasix/src/snapshot/effector/syscalls/path_symlink.rs +++ b/lib/wasix/src/snapshot/effector/syscalls/path_symlink.rs @@ -12,7 +12,7 @@ impl SnapshotEffector { SnapshotLog::CreateSymbolicLink { old_path: old_path.into(), fd, - new_path: new_path.into() + new_path: new_path.into(), }, ) } From 7d023ea13014726ef39c8874d7a3498822d19991 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Mon, 20 Nov 2023 09:37:37 +1100 Subject: [PATCH 032/129] Fixed compile errors without the snapshot feature --- lib/wasi-web/src/pool.rs | 1 - lib/wasix/Cargo.toml | 2 +- lib/wasix/src/runners/wasi.rs | 1 + .../snapshot/effector/syscalls/fd_allocate.rs | 2 +- .../effector/syscalls/fd_duplicate.rs | 2 +- .../effector/syscalls/fd_set_flags.rs | 2 +- .../effector/syscalls/fd_set_rights.rs | 2 +- .../snapshot/effector/syscalls/fd_set_size.rs | 2 +- .../effector/syscalls/fd_set_times.rs | 2 +- .../snapshot/effector/syscalls/path_link.rs | 2 +- .../effector/syscalls/path_set_times.rs | 2 +- .../effector/syscalls/path_symlink.rs | 2 +- lib/wasix/src/snapshot/mod.rs | 6 ++++- lib/wasix/src/state/builder.rs | 26 ++++++++++++------- lib/wasix/src/syscalls/mod.rs | 1 + 15 files changed, 33 insertions(+), 22 deletions(-) diff --git a/lib/wasi-web/src/pool.rs b/lib/wasi-web/src/pool.rs index 87f50f792f9..5515bfc152b 100644 --- a/lib/wasi-web/src/pool.rs +++ b/lib/wasi-web/src/pool.rs @@ -29,7 +29,6 @@ use wasm_bindgen::{prelude::*, JsCast}; use wasm_bindgen_futures::JsFuture; use wasmer::AsStoreRef; use wasmer_wasix::{ - capture_snapshot, runtime::{ task_manager::{ InlineWaker, TaskExecModule, TaskWasm, TaskWasmRun, TaskWasmRunProperties, diff --git a/lib/wasix/Cargo.toml b/lib/wasix/Cargo.toml index d06d8434e70..aace6844936 100644 --- a/lib/wasix/Cargo.toml +++ b/lib/wasix/Cargo.toml @@ -146,7 +146,7 @@ sys-default = [ ] sys-poll = [] sys-thread = ["tokio/rt", "tokio/time", "tokio/rt-multi-thread", "rayon"] -snapshot = [] +snapshot = ["tokio/fs"] # Deprecated. Kept it for compatibility compiler = [] diff --git a/lib/wasix/src/runners/wasi.rs b/lib/wasix/src/runners/wasi.rs index 8c07cd1a973..492cc625109 100644 --- a/lib/wasix/src/runners/wasi.rs +++ b/lib/wasix/src/runners/wasi.rs @@ -318,6 +318,7 @@ impl crate::runners::Runner for WasiRunner { .annotation("wasi")? .unwrap_or_else(|| Wasi::new(command_name)); + #[allow(unused_mut)] let mut env = self .prepare_webc_env(command_name, &wasi, Some(pkg), Arc::clone(&runtime), None) .context("Unable to prepare the WASI environment")?; diff --git a/lib/wasix/src/snapshot/effector/syscalls/fd_allocate.rs b/lib/wasix/src/snapshot/effector/syscalls/fd_allocate.rs index ee20fbfad1a..eb2af859483 100644 --- a/lib/wasix/src/snapshot/effector/syscalls/fd_allocate.rs +++ b/lib/wasix/src/snapshot/effector/syscalls/fd_allocate.rs @@ -24,7 +24,7 @@ impl SnapshotEffector { offset, len, err - ) + ) })?; Ok(()) } diff --git a/lib/wasix/src/snapshot/effector/syscalls/fd_duplicate.rs b/lib/wasix/src/snapshot/effector/syscalls/fd_duplicate.rs index 52083e10d74..9f0c2d5ab2a 100644 --- a/lib/wasix/src/snapshot/effector/syscalls/fd_duplicate.rs +++ b/lib/wasix/src/snapshot/effector/syscalls/fd_duplicate.rs @@ -27,7 +27,7 @@ impl SnapshotEffector { original_fd, copied_fd, err - ) + ) })?; let ret = crate::syscalls::fd_renumber_internal(ctx, ret_fd, copied_fd); diff --git a/lib/wasix/src/snapshot/effector/syscalls/fd_set_flags.rs b/lib/wasix/src/snapshot/effector/syscalls/fd_set_flags.rs index 995de16b3aa..de8208e54f4 100644 --- a/lib/wasix/src/snapshot/effector/syscalls/fd_set_flags.rs +++ b/lib/wasix/src/snapshot/effector/syscalls/fd_set_flags.rs @@ -21,7 +21,7 @@ impl SnapshotEffector { fd, flags, err - ) + ) })?; Ok(()) } diff --git a/lib/wasix/src/snapshot/effector/syscalls/fd_set_rights.rs b/lib/wasix/src/snapshot/effector/syscalls/fd_set_rights.rs index 7c8a59ccd11..266d75e4031 100644 --- a/lib/wasix/src/snapshot/effector/syscalls/fd_set_rights.rs +++ b/lib/wasix/src/snapshot/effector/syscalls/fd_set_rights.rs @@ -31,7 +31,7 @@ impl SnapshotEffector { fs_rights_base, fs_rights_inheriting, err - ) + ) })?; Ok(()) } diff --git a/lib/wasix/src/snapshot/effector/syscalls/fd_set_size.rs b/lib/wasix/src/snapshot/effector/syscalls/fd_set_size.rs index f22b5033089..eb4b0bc2c91 100644 --- a/lib/wasix/src/snapshot/effector/syscalls/fd_set_size.rs +++ b/lib/wasix/src/snapshot/effector/syscalls/fd_set_size.rs @@ -21,7 +21,7 @@ impl SnapshotEffector { fd, st_size, err - ) + ) })?; Ok(()) } diff --git a/lib/wasix/src/snapshot/effector/syscalls/fd_set_times.rs b/lib/wasix/src/snapshot/effector/syscalls/fd_set_times.rs index b419bb6c4ca..e13aa4c0315 100644 --- a/lib/wasix/src/snapshot/effector/syscalls/fd_set_times.rs +++ b/lib/wasix/src/snapshot/effector/syscalls/fd_set_times.rs @@ -35,7 +35,7 @@ impl SnapshotEffector { st_mtim, fst_flags, err - ) + ) })?; Ok(()) } diff --git a/lib/wasix/src/snapshot/effector/syscalls/path_link.rs b/lib/wasix/src/snapshot/effector/syscalls/path_link.rs index b9e2a4dee83..8cf151c9166 100644 --- a/lib/wasix/src/snapshot/effector/syscalls/path_link.rs +++ b/lib/wasix/src/snapshot/effector/syscalls/path_link.rs @@ -39,7 +39,7 @@ impl SnapshotEffector { new_fd, new_path, err - ) + ) })?; Ok(()) } diff --git a/lib/wasix/src/snapshot/effector/syscalls/path_set_times.rs b/lib/wasix/src/snapshot/effector/syscalls/path_set_times.rs index 3ff1886df56..287239542e3 100644 --- a/lib/wasix/src/snapshot/effector/syscalls/path_set_times.rs +++ b/lib/wasix/src/snapshot/effector/syscalls/path_set_times.rs @@ -43,7 +43,7 @@ impl SnapshotEffector { st_mtim, fst_flags, err - ) + ) })?; Ok(()) } diff --git a/lib/wasix/src/snapshot/effector/syscalls/path_symlink.rs b/lib/wasix/src/snapshot/effector/syscalls/path_symlink.rs index 685947c9350..987d9ed1939 100644 --- a/lib/wasix/src/snapshot/effector/syscalls/path_symlink.rs +++ b/lib/wasix/src/snapshot/effector/syscalls/path_symlink.rs @@ -31,7 +31,7 @@ impl SnapshotEffector { fd, new_path, err - ) + ) })?; Ok(()) } diff --git a/lib/wasix/src/snapshot/mod.rs b/lib/wasix/src/snapshot/mod.rs index 0338567f95d..976ce1a3518 100644 --- a/lib/wasix/src/snapshot/mod.rs +++ b/lib/wasix/src/snapshot/mod.rs @@ -2,15 +2,19 @@ mod capturer; mod compactor; #[cfg(feature = "snapshot")] mod effector; +#[cfg(not(feature = "snapshot"))] +#[path = "effector/unimplemented.rs"] +mod effector; mod filter; +#[cfg(feature = "snapshot")] mod log_file; mod unsupported; pub use capturer::*; pub use compactor::*; -#[cfg(feature = "snapshot")] pub use effector::*; pub use filter::*; +#[cfg(feature = "snapshot")] pub use log_file::*; pub use unsupported::*; diff --git a/lib/wasix/src/state/builder.rs b/lib/wasix/src/state/builder.rs index ba74d7431ad..47af0a9517a 100644 --- a/lib/wasix/src/state/builder.rs +++ b/lib/wasix/src/state/builder.rs @@ -13,8 +13,6 @@ use virtual_fs::{ArcFile, FileSystem, FsError, TmpFileSystem, VirtualFile}; use wasmer::{AsStoreMut, Instance, Module, RuntimeError, Store}; use wasmer_wasix_types::wasi::{Errno, ExitCode}; -#[cfg(feature = "snapshot")] -use crate::snapshot::SnapshotTrigger; #[cfg(feature = "sys")] use crate::PluggableRuntime; use crate::{ @@ -25,12 +23,11 @@ use crate::{ runtime::task_manager::InlineWaker, snapshot::DynSnapshotCapturer, state::WasiState, - syscalls::{ - restore_snapshot, - types::{__WASI_STDERR_FILENO, __WASI_STDIN_FILENO, __WASI_STDOUT_FILENO}, - }, + syscalls::types::{__WASI_STDERR_FILENO, __WASI_STDIN_FILENO, __WASI_STDOUT_FILENO}, RewindState, Runtime, WasiEnv, WasiError, WasiFunctionEnv, WasiRuntimeError, }; +#[cfg(feature = "snapshot")] +use crate::{snapshot::SnapshotTrigger, syscalls::restore_snapshot}; use super::env::WasiEnvInit; @@ -522,6 +519,7 @@ impl WasiEnvBuilder { /// Supplies a snapshot capturer which will be used to read the /// snapshot journal events and replay them into the WasiEnv /// rather than starting a new instance from scratch + #[cfg(feature = "snapshot")] pub fn with_snapshot_restore( mut self, restorer: Arc, @@ -536,6 +534,7 @@ impl WasiEnvBuilder { /// Supplies a snapshot capturer where the snapshot journal events /// will be sent to as they are generated + #[cfg(feature = "snapshot")] pub fn with_snapshot_save(mut self, snapshot_capturer: Arc) -> Self { self.snapshot_save.replace(snapshot_capturer); self @@ -934,14 +933,17 @@ impl WasiEnvBuilder { ); } - let restore = self.snapshot_restore.clone(); + #[cfg(feature = "snapshot")] + let snapshot_restore = self.snapshot_restore.clone(); + #[cfg(not(feature = "snapshot"))] + let snapshot_restore = None; let (instance, env) = self.instantiate(module, store)?; let start = instance.exports.get_function("_start")?; env.data(&store).thread.set_status_running(); - let result = crate::run_wasi_func_start(start, store, restore); + let result = crate::run_wasi_func_start(start, store, snapshot_restore); let (result, exit_code) = wasi_exit_code(result); let pid = env.data(&store).pid(); @@ -980,16 +982,20 @@ impl WasiEnvBuilder { #[cfg(feature = "sys-thread")] let _guard = _guard.as_ref().map(|r| r.enter()); + #[cfg(feature = "snapshot")] let snapshot_restore = self.snapshot_restore.clone(); let (_, env) = self.instantiate(module, &mut store)?; env.data(&store).thread.set_status_running(); + #[allow(unused_mut)] let mut rewind_state = None; - if let Some(restore) = snapshot_restore { + + #[cfg(feature = "snapshot")] + if let Some(snapshot_restore) = snapshot_restore { let ctx = env.env.clone().into_mut(&mut store); - let rewind = restore_snapshot(ctx, restore)?; + let rewind = restore_snapshot(ctx, snapshot_restore)?; rewind_state = Some((rewind, None)); } diff --git a/lib/wasix/src/syscalls/mod.rs b/lib/wasix/src/syscalls/mod.rs index 66bfb208a81..4a75d0b1726 100644 --- a/lib/wasix/src/syscalls/mod.rs +++ b/lib/wasix/src/syscalls/mod.rs @@ -1291,6 +1291,7 @@ pub fn anyhow_err_to_runtime_err(err: anyhow::Error) -> WasiRuntimeError { WasiRuntimeError::Runtime(RuntimeError::user(err.into())) } +#[cfg(feature = "snapshot")] pub fn restore_snapshot( mut ctx: FunctionEnvMut<'_, WasiEnv>, restore: SnapshotRestore, From 1223d37d317c5761025876da0db8948333173100 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Mon, 20 Nov 2023 09:38:30 +1100 Subject: [PATCH 033/129] Fixed compile errors without the snapshot feature --- lib/wasix/src/snapshot/effector/unimplemented.rs | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 lib/wasix/src/snapshot/effector/unimplemented.rs diff --git a/lib/wasix/src/snapshot/effector/unimplemented.rs b/lib/wasix/src/snapshot/effector/unimplemented.rs new file mode 100644 index 00000000000..8d8dc18a667 --- /dev/null +++ b/lib/wasix/src/snapshot/effector/unimplemented.rs @@ -0,0 +1,4 @@ +#[derive(Debug, Clone)] +pub struct SnapshotEffector {} + +impl SnapshotEffector {} From 4aa1ec5ca66bcb1c3f45753a0ac459ff3c5c726f Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Mon, 20 Nov 2023 10:02:13 +1100 Subject: [PATCH 034/129] More feature toggle fixes --- lib/wasix/src/state/builder.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/wasix/src/state/builder.rs b/lib/wasix/src/state/builder.rs index 47af0a9517a..6cbc3c2dced 100644 --- a/lib/wasix/src/state/builder.rs +++ b/lib/wasix/src/state/builder.rs @@ -815,7 +815,9 @@ impl WasiEnvBuilder { let runtime = self.runtime.unwrap_or_else(|| { #[cfg(feature = "sys-thread")] { + #[allow(unused_mut)] let mut runtime = PluggableRuntime::new(Arc::new(crate::runtime::task_manager::tokio::TokioTaskManager::default())); + #[cfg(feature = "snapshot")] if let Some(capturer) = self.snapshot_save.clone() { runtime.set_snapshot_capturer(capturer); } From ccc4c29c8622916b980d42eb2129c12798841c74 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Mon, 20 Nov 2023 10:44:43 +1100 Subject: [PATCH 035/129] Refactored the snapshot to journal to make it clearer how it works --- .vscode/settings.json | 3 +- lib/cli/Cargo.toml | 4 +- lib/cli/src/commands/run/mod.rs | 40 +-- lib/cli/src/commands/run/wasi.rs | 79 ++-- lib/wasix/Cargo.toml | 2 +- .../src/{snapshot => journal}/capturer.rs | 12 +- .../src/{snapshot => journal}/compactor.rs | 34 +- .../effector/memory_and_snapshot.rs | 6 +- .../src/{snapshot => journal}/effector/mod.rs | 12 +- .../effector/save_event.rs | 4 +- .../effector/syscalls/chdir.rs | 4 +- .../effector/syscalls/clock_time.rs | 4 +- .../effector/syscalls/epoll_create.rs | 4 +- .../effector/syscalls/epoll_ctl.rs | 4 +- .../effector/syscalls/fd_advise.rs | 4 +- .../effector/syscalls/fd_allocate.rs | 7 +- .../effector/syscalls/fd_close.rs | 4 +- .../effector/syscalls/fd_duplicate.rs | 4 +- .../effector/syscalls/fd_pipe.rs | 4 +- .../effector/syscalls/fd_renumber.rs | 4 +- .../effector/syscalls/fd_seek.rs | 4 +- .../effector/syscalls/fd_set_flags.rs | 4 +- .../effector/syscalls/fd_set_rights.rs | 4 +- .../effector/syscalls/fd_set_size.rs | 4 +- .../effector/syscalls/fd_set_times.rs | 4 +- .../effector/syscalls/fd_write.rs | 4 +- .../syscalls/path_create_directory.rs | 4 +- .../effector/syscalls/path_link.rs | 4 +- .../effector/syscalls/path_open.rs | 4 +- .../syscalls/path_remove_directory.rs | 4 +- .../effector/syscalls/path_rename.rs | 4 +- .../effector/syscalls/path_set_times.rs | 4 +- .../effector/syscalls/path_symlink.rs | 4 +- .../effector/syscalls/path_unlink.rs | 4 +- .../effector/syscalls/tty_set.rs | 4 +- .../effector/thread_exit.rs | 4 +- .../effector/thread_state.rs | 4 +- .../src/journal/effector/unimplemented.rs | 4 + lib/wasix/src/{snapshot => journal}/filter.rs | 78 ++-- .../src/{snapshot => journal}/log_file.rs | 338 +++++++++--------- lib/wasix/src/{snapshot => journal}/mod.rs | 14 +- lib/wasix/src/journal/unsupported.rs | 21 ++ lib/wasix/src/lib.rs | 8 +- lib/wasix/src/os/task/process.rs | 18 +- lib/wasix/src/os/task/thread.rs | 10 +- lib/wasix/src/runners/wasi.rs | 35 +- lib/wasix/src/runners/wasi_common.rs | 8 +- lib/wasix/src/runners/wcgi/runner.rs | 30 +- lib/wasix/src/runtime/mod.rs | 29 +- .../src/snapshot/effector/unimplemented.rs | 4 - lib/wasix/src/snapshot/unsupported.rs | 22 -- lib/wasix/src/state/builder.rs | 59 ++- lib/wasix/src/state/env.rs | 32 +- lib/wasix/src/syscalls/mod.rs | 145 ++++---- lib/wasix/src/syscalls/wasi/clock_time_set.rs | 4 +- lib/wasix/src/syscalls/wasi/environ_get.rs | 2 +- .../src/syscalls/wasi/environ_sizes_get.rs | 2 +- lib/wasix/src/syscalls/wasi/fd_advise.rs | 4 +- lib/wasix/src/syscalls/wasi/fd_allocate.rs | 4 +- lib/wasix/src/syscalls/wasi/fd_close.rs | 4 +- lib/wasix/src/syscalls/wasi/fd_dup.rs | 4 +- .../src/syscalls/wasi/fd_fdstat_set_flags.rs | 4 +- .../src/syscalls/wasi/fd_fdstat_set_rights.rs | 4 +- .../src/syscalls/wasi/fd_filestat_set_size.rs | 4 +- .../syscalls/wasi/fd_filestat_set_times.rs | 4 +- lib/wasix/src/syscalls/wasi/fd_read.rs | 2 +- lib/wasix/src/syscalls/wasi/fd_renumber.rs | 4 +- lib/wasix/src/syscalls/wasi/fd_seek.rs | 4 +- lib/wasix/src/syscalls/wasi/fd_write.rs | 10 +- .../syscalls/wasi/path_create_directory.rs | 4 +- .../syscalls/wasi/path_filestat_set_times.rs | 4 +- lib/wasix/src/syscalls/wasi/path_link.rs | 4 +- lib/wasix/src/syscalls/wasi/path_open.rs | 4 +- .../syscalls/wasi/path_remove_directory.rs | 4 +- lib/wasix/src/syscalls/wasi/path_rename.rs | 4 +- lib/wasix/src/syscalls/wasi/path_symlink.rs | 4 +- .../src/syscalls/wasi/path_unlink_file.rs | 4 +- lib/wasix/src/syscalls/wasix/chdir.rs | 4 +- lib/wasix/src/syscalls/wasix/epoll_create.rs | 4 +- lib/wasix/src/syscalls/wasix/epoll_ctl.rs | 4 +- lib/wasix/src/syscalls/wasix/fd_pipe.rs | 4 +- lib/wasix/src/syscalls/wasix/sock_listen.rs | 2 +- lib/wasix/src/syscalls/wasix/thread_spawn.rs | 4 +- lib/wasix/src/syscalls/wasix/tty_set.rs | 4 +- lib/wasix/src/utils/mod.rs | 2 +- 85 files changed, 619 insertions(+), 663 deletions(-) rename lib/wasix/src/{snapshot => journal}/capturer.rs (94%) rename lib/wasix/src/{snapshot => journal}/compactor.rs (84%) rename lib/wasix/src/{snapshot => journal}/effector/memory_and_snapshot.rs (95%) rename lib/wasix/src/{snapshot => journal}/effector/mod.rs (89%) rename lib/wasix/src/{snapshot => journal}/effector/save_event.rs (90%) rename lib/wasix/src/{snapshot => journal}/effector/syscalls/chdir.rs (82%) rename lib/wasix/src/{snapshot => journal}/effector/syscalls/clock_time.rs (87%) rename lib/wasix/src/{snapshot => journal}/effector/syscalls/epoll_create.rs (91%) rename lib/wasix/src/{snapshot => journal}/effector/syscalls/epoll_ctl.rs (95%) rename lib/wasix/src/{snapshot => journal}/effector/syscalls/fd_advise.rs (93%) rename lib/wasix/src/{snapshot => journal}/effector/syscalls/fd_allocate.rs (84%) rename lib/wasix/src/{snapshot => journal}/effector/syscalls/fd_close.rs (85%) rename lib/wasix/src/{snapshot => journal}/effector/syscalls/fd_duplicate.rs (94%) rename lib/wasix/src/{snapshot => journal}/effector/syscalls/fd_pipe.rs (92%) rename lib/wasix/src/{snapshot => journal}/effector/syscalls/fd_renumber.rs (91%) rename lib/wasix/src/{snapshot => journal}/effector/syscalls/fd_seek.rs (86%) rename lib/wasix/src/{snapshot => journal}/effector/syscalls/fd_set_flags.rs (86%) rename lib/wasix/src/{snapshot => journal}/effector/syscalls/fd_set_rights.rs (93%) rename lib/wasix/src/{snapshot => journal}/effector/syscalls/fd_set_size.rs (86%) rename lib/wasix/src/{snapshot => journal}/effector/syscalls/fd_set_times.rs (93%) rename lib/wasix/src/{snapshot => journal}/effector/syscalls/fd_write.rs (96%) rename lib/wasix/src/{snapshot => journal}/effector/syscalls/path_create_directory.rs (92%) rename lib/wasix/src/{snapshot => journal}/effector/syscalls/path_link.rs (94%) rename lib/wasix/src/{snapshot => journal}/effector/syscalls/path_open.rs (97%) rename lib/wasix/src/{snapshot => journal}/effector/syscalls/path_remove_directory.rs (91%) rename lib/wasix/src/{snapshot => journal}/effector/syscalls/path_rename.rs (94%) rename lib/wasix/src/{snapshot => journal}/effector/syscalls/path_set_times.rs (95%) rename lib/wasix/src/{snapshot => journal}/effector/syscalls/path_symlink.rs (93%) rename lib/wasix/src/{snapshot => journal}/effector/syscalls/path_unlink.rs (92%) rename lib/wasix/src/{snapshot => journal}/effector/syscalls/tty_set.rs (94%) rename lib/wasix/src/{snapshot => journal}/effector/thread_exit.rs (83%) rename lib/wasix/src/{snapshot => journal}/effector/thread_state.rs (90%) create mode 100644 lib/wasix/src/journal/effector/unimplemented.rs rename lib/wasix/src/{snapshot => journal}/filter.rs (74%) rename lib/wasix/src/{snapshot => journal}/log_file.rs (64%) rename lib/wasix/src/{snapshot => journal}/mod.rs (87%) create mode 100644 lib/wasix/src/journal/unsupported.rs delete mode 100644 lib/wasix/src/snapshot/effector/unimplemented.rs delete mode 100644 lib/wasix/src/snapshot/unsupported.rs diff --git a/.vscode/settings.json b/.vscode/settings.json index c0024f29c73..8c5ad19f07b 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -8,5 +8,6 @@ "editor.formatOnSave": true, "editor.formatOnPaste": false, "editor.formatOnType": false - } + }, + "rust-analyzer.showUnlinkedFileNotification": false } \ No newline at end of file diff --git a/lib/cli/Cargo.toml b/lib/cli/Cargo.toml index 959b4ed126c..c4879ed2841 100644 --- a/lib/cli/Cargo.toml +++ b/lib/cli/Cargo.toml @@ -162,11 +162,11 @@ default = [ "wat", "wast", "compiler", - "snapshot", + "journal", "wasmer-artifact-create", "static-artifact-create", ] -snapshot = ["wasmer-wasix/snapshot"] +journal = ["wasmer-wasix/journal"] backend = [] coredump = ["wasm-coredump-builder"] sys = ["compiler", "wasmer-vm"] diff --git a/lib/cli/src/commands/run/mod.rs b/lib/cli/src/commands/run/mod.rs index e50c112e067..0e92a2c9a1f 100644 --- a/lib/cli/src/commands/run/mod.rs +++ b/lib/cli/src/commands/run/mod.rs @@ -30,6 +30,7 @@ use wasmer_compiler::ArtifactBuild; use wasmer_registry::{wasmer_env::WasmerEnv, Package}; use wasmer_wasix::{ bin_factory::BinaryPackage, + journal::{LogFileJournal, SnapshotTrigger}, runners::{MappedCommand, MappedDirectory, Runner}, runtime::{ module_cache::{CacheError, ModuleHash}, @@ -37,7 +38,6 @@ use wasmer_wasix::{ resolver::{PackageSpecifier, QueryError}, task_manager::VirtualTaskManagerExt, }, - snapshot::{LogFileSnapshotCapturer, SnapshotTrigger}, WasiError, }; use wasmer_wasix::{ @@ -242,30 +242,28 @@ impl Run { runner.config().forward_host_env(); } - #[cfg(feature = "snapshot")] + #[cfg(feature = "journal")] for trigger in self.wasi.snapshot_on.iter().cloned() { runner.config().add_snapshot_trigger(trigger); } - #[cfg(feature = "snapshot")] - match (self.wasi.snapshot_to.clone(), self.wasi.resume_from.clone()) { + #[cfg(feature = "journal")] + match (self.wasi.journal.clone(), self.wasi.journal_restore.clone()) { (Some(save), Some(restore)) if save == restore => { return Err(anyhow::format_err!( "The snapshot save path and snapshot restore path can not be the same" )); } (_, _) => { - if let Some(path) = self.wasi.snapshot_to.clone() { + if let Some(path) = self.wasi.journal.clone() { runner .config() - .with_snapshot_save(Arc::new(LogFileSnapshotCapturer::new_std(path)?)); + .with_journal(Arc::new(LogFileJournal::new_std(path)?)); } - if let Some(path) = self.wasi.resume_from.clone() { - let n_snapshots = self.wasi.resume_num_snapshots; - runner.config().with_snapshot_restore( - Arc::new(LogFileSnapshotCapturer::new_std(path)?), - n_snapshots, - ); + if let Some(path) = self.wasi.journal_restore.clone() { + runner + .config() + .with_journal_restore(Arc::new(LogFileJournal::new_std(path)?)); } } } @@ -332,28 +330,24 @@ impl Run { .with_forward_host_env(self.wasi.forward_host_env) .with_capabilities(self.wasi.capabilities()); - #[cfg(feature = "snapshot")] + #[cfg(feature = "journal")] for trigger in self.wasi.snapshot_on.iter().cloned() { runner.add_snapshot_trigger(trigger); } - #[cfg(feature = "snapshot")] - match (self.wasi.snapshot_to.clone(), self.wasi.resume_from.clone()) { + #[cfg(feature = "journal")] + match (self.wasi.journal.clone(), self.wasi.journal_restore.clone()) { (Some(save), Some(restore)) if save == restore => { return Err(anyhow::format_err!( "The snapshot save path and snapshot restore path can not be the same" )); } (_, _) => { - if let Some(path) = self.wasi.snapshot_to.clone() { - runner.with_snapshot_save(Arc::new(LogFileSnapshotCapturer::new_std(path)?)); + if let Some(path) = self.wasi.journal.clone() { + runner.with_journal(Arc::new(LogFileJournal::new_std(path)?)); } - if let Some(path) = self.wasi.resume_from.clone() { - let n_snapshots = self.wasi.resume_num_snapshots; - runner.with_snapshot_restore( - Arc::new(LogFileSnapshotCapturer::new_std(path)?), - n_snapshots, - ); + if let Some(path) = self.wasi.journal_restore.clone() { + runner.with_journal_restore(Arc::new(LogFileJournal::new_std(path)?)); } } } diff --git a/lib/cli/src/commands/run/wasi.rs b/lib/cli/src/commands/run/wasi.rs index 63e299b2100..e693ed5bb0d 100644 --- a/lib/cli/src/commands/run/wasi.rs +++ b/lib/cli/src/commands/run/wasi.rs @@ -19,6 +19,7 @@ use wasmer_wasix::{ capabilities::Capabilities, default_fs_backing, get_wasi_versions, http::HttpClient, + journal::{self, LogFileJournal, SnapshotTrigger}, os::{tty_sys::SysTty, TtyBridge}, rewind_ext, runners::{MappedCommand, MappedDirectory}, @@ -34,7 +35,6 @@ use wasmer_wasix::{ VirtualTaskManagerExt, }, }, - snapshot::{self, LogFileSnapshotCapturer, SnapshotTrigger}, types::__WASI_STDIN_FILENO, wasmer_wasix_types::wasi::Errno, PluggableRuntime, RewindState, Runtime, WasiEnv, WasiEnvBuilder, WasiError, WasiFunctionEnv, @@ -107,40 +107,35 @@ pub struct Wasi { #[clap(long = "enable-async-threads")] pub enable_async_threads: bool, - /// Specifies the snapshot file that Wasmer will use to store - /// the state of the WASM process so that it can be later restored - #[cfg(feature = "snapshot")] - #[clap(long = "snapshot-to")] - pub snapshot_to: Option, + /// Specifies the journal file that Wasmer will use to store and restore + /// the state of the WASM process + #[cfg(feature = "journal")] + #[clap(long = "journal")] + pub journal: Option, + + /// When specified, the runtime will restore a previous snapshot using a different journal + /// then the one specified in the `--journal` argument. If no argument is specified for + /// `--journal` then the state of the process will be restored however no more events + /// will be recorded. + #[cfg(feature = "journal")] + #[clap(long = "journal-restore")] + pub journal_restore: Option, /// Indicates what events will cause a snapshot to be taken - /// and written to the snapshot file. + /// and written to the journal file. /// - /// If not specified, the default is to snapshot on idle plus if a - /// snapshot period is provided it will also default to periodic snapshots - /// as well. - #[cfg(feature = "snapshot")] + /// If not specified, the default is to snapshot when the process idles, when + /// the process exits or periodically if an interval argument is also supplied. + #[cfg(feature = "journal")] #[clap(long = "snapshot-on")] pub snapshot_on: Vec, - /// Adds a timer (measured in seconds) that takes snapshots of the process and dumps the - /// journal of events to the snapshot file. When specifying this parameter it implies - /// that `--snapshot-on timer` has also been specified. - #[cfg(feature = "snapshot")] - #[clap(long = "snapshot-timer")] - pub snapshot_timer: Option, - - /// When specified, the runtime will restore a previous snapshot - /// using the supplied file. - #[cfg(feature = "snapshot")] - #[clap(long = "resume-from")] - pub resume_from: Option, - - /// When specified this limits the number of restoration points - /// that the resume will take before it resumes execution. - #[cfg(feature = "snapshot")] - #[clap(long = "resume-snapshot-limit")] - pub resume_num_snapshots: Option, + /// Adds a periodic interval (measured in seconds) that the runtime will automatically + /// takes snapshots of the running process and write them to the journal. When specifying + /// this parameter it implies that `--snapshot-on interval` has also been specified. + #[cfg(feature = "journal")] + #[clap(long = "snapshot-period")] + pub snapshot_interval: Option, /// Allow instances to send http requests. /// @@ -343,29 +338,25 @@ impl Wasi { *builder.capabilities_mut() = self.capabilities(); - #[cfg(feature = "snapshot")] + #[cfg(feature = "journal")] for trigger in self.snapshot_on.iter().cloned() { builder.add_snapshot_trigger(trigger); } - #[cfg(feature = "snapshot")] - match (self.snapshot_to.clone(), self.resume_from.clone()) { + #[cfg(feature = "journal")] + match (self.journal.clone(), self.journal_restore.clone()) { (Some(save), Some(restore)) if save == restore => { return Err(anyhow::format_err!( "The snapshot save path and snapshot restore path can not be the same" )); } (_, _) => { - if let Some(path) = self.snapshot_to.clone() { - builder = builder - .with_snapshot_save(Arc::new(LogFileSnapshotCapturer::new_std(path)?)); + if let Some(path) = self.journal.clone() { + builder = builder.with_journal(Arc::new(LogFileJournal::new_std(path)?)); } - if let Some(path) = self.resume_from.clone() { - let n_snapshots = self.resume_num_snapshots; - builder = builder.with_snapshot_restore( - Arc::new(LogFileSnapshotCapturer::new_std(path)?), - n_snapshots, - ); + if let Some(path) = self.journal_restore.clone() { + builder = + builder.with_journal_restore(Arc::new(LogFileJournal::new_std(path)?)); } } } @@ -523,9 +514,9 @@ impl Wasi { rt.set_networking_implementation(virtual_net::UnsupportedVirtualNetworking::default()); } - #[cfg(feature = "snapshot")] - if let Some(path) = &self.snapshot_to { - rt.set_snapshot_capturer(Arc::new(snapshot::LogFileSnapshotCapturer::new_std(path)?)); + #[cfg(feature = "journal")] + if let Some(path) = &self.journal { + rt.set_snapshot_capturer(Arc::new(journal::LogFileJournal::new_std(path)?)); } if !self.no_tty { diff --git a/lib/wasix/Cargo.toml b/lib/wasix/Cargo.toml index aace6844936..9cbc8388851 100644 --- a/lib/wasix/Cargo.toml +++ b/lib/wasix/Cargo.toml @@ -146,7 +146,7 @@ sys-default = [ ] sys-poll = [] sys-thread = ["tokio/rt", "tokio/time", "tokio/rt-multi-thread", "rayon"] -snapshot = ["tokio/fs"] +journal = ["tokio/fs"] # Deprecated. Kept it for compatibility compiler = [] diff --git a/lib/wasix/src/snapshot/capturer.rs b/lib/wasix/src/journal/capturer.rs similarity index 94% rename from lib/wasix/src/snapshot/capturer.rs rename to lib/wasix/src/journal/capturer.rs index 777cbaa6864..b68ee906636 100644 --- a/lib/wasix/src/snapshot/capturer.rs +++ b/lib/wasix/src/journal/capturer.rs @@ -15,7 +15,7 @@ use crate::WasiThreadId; use super::SnapshotTrigger; #[derive(Debug, Clone, Serialize, Deserialize)] -pub enum SocketSnapshot { +pub enum SocketJournalEvent { TcpListen { listen_addr: SocketAddr, reuse_port: bool, @@ -39,7 +39,7 @@ pub enum SocketSnapshot { /// Represents a log entry in a snapshot log stream that represents the total /// state of a WASM process at a point in time. #[derive(Debug)] -pub enum SnapshotLog<'a> { +pub enum JournalEntry<'a> { Init { wasm_hash: [u8; 32], }, @@ -194,14 +194,14 @@ pub enum SnapshotLog<'a> { /// a WASM process at a point in time and saves it so that it can be restored. /// It also allows for the restoration of that state at a later moment #[allow(unused_variables)] -pub trait SnapshotCapturer { +pub trait Journal { /// Takes in a stream of snapshot log entries and saves them so that they /// may be restored at a later moment - fn write<'a>(&'a self, entry: SnapshotLog<'a>) -> LocalBoxFuture<'a, anyhow::Result<()>>; + fn write<'a>(&'a self, entry: JournalEntry<'a>) -> LocalBoxFuture<'a, anyhow::Result<()>>; /// Returns a stream of snapshot objects that the runtime will use /// to restore the state of a WASM process to a previous moment in time - fn read<'a>(&'a self) -> LocalBoxFuture<'a, anyhow::Result>>>; + fn read<'a>(&'a self) -> LocalBoxFuture<'a, anyhow::Result>>>; } -pub type DynSnapshotCapturer = dyn SnapshotCapturer + Send + Sync; +pub type DynJournal = dyn Journal + Send + Sync; diff --git a/lib/wasix/src/snapshot/compactor.rs b/lib/wasix/src/journal/compactor.rs similarity index 84% rename from lib/wasix/src/snapshot/compactor.rs rename to lib/wasix/src/journal/compactor.rs index dc4b8a53b94..5598a0515cb 100644 --- a/lib/wasix/src/snapshot/compactor.rs +++ b/lib/wasix/src/journal/compactor.rs @@ -12,20 +12,20 @@ use super::*; struct State { memory_map: HashMap, [u8; 32]>, - open_file: HashMap>, + open_file: HashMap>, close_file: HashSet, } /// Deduplicates memory and stacks to reduce the number of volume of /// log events sent to its inner capturer. Compacting the events occurs /// in line as the events are generated -pub struct CompactingSnapshotCapturer { - inner: Box, +pub struct CompactingJournal { + inner: Box, state: Mutex, } -impl CompactingSnapshotCapturer { - pub fn new(inner: Box) -> Self { +impl CompactingJournal { + pub fn new(inner: Box) -> Self { Self { inner, state: Mutex::new(State { @@ -37,11 +37,11 @@ impl CompactingSnapshotCapturer { } } -impl SnapshotCapturer for CompactingSnapshotCapturer { - fn write<'a>(&'a self, entry: SnapshotLog<'a>) -> LocalBoxFuture<'a, anyhow::Result<()>> { +impl Journal for CompactingJournal { + fn write<'a>(&'a self, entry: JournalEntry<'a>) -> LocalBoxFuture<'a, anyhow::Result<()>> { Box::pin(async { match entry { - SnapshotLog::UpdateMemoryRegion { region, data } => { + JournalEntry::UpdateMemoryRegion { region, data } => { let mut hasher = Sha256::default(); hasher.update(data.as_ref()); let hash: [u8; 32] = hasher.finalize().try_into().unwrap(); @@ -86,15 +86,15 @@ impl SnapshotCapturer for CompactingSnapshotCapturer { } return self .inner - .write(SnapshotLog::UpdateMemoryRegion { region, data }) + .write(JournalEntry::UpdateMemoryRegion { region, data }) .await; } - SnapshotLog::CloseFileDescriptor { fd } => { + JournalEntry::CloseFileDescriptor { fd } => { let mut state = self.state.lock().unwrap(); state.open_file.remove(&fd); state.close_file.insert(fd); } - SnapshotLog::OpenFileDescriptor { + JournalEntry::OpenFileDescriptor { fd, dirfd, dirflags, @@ -109,7 +109,7 @@ impl SnapshotCapturer for CompactingSnapshotCapturer { state.close_file.remove(&fd); state.open_file.insert( fd, - SnapshotLog::OpenFileDescriptor { + JournalEntry::OpenFileDescriptor { fd, dirfd, dirflags, @@ -122,7 +122,7 @@ impl SnapshotCapturer for CompactingSnapshotCapturer { }, ); } - SnapshotLog::Snapshot { .. } => { + JournalEntry::Snapshot { .. } => { let (to_close, to_open) = { let mut state = self.state.lock().unwrap(); ( @@ -132,7 +132,7 @@ impl SnapshotCapturer for CompactingSnapshotCapturer { }; for fd in to_close { self.inner - .write(SnapshotLog::CloseFileDescriptor { fd }) + .write(JournalEntry::CloseFileDescriptor { fd }) .await?; } for (_, entry) in to_open { @@ -148,10 +148,10 @@ impl SnapshotCapturer for CompactingSnapshotCapturer { }) } - fn read<'a>(&'a self) -> LocalBoxFuture<'a, anyhow::Result>>> { + fn read<'a>(&'a self) -> LocalBoxFuture<'a, anyhow::Result>>> { Box::pin(async { Ok(match self.inner.read().await? { - Some(SnapshotLog::UpdateMemoryRegion { region, data }) => { + Some(JournalEntry::UpdateMemoryRegion { region, data }) => { let mut hasher = Sha256::default(); hasher.update(data.as_ref()); let hash: [u8; 32] = hasher.finalize().try_into().unwrap(); @@ -159,7 +159,7 @@ impl SnapshotCapturer for CompactingSnapshotCapturer { let mut state = self.state.lock().unwrap(); state.memory_map.insert(region.clone(), hash); - Some(SnapshotLog::UpdateMemoryRegion { region, data }) + Some(JournalEntry::UpdateMemoryRegion { region, data }) } Some(entry) => Some(entry), None => None, diff --git a/lib/wasix/src/snapshot/effector/memory_and_snapshot.rs b/lib/wasix/src/journal/effector/memory_and_snapshot.rs similarity index 95% rename from lib/wasix/src/snapshot/effector/memory_and_snapshot.rs rename to lib/wasix/src/journal/effector/memory_and_snapshot.rs index 59569e6e446..c5ac86d2a72 100644 --- a/lib/wasix/src/snapshot/effector/memory_and_snapshot.rs +++ b/lib/wasix/src/journal/effector/memory_and_snapshot.rs @@ -1,6 +1,6 @@ use super::*; -impl SnapshotEffector { +impl JournalEffector { pub fn save_memory_and_snapshot( ctx: &mut FunctionEnvMut<'_, WasiEnv>, process: &mut MutexGuard<'_, WasiProcessInner>, @@ -55,7 +55,7 @@ impl SnapshotEffector { // Now we write it to the snap snapshot capturer capturer - .write(SnapshotLog::UpdateMemoryRegion { + .write(JournalEntry::UpdateMemoryRegion { region, data: data.into(), }) @@ -67,7 +67,7 @@ impl SnapshotEffector { // it can act as a restoration point let when = SystemTime::now(); capturer - .write(SnapshotLog::Snapshot { when, trigger }) + .write(JournalEntry::Snapshot { when, trigger }) .await .map_err(map_snapshot_err)?; Ok(()) diff --git a/lib/wasix/src/snapshot/effector/mod.rs b/lib/wasix/src/journal/effector/mod.rs similarity index 89% rename from lib/wasix/src/snapshot/effector/mod.rs rename to lib/wasix/src/journal/effector/mod.rs index 37a77482923..e6916babf12 100644 --- a/lib/wasix/src/snapshot/effector/mod.rs +++ b/lib/wasix/src/journal/effector/mod.rs @@ -24,7 +24,7 @@ pub(super) use crate::{ use super::*; -#[cfg(feature = "snapshot")] +#[cfg(feature = "journal")] mod syscalls { pub(super) use super::*; mod chdir; @@ -53,14 +53,14 @@ mod syscalls { mod path_unlink; mod tty_set; } -#[cfg(feature = "snapshot")] +#[cfg(feature = "journal")] mod memory_and_snapshot; -#[cfg(feature = "snapshot")] +#[cfg(feature = "journal")] mod save_event; -#[cfg(feature = "snapshot")] +#[cfg(feature = "journal")] mod thread_exit; -#[cfg(feature = "snapshot")] +#[cfg(feature = "journal")] mod thread_state; #[derive(Debug, Clone)] -pub struct SnapshotEffector {} +pub struct JournalEffector {} diff --git a/lib/wasix/src/snapshot/effector/save_event.rs b/lib/wasix/src/journal/effector/save_event.rs similarity index 90% rename from lib/wasix/src/snapshot/effector/save_event.rs rename to lib/wasix/src/journal/effector/save_event.rs index 7f213a0d11e..29339cd56e9 100644 --- a/lib/wasix/src/snapshot/effector/save_event.rs +++ b/lib/wasix/src/journal/effector/save_event.rs @@ -1,9 +1,9 @@ use super::*; -impl SnapshotEffector { +impl JournalEffector { pub(super) fn save_event( ctx: &mut FunctionEnvMut<'_, WasiEnv>, - event: SnapshotLog, + event: JournalEntry, ) -> anyhow::Result<()> { let env = ctx.data(); diff --git a/lib/wasix/src/snapshot/effector/syscalls/chdir.rs b/lib/wasix/src/journal/effector/syscalls/chdir.rs similarity index 82% rename from lib/wasix/src/snapshot/effector/syscalls/chdir.rs rename to lib/wasix/src/journal/effector/syscalls/chdir.rs index b62346592d0..ee0d49f7d55 100644 --- a/lib/wasix/src/snapshot/effector/syscalls/chdir.rs +++ b/lib/wasix/src/journal/effector/syscalls/chdir.rs @@ -1,8 +1,8 @@ use super::*; -impl SnapshotEffector { +impl JournalEffector { pub fn save_chdir(ctx: &mut FunctionEnvMut<'_, WasiEnv>, path: String) -> anyhow::Result<()> { - Self::save_event(ctx, SnapshotLog::ChangeDirectory { path: path.into() }) + Self::save_event(ctx, JournalEntry::ChangeDirectory { path: path.into() }) } pub fn apply_chdir(ctx: &mut FunctionEnvMut<'_, WasiEnv>, path: &str) -> anyhow::Result<()> { diff --git a/lib/wasix/src/snapshot/effector/syscalls/clock_time.rs b/lib/wasix/src/journal/effector/syscalls/clock_time.rs similarity index 87% rename from lib/wasix/src/snapshot/effector/syscalls/clock_time.rs rename to lib/wasix/src/journal/effector/syscalls/clock_time.rs index c5bbf67abe8..f77670787d2 100644 --- a/lib/wasix/src/snapshot/effector/syscalls/clock_time.rs +++ b/lib/wasix/src/journal/effector/syscalls/clock_time.rs @@ -1,12 +1,12 @@ use super::*; -impl SnapshotEffector { +impl JournalEffector { pub fn save_clock_time_set( ctx: &mut FunctionEnvMut<'_, WasiEnv>, clock_id: Snapshot0Clockid, time: Timestamp, ) -> anyhow::Result<()> { - Self::save_event(ctx, SnapshotLog::SetClockTime { clock_id, time }) + Self::save_event(ctx, JournalEntry::SetClockTime { clock_id, time }) } pub fn apply_clock_time_set( diff --git a/lib/wasix/src/snapshot/effector/syscalls/epoll_create.rs b/lib/wasix/src/journal/effector/syscalls/epoll_create.rs similarity index 91% rename from lib/wasix/src/snapshot/effector/syscalls/epoll_create.rs rename to lib/wasix/src/journal/effector/syscalls/epoll_create.rs index d98d9977bfc..6546a1344c4 100644 --- a/lib/wasix/src/snapshot/effector/syscalls/epoll_create.rs +++ b/lib/wasix/src/journal/effector/syscalls/epoll_create.rs @@ -1,8 +1,8 @@ use super::*; -impl SnapshotEffector { +impl JournalEffector { pub fn save_epoll_create(ctx: &mut FunctionEnvMut<'_, WasiEnv>, fd: Fd) -> anyhow::Result<()> { - Self::save_event(ctx, SnapshotLog::EpollCreate { fd }) + Self::save_event(ctx, JournalEntry::EpollCreate { fd }) } pub fn apply_epoll_create(ctx: &mut FunctionEnvMut<'_, WasiEnv>, fd: Fd) -> anyhow::Result<()> { diff --git a/lib/wasix/src/snapshot/effector/syscalls/epoll_ctl.rs b/lib/wasix/src/journal/effector/syscalls/epoll_ctl.rs similarity index 95% rename from lib/wasix/src/snapshot/effector/syscalls/epoll_ctl.rs rename to lib/wasix/src/journal/effector/syscalls/epoll_ctl.rs index 4d9c7817c29..fdde696e7ee 100644 --- a/lib/wasix/src/snapshot/effector/syscalls/epoll_ctl.rs +++ b/lib/wasix/src/journal/effector/syscalls/epoll_ctl.rs @@ -1,6 +1,6 @@ use super::*; -impl SnapshotEffector { +impl JournalEffector { pub fn save_epoll_ctl( ctx: &mut FunctionEnvMut<'_, WasiEnv>, epfd: Fd, @@ -10,7 +10,7 @@ impl SnapshotEffector { ) -> anyhow::Result<()> { Self::save_event( ctx, - SnapshotLog::EpollCtl { + JournalEntry::EpollCtl { epfd, op: op.into(), fd, diff --git a/lib/wasix/src/snapshot/effector/syscalls/fd_advise.rs b/lib/wasix/src/journal/effector/syscalls/fd_advise.rs similarity index 93% rename from lib/wasix/src/snapshot/effector/syscalls/fd_advise.rs rename to lib/wasix/src/journal/effector/syscalls/fd_advise.rs index 005ca3918ab..8a6b7a89953 100644 --- a/lib/wasix/src/snapshot/effector/syscalls/fd_advise.rs +++ b/lib/wasix/src/journal/effector/syscalls/fd_advise.rs @@ -1,6 +1,6 @@ use super::*; -impl SnapshotEffector { +impl JournalEffector { pub fn save_fd_advise( ctx: &mut FunctionEnvMut<'_, WasiEnv>, fd: Fd, @@ -10,7 +10,7 @@ impl SnapshotEffector { ) -> anyhow::Result<()> { Self::save_event( ctx, - SnapshotLog::FileDescriptorAdvise { + JournalEntry::FileDescriptorAdvise { fd, offset, len, diff --git a/lib/wasix/src/snapshot/effector/syscalls/fd_allocate.rs b/lib/wasix/src/journal/effector/syscalls/fd_allocate.rs similarity index 84% rename from lib/wasix/src/snapshot/effector/syscalls/fd_allocate.rs rename to lib/wasix/src/journal/effector/syscalls/fd_allocate.rs index eb2af859483..7de1012fcb1 100644 --- a/lib/wasix/src/snapshot/effector/syscalls/fd_allocate.rs +++ b/lib/wasix/src/journal/effector/syscalls/fd_allocate.rs @@ -1,13 +1,16 @@ use super::*; -impl SnapshotEffector { +impl JournalEffector { pub fn save_fd_allocate( ctx: &mut FunctionEnvMut<'_, WasiEnv>, fd: Fd, offset: Filesize, len: Filesize, ) -> anyhow::Result<()> { - Self::save_event(ctx, SnapshotLog::FileDescriptorAllocate { fd, offset, len }) + Self::save_event( + ctx, + JournalEntry::FileDescriptorAllocate { fd, offset, len }, + ) } pub fn apply_fd_allocate( diff --git a/lib/wasix/src/snapshot/effector/syscalls/fd_close.rs b/lib/wasix/src/journal/effector/syscalls/fd_close.rs similarity index 85% rename from lib/wasix/src/snapshot/effector/syscalls/fd_close.rs rename to lib/wasix/src/journal/effector/syscalls/fd_close.rs index 1b07a8b0a33..4d7a593c84d 100644 --- a/lib/wasix/src/snapshot/effector/syscalls/fd_close.rs +++ b/lib/wasix/src/journal/effector/syscalls/fd_close.rs @@ -1,8 +1,8 @@ use super::*; -impl SnapshotEffector { +impl JournalEffector { pub fn save_fd_close(ctx: &mut FunctionEnvMut<'_, WasiEnv>, fd: Fd) -> anyhow::Result<()> { - Self::save_event(ctx, SnapshotLog::CloseFileDescriptor { fd }) + Self::save_event(ctx, JournalEntry::CloseFileDescriptor { fd }) } pub fn apply_fd_close(ctx: &mut FunctionEnvMut<'_, WasiEnv>, fd: Fd) -> anyhow::Result<()> { diff --git a/lib/wasix/src/snapshot/effector/syscalls/fd_duplicate.rs b/lib/wasix/src/journal/effector/syscalls/fd_duplicate.rs similarity index 94% rename from lib/wasix/src/snapshot/effector/syscalls/fd_duplicate.rs rename to lib/wasix/src/journal/effector/syscalls/fd_duplicate.rs index 9f0c2d5ab2a..588c9127e9f 100644 --- a/lib/wasix/src/snapshot/effector/syscalls/fd_duplicate.rs +++ b/lib/wasix/src/journal/effector/syscalls/fd_duplicate.rs @@ -1,6 +1,6 @@ use super::*; -impl SnapshotEffector { +impl JournalEffector { pub fn save_fd_duplicate( ctx: &mut FunctionEnvMut<'_, WasiEnv>, original_fd: Fd, @@ -8,7 +8,7 @@ impl SnapshotEffector { ) -> anyhow::Result<()> { Self::save_event( ctx, - SnapshotLog::DuplicateFileDescriptor { + JournalEntry::DuplicateFileDescriptor { original_fd, copied_fd, }, diff --git a/lib/wasix/src/snapshot/effector/syscalls/fd_pipe.rs b/lib/wasix/src/journal/effector/syscalls/fd_pipe.rs similarity index 92% rename from lib/wasix/src/snapshot/effector/syscalls/fd_pipe.rs rename to lib/wasix/src/journal/effector/syscalls/fd_pipe.rs index 100338cf835..c96b710de60 100644 --- a/lib/wasix/src/snapshot/effector/syscalls/fd_pipe.rs +++ b/lib/wasix/src/journal/effector/syscalls/fd_pipe.rs @@ -1,12 +1,12 @@ use super::*; -impl SnapshotEffector { +impl JournalEffector { pub fn save_fd_pipe( ctx: &mut FunctionEnvMut<'_, WasiEnv>, fd1: Fd, fd2: Fd, ) -> anyhow::Result<()> { - Self::save_event(ctx, SnapshotLog::CreatePipe { fd1, fd2 }) + Self::save_event(ctx, JournalEntry::CreatePipe { fd1, fd2 }) } pub fn apply_fd_pipe( diff --git a/lib/wasix/src/snapshot/effector/syscalls/fd_renumber.rs b/lib/wasix/src/journal/effector/syscalls/fd_renumber.rs similarity index 91% rename from lib/wasix/src/snapshot/effector/syscalls/fd_renumber.rs rename to lib/wasix/src/journal/effector/syscalls/fd_renumber.rs index 1423f7e3a96..b4e59c4d725 100644 --- a/lib/wasix/src/snapshot/effector/syscalls/fd_renumber.rs +++ b/lib/wasix/src/journal/effector/syscalls/fd_renumber.rs @@ -1,6 +1,6 @@ use super::*; -impl SnapshotEffector { +impl JournalEffector { pub fn save_fd_renumber( ctx: &mut FunctionEnvMut<'_, WasiEnv>, from: Fd, @@ -8,7 +8,7 @@ impl SnapshotEffector { ) -> anyhow::Result<()> { Self::save_event( ctx, - SnapshotLog::RenumberFileDescriptor { + JournalEntry::RenumberFileDescriptor { old_fd: from, new_fd: to, }, diff --git a/lib/wasix/src/snapshot/effector/syscalls/fd_seek.rs b/lib/wasix/src/journal/effector/syscalls/fd_seek.rs similarity index 86% rename from lib/wasix/src/snapshot/effector/syscalls/fd_seek.rs rename to lib/wasix/src/journal/effector/syscalls/fd_seek.rs index 10de379ff39..6816b09c7fe 100644 --- a/lib/wasix/src/snapshot/effector/syscalls/fd_seek.rs +++ b/lib/wasix/src/journal/effector/syscalls/fd_seek.rs @@ -1,13 +1,13 @@ use super::*; -impl SnapshotEffector { +impl JournalEffector { pub fn save_fd_seek( ctx: &mut FunctionEnvMut<'_, WasiEnv>, fd: Fd, offset: i64, whence: Whence, ) -> anyhow::Result<()> { - Self::save_event(ctx, SnapshotLog::FileDescriptorSeek { fd, offset, whence }) + Self::save_event(ctx, JournalEntry::FileDescriptorSeek { fd, offset, whence }) } pub async fn apply_fd_seek( diff --git a/lib/wasix/src/snapshot/effector/syscalls/fd_set_flags.rs b/lib/wasix/src/journal/effector/syscalls/fd_set_flags.rs similarity index 86% rename from lib/wasix/src/snapshot/effector/syscalls/fd_set_flags.rs rename to lib/wasix/src/journal/effector/syscalls/fd_set_flags.rs index de8208e54f4..19246c006cf 100644 --- a/lib/wasix/src/snapshot/effector/syscalls/fd_set_flags.rs +++ b/lib/wasix/src/journal/effector/syscalls/fd_set_flags.rs @@ -1,12 +1,12 @@ use super::*; -impl SnapshotEffector { +impl JournalEffector { pub fn save_fd_set_flags( ctx: &mut FunctionEnvMut<'_, WasiEnv>, fd: Fd, flags: Fdflags, ) -> anyhow::Result<()> { - Self::save_event(ctx, SnapshotLog::FileDescriptorSetFlags { fd, flags }) + Self::save_event(ctx, JournalEntry::FileDescriptorSetFlags { fd, flags }) } pub fn apply_fd_set_flags( diff --git a/lib/wasix/src/snapshot/effector/syscalls/fd_set_rights.rs b/lib/wasix/src/journal/effector/syscalls/fd_set_rights.rs similarity index 93% rename from lib/wasix/src/snapshot/effector/syscalls/fd_set_rights.rs rename to lib/wasix/src/journal/effector/syscalls/fd_set_rights.rs index 266d75e4031..95a7ace012a 100644 --- a/lib/wasix/src/snapshot/effector/syscalls/fd_set_rights.rs +++ b/lib/wasix/src/journal/effector/syscalls/fd_set_rights.rs @@ -1,6 +1,6 @@ use super::*; -impl SnapshotEffector { +impl JournalEffector { pub fn save_fd_set_rights( ctx: &mut FunctionEnvMut<'_, WasiEnv>, fd: Fd, @@ -9,7 +9,7 @@ impl SnapshotEffector { ) -> anyhow::Result<()> { Self::save_event( ctx, - SnapshotLog::FileDescriptorSetRights { + JournalEntry::FileDescriptorSetRights { fd, fs_rights_base, fs_rights_inheriting, diff --git a/lib/wasix/src/snapshot/effector/syscalls/fd_set_size.rs b/lib/wasix/src/journal/effector/syscalls/fd_set_size.rs similarity index 86% rename from lib/wasix/src/snapshot/effector/syscalls/fd_set_size.rs rename to lib/wasix/src/journal/effector/syscalls/fd_set_size.rs index eb4b0bc2c91..67a1f1d62a1 100644 --- a/lib/wasix/src/snapshot/effector/syscalls/fd_set_size.rs +++ b/lib/wasix/src/journal/effector/syscalls/fd_set_size.rs @@ -1,12 +1,12 @@ use super::*; -impl SnapshotEffector { +impl JournalEffector { pub fn save_fd_set_size( ctx: &mut FunctionEnvMut<'_, WasiEnv>, fd: Fd, st_size: Filesize, ) -> anyhow::Result<()> { - Self::save_event(ctx, SnapshotLog::FileDescriptorSetSize { fd, st_size }) + Self::save_event(ctx, JournalEntry::FileDescriptorSetSize { fd, st_size }) } pub fn apply_fd_set_size( diff --git a/lib/wasix/src/snapshot/effector/syscalls/fd_set_times.rs b/lib/wasix/src/journal/effector/syscalls/fd_set_times.rs similarity index 93% rename from lib/wasix/src/snapshot/effector/syscalls/fd_set_times.rs rename to lib/wasix/src/journal/effector/syscalls/fd_set_times.rs index e13aa4c0315..492ad413907 100644 --- a/lib/wasix/src/snapshot/effector/syscalls/fd_set_times.rs +++ b/lib/wasix/src/journal/effector/syscalls/fd_set_times.rs @@ -1,6 +1,6 @@ use super::*; -impl SnapshotEffector { +impl JournalEffector { pub fn save_fd_set_times( ctx: &mut FunctionEnvMut<'_, WasiEnv>, fd: Fd, @@ -10,7 +10,7 @@ impl SnapshotEffector { ) -> anyhow::Result<()> { Self::save_event( ctx, - SnapshotLog::FileDescriptorSetTimes { + JournalEntry::FileDescriptorSetTimes { fd, st_atim, st_mtim, diff --git a/lib/wasix/src/snapshot/effector/syscalls/fd_write.rs b/lib/wasix/src/journal/effector/syscalls/fd_write.rs similarity index 96% rename from lib/wasix/src/snapshot/effector/syscalls/fd_write.rs rename to lib/wasix/src/journal/effector/syscalls/fd_write.rs index c9b844207f1..86491d81464 100644 --- a/lib/wasix/src/snapshot/effector/syscalls/fd_write.rs +++ b/lib/wasix/src/journal/effector/syscalls/fd_write.rs @@ -1,6 +1,6 @@ use super::*; -impl SnapshotEffector { +impl JournalEffector { pub fn save_fd_write( ctx: &mut FunctionEnvMut<'_, WasiEnv>, fd: Fd, @@ -31,7 +31,7 @@ impl SnapshotEffector { ctx.data() .runtime() .snapshot_capturer() - .write(SnapshotLog::FileDescriptorWrite { + .write(JournalEntry::FileDescriptorWrite { fd, offset, data: Cow::Borrowed(buf.as_ref()), diff --git a/lib/wasix/src/snapshot/effector/syscalls/path_create_directory.rs b/lib/wasix/src/journal/effector/syscalls/path_create_directory.rs similarity index 92% rename from lib/wasix/src/snapshot/effector/syscalls/path_create_directory.rs rename to lib/wasix/src/journal/effector/syscalls/path_create_directory.rs index 11ce32a96af..5c6c8f83ed5 100644 --- a/lib/wasix/src/snapshot/effector/syscalls/path_create_directory.rs +++ b/lib/wasix/src/journal/effector/syscalls/path_create_directory.rs @@ -1,6 +1,6 @@ use super::*; -impl SnapshotEffector { +impl JournalEffector { pub fn save_path_create_directory( ctx: &mut FunctionEnvMut<'_, WasiEnv>, fd: Fd, @@ -8,7 +8,7 @@ impl SnapshotEffector { ) -> anyhow::Result<()> { Self::save_event( ctx, - SnapshotLog::CreateDirectory { + JournalEntry::CreateDirectory { fd, path: path.into(), }, diff --git a/lib/wasix/src/snapshot/effector/syscalls/path_link.rs b/lib/wasix/src/journal/effector/syscalls/path_link.rs similarity index 94% rename from lib/wasix/src/snapshot/effector/syscalls/path_link.rs rename to lib/wasix/src/journal/effector/syscalls/path_link.rs index 8cf151c9166..24abed63792 100644 --- a/lib/wasix/src/snapshot/effector/syscalls/path_link.rs +++ b/lib/wasix/src/journal/effector/syscalls/path_link.rs @@ -1,6 +1,6 @@ use super::*; -impl SnapshotEffector { +impl JournalEffector { pub fn save_path_link( ctx: &mut FunctionEnvMut<'_, WasiEnv>, old_fd: Fd, @@ -11,7 +11,7 @@ impl SnapshotEffector { ) -> anyhow::Result<()> { Self::save_event( ctx, - SnapshotLog::CreateHardLink { + JournalEntry::CreateHardLink { old_fd, old_flags, old_path: old_path.into(), diff --git a/lib/wasix/src/snapshot/effector/syscalls/path_open.rs b/lib/wasix/src/journal/effector/syscalls/path_open.rs similarity index 97% rename from lib/wasix/src/snapshot/effector/syscalls/path_open.rs rename to lib/wasix/src/journal/effector/syscalls/path_open.rs index b82223b88e5..ecd11695840 100644 --- a/lib/wasix/src/snapshot/effector/syscalls/path_open.rs +++ b/lib/wasix/src/journal/effector/syscalls/path_open.rs @@ -1,6 +1,6 @@ use super::*; -impl SnapshotEffector { +impl JournalEffector { pub fn save_path_open( ctx: &mut FunctionEnvMut<'_, WasiEnv>, fd: Fd, @@ -15,7 +15,7 @@ impl SnapshotEffector { ) -> anyhow::Result<()> { Self::save_event( ctx, - SnapshotLog::OpenFileDescriptor { + JournalEntry::OpenFileDescriptor { fd, dirfd, dirflags, diff --git a/lib/wasix/src/snapshot/effector/syscalls/path_remove_directory.rs b/lib/wasix/src/journal/effector/syscalls/path_remove_directory.rs similarity index 91% rename from lib/wasix/src/snapshot/effector/syscalls/path_remove_directory.rs rename to lib/wasix/src/journal/effector/syscalls/path_remove_directory.rs index 18174fb3885..4b7a2b2a3dd 100644 --- a/lib/wasix/src/snapshot/effector/syscalls/path_remove_directory.rs +++ b/lib/wasix/src/journal/effector/syscalls/path_remove_directory.rs @@ -1,6 +1,6 @@ use super::*; -impl SnapshotEffector { +impl JournalEffector { pub fn save_path_remove_directory( ctx: &mut FunctionEnvMut<'_, WasiEnv>, fd: Fd, @@ -8,7 +8,7 @@ impl SnapshotEffector { ) -> anyhow::Result<()> { Self::save_event( ctx, - SnapshotLog::RemoveDirectory { + JournalEntry::RemoveDirectory { fd, path: Cow::Owned(path), }, diff --git a/lib/wasix/src/snapshot/effector/syscalls/path_rename.rs b/lib/wasix/src/journal/effector/syscalls/path_rename.rs similarity index 94% rename from lib/wasix/src/snapshot/effector/syscalls/path_rename.rs rename to lib/wasix/src/journal/effector/syscalls/path_rename.rs index 13f8f868809..ef692a8f467 100644 --- a/lib/wasix/src/snapshot/effector/syscalls/path_rename.rs +++ b/lib/wasix/src/journal/effector/syscalls/path_rename.rs @@ -1,6 +1,6 @@ use super::*; -impl SnapshotEffector { +impl JournalEffector { pub fn save_path_rename( ctx: &mut FunctionEnvMut<'_, WasiEnv>, old_fd: Fd, @@ -10,7 +10,7 @@ impl SnapshotEffector { ) -> anyhow::Result<()> { Self::save_event( ctx, - SnapshotLog::PathRename { + JournalEntry::PathRename { old_fd, old_path: Cow::Owned(old_path), new_fd, diff --git a/lib/wasix/src/snapshot/effector/syscalls/path_set_times.rs b/lib/wasix/src/journal/effector/syscalls/path_set_times.rs similarity index 95% rename from lib/wasix/src/snapshot/effector/syscalls/path_set_times.rs rename to lib/wasix/src/journal/effector/syscalls/path_set_times.rs index 287239542e3..e0a4cb31e42 100644 --- a/lib/wasix/src/snapshot/effector/syscalls/path_set_times.rs +++ b/lib/wasix/src/journal/effector/syscalls/path_set_times.rs @@ -1,6 +1,6 @@ use super::*; -impl SnapshotEffector { +impl JournalEffector { pub fn save_path_set_times( ctx: &mut FunctionEnvMut<'_, WasiEnv>, fd: Fd, @@ -12,7 +12,7 @@ impl SnapshotEffector { ) -> anyhow::Result<()> { Self::save_event( ctx, - SnapshotLog::PathSetTimes { + JournalEntry::PathSetTimes { fd, flags, path: path.into(), diff --git a/lib/wasix/src/snapshot/effector/syscalls/path_symlink.rs b/lib/wasix/src/journal/effector/syscalls/path_symlink.rs similarity index 93% rename from lib/wasix/src/snapshot/effector/syscalls/path_symlink.rs rename to lib/wasix/src/journal/effector/syscalls/path_symlink.rs index 987d9ed1939..b63f2ca492a 100644 --- a/lib/wasix/src/snapshot/effector/syscalls/path_symlink.rs +++ b/lib/wasix/src/journal/effector/syscalls/path_symlink.rs @@ -1,6 +1,6 @@ use super::*; -impl SnapshotEffector { +impl JournalEffector { pub fn save_path_symlink( ctx: &mut FunctionEnvMut<'_, WasiEnv>, old_path: String, @@ -9,7 +9,7 @@ impl SnapshotEffector { ) -> anyhow::Result<()> { Self::save_event( ctx, - SnapshotLog::CreateSymbolicLink { + JournalEntry::CreateSymbolicLink { old_path: old_path.into(), fd, new_path: new_path.into(), diff --git a/lib/wasix/src/snapshot/effector/syscalls/path_unlink.rs b/lib/wasix/src/journal/effector/syscalls/path_unlink.rs similarity index 92% rename from lib/wasix/src/snapshot/effector/syscalls/path_unlink.rs rename to lib/wasix/src/journal/effector/syscalls/path_unlink.rs index dda892cfdad..073431ab39c 100644 --- a/lib/wasix/src/snapshot/effector/syscalls/path_unlink.rs +++ b/lib/wasix/src/journal/effector/syscalls/path_unlink.rs @@ -1,6 +1,6 @@ use super::*; -impl SnapshotEffector { +impl JournalEffector { pub fn save_path_unlink( ctx: &mut FunctionEnvMut<'_, WasiEnv>, fd: Fd, @@ -8,7 +8,7 @@ impl SnapshotEffector { ) -> anyhow::Result<()> { Self::save_event( ctx, - SnapshotLog::UnlinkFile { + JournalEntry::UnlinkFile { fd, path: Cow::Owned(path), }, diff --git a/lib/wasix/src/snapshot/effector/syscalls/tty_set.rs b/lib/wasix/src/journal/effector/syscalls/tty_set.rs similarity index 94% rename from lib/wasix/src/snapshot/effector/syscalls/tty_set.rs rename to lib/wasix/src/journal/effector/syscalls/tty_set.rs index 4650751107f..bdffe6d3d81 100644 --- a/lib/wasix/src/snapshot/effector/syscalls/tty_set.rs +++ b/lib/wasix/src/journal/effector/syscalls/tty_set.rs @@ -2,14 +2,14 @@ use crate::WasiTtyState; use super::*; -impl SnapshotEffector { +impl JournalEffector { pub fn save_tty_set( ctx: &mut FunctionEnvMut<'_, WasiEnv>, state: WasiTtyState, ) -> anyhow::Result<()> { Self::save_event( ctx, - SnapshotLog::TtySet { + JournalEntry::TtySet { tty: wasmer_wasix_types::wasi::Tty { cols: state.cols, rows: state.rows, diff --git a/lib/wasix/src/snapshot/effector/thread_exit.rs b/lib/wasix/src/journal/effector/thread_exit.rs similarity index 83% rename from lib/wasix/src/snapshot/effector/thread_exit.rs rename to lib/wasix/src/journal/effector/thread_exit.rs index d496eb8c272..316bdb9306a 100644 --- a/lib/wasix/src/snapshot/effector/thread_exit.rs +++ b/lib/wasix/src/journal/effector/thread_exit.rs @@ -1,6 +1,6 @@ use super::*; -impl SnapshotEffector { +impl JournalEffector { pub fn save_thread_exit( env: &WasiEnv, id: WasiThreadId, @@ -9,7 +9,7 @@ impl SnapshotEffector { __asyncify_light(env, None, async { env.runtime() .snapshot_capturer() - .write(SnapshotLog::CloseThread { id, exit_code }) + .write(JournalEntry::CloseThread { id, exit_code }) .await .map_err(map_snapshot_err)?; Ok(()) diff --git a/lib/wasix/src/snapshot/effector/thread_state.rs b/lib/wasix/src/journal/effector/thread_state.rs similarity index 90% rename from lib/wasix/src/snapshot/effector/thread_state.rs rename to lib/wasix/src/journal/effector/thread_state.rs index 925d67a5996..fe2dec80a20 100644 --- a/lib/wasix/src/snapshot/effector/thread_state.rs +++ b/lib/wasix/src/journal/effector/thread_state.rs @@ -1,6 +1,6 @@ use super::*; -impl SnapshotEffector { +impl JournalEffector { pub fn save_thread_state( ctx: &mut FunctionEnvMut<'_, WasiEnv>, id: WasiThreadId, @@ -10,7 +10,7 @@ impl SnapshotEffector { ) -> anyhow::Result<()> { Self::save_event( ctx, - SnapshotLog::SetThread { + JournalEntry::SetThread { id, call_stack: Cow::Owned(rewind_stack.into()), memory_stack: Cow::Owned(memory_stack.into()), diff --git a/lib/wasix/src/journal/effector/unimplemented.rs b/lib/wasix/src/journal/effector/unimplemented.rs new file mode 100644 index 00000000000..6e82767e3bb --- /dev/null +++ b/lib/wasix/src/journal/effector/unimplemented.rs @@ -0,0 +1,4 @@ +#[derive(Debug, Clone)] +pub struct JournalEffector {} + +impl JournalEffector {} diff --git a/lib/wasix/src/snapshot/filter.rs b/lib/wasix/src/journal/filter.rs similarity index 74% rename from lib/wasix/src/snapshot/filter.rs rename to lib/wasix/src/journal/filter.rs index 65b8dca395f..57c77ba2a37 100644 --- a/lib/wasix/src/snapshot/filter.rs +++ b/lib/wasix/src/journal/filter.rs @@ -2,11 +2,11 @@ use futures::future::LocalBoxFuture; use super::*; -/// Filters out a specific set of log events and drops the rest, this -/// capturer can be useful for restoring to a previous call point but +/// Filters out a specific set of journal events and drops the rest, this +/// journal can be useful for restoring to a previous call point but /// retaining the memory changes (e.g. WCGI runner). -pub struct FilteredSnapshotCapturer { - inner: Box, +pub struct FilteredJournal { + inner: Box, filter_memory: bool, filter_threads: bool, filter_files: bool, @@ -18,8 +18,8 @@ pub struct FilteredSnapshotCapturer { filter_epoll: bool, } -impl FilteredSnapshotCapturer { - pub fn new(inner: Box) -> Self { +impl FilteredJournal { + pub fn new(inner: Box) -> Self { Self { inner, filter_memory: false, @@ -80,181 +80,181 @@ impl FilteredSnapshotCapturer { } } -impl SnapshotCapturer for FilteredSnapshotCapturer { - fn write<'a>(&'a self, entry: SnapshotLog<'a>) -> LocalBoxFuture<'a, anyhow::Result<()>> { +impl Journal for FilteredJournal { + fn write<'a>(&'a self, entry: JournalEntry<'a>) -> LocalBoxFuture<'a, anyhow::Result<()>> { Box::pin(async { let evt = match entry { - SnapshotLog::Init { wasm_hash } => SnapshotLog::Init { wasm_hash }, - SnapshotLog::FileDescriptorSeek { .. } => { + JournalEntry::Init { wasm_hash } => JournalEntry::Init { wasm_hash }, + JournalEntry::FileDescriptorSeek { .. } => { if self.filter_descriptors { return Ok(()); } entry } - SnapshotLog::FileDescriptorWrite { .. } => { + JournalEntry::FileDescriptorWrite { .. } => { if self.filter_descriptors { return Ok(()); } entry } - SnapshotLog::UpdateMemoryRegion { .. } => { + JournalEntry::UpdateMemoryRegion { .. } => { if self.filter_memory { return Ok(()); } entry } - SnapshotLog::CloseThread { .. } => { + JournalEntry::CloseThread { .. } => { if self.filter_threads { return Ok(()); } entry } - SnapshotLog::SetThread { .. } => { + JournalEntry::SetThread { .. } => { if self.filter_threads { return Ok(()); } entry } - SnapshotLog::CloseFileDescriptor { .. } => { + JournalEntry::CloseFileDescriptor { .. } => { if self.filter_descriptors { return Ok(()); } entry } - SnapshotLog::OpenFileDescriptor { .. } => { + JournalEntry::OpenFileDescriptor { .. } => { if self.filter_descriptors { return Ok(()); } entry } - SnapshotLog::RemoveDirectory { .. } => { + JournalEntry::RemoveDirectory { .. } => { if self.filter_files { return Ok(()); } entry } - SnapshotLog::UnlinkFile { .. } => { + JournalEntry::UnlinkFile { .. } => { if self.filter_files { return Ok(()); } entry } - SnapshotLog::PathRename { .. } => { + JournalEntry::PathRename { .. } => { if self.filter_files { return Ok(()); } entry } - SnapshotLog::Snapshot { .. } => { + JournalEntry::Snapshot { .. } => { if self.filter_snapshots { return Ok(()); } entry } - SnapshotLog::SetClockTime { .. } => { + JournalEntry::SetClockTime { .. } => { if self.filter_clock { return Ok(()); } entry } - SnapshotLog::RenumberFileDescriptor { .. } => { + JournalEntry::RenumberFileDescriptor { .. } => { if self.filter_terminal { return Ok(()); } entry } - SnapshotLog::DuplicateFileDescriptor { .. } => { + JournalEntry::DuplicateFileDescriptor { .. } => { if self.filter_terminal { return Ok(()); } entry } - SnapshotLog::CreateDirectory { .. } => { + JournalEntry::CreateDirectory { .. } => { if self.filter_files { return Ok(()); } entry } - SnapshotLog::PathSetTimes { .. } => { + JournalEntry::PathSetTimes { .. } => { if self.filter_files { return Ok(()); } entry } - SnapshotLog::FileDescriptorSetFlags { .. } => { + JournalEntry::FileDescriptorSetFlags { .. } => { if self.filter_descriptors { return Ok(()); } entry } - SnapshotLog::FileDescriptorAdvise { .. } => { + JournalEntry::FileDescriptorAdvise { .. } => { if self.filter_descriptors { return Ok(()); } entry } - SnapshotLog::FileDescriptorAllocate { .. } => { + JournalEntry::FileDescriptorAllocate { .. } => { if self.filter_descriptors { return Ok(()); } entry } - SnapshotLog::FileDescriptorSetRights { .. } => { + JournalEntry::FileDescriptorSetRights { .. } => { if self.filter_descriptors { return Ok(()); } entry } - SnapshotLog::FileDescriptorSetTimes { .. } => { + JournalEntry::FileDescriptorSetTimes { .. } => { if self.filter_descriptors { return Ok(()); } entry } - SnapshotLog::FileDescriptorSetSize { .. } => { + JournalEntry::FileDescriptorSetSize { .. } => { if self.filter_descriptors { return Ok(()); } entry } - SnapshotLog::CreateHardLink { .. } => { + JournalEntry::CreateHardLink { .. } => { if self.filter_files { return Ok(()); } entry } - SnapshotLog::CreateSymbolicLink { .. } => { + JournalEntry::CreateSymbolicLink { .. } => { if self.filter_files { return Ok(()); } entry } - SnapshotLog::ChangeDirectory { .. } => { + JournalEntry::ChangeDirectory { .. } => { if self.filter_chdir { return Ok(()); } entry } - SnapshotLog::EpollCreate { .. } => { + JournalEntry::EpollCreate { .. } => { if self.filter_memory { return Ok(()); } entry } - SnapshotLog::EpollCtl { .. } => { + JournalEntry::EpollCtl { .. } => { if self.filter_memory { return Ok(()); } entry } - SnapshotLog::TtySet { .. } => { + JournalEntry::TtySet { .. } => { if self.filter_terminal { return Ok(()); } entry } - SnapshotLog::CreatePipe { .. } => { + JournalEntry::CreatePipe { .. } => { if self.filter_files { return Ok(()); } @@ -265,7 +265,7 @@ impl SnapshotCapturer for FilteredSnapshotCapturer { }) } - fn read<'a>(&'a self) -> LocalBoxFuture<'a, anyhow::Result>>> { + fn read<'a>(&'a self) -> LocalBoxFuture<'a, anyhow::Result>>> { Box::pin(async { self.inner.read().await }) } } diff --git a/lib/wasix/src/snapshot/log_file.rs b/lib/wasix/src/journal/log_file.rs similarity index 64% rename from lib/wasix/src/snapshot/log_file.rs rename to lib/wasix/src/journal/log_file.rs index 787c95355a3..9ed7dd8a768 100644 --- a/lib/wasix/src/snapshot/log_file.rs +++ b/lib/wasix/src/journal/log_file.rs @@ -15,21 +15,21 @@ use crate::WasiThreadId; use super::*; -/// The snapshot log entries are serializable which +/// The journal log entries are serializable which /// allows them to be written directly to a file /// /// Note: This structure is versioned which allows for -/// changes to the log entry types without having to +/// changes to the journal entry types without having to /// worry about backward and forward compatibility #[derive(Debug, Clone, Serialize, Deserialize)] -pub enum SnapshotLogEntry { +pub(crate) enum LogFileJournalEntry { InitV1 { wasm_hash: [u8; 32], }, FileDescriptorSeekV1 { fd: Fd, offset: i64, - whence: WhenceV1, + whence: JournalWhenceV1, }, FileDescriptorWriteV1 { fd: u32, @@ -43,7 +43,7 @@ pub enum SnapshotLogEntry { data: Vec, }, SetClockTimeV1 { - clock_id: Snapshot0ClockidV1, + clock_id: JournalSnapshot0ClockidV1, time: u64, }, SetThreadV1 { @@ -55,7 +55,7 @@ pub enum SnapshotLogEntry { }, CloseThreadV1 { id: WasiThreadId, - exit_code: Option, + exit_code: Option, }, CloseFileDescriptorV1 { fd: u32, @@ -118,7 +118,7 @@ pub enum SnapshotLogEntry { fd: u32, offset: u64, len: u64, - advice: AdviceV1, + advice: JournalAdviceV1, }, FileDescriptorAllocateV1 { fd: u32, @@ -155,7 +155,7 @@ pub enum SnapshotLogEntry { }, EpollCtlV1 { epfd: u32, - op: EpollCtlV1, + op: JournalEpollCtlV1, fd: u32, event: Option, }, @@ -177,12 +177,12 @@ pub enum SnapshotLogEntry { }, SnapshotV1 { when: SystemTime, - trigger: SnapshotTriggerV1, + trigger: JournalSnapshotTriggerV1, }, } #[derive(Debug, Clone, Serialize, Deserialize)] -pub enum Snapshot0ClockidV1 { +pub(crate) enum JournalSnapshot0ClockidV1 { Realtime, Monotonic, ProcessCputimeId, @@ -190,62 +190,62 @@ pub enum Snapshot0ClockidV1 { Unknown = 255, } -impl Into for wasi::Snapshot0Clockid { - fn into(self) -> Snapshot0ClockidV1 { +impl Into for wasi::Snapshot0Clockid { + fn into(self) -> JournalSnapshot0ClockidV1 { match self { - Snapshot0Clockid::Realtime => Snapshot0ClockidV1::Realtime, - Snapshot0Clockid::Monotonic => Snapshot0ClockidV1::Monotonic, - Snapshot0Clockid::ProcessCputimeId => Snapshot0ClockidV1::ProcessCputimeId, - Snapshot0Clockid::ThreadCputimeId => Snapshot0ClockidV1::ThreadCputimeId, - Snapshot0Clockid::Unknown => Snapshot0ClockidV1::Unknown, + Snapshot0Clockid::Realtime => JournalSnapshot0ClockidV1::Realtime, + Snapshot0Clockid::Monotonic => JournalSnapshot0ClockidV1::Monotonic, + Snapshot0Clockid::ProcessCputimeId => JournalSnapshot0ClockidV1::ProcessCputimeId, + Snapshot0Clockid::ThreadCputimeId => JournalSnapshot0ClockidV1::ThreadCputimeId, + Snapshot0Clockid::Unknown => JournalSnapshot0ClockidV1::Unknown, } } } -impl Into for Snapshot0ClockidV1 { +impl Into for JournalSnapshot0ClockidV1 { fn into(self) -> wasi::Snapshot0Clockid { match self { - Snapshot0ClockidV1::Realtime => Snapshot0Clockid::Realtime, - Snapshot0ClockidV1::Monotonic => Snapshot0Clockid::Monotonic, - Snapshot0ClockidV1::ProcessCputimeId => Snapshot0Clockid::ProcessCputimeId, - Snapshot0ClockidV1::ThreadCputimeId => Snapshot0Clockid::ThreadCputimeId, - Snapshot0ClockidV1::Unknown => Snapshot0Clockid::Unknown, + JournalSnapshot0ClockidV1::Realtime => Snapshot0Clockid::Realtime, + JournalSnapshot0ClockidV1::Monotonic => Snapshot0Clockid::Monotonic, + JournalSnapshot0ClockidV1::ProcessCputimeId => Snapshot0Clockid::ProcessCputimeId, + JournalSnapshot0ClockidV1::ThreadCputimeId => Snapshot0Clockid::ThreadCputimeId, + JournalSnapshot0ClockidV1::Unknown => Snapshot0Clockid::Unknown, } } } #[derive(Debug, Clone, Serialize, Deserialize)] -pub enum WhenceV1 { +pub(crate) enum JournalWhenceV1 { Set, Cur, End, Unknown = 255, } -impl Into for wasi::Whence { - fn into(self) -> WhenceV1 { +impl Into for wasi::Whence { + fn into(self) -> JournalWhenceV1 { match self { - wasi::Whence::Set => WhenceV1::Set, - wasi::Whence::Cur => WhenceV1::Cur, - wasi::Whence::End => WhenceV1::End, - wasi::Whence::Unknown => WhenceV1::Unknown, + wasi::Whence::Set => JournalWhenceV1::Set, + wasi::Whence::Cur => JournalWhenceV1::Cur, + wasi::Whence::End => JournalWhenceV1::End, + wasi::Whence::Unknown => JournalWhenceV1::Unknown, } } } -impl Into for WhenceV1 { +impl Into for JournalWhenceV1 { fn into(self) -> wasi::Whence { match self { - WhenceV1::Set => Whence::Set, - WhenceV1::Cur => Whence::Cur, - WhenceV1::End => Whence::End, - WhenceV1::Unknown => Whence::Unknown, + JournalWhenceV1::Set => Whence::Set, + JournalWhenceV1::Cur => Whence::Cur, + JournalWhenceV1::End => Whence::End, + JournalWhenceV1::Unknown => Whence::Unknown, } } } #[derive(Debug, Clone, Serialize, Deserialize)] -pub enum AdviceV1 { +pub(crate) enum JournalAdviceV1 { Normal, Sequential, Random, @@ -255,62 +255,62 @@ pub enum AdviceV1 { Unknown = 255, } -impl Into for wasi::Advice { - fn into(self) -> AdviceV1 { +impl Into for wasi::Advice { + fn into(self) -> JournalAdviceV1 { match self { - Advice::Normal => AdviceV1::Normal, - Advice::Sequential => AdviceV1::Sequential, - Advice::Random => AdviceV1::Random, - Advice::Willneed => AdviceV1::Willneed, - Advice::Dontneed => AdviceV1::Dontneed, - Advice::Noreuse => AdviceV1::Noreuse, - Advice::Unknown => AdviceV1::Unknown, + Advice::Normal => JournalAdviceV1::Normal, + Advice::Sequential => JournalAdviceV1::Sequential, + Advice::Random => JournalAdviceV1::Random, + Advice::Willneed => JournalAdviceV1::Willneed, + Advice::Dontneed => JournalAdviceV1::Dontneed, + Advice::Noreuse => JournalAdviceV1::Noreuse, + Advice::Unknown => JournalAdviceV1::Unknown, } } } -impl Into for AdviceV1 { +impl Into for JournalAdviceV1 { fn into(self) -> wasi::Advice { match self { - AdviceV1::Normal => Advice::Normal, - AdviceV1::Sequential => Advice::Sequential, - AdviceV1::Random => Advice::Random, - AdviceV1::Willneed => Advice::Willneed, - AdviceV1::Dontneed => Advice::Dontneed, - AdviceV1::Noreuse => Advice::Noreuse, - AdviceV1::Unknown => Advice::Unknown, + JournalAdviceV1::Normal => Advice::Normal, + JournalAdviceV1::Sequential => Advice::Sequential, + JournalAdviceV1::Random => Advice::Random, + JournalAdviceV1::Willneed => Advice::Willneed, + JournalAdviceV1::Dontneed => Advice::Dontneed, + JournalAdviceV1::Noreuse => Advice::Noreuse, + JournalAdviceV1::Unknown => Advice::Unknown, } } } #[derive(Debug, Clone, Serialize, Deserialize)] -pub enum ExitCodeV1 { +pub(crate) enum JournalExitCodeV1 { Errno(u16), Other(i32), } -impl Into for wasi::ExitCode { - fn into(self) -> ExitCodeV1 { +impl Into for wasi::ExitCode { + fn into(self) -> JournalExitCodeV1 { match self { - wasi::ExitCode::Errno(errno) => ExitCodeV1::Errno(errno as u16), - wasi::ExitCode::Other(id) => ExitCodeV1::Other(id), + wasi::ExitCode::Errno(errno) => JournalExitCodeV1::Errno(errno as u16), + wasi::ExitCode::Other(id) => JournalExitCodeV1::Other(id), } } } -impl Into for ExitCodeV1 { +impl Into for JournalExitCodeV1 { fn into(self) -> wasi::ExitCode { match self { - ExitCodeV1::Errno(errno) => { + JournalExitCodeV1::Errno(errno) => { wasi::ExitCode::Errno(errno.try_into().unwrap_or(wasi::Errno::Unknown)) } - ExitCodeV1::Other(id) => wasi::ExitCode::Other(id), + JournalExitCodeV1::Other(id) => wasi::ExitCode::Other(id), } } } #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] -pub enum SnapshotTriggerV1 { +pub(crate) enum JournalSnapshotTriggerV1 { Idle, Listen, Environ, @@ -323,75 +323,75 @@ pub enum SnapshotTriggerV1 { NonDeterministicCall, } -impl Into for SnapshotTrigger { - fn into(self) -> SnapshotTriggerV1 { +impl Into for SnapshotTrigger { + fn into(self) -> JournalSnapshotTriggerV1 { match self { - SnapshotTrigger::Idle => SnapshotTriggerV1::Idle, - SnapshotTrigger::Listen => SnapshotTriggerV1::Listen, - SnapshotTrigger::Environ => SnapshotTriggerV1::Environ, - SnapshotTrigger::Stdin => SnapshotTriggerV1::Stdin, - SnapshotTrigger::Timer => SnapshotTriggerV1::Timer, - SnapshotTrigger::Sigint => SnapshotTriggerV1::Sigint, - SnapshotTrigger::Sigalrm => SnapshotTriggerV1::Sigalrm, - SnapshotTrigger::Sigtstp => SnapshotTriggerV1::Sigtstp, - SnapshotTrigger::Sigstop => SnapshotTriggerV1::Sigstop, - SnapshotTrigger::NonDeterministicCall => SnapshotTriggerV1::NonDeterministicCall, + SnapshotTrigger::Idle => JournalSnapshotTriggerV1::Idle, + SnapshotTrigger::Listen => JournalSnapshotTriggerV1::Listen, + SnapshotTrigger::Environ => JournalSnapshotTriggerV1::Environ, + SnapshotTrigger::Stdin => JournalSnapshotTriggerV1::Stdin, + SnapshotTrigger::Interval => JournalSnapshotTriggerV1::Timer, + SnapshotTrigger::Sigint => JournalSnapshotTriggerV1::Sigint, + SnapshotTrigger::Sigalrm => JournalSnapshotTriggerV1::Sigalrm, + SnapshotTrigger::Sigtstp => JournalSnapshotTriggerV1::Sigtstp, + SnapshotTrigger::Sigstop => JournalSnapshotTriggerV1::Sigstop, + SnapshotTrigger::NonDeterministicCall => JournalSnapshotTriggerV1::NonDeterministicCall, } } } -impl Into for SnapshotTriggerV1 { +impl Into for JournalSnapshotTriggerV1 { fn into(self) -> SnapshotTrigger { match self { - SnapshotTriggerV1::Idle => SnapshotTrigger::Idle, - SnapshotTriggerV1::Listen => SnapshotTrigger::Listen, - SnapshotTriggerV1::Environ => SnapshotTrigger::Environ, - SnapshotTriggerV1::Stdin => SnapshotTrigger::Stdin, - SnapshotTriggerV1::Timer => SnapshotTrigger::Timer, - SnapshotTriggerV1::Sigint => SnapshotTrigger::Sigint, - SnapshotTriggerV1::Sigalrm => SnapshotTrigger::Sigalrm, - SnapshotTriggerV1::Sigtstp => SnapshotTrigger::Sigtstp, - SnapshotTriggerV1::Sigstop => SnapshotTrigger::Sigstop, - SnapshotTriggerV1::NonDeterministicCall => SnapshotTrigger::NonDeterministicCall, + JournalSnapshotTriggerV1::Idle => SnapshotTrigger::Idle, + JournalSnapshotTriggerV1::Listen => SnapshotTrigger::Listen, + JournalSnapshotTriggerV1::Environ => SnapshotTrigger::Environ, + JournalSnapshotTriggerV1::Stdin => SnapshotTrigger::Stdin, + JournalSnapshotTriggerV1::Timer => SnapshotTrigger::Interval, + JournalSnapshotTriggerV1::Sigint => SnapshotTrigger::Sigint, + JournalSnapshotTriggerV1::Sigalrm => SnapshotTrigger::Sigalrm, + JournalSnapshotTriggerV1::Sigtstp => SnapshotTrigger::Sigtstp, + JournalSnapshotTriggerV1::Sigstop => SnapshotTrigger::Sigstop, + JournalSnapshotTriggerV1::NonDeterministicCall => SnapshotTrigger::NonDeterministicCall, } } } #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] -pub enum EpollCtlV1 { +pub(crate) enum JournalEpollCtlV1 { Add, Mod, Del, Unknown, } -impl Into for wasi::EpollCtl { - fn into(self) -> EpollCtlV1 { +impl Into for wasi::EpollCtl { + fn into(self) -> JournalEpollCtlV1 { match self { - wasi::EpollCtl::Add => EpollCtlV1::Add, - wasi::EpollCtl::Mod => EpollCtlV1::Mod, - wasi::EpollCtl::Del => EpollCtlV1::Del, - wasi::EpollCtl::Unknown => EpollCtlV1::Unknown, + wasi::EpollCtl::Add => JournalEpollCtlV1::Add, + wasi::EpollCtl::Mod => JournalEpollCtlV1::Mod, + wasi::EpollCtl::Del => JournalEpollCtlV1::Del, + wasi::EpollCtl::Unknown => JournalEpollCtlV1::Unknown, } } } -impl Into for EpollCtlV1 { +impl Into for JournalEpollCtlV1 { fn into(self) -> wasi::EpollCtl { match self { - EpollCtlV1::Add => EpollCtl::Add, - EpollCtlV1::Mod => EpollCtl::Mod, - EpollCtlV1::Del => EpollCtl::Del, - EpollCtlV1::Unknown => EpollCtl::Unknown, + JournalEpollCtlV1::Add => EpollCtl::Add, + JournalEpollCtlV1::Mod => EpollCtl::Mod, + JournalEpollCtlV1::Del => EpollCtl::Del, + JournalEpollCtlV1::Unknown => EpollCtl::Unknown, } } } -impl<'a> From> for SnapshotLogEntry { - fn from(value: SnapshotLog<'a>) -> Self { +impl<'a> From> for LogFileJournalEntry { + fn from(value: JournalEntry<'a>) -> Self { match value { - SnapshotLog::Init { wasm_hash } => Self::InitV1 { wasm_hash }, - SnapshotLog::FileDescriptorWrite { + JournalEntry::Init { wasm_hash } => Self::InitV1 { wasm_hash }, + JournalEntry::FileDescriptorWrite { fd, offset, data, @@ -402,21 +402,21 @@ impl<'a> From> for SnapshotLogEntry { data: data.into_owned(), is_64bit, }, - SnapshotLog::FileDescriptorSeek { fd, offset, whence } => Self::FileDescriptorSeekV1 { + JournalEntry::FileDescriptorSeek { fd, offset, whence } => Self::FileDescriptorSeekV1 { fd, offset, whence: whence.into(), }, - SnapshotLog::UpdateMemoryRegion { region, data } => Self::UpdateMemoryRegionV1 { + JournalEntry::UpdateMemoryRegion { region, data } => Self::UpdateMemoryRegionV1 { start: region.start, end: region.end, data: data.into_owned(), }, - SnapshotLog::CloseThread { id, exit_code } => Self::CloseThreadV1 { + JournalEntry::CloseThread { id, exit_code } => Self::CloseThreadV1 { id, exit_code: exit_code.map(|code| code.into()), }, - SnapshotLog::SetThread { + JournalEntry::SetThread { id, call_stack, memory_stack, @@ -429,8 +429,8 @@ impl<'a> From> for SnapshotLogEntry { store_data: store_data.into_owned(), is_64bit, }, - SnapshotLog::CloseFileDescriptor { fd } => Self::CloseFileDescriptorV1 { fd }, - SnapshotLog::OpenFileDescriptor { + JournalEntry::CloseFileDescriptor { fd } => Self::CloseFileDescriptorV1 { fd }, + JournalEntry::OpenFileDescriptor { fd, dirfd, dirflags, @@ -451,15 +451,15 @@ impl<'a> From> for SnapshotLogEntry { fs_flags: fs_flags.bits(), is_64bit, }, - SnapshotLog::RemoveDirectory { fd, path } => Self::RemoveDirectoryV1 { + JournalEntry::RemoveDirectory { fd, path } => Self::RemoveDirectoryV1 { fd, path: path.into_owned(), }, - SnapshotLog::UnlinkFile { fd, path } => Self::UnlinkFileV1 { + JournalEntry::UnlinkFile { fd, path } => Self::UnlinkFileV1 { fd, path: path.into_owned(), }, - SnapshotLog::PathRename { + JournalEntry::PathRename { old_fd, old_path, new_fd, @@ -470,29 +470,29 @@ impl<'a> From> for SnapshotLogEntry { new_fd, new_path: new_path.into_owned(), }, - SnapshotLog::Snapshot { when, trigger } => Self::SnapshotV1 { + JournalEntry::Snapshot { when, trigger } => Self::SnapshotV1 { when, trigger: trigger.into(), }, - SnapshotLog::SetClockTime { clock_id, time } => Self::SetClockTimeV1 { + JournalEntry::SetClockTime { clock_id, time } => Self::SetClockTimeV1 { clock_id: clock_id.into(), time, }, - SnapshotLog::RenumberFileDescriptor { old_fd, new_fd } => { + JournalEntry::RenumberFileDescriptor { old_fd, new_fd } => { Self::RenumberFileDescriptorV1 { old_fd, new_fd } } - SnapshotLog::DuplicateFileDescriptor { + JournalEntry::DuplicateFileDescriptor { original_fd, copied_fd, } => Self::DuplicateFileDescriptorV1 { original_fd, copied_fd, }, - SnapshotLog::CreateDirectory { fd, path } => Self::CreateDirectoryV1 { + JournalEntry::CreateDirectory { fd, path } => Self::CreateDirectoryV1 { fd, path: path.into_owned(), }, - SnapshotLog::PathSetTimes { + JournalEntry::PathSetTimes { fd, path, flags, @@ -507,7 +507,7 @@ impl<'a> From> for SnapshotLogEntry { st_mtim, fst_flags: fst_flags.bits(), }, - SnapshotLog::FileDescriptorSetTimes { + JournalEntry::FileDescriptorSetTimes { fd, st_atim, st_mtim, @@ -518,14 +518,14 @@ impl<'a> From> for SnapshotLogEntry { st_mtim, fst_flags: fst_flags.bits(), }, - SnapshotLog::FileDescriptorSetSize { fd, st_size } => { + JournalEntry::FileDescriptorSetSize { fd, st_size } => { Self::FileDescriptorSetSizeV1 { fd, st_size } } - SnapshotLog::FileDescriptorSetFlags { fd, flags } => Self::FileDescriptorSetFlagsV1 { + JournalEntry::FileDescriptorSetFlags { fd, flags } => Self::FileDescriptorSetFlagsV1 { fd, flags: flags.bits(), }, - SnapshotLog::FileDescriptorSetRights { + JournalEntry::FileDescriptorSetRights { fd, fs_rights_base, fs_rights_inheriting, @@ -534,7 +534,7 @@ impl<'a> From> for SnapshotLogEntry { fs_rights_base: fs_rights_base.bits(), fs_rights_inheriting: fs_rights_inheriting.bits(), }, - SnapshotLog::FileDescriptorAdvise { + JournalEntry::FileDescriptorAdvise { fd, offset, len, @@ -545,10 +545,10 @@ impl<'a> From> for SnapshotLogEntry { len, advice: advice.into(), }, - SnapshotLog::FileDescriptorAllocate { fd, offset, len } => { + JournalEntry::FileDescriptorAllocate { fd, offset, len } => { Self::FileDescriptorAllocateV1 { fd, offset, len } } - SnapshotLog::CreateHardLink { + JournalEntry::CreateHardLink { old_fd, old_path, old_flags, @@ -561,7 +561,7 @@ impl<'a> From> for SnapshotLogEntry { new_fd, new_path: new_path.into_owned(), }, - SnapshotLog::CreateSymbolicLink { + JournalEntry::CreateSymbolicLink { old_path, fd, new_path, @@ -570,11 +570,11 @@ impl<'a> From> for SnapshotLogEntry { fd, new_path: new_path.into_owned(), }, - SnapshotLog::ChangeDirectory { path } => Self::ChangeDirectoryV1 { + JournalEntry::ChangeDirectory { path } => Self::ChangeDirectoryV1 { path: path.into_owned(), }, - SnapshotLog::EpollCreate { fd } => Self::EpollCreateV1 { fd }, - SnapshotLog::EpollCtl { + JournalEntry::EpollCreate { fd } => Self::EpollCreateV1 { fd }, + JournalEntry::EpollCtl { epfd, op, fd, @@ -585,7 +585,7 @@ impl<'a> From> for SnapshotLogEntry { fd, event, }, - SnapshotLog::TtySet { tty, line_feeds } => Self::TtySetV1 { + JournalEntry::TtySet { tty, line_feeds } => Self::TtySetV1 { cols: tty.cols, rows: tty.rows, width: tty.width, @@ -597,16 +597,16 @@ impl<'a> From> for SnapshotLogEntry { line_buffered: tty.line_buffered, line_feeds, }, - SnapshotLog::CreatePipe { fd1, fd2 } => Self::CreatePipeV1 { fd1, fd2 }, + JournalEntry::CreatePipe { fd1, fd2 } => Self::CreatePipeV1 { fd1, fd2 }, } } } -impl<'a> From for SnapshotLog<'a> { - fn from(value: SnapshotLogEntry) -> Self { +impl<'a> From for JournalEntry<'a> { + fn from(value: LogFileJournalEntry) -> Self { match value { - SnapshotLogEntry::InitV1 { wasm_hash } => Self::Init { wasm_hash }, - SnapshotLogEntry::FileDescriptorWriteV1 { + LogFileJournalEntry::InitV1 { wasm_hash } => Self::Init { wasm_hash }, + LogFileJournalEntry::FileDescriptorWriteV1 { data, fd, offset, @@ -617,24 +617,24 @@ impl<'a> From for SnapshotLog<'a> { offset, is_64bit, }, - SnapshotLogEntry::FileDescriptorSeekV1 { fd, offset, whence } => { + LogFileJournalEntry::FileDescriptorSeekV1 { fd, offset, whence } => { Self::FileDescriptorSeek { fd, offset, whence: whence.into(), } } - SnapshotLogEntry::UpdateMemoryRegionV1 { start, end, data } => { + LogFileJournalEntry::UpdateMemoryRegionV1 { start, end, data } => { Self::UpdateMemoryRegion { region: start..end, data: data.into(), } } - SnapshotLogEntry::CloseThreadV1 { id, exit_code } => Self::CloseThread { + LogFileJournalEntry::CloseThreadV1 { id, exit_code } => Self::CloseThread { id, exit_code: exit_code.map(|code| code.into()), }, - SnapshotLogEntry::SetThreadV1 { + LogFileJournalEntry::SetThreadV1 { id, call_stack, memory_stack, @@ -647,8 +647,8 @@ impl<'a> From for SnapshotLog<'a> { store_data: store_data.into(), is_64bit, }, - SnapshotLogEntry::CloseFileDescriptorV1 { fd } => Self::CloseFileDescriptor { fd }, - SnapshotLogEntry::OpenFileDescriptorV1 { + LogFileJournalEntry::CloseFileDescriptorV1 { fd } => Self::CloseFileDescriptor { fd }, + LogFileJournalEntry::OpenFileDescriptorV1 { fd, dirfd, dirflags, @@ -669,15 +669,15 @@ impl<'a> From for SnapshotLog<'a> { fs_flags: wasi::Fdflags::from_bits_truncate(fs_flags), is_64bit, }, - SnapshotLogEntry::RemoveDirectoryV1 { fd, path } => Self::RemoveDirectory { + LogFileJournalEntry::RemoveDirectoryV1 { fd, path } => Self::RemoveDirectory { fd, path: path.into(), }, - SnapshotLogEntry::UnlinkFileV1 { fd, path } => Self::UnlinkFile { + LogFileJournalEntry::UnlinkFileV1 { fd, path } => Self::UnlinkFile { fd, path: path.into(), }, - SnapshotLogEntry::PathRenameV1 { + LogFileJournalEntry::PathRenameV1 { old_fd, old_path, new_fd, @@ -688,29 +688,29 @@ impl<'a> From for SnapshotLog<'a> { new_fd, new_path: new_path.into(), }, - SnapshotLogEntry::SnapshotV1 { when, trigger } => Self::Snapshot { + LogFileJournalEntry::SnapshotV1 { when, trigger } => Self::Snapshot { when, trigger: trigger.into(), }, - SnapshotLogEntry::SetClockTimeV1 { clock_id, time } => Self::SetClockTime { + LogFileJournalEntry::SetClockTimeV1 { clock_id, time } => Self::SetClockTime { clock_id: clock_id.into(), time, }, - SnapshotLogEntry::RenumberFileDescriptorV1 { old_fd, new_fd } => { + LogFileJournalEntry::RenumberFileDescriptorV1 { old_fd, new_fd } => { Self::RenumberFileDescriptor { old_fd, new_fd } } - SnapshotLogEntry::DuplicateFileDescriptorV1 { + LogFileJournalEntry::DuplicateFileDescriptorV1 { original_fd: old_fd, copied_fd: new_fd, } => Self::DuplicateFileDescriptor { original_fd: old_fd, copied_fd: new_fd, }, - SnapshotLogEntry::CreateDirectoryV1 { fd, path } => Self::CreateDirectory { + LogFileJournalEntry::CreateDirectoryV1 { fd, path } => Self::CreateDirectory { fd, path: path.into(), }, - SnapshotLogEntry::PathSetTimesV1 { + LogFileJournalEntry::PathSetTimesV1 { fd, path, flags, @@ -725,7 +725,7 @@ impl<'a> From for SnapshotLog<'a> { st_mtim, fst_flags: wasi::Fstflags::from_bits_truncate(fst_flags), }, - SnapshotLogEntry::FileDescriptorSetTimesV1 { + LogFileJournalEntry::FileDescriptorSetTimesV1 { fd, st_atim, st_mtim, @@ -736,16 +736,16 @@ impl<'a> From for SnapshotLog<'a> { st_mtim, fst_flags: wasi::Fstflags::from_bits_truncate(fst_flags), }, - SnapshotLogEntry::FileDescriptorSetSizeV1 { fd, st_size } => { + LogFileJournalEntry::FileDescriptorSetSizeV1 { fd, st_size } => { Self::FileDescriptorSetSize { fd, st_size } } - SnapshotLogEntry::FileDescriptorSetFlagsV1 { fd, flags } => { + LogFileJournalEntry::FileDescriptorSetFlagsV1 { fd, flags } => { Self::FileDescriptorSetFlags { fd, flags: Fdflags::from_bits_truncate(flags), } } - SnapshotLogEntry::FileDescriptorSetRightsV1 { + LogFileJournalEntry::FileDescriptorSetRightsV1 { fd, fs_rights_base, fs_rights_inheriting, @@ -754,7 +754,7 @@ impl<'a> From for SnapshotLog<'a> { fs_rights_base: Rights::from_bits_truncate(fs_rights_base), fs_rights_inheriting: Rights::from_bits_truncate(fs_rights_inheriting), }, - SnapshotLogEntry::FileDescriptorAdviseV1 { + LogFileJournalEntry::FileDescriptorAdviseV1 { fd, offset, len, @@ -765,10 +765,10 @@ impl<'a> From for SnapshotLog<'a> { len, advice: advice.into(), }, - SnapshotLogEntry::FileDescriptorAllocateV1 { fd, offset, len } => { + LogFileJournalEntry::FileDescriptorAllocateV1 { fd, offset, len } => { Self::FileDescriptorAllocate { fd, offset, len } } - SnapshotLogEntry::CreateHardLinkV1 { + LogFileJournalEntry::CreateHardLinkV1 { old_fd, old_path, old_flags, @@ -781,7 +781,7 @@ impl<'a> From for SnapshotLog<'a> { new_fd, new_path: new_path.into(), }, - SnapshotLogEntry::CreateSymbolicLinkV1 { + LogFileJournalEntry::CreateSymbolicLinkV1 { old_path, fd, new_path, @@ -790,11 +790,11 @@ impl<'a> From for SnapshotLog<'a> { fd, new_path: new_path.into(), }, - SnapshotLogEntry::ChangeDirectoryV1 { path } => { + LogFileJournalEntry::ChangeDirectoryV1 { path } => { Self::ChangeDirectory { path: path.into() } } - SnapshotLogEntry::EpollCreateV1 { fd } => Self::EpollCreate { fd }, - SnapshotLogEntry::EpollCtlV1 { + LogFileJournalEntry::EpollCreateV1 { fd } => Self::EpollCreate { fd }, + LogFileJournalEntry::EpollCtlV1 { epfd, op, fd, @@ -805,7 +805,7 @@ impl<'a> From for SnapshotLog<'a> { fd, event, }, - SnapshotLogEntry::TtySetV1 { + LogFileJournalEntry::TtySetV1 { cols, rows, width, @@ -830,7 +830,7 @@ impl<'a> From for SnapshotLog<'a> { }, line_feeds, }, - SnapshotLogEntry::CreatePipeV1 { fd1, fd2 } => Self::CreatePipe { fd1, fd2 }, + LogFileJournalEntry::CreatePipeV1 { fd1, fd2 } => Self::CreatePipe { fd1, fd2 }, } } } @@ -851,12 +851,12 @@ struct State { /// /// The logfile snapshot capturer uses a 64bit number as a entry encoding /// delimiter. -pub struct LogFileSnapshotCapturer { +pub struct LogFileJournal { state: tokio::sync::Mutex, handle: Handle, } -impl LogFileSnapshotCapturer { +impl LogFileJournal { pub async fn new(path: impl AsRef) -> io::Result { let state = State { file: tokio::fs::File::options() @@ -891,11 +891,11 @@ impl LogFileSnapshotCapturer { } #[async_trait::async_trait] -impl SnapshotCapturer for LogFileSnapshotCapturer { - fn write<'a>(&'a self, entry: SnapshotLog<'a>) -> LocalBoxFuture<'a, anyhow::Result<()>> { - tracing::debug!("snapshot event: {:?}", entry); +impl Journal for LogFileJournal { + fn write<'a>(&'a self, entry: JournalEntry<'a>) -> LocalBoxFuture<'a, anyhow::Result<()>> { + tracing::debug!("journal event: {:?}", entry); Box::pin(async { - let entry: SnapshotLogEntry = entry.into(); + let entry: LogFileJournalEntry = entry.into(); let _guard = Handle::try_current().map_err(|_| self.handle.enter()); let mut state = self.state.lock().await; @@ -916,7 +916,7 @@ impl SnapshotCapturer for LogFileSnapshotCapturer { /// UNSAFE: This method uses unsafe operations to remove the need to zero /// the buffer before its read the log entries into it - fn read<'a>(&'a self) -> LocalBoxFuture<'a, anyhow::Result>>> { + fn read<'a>(&'a self) -> LocalBoxFuture<'a, anyhow::Result>>> { Box::pin(async { let mut state = self.state.lock().await; @@ -936,7 +936,7 @@ impl SnapshotCapturer for LogFileSnapshotCapturer { data.set_len(data_len as usize); } - let entry: SnapshotLogEntry = bincode::deserialize(&data)?; + let entry: LogFileJournalEntry = bincode::deserialize(&data)?; Ok(Some(entry.into())) }) } diff --git a/lib/wasix/src/snapshot/mod.rs b/lib/wasix/src/journal/mod.rs similarity index 87% rename from lib/wasix/src/snapshot/mod.rs rename to lib/wasix/src/journal/mod.rs index 976ce1a3518..03c8645b53b 100644 --- a/lib/wasix/src/snapshot/mod.rs +++ b/lib/wasix/src/journal/mod.rs @@ -1,12 +1,12 @@ mod capturer; mod compactor; -#[cfg(feature = "snapshot")] +#[cfg(feature = "journal")] mod effector; -#[cfg(not(feature = "snapshot"))] +#[cfg(not(feature = "journal"))] #[path = "effector/unimplemented.rs"] mod effector; mod filter; -#[cfg(feature = "snapshot")] +#[cfg(feature = "journal")] mod log_file; mod unsupported; @@ -14,7 +14,7 @@ pub use capturer::*; pub use compactor::*; pub use effector::*; pub use filter::*; -#[cfg(feature = "snapshot")] +#[cfg(feature = "journal")] pub use log_file::*; pub use unsupported::*; @@ -33,8 +33,8 @@ pub enum SnapshotTrigger { Environ, /// Triggered when the process reads stdin for the first time Stdin, - /// Triggered periodically based on a timer (default 10 seconds) which can be specified using the `snapshot-timer` option - Timer, + /// Triggered periodically based on a interval (default 10 seconds) which can be specified using the `snapshot-interval` option + Interval, /// Issued if the user sends an interrupt signal (Ctrl + C). Sigint, /// Alarm clock signal (used for timers) @@ -57,7 +57,7 @@ impl FromStr for SnapshotTrigger { "listen" => Self::Listen, "stdin" => Self::Stdin, "environ" => Self::Environ, - "periodic" => Self::Timer, + "periodic" => Self::Interval, "intr" | "sigint" | "ctrlc" | "ctrl-c" => Self::Sigint, "alarm" | "timer" | "sigalrm" => Self::Sigalrm, "sigtstp" | "ctrlz" | "ctrl-z" => Self::Sigtstp, diff --git a/lib/wasix/src/journal/unsupported.rs b/lib/wasix/src/journal/unsupported.rs new file mode 100644 index 00000000000..cb712277219 --- /dev/null +++ b/lib/wasix/src/journal/unsupported.rs @@ -0,0 +1,21 @@ +use futures::future::LocalBoxFuture; + +use super::*; + +pub static UNSUPPORTED_SNAPSHOT_CAPTURER: UnsupportedJournal = UnsupportedJournal {}; + +/// The default for runtime is to use the unsupported journal +/// which will fail to write journal entries if one attempts to do so. +#[derive(Debug, Default)] +pub struct UnsupportedJournal {} + +impl Journal for UnsupportedJournal { + fn write<'a>(&'a self, entry: JournalEntry<'a>) -> LocalBoxFuture<'a, anyhow::Result<()>> { + tracing::debug!("snapshot event: {:?}", entry); + Box::pin(async { Err(anyhow::format_err!("unsupported")) }) + } + + fn read<'a>(&'a self) -> LocalBoxFuture<'a, anyhow::Result>>> { + Box::pin(async { Ok(None) }) + } +} diff --git a/lib/wasix/src/lib.rs b/lib/wasix/src/lib.rs index 72f8d89397e..b56703c5183 100644 --- a/lib/wasix/src/lib.rs +++ b/lib/wasix/src/lib.rs @@ -47,10 +47,10 @@ pub mod net; pub mod capabilities; pub mod fs; pub mod http; +pub mod journal; mod rewind; pub mod runners; pub mod runtime; -pub mod snapshot; mod state; mod syscalls; mod utils; @@ -61,7 +61,7 @@ mod bindings; #[allow(unused_imports)] use bytes::{Bytes, BytesMut}; use os::task::control_plane::ControlPlaneError; -use syscalls::state::SnapshotRestore; +use syscalls::state::JournalRestore; use thiserror::Error; use tracing::error; // re-exports needed for OS @@ -232,7 +232,7 @@ impl WasiRuntimeError { pub(crate) fn run_wasi_func( func: &wasmer::Function, store: &mut impl AsStoreMut, - restorer: Option, + restorer: Option, params: &[wasmer::Value], ) -> Result, WasiRuntimeError> { if restorer.is_some() { @@ -264,7 +264,7 @@ pub(crate) fn run_wasi_func( pub(crate) fn run_wasi_func_start( func: &wasmer::Function, store: &mut impl AsStoreMut, - restore: Option, + restore: Option, ) -> Result<(), WasiRuntimeError> { run_wasi_func(func, store, restore, &[])?; Ok(()) diff --git a/lib/wasix/src/os/task/process.rs b/lib/wasix/src/os/task/process.rs index 307ba465a86..33e9927a915 100644 --- a/lib/wasix/src/os/task/process.rs +++ b/lib/wasix/src/os/task/process.rs @@ -1,6 +1,6 @@ -#[cfg(feature = "snapshot")] -use crate::{snapshot::SnapshotEffector, unwind, WasiResult}; -use crate::{snapshot::SnapshotTrigger, WasiEnv, WasiRuntimeError}; +#[cfg(feature = "journal")] +use crate::{journal::JournalEffector, unwind, WasiResult}; +use crate::{journal::SnapshotTrigger, WasiEnv, WasiRuntimeError}; use serde::{Deserialize, Serialize}; use std::{ collections::HashMap, @@ -139,7 +139,7 @@ pub enum MaybeCheckpointResult<'a> { impl WasiProcessInner { /// Checkpoints the process which will cause all other threads to /// pause and for the thread and memory state to be saved - #[cfg(feature = "snapshot")] + #[cfg(feature = "journal")] pub fn checkpoint( inner: LockableWasiProcessInner, ctx: FunctionEnvMut<'_, WasiEnv>, @@ -156,7 +156,7 @@ impl WasiProcessInner { /// If a checkpoint has been started this will block the current process /// until the checkpoint operation has completed - #[cfg(feature = "snapshot")] + #[cfg(feature = "journal")] pub fn maybe_checkpoint( inner: LockableWasiProcessInner, ctx: FunctionEnvMut<'_, WasiEnv>, @@ -196,7 +196,7 @@ impl WasiProcessInner { // Write our thread state to the snapshot let tid = ctx.data().thread.tid(); - if let Err(err) = SnapshotEffector::save_thread_state::( + if let Err(err) = JournalEffector::save_thread_state::( &mut ctx, tid, memory_stack.clone(), @@ -217,9 +217,9 @@ impl WasiProcessInner { // Now if we are the last thread we also write the memory let is_last_thread = guard.threads.values().all(WasiThread::is_check_pointing); if is_last_thread { - if let Err(err) = SnapshotEffector::save_memory_and_snapshot( - &mut ctx, &mut guard, trigger, - ) { + if let Err(err) = + JournalEffector::save_memory_and_snapshot(&mut ctx, &mut guard, trigger) + { inner.1.notify_all(); return wasmer_types::OnCalledAction::Trap(err.into()); } diff --git a/lib/wasix/src/os/task/thread.rs b/lib/wasix/src/os/task/thread.rs index 6e1aa8a9443..5dfe7f4f11f 100644 --- a/lib/wasix/src/os/task/thread.rs +++ b/lib/wasix/src/os/task/thread.rs @@ -1,5 +1,5 @@ use serde::{Deserialize, Serialize}; -#[cfg(feature = "snapshot")] +#[cfg(feature = "journal")] use std::sync::atomic::{AtomicBool, Ordering}; use std::{ collections::HashMap, @@ -131,14 +131,14 @@ impl WasiThread { /// Sets a flag that tells others that this thread is currently /// check pointing itself - #[cfg(feature = "snapshot")] + #[cfg(feature = "journal")] pub(crate) fn set_check_pointing(&self, val: bool) { self.state.check_pointing.store(val, Ordering::SeqCst); } /// Reads a flag that determines if this thread is currently /// check pointing itself or not - #[cfg(feature = "snapshot")] + #[cfg(feature = "journal")] pub(crate) fn is_check_pointing(&self) -> bool { self.state.check_pointing.load(Ordering::SeqCst) } @@ -213,7 +213,7 @@ struct WasiThreadState { signals: Mutex<(Vec, Vec)>, stack: Mutex, status: Arc, - #[cfg(feature = "snapshot")] + #[cfg(feature = "journal")] check_pointing: AtomicBool, // Registers the task termination with the ControlPlane on drop. @@ -240,7 +240,7 @@ impl WasiThread { status, signals: Mutex::new((Vec::new(), Vec::new())), stack: Mutex::new(ThreadStack::default()), - #[cfg(feature = "snapshot")] + #[cfg(feature = "journal")] check_pointing: AtomicBool::new(false), _task_count_guard: guard, }), diff --git a/lib/wasix/src/runners/wasi.rs b/lib/wasix/src/runners/wasi.rs index 492cc625109..95090aff05d 100644 --- a/lib/wasix/src/runners/wasi.rs +++ b/lib/wasix/src/runners/wasi.rs @@ -11,10 +11,10 @@ use webc::metadata::{annotations::Wasi, Command}; use crate::{ bin_factory::BinaryPackage, capabilities::Capabilities, + journal::{DynJournal, SnapshotTrigger}, runners::{wasi_common::CommonWasiOptions, MappedDirectory}, runtime::task_manager::VirtualTaskManagerExt, - snapshot::{DynSnapshotCapturer, SnapshotTrigger}, - Runtime, SnapshotRestore, WasiEnvBuilder, WasiRuntimeError, + JournalRestore, Runtime, WasiEnvBuilder, WasiRuntimeError, }; use super::wasi_common::MappedCommand; @@ -194,20 +194,15 @@ impl WasiRunner { self } - pub fn with_snapshot_restore( - &mut self, - restorer: Arc, - n_snapshots: Option, - ) -> &mut Self { - self.wasi.snapshot_restore.replace(SnapshotRestore { - restorer, - n_snapshots, - }); + pub fn with_journal_restore(&mut self, restorer: Arc) -> &mut Self { + self.wasi + .journal_restore + .replace(JournalRestore { restorer }); self } - pub fn with_snapshot_save(&mut self, capturer: Arc) -> &mut Self { - self.wasi.snapshot_save.replace(capturer); + pub fn with_journal(&mut self, capturer: Arc) -> &mut Self { + self.wasi.journal.replace(capturer); self } @@ -323,19 +318,19 @@ impl crate::runners::Runner for WasiRunner { .prepare_webc_env(command_name, &wasi, Some(pkg), Arc::clone(&runtime), None) .context("Unable to prepare the WASI environment")?; - #[cfg(feature = "snapshot")] + #[cfg(feature = "journal")] for snapshot_trigger in self.wasi.snapshot_on.iter().cloned() { env.add_snapshot_trigger(snapshot_trigger); } - #[cfg(feature = "snapshot")] - if let Some(capturer) = self.wasi.snapshot_save.clone() { - env = env.with_snapshot_save(capturer); + #[cfg(feature = "journal")] + if let Some(capturer) = self.wasi.journal.clone() { + env = env.with_journal(capturer); } - #[cfg(feature = "snapshot")] - if let Some(restore) = self.wasi.snapshot_restore.clone() { - env = env.with_snapshot_restore(restore.restorer, restore.n_snapshots); + #[cfg(feature = "journal")] + if let Some(restore) = self.wasi.journal_restore.clone() { + env = env.with_journal_restore(restore.restorer); } let env = env.build()?; diff --git a/lib/wasix/src/runners/wasi_common.rs b/lib/wasix/src/runners/wasi_common.rs index 31590842160..ff7e70f3550 100644 --- a/lib/wasix/src/runners/wasi_common.rs +++ b/lib/wasix/src/runners/wasi_common.rs @@ -13,9 +13,9 @@ use webc::metadata::annotations::Wasi as WasiAnnotation; use crate::{ bin_factory::BinaryPackage, capabilities::Capabilities, + journal::{DynJournal, SnapshotTrigger}, runners::MappedDirectory, - snapshot::{DynSnapshotCapturer, SnapshotTrigger}, - state::SnapshotRestore, + state::JournalRestore, WasiEnvBuilder, }; @@ -38,9 +38,9 @@ pub(crate) struct CommonWasiOptions { pub(crate) injected_packages: Vec, pub(crate) capabilities: Capabilities, #[derivative(Debug = "ignore")] - pub(crate) snapshot_save: Option>, + pub(crate) journal: Option>, #[derivative(Debug = "ignore")] - pub(crate) snapshot_restore: Option, + pub(crate) journal_restore: Option, pub(crate) snapshot_on: Vec, pub(crate) current_dir: Option, } diff --git a/lib/wasix/src/runners/wcgi/runner.rs b/lib/wasix/src/runners/wcgi/runner.rs index 14c3e0040c1..dfc8b42c6c6 100644 --- a/lib/wasix/src/runners/wcgi/runner.rs +++ b/lib/wasix/src/runners/wcgi/runner.rs @@ -242,32 +242,24 @@ impl Config { &mut self.wasi.capabilities } - #[cfg(feature = "snapshot")] - pub fn add_snapshot_trigger(&mut self, on: crate::snapshot::SnapshotTrigger) { + #[cfg(feature = "journal")] + pub fn add_snapshot_trigger(&mut self, on: crate::journal::SnapshotTrigger) { self.wasi.snapshot_on.push(on); } - #[cfg(feature = "snapshot")] - pub fn with_snapshot_restore( - &mut self, - capturer: Arc, - n_snapshots: Option, - ) -> &mut Self { - use crate::state::SnapshotRestore; + #[cfg(feature = "journal")] + pub fn with_journal_restore(&mut self, capturer: Arc) -> &mut Self { + use crate::state::JournalRestore; - self.wasi.snapshot_restore.replace(SnapshotRestore { - restorer: capturer, - n_snapshots, - }); + self.wasi + .journal_restore + .replace(JournalRestore { restorer: capturer }); self } - #[cfg(feature = "snapshot")] - pub fn with_snapshot_save( - &mut self, - capturer: Arc, - ) -> &mut Self { - self.wasi.snapshot_save.replace(capturer); + #[cfg(feature = "journal")] + pub fn with_journal(&mut self, capturer: Arc) -> &mut Self { + self.wasi.journal.replace(capturer); self } } diff --git a/lib/wasix/src/runtime/mod.rs b/lib/wasix/src/runtime/mod.rs index 19203fb2db6..ce0ac55b2b5 100644 --- a/lib/wasix/src/runtime/mod.rs +++ b/lib/wasix/src/runtime/mod.rs @@ -8,8 +8,8 @@ use self::{ module_cache::{CacheError, ModuleHash}, task_manager::InlineWaker, }; -#[cfg(feature = "snapshot")] -use crate::snapshot::UNSUPPORTED_SNAPSHOT_CAPTURER; +#[cfg(feature = "journal")] +use crate::journal::UNSUPPORTED_SNAPSHOT_CAPTURER; use std::{ fmt, @@ -21,8 +21,8 @@ use futures::future::BoxFuture; use virtual_net::{DynVirtualNetworking, VirtualNetworking}; use wasmer::Module; -#[cfg(feature = "snapshot")] -use crate::snapshot::{DynSnapshotCapturer, UnsupportedSnapshotCapturer}; +#[cfg(feature = "journal")] +use crate::journal::{DynJournal, UnsupportedJournal}; use crate::{ http::{DynHttpClient, HttpClient}, os::TtyBridge, @@ -112,8 +112,8 @@ where /// The snapshot capturer takes and restores snapshots of the WASM process at specific /// points in time by reading and writing log entries - #[cfg(feature = "snapshot")] - fn snapshot_capturer<'a>(&'a self) -> &'_ DynSnapshotCapturer { + #[cfg(feature = "journal")] + fn snapshot_capturer<'a>(&'a self) -> &'_ DynJournal { &UNSUPPORTED_SNAPSHOT_CAPTURER } } @@ -191,9 +191,9 @@ pub struct PluggableRuntime { pub module_cache: Arc, #[derivative(Debug = "ignore")] pub tty: Option>, - #[cfg(feature = "snapshot")] + #[cfg(feature = "journal")] #[derivative(Debug = "ignore")] - pub snapshot_capturer: Arc, + pub snapshot_capturer: Arc, } impl PluggableRuntime { @@ -228,9 +228,8 @@ impl PluggableRuntime { source: Arc::new(source), package_loader: Arc::new(loader), module_cache: Arc::new(module_cache::in_memory()), - #[cfg(feature = "snapshot")] - snapshot_capturer: Arc::new(UnsupportedSnapshotCapturer::default()) - as Arc, + #[cfg(feature = "journal")] + snapshot_capturer: Arc::new(UnsupportedJournal::default()) as Arc, } } @@ -281,8 +280,8 @@ impl PluggableRuntime { self } - #[cfg(feature = "snapshot")] - pub fn set_snapshot_capturer(&mut self, capturer: Arc) -> &mut Self { + #[cfg(feature = "journal")] + pub fn set_snapshot_capturer(&mut self, capturer: Arc) -> &mut Self { self.snapshot_capturer = capturer; self } @@ -332,8 +331,8 @@ impl Runtime for PluggableRuntime { self.module_cache.clone() } - #[cfg(feature = "snapshot")] - fn snapshot_capturer<'a>(&'a self) -> &DynSnapshotCapturer { + #[cfg(feature = "journal")] + fn snapshot_capturer<'a>(&'a self) -> &DynJournal { self.snapshot_capturer.as_ref() } } diff --git a/lib/wasix/src/snapshot/effector/unimplemented.rs b/lib/wasix/src/snapshot/effector/unimplemented.rs deleted file mode 100644 index 8d8dc18a667..00000000000 --- a/lib/wasix/src/snapshot/effector/unimplemented.rs +++ /dev/null @@ -1,4 +0,0 @@ -#[derive(Debug, Clone)] -pub struct SnapshotEffector {} - -impl SnapshotEffector {} diff --git a/lib/wasix/src/snapshot/unsupported.rs b/lib/wasix/src/snapshot/unsupported.rs deleted file mode 100644 index 51d3cb227ba..00000000000 --- a/lib/wasix/src/snapshot/unsupported.rs +++ /dev/null @@ -1,22 +0,0 @@ -use futures::future::LocalBoxFuture; - -use super::*; - -pub static UNSUPPORTED_SNAPSHOT_CAPTURER: UnsupportedSnapshotCapturer = - UnsupportedSnapshotCapturer {}; - -/// The default for runtime is to use the unsupported snapshot capturer -/// which will fail to snapshot if one attempts to do so. -#[derive(Debug, Default)] -pub struct UnsupportedSnapshotCapturer {} - -impl SnapshotCapturer for UnsupportedSnapshotCapturer { - fn write<'a>(&'a self, entry: SnapshotLog<'a>) -> LocalBoxFuture<'a, anyhow::Result<()>> { - tracing::debug!("snapshot event: {:?}", entry); - Box::pin(async { Err(anyhow::format_err!("unsupported")) }) - } - - fn read<'a>(&'a self) -> LocalBoxFuture<'a, anyhow::Result>>> { - Box::pin(async { Ok(None) }) - } -} diff --git a/lib/wasix/src/state/builder.rs b/lib/wasix/src/state/builder.rs index 6cbc3c2dced..5eb49307cfb 100644 --- a/lib/wasix/src/state/builder.rs +++ b/lib/wasix/src/state/builder.rs @@ -19,15 +19,15 @@ use crate::{ bin_factory::{BinFactory, BinaryPackage}, capabilities::Capabilities, fs::{WasiFs, WasiFsRoot, WasiInodes}, + journal::DynJournal, os::task::control_plane::{ControlPlaneConfig, ControlPlaneError, WasiControlPlane}, runtime::task_manager::InlineWaker, - snapshot::DynSnapshotCapturer, state::WasiState, syscalls::types::{__WASI_STDERR_FILENO, __WASI_STDIN_FILENO, __WASI_STDOUT_FILENO}, RewindState, Runtime, WasiEnv, WasiError, WasiFunctionEnv, WasiRuntimeError, }; -#[cfg(feature = "snapshot")] -use crate::{snapshot::SnapshotTrigger, syscalls::restore_snapshot}; +#[cfg(feature = "journal")] +use crate::{journal::SnapshotTrigger, syscalls::restore_snapshot}; use super::env::WasiEnvInit; @@ -75,22 +75,20 @@ pub struct WasiEnvBuilder { pub(super) capabilites: Capabilities, - #[cfg(feature = "snapshot")] + #[cfg(feature = "journal")] pub(super) snapshot_on: Vec, - #[cfg(feature = "snapshot")] - pub(super) snapshot_restore: Option, + #[cfg(feature = "journal")] + pub(super) snapshot_restore: Option, - #[cfg(feature = "snapshot")] - pub(super) snapshot_save: Option>, + #[cfg(feature = "journal")] + pub(super) snapshot_save: Option>, } #[derive(Clone)] -pub struct SnapshotRestore { +pub struct JournalRestore { /// Snapshot capturer that will be used to restore state - pub restorer: Arc, - /// Maximum number of snapshots taken before execution resumes - pub n_snapshots: Option, + pub restorer: Arc, } impl std::fmt::Debug for WasiEnvBuilder { @@ -519,23 +517,16 @@ impl WasiEnvBuilder { /// Supplies a snapshot capturer which will be used to read the /// snapshot journal events and replay them into the WasiEnv /// rather than starting a new instance from scratch - #[cfg(feature = "snapshot")] - pub fn with_snapshot_restore( - mut self, - restorer: Arc, - n_snapshots: Option, - ) -> Self { - self.snapshot_restore.replace(SnapshotRestore { - restorer, - n_snapshots, - }); + #[cfg(feature = "journal")] + pub fn with_journal_restore(mut self, restorer: Arc) -> Self { + self.snapshot_restore.replace(JournalRestore { restorer }); self } /// Supplies a snapshot capturer where the snapshot journal events /// will be sent to as they are generated - #[cfg(feature = "snapshot")] - pub fn with_snapshot_save(mut self, snapshot_capturer: Arc) -> Self { + #[cfg(feature = "journal")] + pub fn with_journal(mut self, snapshot_capturer: Arc) -> Self { self.snapshot_save.replace(snapshot_capturer); self } @@ -642,7 +633,7 @@ impl WasiEnvBuilder { self.capabilites = capabilities; } - #[cfg(feature = "snapshot")] + #[cfg(feature = "journal")] pub fn add_snapshot_trigger(&mut self, on: SnapshotTrigger) { self.snapshot_on.push(on); } @@ -817,7 +808,7 @@ impl WasiEnvBuilder { { #[allow(unused_mut)] let mut runtime = PluggableRuntime::new(Arc::new(crate::runtime::task_manager::tokio::TokioTaskManager::default())); - #[cfg(feature = "snapshot")] + #[cfg(feature = "journal")] if let Some(capturer) = self.snapshot_save.clone() { runtime.set_snapshot_capturer(capturer); } @@ -843,7 +834,7 @@ impl WasiEnvBuilder { }; let control_plane = WasiControlPlane::new(plane_config); - #[cfg(feature = "snapshot")] + #[cfg(feature = "journal")] let snapshot_on = self.snapshot_on; let init = WasiEnvInit { @@ -857,13 +848,13 @@ impl WasiEnvBuilder { memory_ty: None, process: None, thread: None, - #[cfg(feature = "snapshot")] + #[cfg(feature = "journal")] call_initialize: self.snapshot_restore.is_none(), - #[cfg(not(feature = "snapshot"))] + #[cfg(not(feature = "journal"))] call_initialize: true, can_deep_sleep: false, extra_tracing: true, - #[cfg(feature = "snapshot")] + #[cfg(feature = "journal")] snapshot_on, }; @@ -935,9 +926,9 @@ impl WasiEnvBuilder { ); } - #[cfg(feature = "snapshot")] + #[cfg(feature = "journal")] let snapshot_restore = self.snapshot_restore.clone(); - #[cfg(not(feature = "snapshot"))] + #[cfg(not(feature = "journal"))] let snapshot_restore = None; let (instance, env) = self.instantiate(module, store)?; @@ -984,7 +975,7 @@ impl WasiEnvBuilder { #[cfg(feature = "sys-thread")] let _guard = _guard.as_ref().map(|r| r.enter()); - #[cfg(feature = "snapshot")] + #[cfg(feature = "journal")] let snapshot_restore = self.snapshot_restore.clone(); let (_, env) = self.instantiate(module, &mut store)?; @@ -994,7 +985,7 @@ impl WasiEnvBuilder { #[allow(unused_mut)] let mut rewind_state = None; - #[cfg(feature = "snapshot")] + #[cfg(feature = "journal")] if let Some(snapshot_restore) = snapshot_restore { let ctx = env.env.clone().into_mut(&mut store); let rewind = restore_snapshot(ctx, snapshot_restore)?; diff --git a/lib/wasix/src/state/env.rs b/lib/wasix/src/state/env.rs index e5ce02447a4..4f5760c4c7d 100644 --- a/lib/wasix/src/state/env.rs +++ b/lib/wasix/src/state/env.rs @@ -1,4 +1,4 @@ -#[cfg(feature = "snapshot")] +#[cfg(feature = "journal")] use std::collections::HashSet; use std::{ collections::HashMap, @@ -22,8 +22,8 @@ use wasmer_wasix_types::{ wasi::{Errno, ExitCode, Snapshot0Clockid}, }; -#[cfg(feature = "snapshot")] -use crate::snapshot::{SnapshotEffector, SnapshotTrigger}; +#[cfg(feature = "journal")] +use crate::journal::{JournalEffector, SnapshotTrigger}; use crate::{ bin_factory::{BinFactory, BinaryPackage}, capabilities::Capabilities, @@ -235,7 +235,7 @@ pub struct WasiEnvInit { pub extra_tracing: bool, /// Indicates triggers that will cause a snapshot to be taken - #[cfg(feature = "snapshot")] + #[cfg(feature = "journal")] pub snapshot_on: Vec, } @@ -273,7 +273,7 @@ impl WasiEnvInit { call_initialize: self.call_initialize, can_deep_sleep: self.can_deep_sleep, extra_tracing: false, - #[cfg(feature = "snapshot")] + #[cfg(feature = "journal")] snapshot_on: self.snapshot_on.clone(), } } @@ -312,7 +312,7 @@ pub struct WasiEnv { pub enable_snapshot_capture: bool, /// List of situations that the process will checkpoint on - #[cfg(feature = "snapshot")] + #[cfg(feature = "journal")] snapshot_on: HashSet, /// Inner functions and references that are loaded before the environment starts @@ -345,7 +345,7 @@ impl Clone for WasiEnv { capabilities: self.capabilities.clone(), enable_deep_sleep: self.enable_deep_sleep, enable_snapshot_capture: self.enable_snapshot_capture, - #[cfg(feature = "snapshot")] + #[cfg(feature = "journal")] snapshot_on: self.snapshot_on.clone(), } } @@ -384,7 +384,7 @@ impl WasiEnv { capabilities: self.capabilities.clone(), enable_deep_sleep: self.enable_deep_sleep, enable_snapshot_capture: self.enable_snapshot_capture, - #[cfg(feature = "snapshot")] + #[cfg(feature = "journal")] snapshot_on: self.snapshot_on.clone(), }; Ok((new_env, handle)) @@ -449,11 +449,11 @@ impl WasiEnv { bin_factory: init.bin_factory, enable_deep_sleep: init.capabilities.threading.enable_asynchronous_threading, capabilities: init.capabilities, - #[cfg(feature = "snapshot")] + #[cfg(feature = "journal")] enable_snapshot_capture: !init.snapshot_on.is_empty(), - #[cfg(not(feature = "snapshot"))] + #[cfg(not(feature = "journal"))] enable_snapshot_capture: false, - #[cfg(feature = "snapshot")] + #[cfg(feature = "journal")] snapshot_on: init.snapshot_on.into_iter().collect(), }; env.owned_handles.push(thread); @@ -855,19 +855,19 @@ impl WasiEnv { } /// Returns true if the process should perform snapshots or not - #[cfg(feature = "snapshot")] + #[cfg(feature = "journal")] pub fn should_feed_snapshot(&self) -> bool { !self.snapshot_on.is_empty() } /// Returns true if a particular snapshot trigger is enabled - #[cfg(feature = "snapshot")] + #[cfg(feature = "journal")] pub fn has_snapshot_trigger(&self, trigger: SnapshotTrigger) -> bool { self.snapshot_on.contains(&trigger) } /// Returns true if a particular snapshot trigger is enabled - #[cfg(feature = "snapshot")] + #[cfg(feature = "journal")] pub fn pop_snapshot_trigger(&mut self, trigger: SnapshotTrigger) -> bool { self.snapshot_on.remove(&trigger) } @@ -1089,9 +1089,9 @@ impl WasiEnv { const CLEANUP_TIMEOUT: Duration = Duration::from_secs(10); // If snap-shooting is enabled then we should record an event that the thread has exited. - #[cfg(feature = "snapshot")] + #[cfg(feature = "journal")] if self.should_feed_snapshot() { - if let Err(err) = SnapshotEffector::save_thread_exit(self, self.tid(), exit_code) { + if let Err(err) = JournalEffector::save_thread_exit(self, self.tid(), exit_code) { tracing::warn!("failed to save snapshot event for thread exit - {}", err); } } diff --git a/lib/wasix/src/syscalls/mod.rs b/lib/wasix/src/syscalls/mod.rs index 4a75d0b1726..7f0ecc493f7 100644 --- a/lib/wasix/src/syscalls/mod.rs +++ b/lib/wasix/src/syscalls/mod.rs @@ -91,7 +91,7 @@ pub(crate) use self::types::{ *, }; use self::{ - state::{SnapshotRestore, WasiInstanceGuardMemory}, + state::{JournalRestore, WasiInstanceGuardMemory}, utils::WasiDummyWaker, }; pub(crate) use crate::os::task::{ @@ -120,9 +120,9 @@ use crate::{ fs_error_into_wasi_err, virtual_file_type_to_wasi_file_type, Fd, InodeVal, Kind, MAX_SYMLINKS, }, + journal::{DynJournal, JournalEffector}, os::task::{process::MaybeCheckpointResult, thread::RewindResult}, runtime::task_manager::InlineWaker, - snapshot::{DynSnapshotCapturer, SnapshotEffector}, utils::store::InstanceSnapshot, DeepSleepWork, RewindPostProcess, RewindState, SpawnError, WasiInodes, WasiResult, WasiRuntimeError, @@ -1221,18 +1221,18 @@ pub fn rewind_ext( Errno::Success } -#[cfg(not(feature = "snapshot"))] +#[cfg(not(feature = "journal"))] pub fn maybe_snapshot_once( ctx: FunctionEnvMut<'_, WasiEnv>, - _trigger: crate::snapshot::SnapshotTrigger, + _trigger: crate::journal::SnapshotTrigger, ) -> WasiResult> { Ok(Ok(ctx)) } -#[cfg(feature = "snapshot")] +#[cfg(feature = "journal")] pub fn maybe_snapshot_once( mut ctx: FunctionEnvMut<'_, WasiEnv>, - trigger: crate::snapshot::SnapshotTrigger, + trigger: crate::journal::SnapshotTrigger, ) -> WasiResult> { use crate::os::task::process::{WasiProcessCheckpoint, WasiProcessInner}; @@ -1259,14 +1259,14 @@ pub fn maybe_snapshot_once( Ok(Ok(ctx)) } -#[cfg(not(feature = "snapshot"))] +#[cfg(not(feature = "journal"))] pub fn maybe_snapshot( ctx: FunctionEnvMut<'_, WasiEnv>, ) -> WasiResult> { Ok(Ok(ctx)) } -#[cfg(feature = "snapshot")] +#[cfg(feature = "journal")] pub fn maybe_snapshot( mut ctx: FunctionEnvMut<'_, WasiEnv>, ) -> WasiResult> { @@ -1291,47 +1291,45 @@ pub fn anyhow_err_to_runtime_err(err: anyhow::Error) -> WasiRuntimeError { WasiRuntimeError::Runtime(RuntimeError::user(err.into())) } -#[cfg(feature = "snapshot")] +#[cfg(feature = "journal")] pub fn restore_snapshot( mut ctx: FunctionEnvMut<'_, WasiEnv>, - restore: SnapshotRestore, + restore: JournalRestore, ) -> Result { let restorer = restore.restorer; - let mut n_snapshots = restore.n_snapshots; - InlineWaker::block_on(async { let mut rewind = None; while let Some(next) = restorer.read().await.map_err(anyhow_err_to_runtime_err)? { tracing::trace!("Restoring snapshot event - {next:?}"); match next { - crate::snapshot::SnapshotLog::Init { .. } => { + crate::journal::JournalEntry::Init { .. } => { // TODO: Here we need to check the hash matches the binary } - crate::snapshot::SnapshotLog::FileDescriptorWrite { + crate::journal::JournalEntry::FileDescriptorWrite { fd, offset, data, is_64bit, } => { if is_64bit { - SnapshotEffector::apply_fd_write::(&mut ctx, fd, offset, data) + JournalEffector::apply_fd_write::(&mut ctx, fd, offset, data) .await } else { - SnapshotEffector::apply_fd_write::(&mut ctx, fd, offset, data) + JournalEffector::apply_fd_write::(&mut ctx, fd, offset, data) .await } .map_err(anyhow_err_to_runtime_err)?; } - crate::snapshot::SnapshotLog::FileDescriptorSeek { fd, offset, whence } => { - SnapshotEffector::apply_fd_seek(&mut ctx, fd, offset, whence) + crate::journal::JournalEntry::FileDescriptorSeek { fd, offset, whence } => { + JournalEffector::apply_fd_seek(&mut ctx, fd, offset, whence) .await .map_err(anyhow_err_to_runtime_err)?; } - crate::snapshot::SnapshotLog::UpdateMemoryRegion { region, data } => { - SnapshotEffector::apply_memory(&mut ctx, region, &data) + crate::journal::JournalEntry::UpdateMemoryRegion { region, data } => { + JournalEffector::apply_memory(&mut ctx, region, &data) .map_err(anyhow_err_to_runtime_err)?; } - crate::snapshot::SnapshotLog::CloseThread { id, exit_code } => { + crate::journal::JournalEntry::CloseThread { id, exit_code } => { if id == ctx.data().tid() { return Err(WasiRuntimeError::Runtime(RuntimeError::user( anyhow::format_err!( @@ -1341,7 +1339,7 @@ pub fn restore_snapshot( ))); } } - crate::snapshot::SnapshotLog::SetThread { + crate::journal::JournalEntry::SetThread { id, call_stack, memory_stack, @@ -1364,11 +1362,11 @@ pub fn restore_snapshot( ))); } } - crate::snapshot::SnapshotLog::CloseFileDescriptor { fd } => { - SnapshotEffector::apply_fd_close(&mut ctx, fd) + crate::journal::JournalEntry::CloseFileDescriptor { fd } => { + JournalEffector::apply_fd_close(&mut ctx, fd) .map_err(anyhow_err_to_runtime_err)?; } - crate::snapshot::SnapshotLog::OpenFileDescriptor { + crate::journal::JournalEntry::OpenFileDescriptor { fd, dirfd, dirflags, @@ -1379,7 +1377,7 @@ pub fn restore_snapshot( fs_flags, is_64bit, } => { - SnapshotEffector::apply_path_open( + JournalEffector::apply_path_open( &mut ctx, fd, dirfd, @@ -1393,56 +1391,49 @@ pub fn restore_snapshot( ) .map_err(anyhow_err_to_runtime_err)?; } - crate::snapshot::SnapshotLog::RemoveDirectory { fd, path } => { - SnapshotEffector::apply_path_remove_directory(&mut ctx, fd, &path) + crate::journal::JournalEntry::RemoveDirectory { fd, path } => { + JournalEffector::apply_path_remove_directory(&mut ctx, fd, &path) .map_err(anyhow_err_to_runtime_err)?; } - crate::snapshot::SnapshotLog::UnlinkFile { fd, path } => { - SnapshotEffector::apply_path_unlink(&mut ctx, fd, &path) + crate::journal::JournalEntry::UnlinkFile { fd, path } => { + JournalEffector::apply_path_unlink(&mut ctx, fd, &path) .map_err(anyhow_err_to_runtime_err)?; } - crate::snapshot::SnapshotLog::PathRename { + crate::journal::JournalEntry::PathRename { old_fd, old_path, new_fd, new_path, } => { - SnapshotEffector::apply_path_rename( + JournalEffector::apply_path_rename( &mut ctx, old_fd, &old_path, new_fd, &new_path, ) .map_err(anyhow_err_to_runtime_err)?; } - crate::snapshot::SnapshotLog::Snapshot { + crate::journal::JournalEntry::Snapshot { when: _, trigger: _, - } => { - if let Some(n_snapshots) = &mut n_snapshots { - if *n_snapshots <= 1 { - break; - } - *n_snapshots -= 1; - } - } - crate::snapshot::SnapshotLog::SetClockTime { clock_id, time } => { - SnapshotEffector::apply_clock_time_set(&mut ctx, clock_id, time) + } => {} + crate::journal::JournalEntry::SetClockTime { clock_id, time } => { + JournalEffector::apply_clock_time_set(&mut ctx, clock_id, time) .map_err(anyhow_err_to_runtime_err)?; } - crate::snapshot::SnapshotLog::RenumberFileDescriptor { old_fd, new_fd } => { - SnapshotEffector::apply_fd_renumber(&mut ctx, old_fd, new_fd) + crate::journal::JournalEntry::RenumberFileDescriptor { old_fd, new_fd } => { + JournalEffector::apply_fd_renumber(&mut ctx, old_fd, new_fd) .map_err(anyhow_err_to_runtime_err)?; } - crate::snapshot::SnapshotLog::DuplicateFileDescriptor { + crate::journal::JournalEntry::DuplicateFileDescriptor { original_fd, copied_fd, } => { - SnapshotEffector::apply_fd_duplicate(&mut ctx, original_fd, copied_fd) + JournalEffector::apply_fd_duplicate(&mut ctx, original_fd, copied_fd) .map_err(anyhow_err_to_runtime_err)?; } - crate::snapshot::SnapshotLog::CreateDirectory { fd, path } => { - SnapshotEffector::apply_path_create_directory(&mut ctx, fd, &path) + crate::journal::JournalEntry::CreateDirectory { fd, path } => { + JournalEffector::apply_path_create_directory(&mut ctx, fd, &path) .map_err(anyhow_err_to_runtime_err)?; } - crate::snapshot::SnapshotLog::PathSetTimes { + crate::journal::JournalEntry::PathSetTimes { fd, flags, path, @@ -1450,34 +1441,34 @@ pub fn restore_snapshot( st_mtim, fst_flags, } => { - SnapshotEffector::apply_path_set_times( + JournalEffector::apply_path_set_times( &mut ctx, fd, flags, &path, st_atim, st_mtim, fst_flags, ) .map_err(anyhow_err_to_runtime_err)?; } - crate::snapshot::SnapshotLog::FileDescriptorSetTimes { + crate::journal::JournalEntry::FileDescriptorSetTimes { fd, st_atim, st_mtim, fst_flags, } => { - SnapshotEffector::apply_fd_set_times(&mut ctx, fd, st_atim, st_mtim, fst_flags) + JournalEffector::apply_fd_set_times(&mut ctx, fd, st_atim, st_mtim, fst_flags) .map_err(anyhow_err_to_runtime_err)?; } - crate::snapshot::SnapshotLog::FileDescriptorSetSize { fd, st_size } => { - SnapshotEffector::apply_fd_set_size(&mut ctx, fd, st_size) + crate::journal::JournalEntry::FileDescriptorSetSize { fd, st_size } => { + JournalEffector::apply_fd_set_size(&mut ctx, fd, st_size) .map_err(anyhow_err_to_runtime_err)?; } - crate::snapshot::SnapshotLog::FileDescriptorSetFlags { fd, flags } => { - SnapshotEffector::apply_fd_set_flags(&mut ctx, fd, flags) + crate::journal::JournalEntry::FileDescriptorSetFlags { fd, flags } => { + JournalEffector::apply_fd_set_flags(&mut ctx, fd, flags) .map_err(anyhow_err_to_runtime_err)?; } - crate::snapshot::SnapshotLog::FileDescriptorSetRights { + crate::journal::JournalEntry::FileDescriptorSetRights { fd, fs_rights_base, fs_rights_inheriting, } => { - SnapshotEffector::apply_fd_set_rights( + JournalEffector::apply_fd_set_rights( &mut ctx, fd, fs_rights_base, @@ -1485,62 +1476,62 @@ pub fn restore_snapshot( ) .map_err(anyhow_err_to_runtime_err)?; } - crate::snapshot::SnapshotLog::FileDescriptorAdvise { + crate::journal::JournalEntry::FileDescriptorAdvise { fd, offset, len, advice, } => { - SnapshotEffector::apply_fd_advise(&mut ctx, fd, offset, len, advice) + JournalEffector::apply_fd_advise(&mut ctx, fd, offset, len, advice) .map_err(anyhow_err_to_runtime_err)?; } - crate::snapshot::SnapshotLog::FileDescriptorAllocate { fd, offset, len } => { - SnapshotEffector::apply_fd_allocate(&mut ctx, fd, offset, len) + crate::journal::JournalEntry::FileDescriptorAllocate { fd, offset, len } => { + JournalEffector::apply_fd_allocate(&mut ctx, fd, offset, len) .map_err(anyhow_err_to_runtime_err)?; } - crate::snapshot::SnapshotLog::CreateHardLink { + crate::journal::JournalEntry::CreateHardLink { old_fd, old_path, old_flags, new_fd, new_path, } => { - SnapshotEffector::apply_path_link( + JournalEffector::apply_path_link( &mut ctx, old_fd, old_flags, &old_path, new_fd, &new_path, ) .map_err(anyhow_err_to_runtime_err)?; } - crate::snapshot::SnapshotLog::CreateSymbolicLink { + crate::journal::JournalEntry::CreateSymbolicLink { old_path, fd, new_path, } => { - SnapshotEffector::apply_path_symlink(&mut ctx, &old_path, fd, &new_path) + JournalEffector::apply_path_symlink(&mut ctx, &old_path, fd, &new_path) .map_err(anyhow_err_to_runtime_err)?; } - crate::snapshot::SnapshotLog::ChangeDirectory { path } => { - SnapshotEffector::apply_chdir(&mut ctx, &path) + crate::journal::JournalEntry::ChangeDirectory { path } => { + JournalEffector::apply_chdir(&mut ctx, &path) .map_err(anyhow_err_to_runtime_err)?; } - crate::snapshot::SnapshotLog::CreatePipe { fd1, fd2 } => { - SnapshotEffector::apply_fd_pipe(&mut ctx, fd1, fd2) + crate::journal::JournalEntry::CreatePipe { fd1, fd2 } => { + JournalEffector::apply_fd_pipe(&mut ctx, fd1, fd2) .map_err(anyhow_err_to_runtime_err)?; } - crate::snapshot::SnapshotLog::EpollCreate { fd } => { - SnapshotEffector::apply_epoll_create(&mut ctx, fd) + crate::journal::JournalEntry::EpollCreate { fd } => { + JournalEffector::apply_epoll_create(&mut ctx, fd) .map_err(anyhow_err_to_runtime_err)?; } - crate::snapshot::SnapshotLog::EpollCtl { + crate::journal::JournalEntry::EpollCtl { epfd, op, fd, event, } => { - SnapshotEffector::apply_epoll_ctl(&mut ctx, epfd, op, fd, event) + JournalEffector::apply_epoll_ctl(&mut ctx, epfd, op, fd, event) .map_err(anyhow_err_to_runtime_err)?; } - crate::snapshot::SnapshotLog::TtySet { tty, line_feeds } => { - SnapshotEffector::apply_tty_set( + crate::journal::JournalEntry::TtySet { tty, line_feeds } => { + JournalEffector::apply_tty_set( &mut ctx, crate::WasiTtyState { cols: tty.cols, diff --git a/lib/wasix/src/syscalls/wasi/clock_time_set.rs b/lib/wasix/src/syscalls/wasi/clock_time_set.rs index 02ff2c231d9..13ecbe9715b 100644 --- a/lib/wasix/src/syscalls/wasi/clock_time_set.rs +++ b/lib/wasix/src/syscalls/wasi/clock_time_set.rs @@ -18,9 +18,9 @@ pub fn clock_time_set( let env = ctx.data(); if ret == Errno::Success { - #[cfg(feature = "snapshot")] + #[cfg(feature = "journal")] if env.enable_snapshot_capture { - SnapshotEffector::save_clock_time_set(&mut ctx, clock_id, time).map_err(|err| { + JournalEffector::save_clock_time_set(&mut ctx, clock_id, time).map_err(|err| { tracing::error!("failed to save clock time set event - {}", err); WasiError::Exit(ExitCode::Errno(Errno::Fault)) })?; diff --git a/lib/wasix/src/syscalls/wasi/environ_get.rs b/lib/wasix/src/syscalls/wasi/environ_get.rs index 3c2443e13b4..a169581a652 100644 --- a/lib/wasix/src/syscalls/wasi/environ_get.rs +++ b/lib/wasix/src/syscalls/wasi/environ_get.rs @@ -1,5 +1,5 @@ use super::*; -use crate::{snapshot::SnapshotTrigger, syscalls::*}; +use crate::{journal::SnapshotTrigger, syscalls::*}; /// ### `environ_get()` /// Read environment variable data. diff --git a/lib/wasix/src/syscalls/wasi/environ_sizes_get.rs b/lib/wasix/src/syscalls/wasi/environ_sizes_get.rs index 0e6a9bf8d03..e695fe7dd6d 100644 --- a/lib/wasix/src/syscalls/wasi/environ_sizes_get.rs +++ b/lib/wasix/src/syscalls/wasi/environ_sizes_get.rs @@ -1,5 +1,5 @@ use super::*; -use crate::{snapshot::SnapshotTrigger, syscalls::*}; +use crate::{journal::SnapshotTrigger, syscalls::*}; /// ### `environ_sizes_get()` /// Return command-line argument data sizes. diff --git a/lib/wasix/src/syscalls/wasi/fd_advise.rs b/lib/wasix/src/syscalls/wasi/fd_advise.rs index 38ef1586a94..29f0e7a5608 100644 --- a/lib/wasix/src/syscalls/wasi/fd_advise.rs +++ b/lib/wasix/src/syscalls/wasi/fd_advise.rs @@ -23,9 +23,9 @@ pub fn fd_advise( wasi_try_ok!(fd_advise_internal(&mut ctx, fd, offset, len, advice)); let env = ctx.data(); - #[cfg(feature = "snapshot")] + #[cfg(feature = "journal")] if env.enable_snapshot_capture { - SnapshotEffector::save_fd_advise(&mut ctx, fd, offset, len, advice).map_err(|err| { + JournalEffector::save_fd_advise(&mut ctx, fd, offset, len, advice).map_err(|err| { tracing::error!("failed to save file descriptor advise event - {}", err); WasiError::Exit(ExitCode::Errno(Errno::Fault)) })?; diff --git a/lib/wasix/src/syscalls/wasi/fd_allocate.rs b/lib/wasix/src/syscalls/wasi/fd_allocate.rs index f2a1ab542a7..5747cb61e8e 100644 --- a/lib/wasix/src/syscalls/wasi/fd_allocate.rs +++ b/lib/wasix/src/syscalls/wasi/fd_allocate.rs @@ -20,9 +20,9 @@ pub fn fd_allocate( wasi_try_ok!(fd_allocate_internal(&mut ctx, fd, offset, len)); let env = ctx.data(); - #[cfg(feature = "snapshot")] + #[cfg(feature = "journal")] if env.enable_snapshot_capture { - SnapshotEffector::save_fd_allocate(&mut ctx, fd, offset, len).map_err(|err| { + JournalEffector::save_fd_allocate(&mut ctx, fd, offset, len).map_err(|err| { tracing::error!("failed to save file descriptor allocate event - {}", err); WasiError::Exit(ExitCode::Errno(Errno::Fault)) })?; diff --git a/lib/wasix/src/syscalls/wasi/fd_close.rs b/lib/wasix/src/syscalls/wasi/fd_close.rs index 4d75cc508bd..5ced38a7195 100644 --- a/lib/wasix/src/syscalls/wasi/fd_close.rs +++ b/lib/wasix/src/syscalls/wasi/fd_close.rs @@ -32,9 +32,9 @@ pub fn fd_close(mut ctx: FunctionEnvMut<'_, WasiEnv>, fd: WasiFd) -> Result( let copied_fd = wasi_try_ok!(fd_dup_internal(&mut ctx, fd)); let env = ctx.data(); - #[cfg(feature = "snapshot")] + #[cfg(feature = "journal")] if env.enable_snapshot_capture { - SnapshotEffector::save_fd_duplicate(&mut ctx, fd, copied_fd).map_err(|err| { + JournalEffector::save_fd_duplicate(&mut ctx, fd, copied_fd).map_err(|err| { tracing::error!("failed to save file descriptor renumber event - {}", err); WasiError::Exit(ExitCode::Errno(Errno::Fault)) })?; diff --git a/lib/wasix/src/syscalls/wasi/fd_fdstat_set_flags.rs b/lib/wasix/src/syscalls/wasi/fd_fdstat_set_flags.rs index aed536619da..364c00f34f0 100644 --- a/lib/wasix/src/syscalls/wasi/fd_fdstat_set_flags.rs +++ b/lib/wasix/src/syscalls/wasi/fd_fdstat_set_flags.rs @@ -18,9 +18,9 @@ pub fn fd_fdstat_set_flags( let env = ctx.data(); if ret == Errno::Success { - #[cfg(feature = "snapshot")] + #[cfg(feature = "journal")] if env.enable_snapshot_capture { - SnapshotEffector::save_fd_set_flags(&mut ctx, fd, flags).map_err(|err| { + JournalEffector::save_fd_set_flags(&mut ctx, fd, flags).map_err(|err| { tracing::error!("failed to save file set flags event - {}", err); WasiError::Exit(ExitCode::Errno(Errno::Fault)) })?; diff --git a/lib/wasix/src/syscalls/wasi/fd_fdstat_set_rights.rs b/lib/wasix/src/syscalls/wasi/fd_fdstat_set_rights.rs index c4933444230..ff85b4bc343 100644 --- a/lib/wasix/src/syscalls/wasi/fd_fdstat_set_rights.rs +++ b/lib/wasix/src/syscalls/wasi/fd_fdstat_set_rights.rs @@ -25,9 +25,9 @@ pub fn fd_fdstat_set_rights( )); let env = ctx.data(); - #[cfg(feature = "snapshot")] + #[cfg(feature = "journal")] if env.enable_snapshot_capture { - SnapshotEffector::save_fd_set_rights(&mut ctx, fd, fs_rights_base, fs_rights_inheriting) + JournalEffector::save_fd_set_rights(&mut ctx, fd, fs_rights_base, fs_rights_inheriting) .map_err(|err| { tracing::error!("failed to save file set rights event - {}", err); WasiError::Exit(ExitCode::Errno(Errno::Fault)) diff --git a/lib/wasix/src/syscalls/wasi/fd_filestat_set_size.rs b/lib/wasix/src/syscalls/wasi/fd_filestat_set_size.rs index 2ae923302d3..0a19068bb9f 100644 --- a/lib/wasix/src/syscalls/wasi/fd_filestat_set_size.rs +++ b/lib/wasix/src/syscalls/wasi/fd_filestat_set_size.rs @@ -17,9 +17,9 @@ pub fn fd_filestat_set_size( wasi_try_ok!(fd_filestat_set_size_internal(&mut ctx, fd, st_size)); let env = ctx.data(); - #[cfg(feature = "snapshot")] + #[cfg(feature = "journal")] if env.enable_snapshot_capture { - SnapshotEffector::save_fd_set_size(&mut ctx, fd, st_size).map_err(|err| { + JournalEffector::save_fd_set_size(&mut ctx, fd, st_size).map_err(|err| { tracing::error!("failed to save file set size event - {}", err); WasiError::Exit(ExitCode::Errno(Errno::Fault)) })?; diff --git a/lib/wasix/src/syscalls/wasi/fd_filestat_set_times.rs b/lib/wasix/src/syscalls/wasi/fd_filestat_set_times.rs index 0998286368a..ca8efda0a0e 100644 --- a/lib/wasix/src/syscalls/wasi/fd_filestat_set_times.rs +++ b/lib/wasix/src/syscalls/wasi/fd_filestat_set_times.rs @@ -23,9 +23,9 @@ pub fn fd_filestat_set_times( )); let env = ctx.data(); - #[cfg(feature = "snapshot")] + #[cfg(feature = "journal")] if env.enable_snapshot_capture { - SnapshotEffector::save_fd_set_times(&mut ctx, fd, st_atim, st_mtim, fst_flags).map_err( + JournalEffector::save_fd_set_times(&mut ctx, fd, st_atim, st_mtim, fst_flags).map_err( |err| { tracing::error!("failed to save file set times event - {}", err); WasiError::Exit(ExitCode::Errno(Errno::Fault)) diff --git a/lib/wasix/src/syscalls/wasi/fd_read.rs b/lib/wasix/src/syscalls/wasi/fd_read.rs index 16cbae4c549..1d4a8e006be 100644 --- a/lib/wasix/src/syscalls/wasi/fd_read.rs +++ b/lib/wasix/src/syscalls/wasi/fd_read.rs @@ -5,9 +5,9 @@ use virtual_fs::{AsyncReadExt, DeviceFile, ReadBuf}; use super::*; use crate::{ fs::NotificationInner, + journal::SnapshotTrigger, net::socket::TimeType, os::task::process::{MaybeCheckpointResult, WasiProcessCheckpoint, WasiProcessInner}, - snapshot::SnapshotTrigger, syscalls::*, }; diff --git a/lib/wasix/src/syscalls/wasi/fd_renumber.rs b/lib/wasix/src/syscalls/wasi/fd_renumber.rs index e15cc64d267..d6c4e520410 100644 --- a/lib/wasix/src/syscalls/wasi/fd_renumber.rs +++ b/lib/wasix/src/syscalls/wasi/fd_renumber.rs @@ -18,9 +18,9 @@ pub fn fd_renumber( let env = ctx.data(); if ret == Errno::Success { - #[cfg(feature = "snapshot")] + #[cfg(feature = "journal")] if env.enable_snapshot_capture { - SnapshotEffector::save_fd_renumber(&mut ctx, from, to).map_err(|err| { + JournalEffector::save_fd_renumber(&mut ctx, from, to).map_err(|err| { tracing::error!("failed to save file descriptor renumber event - {}", err); WasiError::Exit(ExitCode::Errno(Errno::Fault)) })?; diff --git a/lib/wasix/src/syscalls/wasi/fd_seek.rs b/lib/wasix/src/syscalls/wasi/fd_seek.rs index 83d575b1a74..77410d6afb4 100644 --- a/lib/wasix/src/syscalls/wasi/fd_seek.rs +++ b/lib/wasix/src/syscalls/wasi/fd_seek.rs @@ -26,9 +26,9 @@ pub fn fd_seek( let new_offset = wasi_try_ok!(fd_seek_internal(&mut ctx, fd, offset, whence)?); let env = ctx.data(); - #[cfg(feature = "snapshot")] + #[cfg(feature = "journal")] if env.enable_snapshot_capture { - SnapshotEffector::save_fd_seek(&mut ctx, fd, offset, whence).map_err(|err| { + JournalEffector::save_fd_seek(&mut ctx, fd, offset, whence).map_err(|err| { tracing::error!("failed to save file descriptor seek event - {}", err); WasiError::Exit(ExitCode::Errno(Errno::Fault)) })?; diff --git a/lib/wasix/src/syscalls/wasi/fd_write.rs b/lib/wasix/src/syscalls/wasi/fd_write.rs index 04f28380184..74a5be0debd 100644 --- a/lib/wasix/src/syscalls/wasi/fd_write.rs +++ b/lib/wasix/src/syscalls/wasi/fd_write.rs @@ -1,12 +1,12 @@ use std::task::Waker; use super::*; -use crate::{net::socket::TimeType, syscalls::*}; -#[cfg(feature = "snapshot")] +#[cfg(feature = "journal")] use crate::{ - snapshot::{SnapshotEffector, SnapshotLog}, + journal::{JournalEffector, JournalEntry}, utils::map_snapshot_err, }; +use crate::{net::socket::TimeType, syscalls::*}; /// ### `fd_write()` /// Write data to the file descriptor @@ -372,10 +372,10 @@ pub(crate) fn fd_write_internal( } }; - #[cfg(feature = "snapshot")] + #[cfg(feature = "journal")] if should_snapshot && can_snapshot && bytes_written > 0 { if let FdWriteSource::Iovs { iovs, iovs_len } = data { - SnapshotEffector::save_fd_write(ctx, fd, offset, bytes_written, iovs, iovs_len) + JournalEffector::save_fd_write(ctx, fd, offset, bytes_written, iovs, iovs_len) .map_err(|err| { tracing::error!( "failed to save terminal data to snapshot capturer - {}", diff --git a/lib/wasix/src/syscalls/wasi/path_create_directory.rs b/lib/wasix/src/syscalls/wasi/path_create_directory.rs index 401ba363e6b..0c25424c25b 100644 --- a/lib/wasix/src/syscalls/wasi/path_create_directory.rs +++ b/lib/wasix/src/syscalls/wasi/path_create_directory.rs @@ -38,9 +38,9 @@ pub fn path_create_directory( wasi_try_ok!(path_create_directory_internal(&mut ctx, fd, &path_string)); let env = ctx.data(); - #[cfg(feature = "snapshot")] + #[cfg(feature = "journal")] if env.enable_snapshot_capture { - SnapshotEffector::save_path_create_directory(&mut ctx, fd, path_string).map_err(|err| { + JournalEffector::save_path_create_directory(&mut ctx, fd, path_string).map_err(|err| { tracing::error!( "failed to save create directory event to snapshot capturer - {}", err diff --git a/lib/wasix/src/syscalls/wasi/path_filestat_set_times.rs b/lib/wasix/src/syscalls/wasi/path_filestat_set_times.rs index ca119fa66d7..e048906ea50 100644 --- a/lib/wasix/src/syscalls/wasi/path_filestat_set_times.rs +++ b/lib/wasix/src/syscalls/wasi/path_filestat_set_times.rs @@ -54,9 +54,9 @@ pub fn path_filestat_set_times( )); let env = ctx.data(); - #[cfg(feature = "snapshot")] + #[cfg(feature = "journal")] if env.enable_snapshot_capture { - SnapshotEffector::save_path_set_times( + JournalEffector::save_path_set_times( &mut ctx, fd, flags, diff --git a/lib/wasix/src/syscalls/wasi/path_link.rs b/lib/wasix/src/syscalls/wasi/path_link.rs index 06dea90a68a..1ad9cff53d6 100644 --- a/lib/wasix/src/syscalls/wasi/path_link.rs +++ b/lib/wasix/src/syscalls/wasi/path_link.rs @@ -49,9 +49,9 @@ pub fn path_link( )); let env = ctx.data(); - #[cfg(feature = "snapshot")] + #[cfg(feature = "journal")] if env.enable_snapshot_capture { - SnapshotEffector::save_path_link( + JournalEffector::save_path_link( &mut ctx, old_fd, old_flags, diff --git a/lib/wasix/src/syscalls/wasi/path_open.rs b/lib/wasix/src/syscalls/wasi/path_open.rs index fa88aecc6ab..3ca3207845f 100644 --- a/lib/wasix/src/syscalls/wasi/path_open.rs +++ b/lib/wasix/src/syscalls/wasi/path_open.rs @@ -79,9 +79,9 @@ pub fn path_open( )?); let env = ctx.data(); - #[cfg(feature = "snapshot")] + #[cfg(feature = "journal")] if env.enable_snapshot_capture { - SnapshotEffector::save_path_open( + JournalEffector::save_path_open( &mut ctx, out_fd, dirfd, diff --git a/lib/wasix/src/syscalls/wasi/path_remove_directory.rs b/lib/wasix/src/syscalls/wasi/path_remove_directory.rs index d03e5a55e94..84a4d46c6d3 100644 --- a/lib/wasix/src/syscalls/wasi/path_remove_directory.rs +++ b/lib/wasix/src/syscalls/wasi/path_remove_directory.rs @@ -28,10 +28,10 @@ pub fn path_remove_directory( wasi_try!(path_remove_directory_internal(&mut ctx, fd, &path_str)); let env = ctx.data(); - #[cfg(feature = "snapshot")] + #[cfg(feature = "journal")] if env.enable_snapshot_capture { wasi_try!( - SnapshotEffector::save_path_remove_directory(&mut ctx, fd, path_str).map_err(|err| { + JournalEffector::save_path_remove_directory(&mut ctx, fd, path_str).map_err(|err| { tracing::error!("failed to save unlink event to snapshot capturer - {}", err); Errno::Fault }) diff --git a/lib/wasix/src/syscalls/wasi/path_rename.rs b/lib/wasix/src/syscalls/wasi/path_rename.rs index 0b1954c25b7..acf7d81f575 100644 --- a/lib/wasix/src/syscalls/wasi/path_rename.rs +++ b/lib/wasix/src/syscalls/wasi/path_rename.rs @@ -39,9 +39,9 @@ pub fn path_rename( let env = ctx.data(); if ret == Errno::Success { - #[cfg(feature = "snapshot")] + #[cfg(feature = "journal")] if env.enable_snapshot_capture { - SnapshotEffector::save_path_rename(&mut ctx, old_fd, source_str, new_fd, target_str) + JournalEffector::save_path_rename(&mut ctx, old_fd, source_str, new_fd, target_str) .map_err(|err| { tracing::error!("failed to save unlink event to snapshot capturer - {}", err); WasiError::Exit(ExitCode::Errno(Errno::Fault)) diff --git a/lib/wasix/src/syscalls/wasi/path_symlink.rs b/lib/wasix/src/syscalls/wasi/path_symlink.rs index b91d2beb9bc..a2cf603914a 100644 --- a/lib/wasix/src/syscalls/wasi/path_symlink.rs +++ b/lib/wasix/src/syscalls/wasi/path_symlink.rs @@ -40,9 +40,9 @@ pub fn path_symlink( )); let env = ctx.data(); - #[cfg(feature = "snapshot")] + #[cfg(feature = "journal")] if env.enable_snapshot_capture { - SnapshotEffector::save_path_symlink(&mut ctx, old_path_str, fd, new_path_str).map_err( + JournalEffector::save_path_symlink(&mut ctx, old_path_str, fd, new_path_str).map_err( |err| { tracing::error!("failed to save path symbolic link event - {}", err); WasiError::Exit(ExitCode::Errno(Errno::Fault)) diff --git a/lib/wasix/src/syscalls/wasi/path_unlink_file.rs b/lib/wasix/src/syscalls/wasi/path_unlink_file.rs index ad05a3fbd00..e888e8e74a5 100644 --- a/lib/wasix/src/syscalls/wasi/path_unlink_file.rs +++ b/lib/wasix/src/syscalls/wasi/path_unlink_file.rs @@ -36,10 +36,10 @@ pub fn path_unlink_file( let env = ctx.data(); if ret == Errno::Success { - #[cfg(feature = "snapshot")] + #[cfg(feature = "journal")] if env.enable_snapshot_capture { wasi_try_ok!( - SnapshotEffector::save_path_unlink(&mut ctx, fd, path_str).map_err(|err| { + JournalEffector::save_path_unlink(&mut ctx, fd, path_str).map_err(|err| { tracing::error!("failed to save unlink event to snapshot capturer - {}", err); Errno::Fault }) diff --git a/lib/wasix/src/syscalls/wasix/chdir.rs b/lib/wasix/src/syscalls/wasix/chdir.rs index b842434f8f9..bb8a40583a0 100644 --- a/lib/wasix/src/syscalls/wasix/chdir.rs +++ b/lib/wasix/src/syscalls/wasix/chdir.rs @@ -17,9 +17,9 @@ pub fn chdir( wasi_try_ok!(chdir_internal(&mut ctx, &path,)); let env = ctx.data(); - #[cfg(feature = "snapshot")] + #[cfg(feature = "journal")] if env.enable_snapshot_capture { - SnapshotEffector::save_chdir(&mut ctx, path).map_err(|err| { + JournalEffector::save_chdir(&mut ctx, path).map_err(|err| { tracing::error!("failed to chdir event - {}", err); WasiError::Exit(ExitCode::Errno(Errno::Fault)) })?; diff --git a/lib/wasix/src/syscalls/wasix/epoll_create.rs b/lib/wasix/src/syscalls/wasix/epoll_create.rs index 92e6ee55261..ac1a3a4a4ca 100644 --- a/lib/wasix/src/syscalls/wasix/epoll_create.rs +++ b/lib/wasix/src/syscalls/wasix/epoll_create.rs @@ -21,9 +21,9 @@ pub fn epoll_create( let fd = wasi_try_ok!(epoll_create_internal(&mut ctx)?); let env = ctx.data(); - #[cfg(feature = "snapshot")] + #[cfg(feature = "journal")] if env.enable_snapshot_capture { - SnapshotEffector::save_epoll_create(&mut ctx, fd).map_err(|err| { + JournalEffector::save_epoll_create(&mut ctx, fd).map_err(|err| { tracing::error!("failed to save epoll_create event - {}", err); WasiError::Exit(ExitCode::Errno(Errno::Fault)) })?; diff --git a/lib/wasix/src/syscalls/wasix/epoll_ctl.rs b/lib/wasix/src/syscalls/wasix/epoll_ctl.rs index 7aee40c2fde..9126e43f7f6 100644 --- a/lib/wasix/src/syscalls/wasix/epoll_ctl.rs +++ b/lib/wasix/src/syscalls/wasix/epoll_ctl.rs @@ -60,9 +60,9 @@ pub fn epoll_ctl( )?); let env = ctx.data(); - #[cfg(feature = "snapshot")] + #[cfg(feature = "journal")] if env.enable_snapshot_capture { - SnapshotEffector::save_epoll_ctl(&mut ctx, epfd, op, fd, event_ctl).map_err(|err| { + JournalEffector::save_epoll_ctl(&mut ctx, epfd, op, fd, event_ctl).map_err(|err| { tracing::error!("failed to save epoll_create event - {}", err); WasiError::Exit(ExitCode::Errno(Errno::Fault)) })?; diff --git a/lib/wasix/src/syscalls/wasix/fd_pipe.rs b/lib/wasix/src/syscalls/wasix/fd_pipe.rs index bf80ffc4165..e79eb777d43 100644 --- a/lib/wasix/src/syscalls/wasix/fd_pipe.rs +++ b/lib/wasix/src/syscalls/wasix/fd_pipe.rs @@ -19,9 +19,9 @@ pub fn fd_pipe( let (fd1, fd2) = wasi_try_ok!(fd_pipe_internal(&mut ctx)); let env = ctx.data(); - #[cfg(feature = "snapshot")] + #[cfg(feature = "journal")] if env.enable_snapshot_capture { - SnapshotEffector::save_fd_pipe(&mut ctx, fd1, fd2).map_err(|err| { + JournalEffector::save_fd_pipe(&mut ctx, fd1, fd2).map_err(|err| { tracing::error!("failed to save create pipe event - {}", err); WasiError::Exit(ExitCode::Errno(Errno::Fault)) })?; diff --git a/lib/wasix/src/syscalls/wasix/sock_listen.rs b/lib/wasix/src/syscalls/wasix/sock_listen.rs index 0fe336db047..397e6905342 100644 --- a/lib/wasix/src/syscalls/wasix/sock_listen.rs +++ b/lib/wasix/src/syscalls/wasix/sock_listen.rs @@ -1,5 +1,5 @@ use super::*; -use crate::{snapshot::SnapshotTrigger, syscalls::*}; +use crate::{journal::SnapshotTrigger, syscalls::*}; /// ### `sock_listen()` /// Listen for connections on a socket diff --git a/lib/wasix/src/syscalls/wasix/thread_spawn.rs b/lib/wasix/src/syscalls/wasix/thread_spawn.rs index 2a4387b6572..22f3124bbcb 100644 --- a/lib/wasix/src/syscalls/wasix/thread_spawn.rs +++ b/lib/wasix/src/syscalls/wasix/thread_spawn.rs @@ -1,8 +1,8 @@ use std::f32::consts::E; use super::*; -#[cfg(feature = "snapshot")] -use crate::snapshot::SnapshotEffector; +#[cfg(feature = "journal")] +use crate::journal::JournalEffector; use crate::{ capture_instance_snapshot, os::task::thread::WasiMemoryLayout, diff --git a/lib/wasix/src/syscalls/wasix/tty_set.rs b/lib/wasix/src/syscalls/wasix/tty_set.rs index 9ded32a0132..e7f28e9d7ee 100644 --- a/lib/wasix/src/syscalls/wasix/tty_set.rs +++ b/lib/wasix/src/syscalls/wasix/tty_set.rs @@ -37,9 +37,9 @@ pub fn tty_set( wasi_try_ok!(tty_set_internal(&mut ctx, state.clone())); let env = ctx.data(); - #[cfg(feature = "snapshot")] + #[cfg(feature = "journal")] if env.enable_snapshot_capture { - SnapshotEffector::save_tty_set(&mut ctx, state).map_err(|err| { + JournalEffector::save_tty_set(&mut ctx, state).map_err(|err| { tracing::error!("failed to save path symbolic link event - {}", err); WasiError::Exit(ExitCode::Errno(Errno::Fault)) })?; diff --git a/lib/wasix/src/utils/mod.rs b/lib/wasix/src/utils/mod.rs index 52fa38e1d5a..cd930052860 100644 --- a/lib/wasix/src/utils/mod.rs +++ b/lib/wasix/src/utils/mod.rs @@ -38,7 +38,7 @@ pub fn map_io_err(err: std::io::Error) -> Errno { From::::from(err) } -#[cfg(feature = "snapshot")] +#[cfg(feature = "journal")] pub fn map_snapshot_err(err: anyhow::Error) -> Errno { tracing::warn!("unknown snapshot error: {}", err); Errno::Unknown From 4906460ea9dca4a22ff84e39071f9b5c14e28181 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Mon, 20 Nov 2023 10:58:57 +1100 Subject: [PATCH 036/129] Updated the journal documentation file --- docs/journal.md | 148 ++++++++++++++++++++++++++++++++++++++++++++++ docs/snapshots.md | 109 ---------------------------------- 2 files changed, 148 insertions(+), 109 deletions(-) create mode 100644 docs/journal.md delete mode 100644 docs/snapshots.md diff --git a/docs/journal.md b/docs/journal.md new file mode 100644 index 00000000000..d18e2033bef --- /dev/null +++ b/docs/journal.md @@ -0,0 +1,148 @@ +# WASM Journal Functionality + +Wasmer now supports journals for the state of a WASM process. This gives the ability +to persist changes made to the temporary file system and to save and store snapshots +of the running process. + +The journal file is a linear history of events that occurred when the process was +running that if replayed will bring the process made to a discrete and deterministic +state. + +Journal files can be concatenated, compacted and filtered to change the discrete state. + +These journals are maintained in a consistent and durable way thus ensuring that +failures of the system while the process is running does not corrupt the journal. + +# Snapshot Triggers + +The journal will record state changes to the sandbox built around the WASM process as +it runs however it may be important to certain use-cases to take explicit snapshot +restoration points in the journal at key points that make sense. + +When a snapshot is triggered all the running threads of the process are paused and +the state of the WASM memory and thread stacks are recorded into the journal so that +they can be restored. + +In order to use the snapshot functionality the WASM process must be compiled with +the `asyncify` modifications, this can be done using the `wasm-opt` tool. + +Note: If a process does not have the `asyncify` modifications you can still use +the journal functionality for recording the file system and WASM memory state +however the stacks of the threads will be omitted meaning a restoration will +restart the main thread. + +Various triggers are possible that will cause a snapshot to be taken at a specific +point in time, these are as follows: + +## On Idle + +Triggered when all the threads in the process go into an idle state. This trigger +is useful to take snapshots at convenient moments without causing unnecessary overhead. + +For processes that have TTY/STDIN input this is particularly useful. + +## On Listen / FirstListen + +Triggered when a listen syscall is invoked on a socket. This can be an important +milestone to take a snapshot when one wants to speed up the boot time of a WASM process +up to the moment where it is ready to accept requests. + +## On Stdin / FirstStdin + +Triggered when the process reads stdin for the first time. This can be useful to +speed up the boot time of a WASM process. + +## On Environ / FirstEnviron + +Triggered when the process reads an environment variable for the first time. This can +be useful to speed up the boot time of a CGI WASM process which reads the environment +variables to parse the request that it must execute. + +## On Timer Interval + +Triggered periodically based on a timer (default 10 seconds) which can be specified +using the `journal-interval` option. This can be useful for asynchronous replication +of a WASM process from one machine to another with a particular lag latency. + +## On Sigint (Ctrl+C) + +Issued if the user sends an interrupt signal (Ctrl + C). + +## On Sigalrm + +Alarm clock signal (used for timers) +(see `man alarm`) + +## On Sigtstp + +The SIGTSTP signal is sent to a process by its controlling terminal to request it to stop +temporarily. It is commonly initiated by the user pressing Ctrl-Z. + +# On Sigstop + +The SIGSTOP signal instructs the operating system to stop a process for later resumption + +# On Non Deterministic Call + +When a non-deterministic call is made from WASM process to the outside world (i.e. it reaches +out of the sandbox) + +# Limitations + +- The WASM process that wish to record the state of the threads must have had the `asyncify` + post processing step applied to the binary (see `wasm-opt`). +- Taking a snapshot can consume large amounts of memory while its processing. +- Snapshots are not instant and have overhead when generating. +- The layout of the memory must be known by the runtime in order to take snapshots. + +# Design + +On startup if the restore journal file is specified then the runtime will restore the +state of the WASM process by reading and processing the log entries in the snapshot +journal. This restoration will bring the memory and the thread stacks back to a previous +point in time and then resume all the threads. + +When a trigger occurs a new journal will be taken of the WASM process which will +take the following steps: + +1. Pause all threads +2. Capture the stack of each thread +3. Write the thread state to the journal +4. Write the memory (excluding stacks) to the journal +5. Resume execution. + +The implementation is currently able to save and restore the following: + +- WASM Memory +- Stack memory +- Call stack +- Open sockets +- Open files +- Terminal text + +# Journal Capturer Implementations + +## Log File Journal + +Writes the log events to a linear log file on the local file system +as they are received by the trait. Log files can be concatenated +together to make larger log files. + +## Unsupported Journal + +The default implementation does not support snapshots and will error +out if an attempt is made to send it events. Using the unsupported +capturer as a restoration point will restore nothing but will not +error out. + +## Compacting Journal + +Deduplicates memory and stacks to reduce the number of volume of +log events sent to its inner capturer. Compacting the events occurs +in line as the events are generated + +## Filtered Journal + +Filters out a specific set of log events and drops the rest, this +capturer can be useful for restoring to a previous call point but +retaining the memory changes (e.g. WCGI runner). diff --git a/docs/snapshots.md b/docs/snapshots.md deleted file mode 100644 index 7bba60be9ee..00000000000 --- a/docs/snapshots.md +++ /dev/null @@ -1,109 +0,0 @@ -# WASM Snapshot Functionality - -Wasmer now supports snapshots of the current running process into a journal -log file which allows for the resumption from an earlier point in time. - -# Triggers - -Various triggers are possible that will cause a snapshot to be taken at -a specific point in time, these are: - -## On Idle - -Triggered when all the threads in the process goes idle. - -## On Listen - -Triggered when a listen syscall is invoked on a socket. - -## On Stdin - -Triggered when the process reads stdin for the first time - -## On Environ - -Triggered when the process reads an environment variable for the first time - -## On Timer - -Triggered periodically based on a timer (default 10 seconds) which can be specified using the `snapshot-timer` option - -## On Sigint (Ctrl+C) - -Issued if the user sends an interrupt signal (Ctrl + C). - -## On Sigalrm - -Alarm clock signal (used for timers) -(see `man alarm`) - -## On Sigtstp - -The SIGTSTP signal is sent to a process by its controlling terminal to request it to stop temporarily. It is commonly initiated by the user pressing Ctrl-Z. - -# On Sigstop - -The SIGSTOP signal instructs the operating system to stop a process for later resumption - -# On Non Deterministic Call - -When a non-determinstic call is made from WASM - -# Limitations - -- The WASM process must have had the `asyncify` post processing step applied to the binary. -- Taking a snapshot can consume large amounts of memory while its processing. -- Snapshots are not instant and have overhead when generating. -- The layout of the memory must be known by the runtime in order to take snapshots. - -# Design - -On startup if the restore snapshot file is specified then the runtime will restore the -state of the WASM process by reading and processing the log entries in the snapshot -journal. This restoration will bring the memory and the thread stacks back to a previous -point in time and then resume all the threads. - -When a trigger occurs a new snapshot will be taken of the WASM process which will -take the following steps: - -1. Pause all threads -2. Capture the stack of each thread -3. Write the thread state to the journal -4. Write the memory (excluding stacks) to the journal -5. Resume execution. - -The implementation is currently able to save and restore the following: - -- WASM Memory -- Stack memory -- Call stack -- Open sockets -- Open files -- Terminal text - -# Capturer Implementations - -## Log File Capturer - -Writes the log events to a linear log file on the local file system -as they are received by the trait. Log files can be concatenated -together to make larger log files. - -## Unsupported Capturer - -The default implementation does not support snapshots and will error -out if an attempt is made to send it events. Using the unsupported -capturer as a restoration point will restore nothing but will not -error out. - -## Compacting Capturer - -Deduplicates memory and stacks to reduce the number of volume of -log events sent to its inner capturer. Compacting the events occurs -in line as the events are generated - -## Filtered Capturer - -Filters out a specific set of log events and drops the rest, this -capturer can be useful for restoring to a previous call point but -retaining the memory changes (e.g. WCGI runner). From 00fc6f6065e316430d4d6376f8ec649f11ec2b1b Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Mon, 20 Nov 2023 11:22:08 +1100 Subject: [PATCH 037/129] More wiring of the journal snapshot triggers --- docs/journal.md | 6 +- lib/cli/src/commands/run/mod.rs | 86 ++++++++++++------- lib/cli/src/commands/run/wasi.rs | 6 +- lib/wasix/src/journal/log_file.rs | 16 ++-- lib/wasix/src/journal/mod.rs | 18 ++-- lib/wasix/src/runners/wasi.rs | 39 +++++++-- lib/wasix/src/runners/wasi_common.rs | 1 + lib/wasix/src/runners/wcgi/runner.rs | 30 +++++++ lib/wasix/src/syscalls/wasi/environ_get.rs | 5 +- .../src/syscalls/wasi/environ_sizes_get.rs | 5 +- lib/wasix/src/syscalls/wasi/fd_read.rs | 4 +- lib/wasix/src/syscalls/wasix/sock_listen.rs | 2 +- 12 files changed, 155 insertions(+), 63 deletions(-) diff --git a/docs/journal.md b/docs/journal.md index d18e2033bef..70c1d89788f 100644 --- a/docs/journal.md +++ b/docs/journal.md @@ -41,18 +41,18 @@ is useful to take snapshots at convenient moments without causing unnecessary ov For processes that have TTY/STDIN input this is particularly useful. -## On Listen / FirstListen +## On FirstListen Triggered when a listen syscall is invoked on a socket. This can be an important milestone to take a snapshot when one wants to speed up the boot time of a WASM process up to the moment where it is ready to accept requests. -## On Stdin / FirstStdin +## On FirstStdin Triggered when the process reads stdin for the first time. This can be useful to speed up the boot time of a WASM process. -## On Environ / FirstEnviron +## On FirstEnviron Triggered when the process reads an environment variable for the first time. This can be useful to speed up the boot time of a CGI WASM process which reads the environment diff --git a/lib/cli/src/commands/run/mod.rs b/lib/cli/src/commands/run/mod.rs index 0e92a2c9a1f..acad5fdf163 100644 --- a/lib/cli/src/commands/run/mod.rs +++ b/lib/cli/src/commands/run/mod.rs @@ -243,27 +243,38 @@ impl Run { } #[cfg(feature = "journal")] - for trigger in self.wasi.snapshot_on.iter().cloned() { - runner.config().add_snapshot_trigger(trigger); - } + { + for trigger in self.wasi.snapshot_on.iter().cloned() { + runner.config().add_snapshot_trigger(trigger); + } - #[cfg(feature = "journal")] - match (self.wasi.journal.clone(), self.wasi.journal_restore.clone()) { - (Some(save), Some(restore)) if save == restore => { - return Err(anyhow::format_err!( - "The snapshot save path and snapshot restore path can not be the same" - )); + if self.wasi.snapshot_on.is_empty() && self.wasi.journal.is_some() { + runner.config().add_default_snapshot_triggers(); } - (_, _) => { - if let Some(path) = self.wasi.journal.clone() { - runner - .config() - .with_journal(Arc::new(LogFileJournal::new_std(path)?)); + + if let Some(period) = self.wasi.snapshot_interval { + runner + .config() + .with_snapshot_interval(Duration::from_millis(period)); + } + + match (self.wasi.journal.clone(), self.wasi.journal_restore.clone()) { + (Some(save), Some(restore)) if save == restore => { + return Err(anyhow::format_err!( + "The snapshot save path and snapshot restore path can not be the same" + )); } - if let Some(path) = self.wasi.journal_restore.clone() { - runner - .config() - .with_journal_restore(Arc::new(LogFileJournal::new_std(path)?)); + (_, _) => { + if let Some(path) = self.wasi.journal.clone() { + runner + .config() + .with_journal(Arc::new(LogFileJournal::new_std(path)?)); + } + if let Some(path) = self.wasi.journal_restore.clone() { + runner + .config() + .with_journal_restore(Arc::new(LogFileJournal::new_std(path)?)); + } } } } @@ -331,23 +342,34 @@ impl Run { .with_capabilities(self.wasi.capabilities()); #[cfg(feature = "journal")] - for trigger in self.wasi.snapshot_on.iter().cloned() { - runner.add_snapshot_trigger(trigger); - } + { + for trigger in self.wasi.snapshot_on.iter().cloned() { + runner.add_snapshot_trigger(trigger); + } - #[cfg(feature = "journal")] - match (self.wasi.journal.clone(), self.wasi.journal_restore.clone()) { - (Some(save), Some(restore)) if save == restore => { - return Err(anyhow::format_err!( - "The snapshot save path and snapshot restore path can not be the same" - )); + // If no events are specified then add all the defaults + if self.wasi.snapshot_on.is_empty() && self.wasi.journal.is_some() { + runner.add_default_snapshot_triggers(); } - (_, _) => { - if let Some(path) = self.wasi.journal.clone() { - runner.with_journal(Arc::new(LogFileJournal::new_std(path)?)); + + // If a periodic interval is specified then make sure the event is also added + if let Some(period) = self.wasi.snapshot_interval { + runner.with_snapshot_interval(Duration::from_millis(period)); + } + + match (self.wasi.journal.clone(), self.wasi.journal_restore.clone()) { + (Some(save), Some(restore)) if save == restore => { + return Err(anyhow::format_err!( + "The snapshot save path and snapshot restore path can not be the same" + )); } - if let Some(path) = self.wasi.journal_restore.clone() { - runner.with_journal_restore(Arc::new(LogFileJournal::new_std(path)?)); + (_, _) => { + if let Some(path) = self.wasi.journal.clone() { + runner.with_journal(Arc::new(LogFileJournal::new_std(path)?)); + } + if let Some(path) = self.wasi.journal_restore.clone() { + runner.with_journal_restore(Arc::new(LogFileJournal::new_std(path)?)); + } } } } diff --git a/lib/cli/src/commands/run/wasi.rs b/lib/cli/src/commands/run/wasi.rs index e693ed5bb0d..6678b8b5be3 100644 --- a/lib/cli/src/commands/run/wasi.rs +++ b/lib/cli/src/commands/run/wasi.rs @@ -126,11 +126,15 @@ pub struct Wasi { /// /// If not specified, the default is to snapshot when the process idles, when /// the process exits or periodically if an interval argument is also supplied. + /// + /// Additionally if the snapshot-on is not specified it will also take a snapshot + /// on the first stdin, environ or socket listen - this can be used to accelerate + /// the boot up time of WASM processes. #[cfg(feature = "journal")] #[clap(long = "snapshot-on")] pub snapshot_on: Vec, - /// Adds a periodic interval (measured in seconds) that the runtime will automatically + /// Adds a periodic interval (measured in milli-seconds) that the runtime will automatically /// takes snapshots of the running process and write them to the journal. When specifying /// this parameter it implies that `--snapshot-on interval` has also been specified. #[cfg(feature = "journal")] diff --git a/lib/wasix/src/journal/log_file.rs b/lib/wasix/src/journal/log_file.rs index 9ed7dd8a768..2e021f5f506 100644 --- a/lib/wasix/src/journal/log_file.rs +++ b/lib/wasix/src/journal/log_file.rs @@ -327,10 +327,10 @@ impl Into for SnapshotTrigger { fn into(self) -> JournalSnapshotTriggerV1 { match self { SnapshotTrigger::Idle => JournalSnapshotTriggerV1::Idle, - SnapshotTrigger::Listen => JournalSnapshotTriggerV1::Listen, - SnapshotTrigger::Environ => JournalSnapshotTriggerV1::Environ, - SnapshotTrigger::Stdin => JournalSnapshotTriggerV1::Stdin, - SnapshotTrigger::Interval => JournalSnapshotTriggerV1::Timer, + SnapshotTrigger::FirstListen => JournalSnapshotTriggerV1::Listen, + SnapshotTrigger::FirstEnviron => JournalSnapshotTriggerV1::Environ, + SnapshotTrigger::FirstStdin => JournalSnapshotTriggerV1::Stdin, + SnapshotTrigger::PeriodicInterval => JournalSnapshotTriggerV1::Timer, SnapshotTrigger::Sigint => JournalSnapshotTriggerV1::Sigint, SnapshotTrigger::Sigalrm => JournalSnapshotTriggerV1::Sigalrm, SnapshotTrigger::Sigtstp => JournalSnapshotTriggerV1::Sigtstp, @@ -344,10 +344,10 @@ impl Into for JournalSnapshotTriggerV1 { fn into(self) -> SnapshotTrigger { match self { JournalSnapshotTriggerV1::Idle => SnapshotTrigger::Idle, - JournalSnapshotTriggerV1::Listen => SnapshotTrigger::Listen, - JournalSnapshotTriggerV1::Environ => SnapshotTrigger::Environ, - JournalSnapshotTriggerV1::Stdin => SnapshotTrigger::Stdin, - JournalSnapshotTriggerV1::Timer => SnapshotTrigger::Interval, + JournalSnapshotTriggerV1::Listen => SnapshotTrigger::FirstListen, + JournalSnapshotTriggerV1::Environ => SnapshotTrigger::FirstEnviron, + JournalSnapshotTriggerV1::Stdin => SnapshotTrigger::FirstStdin, + JournalSnapshotTriggerV1::Timer => SnapshotTrigger::PeriodicInterval, JournalSnapshotTriggerV1::Sigint => SnapshotTrigger::Sigint, JournalSnapshotTriggerV1::Sigalrm => SnapshotTrigger::Sigalrm, JournalSnapshotTriggerV1::Sigtstp => SnapshotTrigger::Sigtstp, diff --git a/lib/wasix/src/journal/mod.rs b/lib/wasix/src/journal/mod.rs index 03c8645b53b..2cfc8a64106 100644 --- a/lib/wasix/src/journal/mod.rs +++ b/lib/wasix/src/journal/mod.rs @@ -27,14 +27,14 @@ use std::str::FromStr; pub enum SnapshotTrigger { /// Triggered when all the threads in the process goes idle Idle, - /// Triggered when a listen syscall is invoked on a socket - Listen, + /// Triggered when a listen syscall is invoked on a socket for the first time + FirstListen, /// Triggered on reading the environment variables for the first time - Environ, + FirstEnviron, /// Triggered when the process reads stdin for the first time - Stdin, + FirstStdin, /// Triggered periodically based on a interval (default 10 seconds) which can be specified using the `snapshot-interval` option - Interval, + PeriodicInterval, /// Issued if the user sends an interrupt signal (Ctrl + C). Sigint, /// Alarm clock signal (used for timers) @@ -54,10 +54,10 @@ impl FromStr for SnapshotTrigger { let s = s.trim().to_lowercase(); Ok(match s.as_str() { "idle" => Self::Idle, - "listen" => Self::Listen, - "stdin" => Self::Stdin, - "environ" => Self::Environ, - "periodic" => Self::Interval, + "first-listen" => Self::FirstListen, + "first-stdin" => Self::FirstStdin, + "first-environ" => Self::FirstEnviron, + "periodic-interval" => Self::PeriodicInterval, "intr" | "sigint" | "ctrlc" | "ctrl-c" => Self::Sigint, "alarm" | "timer" | "sigalrm" => Self::Sigalrm, "sigtstp" | "ctrlz" | "ctrl-z" => Self::Sigtstp, diff --git a/lib/wasix/src/runners/wasi.rs b/lib/wasix/src/runners/wasi.rs index 95090aff05d..288392cdce6 100644 --- a/lib/wasix/src/runners/wasi.rs +++ b/lib/wasix/src/runners/wasi.rs @@ -194,6 +194,33 @@ impl WasiRunner { self } + pub fn add_default_snapshot_triggers(&mut self) -> &mut Self { + let defs = [ + SnapshotTrigger::Idle, + SnapshotTrigger::FirstEnviron, + SnapshotTrigger::FirstListen, + SnapshotTrigger::FirstStdin, + ]; + for on in defs { + if self.has_snapshot_trigger(on) == false { + self.add_snapshot_trigger(on); + } + } + self + } + + pub fn has_snapshot_trigger(&self, on: SnapshotTrigger) -> bool { + self.wasi.snapshot_on.iter().any(|t| *t == on) + } + + pub fn with_snapshot_interval(&mut self, period: std::time::Duration) -> &mut Self { + if self.has_snapshot_trigger(SnapshotTrigger::PeriodicInterval) == false { + self.add_snapshot_trigger(SnapshotTrigger::PeriodicInterval); + } + self.wasi.snapshot_interval.replace(period); + self + } + pub fn with_journal_restore(&mut self, restorer: Arc) -> &mut Self { self.wasi .journal_restore @@ -318,11 +345,6 @@ impl crate::runners::Runner for WasiRunner { .prepare_webc_env(command_name, &wasi, Some(pkg), Arc::clone(&runtime), None) .context("Unable to prepare the WASI environment")?; - #[cfg(feature = "journal")] - for snapshot_trigger in self.wasi.snapshot_on.iter().cloned() { - env.add_snapshot_trigger(snapshot_trigger); - } - #[cfg(feature = "journal")] if let Some(capturer) = self.wasi.journal.clone() { env = env.with_journal(capturer); @@ -333,6 +355,13 @@ impl crate::runners::Runner for WasiRunner { env = env.with_journal_restore(restore.restorer); } + #[cfg(feature = "journal")] + { + for snapshot_trigger in self.wasi.snapshot_on.iter().cloned() { + env.add_snapshot_trigger(snapshot_trigger); + } + } + let env = env.build()?; let store = runtime.new_store(); diff --git a/lib/wasix/src/runners/wasi_common.rs b/lib/wasix/src/runners/wasi_common.rs index ff7e70f3550..9e8eae7734e 100644 --- a/lib/wasix/src/runners/wasi_common.rs +++ b/lib/wasix/src/runners/wasi_common.rs @@ -42,6 +42,7 @@ pub(crate) struct CommonWasiOptions { #[derivative(Debug = "ignore")] pub(crate) journal_restore: Option, pub(crate) snapshot_on: Vec, + pub(crate) snapshot_interval: Option, pub(crate) current_dir: Option, } diff --git a/lib/wasix/src/runners/wcgi/runner.rs b/lib/wasix/src/runners/wcgi/runner.rs index dfc8b42c6c6..22800d9d496 100644 --- a/lib/wasix/src/runners/wcgi/runner.rs +++ b/lib/wasix/src/runners/wcgi/runner.rs @@ -247,6 +247,27 @@ impl Config { self.wasi.snapshot_on.push(on); } + #[cfg(feature = "journal")] + pub fn add_default_snapshot_triggers(&mut self) -> &mut Self { + let defs = [ + crate::journal::SnapshotTrigger::Idle, + crate::journal::SnapshotTrigger::FirstEnviron, + crate::journal::SnapshotTrigger::FirstListen, + crate::journal::SnapshotTrigger::FirstStdin, + ]; + for on in defs { + if self.has_snapshot_trigger(on) == false { + self.add_snapshot_trigger(on); + } + } + self + } + + #[cfg(feature = "journal")] + pub fn has_snapshot_trigger(&self, on: crate::journal::SnapshotTrigger) -> bool { + self.wasi.snapshot_on.iter().any(|t| *t == on) + } + #[cfg(feature = "journal")] pub fn with_journal_restore(&mut self, capturer: Arc) -> &mut Self { use crate::state::JournalRestore; @@ -257,6 +278,15 @@ impl Config { self } + #[cfg(feature = "journal")] + pub fn with_snapshot_interval(&mut self, period: std::time::Duration) -> &mut Self { + if self.has_snapshot_trigger(crate::journal::SnapshotTrigger::PeriodicInterval) == false { + self.add_snapshot_trigger(crate::journal::SnapshotTrigger::PeriodicInterval); + } + self.wasi.snapshot_interval.replace(period); + self + } + #[cfg(feature = "journal")] pub fn with_journal(&mut self, capturer: Arc) -> &mut Self { self.wasi.journal.replace(capturer); diff --git a/lib/wasix/src/syscalls/wasi/environ_get.rs b/lib/wasix/src/syscalls/wasi/environ_get.rs index a169581a652..a39a5faa5f6 100644 --- a/lib/wasix/src/syscalls/wasi/environ_get.rs +++ b/lib/wasix/src/syscalls/wasi/environ_get.rs @@ -15,7 +15,10 @@ pub fn environ_get( environ: WasmPtr, M>, environ_buf: WasmPtr, ) -> Result { - ctx = wasi_try_ok!(maybe_snapshot_once::(ctx, SnapshotTrigger::Environ)?); + ctx = wasi_try_ok!(maybe_snapshot_once::( + ctx, + SnapshotTrigger::FirstEnviron + )?); let env = ctx.data(); let (memory, mut state) = unsafe { env.get_memory_and_wasi_state(&ctx, 0) }; diff --git a/lib/wasix/src/syscalls/wasi/environ_sizes_get.rs b/lib/wasix/src/syscalls/wasi/environ_sizes_get.rs index e695fe7dd6d..846b8e92839 100644 --- a/lib/wasix/src/syscalls/wasi/environ_sizes_get.rs +++ b/lib/wasix/src/syscalls/wasi/environ_sizes_get.rs @@ -14,7 +14,10 @@ pub fn environ_sizes_get( environ_count: WasmPtr, environ_buf_size: WasmPtr, ) -> Result { - ctx = wasi_try_ok!(maybe_snapshot_once::(ctx, SnapshotTrigger::Environ)?); + ctx = wasi_try_ok!(maybe_snapshot_once::( + ctx, + SnapshotTrigger::FirstEnviron + )?); let env = ctx.data(); let (memory, mut state) = unsafe { env.get_memory_and_wasi_state(&ctx, 0) }; diff --git a/lib/wasix/src/syscalls/wasi/fd_read.rs b/lib/wasix/src/syscalls/wasi/fd_read.rs index 1d4a8e006be..2d16eda58b4 100644 --- a/lib/wasix/src/syscalls/wasi/fd_read.rs +++ b/lib/wasix/src/syscalls/wasi/fd_read.rs @@ -45,7 +45,7 @@ pub fn fd_read( }; if fd == DeviceFile::STDIN { - ctx = wasi_try_ok!(maybe_snapshot_once::(ctx, SnapshotTrigger::Stdin)?); + ctx = wasi_try_ok!(maybe_snapshot_once::(ctx, SnapshotTrigger::FirstStdin)?); } let res = fd_read_internal::(&mut ctx, fd, iovs, iovs_len, offset, nread, true)?; @@ -80,7 +80,7 @@ pub fn fd_pread( let tid = ctx.data().tid(); if fd == DeviceFile::STDIN { - ctx = wasi_try_ok!(maybe_snapshot_once::(ctx, SnapshotTrigger::Stdin)?); + ctx = wasi_try_ok!(maybe_snapshot_once::(ctx, SnapshotTrigger::FirstStdin)?); } let res = fd_read_internal::(&mut ctx, fd, iovs, iovs_len, offset as usize, nread, false)?; diff --git a/lib/wasix/src/syscalls/wasix/sock_listen.rs b/lib/wasix/src/syscalls/wasix/sock_listen.rs index 397e6905342..0b82d05f7d5 100644 --- a/lib/wasix/src/syscalls/wasix/sock_listen.rs +++ b/lib/wasix/src/syscalls/wasix/sock_listen.rs @@ -19,7 +19,7 @@ pub fn sock_listen( sock: WasiFd, backlog: M::Offset, ) -> Result { - ctx = wasi_try_ok!(maybe_snapshot_once::(ctx, SnapshotTrigger::Listen)?); + ctx = wasi_try_ok!(maybe_snapshot_once::(ctx, SnapshotTrigger::FirstListen)?); let env = ctx.data(); let net = env.net().clone(); From 8337e72ff6672ec6556bf1d0831a2615987bffaa Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Mon, 20 Nov 2023 12:47:03 +1100 Subject: [PATCH 038/129] Clippy fixes --- lib/wasix/src/journal/capturer.rs | 3 +- lib/wasix/src/journal/compactor.rs | 3 +- lib/wasix/src/journal/effector/mod.rs | 2 +- .../journal/effector/syscalls/path_open.rs | 36 ++++++------------- lib/wasix/src/journal/log_file.rs | 5 --- lib/wasix/src/journal/mod.rs | 7 ++++ lib/wasix/src/journal/unsupported.rs | 2 +- lib/wasix/src/runners/wasi.rs | 12 ++----- lib/wasix/src/runners/wcgi/runner.rs | 10 ++---- lib/wasix/src/syscalls/mod.rs | 4 +-- lib/wasix/src/syscalls/wasi/fd_write.rs | 17 +++++---- lib/wasix/src/syscalls/wasi/path_open.rs | 7 ++-- .../syscalls/wasi/path_remove_directory.rs | 2 +- lib/wasix/src/syscalls/wasix/epoll_ctl.rs | 2 +- lib/wasix/src/syscalls/wasix/tty_set.rs | 5 ++- 15 files changed, 45 insertions(+), 72 deletions(-) diff --git a/lib/wasix/src/journal/capturer.rs b/lib/wasix/src/journal/capturer.rs index b68ee906636..91127029266 100644 --- a/lib/wasix/src/journal/capturer.rs +++ b/lib/wasix/src/journal/capturer.rs @@ -85,7 +85,6 @@ pub enum JournalEntry<'a> { fs_rights_base: Rights, fs_rights_inheriting: Rights, fs_flags: Fdflags, - is_64bit: bool, }, RenumberFileDescriptor { old_fd: Fd, @@ -201,7 +200,7 @@ pub trait Journal { /// Returns a stream of snapshot objects that the runtime will use /// to restore the state of a WASM process to a previous moment in time - fn read<'a>(&'a self) -> LocalBoxFuture<'a, anyhow::Result>>>; + fn read(&self) -> LocalBoxFuture<'_, anyhow::Result>>>; } pub type DynJournal = dyn Journal + Send + Sync; diff --git a/lib/wasix/src/journal/compactor.rs b/lib/wasix/src/journal/compactor.rs index 5598a0515cb..c6a10b1a322 100644 --- a/lib/wasix/src/journal/compactor.rs +++ b/lib/wasix/src/journal/compactor.rs @@ -55,6 +55,7 @@ impl Journal for CompactingJournal { *other = hash; } } else { + #[allow(clippy::nonminimal_bool)] let to_remove = state .memory_map .keys() @@ -103,7 +104,6 @@ impl Journal for CompactingJournal { fs_rights_base, fs_rights_inheriting, fs_flags, - is_64bit, } => { let mut state = self.state.lock().unwrap(); state.close_file.remove(&fd); @@ -118,7 +118,6 @@ impl Journal for CompactingJournal { fs_rights_base, fs_rights_inheriting, fs_flags, - is_64bit, }, ); } diff --git a/lib/wasix/src/journal/effector/mod.rs b/lib/wasix/src/journal/effector/mod.rs index e6916babf12..ed319899c7e 100644 --- a/lib/wasix/src/journal/effector/mod.rs +++ b/lib/wasix/src/journal/effector/mod.rs @@ -5,7 +5,7 @@ pub(super) use std::{ pub(super) use anyhow::bail; pub(super) use bytes::Bytes; pub(super) use wasmer::{FunctionEnvMut, RuntimeError, WasmPtr}; -pub(super) use wasmer_types::{Memory32, Memory64, MemorySize}; +pub(super) use wasmer_types::MemorySize; pub(super) use wasmer_wasix_types::{ types::__wasi_ciovec_t, wasi::{ diff --git a/lib/wasix/src/journal/effector/syscalls/path_open.rs b/lib/wasix/src/journal/effector/syscalls/path_open.rs index ecd11695840..d360b1762e3 100644 --- a/lib/wasix/src/journal/effector/syscalls/path_open.rs +++ b/lib/wasix/src/journal/effector/syscalls/path_open.rs @@ -11,7 +11,6 @@ impl JournalEffector { fs_rights_base: Rights, fs_rights_inheriting: Rights, fs_flags: Fdflags, - is_64bit: bool, ) -> anyhow::Result<()> { Self::save_event( ctx, @@ -24,7 +23,6 @@ impl JournalEffector { fs_rights_base, fs_rights_inheriting, fs_flags, - is_64bit, }, ) } @@ -39,31 +37,17 @@ impl JournalEffector { fs_rights_base: Rights, fs_rights_inheriting: Rights, fs_flags: Fdflags, - is_64bit: bool, ) -> anyhow::Result<()> { - let res = if is_64bit { - crate::syscalls::path_open_internal::( - ctx, - dirfd, - dirflags, - path, - o_flags, - fs_rights_base, - fs_rights_inheriting, - fs_flags, - ) - } else { - crate::syscalls::path_open_internal::( - ctx, - dirfd, - dirflags, - path, - o_flags, - fs_rights_base, - fs_rights_inheriting, - fs_flags, - ) - }; + let res = crate::syscalls::path_open_internal( + ctx, + dirfd, + dirflags, + path, + o_flags, + fs_rights_base, + fs_rights_inheriting, + fs_flags, + ); let ret_fd = match res? { Ok(fd) => fd, Err(err) => { diff --git a/lib/wasix/src/journal/log_file.rs b/lib/wasix/src/journal/log_file.rs index 2e021f5f506..9446fe38405 100644 --- a/lib/wasix/src/journal/log_file.rs +++ b/lib/wasix/src/journal/log_file.rs @@ -69,7 +69,6 @@ pub(crate) enum LogFileJournalEntry { fs_rights_base: u64, fs_rights_inheriting: u64, fs_flags: u16, - is_64bit: bool, }, RenumberFileDescriptorV1 { old_fd: u32, @@ -439,7 +438,6 @@ impl<'a> From> for LogFileJournalEntry { fs_rights_base, fs_rights_inheriting, fs_flags, - is_64bit, } => Self::OpenFileDescriptorV1 { fd, dirfd, @@ -449,7 +447,6 @@ impl<'a> From> for LogFileJournalEntry { fs_rights_base: fs_rights_base.bits(), fs_rights_inheriting: fs_rights_inheriting.bits(), fs_flags: fs_flags.bits(), - is_64bit, }, JournalEntry::RemoveDirectory { fd, path } => Self::RemoveDirectoryV1 { fd, @@ -657,7 +654,6 @@ impl<'a> From for JournalEntry<'a> { fs_rights_base, fs_rights_inheriting, fs_flags, - is_64bit, } => Self::OpenFileDescriptor { fd, dirfd, @@ -667,7 +663,6 @@ impl<'a> From for JournalEntry<'a> { fs_rights_base: wasi::Rights::from_bits_truncate(fs_rights_base), fs_rights_inheriting: wasi::Rights::from_bits_truncate(fs_rights_inheriting), fs_flags: wasi::Fdflags::from_bits_truncate(fs_flags), - is_64bit, }, LogFileJournalEntry::RemoveDirectoryV1 { fd, path } => Self::RemoveDirectory { fd, diff --git a/lib/wasix/src/journal/mod.rs b/lib/wasix/src/journal/mod.rs index 2cfc8a64106..28b68b5d21a 100644 --- a/lib/wasix/src/journal/mod.rs +++ b/lib/wasix/src/journal/mod.rs @@ -47,6 +47,13 @@ pub enum SnapshotTrigger { NonDeterministicCall, } +pub const DEFAULT_SNAPSHOT_TRIGGERS: [SnapshotTrigger; 4] = [ + SnapshotTrigger::Idle, + SnapshotTrigger::FirstEnviron, + SnapshotTrigger::FirstListen, + SnapshotTrigger::FirstStdin, +]; + impl FromStr for SnapshotTrigger { type Err = anyhow::Error; diff --git a/lib/wasix/src/journal/unsupported.rs b/lib/wasix/src/journal/unsupported.rs index cb712277219..41f8005f416 100644 --- a/lib/wasix/src/journal/unsupported.rs +++ b/lib/wasix/src/journal/unsupported.rs @@ -15,7 +15,7 @@ impl Journal for UnsupportedJournal { Box::pin(async { Err(anyhow::format_err!("unsupported")) }) } - fn read<'a>(&'a self) -> LocalBoxFuture<'a, anyhow::Result>>> { + fn read(&self) -> LocalBoxFuture<'_, anyhow::Result>>> { Box::pin(async { Ok(None) }) } } diff --git a/lib/wasix/src/runners/wasi.rs b/lib/wasix/src/runners/wasi.rs index 288392cdce6..6963820cb6e 100644 --- a/lib/wasix/src/runners/wasi.rs +++ b/lib/wasix/src/runners/wasi.rs @@ -195,14 +195,8 @@ impl WasiRunner { } pub fn add_default_snapshot_triggers(&mut self) -> &mut Self { - let defs = [ - SnapshotTrigger::Idle, - SnapshotTrigger::FirstEnviron, - SnapshotTrigger::FirstListen, - SnapshotTrigger::FirstStdin, - ]; - for on in defs { - if self.has_snapshot_trigger(on) == false { + for on in crate::journal::DEFAULT_SNAPSHOT_TRIGGERS { + if !self.has_snapshot_trigger(on) { self.add_snapshot_trigger(on); } } @@ -214,7 +208,7 @@ impl WasiRunner { } pub fn with_snapshot_interval(&mut self, period: std::time::Duration) -> &mut Self { - if self.has_snapshot_trigger(SnapshotTrigger::PeriodicInterval) == false { + if !self.has_snapshot_trigger(SnapshotTrigger::PeriodicInterval) { self.add_snapshot_trigger(SnapshotTrigger::PeriodicInterval); } self.wasi.snapshot_interval.replace(period); diff --git a/lib/wasix/src/runners/wcgi/runner.rs b/lib/wasix/src/runners/wcgi/runner.rs index 22800d9d496..00cda9fd70c 100644 --- a/lib/wasix/src/runners/wcgi/runner.rs +++ b/lib/wasix/src/runners/wcgi/runner.rs @@ -249,14 +249,8 @@ impl Config { #[cfg(feature = "journal")] pub fn add_default_snapshot_triggers(&mut self) -> &mut Self { - let defs = [ - crate::journal::SnapshotTrigger::Idle, - crate::journal::SnapshotTrigger::FirstEnviron, - crate::journal::SnapshotTrigger::FirstListen, - crate::journal::SnapshotTrigger::FirstStdin, - ]; - for on in defs { - if self.has_snapshot_trigger(on) == false { + for on in crate::journal::DEFAULT_SNAPSHOT_TRIGGERS { + if !self.has_snapshot_trigger(on) { self.add_snapshot_trigger(on); } } diff --git a/lib/wasix/src/syscalls/mod.rs b/lib/wasix/src/syscalls/mod.rs index 7f0ecc493f7..092808ac10d 100644 --- a/lib/wasix/src/syscalls/mod.rs +++ b/lib/wasix/src/syscalls/mod.rs @@ -1221,6 +1221,7 @@ pub fn rewind_ext( Errno::Success } +#[allow(unused)] #[cfg(not(feature = "journal"))] pub fn maybe_snapshot_once( ctx: FunctionEnvMut<'_, WasiEnv>, @@ -1259,6 +1260,7 @@ pub fn maybe_snapshot_once( Ok(Ok(ctx)) } +#[allow(unused)] #[cfg(not(feature = "journal"))] pub fn maybe_snapshot( ctx: FunctionEnvMut<'_, WasiEnv>, @@ -1375,7 +1377,6 @@ pub fn restore_snapshot( fs_rights_base, fs_rights_inheriting, fs_flags, - is_64bit, } => { JournalEffector::apply_path_open( &mut ctx, @@ -1387,7 +1388,6 @@ pub fn restore_snapshot( fs_rights_base, fs_rights_inheriting, fs_flags, - is_64bit, ) .map_err(anyhow_err_to_runtime_err)?; } diff --git a/lib/wasix/src/syscalls/wasi/fd_write.rs b/lib/wasix/src/syscalls/wasi/fd_write.rs index 74a5be0debd..916b0b81806 100644 --- a/lib/wasix/src/syscalls/wasi/fd_write.rs +++ b/lib/wasix/src/syscalls/wasi/fd_write.rs @@ -79,7 +79,7 @@ pub fn fd_pwrite( &mut ctx, fd, FdWriteSource::Iovs { iovs, iovs_len }, - offset as u64, + offset, Some(nwritten), false, enable_snapshot_capture, @@ -139,7 +139,7 @@ pub(crate) fn fd_write_internal( let mut handle = handle.write().unwrap(); if !is_stdio { handle - .seek(std::io::SeekFrom::Start(offset as u64)) + .seek(std::io::SeekFrom::Start(offset)) .await .map_err(map_io_err)?; } @@ -149,7 +149,7 @@ pub(crate) fn fd_write_internal( match &data { FdWriteSource::Iovs { iovs, iovs_len } => { let iovs_arr = iovs - .slice(&memory, iovs_len.clone()) + .slice(&memory, *iovs_len) .map_err(mem_error_to_wasi)?; let iovs_arr = iovs_arr.access().map_err(mem_error_to_wasi)?; @@ -211,9 +211,8 @@ pub(crate) fn fd_write_internal( match &data { FdWriteSource::Iovs { iovs, iovs_len } => { - let iovs_arr = iovs - .slice(&memory, iovs_len.clone()) - .map_err(mem_error_to_wasi)?; + let iovs_arr = + iovs.slice(&memory, *iovs_len).map_err(mem_error_to_wasi)?; let iovs_arr = iovs_arr.access().map_err(mem_error_to_wasi)?; for iovs in iovs_arr.iter() { let buf = WasmPtr::::new(iovs.buf) @@ -252,7 +251,7 @@ pub(crate) fn fd_write_internal( match &data { FdWriteSource::Iovs { iovs, iovs_len } => { let iovs_arr = wasi_try_ok!(iovs - .slice(&memory, iovs_len.clone()) + .slice(&memory, *iovs_len) .map_err(mem_error_to_wasi)); let iovs_arr = wasi_try_ok!(iovs_arr.access().map_err(mem_error_to_wasi)); @@ -289,7 +288,7 @@ pub(crate) fn fd_write_internal( match &data { FdWriteSource::Iovs { iovs, iovs_len } => { let iovs_arr = wasi_try_ok!(iovs - .slice(&memory, iovs_len.clone()) + .slice(&memory, *iovs_len) .map_err(mem_error_to_wasi)); let iovs_arr = wasi_try_ok!(iovs_arr.access().map_err(mem_error_to_wasi)); @@ -341,7 +340,7 @@ pub(crate) fn fd_write_internal( match &data { FdWriteSource::Iovs { iovs, iovs_len } => { let iovs_arr = wasi_try_ok!(iovs - .slice(&memory, iovs_len.clone()) + .slice(&memory, *iovs_len) .map_err(mem_error_to_wasi)); let iovs_arr = wasi_try_ok!(iovs_arr.access().map_err(mem_error_to_wasi)); diff --git a/lib/wasix/src/syscalls/wasi/path_open.rs b/lib/wasix/src/syscalls/wasi/path_open.rs index 3ca3207845f..db9f2320ea0 100644 --- a/lib/wasix/src/syscalls/wasi/path_open.rs +++ b/lib/wasix/src/syscalls/wasi/path_open.rs @@ -67,7 +67,7 @@ pub fn path_open( ); } - let out_fd = wasi_try_ok!(path_open_internal::( + let out_fd = wasi_try_ok!(path_open_internal( &mut ctx, dirfd, dirflags, @@ -91,7 +91,6 @@ pub fn path_open( fs_rights_base, fs_rights_inheriting, fs_flags, - M::is_64bit(), ) .map_err(|err| { tracing::error!("failed to save unlink event to snapshot capturer - {}", err); @@ -111,7 +110,7 @@ pub fn path_open( Ok(Errno::Success) } -pub(crate) fn path_open_internal( +pub(crate) fn path_open_internal( ctx: &mut FunctionEnvMut<'_, WasiEnv>, dirfd: WasiFd, dirflags: LookupFlags, @@ -129,7 +128,7 @@ pub(crate) fn path_open_internal( let maybe_inode = state.fs.get_inode_at_path( inodes, dirfd, - &path, + path, dirflags & __WASI_LOOKUP_SYMLINK_FOLLOW != 0, ); diff --git a/lib/wasix/src/syscalls/wasi/path_remove_directory.rs b/lib/wasix/src/syscalls/wasi/path_remove_directory.rs index 84a4d46c6d3..c547075eaaf 100644 --- a/lib/wasix/src/syscalls/wasi/path_remove_directory.rs +++ b/lib/wasix/src/syscalls/wasi/path_remove_directory.rs @@ -49,7 +49,7 @@ pub(crate) fn path_remove_directory_internal( let env = ctx.data(); let (memory, mut state, inodes) = unsafe { env.get_memory_and_wasi_state_and_inodes(&ctx, 0) }; - let inode = state.fs.get_inode_at_path(inodes, fd, &path, false)?; + let inode = state.fs.get_inode_at_path(inodes, fd, path, false)?; let (parent_inode, childs_name) = state .fs diff --git a/lib/wasix/src/syscalls/wasix/epoll_ctl.rs b/lib/wasix/src/syscalls/wasix/epoll_ctl.rs index 9126e43f7f6..0f42d324c20 100644 --- a/lib/wasix/src/syscalls/wasix/epoll_ctl.rs +++ b/lib/wasix/src/syscalls/wasix/epoll_ctl.rs @@ -97,7 +97,7 @@ pub(crate) fn epoll_ctl_internal( if let Some(event) = event_ctl { let epoll_fd = EpollFd { events: event.events, - ptr: wasi_try_ok_ok!(event.ptr.try_into().map_err(|_| Errno::Overflow)), + ptr: event.ptr, fd: event.fd, data1: event.data1, data2: event.data2, diff --git a/lib/wasix/src/syscalls/wasix/tty_set.rs b/lib/wasix/src/syscalls/wasix/tty_set.rs index e7f28e9d7ee..d062b77f9e0 100644 --- a/lib/wasix/src/syscalls/wasix/tty_set.rs +++ b/lib/wasix/src/syscalls/wasix/tty_set.rs @@ -34,7 +34,10 @@ pub fn tty_set( line_feeds, }; - wasi_try_ok!(tty_set_internal(&mut ctx, state.clone())); + wasi_try_ok!({ + #[allow(clippy::redundant_clone)] + tty_set_internal(&mut ctx, state.clone()) + }); let env = ctx.data(); #[cfg(feature = "journal")] From 687667d4084d209869f89fd4734180c6840c8c3b Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Mon, 20 Nov 2023 12:53:06 +1100 Subject: [PATCH 039/129] Fixed the JSC compile errors --- lib/cli/src/commands/run/mod.rs | 3 ++- lib/cli/src/commands/run/wasi.rs | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/cli/src/commands/run/mod.rs b/lib/cli/src/commands/run/mod.rs index acad5fdf163..16556151e22 100644 --- a/lib/cli/src/commands/run/mod.rs +++ b/lib/cli/src/commands/run/mod.rs @@ -28,9 +28,10 @@ use wasmer::{ #[cfg(feature = "compiler")] use wasmer_compiler::ArtifactBuild; use wasmer_registry::{wasmer_env::WasmerEnv, Package}; +#[cfg(feature = "journal")] +use wasmer_wasix::journal::{LogFileJournal, SnapshotTrigger}; use wasmer_wasix::{ bin_factory::BinaryPackage, - journal::{LogFileJournal, SnapshotTrigger}, runners::{MappedCommand, MappedDirectory, Runner}, runtime::{ module_cache::{CacheError, ModuleHash}, diff --git a/lib/cli/src/commands/run/wasi.rs b/lib/cli/src/commands/run/wasi.rs index 6678b8b5be3..b4b5d0b506f 100644 --- a/lib/cli/src/commands/run/wasi.rs +++ b/lib/cli/src/commands/run/wasi.rs @@ -14,12 +14,13 @@ use url::Url; use virtual_fs::{DeviceFile, FileSystem, PassthruFileSystem, RootFileSystemBuilder}; use wasmer::{Engine, Function, Instance, Memory32, Memory64, Module, RuntimeError, Store, Value}; use wasmer_registry::wasmer_env::WasmerEnv; +#[cfg(feature = "journal")] +use wasmer_wasix::journal::{LogFileJournal, SnapshotTrigger}; use wasmer_wasix::{ bin_factory::BinaryPackage, capabilities::Capabilities, default_fs_backing, get_wasi_versions, http::HttpClient, - journal::{self, LogFileJournal, SnapshotTrigger}, os::{tty_sys::SysTty, TtyBridge}, rewind_ext, runners::{MappedCommand, MappedDirectory}, @@ -520,7 +521,7 @@ impl Wasi { #[cfg(feature = "journal")] if let Some(path) = &self.journal { - rt.set_snapshot_capturer(Arc::new(journal::LogFileJournal::new_std(path)?)); + rt.set_snapshot_capturer(Arc::new(LogFileJournal::new_std(path)?)); } if !self.no_tty { From 897b8d700a2a8eb9de280b5fd2c84691dc052a4d Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Mon, 20 Nov 2023 13:02:31 +1100 Subject: [PATCH 040/129] Added a panic journal event --- lib/wasix/src/journal/capturer.rs | 4 ++++ lib/wasix/src/journal/compactor.rs | 2 +- lib/wasix/src/journal/filter.rs | 15 ++++++++++++++- lib/wasix/src/journal/log_file.rs | 14 +++++++++++++- lib/wasix/src/syscalls/mod.rs | 10 ++++++++-- 5 files changed, 40 insertions(+), 5 deletions(-) diff --git a/lib/wasix/src/journal/capturer.rs b/lib/wasix/src/journal/capturer.rs index 91127029266..d53cf31f0e9 100644 --- a/lib/wasix/src/journal/capturer.rs +++ b/lib/wasix/src/journal/capturer.rs @@ -182,6 +182,10 @@ pub enum JournalEntry<'a> { fd1: Fd, fd2: Fd, }, + Panic { + when: SystemTime, + stack_trace: Cow<'a, str>, + }, /// Represents the marker for the end of a snapshot Snapshot { when: SystemTime, diff --git a/lib/wasix/src/journal/compactor.rs b/lib/wasix/src/journal/compactor.rs index c6a10b1a322..3c4d75c7681 100644 --- a/lib/wasix/src/journal/compactor.rs +++ b/lib/wasix/src/journal/compactor.rs @@ -147,7 +147,7 @@ impl Journal for CompactingJournal { }) } - fn read<'a>(&'a self) -> LocalBoxFuture<'a, anyhow::Result>>> { + fn read(&self) -> LocalBoxFuture<'_, anyhow::Result>>> { Box::pin(async { Ok(match self.inner.read().await? { Some(JournalEntry::UpdateMemoryRegion { region, data }) => { diff --git a/lib/wasix/src/journal/filter.rs b/lib/wasix/src/journal/filter.rs index 57c77ba2a37..4714f79c88c 100644 --- a/lib/wasix/src/journal/filter.rs +++ b/lib/wasix/src/journal/filter.rs @@ -16,6 +16,7 @@ pub struct FilteredJournal { filter_snapshots: bool, filter_descriptors: bool, filter_epoll: bool, + filter_panics: bool, } impl FilteredJournal { @@ -31,6 +32,7 @@ impl FilteredJournal { filter_snapshots: false, filter_descriptors: false, filter_epoll: false, + filter_panics: false, } } @@ -78,6 +80,11 @@ impl FilteredJournal { self.filter_descriptors = val; self } + + pub fn with_ignore_panics(mut self, val: bool) -> Self { + self.filter_panics = val; + self + } } impl Journal for FilteredJournal { @@ -260,12 +267,18 @@ impl Journal for FilteredJournal { } entry } + JournalEntry::Panic { .. } => { + if self.filter_panics { + return Ok(()); + } + entry + } }; self.inner.write(evt).await }) } - fn read<'a>(&'a self) -> LocalBoxFuture<'a, anyhow::Result>>> { + fn read(&self) -> LocalBoxFuture<'_, anyhow::Result>>> { Box::pin(async { self.inner.read().await }) } } diff --git a/lib/wasix/src/journal/log_file.rs b/lib/wasix/src/journal/log_file.rs index 9446fe38405..56056beb0b7 100644 --- a/lib/wasix/src/journal/log_file.rs +++ b/lib/wasix/src/journal/log_file.rs @@ -178,6 +178,10 @@ pub(crate) enum LogFileJournalEntry { when: SystemTime, trigger: JournalSnapshotTriggerV1, }, + Panic { + when: SystemTime, + stack_trace: String, + }, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -595,6 +599,10 @@ impl<'a> From> for LogFileJournalEntry { line_feeds, }, JournalEntry::CreatePipe { fd1, fd2 } => Self::CreatePipeV1 { fd1, fd2 }, + JournalEntry::Panic { when, stack_trace } => Self::Panic { + when, + stack_trace: stack_trace.into_owned(), + }, } } } @@ -826,6 +834,10 @@ impl<'a> From for JournalEntry<'a> { line_feeds, }, LogFileJournalEntry::CreatePipeV1 { fd1, fd2 } => Self::CreatePipe { fd1, fd2 }, + LogFileJournalEntry::Panic { when, stack_trace } => Self::Panic { + when, + stack_trace: stack_trace.into(), + }, } } } @@ -911,7 +923,7 @@ impl Journal for LogFileJournal { /// UNSAFE: This method uses unsafe operations to remove the need to zero /// the buffer before its read the log entries into it - fn read<'a>(&'a self) -> LocalBoxFuture<'a, anyhow::Result>>> { + fn read(&self) -> LocalBoxFuture<'_, anyhow::Result>>> { Box::pin(async { let mut state = self.state.lock().await; diff --git a/lib/wasix/src/syscalls/mod.rs b/lib/wasix/src/syscalls/mod.rs index 092808ac10d..e8e8e432d5d 100644 --- a/lib/wasix/src/syscalls/mod.rs +++ b/lib/wasix/src/syscalls/mod.rs @@ -1221,7 +1221,7 @@ pub fn rewind_ext( Errno::Success } -#[allow(unused)] +#[allow(clippy::extra_unused_type_parameters)] #[cfg(not(feature = "journal"))] pub fn maybe_snapshot_once( ctx: FunctionEnvMut<'_, WasiEnv>, @@ -1260,7 +1260,7 @@ pub fn maybe_snapshot_once( Ok(Ok(ctx)) } -#[allow(unused)] +#[allow(clippy::extra_unused_type_parameters)] #[cfg(not(feature = "journal"))] pub fn maybe_snapshot( ctx: FunctionEnvMut<'_, WasiEnv>, @@ -1548,6 +1548,12 @@ pub fn restore_snapshot( ) .map_err(anyhow_err_to_runtime_err)?; } + crate::journal::JournalEntry::Panic { stack_trace, .. } => { + return Err(WasiRuntimeError::Runtime(RuntimeError::new(format!( + "panic\r\nstack: {}", + stack_trace + )))); + } } } rewind.ok_or(WasiRuntimeError::Runtime(RuntimeError::user(anyhow::format_err!("The restored snapshot journal does not have a thread stack events and hence we can not restore the state of the process.").into()))) From 2f4baeb984d335bc81f90872faaac4cd30b3a08d Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Mon, 20 Nov 2023 13:23:34 +1100 Subject: [PATCH 041/129] More clippy fixes --- lib/wasi-types/src/wasi/bindings.rs | 6 +- .../journal/effector/memory_and_snapshot.rs | 4 +- .../journal/effector/syscalls/epoll_ctl.rs | 2 +- .../journal/effector/syscalls/fd_advise.rs | 2 +- .../journal/effector/syscalls/path_open.rs | 2 + lib/wasix/src/journal/log_file.rs | 76 +++++++++---------- lib/wasix/src/runners/wcgi/runner.rs | 2 +- lib/wasix/src/runtime/mod.rs | 4 +- lib/wasix/src/syscalls/mod.rs | 11 +-- 9 files changed, 57 insertions(+), 52 deletions(-) diff --git a/lib/wasi-types/src/wasi/bindings.rs b/lib/wasi-types/src/wasi/bindings.rs index 41fa23c65eb..47e4a668428 100644 --- a/lib/wasi-types/src/wasi/bindings.rs +++ b/lib/wasi-types/src/wasi/bindings.rs @@ -640,7 +640,8 @@ impl core::fmt::Debug for Dirent { } #[doc = " File or memory access pattern advisory information."] #[repr(u8)] -#[derive(Clone, Serialize, Deserialize, Copy, PartialEq, Eq)] +#[derive(Clone, Copy, PartialEq, Eq)] +#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] pub enum Advice { #[doc = " The application has no advice to give on its behavior with respect to the specified data."] Normal, @@ -1187,7 +1188,8 @@ impl core::fmt::Debug for Whence { } } #[repr(C)] -#[derive(Copy, Clone, Serialize, Deserialize)] +#[derive(Copy, Clone)] +#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] pub struct Tty { pub cols: u32, pub rows: u32, diff --git a/lib/wasix/src/journal/effector/memory_and_snapshot.rs b/lib/wasix/src/journal/effector/memory_and_snapshot.rs index c5ac86d2a72..f42abf753b5 100644 --- a/lib/wasix/src/journal/effector/memory_and_snapshot.rs +++ b/lib/wasix/src/journal/effector/memory_and_snapshot.rs @@ -81,8 +81,8 @@ impl JournalEffector { region: Range, data: &[u8], ) -> anyhow::Result<()> { - let (env, mut store) = ctx.data_and_store_mut(); - let memory = unsafe { env.memory_view(&mut store) }; + let (env, store) = ctx.data_and_store_mut(); + let memory = unsafe { env.memory_view(&store) }; memory .write(region.start, data.as_ref()) .map_err(|err| WasiRuntimeError::Runtime(RuntimeError::user(err.into())))?; diff --git a/lib/wasix/src/journal/effector/syscalls/epoll_ctl.rs b/lib/wasix/src/journal/effector/syscalls/epoll_ctl.rs index fdde696e7ee..4906711840d 100644 --- a/lib/wasix/src/journal/effector/syscalls/epoll_ctl.rs +++ b/lib/wasix/src/journal/effector/syscalls/epoll_ctl.rs @@ -12,7 +12,7 @@ impl JournalEffector { ctx, JournalEntry::EpollCtl { epfd, - op: op.into(), + op, fd, event, }, diff --git a/lib/wasix/src/journal/effector/syscalls/fd_advise.rs b/lib/wasix/src/journal/effector/syscalls/fd_advise.rs index 8a6b7a89953..96424ed1872 100644 --- a/lib/wasix/src/journal/effector/syscalls/fd_advise.rs +++ b/lib/wasix/src/journal/effector/syscalls/fd_advise.rs @@ -14,7 +14,7 @@ impl JournalEffector { fd, offset, len, - advice: advice.into(), + advice, }, ) } diff --git a/lib/wasix/src/journal/effector/syscalls/path_open.rs b/lib/wasix/src/journal/effector/syscalls/path_open.rs index d360b1762e3..a2af4bc8a13 100644 --- a/lib/wasix/src/journal/effector/syscalls/path_open.rs +++ b/lib/wasix/src/journal/effector/syscalls/path_open.rs @@ -1,6 +1,7 @@ use super::*; impl JournalEffector { + #[allow(clippy::too_many_arguments)] pub fn save_path_open( ctx: &mut FunctionEnvMut<'_, WasiEnv>, fd: Fd, @@ -27,6 +28,7 @@ impl JournalEffector { ) } + #[allow(clippy::too_many_arguments)] pub fn apply_path_open( ctx: &mut FunctionEnvMut<'_, WasiEnv>, fd: Fd, diff --git a/lib/wasix/src/journal/log_file.rs b/lib/wasix/src/journal/log_file.rs index 56056beb0b7..88c32308da3 100644 --- a/lib/wasix/src/journal/log_file.rs +++ b/lib/wasix/src/journal/log_file.rs @@ -193,9 +193,9 @@ pub(crate) enum JournalSnapshot0ClockidV1 { Unknown = 255, } -impl Into for wasi::Snapshot0Clockid { - fn into(self) -> JournalSnapshot0ClockidV1 { - match self { +impl From for JournalSnapshot0ClockidV1 { + fn from(val: wasi::Snapshot0Clockid) -> Self { + match val { Snapshot0Clockid::Realtime => JournalSnapshot0ClockidV1::Realtime, Snapshot0Clockid::Monotonic => JournalSnapshot0ClockidV1::Monotonic, Snapshot0Clockid::ProcessCputimeId => JournalSnapshot0ClockidV1::ProcessCputimeId, @@ -205,9 +205,9 @@ impl Into for wasi::Snapshot0Clockid { } } -impl Into for JournalSnapshot0ClockidV1 { - fn into(self) -> wasi::Snapshot0Clockid { - match self { +impl From for wasi::Snapshot0Clockid { + fn from(val: JournalSnapshot0ClockidV1) -> Self { + match val { JournalSnapshot0ClockidV1::Realtime => Snapshot0Clockid::Realtime, JournalSnapshot0ClockidV1::Monotonic => Snapshot0Clockid::Monotonic, JournalSnapshot0ClockidV1::ProcessCputimeId => Snapshot0Clockid::ProcessCputimeId, @@ -225,9 +225,9 @@ pub(crate) enum JournalWhenceV1 { Unknown = 255, } -impl Into for wasi::Whence { - fn into(self) -> JournalWhenceV1 { - match self { +impl From for JournalWhenceV1 { + fn from(val: wasi::Whence) -> Self { + match val { wasi::Whence::Set => JournalWhenceV1::Set, wasi::Whence::Cur => JournalWhenceV1::Cur, wasi::Whence::End => JournalWhenceV1::End, @@ -236,9 +236,9 @@ impl Into for wasi::Whence { } } -impl Into for JournalWhenceV1 { - fn into(self) -> wasi::Whence { - match self { +impl From for wasi::Whence { + fn from(val: JournalWhenceV1) -> Self { + match val { JournalWhenceV1::Set => Whence::Set, JournalWhenceV1::Cur => Whence::Cur, JournalWhenceV1::End => Whence::End, @@ -258,9 +258,9 @@ pub(crate) enum JournalAdviceV1 { Unknown = 255, } -impl Into for wasi::Advice { - fn into(self) -> JournalAdviceV1 { - match self { +impl From for JournalAdviceV1 { + fn from(val: wasi::Advice) -> Self { + match val { Advice::Normal => JournalAdviceV1::Normal, Advice::Sequential => JournalAdviceV1::Sequential, Advice::Random => JournalAdviceV1::Random, @@ -272,9 +272,9 @@ impl Into for wasi::Advice { } } -impl Into for JournalAdviceV1 { - fn into(self) -> wasi::Advice { - match self { +impl From for wasi::Advice { + fn from(val: JournalAdviceV1) -> Self { + match val { JournalAdviceV1::Normal => Advice::Normal, JournalAdviceV1::Sequential => Advice::Sequential, JournalAdviceV1::Random => Advice::Random, @@ -292,18 +292,18 @@ pub(crate) enum JournalExitCodeV1 { Other(i32), } -impl Into for wasi::ExitCode { - fn into(self) -> JournalExitCodeV1 { - match self { +impl From for JournalExitCodeV1 { + fn from(val: wasi::ExitCode) -> Self { + match val { wasi::ExitCode::Errno(errno) => JournalExitCodeV1::Errno(errno as u16), wasi::ExitCode::Other(id) => JournalExitCodeV1::Other(id), } } } -impl Into for JournalExitCodeV1 { - fn into(self) -> wasi::ExitCode { - match self { +impl From for wasi::ExitCode { + fn from(val: JournalExitCodeV1) -> Self { + match val { JournalExitCodeV1::Errno(errno) => { wasi::ExitCode::Errno(errno.try_into().unwrap_or(wasi::Errno::Unknown)) } @@ -326,9 +326,9 @@ pub(crate) enum JournalSnapshotTriggerV1 { NonDeterministicCall, } -impl Into for SnapshotTrigger { - fn into(self) -> JournalSnapshotTriggerV1 { - match self { +impl From for JournalSnapshotTriggerV1 { + fn from(val: SnapshotTrigger) -> Self { + match val { SnapshotTrigger::Idle => JournalSnapshotTriggerV1::Idle, SnapshotTrigger::FirstListen => JournalSnapshotTriggerV1::Listen, SnapshotTrigger::FirstEnviron => JournalSnapshotTriggerV1::Environ, @@ -343,9 +343,9 @@ impl Into for SnapshotTrigger { } } -impl Into for JournalSnapshotTriggerV1 { - fn into(self) -> SnapshotTrigger { - match self { +impl From for SnapshotTrigger { + fn from(val: JournalSnapshotTriggerV1) -> Self { + match val { JournalSnapshotTriggerV1::Idle => SnapshotTrigger::Idle, JournalSnapshotTriggerV1::Listen => SnapshotTrigger::FirstListen, JournalSnapshotTriggerV1::Environ => SnapshotTrigger::FirstEnviron, @@ -368,9 +368,9 @@ pub(crate) enum JournalEpollCtlV1 { Unknown, } -impl Into for wasi::EpollCtl { - fn into(self) -> JournalEpollCtlV1 { - match self { +impl From for JournalEpollCtlV1 { + fn from(val: wasi::EpollCtl) -> Self { + match val { wasi::EpollCtl::Add => JournalEpollCtlV1::Add, wasi::EpollCtl::Mod => JournalEpollCtlV1::Mod, wasi::EpollCtl::Del => JournalEpollCtlV1::Del, @@ -379,9 +379,9 @@ impl Into for wasi::EpollCtl { } } -impl Into for JournalEpollCtlV1 { - fn into(self) -> wasi::EpollCtl { - match self { +impl From for wasi::EpollCtl { + fn from(val: JournalEpollCtlV1) -> Self { + match val { JournalEpollCtlV1::Add => EpollCtl::Add, JournalEpollCtlV1::Mod => EpollCtl::Mod, JournalEpollCtlV1::Del => EpollCtl::Del, @@ -646,7 +646,7 @@ impl<'a> From for JournalEntry<'a> { store_data, is_64bit, } => Self::SetThread { - id: id, + id, call_stack: call_stack.into(), memory_stack: memory_stack.into(), store_data: store_data.into(), @@ -906,7 +906,7 @@ impl Journal for LogFileJournal { let _guard = Handle::try_current().map_err(|_| self.handle.enter()); let mut state = self.state.lock().await; - if state.at_end == false { + if !state.at_end { state.file.seek(SeekFrom::End(0)).await?; state.at_end = true; } diff --git a/lib/wasix/src/runners/wcgi/runner.rs b/lib/wasix/src/runners/wcgi/runner.rs index 00cda9fd70c..18438d50f0f 100644 --- a/lib/wasix/src/runners/wcgi/runner.rs +++ b/lib/wasix/src/runners/wcgi/runner.rs @@ -274,7 +274,7 @@ impl Config { #[cfg(feature = "journal")] pub fn with_snapshot_interval(&mut self, period: std::time::Duration) -> &mut Self { - if self.has_snapshot_trigger(crate::journal::SnapshotTrigger::PeriodicInterval) == false { + if !self.has_snapshot_trigger(crate::journal::SnapshotTrigger::PeriodicInterval) { self.add_snapshot_trigger(crate::journal::SnapshotTrigger::PeriodicInterval); } self.wasi.snapshot_interval.replace(period); diff --git a/lib/wasix/src/runtime/mod.rs b/lib/wasix/src/runtime/mod.rs index ce0ac55b2b5..446557890ac 100644 --- a/lib/wasix/src/runtime/mod.rs +++ b/lib/wasix/src/runtime/mod.rs @@ -113,7 +113,7 @@ where /// The snapshot capturer takes and restores snapshots of the WASM process at specific /// points in time by reading and writing log entries #[cfg(feature = "journal")] - fn snapshot_capturer<'a>(&'a self) -> &'_ DynJournal { + fn snapshot_capturer(&self) -> &'_ DynJournal { &UNSUPPORTED_SNAPSHOT_CAPTURER } } @@ -332,7 +332,7 @@ impl Runtime for PluggableRuntime { } #[cfg(feature = "journal")] - fn snapshot_capturer<'a>(&'a self) -> &DynJournal { + fn snapshot_capturer(&self) -> &DynJournal { self.snapshot_capturer.as_ref() } } diff --git a/lib/wasix/src/syscalls/mod.rs b/lib/wasix/src/syscalls/mod.rs index e8e8e432d5d..00c7648b214 100644 --- a/lib/wasix/src/syscalls/mod.rs +++ b/lib/wasix/src/syscalls/mod.rs @@ -1239,7 +1239,7 @@ pub fn maybe_snapshot_once( unsafe { handle_rewind_ext::(&mut ctx, HandleRewindType::Resultless) }; - if ctx.data().enable_snapshot_capture == false { + if !ctx.data().enable_snapshot_capture { return Ok(Ok(ctx)); } @@ -1248,7 +1248,7 @@ pub fn maybe_snapshot_once( let res = wasi_try_ok_ok!(WasiProcessInner::checkpoint::( inner, ctx, - WasiProcessCheckpoint::Snapshot { trigger: trigger }, + WasiProcessCheckpoint::Snapshot { trigger }, )?); match res { MaybeCheckpointResult::Unwinding => return Ok(Err(Errno::Success)), @@ -1274,7 +1274,7 @@ pub fn maybe_snapshot( ) -> WasiResult> { use crate::os::task::process::{WasiProcessCheckpoint, WasiProcessInner}; - if ctx.data().enable_snapshot_capture == false { + if !ctx.data().enable_snapshot_capture { return Ok(Ok(ctx)); } @@ -1293,6 +1293,7 @@ pub fn anyhow_err_to_runtime_err(err: anyhow::Error) -> WasiRuntimeError { WasiRuntimeError::Runtime(RuntimeError::user(err.into())) } +#[allow(clippy::result_large_err)] #[cfg(feature = "journal")] pub fn restore_snapshot( mut ctx: FunctionEnvMut<'_, WasiEnv>, @@ -1353,7 +1354,7 @@ pub fn restore_snapshot( memory_stack: memory_stack.to_vec().into(), rewind_stack: call_stack.to_vec().into(), store_data: store_data.to_vec().into(), - is_64bit: is_64bit, + is_64bit, }); } else { return Err(WasiRuntimeError::Runtime(RuntimeError::user( @@ -1543,7 +1544,7 @@ pub fn restore_snapshot( stderr_tty: tty.stderr_tty, echo: tty.echo, line_buffered: tty.line_buffered, - line_feeds: line_feeds, + line_feeds, }, ) .map_err(anyhow_err_to_runtime_err)?; From 9cd4c446562d57f0011ae197ac38605508ddc5e4 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Mon, 20 Nov 2023 14:18:52 +1100 Subject: [PATCH 042/129] More cleanup of the CLI and naming --- lib/cli/src/cli.rs | 1 + lib/cli/src/commands/run/mod.rs | 58 +++++----------- lib/cli/src/commands/run/wasi.rs | 51 ++++++-------- lib/wasi-types/src/wasi/wasix_manual.rs | 8 +-- .../journal/effector/memory_and_snapshot.rs | 6 +- lib/wasix/src/journal/effector/save_event.rs | 3 +- .../src/journal/effector/syscalls/fd_write.rs | 3 +- lib/wasix/src/journal/effector/thread_exit.rs | 3 +- .../src/journal/{capturer.rs => journal.rs} | 1 + lib/wasix/src/journal/log_file.rs | 1 + lib/wasix/src/journal/mod.rs | 4 +- lib/wasix/src/lib.rs | 12 ++-- lib/wasix/src/runners/wasi.rs | 27 ++------ lib/wasix/src/runners/wasi_common.rs | 5 +- lib/wasix/src/runners/wcgi/runner.rs | 14 +--- lib/wasix/src/runtime/mod.rs | 30 +++++--- lib/wasix/src/state/builder.rs | 69 +++++++++---------- lib/wasix/src/state/env.rs | 34 +++++---- lib/wasix/src/syscalls/mod.rs | 14 ++-- lib/wasix/src/syscalls/wasi/clock_time_set.rs | 2 +- lib/wasix/src/syscalls/wasi/fd_advise.rs | 2 +- lib/wasix/src/syscalls/wasi/fd_allocate.rs | 2 +- lib/wasix/src/syscalls/wasi/fd_close.rs | 2 +- lib/wasix/src/syscalls/wasi/fd_dup.rs | 2 +- .../src/syscalls/wasi/fd_fdstat_set_flags.rs | 2 +- .../src/syscalls/wasi/fd_fdstat_set_rights.rs | 2 +- .../src/syscalls/wasi/fd_filestat_set_size.rs | 2 +- .../syscalls/wasi/fd_filestat_set_times.rs | 2 +- lib/wasix/src/syscalls/wasi/fd_renumber.rs | 2 +- lib/wasix/src/syscalls/wasi/fd_seek.rs | 2 +- lib/wasix/src/syscalls/wasi/fd_write.rs | 4 +- .../syscalls/wasi/path_create_directory.rs | 2 +- .../syscalls/wasi/path_filestat_set_times.rs | 2 +- lib/wasix/src/syscalls/wasi/path_link.rs | 2 +- lib/wasix/src/syscalls/wasi/path_open.rs | 2 +- .../syscalls/wasi/path_remove_directory.rs | 2 +- lib/wasix/src/syscalls/wasi/path_rename.rs | 2 +- lib/wasix/src/syscalls/wasi/path_symlink.rs | 2 +- .../src/syscalls/wasi/path_unlink_file.rs | 2 +- lib/wasix/src/syscalls/wasix/chdir.rs | 2 +- lib/wasix/src/syscalls/wasix/epoll_create.rs | 2 +- lib/wasix/src/syscalls/wasix/epoll_ctl.rs | 2 +- lib/wasix/src/syscalls/wasix/fd_pipe.rs | 2 +- lib/wasix/src/syscalls/wasix/tty_set.rs | 2 +- 44 files changed, 172 insertions(+), 224 deletions(-) rename lib/wasix/src/journal/{capturer.rs => journal.rs} (99%) diff --git a/lib/cli/src/cli.rs b/lib/cli/src/cli.rs index 22d1dad04ff..7c38362a4f1 100644 --- a/lib/cli/src/cli.rs +++ b/lib/cli/src/cli.rs @@ -142,6 +142,7 @@ impl Args { } #[derive(Parser, Debug)] +#[allow(clippy::large_enum_variant)] /// The options for the wasmer Command Line Interface enum Cmd { /// Login into a wasmer.io-like registry diff --git a/lib/cli/src/commands/run/mod.rs b/lib/cli/src/commands/run/mod.rs index 16556151e22..d8031049362 100644 --- a/lib/cli/src/commands/run/mod.rs +++ b/lib/cli/src/commands/run/mod.rs @@ -248,35 +248,23 @@ impl Run { for trigger in self.wasi.snapshot_on.iter().cloned() { runner.config().add_snapshot_trigger(trigger); } - - if self.wasi.snapshot_on.is_empty() && self.wasi.journal.is_some() { + if self.wasi.snapshot_on.is_empty() && !self.wasi.journals.is_empty() { runner.config().add_default_snapshot_triggers(); } - if let Some(period) = self.wasi.snapshot_interval { + if self.wasi.journals.is_empty() { + return Err(anyhow::format_err!( + "If you specify a snapshot interval then you must also specify a journal file" + )); + } runner .config() .with_snapshot_interval(Duration::from_millis(period)); } - - match (self.wasi.journal.clone(), self.wasi.journal_restore.clone()) { - (Some(save), Some(restore)) if save == restore => { - return Err(anyhow::format_err!( - "The snapshot save path and snapshot restore path can not be the same" - )); - } - (_, _) => { - if let Some(path) = self.wasi.journal.clone() { - runner - .config() - .with_journal(Arc::new(LogFileJournal::new_std(path)?)); - } - if let Some(path) = self.wasi.journal_restore.clone() { - runner - .config() - .with_journal_restore(Arc::new(LogFileJournal::new_std(path)?)); - } - } + for journal in self.wasi.journals.clone() { + runner + .config() + .add_journal(Arc::new(LogFileJournal::new_std(journal)?)); } } @@ -347,31 +335,19 @@ impl Run { for trigger in self.wasi.snapshot_on.iter().cloned() { runner.add_snapshot_trigger(trigger); } - - // If no events are specified then add all the defaults - if self.wasi.snapshot_on.is_empty() && self.wasi.journal.is_some() { + if self.wasi.snapshot_on.is_empty() && !self.wasi.journals.is_empty() { runner.add_default_snapshot_triggers(); } - - // If a periodic interval is specified then make sure the event is also added if let Some(period) = self.wasi.snapshot_interval { - runner.with_snapshot_interval(Duration::from_millis(period)); - } - - match (self.wasi.journal.clone(), self.wasi.journal_restore.clone()) { - (Some(save), Some(restore)) if save == restore => { + if self.wasi.journals.is_empty() { return Err(anyhow::format_err!( - "The snapshot save path and snapshot restore path can not be the same" + "If you specify a snapshot interval then you must also specify a journal file" )); } - (_, _) => { - if let Some(path) = self.wasi.journal.clone() { - runner.with_journal(Arc::new(LogFileJournal::new_std(path)?)); - } - if let Some(path) = self.wasi.journal_restore.clone() { - runner.with_journal_restore(Arc::new(LogFileJournal::new_std(path)?)); - } - } + runner.with_snapshot_interval(Duration::from_millis(period)); + } + for journal in self.wasi.journals.clone() { + runner.add_journal(Arc::new(LogFileJournal::new_std(journal)?)); } } diff --git a/lib/cli/src/commands/run/wasi.rs b/lib/cli/src/commands/run/wasi.rs index b4b5d0b506f..bd442859d91 100644 --- a/lib/cli/src/commands/run/wasi.rs +++ b/lib/cli/src/commands/run/wasi.rs @@ -108,19 +108,18 @@ pub struct Wasi { #[clap(long = "enable-async-threads")] pub enable_async_threads: bool, - /// Specifies the journal file that Wasmer will use to store and restore - /// the state of the WASM process + /// Specifies one or more journal files that Wasmer will use to restore + /// and save the state of the WASM process as it executes. + /// + /// The state of the WASM process and its sandbox will be reapplied using + /// the journals in the order that you specify here. + /// + /// The last journal file specified will be created if it does not exist + /// and opened for read and write. New journal events will be written to this + /// file #[cfg(feature = "journal")] #[clap(long = "journal")] - pub journal: Option, - - /// When specified, the runtime will restore a previous snapshot using a different journal - /// then the one specified in the `--journal` argument. If no argument is specified for - /// `--journal` then the state of the process will be restored however no more events - /// will be recorded. - #[cfg(feature = "journal")] - #[clap(long = "journal-restore")] - pub journal_restore: Option, + pub journals: Vec, /// Indicates what events will cause a snapshot to be taken /// and written to the journal file. @@ -344,25 +343,15 @@ impl Wasi { *builder.capabilities_mut() = self.capabilities(); #[cfg(feature = "journal")] - for trigger in self.snapshot_on.iter().cloned() { - builder.add_snapshot_trigger(trigger); - } - - #[cfg(feature = "journal")] - match (self.journal.clone(), self.journal_restore.clone()) { - (Some(save), Some(restore)) if save == restore => { - return Err(anyhow::format_err!( - "The snapshot save path and snapshot restore path can not be the same" - )); + { + for trigger in self.snapshot_on.iter().cloned() { + builder.add_snapshot_trigger(trigger); } - (_, _) => { - if let Some(path) = self.journal.clone() { - builder = builder.with_journal(Arc::new(LogFileJournal::new_std(path)?)); - } - if let Some(path) = self.journal_restore.clone() { - builder = - builder.with_journal_restore(Arc::new(LogFileJournal::new_std(path)?)); - } + if let Some(interval) = self.snapshot_interval.clone() { + builder.with_snapshot_interval(std::time::Duration::from_millis(interval)); + } + for journal in self.journals.iter() { + builder.add_journal(Arc::new(LogFileJournal::new_std(journal)?)); } } @@ -520,8 +509,8 @@ impl Wasi { } #[cfg(feature = "journal")] - if let Some(path) = &self.journal { - rt.set_snapshot_capturer(Arc::new(LogFileJournal::new_std(path)?)); + for journal in self.journals.clone() { + rt.add_journal(Arc::new(LogFileJournal::new_std(journal)?)); } if !self.no_tty { diff --git a/lib/wasi-types/src/wasi/wasix_manual.rs b/lib/wasi-types/src/wasi/wasix_manual.rs index 13c1d2b0547..b8ee2e53d8b 100644 --- a/lib/wasi-types/src/wasi/wasix_manual.rs +++ b/lib/wasi-types/src/wasi/wasix_manual.rs @@ -376,9 +376,8 @@ wai_bindgen_rust::bitflags::bitflags! { #[doc = " Epoll operation."] #[repr(u32)] -#[derive( - Clone, Copy, Serialize, Deserialize, PartialEq, Eq, num_enum :: TryFromPrimitive, Hash, -)] +#[derive(Clone, Copy, PartialEq, Eq, num_enum :: TryFromPrimitive, Hash)] +#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] pub enum EpollCtl { #[doc = " Add an entry to the interest list of the epoll file descriptor, epfd."] Add, @@ -430,7 +429,8 @@ unsafe impl wasmer::FromToNativeWasmType for EpollCtl { } } -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] pub struct EpollEventCtl { pub events: EpollType, pub ptr: u64, diff --git a/lib/wasix/src/journal/effector/memory_and_snapshot.rs b/lib/wasix/src/journal/effector/memory_and_snapshot.rs index f42abf753b5..7bbdfd564cb 100644 --- a/lib/wasix/src/journal/effector/memory_and_snapshot.rs +++ b/lib/wasix/src/journal/effector/memory_and_snapshot.rs @@ -43,7 +43,7 @@ impl JournalEffector { // file in an orderly manner. __asyncify_light(env, None, async { let memory = unsafe { env.memory_view(ctx) }; - let capturer = ctx.data().runtime().snapshot_capturer(); + let journal = ctx.data().active_journal()?; for region in regions { // We grab this region of memory as a vector and hash @@ -54,7 +54,7 @@ impl JournalEffector { .map_err(mem_error_to_wasi)?; // Now we write it to the snap snapshot capturer - capturer + journal .write(JournalEntry::UpdateMemoryRegion { region, data: data.into(), @@ -66,7 +66,7 @@ impl JournalEffector { // Finally we mark the end of the snapshot so that // it can act as a restoration point let when = SystemTime::now(); - capturer + journal .write(JournalEntry::Snapshot { when, trigger }) .await .map_err(map_snapshot_err)?; diff --git a/lib/wasix/src/journal/effector/save_event.rs b/lib/wasix/src/journal/effector/save_event.rs index 29339cd56e9..ccc1d4d32d4 100644 --- a/lib/wasix/src/journal/effector/save_event.rs +++ b/lib/wasix/src/journal/effector/save_event.rs @@ -9,8 +9,7 @@ impl JournalEffector { __asyncify_light(env, None, async { ctx.data() - .runtime() - .snapshot_capturer() + .active_journal()? .write(event) .await .map_err(map_snapshot_err)?; diff --git a/lib/wasix/src/journal/effector/syscalls/fd_write.rs b/lib/wasix/src/journal/effector/syscalls/fd_write.rs index 86491d81464..493ff41470c 100644 --- a/lib/wasix/src/journal/effector/syscalls/fd_write.rs +++ b/lib/wasix/src/journal/effector/syscalls/fd_write.rs @@ -29,8 +29,7 @@ impl JournalEffector { .access() .map_err(mem_error_to_wasi)?; ctx.data() - .runtime() - .snapshot_capturer() + .active_journal()? .write(JournalEntry::FileDescriptorWrite { fd, offset, diff --git a/lib/wasix/src/journal/effector/thread_exit.rs b/lib/wasix/src/journal/effector/thread_exit.rs index 316bdb9306a..30c05ea972f 100644 --- a/lib/wasix/src/journal/effector/thread_exit.rs +++ b/lib/wasix/src/journal/effector/thread_exit.rs @@ -7,8 +7,7 @@ impl JournalEffector { exit_code: Option, ) -> anyhow::Result<()> { __asyncify_light(env, None, async { - env.runtime() - .snapshot_capturer() + env.active_journal()? .write(JournalEntry::CloseThread { id, exit_code }) .await .map_err(map_snapshot_err)?; diff --git a/lib/wasix/src/journal/capturer.rs b/lib/wasix/src/journal/journal.rs similarity index 99% rename from lib/wasix/src/journal/capturer.rs rename to lib/wasix/src/journal/journal.rs index d53cf31f0e9..77ca7a41ad5 100644 --- a/lib/wasix/src/journal/capturer.rs +++ b/lib/wasix/src/journal/journal.rs @@ -38,6 +38,7 @@ pub enum SocketJournalEvent { /// Represents a log entry in a snapshot log stream that represents the total /// state of a WASM process at a point in time. +#[allow(clippy::large_enum_variant)] #[derive(Debug)] pub enum JournalEntry<'a> { Init { diff --git a/lib/wasix/src/journal/log_file.rs b/lib/wasix/src/journal/log_file.rs index 88c32308da3..b9292ae3598 100644 --- a/lib/wasix/src/journal/log_file.rs +++ b/lib/wasix/src/journal/log_file.rs @@ -21,6 +21,7 @@ use super::*; /// Note: This structure is versioned which allows for /// changes to the journal entry types without having to /// worry about backward and forward compatibility +#[allow(clippy::large_enum_variant)] #[derive(Debug, Clone, Serialize, Deserialize)] pub(crate) enum LogFileJournalEntry { InitV1 { diff --git a/lib/wasix/src/journal/mod.rs b/lib/wasix/src/journal/mod.rs index 28b68b5d21a..0dee6884595 100644 --- a/lib/wasix/src/journal/mod.rs +++ b/lib/wasix/src/journal/mod.rs @@ -1,4 +1,3 @@ -mod capturer; mod compactor; #[cfg(feature = "journal")] mod effector; @@ -6,14 +5,15 @@ mod effector; #[path = "effector/unimplemented.rs"] mod effector; mod filter; +mod journal; #[cfg(feature = "journal")] mod log_file; mod unsupported; -pub use capturer::*; pub use compactor::*; pub use effector::*; pub use filter::*; +pub use journal::*; #[cfg(feature = "journal")] pub use log_file::*; pub use unsupported::*; diff --git a/lib/wasix/src/lib.rs b/lib/wasix/src/lib.rs index b56703c5183..3205e2ed5b5 100644 --- a/lib/wasix/src/lib.rs +++ b/lib/wasix/src/lib.rs @@ -58,10 +58,12 @@ mod utils; /// WAI based bindings. mod bindings; +use std::sync::Arc; + #[allow(unused_imports)] use bytes::{Bytes, BytesMut}; +use journal::DynJournal; use os::task::control_plane::ControlPlaneError; -use syscalls::state::JournalRestore; use thiserror::Error; use tracing::error; // re-exports needed for OS @@ -232,10 +234,10 @@ impl WasiRuntimeError { pub(crate) fn run_wasi_func( func: &wasmer::Function, store: &mut impl AsStoreMut, - restorer: Option, + journals: Vec>, params: &[wasmer::Value], ) -> Result, WasiRuntimeError> { - if restorer.is_some() { + if !journals.is_empty() { return Err(WasiRuntimeError::Runtime(RuntimeError::user( anyhow::format_err!( "snapshot restoration is not currently supported when running specific functions" @@ -264,9 +266,9 @@ pub(crate) fn run_wasi_func( pub(crate) fn run_wasi_func_start( func: &wasmer::Function, store: &mut impl AsStoreMut, - restore: Option, + journals: Vec>, ) -> Result<(), WasiRuntimeError> { - run_wasi_func(func, store, restore, &[])?; + run_wasi_func(func, store, journals, &[])?; Ok(()) } diff --git a/lib/wasix/src/runners/wasi.rs b/lib/wasix/src/runners/wasi.rs index 6963820cb6e..596d09d13e9 100644 --- a/lib/wasix/src/runners/wasi.rs +++ b/lib/wasix/src/runners/wasi.rs @@ -14,7 +14,7 @@ use crate::{ journal::{DynJournal, SnapshotTrigger}, runners::{wasi_common::CommonWasiOptions, MappedDirectory}, runtime::task_manager::VirtualTaskManagerExt, - JournalRestore, Runtime, WasiEnvBuilder, WasiRuntimeError, + Runtime, WasiEnvBuilder, WasiRuntimeError, }; use super::wasi_common::MappedCommand; @@ -215,15 +215,8 @@ impl WasiRunner { self } - pub fn with_journal_restore(&mut self, restorer: Arc) -> &mut Self { - self.wasi - .journal_restore - .replace(JournalRestore { restorer }); - self - } - - pub fn with_journal(&mut self, capturer: Arc) -> &mut Self { - self.wasi.journal.replace(capturer); + pub fn add_journal(&mut self, journal: Arc) -> &mut Self { + self.wasi.journals.push(journal); self } @@ -339,18 +332,12 @@ impl crate::runners::Runner for WasiRunner { .prepare_webc_env(command_name, &wasi, Some(pkg), Arc::clone(&runtime), None) .context("Unable to prepare the WASI environment")?; - #[cfg(feature = "journal")] - if let Some(capturer) = self.wasi.journal.clone() { - env = env.with_journal(capturer); - } - - #[cfg(feature = "journal")] - if let Some(restore) = self.wasi.journal_restore.clone() { - env = env.with_journal_restore(restore.restorer); - } - #[cfg(feature = "journal")] { + for journal in self.wasi.journals.clone() { + env.add_journal(journal); + } + for snapshot_trigger in self.wasi.snapshot_on.iter().cloned() { env.add_snapshot_trigger(snapshot_trigger); } diff --git a/lib/wasix/src/runners/wasi_common.rs b/lib/wasix/src/runners/wasi_common.rs index 9e8eae7734e..f037c3d7f1e 100644 --- a/lib/wasix/src/runners/wasi_common.rs +++ b/lib/wasix/src/runners/wasi_common.rs @@ -15,7 +15,6 @@ use crate::{ capabilities::Capabilities, journal::{DynJournal, SnapshotTrigger}, runners::MappedDirectory, - state::JournalRestore, WasiEnvBuilder, }; @@ -38,9 +37,7 @@ pub(crate) struct CommonWasiOptions { pub(crate) injected_packages: Vec, pub(crate) capabilities: Capabilities, #[derivative(Debug = "ignore")] - pub(crate) journal: Option>, - #[derivative(Debug = "ignore")] - pub(crate) journal_restore: Option, + pub(crate) journals: Vec>, pub(crate) snapshot_on: Vec, pub(crate) snapshot_interval: Option, pub(crate) current_dir: Option, diff --git a/lib/wasix/src/runners/wcgi/runner.rs b/lib/wasix/src/runners/wcgi/runner.rs index 18438d50f0f..2c2ba3b5278 100644 --- a/lib/wasix/src/runners/wcgi/runner.rs +++ b/lib/wasix/src/runners/wcgi/runner.rs @@ -262,16 +262,6 @@ impl Config { self.wasi.snapshot_on.iter().any(|t| *t == on) } - #[cfg(feature = "journal")] - pub fn with_journal_restore(&mut self, capturer: Arc) -> &mut Self { - use crate::state::JournalRestore; - - self.wasi - .journal_restore - .replace(JournalRestore { restorer: capturer }); - self - } - #[cfg(feature = "journal")] pub fn with_snapshot_interval(&mut self, period: std::time::Duration) -> &mut Self { if !self.has_snapshot_trigger(crate::journal::SnapshotTrigger::PeriodicInterval) { @@ -282,8 +272,8 @@ impl Config { } #[cfg(feature = "journal")] - pub fn with_journal(&mut self, capturer: Arc) -> &mut Self { - self.wasi.journal.replace(capturer); + pub fn add_journal(&mut self, journal: Arc) -> &mut Self { + self.wasi.journals.push(journal); self } } diff --git a/lib/wasix/src/runtime/mod.rs b/lib/wasix/src/runtime/mod.rs index 446557890ac..76cc56db94b 100644 --- a/lib/wasix/src/runtime/mod.rs +++ b/lib/wasix/src/runtime/mod.rs @@ -8,8 +8,6 @@ use self::{ module_cache::{CacheError, ModuleHash}, task_manager::InlineWaker, }; -#[cfg(feature = "journal")] -use crate::journal::UNSUPPORTED_SNAPSHOT_CAPTURER; use std::{ fmt, @@ -22,7 +20,7 @@ use virtual_net::{DynVirtualNetworking, VirtualNetworking}; use wasmer::Module; #[cfg(feature = "journal")] -use crate::journal::{DynJournal, UnsupportedJournal}; +use crate::journal::DynJournal; use crate::{ http::{DynHttpClient, HttpClient}, os::TtyBridge, @@ -110,14 +108,24 @@ where InlineWaker::block_on(self.load_module(wasm)) } + /// The list of journals which will be used to restore the state of the + /// runtime at a particular point in time + #[cfg(feature = "journal")] + fn journals(&self) -> &'static Vec> { + &EMPTY_JOURNAL_LIST + } + /// The snapshot capturer takes and restores snapshots of the WASM process at specific /// points in time by reading and writing log entries #[cfg(feature = "journal")] - fn snapshot_capturer(&self) -> &'_ DynJournal { - &UNSUPPORTED_SNAPSHOT_CAPTURER + fn active_journal(&self) -> Option<&'_ DynJournal> { + None } } +#[cfg(feature = "journal")] +static EMPTY_JOURNAL_LIST: Vec> = Vec::new(); + /// Load a a Webassembly module, trying to use a pre-compiled version if possible. /// // This function exists to provide a reusable baseline implementation for @@ -193,7 +201,7 @@ pub struct PluggableRuntime { pub tty: Option>, #[cfg(feature = "journal")] #[derivative(Debug = "ignore")] - pub snapshot_capturer: Arc, + pub journals: Vec>, } impl PluggableRuntime { @@ -229,7 +237,7 @@ impl PluggableRuntime { package_loader: Arc::new(loader), module_cache: Arc::new(module_cache::in_memory()), #[cfg(feature = "journal")] - snapshot_capturer: Arc::new(UnsupportedJournal::default()) as Arc, + journals: Vec::new(), } } @@ -281,8 +289,8 @@ impl PluggableRuntime { } #[cfg(feature = "journal")] - pub fn set_snapshot_capturer(&mut self, capturer: Arc) -> &mut Self { - self.snapshot_capturer = capturer; + pub fn add_journal(&mut self, journal: Arc) -> &mut Self { + self.journals.push(journal); self } } @@ -332,7 +340,7 @@ impl Runtime for PluggableRuntime { } #[cfg(feature = "journal")] - fn snapshot_capturer(&self) -> &DynJournal { - self.snapshot_capturer.as_ref() + fn active_journal(&self) -> Option<&DynJournal> { + self.journals.iter().last().map(|a| a.as_ref()) } } diff --git a/lib/wasix/src/state/builder.rs b/lib/wasix/src/state/builder.rs index 5eb49307cfb..bb303addb43 100644 --- a/lib/wasix/src/state/builder.rs +++ b/lib/wasix/src/state/builder.rs @@ -19,7 +19,6 @@ use crate::{ bin_factory::{BinFactory, BinaryPackage}, capabilities::Capabilities, fs::{WasiFs, WasiFsRoot, WasiInodes}, - journal::DynJournal, os::task::control_plane::{ControlPlaneConfig, ControlPlaneError, WasiControlPlane}, runtime::task_manager::InlineWaker, state::WasiState, @@ -27,7 +26,10 @@ use crate::{ RewindState, Runtime, WasiEnv, WasiError, WasiFunctionEnv, WasiRuntimeError, }; #[cfg(feature = "journal")] -use crate::{journal::SnapshotTrigger, syscalls::restore_snapshot}; +use crate::{ + journal::{DynJournal, SnapshotTrigger}, + syscalls::restore_snapshot, +}; use super::env::WasiEnvInit; @@ -79,16 +81,10 @@ pub struct WasiEnvBuilder { pub(super) snapshot_on: Vec, #[cfg(feature = "journal")] - pub(super) snapshot_restore: Option, + pub(super) snapshot_interval: Option, #[cfg(feature = "journal")] - pub(super) snapshot_save: Option>, -} - -#[derive(Clone)] -pub struct JournalRestore { - /// Snapshot capturer that will be used to restore state - pub restorer: Arc, + pub(super) journals: Vec>, } impl std::fmt::Debug for WasiEnvBuilder { @@ -514,21 +510,18 @@ impl WasiEnvBuilder { Ok(self) } - /// Supplies a snapshot capturer which will be used to read the - /// snapshot journal events and replay them into the WasiEnv - /// rather than starting a new instance from scratch - #[cfg(feature = "journal")] - pub fn with_journal_restore(mut self, restorer: Arc) -> Self { - self.snapshot_restore.replace(JournalRestore { restorer }); - self - } - - /// Supplies a snapshot capturer where the snapshot journal events - /// will be sent to as they are generated + /// Specifies one or more journal files that Wasmer will use to restore + /// the state of the WASM process. + /// + /// The state of the WASM process and its sandbox will be reapplied use + /// the journals in the order that you specify here. + /// + /// The last journal file specified will be created if it does not exist + /// and opened for read and write. New journal events will be written to this + /// file #[cfg(feature = "journal")] - pub fn with_journal(mut self, snapshot_capturer: Arc) -> Self { - self.snapshot_save.replace(snapshot_capturer); - self + pub fn add_journal(&mut self, journal: Arc) { + self.journals.push(journal); } pub fn set_current_dir(&mut self, dir: impl Into) { @@ -638,6 +631,11 @@ impl WasiEnvBuilder { self.snapshot_on.push(on); } + #[cfg(feature = "journal")] + pub fn with_snapshot_interval(&mut self, interval: std::time::Duration) { + self.snapshot_interval.replace(interval); + } + /// Consumes the [`WasiEnvBuilder`] and produces a [`WasiEnvInit`], which /// can be used to construct a new [`WasiEnv`]. /// @@ -809,8 +807,8 @@ impl WasiEnvBuilder { #[allow(unused_mut)] let mut runtime = PluggableRuntime::new(Arc::new(crate::runtime::task_manager::tokio::TokioTaskManager::default())); #[cfg(feature = "journal")] - if let Some(capturer) = self.snapshot_save.clone() { - runtime.set_snapshot_capturer(capturer); + for journal in self.journals.clone() { + runtime.add_journal(journal); } Arc::new(runtime) } @@ -834,9 +832,6 @@ impl WasiEnvBuilder { }; let control_plane = WasiControlPlane::new(plane_config); - #[cfg(feature = "journal")] - let snapshot_on = self.snapshot_on; - let init = WasiEnvInit { state, runtime, @@ -849,13 +844,13 @@ impl WasiEnvBuilder { process: None, thread: None, #[cfg(feature = "journal")] - call_initialize: self.snapshot_restore.is_none(), + call_initialize: self.journals.is_empty(), #[cfg(not(feature = "journal"))] call_initialize: true, can_deep_sleep: false, extra_tracing: true, #[cfg(feature = "journal")] - snapshot_on, + snapshot_on: self.snapshot_on, }; Ok(init) @@ -927,16 +922,16 @@ impl WasiEnvBuilder { } #[cfg(feature = "journal")] - let snapshot_restore = self.snapshot_restore.clone(); + let journals: Vec> = self.journals.clone(); #[cfg(not(feature = "journal"))] - let snapshot_restore = None; + let journals = Vec::new(); let (instance, env) = self.instantiate(module, store)?; let start = instance.exports.get_function("_start")?; env.data(&store).thread.set_status_running(); - let result = crate::run_wasi_func_start(start, store, snapshot_restore); + let result = crate::run_wasi_func_start(start, store, journals); let (result, exit_code) = wasi_exit_code(result); let pid = env.data(&store).pid(); @@ -976,7 +971,7 @@ impl WasiEnvBuilder { let _guard = _guard.as_ref().map(|r| r.enter()); #[cfg(feature = "journal")] - let snapshot_restore = self.snapshot_restore.clone(); + let journals = self.journals.clone(); let (_, env) = self.instantiate(module, &mut store)?; @@ -986,9 +981,9 @@ impl WasiEnvBuilder { let mut rewind_state = None; #[cfg(feature = "journal")] - if let Some(snapshot_restore) = snapshot_restore { + for journal in journals { let ctx = env.env.clone().into_mut(&mut store); - let rewind = restore_snapshot(ctx, snapshot_restore)?; + let rewind = restore_snapshot(ctx, journal)?; rewind_state = Some((rewind, None)); } diff --git a/lib/wasix/src/state/env.rs b/lib/wasix/src/state/env.rs index 4f5760c4c7d..de74f7bff0d 100644 --- a/lib/wasix/src/state/env.rs +++ b/lib/wasix/src/state/env.rs @@ -23,7 +23,7 @@ use wasmer_wasix_types::{ }; #[cfg(feature = "journal")] -use crate::journal::{JournalEffector, SnapshotTrigger}; +use crate::journal::{DynJournal, JournalEffector, SnapshotTrigger}; use crate::{ bin_factory::{BinFactory, BinaryPackage}, capabilities::Capabilities, @@ -309,7 +309,7 @@ pub struct WasiEnv { pub enable_deep_sleep: bool, /// Enables the snap shotting functionality - pub enable_snapshot_capture: bool, + pub enable_journal: bool, /// List of situations that the process will checkpoint on #[cfg(feature = "journal")] @@ -344,7 +344,7 @@ impl Clone for WasiEnv { runtime: self.runtime.clone(), capabilities: self.capabilities.clone(), enable_deep_sleep: self.enable_deep_sleep, - enable_snapshot_capture: self.enable_snapshot_capture, + enable_journal: self.enable_journal, #[cfg(feature = "journal")] snapshot_on: self.snapshot_on.clone(), } @@ -383,7 +383,7 @@ impl WasiEnv { runtime: self.runtime.clone(), capabilities: self.capabilities.clone(), enable_deep_sleep: self.enable_deep_sleep, - enable_snapshot_capture: self.enable_snapshot_capture, + enable_journal: self.enable_journal, #[cfg(feature = "journal")] snapshot_on: self.snapshot_on.clone(), }; @@ -445,15 +445,15 @@ impl WasiEnv { state: Arc::new(init.state), inner: Default::default(), owned_handles: Vec::new(), + #[cfg(feature = "journal")] + enable_journal: init.runtime.active_journal().is_some(), + #[cfg(not(feature = "journal"))] + enable_journal: false, + enable_deep_sleep: init.capabilities.threading.enable_asynchronous_threading, runtime: init.runtime, bin_factory: init.bin_factory, - enable_deep_sleep: init.capabilities.threading.enable_asynchronous_threading, capabilities: init.capabilities, #[cfg(feature = "journal")] - enable_snapshot_capture: !init.snapshot_on.is_empty(), - #[cfg(not(feature = "journal"))] - enable_snapshot_capture: false, - #[cfg(feature = "journal")] snapshot_on: init.snapshot_on.into_iter().collect(), }; env.owned_handles.push(thread); @@ -556,7 +556,7 @@ impl WasiEnv { // If this module exports an _initialize function, run that first. if call_initialize { if let Ok(initialize) = instance.exports.get_function("_initialize") { - if let Err(err) = crate::run_wasi_func_start(initialize, &mut store, None) { + if let Err(err) = crate::run_wasi_func_start(initialize, &mut store, Vec::new()) { func_env .data(&store) .blocking_cleanup(Some(Errno::Noexec.into())); @@ -855,9 +855,17 @@ impl WasiEnv { } /// Returns true if the process should perform snapshots or not + pub fn should_journal(&self) -> bool { + self.enable_journal + } + + /// Returns the active journal or fails with an error #[cfg(feature = "journal")] - pub fn should_feed_snapshot(&self) -> bool { - !self.snapshot_on.is_empty() + pub fn active_journal(&self) -> Result<&DynJournal, Errno> { + self.runtime().active_journal().ok_or({ + tracing::warn!("failed to save thread exit as there is not active journal"); + Errno::Fault + }) } /// Returns true if a particular snapshot trigger is enabled @@ -1090,7 +1098,7 @@ impl WasiEnv { // If snap-shooting is enabled then we should record an event that the thread has exited. #[cfg(feature = "journal")] - if self.should_feed_snapshot() { + if self.should_journal() { if let Err(err) = JournalEffector::save_thread_exit(self, self.tid(), exit_code) { tracing::warn!("failed to save snapshot event for thread exit - {}", err); } diff --git a/lib/wasix/src/syscalls/mod.rs b/lib/wasix/src/syscalls/mod.rs index 00c7648b214..4104f2d5fa4 100644 --- a/lib/wasix/src/syscalls/mod.rs +++ b/lib/wasix/src/syscalls/mod.rs @@ -90,10 +90,7 @@ pub(crate) use self::types::{ }, *, }; -use self::{ - state::{JournalRestore, WasiInstanceGuardMemory}, - utils::WasiDummyWaker, -}; +use self::{state::WasiInstanceGuardMemory, utils::WasiDummyWaker}; pub(crate) use crate::os::task::{ process::{WasiProcessId, WasiProcessWait}, thread::{WasiThread, WasiThreadId}, @@ -1239,7 +1236,7 @@ pub fn maybe_snapshot_once( unsafe { handle_rewind_ext::(&mut ctx, HandleRewindType::Resultless) }; - if !ctx.data().enable_snapshot_capture { + if !ctx.data().enable_journal { return Ok(Ok(ctx)); } @@ -1274,7 +1271,7 @@ pub fn maybe_snapshot( ) -> WasiResult> { use crate::os::task::process::{WasiProcessCheckpoint, WasiProcessInner}; - if !ctx.data().enable_snapshot_capture { + if !ctx.data().enable_journal { return Ok(Ok(ctx)); } @@ -1297,12 +1294,11 @@ pub fn anyhow_err_to_runtime_err(err: anyhow::Error) -> WasiRuntimeError { #[cfg(feature = "journal")] pub fn restore_snapshot( mut ctx: FunctionEnvMut<'_, WasiEnv>, - restore: JournalRestore, + journal: Arc, ) -> Result { - let restorer = restore.restorer; InlineWaker::block_on(async { let mut rewind = None; - while let Some(next) = restorer.read().await.map_err(anyhow_err_to_runtime_err)? { + while let Some(next) = journal.read().await.map_err(anyhow_err_to_runtime_err)? { tracing::trace!("Restoring snapshot event - {next:?}"); match next { crate::journal::JournalEntry::Init { .. } => { diff --git a/lib/wasix/src/syscalls/wasi/clock_time_set.rs b/lib/wasix/src/syscalls/wasi/clock_time_set.rs index 13ecbe9715b..03c2df57229 100644 --- a/lib/wasix/src/syscalls/wasi/clock_time_set.rs +++ b/lib/wasix/src/syscalls/wasi/clock_time_set.rs @@ -19,7 +19,7 @@ pub fn clock_time_set( if ret == Errno::Success { #[cfg(feature = "journal")] - if env.enable_snapshot_capture { + if env.enable_journal { JournalEffector::save_clock_time_set(&mut ctx, clock_id, time).map_err(|err| { tracing::error!("failed to save clock time set event - {}", err); WasiError::Exit(ExitCode::Errno(Errno::Fault)) diff --git a/lib/wasix/src/syscalls/wasi/fd_advise.rs b/lib/wasix/src/syscalls/wasi/fd_advise.rs index 29f0e7a5608..ed2a43684fe 100644 --- a/lib/wasix/src/syscalls/wasi/fd_advise.rs +++ b/lib/wasix/src/syscalls/wasi/fd_advise.rs @@ -24,7 +24,7 @@ pub fn fd_advise( let env = ctx.data(); #[cfg(feature = "journal")] - if env.enable_snapshot_capture { + if env.enable_journal { JournalEffector::save_fd_advise(&mut ctx, fd, offset, len, advice).map_err(|err| { tracing::error!("failed to save file descriptor advise event - {}", err); WasiError::Exit(ExitCode::Errno(Errno::Fault)) diff --git a/lib/wasix/src/syscalls/wasi/fd_allocate.rs b/lib/wasix/src/syscalls/wasi/fd_allocate.rs index 5747cb61e8e..23462cbe48d 100644 --- a/lib/wasix/src/syscalls/wasi/fd_allocate.rs +++ b/lib/wasix/src/syscalls/wasi/fd_allocate.rs @@ -21,7 +21,7 @@ pub fn fd_allocate( let env = ctx.data(); #[cfg(feature = "journal")] - if env.enable_snapshot_capture { + if env.enable_journal { JournalEffector::save_fd_allocate(&mut ctx, fd, offset, len).map_err(|err| { tracing::error!("failed to save file descriptor allocate event - {}", err); WasiError::Exit(ExitCode::Errno(Errno::Fault)) diff --git a/lib/wasix/src/syscalls/wasi/fd_close.rs b/lib/wasix/src/syscalls/wasi/fd_close.rs index 5ced38a7195..c79b0023d4d 100644 --- a/lib/wasix/src/syscalls/wasi/fd_close.rs +++ b/lib/wasix/src/syscalls/wasi/fd_close.rs @@ -33,7 +33,7 @@ pub fn fd_close(mut ctx: FunctionEnvMut<'_, WasiEnv>, fd: WasiFd) -> Result( let env = ctx.data(); #[cfg(feature = "journal")] - if env.enable_snapshot_capture { + if env.enable_journal { JournalEffector::save_fd_duplicate(&mut ctx, fd, copied_fd).map_err(|err| { tracing::error!("failed to save file descriptor renumber event - {}", err); WasiError::Exit(ExitCode::Errno(Errno::Fault)) diff --git a/lib/wasix/src/syscalls/wasi/fd_fdstat_set_flags.rs b/lib/wasix/src/syscalls/wasi/fd_fdstat_set_flags.rs index 364c00f34f0..cedd78fecc9 100644 --- a/lib/wasix/src/syscalls/wasi/fd_fdstat_set_flags.rs +++ b/lib/wasix/src/syscalls/wasi/fd_fdstat_set_flags.rs @@ -19,7 +19,7 @@ pub fn fd_fdstat_set_flags( if ret == Errno::Success { #[cfg(feature = "journal")] - if env.enable_snapshot_capture { + if env.enable_journal { JournalEffector::save_fd_set_flags(&mut ctx, fd, flags).map_err(|err| { tracing::error!("failed to save file set flags event - {}", err); WasiError::Exit(ExitCode::Errno(Errno::Fault)) diff --git a/lib/wasix/src/syscalls/wasi/fd_fdstat_set_rights.rs b/lib/wasix/src/syscalls/wasi/fd_fdstat_set_rights.rs index ff85b4bc343..ce9aa91d2dc 100644 --- a/lib/wasix/src/syscalls/wasi/fd_fdstat_set_rights.rs +++ b/lib/wasix/src/syscalls/wasi/fd_fdstat_set_rights.rs @@ -26,7 +26,7 @@ pub fn fd_fdstat_set_rights( let env = ctx.data(); #[cfg(feature = "journal")] - if env.enable_snapshot_capture { + if env.enable_journal { JournalEffector::save_fd_set_rights(&mut ctx, fd, fs_rights_base, fs_rights_inheriting) .map_err(|err| { tracing::error!("failed to save file set rights event - {}", err); diff --git a/lib/wasix/src/syscalls/wasi/fd_filestat_set_size.rs b/lib/wasix/src/syscalls/wasi/fd_filestat_set_size.rs index 0a19068bb9f..ddb361744b2 100644 --- a/lib/wasix/src/syscalls/wasi/fd_filestat_set_size.rs +++ b/lib/wasix/src/syscalls/wasi/fd_filestat_set_size.rs @@ -18,7 +18,7 @@ pub fn fd_filestat_set_size( let env = ctx.data(); #[cfg(feature = "journal")] - if env.enable_snapshot_capture { + if env.enable_journal { JournalEffector::save_fd_set_size(&mut ctx, fd, st_size).map_err(|err| { tracing::error!("failed to save file set size event - {}", err); WasiError::Exit(ExitCode::Errno(Errno::Fault)) diff --git a/lib/wasix/src/syscalls/wasi/fd_filestat_set_times.rs b/lib/wasix/src/syscalls/wasi/fd_filestat_set_times.rs index ca8efda0a0e..df7db200bc1 100644 --- a/lib/wasix/src/syscalls/wasi/fd_filestat_set_times.rs +++ b/lib/wasix/src/syscalls/wasi/fd_filestat_set_times.rs @@ -24,7 +24,7 @@ pub fn fd_filestat_set_times( let env = ctx.data(); #[cfg(feature = "journal")] - if env.enable_snapshot_capture { + if env.enable_journal { JournalEffector::save_fd_set_times(&mut ctx, fd, st_atim, st_mtim, fst_flags).map_err( |err| { tracing::error!("failed to save file set times event - {}", err); diff --git a/lib/wasix/src/syscalls/wasi/fd_renumber.rs b/lib/wasix/src/syscalls/wasi/fd_renumber.rs index d6c4e520410..55add2ebea7 100644 --- a/lib/wasix/src/syscalls/wasi/fd_renumber.rs +++ b/lib/wasix/src/syscalls/wasi/fd_renumber.rs @@ -19,7 +19,7 @@ pub fn fd_renumber( if ret == Errno::Success { #[cfg(feature = "journal")] - if env.enable_snapshot_capture { + if env.enable_journal { JournalEffector::save_fd_renumber(&mut ctx, from, to).map_err(|err| { tracing::error!("failed to save file descriptor renumber event - {}", err); WasiError::Exit(ExitCode::Errno(Errno::Fault)) diff --git a/lib/wasix/src/syscalls/wasi/fd_seek.rs b/lib/wasix/src/syscalls/wasi/fd_seek.rs index 77410d6afb4..02c6b063ad0 100644 --- a/lib/wasix/src/syscalls/wasi/fd_seek.rs +++ b/lib/wasix/src/syscalls/wasi/fd_seek.rs @@ -27,7 +27,7 @@ pub fn fd_seek( let env = ctx.data(); #[cfg(feature = "journal")] - if env.enable_snapshot_capture { + if env.enable_journal { JournalEffector::save_fd_seek(&mut ctx, fd, offset, whence).map_err(|err| { tracing::error!("failed to save file descriptor seek event - {}", err); WasiError::Exit(ExitCode::Errno(Errno::Fault)) diff --git a/lib/wasix/src/syscalls/wasi/fd_write.rs b/lib/wasix/src/syscalls/wasi/fd_write.rs index 916b0b81806..2590b0abb70 100644 --- a/lib/wasix/src/syscalls/wasi/fd_write.rs +++ b/lib/wasix/src/syscalls/wasi/fd_write.rs @@ -38,7 +38,7 @@ pub fn fd_write( let fd_entry = wasi_try_ok!(state.fs.get_fd(fd)); fd_entry.offset.load(Ordering::Acquire) as usize }; - let enable_snapshot_capture = env.enable_snapshot_capture; + let enable_snapshot_capture = env.enable_journal; fd_write_internal::( &mut ctx, @@ -74,7 +74,7 @@ pub fn fd_pwrite( offset: Filesize, nwritten: WasmPtr, ) -> Result { - let enable_snapshot_capture = ctx.data().enable_snapshot_capture; + let enable_snapshot_capture = ctx.data().enable_journal; fd_write_internal::( &mut ctx, fd, diff --git a/lib/wasix/src/syscalls/wasi/path_create_directory.rs b/lib/wasix/src/syscalls/wasi/path_create_directory.rs index 0c25424c25b..23f10c0ab95 100644 --- a/lib/wasix/src/syscalls/wasi/path_create_directory.rs +++ b/lib/wasix/src/syscalls/wasi/path_create_directory.rs @@ -39,7 +39,7 @@ pub fn path_create_directory( let env = ctx.data(); #[cfg(feature = "journal")] - if env.enable_snapshot_capture { + if env.enable_journal { JournalEffector::save_path_create_directory(&mut ctx, fd, path_string).map_err(|err| { tracing::error!( "failed to save create directory event to snapshot capturer - {}", diff --git a/lib/wasix/src/syscalls/wasi/path_filestat_set_times.rs b/lib/wasix/src/syscalls/wasi/path_filestat_set_times.rs index e048906ea50..c471374e20e 100644 --- a/lib/wasix/src/syscalls/wasi/path_filestat_set_times.rs +++ b/lib/wasix/src/syscalls/wasi/path_filestat_set_times.rs @@ -55,7 +55,7 @@ pub fn path_filestat_set_times( let env = ctx.data(); #[cfg(feature = "journal")] - if env.enable_snapshot_capture { + if env.enable_journal { JournalEffector::save_path_set_times( &mut ctx, fd, diff --git a/lib/wasix/src/syscalls/wasi/path_link.rs b/lib/wasix/src/syscalls/wasi/path_link.rs index 1ad9cff53d6..9b0388d6cd2 100644 --- a/lib/wasix/src/syscalls/wasi/path_link.rs +++ b/lib/wasix/src/syscalls/wasi/path_link.rs @@ -50,7 +50,7 @@ pub fn path_link( let env = ctx.data(); #[cfg(feature = "journal")] - if env.enable_snapshot_capture { + if env.enable_journal { JournalEffector::save_path_link( &mut ctx, old_fd, diff --git a/lib/wasix/src/syscalls/wasi/path_open.rs b/lib/wasix/src/syscalls/wasi/path_open.rs index db9f2320ea0..803a4a0be24 100644 --- a/lib/wasix/src/syscalls/wasi/path_open.rs +++ b/lib/wasix/src/syscalls/wasi/path_open.rs @@ -80,7 +80,7 @@ pub fn path_open( let env = ctx.data(); #[cfg(feature = "journal")] - if env.enable_snapshot_capture { + if env.enable_journal { JournalEffector::save_path_open( &mut ctx, out_fd, diff --git a/lib/wasix/src/syscalls/wasi/path_remove_directory.rs b/lib/wasix/src/syscalls/wasi/path_remove_directory.rs index c547075eaaf..24d9dc3b7d7 100644 --- a/lib/wasix/src/syscalls/wasi/path_remove_directory.rs +++ b/lib/wasix/src/syscalls/wasi/path_remove_directory.rs @@ -29,7 +29,7 @@ pub fn path_remove_directory( let env = ctx.data(); #[cfg(feature = "journal")] - if env.enable_snapshot_capture { + if env.enable_journal { wasi_try!( JournalEffector::save_path_remove_directory(&mut ctx, fd, path_str).map_err(|err| { tracing::error!("failed to save unlink event to snapshot capturer - {}", err); diff --git a/lib/wasix/src/syscalls/wasi/path_rename.rs b/lib/wasix/src/syscalls/wasi/path_rename.rs index acf7d81f575..ca5084dc4f2 100644 --- a/lib/wasix/src/syscalls/wasi/path_rename.rs +++ b/lib/wasix/src/syscalls/wasi/path_rename.rs @@ -40,7 +40,7 @@ pub fn path_rename( if ret == Errno::Success { #[cfg(feature = "journal")] - if env.enable_snapshot_capture { + if env.enable_journal { JournalEffector::save_path_rename(&mut ctx, old_fd, source_str, new_fd, target_str) .map_err(|err| { tracing::error!("failed to save unlink event to snapshot capturer - {}", err); diff --git a/lib/wasix/src/syscalls/wasi/path_symlink.rs b/lib/wasix/src/syscalls/wasi/path_symlink.rs index a2cf603914a..66a45988777 100644 --- a/lib/wasix/src/syscalls/wasi/path_symlink.rs +++ b/lib/wasix/src/syscalls/wasi/path_symlink.rs @@ -41,7 +41,7 @@ pub fn path_symlink( let env = ctx.data(); #[cfg(feature = "journal")] - if env.enable_snapshot_capture { + if env.enable_journal { JournalEffector::save_path_symlink(&mut ctx, old_path_str, fd, new_path_str).map_err( |err| { tracing::error!("failed to save path symbolic link event - {}", err); diff --git a/lib/wasix/src/syscalls/wasi/path_unlink_file.rs b/lib/wasix/src/syscalls/wasi/path_unlink_file.rs index e888e8e74a5..0159bf6eeac 100644 --- a/lib/wasix/src/syscalls/wasi/path_unlink_file.rs +++ b/lib/wasix/src/syscalls/wasi/path_unlink_file.rs @@ -37,7 +37,7 @@ pub fn path_unlink_file( if ret == Errno::Success { #[cfg(feature = "journal")] - if env.enable_snapshot_capture { + if env.enable_journal { wasi_try_ok!( JournalEffector::save_path_unlink(&mut ctx, fd, path_str).map_err(|err| { tracing::error!("failed to save unlink event to snapshot capturer - {}", err); diff --git a/lib/wasix/src/syscalls/wasix/chdir.rs b/lib/wasix/src/syscalls/wasix/chdir.rs index bb8a40583a0..beb4b93a75d 100644 --- a/lib/wasix/src/syscalls/wasix/chdir.rs +++ b/lib/wasix/src/syscalls/wasix/chdir.rs @@ -18,7 +18,7 @@ pub fn chdir( let env = ctx.data(); #[cfg(feature = "journal")] - if env.enable_snapshot_capture { + if env.enable_journal { JournalEffector::save_chdir(&mut ctx, path).map_err(|err| { tracing::error!("failed to chdir event - {}", err); WasiError::Exit(ExitCode::Errno(Errno::Fault)) diff --git a/lib/wasix/src/syscalls/wasix/epoll_create.rs b/lib/wasix/src/syscalls/wasix/epoll_create.rs index ac1a3a4a4ca..cf9d942176f 100644 --- a/lib/wasix/src/syscalls/wasix/epoll_create.rs +++ b/lib/wasix/src/syscalls/wasix/epoll_create.rs @@ -22,7 +22,7 @@ pub fn epoll_create( let env = ctx.data(); #[cfg(feature = "journal")] - if env.enable_snapshot_capture { + if env.enable_journal { JournalEffector::save_epoll_create(&mut ctx, fd).map_err(|err| { tracing::error!("failed to save epoll_create event - {}", err); WasiError::Exit(ExitCode::Errno(Errno::Fault)) diff --git a/lib/wasix/src/syscalls/wasix/epoll_ctl.rs b/lib/wasix/src/syscalls/wasix/epoll_ctl.rs index 0f42d324c20..06ff54159af 100644 --- a/lib/wasix/src/syscalls/wasix/epoll_ctl.rs +++ b/lib/wasix/src/syscalls/wasix/epoll_ctl.rs @@ -61,7 +61,7 @@ pub fn epoll_ctl( let env = ctx.data(); #[cfg(feature = "journal")] - if env.enable_snapshot_capture { + if env.enable_journal { JournalEffector::save_epoll_ctl(&mut ctx, epfd, op, fd, event_ctl).map_err(|err| { tracing::error!("failed to save epoll_create event - {}", err); WasiError::Exit(ExitCode::Errno(Errno::Fault)) diff --git a/lib/wasix/src/syscalls/wasix/fd_pipe.rs b/lib/wasix/src/syscalls/wasix/fd_pipe.rs index e79eb777d43..7752053d066 100644 --- a/lib/wasix/src/syscalls/wasix/fd_pipe.rs +++ b/lib/wasix/src/syscalls/wasix/fd_pipe.rs @@ -20,7 +20,7 @@ pub fn fd_pipe( let env = ctx.data(); #[cfg(feature = "journal")] - if env.enable_snapshot_capture { + if env.enable_journal { JournalEffector::save_fd_pipe(&mut ctx, fd1, fd2).map_err(|err| { tracing::error!("failed to save create pipe event - {}", err); WasiError::Exit(ExitCode::Errno(Errno::Fault)) diff --git a/lib/wasix/src/syscalls/wasix/tty_set.rs b/lib/wasix/src/syscalls/wasix/tty_set.rs index d062b77f9e0..24866d9f5b6 100644 --- a/lib/wasix/src/syscalls/wasix/tty_set.rs +++ b/lib/wasix/src/syscalls/wasix/tty_set.rs @@ -41,7 +41,7 @@ pub fn tty_set( let env = ctx.data(); #[cfg(feature = "journal")] - if env.enable_snapshot_capture { + if env.enable_journal { JournalEffector::save_tty_set(&mut ctx, state).map_err(|err| { tracing::error!("failed to save path symbolic link event - {}", err); WasiError::Exit(ExitCode::Errno(Errno::Fault)) From 305fc4c4cb556c2d8dcdc64fb4618e051b201500 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Mon, 20 Nov 2023 14:24:34 +1100 Subject: [PATCH 043/129] Renamed the journaling submodule --- lib/cli/src/commands/run/mod.rs | 2 +- lib/cli/src/commands/run/wasi.rs | 2 +- .../src/{journal => journaling}/compactor.rs | 0 .../effector/memory_and_snapshot.rs | 2 +- .../{journal => journaling}/effector/mod.rs | 0 .../effector/save_event.rs | 0 .../effector/syscalls/chdir.rs | 0 .../effector/syscalls/clock_time.rs | 0 .../effector/syscalls/epoll_create.rs | 0 .../effector/syscalls/epoll_ctl.rs | 0 .../effector/syscalls/fd_advise.rs | 0 .../effector/syscalls/fd_allocate.rs | 0 .../effector/syscalls/fd_close.rs | 0 .../effector/syscalls/fd_duplicate.rs | 0 .../effector/syscalls/fd_pipe.rs | 0 .../effector/syscalls/fd_renumber.rs | 0 .../effector/syscalls/fd_seek.rs | 0 .../effector/syscalls/fd_set_flags.rs | 0 .../effector/syscalls/fd_set_rights.rs | 0 .../effector/syscalls/fd_set_size.rs | 0 .../effector/syscalls/fd_set_times.rs | 0 .../effector/syscalls/fd_write.rs | 0 .../syscalls/path_create_directory.rs | 0 .../effector/syscalls/path_link.rs | 0 .../effector/syscalls/path_open.rs | 0 .../syscalls/path_remove_directory.rs | 0 .../effector/syscalls/path_rename.rs | 0 .../effector/syscalls/path_set_times.rs | 0 .../effector/syscalls/path_symlink.rs | 0 .../effector/syscalls/path_unlink.rs | 0 .../effector/syscalls/tty_set.rs | 0 .../effector/thread_exit.rs | 0 .../effector/thread_state.rs | 0 .../effector/unimplemented.rs | 0 .../src/{journal => journaling}/filter.rs | 0 .../src/{journal => journaling}/journal.rs | 0 .../src/{journal => journaling}/log_file.rs | 0 lib/wasix/src/{journal => journaling}/mod.rs | 0 .../{journal => journaling}/unsupported.rs | 0 lib/wasix/src/lib.rs | 4 +- lib/wasix/src/os/task/process.rs | 4 +- lib/wasix/src/runners/wasi.rs | 4 +- lib/wasix/src/runners/wasi_common.rs | 2 +- lib/wasix/src/runners/wcgi/runner.rs | 12 ++-- lib/wasix/src/runtime/mod.rs | 2 +- lib/wasix/src/state/builder.rs | 2 +- lib/wasix/src/state/env.rs | 2 +- lib/wasix/src/syscalls/mod.rs | 68 +++++++++---------- lib/wasix/src/syscalls/wasi/environ_get.rs | 2 +- .../src/syscalls/wasi/environ_sizes_get.rs | 2 +- lib/wasix/src/syscalls/wasi/fd_read.rs | 2 +- lib/wasix/src/syscalls/wasi/fd_write.rs | 2 +- lib/wasix/src/syscalls/wasix/sock_listen.rs | 2 +- lib/wasix/src/syscalls/wasix/thread_spawn.rs | 2 +- 54 files changed, 59 insertions(+), 59 deletions(-) rename lib/wasix/src/{journal => journaling}/compactor.rs (100%) rename lib/wasix/src/{journal => journaling}/effector/memory_and_snapshot.rs (97%) rename lib/wasix/src/{journal => journaling}/effector/mod.rs (100%) rename lib/wasix/src/{journal => journaling}/effector/save_event.rs (100%) rename lib/wasix/src/{journal => journaling}/effector/syscalls/chdir.rs (100%) rename lib/wasix/src/{journal => journaling}/effector/syscalls/clock_time.rs (100%) rename lib/wasix/src/{journal => journaling}/effector/syscalls/epoll_create.rs (100%) rename lib/wasix/src/{journal => journaling}/effector/syscalls/epoll_ctl.rs (100%) rename lib/wasix/src/{journal => journaling}/effector/syscalls/fd_advise.rs (100%) rename lib/wasix/src/{journal => journaling}/effector/syscalls/fd_allocate.rs (100%) rename lib/wasix/src/{journal => journaling}/effector/syscalls/fd_close.rs (100%) rename lib/wasix/src/{journal => journaling}/effector/syscalls/fd_duplicate.rs (100%) rename lib/wasix/src/{journal => journaling}/effector/syscalls/fd_pipe.rs (100%) rename lib/wasix/src/{journal => journaling}/effector/syscalls/fd_renumber.rs (100%) rename lib/wasix/src/{journal => journaling}/effector/syscalls/fd_seek.rs (100%) rename lib/wasix/src/{journal => journaling}/effector/syscalls/fd_set_flags.rs (100%) rename lib/wasix/src/{journal => journaling}/effector/syscalls/fd_set_rights.rs (100%) rename lib/wasix/src/{journal => journaling}/effector/syscalls/fd_set_size.rs (100%) rename lib/wasix/src/{journal => journaling}/effector/syscalls/fd_set_times.rs (100%) rename lib/wasix/src/{journal => journaling}/effector/syscalls/fd_write.rs (100%) rename lib/wasix/src/{journal => journaling}/effector/syscalls/path_create_directory.rs (100%) rename lib/wasix/src/{journal => journaling}/effector/syscalls/path_link.rs (100%) rename lib/wasix/src/{journal => journaling}/effector/syscalls/path_open.rs (100%) rename lib/wasix/src/{journal => journaling}/effector/syscalls/path_remove_directory.rs (100%) rename lib/wasix/src/{journal => journaling}/effector/syscalls/path_rename.rs (100%) rename lib/wasix/src/{journal => journaling}/effector/syscalls/path_set_times.rs (100%) rename lib/wasix/src/{journal => journaling}/effector/syscalls/path_symlink.rs (100%) rename lib/wasix/src/{journal => journaling}/effector/syscalls/path_unlink.rs (100%) rename lib/wasix/src/{journal => journaling}/effector/syscalls/tty_set.rs (100%) rename lib/wasix/src/{journal => journaling}/effector/thread_exit.rs (100%) rename lib/wasix/src/{journal => journaling}/effector/thread_state.rs (100%) rename lib/wasix/src/{journal => journaling}/effector/unimplemented.rs (100%) rename lib/wasix/src/{journal => journaling}/filter.rs (100%) rename lib/wasix/src/{journal => journaling}/journal.rs (100%) rename lib/wasix/src/{journal => journaling}/log_file.rs (100%) rename lib/wasix/src/{journal => journaling}/mod.rs (100%) rename lib/wasix/src/{journal => journaling}/unsupported.rs (100%) diff --git a/lib/cli/src/commands/run/mod.rs b/lib/cli/src/commands/run/mod.rs index d8031049362..aebe12e6428 100644 --- a/lib/cli/src/commands/run/mod.rs +++ b/lib/cli/src/commands/run/mod.rs @@ -29,7 +29,7 @@ use wasmer::{ use wasmer_compiler::ArtifactBuild; use wasmer_registry::{wasmer_env::WasmerEnv, Package}; #[cfg(feature = "journal")] -use wasmer_wasix::journal::{LogFileJournal, SnapshotTrigger}; +use wasmer_wasix::journaling::{LogFileJournal, SnapshotTrigger}; use wasmer_wasix::{ bin_factory::BinaryPackage, runners::{MappedCommand, MappedDirectory, Runner}, diff --git a/lib/cli/src/commands/run/wasi.rs b/lib/cli/src/commands/run/wasi.rs index bd442859d91..dd79828acde 100644 --- a/lib/cli/src/commands/run/wasi.rs +++ b/lib/cli/src/commands/run/wasi.rs @@ -15,7 +15,7 @@ use virtual_fs::{DeviceFile, FileSystem, PassthruFileSystem, RootFileSystemBuild use wasmer::{Engine, Function, Instance, Memory32, Memory64, Module, RuntimeError, Store, Value}; use wasmer_registry::wasmer_env::WasmerEnv; #[cfg(feature = "journal")] -use wasmer_wasix::journal::{LogFileJournal, SnapshotTrigger}; +use wasmer_wasix::journaling::{LogFileJournal, SnapshotTrigger}; use wasmer_wasix::{ bin_factory::BinaryPackage, capabilities::Capabilities, diff --git a/lib/wasix/src/journal/compactor.rs b/lib/wasix/src/journaling/compactor.rs similarity index 100% rename from lib/wasix/src/journal/compactor.rs rename to lib/wasix/src/journaling/compactor.rs diff --git a/lib/wasix/src/journal/effector/memory_and_snapshot.rs b/lib/wasix/src/journaling/effector/memory_and_snapshot.rs similarity index 97% rename from lib/wasix/src/journal/effector/memory_and_snapshot.rs rename to lib/wasix/src/journaling/effector/memory_and_snapshot.rs index 7bbdfd564cb..46c8898274a 100644 --- a/lib/wasix/src/journal/effector/memory_and_snapshot.rs +++ b/lib/wasix/src/journaling/effector/memory_and_snapshot.rs @@ -38,7 +38,7 @@ impl JournalEffector { cur = end; } - // Now that we known all the regions that need to be saved we + // Now that we know all the regions that need to be saved we // enter a processing loop that dumps all the data to the log // file in an orderly manner. __asyncify_light(env, None, async { diff --git a/lib/wasix/src/journal/effector/mod.rs b/lib/wasix/src/journaling/effector/mod.rs similarity index 100% rename from lib/wasix/src/journal/effector/mod.rs rename to lib/wasix/src/journaling/effector/mod.rs diff --git a/lib/wasix/src/journal/effector/save_event.rs b/lib/wasix/src/journaling/effector/save_event.rs similarity index 100% rename from lib/wasix/src/journal/effector/save_event.rs rename to lib/wasix/src/journaling/effector/save_event.rs diff --git a/lib/wasix/src/journal/effector/syscalls/chdir.rs b/lib/wasix/src/journaling/effector/syscalls/chdir.rs similarity index 100% rename from lib/wasix/src/journal/effector/syscalls/chdir.rs rename to lib/wasix/src/journaling/effector/syscalls/chdir.rs diff --git a/lib/wasix/src/journal/effector/syscalls/clock_time.rs b/lib/wasix/src/journaling/effector/syscalls/clock_time.rs similarity index 100% rename from lib/wasix/src/journal/effector/syscalls/clock_time.rs rename to lib/wasix/src/journaling/effector/syscalls/clock_time.rs diff --git a/lib/wasix/src/journal/effector/syscalls/epoll_create.rs b/lib/wasix/src/journaling/effector/syscalls/epoll_create.rs similarity index 100% rename from lib/wasix/src/journal/effector/syscalls/epoll_create.rs rename to lib/wasix/src/journaling/effector/syscalls/epoll_create.rs diff --git a/lib/wasix/src/journal/effector/syscalls/epoll_ctl.rs b/lib/wasix/src/journaling/effector/syscalls/epoll_ctl.rs similarity index 100% rename from lib/wasix/src/journal/effector/syscalls/epoll_ctl.rs rename to lib/wasix/src/journaling/effector/syscalls/epoll_ctl.rs diff --git a/lib/wasix/src/journal/effector/syscalls/fd_advise.rs b/lib/wasix/src/journaling/effector/syscalls/fd_advise.rs similarity index 100% rename from lib/wasix/src/journal/effector/syscalls/fd_advise.rs rename to lib/wasix/src/journaling/effector/syscalls/fd_advise.rs diff --git a/lib/wasix/src/journal/effector/syscalls/fd_allocate.rs b/lib/wasix/src/journaling/effector/syscalls/fd_allocate.rs similarity index 100% rename from lib/wasix/src/journal/effector/syscalls/fd_allocate.rs rename to lib/wasix/src/journaling/effector/syscalls/fd_allocate.rs diff --git a/lib/wasix/src/journal/effector/syscalls/fd_close.rs b/lib/wasix/src/journaling/effector/syscalls/fd_close.rs similarity index 100% rename from lib/wasix/src/journal/effector/syscalls/fd_close.rs rename to lib/wasix/src/journaling/effector/syscalls/fd_close.rs diff --git a/lib/wasix/src/journal/effector/syscalls/fd_duplicate.rs b/lib/wasix/src/journaling/effector/syscalls/fd_duplicate.rs similarity index 100% rename from lib/wasix/src/journal/effector/syscalls/fd_duplicate.rs rename to lib/wasix/src/journaling/effector/syscalls/fd_duplicate.rs diff --git a/lib/wasix/src/journal/effector/syscalls/fd_pipe.rs b/lib/wasix/src/journaling/effector/syscalls/fd_pipe.rs similarity index 100% rename from lib/wasix/src/journal/effector/syscalls/fd_pipe.rs rename to lib/wasix/src/journaling/effector/syscalls/fd_pipe.rs diff --git a/lib/wasix/src/journal/effector/syscalls/fd_renumber.rs b/lib/wasix/src/journaling/effector/syscalls/fd_renumber.rs similarity index 100% rename from lib/wasix/src/journal/effector/syscalls/fd_renumber.rs rename to lib/wasix/src/journaling/effector/syscalls/fd_renumber.rs diff --git a/lib/wasix/src/journal/effector/syscalls/fd_seek.rs b/lib/wasix/src/journaling/effector/syscalls/fd_seek.rs similarity index 100% rename from lib/wasix/src/journal/effector/syscalls/fd_seek.rs rename to lib/wasix/src/journaling/effector/syscalls/fd_seek.rs diff --git a/lib/wasix/src/journal/effector/syscalls/fd_set_flags.rs b/lib/wasix/src/journaling/effector/syscalls/fd_set_flags.rs similarity index 100% rename from lib/wasix/src/journal/effector/syscalls/fd_set_flags.rs rename to lib/wasix/src/journaling/effector/syscalls/fd_set_flags.rs diff --git a/lib/wasix/src/journal/effector/syscalls/fd_set_rights.rs b/lib/wasix/src/journaling/effector/syscalls/fd_set_rights.rs similarity index 100% rename from lib/wasix/src/journal/effector/syscalls/fd_set_rights.rs rename to lib/wasix/src/journaling/effector/syscalls/fd_set_rights.rs diff --git a/lib/wasix/src/journal/effector/syscalls/fd_set_size.rs b/lib/wasix/src/journaling/effector/syscalls/fd_set_size.rs similarity index 100% rename from lib/wasix/src/journal/effector/syscalls/fd_set_size.rs rename to lib/wasix/src/journaling/effector/syscalls/fd_set_size.rs diff --git a/lib/wasix/src/journal/effector/syscalls/fd_set_times.rs b/lib/wasix/src/journaling/effector/syscalls/fd_set_times.rs similarity index 100% rename from lib/wasix/src/journal/effector/syscalls/fd_set_times.rs rename to lib/wasix/src/journaling/effector/syscalls/fd_set_times.rs diff --git a/lib/wasix/src/journal/effector/syscalls/fd_write.rs b/lib/wasix/src/journaling/effector/syscalls/fd_write.rs similarity index 100% rename from lib/wasix/src/journal/effector/syscalls/fd_write.rs rename to lib/wasix/src/journaling/effector/syscalls/fd_write.rs diff --git a/lib/wasix/src/journal/effector/syscalls/path_create_directory.rs b/lib/wasix/src/journaling/effector/syscalls/path_create_directory.rs similarity index 100% rename from lib/wasix/src/journal/effector/syscalls/path_create_directory.rs rename to lib/wasix/src/journaling/effector/syscalls/path_create_directory.rs diff --git a/lib/wasix/src/journal/effector/syscalls/path_link.rs b/lib/wasix/src/journaling/effector/syscalls/path_link.rs similarity index 100% rename from lib/wasix/src/journal/effector/syscalls/path_link.rs rename to lib/wasix/src/journaling/effector/syscalls/path_link.rs diff --git a/lib/wasix/src/journal/effector/syscalls/path_open.rs b/lib/wasix/src/journaling/effector/syscalls/path_open.rs similarity index 100% rename from lib/wasix/src/journal/effector/syscalls/path_open.rs rename to lib/wasix/src/journaling/effector/syscalls/path_open.rs diff --git a/lib/wasix/src/journal/effector/syscalls/path_remove_directory.rs b/lib/wasix/src/journaling/effector/syscalls/path_remove_directory.rs similarity index 100% rename from lib/wasix/src/journal/effector/syscalls/path_remove_directory.rs rename to lib/wasix/src/journaling/effector/syscalls/path_remove_directory.rs diff --git a/lib/wasix/src/journal/effector/syscalls/path_rename.rs b/lib/wasix/src/journaling/effector/syscalls/path_rename.rs similarity index 100% rename from lib/wasix/src/journal/effector/syscalls/path_rename.rs rename to lib/wasix/src/journaling/effector/syscalls/path_rename.rs diff --git a/lib/wasix/src/journal/effector/syscalls/path_set_times.rs b/lib/wasix/src/journaling/effector/syscalls/path_set_times.rs similarity index 100% rename from lib/wasix/src/journal/effector/syscalls/path_set_times.rs rename to lib/wasix/src/journaling/effector/syscalls/path_set_times.rs diff --git a/lib/wasix/src/journal/effector/syscalls/path_symlink.rs b/lib/wasix/src/journaling/effector/syscalls/path_symlink.rs similarity index 100% rename from lib/wasix/src/journal/effector/syscalls/path_symlink.rs rename to lib/wasix/src/journaling/effector/syscalls/path_symlink.rs diff --git a/lib/wasix/src/journal/effector/syscalls/path_unlink.rs b/lib/wasix/src/journaling/effector/syscalls/path_unlink.rs similarity index 100% rename from lib/wasix/src/journal/effector/syscalls/path_unlink.rs rename to lib/wasix/src/journaling/effector/syscalls/path_unlink.rs diff --git a/lib/wasix/src/journal/effector/syscalls/tty_set.rs b/lib/wasix/src/journaling/effector/syscalls/tty_set.rs similarity index 100% rename from lib/wasix/src/journal/effector/syscalls/tty_set.rs rename to lib/wasix/src/journaling/effector/syscalls/tty_set.rs diff --git a/lib/wasix/src/journal/effector/thread_exit.rs b/lib/wasix/src/journaling/effector/thread_exit.rs similarity index 100% rename from lib/wasix/src/journal/effector/thread_exit.rs rename to lib/wasix/src/journaling/effector/thread_exit.rs diff --git a/lib/wasix/src/journal/effector/thread_state.rs b/lib/wasix/src/journaling/effector/thread_state.rs similarity index 100% rename from lib/wasix/src/journal/effector/thread_state.rs rename to lib/wasix/src/journaling/effector/thread_state.rs diff --git a/lib/wasix/src/journal/effector/unimplemented.rs b/lib/wasix/src/journaling/effector/unimplemented.rs similarity index 100% rename from lib/wasix/src/journal/effector/unimplemented.rs rename to lib/wasix/src/journaling/effector/unimplemented.rs diff --git a/lib/wasix/src/journal/filter.rs b/lib/wasix/src/journaling/filter.rs similarity index 100% rename from lib/wasix/src/journal/filter.rs rename to lib/wasix/src/journaling/filter.rs diff --git a/lib/wasix/src/journal/journal.rs b/lib/wasix/src/journaling/journal.rs similarity index 100% rename from lib/wasix/src/journal/journal.rs rename to lib/wasix/src/journaling/journal.rs diff --git a/lib/wasix/src/journal/log_file.rs b/lib/wasix/src/journaling/log_file.rs similarity index 100% rename from lib/wasix/src/journal/log_file.rs rename to lib/wasix/src/journaling/log_file.rs diff --git a/lib/wasix/src/journal/mod.rs b/lib/wasix/src/journaling/mod.rs similarity index 100% rename from lib/wasix/src/journal/mod.rs rename to lib/wasix/src/journaling/mod.rs diff --git a/lib/wasix/src/journal/unsupported.rs b/lib/wasix/src/journaling/unsupported.rs similarity index 100% rename from lib/wasix/src/journal/unsupported.rs rename to lib/wasix/src/journaling/unsupported.rs diff --git a/lib/wasix/src/lib.rs b/lib/wasix/src/lib.rs index 3205e2ed5b5..031c86afa0b 100644 --- a/lib/wasix/src/lib.rs +++ b/lib/wasix/src/lib.rs @@ -47,7 +47,7 @@ pub mod net; pub mod capabilities; pub mod fs; pub mod http; -pub mod journal; +pub mod journaling; mod rewind; pub mod runners; pub mod runtime; @@ -62,7 +62,7 @@ use std::sync::Arc; #[allow(unused_imports)] use bytes::{Bytes, BytesMut}; -use journal::DynJournal; +use journaling::DynJournal; use os::task::control_plane::ControlPlaneError; use thiserror::Error; use tracing::error; diff --git a/lib/wasix/src/os/task/process.rs b/lib/wasix/src/os/task/process.rs index 33e9927a915..a6950542b88 100644 --- a/lib/wasix/src/os/task/process.rs +++ b/lib/wasix/src/os/task/process.rs @@ -1,6 +1,6 @@ #[cfg(feature = "journal")] -use crate::{journal::JournalEffector, unwind, WasiResult}; -use crate::{journal::SnapshotTrigger, WasiEnv, WasiRuntimeError}; +use crate::{journaling::JournalEffector, unwind, WasiResult}; +use crate::{journaling::SnapshotTrigger, WasiEnv, WasiRuntimeError}; use serde::{Deserialize, Serialize}; use std::{ collections::HashMap, diff --git a/lib/wasix/src/runners/wasi.rs b/lib/wasix/src/runners/wasi.rs index 596d09d13e9..ef4eb554926 100644 --- a/lib/wasix/src/runners/wasi.rs +++ b/lib/wasix/src/runners/wasi.rs @@ -11,7 +11,7 @@ use webc::metadata::{annotations::Wasi, Command}; use crate::{ bin_factory::BinaryPackage, capabilities::Capabilities, - journal::{DynJournal, SnapshotTrigger}, + journaling::{DynJournal, SnapshotTrigger}, runners::{wasi_common::CommonWasiOptions, MappedDirectory}, runtime::task_manager::VirtualTaskManagerExt, Runtime, WasiEnvBuilder, WasiRuntimeError, @@ -195,7 +195,7 @@ impl WasiRunner { } pub fn add_default_snapshot_triggers(&mut self) -> &mut Self { - for on in crate::journal::DEFAULT_SNAPSHOT_TRIGGERS { + for on in crate::journaling::DEFAULT_SNAPSHOT_TRIGGERS { if !self.has_snapshot_trigger(on) { self.add_snapshot_trigger(on); } diff --git a/lib/wasix/src/runners/wasi_common.rs b/lib/wasix/src/runners/wasi_common.rs index f037c3d7f1e..6df106be212 100644 --- a/lib/wasix/src/runners/wasi_common.rs +++ b/lib/wasix/src/runners/wasi_common.rs @@ -13,7 +13,7 @@ use webc::metadata::annotations::Wasi as WasiAnnotation; use crate::{ bin_factory::BinaryPackage, capabilities::Capabilities, - journal::{DynJournal, SnapshotTrigger}, + journaling::{DynJournal, SnapshotTrigger}, runners::MappedDirectory, WasiEnvBuilder, }; diff --git a/lib/wasix/src/runners/wcgi/runner.rs b/lib/wasix/src/runners/wcgi/runner.rs index 2c2ba3b5278..5022e826f9a 100644 --- a/lib/wasix/src/runners/wcgi/runner.rs +++ b/lib/wasix/src/runners/wcgi/runner.rs @@ -243,13 +243,13 @@ impl Config { } #[cfg(feature = "journal")] - pub fn add_snapshot_trigger(&mut self, on: crate::journal::SnapshotTrigger) { + pub fn add_snapshot_trigger(&mut self, on: crate::journaling::SnapshotTrigger) { self.wasi.snapshot_on.push(on); } #[cfg(feature = "journal")] pub fn add_default_snapshot_triggers(&mut self) -> &mut Self { - for on in crate::journal::DEFAULT_SNAPSHOT_TRIGGERS { + for on in crate::journaling::DEFAULT_SNAPSHOT_TRIGGERS { if !self.has_snapshot_trigger(on) { self.add_snapshot_trigger(on); } @@ -258,21 +258,21 @@ impl Config { } #[cfg(feature = "journal")] - pub fn has_snapshot_trigger(&self, on: crate::journal::SnapshotTrigger) -> bool { + pub fn has_snapshot_trigger(&self, on: crate::journaling::SnapshotTrigger) -> bool { self.wasi.snapshot_on.iter().any(|t| *t == on) } #[cfg(feature = "journal")] pub fn with_snapshot_interval(&mut self, period: std::time::Duration) -> &mut Self { - if !self.has_snapshot_trigger(crate::journal::SnapshotTrigger::PeriodicInterval) { - self.add_snapshot_trigger(crate::journal::SnapshotTrigger::PeriodicInterval); + if !self.has_snapshot_trigger(crate::journaling::SnapshotTrigger::PeriodicInterval) { + self.add_snapshot_trigger(crate::journaling::SnapshotTrigger::PeriodicInterval); } self.wasi.snapshot_interval.replace(period); self } #[cfg(feature = "journal")] - pub fn add_journal(&mut self, journal: Arc) -> &mut Self { + pub fn add_journal(&mut self, journal: Arc) -> &mut Self { self.wasi.journals.push(journal); self } diff --git a/lib/wasix/src/runtime/mod.rs b/lib/wasix/src/runtime/mod.rs index 76cc56db94b..9dd8b7d8751 100644 --- a/lib/wasix/src/runtime/mod.rs +++ b/lib/wasix/src/runtime/mod.rs @@ -20,7 +20,7 @@ use virtual_net::{DynVirtualNetworking, VirtualNetworking}; use wasmer::Module; #[cfg(feature = "journal")] -use crate::journal::DynJournal; +use crate::journaling::DynJournal; use crate::{ http::{DynHttpClient, HttpClient}, os::TtyBridge, diff --git a/lib/wasix/src/state/builder.rs b/lib/wasix/src/state/builder.rs index bb303addb43..87f04ebca8b 100644 --- a/lib/wasix/src/state/builder.rs +++ b/lib/wasix/src/state/builder.rs @@ -27,7 +27,7 @@ use crate::{ }; #[cfg(feature = "journal")] use crate::{ - journal::{DynJournal, SnapshotTrigger}, + journaling::{DynJournal, SnapshotTrigger}, syscalls::restore_snapshot, }; diff --git a/lib/wasix/src/state/env.rs b/lib/wasix/src/state/env.rs index de74f7bff0d..b6b01c6f988 100644 --- a/lib/wasix/src/state/env.rs +++ b/lib/wasix/src/state/env.rs @@ -23,7 +23,7 @@ use wasmer_wasix_types::{ }; #[cfg(feature = "journal")] -use crate::journal::{DynJournal, JournalEffector, SnapshotTrigger}; +use crate::journaling::{DynJournal, JournalEffector, SnapshotTrigger}; use crate::{ bin_factory::{BinFactory, BinaryPackage}, capabilities::Capabilities, diff --git a/lib/wasix/src/syscalls/mod.rs b/lib/wasix/src/syscalls/mod.rs index 4104f2d5fa4..5c5b5b69eca 100644 --- a/lib/wasix/src/syscalls/mod.rs +++ b/lib/wasix/src/syscalls/mod.rs @@ -117,7 +117,7 @@ use crate::{ fs_error_into_wasi_err, virtual_file_type_to_wasi_file_type, Fd, InodeVal, Kind, MAX_SYMLINKS, }, - journal::{DynJournal, JournalEffector}, + journaling::{DynJournal, JournalEffector}, os::task::{process::MaybeCheckpointResult, thread::RewindResult}, runtime::task_manager::InlineWaker, utils::store::InstanceSnapshot, @@ -1222,7 +1222,7 @@ pub fn rewind_ext( #[cfg(not(feature = "journal"))] pub fn maybe_snapshot_once( ctx: FunctionEnvMut<'_, WasiEnv>, - _trigger: crate::journal::SnapshotTrigger, + _trigger: crate::journaling::SnapshotTrigger, ) -> WasiResult> { Ok(Ok(ctx)) } @@ -1230,7 +1230,7 @@ pub fn maybe_snapshot_once( #[cfg(feature = "journal")] pub fn maybe_snapshot_once( mut ctx: FunctionEnvMut<'_, WasiEnv>, - trigger: crate::journal::SnapshotTrigger, + trigger: crate::journaling::SnapshotTrigger, ) -> WasiResult> { use crate::os::task::process::{WasiProcessCheckpoint, WasiProcessInner}; @@ -1301,10 +1301,10 @@ pub fn restore_snapshot( while let Some(next) = journal.read().await.map_err(anyhow_err_to_runtime_err)? { tracing::trace!("Restoring snapshot event - {next:?}"); match next { - crate::journal::JournalEntry::Init { .. } => { + crate::journaling::JournalEntry::Init { .. } => { // TODO: Here we need to check the hash matches the binary } - crate::journal::JournalEntry::FileDescriptorWrite { + crate::journaling::JournalEntry::FileDescriptorWrite { fd, offset, data, @@ -1319,16 +1319,16 @@ pub fn restore_snapshot( } .map_err(anyhow_err_to_runtime_err)?; } - crate::journal::JournalEntry::FileDescriptorSeek { fd, offset, whence } => { + crate::journaling::JournalEntry::FileDescriptorSeek { fd, offset, whence } => { JournalEffector::apply_fd_seek(&mut ctx, fd, offset, whence) .await .map_err(anyhow_err_to_runtime_err)?; } - crate::journal::JournalEntry::UpdateMemoryRegion { region, data } => { + crate::journaling::JournalEntry::UpdateMemoryRegion { region, data } => { JournalEffector::apply_memory(&mut ctx, region, &data) .map_err(anyhow_err_to_runtime_err)?; } - crate::journal::JournalEntry::CloseThread { id, exit_code } => { + crate::journaling::JournalEntry::CloseThread { id, exit_code } => { if id == ctx.data().tid() { return Err(WasiRuntimeError::Runtime(RuntimeError::user( anyhow::format_err!( @@ -1338,7 +1338,7 @@ pub fn restore_snapshot( ))); } } - crate::journal::JournalEntry::SetThread { + crate::journaling::JournalEntry::SetThread { id, call_stack, memory_stack, @@ -1361,11 +1361,11 @@ pub fn restore_snapshot( ))); } } - crate::journal::JournalEntry::CloseFileDescriptor { fd } => { + crate::journaling::JournalEntry::CloseFileDescriptor { fd } => { JournalEffector::apply_fd_close(&mut ctx, fd) .map_err(anyhow_err_to_runtime_err)?; } - crate::journal::JournalEntry::OpenFileDescriptor { + crate::journaling::JournalEntry::OpenFileDescriptor { fd, dirfd, dirflags, @@ -1388,15 +1388,15 @@ pub fn restore_snapshot( ) .map_err(anyhow_err_to_runtime_err)?; } - crate::journal::JournalEntry::RemoveDirectory { fd, path } => { + crate::journaling::JournalEntry::RemoveDirectory { fd, path } => { JournalEffector::apply_path_remove_directory(&mut ctx, fd, &path) .map_err(anyhow_err_to_runtime_err)?; } - crate::journal::JournalEntry::UnlinkFile { fd, path } => { + crate::journaling::JournalEntry::UnlinkFile { fd, path } => { JournalEffector::apply_path_unlink(&mut ctx, fd, &path) .map_err(anyhow_err_to_runtime_err)?; } - crate::journal::JournalEntry::PathRename { + crate::journaling::JournalEntry::PathRename { old_fd, old_path, new_fd, @@ -1407,30 +1407,30 @@ pub fn restore_snapshot( ) .map_err(anyhow_err_to_runtime_err)?; } - crate::journal::JournalEntry::Snapshot { + crate::journaling::JournalEntry::Snapshot { when: _, trigger: _, } => {} - crate::journal::JournalEntry::SetClockTime { clock_id, time } => { + crate::journaling::JournalEntry::SetClockTime { clock_id, time } => { JournalEffector::apply_clock_time_set(&mut ctx, clock_id, time) .map_err(anyhow_err_to_runtime_err)?; } - crate::journal::JournalEntry::RenumberFileDescriptor { old_fd, new_fd } => { + crate::journaling::JournalEntry::RenumberFileDescriptor { old_fd, new_fd } => { JournalEffector::apply_fd_renumber(&mut ctx, old_fd, new_fd) .map_err(anyhow_err_to_runtime_err)?; } - crate::journal::JournalEntry::DuplicateFileDescriptor { + crate::journaling::JournalEntry::DuplicateFileDescriptor { original_fd, copied_fd, } => { JournalEffector::apply_fd_duplicate(&mut ctx, original_fd, copied_fd) .map_err(anyhow_err_to_runtime_err)?; } - crate::journal::JournalEntry::CreateDirectory { fd, path } => { + crate::journaling::JournalEntry::CreateDirectory { fd, path } => { JournalEffector::apply_path_create_directory(&mut ctx, fd, &path) .map_err(anyhow_err_to_runtime_err)?; } - crate::journal::JournalEntry::PathSetTimes { + crate::journaling::JournalEntry::PathSetTimes { fd, flags, path, @@ -1443,7 +1443,7 @@ pub fn restore_snapshot( ) .map_err(anyhow_err_to_runtime_err)?; } - crate::journal::JournalEntry::FileDescriptorSetTimes { + crate::journaling::JournalEntry::FileDescriptorSetTimes { fd, st_atim, st_mtim, @@ -1452,15 +1452,15 @@ pub fn restore_snapshot( JournalEffector::apply_fd_set_times(&mut ctx, fd, st_atim, st_mtim, fst_flags) .map_err(anyhow_err_to_runtime_err)?; } - crate::journal::JournalEntry::FileDescriptorSetSize { fd, st_size } => { + crate::journaling::JournalEntry::FileDescriptorSetSize { fd, st_size } => { JournalEffector::apply_fd_set_size(&mut ctx, fd, st_size) .map_err(anyhow_err_to_runtime_err)?; } - crate::journal::JournalEntry::FileDescriptorSetFlags { fd, flags } => { + crate::journaling::JournalEntry::FileDescriptorSetFlags { fd, flags } => { JournalEffector::apply_fd_set_flags(&mut ctx, fd, flags) .map_err(anyhow_err_to_runtime_err)?; } - crate::journal::JournalEntry::FileDescriptorSetRights { + crate::journaling::JournalEntry::FileDescriptorSetRights { fd, fs_rights_base, fs_rights_inheriting, @@ -1473,7 +1473,7 @@ pub fn restore_snapshot( ) .map_err(anyhow_err_to_runtime_err)?; } - crate::journal::JournalEntry::FileDescriptorAdvise { + crate::journaling::JournalEntry::FileDescriptorAdvise { fd, offset, len, @@ -1482,11 +1482,11 @@ pub fn restore_snapshot( JournalEffector::apply_fd_advise(&mut ctx, fd, offset, len, advice) .map_err(anyhow_err_to_runtime_err)?; } - crate::journal::JournalEntry::FileDescriptorAllocate { fd, offset, len } => { + crate::journaling::JournalEntry::FileDescriptorAllocate { fd, offset, len } => { JournalEffector::apply_fd_allocate(&mut ctx, fd, offset, len) .map_err(anyhow_err_to_runtime_err)?; } - crate::journal::JournalEntry::CreateHardLink { + crate::journaling::JournalEntry::CreateHardLink { old_fd, old_path, old_flags, @@ -1498,7 +1498,7 @@ pub fn restore_snapshot( ) .map_err(anyhow_err_to_runtime_err)?; } - crate::journal::JournalEntry::CreateSymbolicLink { + crate::journaling::JournalEntry::CreateSymbolicLink { old_path, fd, new_path, @@ -1506,19 +1506,19 @@ pub fn restore_snapshot( JournalEffector::apply_path_symlink(&mut ctx, &old_path, fd, &new_path) .map_err(anyhow_err_to_runtime_err)?; } - crate::journal::JournalEntry::ChangeDirectory { path } => { + crate::journaling::JournalEntry::ChangeDirectory { path } => { JournalEffector::apply_chdir(&mut ctx, &path) .map_err(anyhow_err_to_runtime_err)?; } - crate::journal::JournalEntry::CreatePipe { fd1, fd2 } => { + crate::journaling::JournalEntry::CreatePipe { fd1, fd2 } => { JournalEffector::apply_fd_pipe(&mut ctx, fd1, fd2) .map_err(anyhow_err_to_runtime_err)?; } - crate::journal::JournalEntry::EpollCreate { fd } => { + crate::journaling::JournalEntry::EpollCreate { fd } => { JournalEffector::apply_epoll_create(&mut ctx, fd) .map_err(anyhow_err_to_runtime_err)?; } - crate::journal::JournalEntry::EpollCtl { + crate::journaling::JournalEntry::EpollCtl { epfd, op, fd, @@ -1527,7 +1527,7 @@ pub fn restore_snapshot( JournalEffector::apply_epoll_ctl(&mut ctx, epfd, op, fd, event) .map_err(anyhow_err_to_runtime_err)?; } - crate::journal::JournalEntry::TtySet { tty, line_feeds } => { + crate::journaling::JournalEntry::TtySet { tty, line_feeds } => { JournalEffector::apply_tty_set( &mut ctx, crate::WasiTtyState { @@ -1545,7 +1545,7 @@ pub fn restore_snapshot( ) .map_err(anyhow_err_to_runtime_err)?; } - crate::journal::JournalEntry::Panic { stack_trace, .. } => { + crate::journaling::JournalEntry::Panic { stack_trace, .. } => { return Err(WasiRuntimeError::Runtime(RuntimeError::new(format!( "panic\r\nstack: {}", stack_trace diff --git a/lib/wasix/src/syscalls/wasi/environ_get.rs b/lib/wasix/src/syscalls/wasi/environ_get.rs index a39a5faa5f6..201c73440b2 100644 --- a/lib/wasix/src/syscalls/wasi/environ_get.rs +++ b/lib/wasix/src/syscalls/wasi/environ_get.rs @@ -1,5 +1,5 @@ use super::*; -use crate::{journal::SnapshotTrigger, syscalls::*}; +use crate::{journaling::SnapshotTrigger, syscalls::*}; /// ### `environ_get()` /// Read environment variable data. diff --git a/lib/wasix/src/syscalls/wasi/environ_sizes_get.rs b/lib/wasix/src/syscalls/wasi/environ_sizes_get.rs index 846b8e92839..9a68b784d4f 100644 --- a/lib/wasix/src/syscalls/wasi/environ_sizes_get.rs +++ b/lib/wasix/src/syscalls/wasi/environ_sizes_get.rs @@ -1,5 +1,5 @@ use super::*; -use crate::{journal::SnapshotTrigger, syscalls::*}; +use crate::{journaling::SnapshotTrigger, syscalls::*}; /// ### `environ_sizes_get()` /// Return command-line argument data sizes. diff --git a/lib/wasix/src/syscalls/wasi/fd_read.rs b/lib/wasix/src/syscalls/wasi/fd_read.rs index 2d16eda58b4..cc26a529543 100644 --- a/lib/wasix/src/syscalls/wasi/fd_read.rs +++ b/lib/wasix/src/syscalls/wasi/fd_read.rs @@ -5,7 +5,7 @@ use virtual_fs::{AsyncReadExt, DeviceFile, ReadBuf}; use super::*; use crate::{ fs::NotificationInner, - journal::SnapshotTrigger, + journaling::SnapshotTrigger, net::socket::TimeType, os::task::process::{MaybeCheckpointResult, WasiProcessCheckpoint, WasiProcessInner}, syscalls::*, diff --git a/lib/wasix/src/syscalls/wasi/fd_write.rs b/lib/wasix/src/syscalls/wasi/fd_write.rs index 2590b0abb70..95510661c20 100644 --- a/lib/wasix/src/syscalls/wasi/fd_write.rs +++ b/lib/wasix/src/syscalls/wasi/fd_write.rs @@ -3,7 +3,7 @@ use std::task::Waker; use super::*; #[cfg(feature = "journal")] use crate::{ - journal::{JournalEffector, JournalEntry}, + journaling::{JournalEffector, JournalEntry}, utils::map_snapshot_err, }; use crate::{net::socket::TimeType, syscalls::*}; diff --git a/lib/wasix/src/syscalls/wasix/sock_listen.rs b/lib/wasix/src/syscalls/wasix/sock_listen.rs index 0b82d05f7d5..f1614789ca7 100644 --- a/lib/wasix/src/syscalls/wasix/sock_listen.rs +++ b/lib/wasix/src/syscalls/wasix/sock_listen.rs @@ -1,5 +1,5 @@ use super::*; -use crate::{journal::SnapshotTrigger, syscalls::*}; +use crate::{journaling::SnapshotTrigger, syscalls::*}; /// ### `sock_listen()` /// Listen for connections on a socket diff --git a/lib/wasix/src/syscalls/wasix/thread_spawn.rs b/lib/wasix/src/syscalls/wasix/thread_spawn.rs index 22f3124bbcb..5735abaceaa 100644 --- a/lib/wasix/src/syscalls/wasix/thread_spawn.rs +++ b/lib/wasix/src/syscalls/wasix/thread_spawn.rs @@ -2,7 +2,7 @@ use std::f32::consts::E; use super::*; #[cfg(feature = "journal")] -use crate::journal::JournalEffector; +use crate::journaling::JournalEffector; use crate::{ capture_instance_snapshot, os::task::thread::WasiMemoryLayout, From ffad63c077226e07b937e6332155178e1517332f Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Mon, 20 Nov 2023 14:28:11 +1100 Subject: [PATCH 044/129] Update of gramma --- lib/wasix/src/journaling/effector/syscalls/clock_time.rs | 2 +- lib/wasix/src/journaling/effector/syscalls/epoll_create.rs | 6 +++--- lib/wasix/src/journaling/effector/syscalls/epoll_ctl.rs | 4 ++-- lib/wasix/src/journaling/effector/syscalls/fd_advise.rs | 2 +- lib/wasix/src/journaling/effector/syscalls/fd_allocate.rs | 2 +- lib/wasix/src/journaling/effector/syscalls/fd_close.rs | 2 +- lib/wasix/src/journaling/effector/syscalls/fd_duplicate.rs | 4 ++-- lib/wasix/src/journaling/effector/syscalls/fd_pipe.rs | 6 +++--- lib/wasix/src/journaling/effector/syscalls/fd_renumber.rs | 2 +- lib/wasix/src/journaling/effector/syscalls/fd_seek.rs | 2 +- lib/wasix/src/journaling/effector/syscalls/fd_set_flags.rs | 2 +- lib/wasix/src/journaling/effector/syscalls/fd_set_rights.rs | 2 +- lib/wasix/src/journaling/effector/syscalls/fd_set_size.rs | 2 +- lib/wasix/src/journaling/effector/syscalls/fd_set_times.rs | 2 +- lib/wasix/src/journaling/effector/syscalls/fd_write.rs | 2 +- .../journaling/effector/syscalls/path_create_directory.rs | 2 +- lib/wasix/src/journaling/effector/syscalls/path_link.rs | 2 +- lib/wasix/src/journaling/effector/syscalls/path_open.rs | 4 ++-- .../journaling/effector/syscalls/path_remove_directory.rs | 2 +- lib/wasix/src/journaling/effector/syscalls/path_rename.rs | 2 +- .../src/journaling/effector/syscalls/path_set_times.rs | 2 +- lib/wasix/src/journaling/effector/syscalls/path_symlink.rs | 2 +- lib/wasix/src/journaling/effector/syscalls/path_unlink.rs | 2 +- lib/wasix/src/journaling/effector/syscalls/tty_set.rs | 2 +- lib/wasix/src/journaling/unsupported.rs | 2 +- lib/wasix/src/lib.rs | 2 +- 26 files changed, 33 insertions(+), 33 deletions(-) diff --git a/lib/wasix/src/journaling/effector/syscalls/clock_time.rs b/lib/wasix/src/journaling/effector/syscalls/clock_time.rs index f77670787d2..4100c754797 100644 --- a/lib/wasix/src/journaling/effector/syscalls/clock_time.rs +++ b/lib/wasix/src/journaling/effector/syscalls/clock_time.rs @@ -17,7 +17,7 @@ impl JournalEffector { let ret = crate::syscalls::clock_time_set_internal(ctx, clock_id, time); if ret != Errno::Success { bail!( - "snapshot restore error: failed to set clock time (clock_id={:?}, time={}) - {}", + "journal restore error: failed to set clock time (clock_id={:?}, time={}) - {}", clock_id, time, ret diff --git a/lib/wasix/src/journaling/effector/syscalls/epoll_create.rs b/lib/wasix/src/journaling/effector/syscalls/epoll_create.rs index 6546a1344c4..de75591a5e7 100644 --- a/lib/wasix/src/journaling/effector/syscalls/epoll_create.rs +++ b/lib/wasix/src/journaling/effector/syscalls/epoll_create.rs @@ -8,16 +8,16 @@ impl JournalEffector { pub fn apply_epoll_create(ctx: &mut FunctionEnvMut<'_, WasiEnv>, fd: Fd) -> anyhow::Result<()> { let ret_fd = crate::syscalls::epoll_create_internal(ctx) .map_err(|err| { - anyhow::format_err!("snapshot restore error: failed to create epoll - {}", err) + anyhow::format_err!("journal restore error: failed to create epoll - {}", err) })? .map_err(|err| { - anyhow::format_err!("snapshot restore error: failed to create epoll - {}", err) + anyhow::format_err!("journal restore error: failed to create epoll - {}", err) })?; let ret = crate::syscalls::fd_renumber_internal(ctx, ret_fd, fd); if ret != Errno::Success { bail!( - "snapshot restore error: failed renumber file descriptor after epoll create (from={}, to={}) - {}", + "journal restore error: failed renumber file descriptor after epoll create (from={}, to={}) - {}", ret_fd, fd, ret diff --git a/lib/wasix/src/journaling/effector/syscalls/epoll_ctl.rs b/lib/wasix/src/journaling/effector/syscalls/epoll_ctl.rs index 4906711840d..6aaf463b56f 100644 --- a/lib/wasix/src/journaling/effector/syscalls/epoll_ctl.rs +++ b/lib/wasix/src/journaling/effector/syscalls/epoll_ctl.rs @@ -29,7 +29,7 @@ impl JournalEffector { crate::syscalls::epoll_ctl_internal(ctx, pfd, op, fd, event.as_ref()) .map_err(|err| { anyhow::format_err!( - "snapshot restore error: failed to epoll ctl (pfd={}, op={:?}, fd={}) - {}", + "journal restore error: failed to epoll ctl (pfd={}, op={:?}, fd={}) - {}", pfd, op, fd, @@ -38,7 +38,7 @@ impl JournalEffector { })? .map_err(|err| { anyhow::format_err!( - "snapshot restore error: failed to epoll ctl (pfd={}, op={:?}, fd={}) - {}", + "journal restore error: failed to epoll ctl (pfd={}, op={:?}, fd={}) - {}", pfd, op, fd, diff --git a/lib/wasix/src/journaling/effector/syscalls/fd_advise.rs b/lib/wasix/src/journaling/effector/syscalls/fd_advise.rs index 96424ed1872..900db9aca79 100644 --- a/lib/wasix/src/journaling/effector/syscalls/fd_advise.rs +++ b/lib/wasix/src/journaling/effector/syscalls/fd_advise.rs @@ -28,7 +28,7 @@ impl JournalEffector { ) -> anyhow::Result<()> { crate::syscalls::fd_advise_internal(ctx, fd, offset, len, advice).map_err(|err| { anyhow::format_err!( - "snapshot restore error: failed to advise file descriptor (fd={}, offset={}, len={}, advice={:?}) - {}", + "journal restore error: failed to advise file descriptor (fd={}, offset={}, len={}, advice={:?}) - {}", fd, offset, len, diff --git a/lib/wasix/src/journaling/effector/syscalls/fd_allocate.rs b/lib/wasix/src/journaling/effector/syscalls/fd_allocate.rs index 7de1012fcb1..3ad49bb02fd 100644 --- a/lib/wasix/src/journaling/effector/syscalls/fd_allocate.rs +++ b/lib/wasix/src/journaling/effector/syscalls/fd_allocate.rs @@ -22,7 +22,7 @@ impl JournalEffector { crate::syscalls::fd_allocate_internal(ctx, fd, offset, len) .map_err(|err| { anyhow::format_err!( - "snapshot restore error: failed to allocate on file descriptor (fd={}, offset={}, len={}) - {}", + "journal restore error: failed to allocate on file descriptor (fd={}, offset={}, len={}) - {}", fd, offset, len, diff --git a/lib/wasix/src/journaling/effector/syscalls/fd_close.rs b/lib/wasix/src/journaling/effector/syscalls/fd_close.rs index 4d7a593c84d..d1d0b25aa2c 100644 --- a/lib/wasix/src/journaling/effector/syscalls/fd_close.rs +++ b/lib/wasix/src/journaling/effector/syscalls/fd_close.rs @@ -10,7 +10,7 @@ impl JournalEffector { let (_, state) = unsafe { env.get_memory_and_wasi_state(&ctx, 0) }; if let Err(err) = state.fs.close_fd(fd) { bail!( - "snapshot restore error: failed to close descriptor (fd={}) - {}", + "journal restore error: failed to close descriptor (fd={}) - {}", fd, err ); diff --git a/lib/wasix/src/journaling/effector/syscalls/fd_duplicate.rs b/lib/wasix/src/journaling/effector/syscalls/fd_duplicate.rs index 588c9127e9f..1d489ec5420 100644 --- a/lib/wasix/src/journaling/effector/syscalls/fd_duplicate.rs +++ b/lib/wasix/src/journaling/effector/syscalls/fd_duplicate.rs @@ -23,7 +23,7 @@ impl JournalEffector { let ret_fd = crate::syscalls::fd_dup_internal(ctx, original_fd) .map_err(|err| { anyhow::format_err!( - "snapshot restore error: failed to duplicate file descriptor (original={}, copied={}) - {}", + "journal restore error: failed to duplicate file descriptor (original={}, copied={}) - {}", original_fd, copied_fd, err @@ -33,7 +33,7 @@ impl JournalEffector { let ret = crate::syscalls::fd_renumber_internal(ctx, ret_fd, copied_fd); if ret != Errno::Success { bail!( - "snapshot restore error: failed renumber file descriptor after duplicate (from={}, to={}) - {}", + "journal restore error: failed renumber file descriptor after duplicate (from={}, to={}) - {}", ret_fd, copied_fd, ret diff --git a/lib/wasix/src/journaling/effector/syscalls/fd_pipe.rs b/lib/wasix/src/journaling/effector/syscalls/fd_pipe.rs index c96b710de60..5238dd5ac14 100644 --- a/lib/wasix/src/journaling/effector/syscalls/fd_pipe.rs +++ b/lib/wasix/src/journaling/effector/syscalls/fd_pipe.rs @@ -15,13 +15,13 @@ impl JournalEffector { fd2: Fd, ) -> anyhow::Result<()> { let (ret_fd1, ret_fd2) = crate::syscalls::fd_pipe_internal(ctx).map_err(|err| { - anyhow::format_err!("snapshot restore error: failed to create pipe - {}", err) + anyhow::format_err!("journal restore error: failed to create pipe - {}", err) })?; let ret = crate::syscalls::fd_renumber_internal(ctx, ret_fd1, fd1); if ret != Errno::Success { bail!( - "snapshot restore error: failed renumber file descriptor after create pipe (from={}, to={}) - {}", + "journal restore error: failed renumber file descriptor after create pipe (from={}, to={}) - {}", ret_fd1, fd1, ret @@ -31,7 +31,7 @@ impl JournalEffector { let ret = crate::syscalls::fd_renumber_internal(ctx, ret_fd2, fd2); if ret != Errno::Success { bail!( - "snapshot restore error: failed renumber file descriptor after create pipe (from={}, to={}) - {}", + "journal restore error: failed renumber file descriptor after create pipe (from={}, to={}) - {}", ret_fd2, fd2, ret diff --git a/lib/wasix/src/journaling/effector/syscalls/fd_renumber.rs b/lib/wasix/src/journaling/effector/syscalls/fd_renumber.rs index b4e59c4d725..5d72902449d 100644 --- a/lib/wasix/src/journaling/effector/syscalls/fd_renumber.rs +++ b/lib/wasix/src/journaling/effector/syscalls/fd_renumber.rs @@ -23,7 +23,7 @@ impl JournalEffector { let ret = crate::syscalls::fd_renumber_internal(ctx, from, to); if ret != Errno::Success { bail!( - "snapshot restore error: failed to renumber descriptor (from={}, to={}) - {}", + "journal restore error: failed to renumber descriptor (from={}, to={}) - {}", from, to, ret diff --git a/lib/wasix/src/journaling/effector/syscalls/fd_seek.rs b/lib/wasix/src/journaling/effector/syscalls/fd_seek.rs index 6816b09c7fe..ecb19c1159b 100644 --- a/lib/wasix/src/journaling/effector/syscalls/fd_seek.rs +++ b/lib/wasix/src/journaling/effector/syscalls/fd_seek.rs @@ -19,7 +19,7 @@ impl JournalEffector { crate::syscalls::fd_seek_internal(ctx, fd, offset, whence)? .map_err(|err| { anyhow::format_err!( - "snapshot restore error: failed to seek descriptor (fd={}, offset={}, whence={:?}) - {}", + "journal restore error: failed to seek descriptor (fd={}, offset={}, whence={:?}) - {}", fd, offset, whence, diff --git a/lib/wasix/src/journaling/effector/syscalls/fd_set_flags.rs b/lib/wasix/src/journaling/effector/syscalls/fd_set_flags.rs index 19246c006cf..ba9e118b92e 100644 --- a/lib/wasix/src/journaling/effector/syscalls/fd_set_flags.rs +++ b/lib/wasix/src/journaling/effector/syscalls/fd_set_flags.rs @@ -17,7 +17,7 @@ impl JournalEffector { crate::syscalls::fd_fdstat_set_flags_internal(ctx, fd, flags) .map_err(|err| { anyhow::format_err!( - "snapshot restore error: failed to duplicate file descriptor (fd={}, flags={:?}) - {}", + "journal restore error: failed to duplicate file descriptor (fd={}, flags={:?}) - {}", fd, flags, err diff --git a/lib/wasix/src/journaling/effector/syscalls/fd_set_rights.rs b/lib/wasix/src/journaling/effector/syscalls/fd_set_rights.rs index 95a7ace012a..b86539c0c14 100644 --- a/lib/wasix/src/journaling/effector/syscalls/fd_set_rights.rs +++ b/lib/wasix/src/journaling/effector/syscalls/fd_set_rights.rs @@ -26,7 +26,7 @@ impl JournalEffector { crate::syscalls::fd_fdstat_set_rights_internal(ctx, fd, fs_rights_base, fs_rights_inheriting) .map_err(|err| { anyhow::format_err!( - "snapshot restore error: failed to duplicate file descriptor (fd={}, fs_rights_base={:?}, fs_rights_inheriting={:?}) - {}", + "journal restore error: failed to duplicate file descriptor (fd={}, fs_rights_base={:?}, fs_rights_inheriting={:?}) - {}", fd, fs_rights_base, fs_rights_inheriting, diff --git a/lib/wasix/src/journaling/effector/syscalls/fd_set_size.rs b/lib/wasix/src/journaling/effector/syscalls/fd_set_size.rs index 67a1f1d62a1..ec24f9ed0ad 100644 --- a/lib/wasix/src/journaling/effector/syscalls/fd_set_size.rs +++ b/lib/wasix/src/journaling/effector/syscalls/fd_set_size.rs @@ -17,7 +17,7 @@ impl JournalEffector { crate::syscalls::fd_filestat_set_size_internal(ctx, fd, st_size) .map_err(|err| { anyhow::format_err!( - "snapshot restore error: failed to duplicate file descriptor (fd={}, st_size={}) - {}", + "journal restore error: failed to duplicate file descriptor (fd={}, st_size={}) - {}", fd, st_size, err diff --git a/lib/wasix/src/journaling/effector/syscalls/fd_set_times.rs b/lib/wasix/src/journaling/effector/syscalls/fd_set_times.rs index 492ad413907..6500378d392 100644 --- a/lib/wasix/src/journaling/effector/syscalls/fd_set_times.rs +++ b/lib/wasix/src/journaling/effector/syscalls/fd_set_times.rs @@ -29,7 +29,7 @@ impl JournalEffector { crate::syscalls::fd_filestat_set_times_internal(ctx, fd, st_atim, st_mtim, fst_flags) .map_err(|err| { anyhow::format_err!( - "snapshot restore error: failed to duplicate file descriptor (fd={}, st_atim={}, st_mtim={}, fst_flags={:?}) - {}", + "journal restore error: failed to duplicate file descriptor (fd={}, st_atim={}, st_mtim={}, fst_flags={:?}) - {}", fd, st_atim, st_mtim, diff --git a/lib/wasix/src/journaling/effector/syscalls/fd_write.rs b/lib/wasix/src/journaling/effector/syscalls/fd_write.rs index 493ff41470c..2b08daf38ba 100644 --- a/lib/wasix/src/journaling/effector/syscalls/fd_write.rs +++ b/lib/wasix/src/journaling/effector/syscalls/fd_write.rs @@ -62,7 +62,7 @@ impl JournalEffector { )?; if ret != Errno::Success { bail!( - "snapshot restore error: failed to write to descriptor (fd={}, offset={}) - {}", + "journal restore error: failed to write to descriptor (fd={}, offset={}) - {}", fd, offset, ret diff --git a/lib/wasix/src/journaling/effector/syscalls/path_create_directory.rs b/lib/wasix/src/journaling/effector/syscalls/path_create_directory.rs index 5c6c8f83ed5..249a9df128c 100644 --- a/lib/wasix/src/journaling/effector/syscalls/path_create_directory.rs +++ b/lib/wasix/src/journaling/effector/syscalls/path_create_directory.rs @@ -22,7 +22,7 @@ impl JournalEffector { ) -> anyhow::Result<()> { crate::syscalls::path_create_directory_internal(ctx, fd, path).map_err(|err| { anyhow::format_err!( - "snapshot restore error: failed to create directory path (fd={}, path={}) - {}", + "journal restore error: failed to create directory path (fd={}, path={}) - {}", fd, path, err diff --git a/lib/wasix/src/journaling/effector/syscalls/path_link.rs b/lib/wasix/src/journaling/effector/syscalls/path_link.rs index 24abed63792..fd85faf35fa 100644 --- a/lib/wasix/src/journaling/effector/syscalls/path_link.rs +++ b/lib/wasix/src/journaling/effector/syscalls/path_link.rs @@ -32,7 +32,7 @@ impl JournalEffector { crate::syscalls::path_link_internal(ctx, old_fd, old_flags, old_path, new_fd, new_path) .map_err(|err| { anyhow::format_err!( - "snapshot restore error: failed to create hard link (old_fd={}, old_flags={}, old_path={}, new_fd={}, new_path={}) - {}", + "journal restore error: failed to create hard link (old_fd={}, old_flags={}, old_path={}, new_fd={}, new_path={}) - {}", old_fd, old_flags, old_path, diff --git a/lib/wasix/src/journaling/effector/syscalls/path_open.rs b/lib/wasix/src/journaling/effector/syscalls/path_open.rs index a2af4bc8a13..5119f3570a5 100644 --- a/lib/wasix/src/journaling/effector/syscalls/path_open.rs +++ b/lib/wasix/src/journaling/effector/syscalls/path_open.rs @@ -54,7 +54,7 @@ impl JournalEffector { Ok(fd) => fd, Err(err) => { bail!( - "snapshot restore error: failed to open descriptor (fd={}, path={}) - {}", + "journal restore error: failed to open descriptor (fd={}, path={}) - {}", fd, path, err @@ -65,7 +65,7 @@ impl JournalEffector { let ret = crate::syscalls::fd_renumber_internal(ctx, ret_fd, fd); if ret != Errno::Success { bail!( - "snapshot restore error: failed renumber file descriptor after open (from={}, to={}) - {}", + "journal restore error: failed renumber file descriptor after open (from={}, to={}) - {}", ret_fd, fd, ret diff --git a/lib/wasix/src/journaling/effector/syscalls/path_remove_directory.rs b/lib/wasix/src/journaling/effector/syscalls/path_remove_directory.rs index 4b7a2b2a3dd..d607cca4520 100644 --- a/lib/wasix/src/journaling/effector/syscalls/path_remove_directory.rs +++ b/lib/wasix/src/journaling/effector/syscalls/path_remove_directory.rs @@ -22,7 +22,7 @@ impl JournalEffector { ) -> anyhow::Result<()> { if let Err(err) = crate::syscalls::path_remove_directory_internal(ctx, fd, path) { bail!( - "snapshot restore error: failed to remove directory - {}", + "journal restore error: failed to remove directory - {}", err ); } diff --git a/lib/wasix/src/journaling/effector/syscalls/path_rename.rs b/lib/wasix/src/journaling/effector/syscalls/path_rename.rs index ef692a8f467..9eaafaff42c 100644 --- a/lib/wasix/src/journaling/effector/syscalls/path_rename.rs +++ b/lib/wasix/src/journaling/effector/syscalls/path_rename.rs @@ -29,7 +29,7 @@ impl JournalEffector { let ret = crate::syscalls::path_rename_internal(ctx, old_fd, old_path, new_fd, new_path)?; if ret != Errno::Success { bail!( - "snapshot restore error: failed to rename path (old_fd={}, old_path={}, new_fd={}, new_path={}) - {}", + "journal restore error: failed to rename path (old_fd={}, old_path={}, new_fd={}, new_path={}) - {}", old_fd, old_path, new_fd, diff --git a/lib/wasix/src/journaling/effector/syscalls/path_set_times.rs b/lib/wasix/src/journaling/effector/syscalls/path_set_times.rs index e0a4cb31e42..b9e72a0c5b4 100644 --- a/lib/wasix/src/journaling/effector/syscalls/path_set_times.rs +++ b/lib/wasix/src/journaling/effector/syscalls/path_set_times.rs @@ -35,7 +35,7 @@ impl JournalEffector { crate::syscalls::path_filestat_set_times_internal(ctx, fd, flags, path, st_atim, st_mtim, fst_flags) .map_err(|err| { anyhow::format_err!( - "snapshot restore error: failed to set path times (fd={}, flags={}, path={}, st_atim={}, st_mtim={}, fst_flags={:?}) - {}", + "journal restore error: failed to set path times (fd={}, flags={}, path={}, st_atim={}, st_mtim={}, fst_flags={:?}) - {}", fd, flags, path, diff --git a/lib/wasix/src/journaling/effector/syscalls/path_symlink.rs b/lib/wasix/src/journaling/effector/syscalls/path_symlink.rs index b63f2ca492a..3f54de76412 100644 --- a/lib/wasix/src/journaling/effector/syscalls/path_symlink.rs +++ b/lib/wasix/src/journaling/effector/syscalls/path_symlink.rs @@ -26,7 +26,7 @@ impl JournalEffector { crate::syscalls::path_symlink_internal(ctx, old_path, fd, new_path) .map_err(|err| { anyhow::format_err!( - "snapshot restore error: failed to create symlink (old_path={}, fd={}, new_path={}) - {}", + "journal restore error: failed to create symlink (old_path={}, fd={}, new_path={}) - {}", old_path, fd, new_path, diff --git a/lib/wasix/src/journaling/effector/syscalls/path_unlink.rs b/lib/wasix/src/journaling/effector/syscalls/path_unlink.rs index 073431ab39c..a21825b9258 100644 --- a/lib/wasix/src/journaling/effector/syscalls/path_unlink.rs +++ b/lib/wasix/src/journaling/effector/syscalls/path_unlink.rs @@ -23,7 +23,7 @@ impl JournalEffector { let ret = crate::syscalls::path_unlink_file_internal(ctx, fd, path)?; if ret != Errno::Success { bail!( - "snapshot restore error: failed to remove file (fd={}, path={}) - {}", + "journal restore error: failed to remove file (fd={}, path={}) - {}", fd, path, ret diff --git a/lib/wasix/src/journaling/effector/syscalls/tty_set.rs b/lib/wasix/src/journaling/effector/syscalls/tty_set.rs index bdffe6d3d81..76d7ef44ac0 100644 --- a/lib/wasix/src/journaling/effector/syscalls/tty_set.rs +++ b/lib/wasix/src/journaling/effector/syscalls/tty_set.rs @@ -31,7 +31,7 @@ impl JournalEffector { state: WasiTtyState, ) -> anyhow::Result<()> { crate::syscalls::tty_set_internal(ctx, state).map_err(|err| { - anyhow::format_err!("snapshot restore error: failed to set tty - {}", err) + anyhow::format_err!("journal restore error: failed to set tty - {}", err) })?; Ok(()) } diff --git a/lib/wasix/src/journaling/unsupported.rs b/lib/wasix/src/journaling/unsupported.rs index 41f8005f416..6256c18dd59 100644 --- a/lib/wasix/src/journaling/unsupported.rs +++ b/lib/wasix/src/journaling/unsupported.rs @@ -11,7 +11,7 @@ pub struct UnsupportedJournal {} impl Journal for UnsupportedJournal { fn write<'a>(&'a self, entry: JournalEntry<'a>) -> LocalBoxFuture<'a, anyhow::Result<()>> { - tracing::debug!("snapshot event: {:?}", entry); + tracing::debug!("journal event: {:?}", entry); Box::pin(async { Err(anyhow::format_err!("unsupported")) }) } diff --git a/lib/wasix/src/lib.rs b/lib/wasix/src/lib.rs index 031c86afa0b..9bc3716632c 100644 --- a/lib/wasix/src/lib.rs +++ b/lib/wasix/src/lib.rs @@ -240,7 +240,7 @@ pub(crate) fn run_wasi_func( if !journals.is_empty() { return Err(WasiRuntimeError::Runtime(RuntimeError::user( anyhow::format_err!( - "snapshot restoration is not currently supported when running specific functions" + "journal restoration is not currently supported when running specific functions" ) .into(), ))); From cbeff30a21c0784da4ce12e8df7cf2d493b60009 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Mon, 20 Nov 2023 14:29:42 +1100 Subject: [PATCH 045/129] Fix for a linting issue --- lib/cli/src/commands/run/wasi.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cli/src/commands/run/wasi.rs b/lib/cli/src/commands/run/wasi.rs index dd79828acde..b1b00a1ca73 100644 --- a/lib/cli/src/commands/run/wasi.rs +++ b/lib/cli/src/commands/run/wasi.rs @@ -347,7 +347,7 @@ impl Wasi { for trigger in self.snapshot_on.iter().cloned() { builder.add_snapshot_trigger(trigger); } - if let Some(interval) = self.snapshot_interval.clone() { + if let Some(interval) = self.snapshot_interval { builder.with_snapshot_interval(std::time::Duration::from_millis(interval)); } for journal in self.journals.iter() { From e36b898895922593cd3b9971e5fae5a915951146 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Mon, 20 Nov 2023 16:54:31 +1100 Subject: [PATCH 046/129] Added an event for process exit and a module hash to validate the same memory can be used --- examples/wasi_manual_setup.rs | 2 +- lib/c-api/src/wasm_c_api/wasi/mod.rs | 2 +- lib/cli/src/commands/run/mod.rs | 25 +++- lib/cli/src/commands/run/wasi.rs | 5 +- lib/wasix/src/bin_factory/exec.rs | 16 +-- lib/wasix/src/journaling/effector/mod.rs | 2 + .../src/journaling/effector/process_exit.rs | 20 +++ .../src/journaling/effector/save_event.rs | 5 +- lib/wasix/src/journaling/filter.rs | 44 +++--- lib/wasix/src/journaling/journal.rs | 39 +++-- lib/wasix/src/journaling/log_file.rs | 135 +++++++++--------- lib/wasix/src/os/task/control_plane.rs | 14 +- lib/wasix/src/os/task/process.rs | 9 +- lib/wasix/src/runners/wasi.rs | 7 +- lib/wasix/src/runners/wcgi/handler.rs | 8 +- lib/wasix/src/runners/wcgi/runner.rs | 1 + lib/wasix/src/runtime/module_cache/types.rs | 11 +- lib/wasix/src/state/builder.rs | 61 +++++--- lib/wasix/src/state/env.rs | 40 ++++-- lib/wasix/src/state/func_env.rs | 6 +- lib/wasix/src/syscalls/mod.rs | 39 +++-- lib/wasix/src/syscalls/wasix/proc_fork.rs | 2 +- lib/wasix/src/syscalls/wasix/thread_spawn.rs | 2 +- tests/lib/wast/src/wasi_wast.rs | 11 +- 24 files changed, 317 insertions(+), 189 deletions(-) create mode 100644 lib/wasix/src/journaling/effector/process_exit.rs diff --git a/examples/wasi_manual_setup.rs b/examples/wasi_manual_setup.rs index fceb44b8db1..d08de6d01b9 100644 --- a/examples/wasi_manual_setup.rs +++ b/examples/wasi_manual_setup.rs @@ -65,7 +65,7 @@ fn main() -> Result<(), Box> { let start = instance.exports.get_function("_start")?; start.call(&mut store, &[])?; - wasi_env.cleanup(&mut store, None); + wasi_env.on_exit(&mut store, None); Ok(()) } diff --git a/lib/c-api/src/wasm_c_api/wasi/mod.rs b/lib/c-api/src/wasm_c_api/wasi/mod.rs index 508293dcb93..f8e52e9ae5e 100644 --- a/lib/c-api/src/wasm_c_api/wasi/mod.rs +++ b/lib/c-api/src/wasm_c_api/wasi/mod.rs @@ -376,7 +376,7 @@ pub unsafe extern "C" fn wasi_env_new( pub extern "C" fn wasi_env_delete(state: Option>) { if let Some(mut env) = state { env.inner - .cleanup(unsafe { &mut env.store.store_mut() }, None); + .on_exit(unsafe { &mut env.store.store_mut() }, None); } } diff --git a/lib/cli/src/commands/run/mod.rs b/lib/cli/src/commands/run/mod.rs index aebe12e6428..3d4dce6a8ff 100644 --- a/lib/cli/src/commands/run/mod.rs +++ b/lib/cli/src/commands/run/mod.rs @@ -123,9 +123,11 @@ impl Run { let result = { match target { - ExecutableTarget::WebAssembly { module, path } => { - self.execute_wasm(&path, &module, store, runtime) - } + ExecutableTarget::WebAssembly { + module, + module_hash, + path, + } => self.execute_wasm(&path, &module, module_hash, store, runtime), ExecutableTarget::Package(pkg) => self.execute_webc(&pkg, runtime), } }; @@ -142,13 +144,14 @@ impl Run { &self, path: &Path, module: &Module, + module_hash: ModuleHash, mut store: Store, runtime: Arc, ) -> Result<(), Error> { if wasmer_emscripten::is_emscripten_module(module) { self.execute_emscripten_module() } else if wasmer_wasix::is_wasi_module(module) || wasmer_wasix::is_wasix_module(module) { - self.execute_wasi_module(path, module, runtime, store) + self.execute_wasi_module(path, module, module_hash, runtime, store) } else { self.execute_pure_wasm_module(module, &mut store) } @@ -359,6 +362,7 @@ impl Run { &self, wasm_path: &Path, module: &Module, + module_hash: ModuleHash, runtime: Arc, mut store: Store, ) -> Result<(), Error> { @@ -369,6 +373,7 @@ impl Run { runtime, &program_name, module, + module_hash, self.wasi.enable_async_threads, ) } @@ -604,7 +609,11 @@ impl TargetOnDisk { #[derive(Debug, Clone)] enum ExecutableTarget { - WebAssembly { module: Module, path: PathBuf }, + WebAssembly { + module: Module, + module_hash: ModuleHash, + path: PathBuf, + }, Package(BinaryPackage), } @@ -652,6 +661,7 @@ impl ExecutableTarget { Ok(ExecutableTarget::WebAssembly { module, + module_hash: ModuleHash::sha256(&wasm), path: path.to_path_buf(), }) } @@ -659,9 +669,14 @@ impl ExecutableTarget { let engine = runtime.engine(); pb.set_message("Deserializing pre-compiled WebAssembly module"); let module = unsafe { Module::deserialize_from_file(&engine, path)? }; + let module_hash = { + let wasm = std::fs::read(path)?; + ModuleHash::sha256(&wasm) + }; Ok(ExecutableTarget::WebAssembly { module, + module_hash, path: path.to_path_buf(), }) } diff --git a/lib/cli/src/commands/run/wasi.rs b/lib/cli/src/commands/run/wasi.rs index b1b00a1ca73..bdf55fb0429 100644 --- a/lib/cli/src/commands/run/wasi.rs +++ b/lib/cli/src/commands/run/wasi.rs @@ -25,7 +25,7 @@ use wasmer_wasix::{ rewind_ext, runners::{MappedCommand, MappedDirectory}, runtime::{ - module_cache::{FileSystemCache, ModuleCache}, + module_cache::{FileSystemCache, ModuleCache, ModuleHash}, package_loader::{BuiltinPackageLoader, PackageLoader}, resolver::{ FileSystemSource, InMemorySource, MultiSource, PackageSpecifier, Source, WapmSource, @@ -545,13 +545,14 @@ impl Wasi { pub fn instantiate( &self, module: &Module, + module_hash: ModuleHash, program_name: String, args: Vec, runtime: Arc, store: &mut Store, ) -> Result<(WasiFunctionEnv, Instance)> { let builder = self.prepare(module, program_name, args, runtime)?; - let (instance, wasi_env) = builder.instantiate(module.clone(), store)?; + let (instance, wasi_env) = builder.instantiate(module.clone(), module_hash, store)?; Ok((wasi_env, instance)) } diff --git a/lib/wasix/src/bin_factory/exec.rs b/lib/wasix/src/bin_factory/exec.rs index 66420279d58..d6bc8b7762d 100644 --- a/lib/wasix/src/bin_factory/exec.rs +++ b/lib/wasix/src/bin_factory/exec.rs @@ -34,7 +34,7 @@ pub async fn spawn_exec( pkg.version=%binary.version, "Unable to spawn a command because its package has no entrypoint", ); - env.cleanup(Some(Errno::Noexec.into())).await; + env.on_exit(Some(Errno::Noexec.into())).await; return Err(SpawnError::CompileError); }; @@ -46,7 +46,7 @@ pub async fn spawn_exec( error = &*err, "Failed to compile the module", ); - env.cleanup(Some(Errno::Noexec.into())).await; + env.on_exit(Some(Errno::Noexec.into())).await; return Err(SpawnError::CompileError); } }; @@ -111,7 +111,7 @@ pub fn spawn_exec_module( if let Err(err) = initialize.call(&mut store, &[]) { thread.thread.set_status_finished(Err(err.into())); ctx.data(&store) - .blocking_cleanup(Some(Errno::Noexec.into())); + .blocking_on_exit(Some(Errno::Noexec.into())); return; } } @@ -171,7 +171,7 @@ fn call_module( Some(rewind_result), ); if res != Errno::Success { - ctx.data(&store).blocking_cleanup(Some(res.into())); + ctx.data(&store).blocking_on_exit(Some(res.into())); return; } } else { @@ -183,7 +183,7 @@ fn call_module( Some(rewind_result), ); if res != Errno::Success { - ctx.data(&store).blocking_cleanup(Some(res.into())); + ctx.data(&store).blocking_on_exit(Some(res.into())); return; } }; @@ -197,7 +197,7 @@ fn call_module( } else { debug!("wasi[{}]::exec-failed: missing _start function", pid); ctx.data(&store) - .blocking_cleanup(Some(Errno::Noexec.into())); + .blocking_on_exit(Some(Errno::Noexec.into())); return; }; @@ -241,7 +241,7 @@ fn call_module( }; // Cleanup the environment - ctx.data(&store).blocking_cleanup(Some(code)); + ctx.data(&store).blocking_on_exit(Some(code)); debug!("wasi[{pid}]::main() has exited with {code}"); handle.thread.set_status_finished(ret.map(|a| a.into())); @@ -261,7 +261,7 @@ impl BinFactory { .await .ok_or(SpawnError::NotFound); if binary.is_err() { - env.cleanup(Some(Errno::Noent.into())).await; + env.on_exit(Some(Errno::Noent.into())).await; } let binary = binary?; diff --git a/lib/wasix/src/journaling/effector/mod.rs b/lib/wasix/src/journaling/effector/mod.rs index ed319899c7e..7059855a76c 100644 --- a/lib/wasix/src/journaling/effector/mod.rs +++ b/lib/wasix/src/journaling/effector/mod.rs @@ -56,6 +56,8 @@ mod syscalls { #[cfg(feature = "journal")] mod memory_and_snapshot; #[cfg(feature = "journal")] +mod process_exit; +#[cfg(feature = "journal")] mod save_event; #[cfg(feature = "journal")] mod thread_exit; diff --git a/lib/wasix/src/journaling/effector/process_exit.rs b/lib/wasix/src/journaling/effector/process_exit.rs new file mode 100644 index 00000000000..45f32eb4498 --- /dev/null +++ b/lib/wasix/src/journaling/effector/process_exit.rs @@ -0,0 +1,20 @@ +use super::*; + +impl JournalEffector { + pub fn save_process_exit(env: &WasiEnv, exit_code: Option) -> anyhow::Result<()> { + __asyncify_light(env, None, async { + env.active_journal()? + .write(JournalEntry::ProcessExit { exit_code }) + .await + .map_err(map_snapshot_err)?; + Ok(()) + })? + .map_err(|err| WasiError::Exit(ExitCode::Errno(err)))?; + Ok(()) + } + + pub fn apply_process_exit(env: &WasiEnv, exit_code: Option) -> anyhow::Result<()> { + env.blocking_on_exit(exit_code); + Ok(()) + } +} diff --git a/lib/wasix/src/journaling/effector/save_event.rs b/lib/wasix/src/journaling/effector/save_event.rs index ccc1d4d32d4..d214d973263 100644 --- a/lib/wasix/src/journaling/effector/save_event.rs +++ b/lib/wasix/src/journaling/effector/save_event.rs @@ -1,11 +1,14 @@ use super::*; impl JournalEffector { - pub(super) fn save_event( + pub(crate) fn save_event( ctx: &mut FunctionEnvMut<'_, WasiEnv>, event: JournalEntry, ) -> anyhow::Result<()> { let env = ctx.data(); + if !env.should_journal() { + return Ok(()); + } __asyncify_light(env, None, async { ctx.data() diff --git a/lib/wasix/src/journaling/filter.rs b/lib/wasix/src/journaling/filter.rs index 4714f79c88c..e55fb9e617b 100644 --- a/lib/wasix/src/journaling/filter.rs +++ b/lib/wasix/src/journaling/filter.rs @@ -16,7 +16,6 @@ pub struct FilteredJournal { filter_snapshots: bool, filter_descriptors: bool, filter_epoll: bool, - filter_panics: bool, } impl FilteredJournal { @@ -32,7 +31,6 @@ impl FilteredJournal { filter_snapshots: false, filter_descriptors: false, filter_epoll: false, - filter_panics: false, } } @@ -80,49 +78,49 @@ impl FilteredJournal { self.filter_descriptors = val; self } - - pub fn with_ignore_panics(mut self, val: bool) -> Self { - self.filter_panics = val; - self - } } impl Journal for FilteredJournal { fn write<'a>(&'a self, entry: JournalEntry<'a>) -> LocalBoxFuture<'a, anyhow::Result<()>> { Box::pin(async { let evt = match entry { - JournalEntry::Init { wasm_hash } => JournalEntry::Init { wasm_hash }, - JournalEntry::FileDescriptorSeek { .. } => { - if self.filter_descriptors { + JournalEntry::InitModule { .. } => { + if self.filter_threads { return Ok(()); } entry } - JournalEntry::FileDescriptorWrite { .. } => { - if self.filter_descriptors { + JournalEntry::UpdateMemoryRegion { .. } => { + if self.filter_memory { return Ok(()); } entry } - JournalEntry::UpdateMemoryRegion { .. } => { - if self.filter_memory { + JournalEntry::ProcessExit { .. } => { + if self.filter_threads { return Ok(()); } entry } - JournalEntry::CloseThread { .. } => { + JournalEntry::SetThread { .. } => { if self.filter_threads { return Ok(()); } entry } - JournalEntry::SetThread { .. } => { + JournalEntry::CloseThread { .. } => { if self.filter_threads { return Ok(()); } entry } - JournalEntry::CloseFileDescriptor { .. } => { + JournalEntry::FileDescriptorSeek { .. } => { + if self.filter_descriptors { + return Ok(()); + } + entry + } + JournalEntry::FileDescriptorWrite { .. } => { if self.filter_descriptors { return Ok(()); } @@ -134,6 +132,12 @@ impl Journal for FilteredJournal { } entry } + JournalEntry::CloseFileDescriptor { .. } => { + if self.filter_descriptors { + return Ok(()); + } + entry + } JournalEntry::RemoveDirectory { .. } => { if self.filter_files { return Ok(()); @@ -267,12 +271,6 @@ impl Journal for FilteredJournal { } entry } - JournalEntry::Panic { .. } => { - if self.filter_panics { - return Ok(()); - } - entry - } }; self.inner.write(evt).await }) diff --git a/lib/wasix/src/journaling/journal.rs b/lib/wasix/src/journaling/journal.rs index 77ca7a41ad5..95c51fe8a48 100644 --- a/lib/wasix/src/journaling/journal.rs +++ b/lib/wasix/src/journaling/journal.rs @@ -41,9 +41,27 @@ pub enum SocketJournalEvent { #[allow(clippy::large_enum_variant)] #[derive(Debug)] pub enum JournalEntry<'a> { - Init { + InitModule { wasm_hash: [u8; 32], }, + UpdateMemoryRegion { + region: Range, + data: Cow<'a, [u8]>, + }, + ProcessExit { + exit_code: Option, + }, + SetThread { + id: WasiThreadId, + call_stack: Cow<'a, [u8]>, + memory_stack: Cow<'a, [u8]>, + store_data: Cow<'a, [u8]>, + is_64bit: bool, + }, + CloseThread { + id: WasiThreadId, + exit_code: Option, + }, FileDescriptorSeek { fd: Fd, offset: FileDelta, @@ -55,25 +73,10 @@ pub enum JournalEntry<'a> { data: Cow<'a, [u8]>, is_64bit: bool, }, - UpdateMemoryRegion { - region: Range, - data: Cow<'a, [u8]>, - }, SetClockTime { clock_id: Snapshot0Clockid, time: Timestamp, }, - CloseThread { - id: WasiThreadId, - exit_code: Option, - }, - SetThread { - id: WasiThreadId, - call_stack: Cow<'a, [u8]>, - memory_stack: Cow<'a, [u8]>, - store_data: Cow<'a, [u8]>, - is_64bit: bool, - }, CloseFileDescriptor { fd: Fd, }, @@ -183,10 +186,6 @@ pub enum JournalEntry<'a> { fd1: Fd, fd2: Fd, }, - Panic { - when: SystemTime, - stack_trace: Cow<'a, str>, - }, /// Represents the marker for the end of a snapshot Snapshot { when: SystemTime, diff --git a/lib/wasix/src/journaling/log_file.rs b/lib/wasix/src/journaling/log_file.rs index b9292ae3598..8a0f171697d 100644 --- a/lib/wasix/src/journaling/log_file.rs +++ b/lib/wasix/src/journaling/log_file.rs @@ -24,9 +24,23 @@ use super::*; #[allow(clippy::large_enum_variant)] #[derive(Debug, Clone, Serialize, Deserialize)] pub(crate) enum LogFileJournalEntry { - InitV1 { + InitModuleV1 { wasm_hash: [u8; 32], }, + ProcessExitV1 { + exit_code: Option, + }, + SetThreadV1 { + id: WasiThreadId, + call_stack: Vec, + memory_stack: Vec, + store_data: Vec, + is_64bit: bool, + }, + CloseThreadV1 { + id: WasiThreadId, + exit_code: Option, + }, FileDescriptorSeekV1 { fd: Fd, offset: i64, @@ -47,20 +61,6 @@ pub(crate) enum LogFileJournalEntry { clock_id: JournalSnapshot0ClockidV1, time: u64, }, - SetThreadV1 { - id: WasiThreadId, - call_stack: Vec, - memory_stack: Vec, - store_data: Vec, - is_64bit: bool, - }, - CloseThreadV1 { - id: WasiThreadId, - exit_code: Option, - }, - CloseFileDescriptorV1 { - fd: u32, - }, OpenFileDescriptorV1 { fd: u32, dirfd: u32, @@ -71,6 +71,9 @@ pub(crate) enum LogFileJournalEntry { fs_rights_inheriting: u64, fs_flags: u16, }, + CloseFileDescriptorV1 { + fd: u32, + }, RenumberFileDescriptorV1 { old_fd: u32, new_fd: u32, @@ -179,10 +182,6 @@ pub(crate) enum LogFileJournalEntry { when: SystemTime, trigger: JournalSnapshotTriggerV1, }, - Panic { - when: SystemTime, - stack_trace: String, - }, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -394,30 +393,13 @@ impl From for wasi::EpollCtl { impl<'a> From> for LogFileJournalEntry { fn from(value: JournalEntry<'a>) -> Self { match value { - JournalEntry::Init { wasm_hash } => Self::InitV1 { wasm_hash }, - JournalEntry::FileDescriptorWrite { - fd, - offset, - data, - is_64bit, - } => Self::FileDescriptorWriteV1 { - fd, - offset, - data: data.into_owned(), - is_64bit, - }, - JournalEntry::FileDescriptorSeek { fd, offset, whence } => Self::FileDescriptorSeekV1 { - fd, - offset, - whence: whence.into(), - }, + JournalEntry::InitModule { wasm_hash } => Self::InitModuleV1 { wasm_hash }, JournalEntry::UpdateMemoryRegion { region, data } => Self::UpdateMemoryRegionV1 { start: region.start, end: region.end, data: data.into_owned(), }, - JournalEntry::CloseThread { id, exit_code } => Self::CloseThreadV1 { - id, + JournalEntry::ProcessExit { exit_code } => Self::ProcessExitV1 { exit_code: exit_code.map(|code| code.into()), }, JournalEntry::SetThread { @@ -433,7 +415,26 @@ impl<'a> From> for LogFileJournalEntry { store_data: store_data.into_owned(), is_64bit, }, - JournalEntry::CloseFileDescriptor { fd } => Self::CloseFileDescriptorV1 { fd }, + JournalEntry::CloseThread { id, exit_code } => Self::CloseThreadV1 { + id, + exit_code: exit_code.map(|code| code.into()), + }, + JournalEntry::FileDescriptorWrite { + fd, + offset, + data, + is_64bit, + } => Self::FileDescriptorWriteV1 { + fd, + offset, + data: data.into_owned(), + is_64bit, + }, + JournalEntry::FileDescriptorSeek { fd, offset, whence } => Self::FileDescriptorSeekV1 { + fd, + offset, + whence: whence.into(), + }, JournalEntry::OpenFileDescriptor { fd, dirfd, @@ -453,6 +454,7 @@ impl<'a> From> for LogFileJournalEntry { fs_rights_inheriting: fs_rights_inheriting.bits(), fs_flags: fs_flags.bits(), }, + JournalEntry::CloseFileDescriptor { fd } => Self::CloseFileDescriptorV1 { fd }, JournalEntry::RemoveDirectory { fd, path } => Self::RemoveDirectoryV1 { fd, path: path.into_owned(), @@ -600,10 +602,6 @@ impl<'a> From> for LogFileJournalEntry { line_feeds, }, JournalEntry::CreatePipe { fd1, fd2 } => Self::CreatePipeV1 { fd1, fd2 }, - JournalEntry::Panic { when, stack_trace } => Self::Panic { - when, - stack_trace: stack_trace.into_owned(), - }, } } } @@ -611,33 +609,14 @@ impl<'a> From> for LogFileJournalEntry { impl<'a> From for JournalEntry<'a> { fn from(value: LogFileJournalEntry) -> Self { match value { - LogFileJournalEntry::InitV1 { wasm_hash } => Self::Init { wasm_hash }, - LogFileJournalEntry::FileDescriptorWriteV1 { - data, - fd, - offset, - is_64bit, - } => Self::FileDescriptorWrite { - data: data.into(), - fd, - offset, - is_64bit, - }, - LogFileJournalEntry::FileDescriptorSeekV1 { fd, offset, whence } => { - Self::FileDescriptorSeek { - fd, - offset, - whence: whence.into(), - } - } + LogFileJournalEntry::InitModuleV1 { wasm_hash } => Self::InitModule { wasm_hash }, LogFileJournalEntry::UpdateMemoryRegionV1 { start, end, data } => { Self::UpdateMemoryRegion { region: start..end, data: data.into(), } } - LogFileJournalEntry::CloseThreadV1 { id, exit_code } => Self::CloseThread { - id, + LogFileJournalEntry::ProcessExitV1 { exit_code } => Self::ProcessExit { exit_code: exit_code.map(|code| code.into()), }, LogFileJournalEntry::SetThreadV1 { @@ -653,7 +632,28 @@ impl<'a> From for JournalEntry<'a> { store_data: store_data.into(), is_64bit, }, - LogFileJournalEntry::CloseFileDescriptorV1 { fd } => Self::CloseFileDescriptor { fd }, + LogFileJournalEntry::CloseThreadV1 { id, exit_code } => Self::CloseThread { + id, + exit_code: exit_code.map(|code| code.into()), + }, + LogFileJournalEntry::FileDescriptorWriteV1 { + data, + fd, + offset, + is_64bit, + } => Self::FileDescriptorWrite { + data: data.into(), + fd, + offset, + is_64bit, + }, + LogFileJournalEntry::FileDescriptorSeekV1 { fd, offset, whence } => { + Self::FileDescriptorSeek { + fd, + offset, + whence: whence.into(), + } + } LogFileJournalEntry::OpenFileDescriptorV1 { fd, dirfd, @@ -673,6 +673,7 @@ impl<'a> From for JournalEntry<'a> { fs_rights_inheriting: wasi::Rights::from_bits_truncate(fs_rights_inheriting), fs_flags: wasi::Fdflags::from_bits_truncate(fs_flags), }, + LogFileJournalEntry::CloseFileDescriptorV1 { fd } => Self::CloseFileDescriptor { fd }, LogFileJournalEntry::RemoveDirectoryV1 { fd, path } => Self::RemoveDirectory { fd, path: path.into(), @@ -835,10 +836,6 @@ impl<'a> From for JournalEntry<'a> { line_feeds, }, LogFileJournalEntry::CreatePipeV1 { fd1, fd2 } => Self::CreatePipe { fd1, fd2 }, - LogFileJournalEntry::Panic { when, stack_trace } => Self::Panic { - when, - stack_trace: stack_trace.into(), - }, } } } diff --git a/lib/wasix/src/os/task/control_plane.rs b/lib/wasix/src/os/task/control_plane.rs index af432d5e0d4..7bbc7894887 100644 --- a/lib/wasix/src/os/task/control_plane.rs +++ b/lib/wasix/src/os/task/control_plane.rs @@ -6,7 +6,7 @@ use std::{ }, }; -use crate::{WasiProcess, WasiProcessId}; +use crate::{runtime::module_cache::ModuleHash, WasiProcess, WasiProcessId}; #[derive(Debug, Clone)] pub struct WasiControlPlane { @@ -123,7 +123,7 @@ impl WasiControlPlane { /// Creates a new process // FIXME: De-register terminated processes! // Currently they just accumulate. - pub fn new_process(&self) -> Result { + pub fn new_process(&self, module_hash: ModuleHash) -> Result { if let Some(max) = self.state.config.max_task_count { if self.active_task_count() >= max { // NOTE: task count is not incremented here, only when new threads are spawned. @@ -133,7 +133,7 @@ impl WasiControlPlane { } // Create the process first to do all the allocations before locking. - let mut proc = WasiProcess::new(WasiProcessId::from(0), self.handle()); + let mut proc = WasiProcess::new(WasiProcessId::from(0), module_hash, self.handle()); let mut mutable = self.state.mutable.write().unwrap(); @@ -215,12 +215,12 @@ mod tests { enable_asynchronous_threading: false, }); - let p1 = p.new_process().unwrap(); + let p1 = p.new_process(ModuleHash::random()).unwrap(); let _t1 = p1.new_thread(WasiMemoryLayout::default()).unwrap(); let _t2 = p1.new_thread(WasiMemoryLayout::default()).unwrap(); assert_eq!( - p.new_process().unwrap_err(), + p.new_process(ModuleHash::random()).unwrap_err(), ControlPlaneError::TaskLimitReached { max: 2 } ); } @@ -233,7 +233,7 @@ mod tests { enable_asynchronous_threading: false, }); - let p1 = p.new_process().unwrap(); + let p1 = p.new_process(ModuleHash::random()).unwrap(); for _ in 0..10 { let _thread = p1.new_thread(WasiMemoryLayout::default()).unwrap(); @@ -243,7 +243,7 @@ mod tests { let _t2 = p1.new_thread(WasiMemoryLayout::default()).unwrap(); assert_eq!( - p.new_process().unwrap_err(), + p.new_process(ModuleHash::random()).unwrap_err(), ControlPlaneError::TaskLimitReached { max: 2 } ); } diff --git a/lib/wasix/src/os/task/process.rs b/lib/wasix/src/os/task/process.rs index a6950542b88..0b1ad096e32 100644 --- a/lib/wasix/src/os/task/process.rs +++ b/lib/wasix/src/os/task/process.rs @@ -1,6 +1,8 @@ #[cfg(feature = "journal")] use crate::{journaling::JournalEffector, unwind, WasiResult}; -use crate::{journaling::SnapshotTrigger, WasiEnv, WasiRuntimeError}; +use crate::{ + journaling::SnapshotTrigger, runtime::module_cache::ModuleHash, WasiEnv, WasiRuntimeError, +}; use serde::{Deserialize, Serialize}; use std::{ collections::HashMap, @@ -84,6 +86,8 @@ pub type LockableWasiProcessInner = Arc<(Mutex, Condvar)>; pub struct WasiProcess { /// Unique ID of this process pub(crate) pid: WasiProcessId, + /// Hash of the module that this process is using + pub(crate) module_hash: ModuleHash, /// List of all the children spawned from this thread pub(crate) parent: Option>>, /// The inner protected region of the process with a conditional @@ -277,9 +281,10 @@ impl Drop for WasiProcessWait { } impl WasiProcess { - pub fn new(pid: WasiProcessId, plane: WasiControlPlaneHandle) -> Self { + pub fn new(pid: WasiProcessId, module_hash: ModuleHash, plane: WasiControlPlaneHandle) -> Self { WasiProcess { pid, + module_hash, parent: None, compute: plane, inner: Arc::new(( diff --git a/lib/wasix/src/runners/wasi.rs b/lib/wasix/src/runners/wasi.rs index ef4eb554926..bf50e54defb 100644 --- a/lib/wasix/src/runners/wasi.rs +++ b/lib/wasix/src/runners/wasi.rs @@ -13,7 +13,7 @@ use crate::{ capabilities::Capabilities, journaling::{DynJournal, SnapshotTrigger}, runners::{wasi_common::CommonWasiOptions, MappedDirectory}, - runtime::task_manager::VirtualTaskManagerExt, + runtime::{module_cache::ModuleHash, task_manager::VirtualTaskManagerExt}, Runtime, WasiEnvBuilder, WasiRuntimeError, }; @@ -289,6 +289,7 @@ impl WasiRunner { runtime: Arc, program_name: &str, module: &Module, + module_hash: ModuleHash, asyncify: bool, ) -> Result<(), Error> { let wasi = webc::metadata::annotations::Wasi::new(program_name); @@ -296,9 +297,9 @@ impl WasiRunner { let env = self.prepare_webc_env(program_name, &wasi, None, runtime, None)?; if asyncify { - env.run_with_store_async(module.clone(), store)?; + env.run_with_store_async(module.clone(), module_hash, store)?; } else { - env.run_with_store(module.clone(), &mut store)?; + env.run_with_store(module.clone(), module_hash, &mut store)?; } Ok(()) diff --git a/lib/wasix/src/runners/wcgi/handler.rs b/lib/wasix/src/runners/wcgi/handler.rs index 2101142e3ca..44618179daa 100644 --- a/lib/wasix/src/runners/wcgi/handler.rs +++ b/lib/wasix/src/runners/wcgi/handler.rs @@ -10,8 +10,8 @@ use wasmer::Module; use wcgi_host::CgiDialect; use crate::{ - capabilities::Capabilities, http::HttpClientCapabilityV1, runners::wcgi::Callbacks, Pipe, - Runtime, VirtualTaskManager, WasiEnvBuilder, + capabilities::Capabilities, http::HttpClientCapabilityV1, runners::wcgi::Callbacks, + runtime::module_cache::ModuleHash, Pipe, Runtime, VirtualTaskManager, WasiEnvBuilder, }; /// The shared object that manages the instantiaion of WASI executables and @@ -59,6 +59,7 @@ impl Handler { }); let module = self.module.clone(); + let module_hash = self.module_hash.clone(); tracing::debug!( dialect=%self.dialect, @@ -71,7 +72,7 @@ impl Handler { let (run_tx, mut run_rx) = tokio::sync::mpsc::unbounded_channel(); task_manager.task_dedicated(Box::new(move || { run_tx - .send(builder.run_with_store_async(module, store)) + .send(builder.run_with_store_async(module, module_hash, store)) .ok(); }))?; let done = async move { run_rx.recv().await.unwrap().map_err(Error::from) }; @@ -203,6 +204,7 @@ type SetupBuilder = Box Result<(), anyhow::Error> #[derivative(Debug)] pub(crate) struct SharedState { pub(crate) module: Module, + pub(crate) module_hash: ModuleHash, pub(crate) dialect: CgiDialect, pub(crate) program_name: String, #[derivative(Debug = "ignore")] diff --git a/lib/wasix/src/runners/wcgi/runner.rs b/lib/wasix/src/runners/wcgi/runner.rs index 5022e826f9a..eff7b1b402d 100644 --- a/lib/wasix/src/runners/wcgi/runner.rs +++ b/lib/wasix/src/runners/wcgi/runner.rs @@ -75,6 +75,7 @@ impl WcgiRunner { let shared = SharedState { module, + module_hash: pkg.hash(), dialect, program_name: command_name.to_string(), setup_builder: Box::new(setup_builder), diff --git a/lib/wasix/src/runtime/module_cache/types.rs b/lib/wasix/src/runtime/module_cache/types.rs index 9d801007c12..fb8ad5ffae2 100644 --- a/lib/wasix/src/runtime/module_cache/types.rs +++ b/lib/wasix/src/runtime/module_cache/types.rs @@ -4,6 +4,7 @@ use std::{ path::PathBuf, }; +use rand::RngCore; use sha2::{Digest, Sha256}; use wasmer::{Engine, Module}; @@ -128,7 +129,15 @@ pub struct ModuleHash([u8; 32]); impl ModuleHash { /// Create a new [`ModuleHash`] from the raw SHA-256 hash. pub fn from_bytes(key: [u8; 32]) -> Self { - ModuleHash(key) + Self(key) + } + + // Creates a random hash for the module + pub fn random() -> Self { + let mut rand = rand::thread_rng(); + let mut key = [0u8; 32]; + rand.fill_bytes(&mut key); + Self(key) } /// Parse a sha256 hash from a hex-encoded string. diff --git a/lib/wasix/src/state/builder.rs b/lib/wasix/src/state/builder.rs index 87f04ebca8b..5fadc74365d 100644 --- a/lib/wasix/src/state/builder.rs +++ b/lib/wasix/src/state/builder.rs @@ -20,7 +20,7 @@ use crate::{ capabilities::Capabilities, fs::{WasiFs, WasiFsRoot, WasiInodes}, os::task::control_plane::{ControlPlaneConfig, ControlPlaneError, WasiControlPlane}, - runtime::task_manager::InlineWaker, + runtime::{module_cache::ModuleHash, task_manager::InlineWaker}, state::WasiState, syscalls::types::{__WASI_STDERR_FILENO, __WASI_STDIN_FILENO, __WASI_STDOUT_FILENO}, RewindState, Runtime, WasiEnv, WasiError, WasiFunctionEnv, WasiRuntimeError, @@ -887,20 +887,26 @@ impl WasiEnvBuilder { pub fn instantiate( self, module: Module, + module_hash: ModuleHash, store: &mut impl AsStoreMut, ) -> Result<(Instance, WasiFunctionEnv), WasiRuntimeError> { let init = self.build_init()?; - WasiEnv::instantiate(init, module, store) + WasiEnv::instantiate(init, module, module_hash, store) } #[allow(clippy::result_large_err)] - pub fn run(self, module: Module) -> Result<(), WasiRuntimeError> { + pub fn run(self, module: Module, module_hash: ModuleHash) -> Result<(), WasiRuntimeError> { let mut store = wasmer::Store::default(); - self.run_with_store(module, &mut store) + self.run_with_store(module, module_hash, &mut store) } #[allow(clippy::result_large_err)] - pub fn run_with_store(self, module: Module, store: &mut Store) -> Result<(), WasiRuntimeError> { + pub fn run_with_store( + self, + module: Module, + module_hash: ModuleHash, + store: &mut Store, + ) -> Result<(), WasiRuntimeError> { // If no handle or runtime exists then create one #[cfg(feature = "sys-thread")] let _guard = if tokio::runtime::Handle::try_current().is_err() { @@ -926,7 +932,7 @@ impl WasiEnvBuilder { #[cfg(not(feature = "journal"))] let journals = Vec::new(); - let (instance, env) = self.instantiate(module, store)?; + let (instance, env) = self.instantiate(module, module_hash, store)?; let start = instance.exports.get_function("_start")?; env.data(&store).thread.set_status_running(); @@ -944,7 +950,7 @@ impl WasiEnvBuilder { "main exit", ); - env.cleanup(store, Some(exit_code)); + env.on_exit(store, Some(exit_code)); result } @@ -954,6 +960,7 @@ impl WasiEnvBuilder { pub fn run_with_store_async( self, module: Module, + module_hash: ModuleHash, mut store: Store, ) -> Result<(), WasiRuntimeError> { // If no handle or runtime exists then create one @@ -973,7 +980,7 @@ impl WasiEnvBuilder { #[cfg(feature = "journal")] let journals = self.journals.clone(); - let (_, env) = self.instantiate(module, &mut store)?; + let (_, env) = self.instantiate(module, module_hash, &mut store)?; env.data(&store).thread.set_status_running(); @@ -981,10 +988,32 @@ impl WasiEnvBuilder { let mut rewind_state = None; #[cfg(feature = "journal")] - for journal in journals { - let ctx = env.env.clone().into_mut(&mut store); - let rewind = restore_snapshot(ctx, journal)?; - rewind_state = Some((rewind, None)); + { + env.data_mut(&mut store).replaying_journal = true; + + for journal in journals { + let ctx = env.env.clone().into_mut(&mut store); + let rewind = restore_snapshot(ctx, journal)?; + rewind_state = Some((rewind, None)); + } + + env.data_mut(&mut store).replaying_journal = false; + + // The first event we save is an event that records the module hash. + // Note: This is used to detect if an incorrect journal is used on the wrong + // process or if a process has been recompiled + let wasm_hash = env.data(&store).process.module_hash.as_bytes(); + let mut ctx = env.env.clone().into_mut(&mut store); + crate::journaling::JournalEffector::save_event( + &mut ctx, + crate::journaling::JournalEntry::InitModule { wasm_hash }, + ) + .map_err(|err| { + WasiRuntimeError::Runtime(RuntimeError::new(format!( + "journal failied to save the module initialization event - {}", + err + ))) + })?; } let tasks = env.data(&store).tasks().clone(); @@ -1073,7 +1102,7 @@ fn run_with_deep_sleep( if errno != Errno::Success { let exit_code = ExitCode::from(errno); - env.cleanup(&mut store, Some(exit_code)); + env.on_exit(&mut store, Some(exit_code)); let _ = sender.send(Err(WasiRuntimeError::Wasi(WasiError::Exit(exit_code)))); return; } @@ -1083,7 +1112,7 @@ fn run_with_deep_sleep( Some(instance) => instance, None => { tracing::debug!("Unable to clone the instance"); - env.cleanup(&mut store, None); + env.on_exit(&mut store, None); let _ = sender.send(Err(WasiRuntimeError::Wasi(WasiError::Exit( Errno::Noexec.into(), )))); @@ -1095,7 +1124,7 @@ fn run_with_deep_sleep( Ok(start) => start, Err(e) => { tracing::debug!("Unable to get the _start function"); - env.cleanup(&mut store, None); + env.on_exit(&mut store, None); let _ = sender.send(Err(e.into())); return; } @@ -1138,7 +1167,7 @@ fn handle_result( }; let (result, exit_code) = wasi_exit_code(result); - env.cleanup(&mut store, Some(exit_code)); + env.on_exit(&mut store, Some(exit_code)); sender.send(result).ok(); } diff --git a/lib/wasix/src/state/env.rs b/lib/wasix/src/state/env.rs index b6b01c6f988..43b52c6b968 100644 --- a/lib/wasix/src/state/env.rs +++ b/lib/wasix/src/state/env.rs @@ -34,7 +34,10 @@ use crate::{ process::{WasiProcess, WasiProcessId}, thread::{WasiMemoryLayout, WasiThread, WasiThreadHandle, WasiThreadId}, }, - runtime::{resolver::PackageSpecifier, task_manager::InlineWaker, SpawnMemoryType}, + runtime::{ + module_cache::ModuleHash, resolver::PackageSpecifier, task_manager::InlineWaker, + SpawnMemoryType, + }, syscalls::platform_clock_time_get, Runtime, VirtualTaskManager, WasiControlPlane, WasiEnvBuilder, WasiError, WasiFunctionEnv, WasiResult, WasiRuntimeError, WasiStateCreationError, WasiVFork, @@ -311,6 +314,10 @@ pub struct WasiEnv { /// Enables the snap shotting functionality pub enable_journal: bool, + /// Flag that indicatees if the environment is currently replaying the journal + /// (and hence it should not record new events) + pub replaying_journal: bool, + /// List of situations that the process will checkpoint on #[cfg(feature = "journal")] snapshot_on: HashSet, @@ -345,6 +352,7 @@ impl Clone for WasiEnv { capabilities: self.capabilities.clone(), enable_deep_sleep: self.enable_deep_sleep, enable_journal: self.enable_journal, + replaying_journal: self.replaying_journal, #[cfg(feature = "journal")] snapshot_on: self.snapshot_on.clone(), } @@ -359,7 +367,7 @@ impl WasiEnv { /// Forking the WasiState is used when either fork or vfork is called pub fn fork(&self) -> Result<(Self, WasiThreadHandle), ControlPlaneError> { - let process = self.control_plane.new_process()?; + let process = self.control_plane.new_process(self.process.module_hash)?; let handle = process.new_thread(self.layout.clone())?; let thread = handle.as_thread(); @@ -384,6 +392,7 @@ impl WasiEnv { capabilities: self.capabilities.clone(), enable_deep_sleep: self.enable_deep_sleep, enable_journal: self.enable_journal, + replaying_journal: false, #[cfg(feature = "journal")] snapshot_on: self.snapshot_on.clone(), }; @@ -425,7 +434,7 @@ impl WasiEnv { let process = if let Some(p) = init.process { p } else { - init.control_plane.new_process()? + init.control_plane.new_process(ModuleHash::random())? }; let layout = WasiMemoryLayout::default(); @@ -449,6 +458,7 @@ impl WasiEnv { enable_journal: init.runtime.active_journal().is_some(), #[cfg(not(feature = "journal"))] enable_journal: false, + replaying_journal: false, enable_deep_sleep: init.capabilities.threading.enable_asynchronous_threading, runtime: init.runtime, bin_factory: init.bin_factory, @@ -474,6 +484,7 @@ impl WasiEnv { pub(crate) fn instantiate( mut init: WasiEnvInit, module: Module, + module_hash: ModuleHash, store: &mut impl AsStoreMut, ) -> Result<(Instance, WasiFunctionEnv), WasiRuntimeError> { let call_initialize = init.call_initialize; @@ -485,7 +496,8 @@ impl WasiEnv { } } - let env = Self::from_init(init)?; + let mut env = Self::from_init(init)?; + env.process.module_hash = module_hash; let pid = env.process.pid(); @@ -530,7 +542,7 @@ impl WasiEnv { ); func_env .data(&store) - .blocking_cleanup(Some(Errno::Noexec.into())); + .blocking_on_exit(Some(Errno::Noexec.into())); return Err(err.into()); } }; @@ -549,7 +561,7 @@ impl WasiEnv { ); func_env .data(&store) - .blocking_cleanup(Some(Errno::Noexec.into())); + .blocking_on_exit(Some(Errno::Noexec.into())); return Err(err.into()); } @@ -559,7 +571,7 @@ impl WasiEnv { if let Err(err) = crate::run_wasi_func_start(initialize, &mut store, Vec::new()) { func_env .data(&store) - .blocking_cleanup(Some(Errno::Noexec.into())); + .blocking_on_exit(Some(Errno::Noexec.into())); return Err(err); } } @@ -856,7 +868,7 @@ impl WasiEnv { /// Returns true if the process should perform snapshots or not pub fn should_journal(&self) -> bool { - self.enable_journal + self.enable_journal && !self.replaying_journal } /// Returns the active journal or fails with an error @@ -1086,14 +1098,14 @@ impl WasiEnv { /// Cleans up all the open files (if this is the main thread) #[allow(clippy::await_holding_lock)] - pub fn blocking_cleanup(&self, exit_code: Option) { - let cleanup = self.cleanup(exit_code); + pub fn blocking_on_exit(&self, exit_code: Option) { + let cleanup = self.on_exit(exit_code); InlineWaker::block_on(cleanup); } /// Cleans up all the open files (if this is the main thread) #[allow(clippy::await_holding_lock)] - pub fn cleanup(&self, exit_code: Option) -> BoxFuture<'static, ()> { + pub fn on_exit(&self, exit_code: Option) -> BoxFuture<'static, ()> { const CLEANUP_TIMEOUT: Duration = Duration::from_secs(10); // If snap-shooting is enabled then we should record an event that the thread has exited. @@ -1102,6 +1114,12 @@ impl WasiEnv { if let Err(err) = JournalEffector::save_thread_exit(self, self.tid(), exit_code) { tracing::warn!("failed to save snapshot event for thread exit - {}", err); } + + if self.thread.is_main() { + if let Err(err) = JournalEffector::save_process_exit(self, exit_code) { + tracing::warn!("failed to save snapshot event for process exit - {}", err); + } + } } // If this is the main thread then also close all the files diff --git a/lib/wasix/src/state/func_env.rs b/lib/wasix/src/state/func_env.rs index 0101c521b07..0a3983e78f5 100644 --- a/lib/wasix/src/state/func_env.rs +++ b/lib/wasix/src/state/func_env.rs @@ -220,14 +220,14 @@ impl WasiFunctionEnv { /// This function should only be called from within a syscall /// as it can potentially execute local thread variable cleanup /// code - pub fn cleanup(&self, store: &mut impl AsStoreMut, exit_code: Option) { + pub fn on_exit(&self, store: &mut impl AsStoreMut, exit_code: Option) { trace!( - "wasi[{}:{}]::cleanup", + "wasi[{}:{}]::on_exit", self.data(store).pid(), self.data(store).tid() ); // Cleans up all the open files (if this is the main thread) - self.data(store).blocking_cleanup(exit_code); + self.data(store).blocking_on_exit(exit_code); } } diff --git a/lib/wasix/src/syscalls/mod.rs b/lib/wasix/src/syscalls/mod.rs index 5c5b5b69eca..bd4e48a1b18 100644 --- a/lib/wasix/src/syscalls/mod.rs +++ b/lib/wasix/src/syscalls/mod.rs @@ -1297,12 +1297,17 @@ pub fn restore_snapshot( journal: Arc, ) -> Result { InlineWaker::block_on(async { + let mut is_same_module = false; let mut rewind = None; while let Some(next) = journal.read().await.map_err(anyhow_err_to_runtime_err)? { tracing::trace!("Restoring snapshot event - {next:?}"); match next { - crate::journaling::JournalEntry::Init { .. } => { - // TODO: Here we need to check the hash matches the binary + crate::journaling::JournalEntry::InitModule { wasm_hash } => { + is_same_module = ctx.data().process.module_hash.as_bytes() == wasm_hash; + } + crate::journaling::JournalEntry::ProcessExit { exit_code } => { + JournalEffector::apply_process_exit(ctx.data(), exit_code) + .map_err(anyhow_err_to_runtime_err)?; } crate::journaling::JournalEntry::FileDescriptorWrite { fd, @@ -1325,6 +1330,9 @@ pub fn restore_snapshot( .map_err(anyhow_err_to_runtime_err)?; } crate::journaling::JournalEntry::UpdateMemoryRegion { region, data } => { + if is_same_module == false { + continue; + } JournalEffector::apply_memory(&mut ctx, region, &data) .map_err(anyhow_err_to_runtime_err)?; } @@ -1345,6 +1353,9 @@ pub fn restore_snapshot( store_data, is_64bit, } => { + if is_same_module == false { + continue; + } if id == ctx.data().tid() { rewind.replace(RewindState { memory_stack: memory_stack.to_vec().into(), @@ -1410,7 +1421,11 @@ pub fn restore_snapshot( crate::journaling::JournalEntry::Snapshot { when: _, trigger: _, - } => {} + } => { + if is_same_module == false { + continue; + } + } crate::journaling::JournalEntry::SetClockTime { clock_id, time } => { JournalEffector::apply_clock_time_set(&mut ctx, clock_id, time) .map_err(anyhow_err_to_runtime_err)?; @@ -1545,14 +1560,20 @@ pub fn restore_snapshot( ) .map_err(anyhow_err_to_runtime_err)?; } - crate::journaling::JournalEntry::Panic { stack_trace, .. } => { - return Err(WasiRuntimeError::Runtime(RuntimeError::new(format!( - "panic\r\nstack: {}", - stack_trace - )))); - } } } + // If we are not in the same module then we fire off an exit + // that simulates closing the process (hence keeps everything + // in a clean state) + if !is_same_module { + JournalEffector::apply_process_exit(ctx.data(), None) + .map_err(anyhow_err_to_runtime_err)?; + } else { + tracing::debug!( + "journal used on a different module - the process will simulate a restart." + ); + } + rewind.ok_or(WasiRuntimeError::Runtime(RuntimeError::user(anyhow::format_err!("The restored snapshot journal does not have a thread stack events and hence we can not restore the state of the process.").into()))) }) } diff --git a/lib/wasix/src/syscalls/wasix/proc_fork.rs b/lib/wasix/src/syscalls/wasix/proc_fork.rs index 32a03599423..9b87ee9b978 100644 --- a/lib/wasix/src/syscalls/wasix/proc_fork.rs +++ b/lib/wasix/src/syscalls/wasix/proc_fork.rs @@ -306,7 +306,7 @@ fn run( trace!(%pid, %tid, "child exited (code = {})", ret); // Clean up the environment and return the result - ctx.cleanup((&mut store), Some(ret)); + ctx.on_exit((&mut store), Some(ret)); // We drop the handle at the last moment which will close the thread drop(child_handle); diff --git a/lib/wasix/src/syscalls/wasix/thread_spawn.rs b/lib/wasix/src/syscalls/wasix/thread_spawn.rs index 5735abaceaa..048c35c8514 100644 --- a/lib/wasix/src/syscalls/wasix/thread_spawn.rs +++ b/lib/wasix/src/syscalls/wasix/thread_spawn.rs @@ -197,7 +197,7 @@ fn call_module( trace!("callback finished (ret={})", ret); // Clean up the environment - env.cleanup(store, Some(ret.into())); + env.on_exit(store, Some(ret.into())); // Return the result Ok(ret as u32) diff --git a/tests/lib/wast/src/wasi_wast.rs b/tests/lib/wast/src/wasi_wast.rs index 8354ea775da..c741d2a8638 100644 --- a/tests/lib/wast/src/wasi_wast.rs +++ b/tests/lib/wast/src/wasi_wast.rs @@ -14,7 +14,10 @@ use virtual_fs::{ AsyncWriteExt, FileSystem, Pipe, ReadBuf, RootFileSystemBuilder, }; use wasmer::{FunctionEnv, Imports, Module, Store}; -use wasmer_wasix::runtime::task_manager::{tokio::TokioTaskManager, InlineWaker}; +use wasmer_wasix::runtime::{ + module_cache::ModuleHash, + task_manager::{tokio::TokioTaskManager, InlineWaker}, +}; use wasmer_wasix::types::wasi::{Filesize, Timestamp}; use wasmer_wasix::{ generate_import_object_from_env, get_wasi_version, FsError, PluggableRuntime, VirtualFile, @@ -119,12 +122,16 @@ impl<'a> WasiTest<'a> { wasm_module.read_to_end(&mut out)?; out }; + let module_hash = ModuleHash::sha256(&wasm_bytes); let module = Module::new(store, wasm_bytes)?; let (builder, _tempdirs, mut stdin_tx, stdout_rx, stderr_rx) = { InlineWaker::block_on(async { self.create_wasi_env(filesystem_kind).await }) }?; - let (instance, _wasi_env) = builder.runtime(Arc::new(rt)).instantiate(module, store)?; + let (instance, _wasi_env) = + builder + .runtime(Arc::new(rt)) + .instantiate(module, module_hash, store)?; let start = instance.exports.get_function("_start")?; From 85d4b6d6b7a94468a59e040822a4f3d850c3867d Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Mon, 20 Nov 2023 17:01:06 +1100 Subject: [PATCH 047/129] Fixed some linting issues --- lib/wasix/src/runners/wcgi/handler.rs | 2 +- lib/wasix/src/syscalls/mod.rs | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/wasix/src/runners/wcgi/handler.rs b/lib/wasix/src/runners/wcgi/handler.rs index 44618179daa..f3aee79e3eb 100644 --- a/lib/wasix/src/runners/wcgi/handler.rs +++ b/lib/wasix/src/runners/wcgi/handler.rs @@ -59,7 +59,7 @@ impl Handler { }); let module = self.module.clone(); - let module_hash = self.module_hash.clone(); + let module_hash = self.module_hash; tracing::debug!( dialect=%self.dialect, diff --git a/lib/wasix/src/syscalls/mod.rs b/lib/wasix/src/syscalls/mod.rs index bd4e48a1b18..18cac2b3b7f 100644 --- a/lib/wasix/src/syscalls/mod.rs +++ b/lib/wasix/src/syscalls/mod.rs @@ -1330,7 +1330,7 @@ pub fn restore_snapshot( .map_err(anyhow_err_to_runtime_err)?; } crate::journaling::JournalEntry::UpdateMemoryRegion { region, data } => { - if is_same_module == false { + if !is_same_module { continue; } JournalEffector::apply_memory(&mut ctx, region, &data) @@ -1353,7 +1353,7 @@ pub fn restore_snapshot( store_data, is_64bit, } => { - if is_same_module == false { + if !is_same_module { continue; } if id == ctx.data().tid() { @@ -1422,7 +1422,7 @@ pub fn restore_snapshot( when: _, trigger: _, } => { - if is_same_module == false { + if !is_same_module { continue; } } From 0dda34cdc2b0a6eeb96df539fae05360f6802818 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Mon, 20 Nov 2023 17:02:49 +1100 Subject: [PATCH 048/129] Fixed some unit tests --- lib/wasix/tests/stdio.rs | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/lib/wasix/tests/stdio.rs b/lib/wasix/tests/stdio.rs index 9875aae75b9..22b4051fa56 100644 --- a/lib/wasix/tests/stdio.rs +++ b/lib/wasix/tests/stdio.rs @@ -1,6 +1,6 @@ use virtual_fs::{AsyncReadExt, AsyncWriteExt}; use wasmer::{Module, Store}; -use wasmer_wasix::{Pipe, WasiEnv}; +use wasmer_wasix::{runtime::module_cache::ModuleHash, Pipe, WasiEnv}; mod sys { #[tokio::test] @@ -80,14 +80,18 @@ async fn test_stdout() { #[cfg(feature = "js")] { - builder.run_with_store(module, &mut store).unwrap(); + builder + .run_with_store(module, ModuleHash::random(), &mut store) + .unwrap(); } #[cfg(not(feature = "js"))] { - std::thread::spawn(move || builder.run_with_store(module, &mut store)) - .join() - .unwrap() - .unwrap(); + std::thread::spawn(move || { + builder.run_with_store(module, ModuleHash::random(), &mut store) + }) + .join() + .unwrap() + .unwrap(); } let mut stdout_str = String::new(); From b53957248e29b5523f04b7740435ead192fd7625 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Mon, 20 Nov 2023 17:09:19 +1100 Subject: [PATCH 049/129] Fixed a clippy error --- lib/cli/src/commands/run/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cli/src/commands/run/mod.rs b/lib/cli/src/commands/run/mod.rs index 3d4dce6a8ff..34d79105aba 100644 --- a/lib/cli/src/commands/run/mod.rs +++ b/lib/cli/src/commands/run/mod.rs @@ -671,7 +671,7 @@ impl ExecutableTarget { let module = unsafe { Module::deserialize_from_file(&engine, path)? }; let module_hash = { let wasm = std::fs::read(path)?; - ModuleHash::sha256(&wasm) + ModuleHash::sha256(wasm) }; Ok(ExecutableTarget::WebAssembly { From e80ddb4b139a5c6565b20ad8f95854678941f873 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Mon, 20 Nov 2023 17:20:39 +1100 Subject: [PATCH 050/129] Fix for some unit tests --- lib/wasix/tests/stdio.rs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/lib/wasix/tests/stdio.rs b/lib/wasix/tests/stdio.rs index 22b4051fa56..7f796371de5 100644 --- a/lib/wasix/tests/stdio.rs +++ b/lib/wasix/tests/stdio.rs @@ -102,7 +102,9 @@ async fn test_stdout() { async fn test_env() { let mut store = Store::default(); - let module = Module::new(&store, include_bytes!("envvar.wasm")).unwrap(); + let module_bytes = include_bytes!("envvar.wasm"); + let module = Module::new(&store, module_bytes).unwrap(); + let module_hash = ModuleHash::sha256(module_bytes); #[cfg(feature = "js")] tracing_wasm::set_as_global_default_with_config({ @@ -128,7 +130,7 @@ async fn test_env() { #[cfg(not(feature = "js"))] { - std::thread::spawn(move || builder.run_with_store(module, &mut store)) + std::thread::spawn(move || builder.run_with_store(module, module_hash, &mut store)) .join() .unwrap() .unwrap(); @@ -142,7 +144,9 @@ async fn test_env() { async fn test_stdin() { let mut store = Store::default(); - let module = Module::new(&store, include_bytes!("stdin-hello.wasm")).unwrap(); + let module_bytes = include_bytes!("stdin-hello.wasm"); + let module = Module::new(&store, module_bytes).unwrap(); + let module_hash = ModuleHash::sha256(module_bytes); // Create the `WasiEnv`. let (mut pipe_tx, pipe_rx) = Pipe::channel(); @@ -162,7 +166,7 @@ async fn test_stdin() { #[cfg(not(feature = "js"))] { - std::thread::spawn(move || builder.run_with_store(module, &mut store)) + std::thread::spawn(move || builder.run_with_store(module, module_hash, &mut store)) .join() .unwrap() .unwrap(); From e6e347817e724c513beff1d317c22cbfc4047514 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Mon, 20 Nov 2023 17:31:38 +1100 Subject: [PATCH 051/129] Made the run commands backwards compatible --- examples/wasi.rs | 2 +- examples/wasi_pipes.rs | 4 ++-- lib/cli/src/commands/run/wasi.rs | 2 +- lib/wasix/src/runners/wasi.rs | 2 +- lib/wasix/src/state/builder.rs | 29 ++++++++++++++++++++++++----- lib/wasix/tests/stdio.rs | 6 +++--- tests/lib/wast/src/wasi_wast.rs | 2 +- 7 files changed, 33 insertions(+), 14 deletions(-) diff --git a/examples/wasi.rs b/examples/wasi.rs index c055d356a80..957125a803a 100644 --- a/examples/wasi.rs +++ b/examples/wasi.rs @@ -41,7 +41,7 @@ fn main() -> Result<(), Box> { // .args(&["world"]) // .env("KEY", "Value") .stdout(Box::new(stdout_tx)) - .run_with_store(module, &mut store)?; + .run_with_store_ext(module, &mut store)?; eprintln!("Run complete - reading output"); diff --git a/examples/wasi_pipes.rs b/examples/wasi_pipes.rs index 925c8801959..3f84acaded7 100644 --- a/examples/wasi_pipes.rs +++ b/examples/wasi_pipes.rs @@ -13,7 +13,7 @@ use std::io::{Read, Write}; use wasmer::{Module, Store}; -use wasmer_wasix::{Pipe, WasiEnv}; +use wasmer_wasix::{runtime::module_cache::ModuleHash, Pipe, WasiEnv}; fn main() -> Result<(), Box> { let wasm_path = concat!( @@ -43,7 +43,7 @@ fn main() -> Result<(), Box> { WasiEnv::builder("hello") .stdin(Box::new(stdin_reader)) .stdout(Box::new(stdout_sender)) - .run_with_store(module, &mut store)?; + .run_with_store_ext(module, module_hash, &mut store)?; // To read from the stdout let mut buf = String::new(); diff --git a/lib/cli/src/commands/run/wasi.rs b/lib/cli/src/commands/run/wasi.rs index bdf55fb0429..0ebbbbf8473 100644 --- a/lib/cli/src/commands/run/wasi.rs +++ b/lib/cli/src/commands/run/wasi.rs @@ -552,7 +552,7 @@ impl Wasi { store: &mut Store, ) -> Result<(WasiFunctionEnv, Instance)> { let builder = self.prepare(module, program_name, args, runtime)?; - let (instance, wasi_env) = builder.instantiate(module.clone(), module_hash, store)?; + let (instance, wasi_env) = builder.instantiate_ext(module.clone(), module_hash, store)?; Ok((wasi_env, instance)) } diff --git a/lib/wasix/src/runners/wasi.rs b/lib/wasix/src/runners/wasi.rs index bf50e54defb..b351f38db18 100644 --- a/lib/wasix/src/runners/wasi.rs +++ b/lib/wasix/src/runners/wasi.rs @@ -299,7 +299,7 @@ impl WasiRunner { if asyncify { env.run_with_store_async(module.clone(), module_hash, store)?; } else { - env.run_with_store(module.clone(), module_hash, &mut store)?; + env.run_with_store_ext(module.clone(), module_hash, &mut store)?; } Ok(()) diff --git a/lib/wasix/src/state/builder.rs b/lib/wasix/src/state/builder.rs index 5fadc74365d..827c52ba936 100644 --- a/lib/wasix/src/state/builder.rs +++ b/lib/wasix/src/state/builder.rs @@ -885,6 +885,15 @@ impl WasiEnvBuilder { // FIXME: use a proper custom error type #[allow(clippy::result_large_err)] pub fn instantiate( + self, + module: Module, + store: &mut impl AsStoreMut, + ) -> Result<(Instance, WasiFunctionEnv), WasiRuntimeError> { + self.instantiate_ext(module, ModuleHash::random(), store) + } + + #[allow(clippy::result_large_err)] + pub fn instantiate_ext( self, module: Module, module_hash: ModuleHash, @@ -895,13 +904,23 @@ impl WasiEnvBuilder { } #[allow(clippy::result_large_err)] - pub fn run(self, module: Module, module_hash: ModuleHash) -> Result<(), WasiRuntimeError> { + pub fn run(self, module: Module) -> Result<(), WasiRuntimeError> { + self.run_ext(module, ModuleHash::random()) + } + + #[allow(clippy::result_large_err)] + pub fn run_ext(self, module: Module, module_hash: ModuleHash) -> Result<(), WasiRuntimeError> { let mut store = wasmer::Store::default(); - self.run_with_store(module, module_hash, &mut store) + self.run_with_store_ext(module, module_hash, &mut store) + } + + #[allow(clippy::result_large_err)] + pub fn run_with_store(self, module: Module, store: &mut Store) -> Result<(), WasiRuntimeError> { + self.run_with_store_ext(module, ModuleHash::random(), store) } #[allow(clippy::result_large_err)] - pub fn run_with_store( + pub fn run_with_store_ext( self, module: Module, module_hash: ModuleHash, @@ -932,7 +951,7 @@ impl WasiEnvBuilder { #[cfg(not(feature = "journal"))] let journals = Vec::new(); - let (instance, env) = self.instantiate(module, module_hash, store)?; + let (instance, env) = self.instantiate_ext(module, module_hash, store)?; let start = instance.exports.get_function("_start")?; env.data(&store).thread.set_status_running(); @@ -980,7 +999,7 @@ impl WasiEnvBuilder { #[cfg(feature = "journal")] let journals = self.journals.clone(); - let (_, env) = self.instantiate(module, module_hash, &mut store)?; + let (_, env) = self.instantiate_ext(module, module_hash, &mut store)?; env.data(&store).thread.set_status_running(); diff --git a/lib/wasix/tests/stdio.rs b/lib/wasix/tests/stdio.rs index 7f796371de5..804558a8319 100644 --- a/lib/wasix/tests/stdio.rs +++ b/lib/wasix/tests/stdio.rs @@ -87,7 +87,7 @@ async fn test_stdout() { #[cfg(not(feature = "js"))] { std::thread::spawn(move || { - builder.run_with_store(module, ModuleHash::random(), &mut store) + builder.run_with_store_ext(module, ModuleHash::random(), &mut store) }) .join() .unwrap() @@ -130,7 +130,7 @@ async fn test_env() { #[cfg(not(feature = "js"))] { - std::thread::spawn(move || builder.run_with_store(module, module_hash, &mut store)) + std::thread::spawn(move || builder.run_with_store_ext(module, module_hash, &mut store)) .join() .unwrap() .unwrap(); @@ -166,7 +166,7 @@ async fn test_stdin() { #[cfg(not(feature = "js"))] { - std::thread::spawn(move || builder.run_with_store(module, module_hash, &mut store)) + std::thread::spawn(move || builder.run_with_store_ext(module, module_hash, &mut store)) .join() .unwrap() .unwrap(); diff --git a/tests/lib/wast/src/wasi_wast.rs b/tests/lib/wast/src/wasi_wast.rs index c741d2a8638..e3bf84d8e10 100644 --- a/tests/lib/wast/src/wasi_wast.rs +++ b/tests/lib/wast/src/wasi_wast.rs @@ -131,7 +131,7 @@ impl<'a> WasiTest<'a> { let (instance, _wasi_env) = builder .runtime(Arc::new(rt)) - .instantiate(module, module_hash, store)?; + .instantiate_ext(module, module_hash, store)?; let start = instance.exports.get_function("_start")?; From a4bc07124ff1405a8dd85bbc3c04cc877c8cfa8b Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Mon, 20 Nov 2023 17:32:47 +1100 Subject: [PATCH 052/129] Undid the example refactor --- examples/wasi.rs | 2 +- examples/wasi_pipes.rs | 4 ++-- lib/wasix/tests/stdio.rs | 28 ++++++++++------------------ 3 files changed, 13 insertions(+), 21 deletions(-) diff --git a/examples/wasi.rs b/examples/wasi.rs index 957125a803a..c055d356a80 100644 --- a/examples/wasi.rs +++ b/examples/wasi.rs @@ -41,7 +41,7 @@ fn main() -> Result<(), Box> { // .args(&["world"]) // .env("KEY", "Value") .stdout(Box::new(stdout_tx)) - .run_with_store_ext(module, &mut store)?; + .run_with_store(module, &mut store)?; eprintln!("Run complete - reading output"); diff --git a/examples/wasi_pipes.rs b/examples/wasi_pipes.rs index 3f84acaded7..925c8801959 100644 --- a/examples/wasi_pipes.rs +++ b/examples/wasi_pipes.rs @@ -13,7 +13,7 @@ use std::io::{Read, Write}; use wasmer::{Module, Store}; -use wasmer_wasix::{runtime::module_cache::ModuleHash, Pipe, WasiEnv}; +use wasmer_wasix::{Pipe, WasiEnv}; fn main() -> Result<(), Box> { let wasm_path = concat!( @@ -43,7 +43,7 @@ fn main() -> Result<(), Box> { WasiEnv::builder("hello") .stdin(Box::new(stdin_reader)) .stdout(Box::new(stdout_sender)) - .run_with_store_ext(module, module_hash, &mut store)?; + .run_with_store(module, &mut store)?; // To read from the stdout let mut buf = String::new(); diff --git a/lib/wasix/tests/stdio.rs b/lib/wasix/tests/stdio.rs index 804558a8319..9875aae75b9 100644 --- a/lib/wasix/tests/stdio.rs +++ b/lib/wasix/tests/stdio.rs @@ -1,6 +1,6 @@ use virtual_fs::{AsyncReadExt, AsyncWriteExt}; use wasmer::{Module, Store}; -use wasmer_wasix::{runtime::module_cache::ModuleHash, Pipe, WasiEnv}; +use wasmer_wasix::{Pipe, WasiEnv}; mod sys { #[tokio::test] @@ -80,18 +80,14 @@ async fn test_stdout() { #[cfg(feature = "js")] { - builder - .run_with_store(module, ModuleHash::random(), &mut store) - .unwrap(); + builder.run_with_store(module, &mut store).unwrap(); } #[cfg(not(feature = "js"))] { - std::thread::spawn(move || { - builder.run_with_store_ext(module, ModuleHash::random(), &mut store) - }) - .join() - .unwrap() - .unwrap(); + std::thread::spawn(move || builder.run_with_store(module, &mut store)) + .join() + .unwrap() + .unwrap(); } let mut stdout_str = String::new(); @@ -102,9 +98,7 @@ async fn test_stdout() { async fn test_env() { let mut store = Store::default(); - let module_bytes = include_bytes!("envvar.wasm"); - let module = Module::new(&store, module_bytes).unwrap(); - let module_hash = ModuleHash::sha256(module_bytes); + let module = Module::new(&store, include_bytes!("envvar.wasm")).unwrap(); #[cfg(feature = "js")] tracing_wasm::set_as_global_default_with_config({ @@ -130,7 +124,7 @@ async fn test_env() { #[cfg(not(feature = "js"))] { - std::thread::spawn(move || builder.run_with_store_ext(module, module_hash, &mut store)) + std::thread::spawn(move || builder.run_with_store(module, &mut store)) .join() .unwrap() .unwrap(); @@ -144,9 +138,7 @@ async fn test_env() { async fn test_stdin() { let mut store = Store::default(); - let module_bytes = include_bytes!("stdin-hello.wasm"); - let module = Module::new(&store, module_bytes).unwrap(); - let module_hash = ModuleHash::sha256(module_bytes); + let module = Module::new(&store, include_bytes!("stdin-hello.wasm")).unwrap(); // Create the `WasiEnv`. let (mut pipe_tx, pipe_rx) = Pipe::channel(); @@ -166,7 +158,7 @@ async fn test_stdin() { #[cfg(not(feature = "js"))] { - std::thread::spawn(move || builder.run_with_store_ext(module, module_hash, &mut store)) + std::thread::spawn(move || builder.run_with_store(module, &mut store)) .join() .unwrap() .unwrap(); From 8a9d5a860068dddb27408eebfa196e4d16ffabad Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Tue, 21 Nov 2023 06:53:21 +1100 Subject: [PATCH 053/129] Added events for ports and sockets --- lib/wasi-types/src/wasi/bindings.rs | 2 +- lib/wasix/src/journaling/filter.rs | 264 ++++--------- lib/wasix/src/journaling/journal.rs | 110 +++++- lib/wasix/src/journaling/log_file.rs | 540 ++++++++++++++++++++++++++- 4 files changed, 719 insertions(+), 197 deletions(-) diff --git a/lib/wasi-types/src/wasi/bindings.rs b/lib/wasi-types/src/wasi/bindings.rs index 47e4a668428..bbf707997c6 100644 --- a/lib/wasi-types/src/wasi/bindings.rs +++ b/lib/wasi-types/src/wasi/bindings.rs @@ -1351,7 +1351,7 @@ impl core::fmt::Debug for StdioMode { } } #[repr(u16)] -#[derive(Clone, Copy, PartialEq, Eq)] +#[derive(Clone, Copy, PartialEq, Eq, IntoPrimitive, TryFromPrimitive)] pub enum SockProto { Ip, Icmp, diff --git a/lib/wasix/src/journaling/filter.rs b/lib/wasix/src/journaling/filter.rs index e55fb9e617b..bdba9226596 100644 --- a/lib/wasix/src/journaling/filter.rs +++ b/lib/wasix/src/journaling/filter.rs @@ -9,13 +9,10 @@ pub struct FilteredJournal { inner: Box, filter_memory: bool, filter_threads: bool, - filter_files: bool, - filter_chdir: bool, - filter_clock: bool, - filter_terminal: bool, + filter_fs: bool, + filter_core: bool, filter_snapshots: bool, - filter_descriptors: bool, - filter_epoll: bool, + filter_net: bool, } impl FilteredJournal { @@ -24,13 +21,10 @@ impl FilteredJournal { inner, filter_memory: false, filter_threads: false, - filter_files: false, - filter_chdir: false, - filter_clock: false, - filter_terminal: false, + filter_fs: false, + filter_core: false, filter_snapshots: false, - filter_descriptors: false, - filter_epoll: false, + filter_net: false, } } @@ -44,28 +38,13 @@ impl FilteredJournal { self } - pub fn with_ignore_files(mut self, val: bool) -> Self { - self.filter_files = val; + pub fn with_ignore_fs(mut self, val: bool) -> Self { + self.filter_fs = val; self } - pub fn with_ignore_chdir(mut self, val: bool) -> Self { - self.filter_chdir = val; - self - } - - pub fn with_ignore_clock(mut self, val: bool) -> Self { - self.filter_clock = val; - self - } - - pub fn with_ignore_epoll(mut self, val: bool) -> Self { - self.filter_epoll = val; - self - } - - pub fn with_ignore_terminal(mut self, val: bool) -> Self { - self.filter_terminal = val; + pub fn with_ignore_core(mut self, val: bool) -> Self { + self.filter_core = val; self } @@ -74,8 +53,8 @@ impl FilteredJournal { self } - pub fn with_ignore_descriptors(mut self, val: bool) -> Self { - self.filter_descriptors = val; + pub fn with_ignore_networking(mut self, val: bool) -> Self { + self.filter_net = val; self } } @@ -84,74 +63,51 @@ impl Journal for FilteredJournal { fn write<'a>(&'a self, entry: JournalEntry<'a>) -> LocalBoxFuture<'a, anyhow::Result<()>> { Box::pin(async { let evt = match entry { - JournalEntry::InitModule { .. } => { - if self.filter_threads { - return Ok(()); - } - entry - } - JournalEntry::UpdateMemoryRegion { .. } => { - if self.filter_memory { - return Ok(()); - } - entry - } - JournalEntry::ProcessExit { .. } => { - if self.filter_threads { + JournalEntry::SetClockTime { .. } + | JournalEntry::InitModule { .. } + | JournalEntry::ProcessExit { .. } + | JournalEntry::EpollCreate { .. } + | JournalEntry::EpollCtl { .. } + | JournalEntry::TtySet { .. } => { + if self.filter_core { return Ok(()); } entry } - JournalEntry::SetThread { .. } => { + JournalEntry::SetThread { .. } | JournalEntry::CloseThread { .. } => { if self.filter_threads { return Ok(()); } entry } - JournalEntry::CloseThread { .. } => { - if self.filter_threads { - return Ok(()); - } - entry - } - JournalEntry::FileDescriptorSeek { .. } => { - if self.filter_descriptors { - return Ok(()); - } - entry - } - JournalEntry::FileDescriptorWrite { .. } => { - if self.filter_descriptors { - return Ok(()); - } - entry - } - JournalEntry::OpenFileDescriptor { .. } => { - if self.filter_descriptors { - return Ok(()); - } - entry - } - JournalEntry::CloseFileDescriptor { .. } => { - if self.filter_descriptors { - return Ok(()); - } - entry - } - JournalEntry::RemoveDirectory { .. } => { - if self.filter_files { - return Ok(()); - } - entry - } - JournalEntry::UnlinkFile { .. } => { - if self.filter_files { + JournalEntry::UpdateMemoryRegion { .. } => { + if self.filter_memory { return Ok(()); } entry } - JournalEntry::PathRename { .. } => { - if self.filter_files { + JournalEntry::FileDescriptorSeek { .. } + | JournalEntry::FileDescriptorWrite { .. } + | JournalEntry::OpenFileDescriptor { .. } + | JournalEntry::CloseFileDescriptor { .. } + | JournalEntry::RemoveDirectory { .. } + | JournalEntry::UnlinkFile { .. } + | JournalEntry::PathRename { .. } + | JournalEntry::RenumberFileDescriptor { .. } + | JournalEntry::DuplicateFileDescriptor { .. } + | JournalEntry::CreateDirectory { .. } + | JournalEntry::PathSetTimes { .. } + | JournalEntry::FileDescriptorSetFlags { .. } + | JournalEntry::FileDescriptorAdvise { .. } + | JournalEntry::FileDescriptorAllocate { .. } + | JournalEntry::FileDescriptorSetRights { .. } + | JournalEntry::FileDescriptorSetTimes { .. } + | JournalEntry::FileDescriptorSetSize { .. } + | JournalEntry::CreateHardLink { .. } + | JournalEntry::CreateSymbolicLink { .. } + | JournalEntry::ChangeDirectory { .. } + | JournalEntry::CreatePipe { .. } => { + if self.filter_fs { return Ok(()); } entry @@ -162,111 +118,33 @@ impl Journal for FilteredJournal { } entry } - JournalEntry::SetClockTime { .. } => { - if self.filter_clock { - return Ok(()); - } - entry - } - JournalEntry::RenumberFileDescriptor { .. } => { - if self.filter_terminal { - return Ok(()); - } - entry - } - JournalEntry::DuplicateFileDescriptor { .. } => { - if self.filter_terminal { - return Ok(()); - } - entry - } - JournalEntry::CreateDirectory { .. } => { - if self.filter_files { - return Ok(()); - } - entry - } - JournalEntry::PathSetTimes { .. } => { - if self.filter_files { - return Ok(()); - } - entry - } - JournalEntry::FileDescriptorSetFlags { .. } => { - if self.filter_descriptors { - return Ok(()); - } - entry - } - JournalEntry::FileDescriptorAdvise { .. } => { - if self.filter_descriptors { - return Ok(()); - } - entry - } - JournalEntry::FileDescriptorAllocate { .. } => { - if self.filter_descriptors { - return Ok(()); - } - entry - } - - JournalEntry::FileDescriptorSetRights { .. } => { - if self.filter_descriptors { - return Ok(()); - } - entry - } - JournalEntry::FileDescriptorSetTimes { .. } => { - if self.filter_descriptors { - return Ok(()); - } - entry - } - JournalEntry::FileDescriptorSetSize { .. } => { - if self.filter_descriptors { - return Ok(()); - } - entry - } - JournalEntry::CreateHardLink { .. } => { - if self.filter_files { - return Ok(()); - } - entry - } - JournalEntry::CreateSymbolicLink { .. } => { - if self.filter_files { - return Ok(()); - } - entry - } - JournalEntry::ChangeDirectory { .. } => { - if self.filter_chdir { - return Ok(()); - } - entry - } - JournalEntry::EpollCreate { .. } => { - if self.filter_memory { - return Ok(()); - } - entry - } - JournalEntry::EpollCtl { .. } => { - if self.filter_memory { - return Ok(()); - } - entry - } - JournalEntry::TtySet { .. } => { - if self.filter_terminal { - return Ok(()); - } - entry - } - JournalEntry::CreatePipe { .. } => { - if self.filter_files { + JournalEntry::PortAddAddr { .. } + | JournalEntry::PortDelAddr { .. } + | JournalEntry::PortAddrClear + | JournalEntry::PortBridge { .. } + | JournalEntry::PortUnbridge + | JournalEntry::PortDhcpAcquire + | JournalEntry::PortGatewaySet { .. } + | JournalEntry::PortRouteAdd { .. } + | JournalEntry::PortRouteClear + | JournalEntry::PortRouteDel { .. } + | JournalEntry::SocketOpen { .. } + | JournalEntry::SocketListen { .. } + | JournalEntry::SocketBind { .. } + | JournalEntry::SocketConnect { .. } + | JournalEntry::SocketAccept { .. } + | JournalEntry::SocketJoinIpv4Multicast { .. } + | JournalEntry::SocketJoinIpv6Multicast { .. } + | JournalEntry::SocketLeaveIpv4Multicast { .. } + | JournalEntry::SocketLeaveIpv6Multicast { .. } + | JournalEntry::SocketSendFile { .. } + | JournalEntry::SocketSendTo { .. } + | JournalEntry::SocketSend { .. } + | JournalEntry::SocketSetOptFlag { .. } + | JournalEntry::SocketSetOptSize { .. } + | JournalEntry::SocketSetOptTime { .. } + | JournalEntry::SocketShutdown { .. } => { + if self.filter_net { return Ok(()); } entry diff --git a/lib/wasix/src/journaling/journal.rs b/lib/wasix/src/journaling/journal.rs index 95c51fe8a48..7d5bf1df265 100644 --- a/lib/wasix/src/journaling/journal.rs +++ b/lib/wasix/src/journaling/journal.rs @@ -2,9 +2,11 @@ use serde::{Deserialize, Serialize}; use std::net::SocketAddr; use std::time::SystemTime; use std::{borrow::Cow, ops::Range}; +use virtual_net::{IpAddr, IpCidr, Ipv4Addr, Ipv6Addr}; use wasmer_wasix_types::wasi::{ - Advice, EpollCtl, EpollEventCtl, ExitCode, Fdflags, FileDelta, Filesize, Fstflags, LookupFlags, - Oflags, Rights, Snapshot0Clockid, Timestamp, Tty, Whence, + Addressfamily, Advice, EpollCtl, EpollEventCtl, ExitCode, Fdflags, FileDelta, Filesize, + Fstflags, LookupFlags, Oflags, Rights, SdFlags, SiFlags, Snapshot0Clockid, SockProto, + Sockoption, Socktype, Streamsecurity, Timestamp, Tty, Whence, }; use futures::future::LocalBoxFuture; @@ -186,6 +188,110 @@ pub enum JournalEntry<'a> { fd1: Fd, fd2: Fd, }, + PortAddAddr { + cidr: IpCidr, + }, + PortDelAddr { + addr: IpAddr, + }, + PortAddrClear, + PortBridge { + network: String, + token: String, + security: Streamsecurity, + }, + PortUnbridge, + PortDhcpAcquire, + PortGatewaySet { + ip: IpAddr, + }, + PortRouteAdd { + cidr: IpCidr, + via_router: IpAddr, + preferred_until: Option, + expires_at: Option, + }, + PortRouteClear, + PortRouteDel { + ip: IpAddr, + }, + SocketOpen { + af: Addressfamily, + ty: Socktype, + pt: SockProto, + fd: Fd, + }, + SocketListen { + fd: Fd, + backlog: u32, + }, + SocketBind { + fd: Fd, + addr: SocketAddr, + }, + SocketConnect { + fd: Fd, + addr: SocketAddr, + }, + SocketAccept { + listen_fd: Fd, + fd: Fd, + peer_addr: SocketAddr, + }, + SocketJoinIpv4Multicast { + fd: Fd, + multiaddr: Ipv4Addr, + iface: Ipv4Addr, + }, + SocketJoinIpv6Multicast { + fd: Fd, + multiaddr: Ipv6Addr, + iface: u32, + }, + SocketLeaveIpv4Multicast { + fd: Fd, + multiaddr: Ipv4Addr, + iface: Ipv4Addr, + }, + SocketLeaveIpv6Multicast { + fd: Fd, + multiaddr: Ipv6Addr, + iface: u32, + }, + SocketSendFile { + socket_fd: Fd, + file_fd: Fd, + }, + SocketSendTo { + fd: Fd, + data: Cow<'a, [u8]>, + flags: SiFlags, + addr: SocketAddr, + }, + SocketSend { + fd: Fd, + data: Cow<'a, [u8]>, + flags: SiFlags, + }, + SocketSetOptFlag { + fd: Fd, + opt: Sockoption, + flag: bool, + }, + SocketSetOptSize { + fd: Fd, + opt: Sockoption, + size: u64, + }, + SocketSetOptTime { + fd: Fd, + opt: Sockoption, + size: Option, + }, + SocketShutdown { + fd: Fd, + how: SdFlags, + }, /// Represents the marker for the end of a snapshot Snapshot { when: SystemTime, diff --git a/lib/wasix/src/journaling/log_file.rs b/lib/wasix/src/journaling/log_file.rs index 8a0f171697d..79324784431 100644 --- a/lib/wasix/src/journaling/log_file.rs +++ b/lib/wasix/src/journaling/log_file.rs @@ -6,7 +6,10 @@ use std::{ time::SystemTime, }; use tokio::runtime::Handle; -use wasmer_wasix_types::wasi::{self, EpollEventCtl}; +use virtual_net::{IpAddr, IpCidr, Ipv4Addr, Ipv6Addr, SocketAddr}; +use wasmer_wasix_types::wasi::{ + self, Addressfamily, EpollEventCtl, Sockoption, Socktype, Streamsecurity, +}; use futures::future::LocalBoxFuture; use virtual_fs::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt}; @@ -178,6 +181,110 @@ pub(crate) enum LogFileJournalEntry { fd1: u32, fd2: u32, }, + PortAddAddrV1 { + cidr: IpCidr, + }, + PortDelAddrV1 { + addr: IpAddr, + }, + PortAddrClearV1, + PortBridgeV1 { + network: String, + token: String, + security: JournalStreamSecurityV1, + }, + PortUnbridgeV1, + PortDhcpAcquireV1, + PortGatewaySetV1 { + ip: IpAddr, + }, + PortRouteAddV1 { + cidr: IpCidr, + via_router: IpAddr, + preferred_until: Option, + expires_at: Option, + }, + PortRouteClearV1, + PortRouteDelV1 { + ip: IpAddr, + }, + SocketOpenV1 { + af: JournalAddressfamilyV1, + ty: JournalSocktypeV1, + pt: u16, + fd: u32, + }, + SocketListenV1 { + fd: u32, + backlog: u32, + }, + SocketBindV1 { + fd: u32, + addr: SocketAddr, + }, + SocketConnectV1 { + fd: u32, + addr: SocketAddr, + }, + SocketAcceptV1 { + listen_fd: u32, + fd: u32, + peer_addr: SocketAddr, + }, + SocketJoinIpv4MulticastV1 { + fd: u32, + multiaddr: Ipv4Addr, + iface: Ipv4Addr, + }, + SocketJoinIpv6MulticastV1 { + fd: u32, + multiaddr: Ipv6Addr, + iface: u32, + }, + SocketLeaveIpv4MulticastV1 { + fd: u32, + multiaddr: Ipv4Addr, + iface: Ipv4Addr, + }, + SocketLeaveIpv6MulticastV1 { + fd: u32, + multiaddr: Ipv6Addr, + iface: u32, + }, + SocketSendFileV1 { + socket_fd: u32, + file_fd: u32, + }, + SocketSendToV1 { + fd: u32, + data: Vec, + flags: u16, + addr: SocketAddr, + }, + SocketSendV1 { + fd: u32, + data: Vec, + flags: u16, + }, + SocketSetOptFlagV1 { + fd: u32, + opt: JournalSockoptionV1, + flag: bool, + }, + SocketSetOptSizeV1 { + fd: u32, + opt: JournalSockoptionV1, + size: u64, + }, + SocketSetOptTimeV1 { + fd: u32, + opt: JournalSockoptionV1, + size: Option, + }, + SocketShutdownV1 { + fd: u32, + how: u8, + }, SnapshotV1 { when: SystemTime, trigger: JournalSnapshotTriggerV1, @@ -390,6 +497,201 @@ impl From for wasi::EpollCtl { } } +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +pub enum JournalStreamSecurityV1 { + Unencrypted, + AnyEncryption, + ClassicEncryption, + DoubleEncryption, + Unknown, +} + +impl From for JournalStreamSecurityV1 { + fn from(val: Streamsecurity) -> Self { + match val { + Streamsecurity::Unencrypted => JournalStreamSecurityV1::Unencrypted, + Streamsecurity::AnyEncryption => JournalStreamSecurityV1::AnyEncryption, + Streamsecurity::ClassicEncryption => JournalStreamSecurityV1::ClassicEncryption, + Streamsecurity::DoubleEncryption => JournalStreamSecurityV1::DoubleEncryption, + Streamsecurity::Unknown => JournalStreamSecurityV1::Unknown, + } + } +} + +impl From for Streamsecurity { + fn from(val: JournalStreamSecurityV1) -> Self { + match val { + JournalStreamSecurityV1::Unencrypted => Streamsecurity::Unencrypted, + JournalStreamSecurityV1::AnyEncryption => Streamsecurity::AnyEncryption, + JournalStreamSecurityV1::ClassicEncryption => Streamsecurity::ClassicEncryption, + JournalStreamSecurityV1::DoubleEncryption => Streamsecurity::DoubleEncryption, + JournalStreamSecurityV1::Unknown => Streamsecurity::Unknown, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +pub enum JournalAddressfamilyV1 { + Unspec, + Inet4, + Inet6, + Unix, +} + +impl From for JournalAddressfamilyV1 { + fn from(val: Addressfamily) -> Self { + match val { + Addressfamily::Unspec => JournalAddressfamilyV1::Unspec, + Addressfamily::Inet4 => JournalAddressfamilyV1::Inet4, + Addressfamily::Inet6 => JournalAddressfamilyV1::Inet6, + Addressfamily::Unix => JournalAddressfamilyV1::Unix, + } + } +} + +impl From for Addressfamily { + fn from(val: JournalAddressfamilyV1) -> Self { + match val { + JournalAddressfamilyV1::Unspec => Addressfamily::Unspec, + JournalAddressfamilyV1::Inet4 => Addressfamily::Inet4, + JournalAddressfamilyV1::Inet6 => Addressfamily::Inet6, + JournalAddressfamilyV1::Unix => Addressfamily::Unix, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +pub enum JournalSocktypeV1 { + Unknown, + Stream, + Dgram, + Raw, + Seqpacket, +} + +impl From for JournalSocktypeV1 { + fn from(val: Socktype) -> Self { + match val { + Socktype::Stream => JournalSocktypeV1::Stream, + Socktype::Dgram => JournalSocktypeV1::Dgram, + Socktype::Raw => JournalSocktypeV1::Raw, + Socktype::Seqpacket => JournalSocktypeV1::Seqpacket, + Socktype::Unknown => JournalSocktypeV1::Unknown, + } + } +} + +impl From for Socktype { + fn from(val: JournalSocktypeV1) -> Self { + match val { + JournalSocktypeV1::Stream => Socktype::Stream, + JournalSocktypeV1::Dgram => Socktype::Dgram, + JournalSocktypeV1::Raw => Socktype::Raw, + JournalSocktypeV1::Seqpacket => Socktype::Seqpacket, + JournalSocktypeV1::Unknown => Socktype::Unknown, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +pub enum JournalSockoptionV1 { + Noop, + ReusePort, + ReuseAddr, + NoDelay, + DontRoute, + OnlyV6, + Broadcast, + MulticastLoopV4, + MulticastLoopV6, + Promiscuous, + Listening, + LastError, + KeepAlive, + Linger, + OobInline, + RecvBufSize, + SendBufSize, + RecvLowat, + SendLowat, + RecvTimeout, + SendTimeout, + ConnectTimeout, + AcceptTimeout, + Ttl, + MulticastTtlV4, + Type, + Proto, +} + +impl From for JournalSockoptionV1 { + fn from(val: Sockoption) -> Self { + match val { + Sockoption::Noop => JournalSockoptionV1::Noop, + Sockoption::ReusePort => JournalSockoptionV1::ReusePort, + Sockoption::ReuseAddr => JournalSockoptionV1::ReuseAddr, + Sockoption::NoDelay => JournalSockoptionV1::NoDelay, + Sockoption::DontRoute => JournalSockoptionV1::DontRoute, + Sockoption::OnlyV6 => JournalSockoptionV1::OnlyV6, + Sockoption::Broadcast => JournalSockoptionV1::Broadcast, + Sockoption::MulticastLoopV4 => JournalSockoptionV1::MulticastLoopV4, + Sockoption::MulticastLoopV6 => JournalSockoptionV1::MulticastLoopV6, + Sockoption::Promiscuous => JournalSockoptionV1::Promiscuous, + Sockoption::Listening => JournalSockoptionV1::Listening, + Sockoption::LastError => JournalSockoptionV1::LastError, + Sockoption::KeepAlive => JournalSockoptionV1::KeepAlive, + Sockoption::Linger => JournalSockoptionV1::Linger, + Sockoption::OobInline => JournalSockoptionV1::OobInline, + Sockoption::RecvBufSize => JournalSockoptionV1::RecvBufSize, + Sockoption::SendBufSize => JournalSockoptionV1::SendBufSize, + Sockoption::RecvLowat => JournalSockoptionV1::RecvLowat, + Sockoption::SendLowat => JournalSockoptionV1::SendLowat, + Sockoption::RecvTimeout => JournalSockoptionV1::RecvTimeout, + Sockoption::SendTimeout => JournalSockoptionV1::SendTimeout, + Sockoption::ConnectTimeout => JournalSockoptionV1::ConnectTimeout, + Sockoption::AcceptTimeout => JournalSockoptionV1::AcceptTimeout, + Sockoption::Ttl => JournalSockoptionV1::Ttl, + Sockoption::MulticastTtlV4 => JournalSockoptionV1::MulticastTtlV4, + Sockoption::Type => JournalSockoptionV1::Type, + Sockoption::Proto => JournalSockoptionV1::Proto, + } + } +} + +impl From for Sockoption { + fn from(val: JournalSockoptionV1) -> Self { + match val { + JournalSockoptionV1::Noop => Sockoption::Noop, + JournalSockoptionV1::ReusePort => Sockoption::ReusePort, + JournalSockoptionV1::ReuseAddr => Sockoption::ReuseAddr, + JournalSockoptionV1::NoDelay => Sockoption::NoDelay, + JournalSockoptionV1::DontRoute => Sockoption::DontRoute, + JournalSockoptionV1::OnlyV6 => Sockoption::OnlyV6, + JournalSockoptionV1::Broadcast => Sockoption::Broadcast, + JournalSockoptionV1::MulticastLoopV4 => Sockoption::MulticastLoopV4, + JournalSockoptionV1::MulticastLoopV6 => Sockoption::MulticastLoopV6, + JournalSockoptionV1::Promiscuous => Sockoption::Promiscuous, + JournalSockoptionV1::Listening => Sockoption::Listening, + JournalSockoptionV1::LastError => Sockoption::LastError, + JournalSockoptionV1::KeepAlive => Sockoption::KeepAlive, + JournalSockoptionV1::Linger => Sockoption::Linger, + JournalSockoptionV1::OobInline => Sockoption::OobInline, + JournalSockoptionV1::RecvBufSize => Sockoption::RecvBufSize, + JournalSockoptionV1::SendBufSize => Sockoption::SendBufSize, + JournalSockoptionV1::RecvLowat => Sockoption::RecvLowat, + JournalSockoptionV1::SendLowat => Sockoption::SendLowat, + JournalSockoptionV1::RecvTimeout => Sockoption::RecvTimeout, + JournalSockoptionV1::SendTimeout => Sockoption::SendTimeout, + JournalSockoptionV1::ConnectTimeout => Sockoption::ConnectTimeout, + JournalSockoptionV1::AcceptTimeout => Sockoption::AcceptTimeout, + JournalSockoptionV1::Ttl => Sockoption::Ttl, + JournalSockoptionV1::MulticastTtlV4 => Sockoption::MulticastTtlV4, + JournalSockoptionV1::Type => Sockoption::Type, + JournalSockoptionV1::Proto => Sockoption::Proto, + } + } +} + impl<'a> From> for LogFileJournalEntry { fn from(value: JournalEntry<'a>) -> Self { match value { @@ -602,6 +904,123 @@ impl<'a> From> for LogFileJournalEntry { line_feeds, }, JournalEntry::CreatePipe { fd1, fd2 } => Self::CreatePipeV1 { fd1, fd2 }, + JournalEntry::PortAddAddr { cidr } => Self::PortAddAddrV1 { cidr }, + JournalEntry::PortDelAddr { addr } => Self::PortDelAddrV1 { addr }, + JournalEntry::PortAddrClear => Self::PortAddrClearV1, + JournalEntry::PortBridge { + network, + token, + security, + } => Self::PortBridgeV1 { + network: network.into(), + token: token.into(), + security: security.into(), + }, + JournalEntry::PortUnbridge => Self::PortUnbridgeV1, + JournalEntry::PortDhcpAcquire => Self::PortDhcpAcquireV1, + JournalEntry::PortGatewaySet { ip } => Self::PortGatewaySetV1 { ip }, + JournalEntry::PortRouteAdd { + cidr, + via_router, + preferred_until, + expires_at, + } => Self::PortRouteAddV1 { + cidr, + via_router, + preferred_until, + expires_at, + }, + JournalEntry::PortRouteClear => Self::PortRouteClearV1, + JournalEntry::PortRouteDel { ip } => Self::PortRouteDelV1 { ip }, + JournalEntry::SocketOpen { af, ty, pt, fd } => Self::SocketOpenV1 { + af: af.into(), + ty: ty.into(), + pt: pt as u16, + fd, + }, + JournalEntry::SocketListen { fd, backlog } => Self::SocketListenV1 { fd, backlog }, + JournalEntry::SocketBind { fd, addr } => Self::SocketBindV1 { fd, addr }, + JournalEntry::SocketConnect { fd, addr } => Self::SocketConnectV1 { fd, addr }, + JournalEntry::SocketAccept { + listen_fd, + fd, + peer_addr, + } => Self::SocketAcceptV1 { + listen_fd, + fd, + peer_addr, + }, + JournalEntry::SocketJoinIpv4Multicast { + fd, + multiaddr, + iface, + } => Self::SocketJoinIpv4MulticastV1 { + fd, + multiaddr, + iface, + }, + JournalEntry::SocketJoinIpv6Multicast { + fd, + multiaddr, + iface, + } => Self::SocketJoinIpv6MulticastV1 { + fd, + multiaddr, + iface, + }, + JournalEntry::SocketLeaveIpv4Multicast { + fd, + multiaddr, + iface, + } => Self::SocketLeaveIpv4MulticastV1 { + fd, + multiaddr, + iface, + }, + JournalEntry::SocketLeaveIpv6Multicast { + fd, + multiaddr, + iface, + } => Self::SocketLeaveIpv6MulticastV1 { + fd, + multiaddr, + iface, + }, + JournalEntry::SocketSendFile { socket_fd, file_fd } => { + Self::SocketSendFileV1 { socket_fd, file_fd } + } + JournalEntry::SocketSendTo { + fd, + data, + flags, + addr, + } => Self::SocketSendToV1 { + fd, + data: data.into(), + flags: flags as u16, + addr, + }, + JournalEntry::SocketSend { fd, data, flags } => Self::SocketSendV1 { + fd, + data: data.into(), + flags: flags as u16, + }, + JournalEntry::SocketSetOptFlag { fd, opt, flag } => Self::SocketSetOptFlagV1 { + fd, + opt: opt.into(), + flag, + }, + JournalEntry::SocketSetOptSize { fd, opt, size } => Self::SocketSetOptSizeV1 { + fd, + opt: opt.into(), + size, + }, + JournalEntry::SocketSetOptTime { fd, opt, size } => Self::SocketSetOptTimeV1 { + fd, + opt: opt.into(), + size, + }, + JournalEntry::SocketShutdown { fd, how } => Self::SocketShutdownV1 { fd, how }, } } } @@ -836,6 +1255,125 @@ impl<'a> From for JournalEntry<'a> { line_feeds, }, LogFileJournalEntry::CreatePipeV1 { fd1, fd2 } => Self::CreatePipe { fd1, fd2 }, + LogFileJournalEntry::PortAddAddrV1 { cidr } => Self::PortAddAddr { cidr }, + LogFileJournalEntry::PortDelAddrV1 { addr } => Self::PortDelAddr { addr }, + LogFileJournalEntry::PortAddrClearV1 => Self::PortAddrClear, + LogFileJournalEntry::PortBridgeV1 { + network, + token, + security, + } => Self::PortBridge { + network: network.into(), + token: token.into(), + security: security.into(), + }, + LogFileJournalEntry::PortUnbridgeV1 => Self::PortUnbridge, + LogFileJournalEntry::PortDhcpAcquireV1 => Self::PortDhcpAcquire, + LogFileJournalEntry::PortGatewaySetV1 { ip } => Self::PortGatewaySet { ip }, + LogFileJournalEntry::PortRouteAddV1 { + cidr, + via_router, + preferred_until, + expires_at, + } => Self::PortRouteAdd { + cidr, + via_router, + preferred_until, + expires_at, + }, + LogFileJournalEntry::PortRouteClearV1 => Self::PortRouteClear, + LogFileJournalEntry::PortRouteDelV1 { ip } => Self::PortRouteDel { ip }, + LogFileJournalEntry::SocketOpenV1 { af, ty, pt, fd } => Self::SocketOpen { + af: af.into(), + ty: ty.into(), + pt: pt.try_into().unwrap_or(wasi::SockProto::Max), + fd, + }, + LogFileJournalEntry::SocketListenV1 { fd, backlog } => { + Self::SocketListen { fd, backlog } + } + LogFileJournalEntry::SocketBindV1 { fd, addr } => Self::SocketBind { fd, addr }, + LogFileJournalEntry::SocketConnectV1 { fd, addr } => Self::SocketConnect { fd, addr }, + LogFileJournalEntry::SocketAcceptV1 { + listen_fd, + fd, + peer_addr, + } => Self::SocketAccept { + listen_fd, + fd, + peer_addr, + }, + LogFileJournalEntry::SocketJoinIpv4MulticastV1 { + fd, + multiaddr, + iface, + } => Self::SocketJoinIpv4Multicast { + fd, + multiaddr, + iface, + }, + LogFileJournalEntry::SocketJoinIpv6MulticastV1 { + fd, + multiaddr, + iface, + } => Self::SocketJoinIpv6Multicast { + fd, + multiaddr, + iface, + }, + LogFileJournalEntry::SocketLeaveIpv4MulticastV1 { + fd, + multiaddr, + iface, + } => Self::SocketLeaveIpv4Multicast { + fd, + multiaddr, + iface, + }, + LogFileJournalEntry::SocketLeaveIpv6MulticastV1 { + fd, + multiaddr, + iface, + } => Self::SocketLeaveIpv6Multicast { + fd, + multiaddr, + iface, + }, + LogFileJournalEntry::SocketSendFileV1 { socket_fd, file_fd } => { + Self::SocketSendFile { socket_fd, file_fd } + } + LogFileJournalEntry::SocketSendToV1 { + fd, + data, + flags, + addr, + } => Self::SocketSendTo { + fd, + data: data.into(), + flags, + addr, + }, + LogFileJournalEntry::SocketSendV1 { fd, data, flags } => Self::SocketSend { + fd, + data: data.into(), + flags, + }, + LogFileJournalEntry::SocketSetOptFlagV1 { fd, opt, flag } => Self::SocketSetOptFlag { + fd, + opt: opt.into(), + flag, + }, + LogFileJournalEntry::SocketSetOptSizeV1 { fd, opt, size } => Self::SocketSetOptSize { + fd, + opt: opt.into(), + size, + }, + LogFileJournalEntry::SocketSetOptTimeV1 { fd, opt, size } => Self::SocketSetOptTime { + fd, + opt: opt.into(), + size, + }, + LogFileJournalEntry::SocketShutdownV1 { fd, how } => Self::SocketShutdown { fd, how }, } } } From 61c68250bde297034061f425fd4b7e94268d7819 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Tue, 21 Nov 2023 08:35:55 +1100 Subject: [PATCH 054/129] Split the port and socket syscalls so they can be invokes seperately --- .../journaling/effector/syscalls/fd_write.rs | 19 +- lib/wasix/src/syscalls/mod.rs | 60 +++ lib/wasix/src/syscalls/wasi/fd_write.rs | 121 +++--- lib/wasix/src/syscalls/wasix/port_addr_add.rs | 15 +- .../src/syscalls/wasix/port_addr_clear.rs | 11 +- .../src/syscalls/wasix/port_addr_remove.rs | 14 +- lib/wasix/src/syscalls/wasix/port_bridge.rs | 24 +- .../src/syscalls/wasix/port_dhcp_acquire.rs | 12 +- .../src/syscalls/wasix/port_gateway_set.rs | 14 +- .../src/syscalls/wasix/port_route_add.rs | 26 +- .../src/syscalls/wasix/port_route_clear.rs | 12 +- .../src/syscalls/wasix/port_route_remove.rs | 14 +- lib/wasix/src/syscalls/wasix/port_unbridge.rs | 11 +- lib/wasix/src/syscalls/wasix/sock_accept.rs | 20 +- lib/wasix/src/syscalls/wasix/sock_bind.rs | 15 +- lib/wasix/src/syscalls/wasix/sock_connect.rs | 23 +- .../syscalls/wasix/sock_join_multicast_v4.rs | 32 +- .../syscalls/wasix/sock_join_multicast_v6.rs | 30 +- .../syscalls/wasix/sock_leave_multicast_v4.rs | 32 +- .../syscalls/wasix/sock_leave_multicast_v6.rs | 25 +- lib/wasix/src/syscalls/wasix/sock_listen.rs | 17 +- lib/wasix/src/syscalls/wasix/sock_open.rs | 39 +- lib/wasix/src/syscalls/wasix/sock_send.rs | 171 +++++---- .../src/syscalls/wasix/sock_send_file.rs | 349 +++++++++--------- lib/wasix/src/syscalls/wasix/sock_send_to.rs | 38 +- .../src/syscalls/wasix/sock_set_opt_flag.rs | 21 +- .../src/syscalls/wasix/sock_set_opt_size.rs | 21 +- .../src/syscalls/wasix/sock_set_opt_time.rs | 32 +- lib/wasix/src/syscalls/wasix/sock_shutdown.rs | 30 +- 29 files changed, 807 insertions(+), 441 deletions(-) diff --git a/lib/wasix/src/journaling/effector/syscalls/fd_write.rs b/lib/wasix/src/journaling/effector/syscalls/fd_write.rs index 2b08daf38ba..f9a71c625e2 100644 --- a/lib/wasix/src/journaling/effector/syscalls/fd_write.rs +++ b/lib/wasix/src/journaling/effector/syscalls/fd_write.rs @@ -2,7 +2,7 @@ use super::*; impl JournalEffector { pub fn save_fd_write( - ctx: &mut FunctionEnvMut<'_, WasiEnv>, + ctx: &FunctionEnvMut<'_, WasiEnv>, fd: Fd, offset: u64, written: usize, @@ -46,28 +46,27 @@ impl JournalEffector { } pub async fn apply_fd_write( - ctx: &mut FunctionEnvMut<'_, WasiEnv>, + ctx: &FunctionEnvMut<'_, WasiEnv>, fd: Fd, offset: u64, data: Cow<'_, [u8]>, ) -> anyhow::Result<()> { - let ret = fd_write_internal( + fd_write_internal( ctx, fd, FdWriteSource::<'_, M>::Buffer(data), offset, - None, true, false, - )?; - if ret != Errno::Success { - bail!( + )? + .map_err(|err| { + anyhow::format_err!( "journal restore error: failed to write to descriptor (fd={}, offset={}) - {}", fd, offset, - ret - ); - } + err + ) + })?; Ok(()) } } diff --git a/lib/wasix/src/syscalls/mod.rs b/lib/wasix/src/syscalls/mod.rs index 18cac2b3b7f..1e920c715c1 100644 --- a/lib/wasix/src/syscalls/mod.rs +++ b/lib/wasix/src/syscalls/mod.rs @@ -1560,6 +1560,66 @@ pub fn restore_snapshot( ) .map_err(anyhow_err_to_runtime_err)?; } + crate::journaling::JournalEntry::PortAddAddr { cidr } => todo!(), + crate::journaling::JournalEntry::PortDelAddr { addr } => todo!(), + crate::journaling::JournalEntry::PortAddrClear => todo!(), + crate::journaling::JournalEntry::PortBridge { + network, + token, + security, + } => todo!(), + crate::journaling::JournalEntry::PortUnbridge => todo!(), + crate::journaling::JournalEntry::PortDhcpAcquire => todo!(), + crate::journaling::JournalEntry::PortGatewaySet { ip } => todo!(), + crate::journaling::JournalEntry::PortRouteAdd { + cidr, + via_router, + preferred_until, + expires_at, + } => todo!(), + crate::journaling::JournalEntry::PortRouteClear => todo!(), + crate::journaling::JournalEntry::PortRouteDel { ip } => todo!(), + crate::journaling::JournalEntry::SocketOpen { af, ty, pt, fd } => todo!(), + crate::journaling::JournalEntry::SocketListen { fd, backlog } => todo!(), + crate::journaling::JournalEntry::SocketBind { fd, addr } => todo!(), + crate::journaling::JournalEntry::SocketConnect { fd, addr } => todo!(), + crate::journaling::JournalEntry::SocketAccept { + listen_fd, + fd, + peer_addr, + } => todo!(), + crate::journaling::JournalEntry::SocketJoinIpv4Multicast { + fd, + multiaddr, + iface, + } => todo!(), + crate::journaling::JournalEntry::SocketJoinIpv6Multicast { + fd, + multiaddr, + iface, + } => todo!(), + crate::journaling::JournalEntry::SocketLeaveIpv4Multicast { + fd, + multiaddr, + iface, + } => todo!(), + crate::journaling::JournalEntry::SocketLeaveIpv6Multicast { + fd, + multiaddr, + iface, + } => todo!(), + crate::journaling::JournalEntry::SocketSendFile { socket_fd, file_fd } => todo!(), + crate::journaling::JournalEntry::SocketSendTo { + fd, + data, + flags, + addr, + } => todo!(), + crate::journaling::JournalEntry::SocketSend { fd, data, flags } => todo!(), + crate::journaling::JournalEntry::SocketSetOptFlag { fd, opt, flag } => todo!(), + crate::journaling::JournalEntry::SocketSetOptSize { fd, opt, size } => todo!(), + crate::journaling::JournalEntry::SocketSetOptTime { fd, opt, size } => todo!(), + crate::journaling::JournalEntry::SocketShutdown { fd, how } => todo!(), } } // If we are not in the same module then we fire off an exit diff --git a/lib/wasix/src/syscalls/wasi/fd_write.rs b/lib/wasix/src/syscalls/wasi/fd_write.rs index 95510661c20..220190ef16f 100644 --- a/lib/wasix/src/syscalls/wasi/fd_write.rs +++ b/lib/wasix/src/syscalls/wasi/fd_write.rs @@ -30,6 +30,8 @@ pub fn fd_write( iovs_len: M::Offset, nwritten: WasmPtr, ) -> Result { + wasi_try_ok!(WasiEnv::process_signals_and_exit(&mut ctx)?); + let env = ctx.data(); let offset = { let state = env.state.clone(); @@ -38,17 +40,26 @@ pub fn fd_write( let fd_entry = wasi_try_ok!(state.fs.get_fd(fd)); fd_entry.offset.load(Ordering::Acquire) as usize }; - let enable_snapshot_capture = env.enable_journal; - fd_write_internal::( - &mut ctx, + let bytes_written = wasi_try_ok!(fd_write_internal::( + &ctx, fd, FdWriteSource::Iovs { iovs, iovs_len }, offset as u64, - Some(nwritten), true, - enable_snapshot_capture, - ) + env.enable_journal, + )?); + + Span::current().record("nwritten", bytes_written); + + let mut env = ctx.data(); + let memory = unsafe { env.memory_view(&ctx) }; + let nwritten_ref = nwritten.deref(&memory); + let bytes_written: M::Offset = + wasi_try_ok!(bytes_written.try_into().map_err(|_| Errno::Overflow)); + wasi_try_mem_ok!(nwritten_ref.write(bytes_written)); + + Ok(Errno::Success) } /// ### `fd_pwrite()` @@ -74,16 +85,29 @@ pub fn fd_pwrite( offset: Filesize, nwritten: WasmPtr, ) -> Result { + wasi_try_ok!(WasiEnv::process_signals_and_exit(&mut ctx)?); + let enable_snapshot_capture = ctx.data().enable_journal; - fd_write_internal::( - &mut ctx, + + let bytes_written = wasi_try_ok!(fd_write_internal::( + &ctx, fd, FdWriteSource::Iovs { iovs, iovs_len }, offset, - Some(nwritten), false, enable_snapshot_capture, - ) + )?); + + Span::current().record("nwritten", bytes_written); + + let mut env = ctx.data(); + let memory = unsafe { env.memory_view(&ctx) }; + let nwritten_ref = nwritten.deref(&memory); + let bytes_written: M::Offset = + wasi_try_ok!(bytes_written.try_into().map_err(|_| Errno::Overflow)); + wasi_try_mem_ok!(nwritten_ref.write(bytes_written)); + + Ok(Errno::Success) } pub(crate) enum FdWriteSource<'a, M: MemorySize> { @@ -95,25 +119,22 @@ pub(crate) enum FdWriteSource<'a, M: MemorySize> { } pub(crate) fn fd_write_internal( - ctx: &mut FunctionEnvMut<'_, WasiEnv>, + ctx: &FunctionEnvMut<'_, WasiEnv>, fd: WasiFd, data: FdWriteSource<'_, M>, offset: u64, - nwritten: Option>, should_update_cursor: bool, should_snapshot: bool, -) -> Result { - wasi_try_ok!(WasiEnv::process_signals_and_exit(ctx)?); - +) -> Result, WasiError> { let mut env = ctx.data(); let state = env.state.clone(); - let fd_entry = wasi_try_ok!(state.fs.get_fd(fd)); + let fd_entry = wasi_try_ok_ok!(state.fs.get_fd(fd)); let is_stdio = fd_entry.is_stdio; let bytes_written = { if !is_stdio && !fd_entry.rights.contains(Rights::FD_WRITE) { - return Ok(Errno::Access); + return Ok(Err(Errno::Access)); } let fd_flags = fd_entry.flags; @@ -183,14 +204,14 @@ pub(crate) fn fd_write_internal( Ok(written) }, ); - let written = wasi_try_ok!(res?.map_err(|err| match err { + let written = wasi_try_ok_ok!(res?.map_err(|err| match err { Errno::Timedout => Errno::Again, a => a, })); (written, true, true) } else { - return Ok(Errno::Inval); + return Ok(Err(Errno::Inval)); } } Kind::Socket { socket } => { @@ -242,7 +263,7 @@ pub(crate) fn fd_write_internal( } Ok(sent) }); - let written = wasi_try_ok!(res?); + let written = wasi_try_ok_ok!(res?); (written, false, false) } Kind::Pipe { pipe } => { @@ -250,18 +271,18 @@ pub(crate) fn fd_write_internal( match &data { FdWriteSource::Iovs { iovs, iovs_len } => { - let iovs_arr = wasi_try_ok!(iovs + let iovs_arr = wasi_try_ok_ok!(iovs .slice(&memory, *iovs_len) .map_err(mem_error_to_wasi)); let iovs_arr = - wasi_try_ok!(iovs_arr.access().map_err(mem_error_to_wasi)); + wasi_try_ok_ok!(iovs_arr.access().map_err(mem_error_to_wasi)); for iovs in iovs_arr.iter() { - let buf = wasi_try_ok!(WasmPtr::::new(iovs.buf) + let buf = wasi_try_ok_ok!(WasmPtr::::new(iovs.buf) .slice(&memory, iovs.buf_len) .map_err(mem_error_to_wasi)); - let buf = wasi_try_ok!(buf.access().map_err(mem_error_to_wasi)); + let buf = wasi_try_ok_ok!(buf.access().map_err(mem_error_to_wasi)); let local_written = - wasi_try_ok!(std::io::Write::write(pipe, buf.as_ref()) + wasi_try_ok_ok!(std::io::Write::write(pipe, buf.as_ref()) .map_err(map_io_err)); written += local_written; @@ -271,7 +292,9 @@ pub(crate) fn fd_write_internal( } } FdWriteSource::Buffer(data) => { - wasi_try_ok!(std::io::Write::write_all(pipe, data).map_err(map_io_err)); + wasi_try_ok_ok!( + std::io::Write::write_all(pipe, data).map_err(map_io_err) + ); written += data.len(); } } @@ -280,31 +303,34 @@ pub(crate) fn fd_write_internal( } Kind::Dir { .. } | Kind::Root { .. } => { // TODO: verify - return Ok(Errno::Isdir); + return Ok(Err(Errno::Isdir)); } Kind::EventNotifications { inner } => { let mut written = 0usize; match &data { FdWriteSource::Iovs { iovs, iovs_len } => { - let iovs_arr = wasi_try_ok!(iovs + let iovs_arr = wasi_try_ok_ok!(iovs .slice(&memory, *iovs_len) .map_err(mem_error_to_wasi)); let iovs_arr = - wasi_try_ok!(iovs_arr.access().map_err(mem_error_to_wasi)); + wasi_try_ok_ok!(iovs_arr.access().map_err(mem_error_to_wasi)); for iovs in iovs_arr.iter() { - let buf_len: usize = - wasi_try_ok!(iovs.buf_len.try_into().map_err(|_| Errno::Inval)); + let buf_len: usize = wasi_try_ok_ok!(iovs + .buf_len + .try_into() + .map_err(|_| Errno::Inval)); let will_be_written = buf_len; let val_cnt = buf_len / std::mem::size_of::(); let val_cnt: M::Offset = - wasi_try_ok!(val_cnt.try_into().map_err(|_| Errno::Inval)); + wasi_try_ok_ok!(val_cnt.try_into().map_err(|_| Errno::Inval)); - let vals = wasi_try_ok!(WasmPtr::::new(iovs.buf) + let vals = wasi_try_ok_ok!(WasmPtr::::new(iovs.buf) .slice(&memory, val_cnt as M::Offset) .map_err(mem_error_to_wasi)); - let vals = wasi_try_ok!(vals.access().map_err(mem_error_to_wasi)); + let vals = + wasi_try_ok_ok!(vals.access().map_err(mem_error_to_wasi)); for val in vals.iter() { inner.write(*val); } @@ -333,24 +359,24 @@ pub(crate) fn fd_write_internal( (written, false, true) } - Kind::Symlink { .. } | Kind::Epoll { .. } => return Ok(Errno::Inval), + Kind::Symlink { .. } | Kind::Epoll { .. } => return Ok(Err(Errno::Inval)), Kind::Buffer { buffer } => { let mut written = 0usize; match &data { FdWriteSource::Iovs { iovs, iovs_len } => { - let iovs_arr = wasi_try_ok!(iovs + let iovs_arr = wasi_try_ok_ok!(iovs .slice(&memory, *iovs_len) .map_err(mem_error_to_wasi)); let iovs_arr = - wasi_try_ok!(iovs_arr.access().map_err(mem_error_to_wasi)); + wasi_try_ok_ok!(iovs_arr.access().map_err(mem_error_to_wasi)); for iovs in iovs_arr.iter() { - let buf = wasi_try_ok!(WasmPtr::::new(iovs.buf) + let buf = wasi_try_ok_ok!(WasmPtr::::new(iovs.buf) .slice(&memory, iovs.buf_len) .map_err(mem_error_to_wasi)); - let buf = wasi_try_ok!(buf.access().map_err(mem_error_to_wasi)); + let buf = wasi_try_ok_ok!(buf.access().map_err(mem_error_to_wasi)); let local_written = - wasi_try_ok!(std::io::Write::write(buffer, buf.as_ref()) + wasi_try_ok_ok!(std::io::Write::write(buffer, buf.as_ref()) .map_err(map_io_err)); written += local_written; if local_written != buf.len() { @@ -359,7 +385,7 @@ pub(crate) fn fd_write_internal( } } FdWriteSource::Buffer(data) => { - wasi_try_ok!( + wasi_try_ok_ok!( std::io::Write::write_all(buffer, data).map_err(map_io_err) ); written += data.len(); @@ -392,7 +418,7 @@ pub(crate) fn fd_write_internal( if !is_stdio { if can_update_cursor && should_update_cursor { let mut fd_map = state.fs.fd_map.write().unwrap(); - let fd_entry = wasi_try_ok!(fd_map.get_mut(&fd).ok_or(Errno::Badf)); + let fd_entry = wasi_try_ok_ok!(fd_map.get_mut(&fd).ok_or(Errno::Badf)); fd_entry .offset .fetch_add(bytes_written as u64, Ordering::AcqRel); @@ -407,15 +433,6 @@ pub(crate) fn fd_write_internal( } bytes_written }; - if let Some(nwritten) = nwritten { - Span::current().record("nwritten", bytes_written); - let memory = unsafe { env.memory_view(&ctx) }; - let nwritten_ref = nwritten.deref(&memory); - let bytes_written: M::Offset = - wasi_try_ok!(bytes_written.try_into().map_err(|_| Errno::Overflow)); - wasi_try_mem_ok!(nwritten_ref.write(bytes_written)); - } - - Ok(Errno::Success) + Ok(Ok(bytes_written)) } diff --git a/lib/wasix/src/syscalls/wasix/port_addr_add.rs b/lib/wasix/src/syscalls/wasix/port_addr_add.rs index 5feabed467b..37b4f1d53a7 100644 --- a/lib/wasix/src/syscalls/wasix/port_addr_add.rs +++ b/lib/wasix/src/syscalls/wasix/port_addr_add.rs @@ -1,3 +1,5 @@ +use virtual_net::IpCidr; + use super::*; use crate::syscalls::*; @@ -18,11 +20,20 @@ pub fn port_addr_add( let cidr = wasi_try_ok!(crate::net::read_cidr(&memory, ip)); Span::current().record("ip", &format!("{:?}", cidr)); + wasi_try_ok!(port_addr_add_internal(&mut ctx, cidr)?); + Ok(Errno::Success) +} + +pub(crate) fn port_addr_add_internal( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + cidr: IpCidr, +) -> Result, WasiError> { + let env = ctx.data(); let net = env.net().clone(); - wasi_try_ok!(__asyncify(&mut ctx, None, async { + wasi_try_ok_ok!(__asyncify(ctx, None, async { net.ip_add(cidr.ip, cidr.prefix) .await .map_err(net_error_into_wasi_err) })?); - Ok(Errno::Success) + Ok(Ok(())) } diff --git a/lib/wasix/src/syscalls/wasix/port_addr_clear.rs b/lib/wasix/src/syscalls/wasix/port_addr_clear.rs index a1fec934330..e4bb6c28991 100644 --- a/lib/wasix/src/syscalls/wasix/port_addr_clear.rs +++ b/lib/wasix/src/syscalls/wasix/port_addr_clear.rs @@ -5,10 +5,17 @@ use crate::syscalls::*; /// Clears all the addresses on the local port #[instrument(level = "debug", skip_all, ret)] pub fn port_addr_clear(mut ctx: FunctionEnvMut<'_, WasiEnv>) -> Result { + wasi_try_ok!(port_addr_clear_internal(&mut ctx)?); + Ok(Errno::Success) +} + +pub(crate) fn port_addr_clear_internal( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, +) -> Result, WasiError> { let env = ctx.data(); let net = env.net().clone(); - wasi_try_ok!(__asyncify(&mut ctx, None, async { + wasi_try_ok_ok!(__asyncify(ctx, None, async { net.ip_clear().await.map_err(net_error_into_wasi_err) })?); - Ok(Errno::Success) + Ok(Ok(())) } diff --git a/lib/wasix/src/syscalls/wasix/port_addr_remove.rs b/lib/wasix/src/syscalls/wasix/port_addr_remove.rs index accae6cba6f..9d91eec380b 100644 --- a/lib/wasix/src/syscalls/wasix/port_addr_remove.rs +++ b/lib/wasix/src/syscalls/wasix/port_addr_remove.rs @@ -18,9 +18,19 @@ pub fn port_addr_remove( let ip = wasi_try_ok!(crate::net::read_ip(&memory, ip)); Span::current().record("ip", &format!("{:?}", ip)); + wasi_try_ok!(port_addr_remove_internal(&mut ctx, ip)?); + + Ok(Errno::Success) +} + +pub(crate) fn port_addr_remove_internal( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + ip: IpAddr, +) -> Result, WasiError> { + let env = ctx.data(); let net = env.net().clone(); - wasi_try_ok!(__asyncify(&mut ctx, None, async { + wasi_try_ok_ok!(__asyncify(ctx, None, async { net.ip_remove(ip).await.map_err(net_error_into_wasi_err) })?); - Ok(Errno::Success) + Ok(Ok(())) } diff --git a/lib/wasix/src/syscalls/wasix/port_bridge.rs b/lib/wasix/src/syscalls/wasix/port_bridge.rs index 1a428af3c24..47d8b739fac 100644 --- a/lib/wasix/src/syscalls/wasix/port_bridge.rs +++ b/lib/wasix/src/syscalls/wasix/port_bridge.rs @@ -33,11 +33,29 @@ pub fn port_bridge( _ => return Ok(Errno::Inval), }; + wasi_try_ok!(port_bridge_internal( + &mut ctx, + network.as_str(), + token.as_str(), + security + )?); + + Ok(Errno::Success) +} + +pub(crate) fn port_bridge_internal( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + network: &str, + token: &str, + security: StreamSecurity, +) -> Result, WasiError> { + let env = ctx.data(); + let net = env.net().clone(); - wasi_try_ok!(__asyncify(&mut ctx, None, async move { - net.bridge(network.as_str(), token.as_str(), security) + wasi_try_ok_ok!(__asyncify(ctx, None, async move { + net.bridge(network, token, security) .await .map_err(net_error_into_wasi_err) })?); - Ok(Errno::Success) + Ok(Ok(())) } diff --git a/lib/wasix/src/syscalls/wasix/port_dhcp_acquire.rs b/lib/wasix/src/syscalls/wasix/port_dhcp_acquire.rs index 3a7a534ee73..80ba6e4aaba 100644 --- a/lib/wasix/src/syscalls/wasix/port_dhcp_acquire.rs +++ b/lib/wasix/src/syscalls/wasix/port_dhcp_acquire.rs @@ -5,11 +5,19 @@ use crate::syscalls::*; /// Acquires a set of IP addresses using DHCP #[instrument(level = "debug", skip_all, ret)] pub fn port_dhcp_acquire(mut ctx: FunctionEnvMut<'_, WasiEnv>) -> Result { + wasi_try_ok!(port_dhcp_acquire_internal(&mut ctx)?); + + Ok(Errno::Success) +} + +pub(crate) fn port_dhcp_acquire_internal( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, +) -> Result, WasiError> { let env = ctx.data(); let net = env.net().clone(); let tasks = env.tasks().clone(); - wasi_try_ok!(__asyncify(&mut ctx, None, async move { + wasi_try_ok_ok!(__asyncify(ctx, None, async move { net.dhcp_acquire().await.map_err(net_error_into_wasi_err) })?); - Ok(Errno::Success) + Ok(Ok(())) } diff --git a/lib/wasix/src/syscalls/wasix/port_gateway_set.rs b/lib/wasix/src/syscalls/wasix/port_gateway_set.rs index 0c9136a4af3..ef74389b492 100644 --- a/lib/wasix/src/syscalls/wasix/port_gateway_set.rs +++ b/lib/wasix/src/syscalls/wasix/port_gateway_set.rs @@ -18,9 +18,19 @@ pub fn port_gateway_set( let ip = wasi_try_ok!(crate::net::read_ip(&memory, ip)); Span::current().record("ip", &format!("{:?}", ip)); + wasi_try_ok!(port_gateway_set_internal(&mut ctx, ip)?); + + Ok(Errno::Success) +} + +pub(crate) fn port_gateway_set_internal( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + ip: IpAddr, +) -> Result, WasiError> { + let env = ctx.data(); let net = env.net().clone(); - wasi_try_ok!(__asyncify(&mut ctx, None, async { + wasi_try_ok_ok!(__asyncify(ctx, None, async { net.gateway_set(ip).await.map_err(net_error_into_wasi_err) })?); - Ok(Errno::Success) + Ok(Ok(())) } diff --git a/lib/wasix/src/syscalls/wasix/port_route_add.rs b/lib/wasix/src/syscalls/wasix/port_route_add.rs index ccad1ca5a3f..f2d94e95fea 100644 --- a/lib/wasix/src/syscalls/wasix/port_route_add.rs +++ b/lib/wasix/src/syscalls/wasix/port_route_add.rs @@ -1,3 +1,5 @@ +use virtual_net::IpCidr; + use super::*; use crate::syscalls::*; @@ -33,11 +35,31 @@ pub fn port_route_add( _ => return Ok(Errno::Inval), }; + wasi_try_ok!(port_route_add_internal( + &mut ctx, + cidr, + via_router, + preferred_until, + expires_at + )?); + + Ok(Errno::Success) +} + +pub(crate) fn port_route_add_internal( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + cidr: IpCidr, + via_router: IpAddr, + preferred_until: Option, + expires_at: Option, +) -> Result, WasiError> { + let env = ctx.data(); let net = env.net().clone(); - wasi_try_ok!(__asyncify(&mut ctx, None, async { + wasi_try_ok_ok!(__asyncify(ctx, None, async { net.route_add(cidr, via_router, preferred_until, expires_at) .await .map_err(net_error_into_wasi_err) })?); - Ok(Errno::Success) + + Ok(Ok(())) } diff --git a/lib/wasix/src/syscalls/wasix/port_route_clear.rs b/lib/wasix/src/syscalls/wasix/port_route_clear.rs index ba2ea4ab04a..b1a4dc7b3db 100644 --- a/lib/wasix/src/syscalls/wasix/port_route_clear.rs +++ b/lib/wasix/src/syscalls/wasix/port_route_clear.rs @@ -5,10 +5,18 @@ use crate::syscalls::*; /// Clears all the routes in the local port #[instrument(level = "debug", skip_all, ret)] pub fn port_route_clear(mut ctx: FunctionEnvMut<'_, WasiEnv>) -> Result { + wasi_try_ok!(port_route_clear_internal(&mut ctx)?); + + Ok(Errno::Success) +} + +pub(crate) fn port_route_clear_internal( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, +) -> Result, WasiError> { let env = ctx.data(); let net = env.net().clone(); - wasi_try_ok!(__asyncify(&mut ctx, None, async { + wasi_try_ok_ok!(__asyncify(ctx, None, async { net.route_clear().await.map_err(net_error_into_wasi_err) })?); - Ok(Errno::Success) + Ok(Ok(())) } diff --git a/lib/wasix/src/syscalls/wasix/port_route_remove.rs b/lib/wasix/src/syscalls/wasix/port_route_remove.rs index 46e1b9d2335..60e4301d4cc 100644 --- a/lib/wasix/src/syscalls/wasix/port_route_remove.rs +++ b/lib/wasix/src/syscalls/wasix/port_route_remove.rs @@ -14,10 +14,20 @@ pub fn port_route_remove( let ip = wasi_try_ok!(crate::net::read_ip(&memory, ip)); Span::current().record("ip", &format!("{:?}", ip)); + wasi_try_ok!(port_route_remove_internal(&mut ctx, ip)?); + + Ok(Errno::Success) +} + +pub(crate) fn port_route_remove_internal( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + ip: IpAddr, +) -> Result, WasiError> { + let env = ctx.data(); let net = env.net().clone(); - wasi_try_ok!(__asyncify(&mut ctx, None, async { + wasi_try_ok_ok!(__asyncify(ctx, None, async { net.route_remove(ip).await.map_err(net_error_into_wasi_err) })?); - Ok(Errno::Success) + Ok(Ok(())) } diff --git a/lib/wasix/src/syscalls/wasix/port_unbridge.rs b/lib/wasix/src/syscalls/wasix/port_unbridge.rs index 6cc4085a2a6..3d0cd749c8a 100644 --- a/lib/wasix/src/syscalls/wasix/port_unbridge.rs +++ b/lib/wasix/src/syscalls/wasix/port_unbridge.rs @@ -5,10 +5,17 @@ use crate::syscalls::*; /// Disconnects from a remote network #[instrument(level = "debug", skip_all, ret)] pub fn port_unbridge(mut ctx: FunctionEnvMut<'_, WasiEnv>) -> Result { + wasi_try_ok!(port_unbridge_internal(&mut ctx)?); + Ok(Errno::Success) +} + +pub(crate) fn port_unbridge_internal( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, +) -> Result, WasiError> { let env = ctx.data(); let net = env.net().clone(); - wasi_try_ok!(__asyncify(&mut ctx, None, async move { + wasi_try_ok_ok!(__asyncify(ctx, None, async move { net.unbridge().await.map_err(net_error_into_wasi_err) })?); - Ok(Errno::Success) + Ok(Ok(())) } diff --git a/lib/wasix/src/syscalls/wasix/sock_accept.rs b/lib/wasix/src/syscalls/wasix/sock_accept.rs index 1768f4e4d8e..5c9555ca6c6 100644 --- a/lib/wasix/src/syscalls/wasix/sock_accept.rs +++ b/lib/wasix/src/syscalls/wasix/sock_accept.rs @@ -31,7 +31,7 @@ pub fn sock_accept( let nonblocking = fd_flags.contains(Fdflags::NONBLOCK); - let (fd, addr) = wasi_try_ok!(sock_accept_internal(env, sock, fd_flags, nonblocking)); + let (fd, addr) = wasi_try_ok!(sock_accept_internal(env, sock, fd_flags, nonblocking)?); wasi_try_mem_ok!(ro_fd.write(&memory, fd)); @@ -66,7 +66,7 @@ pub fn sock_accept_v2( let nonblocking = fd_flags.contains(Fdflags::NONBLOCK); - let (fd, addr) = wasi_try_ok!(sock_accept_internal(env, sock, fd_flags, nonblocking)); + let (fd, addr) = wasi_try_ok!(sock_accept_internal(env, sock, fd_flags, nonblocking)?); wasi_try_mem_ok!(ro_fd.write(&memory, fd)); wasi_try_ok!(crate::net::write_ip_port( @@ -79,17 +79,17 @@ pub fn sock_accept_v2( Ok(Errno::Success) } -pub fn sock_accept_internal( +pub(crate) fn sock_accept_internal( env: &WasiEnv, sock: WasiFd, mut fd_flags: Fdflags, mut nonblocking: bool, -) -> Result<(WasiFd, SocketAddr), Errno> { +) -> Result, WasiError> { let state = env.state(); let inodes = &state.inodes; let tasks = env.tasks().clone(); - let (child, addr, fd_flags) = __sock_asyncify( + let (child, addr, fd_flags) = wasi_try_ok_ok!(__sock_asyncify( env, sock, Rights::SOCK_ACCEPT, @@ -108,15 +108,15 @@ pub fn sock_accept_internal( .await .map(|a| (a.0, a.1, fd_flags)) }, - )?; + )); let kind = Kind::Socket { - socket: InodeSocket::new(InodeSocketKind::TcpStream { + socket: wasi_try_ok_ok!(InodeSocket::new(InodeSocketKind::TcpStream { socket: child, write_timeout: None, read_timeout: None, }) - .map_err(net_error_into_wasi_err)?, + .map_err(net_error_into_wasi_err)), }; let inode = state .fs @@ -133,8 +133,8 @@ pub fn sock_accept_internal( } let rights = Rights::all_socket(); - let fd = state.fs.create_fd(rights, rights, new_flags, 0, inode)?; + let fd = wasi_try_ok_ok!(state.fs.create_fd(rights, rights, new_flags, 0, inode)); Span::current().record("fd", fd); - Ok((fd, addr)) + Ok(Ok((fd, addr))) } diff --git a/lib/wasix/src/syscalls/wasix/sock_bind.rs b/lib/wasix/src/syscalls/wasix/sock_bind.rs index e8c187dd58f..7f72f98162d 100644 --- a/lib/wasix/src/syscalls/wasix/sock_bind.rs +++ b/lib/wasix/src/syscalls/wasix/sock_bind.rs @@ -21,15 +21,24 @@ pub fn sock_bind( let addr = SocketAddr::new(addr.0, addr.1); Span::current().record("addr", &format!("{:?}", addr)); + Errno::Success +} + +pub(crate) fn sock_bind_internal( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + sock: WasiFd, + addr: SocketAddr, +) -> Result, WasiError> { + let env = ctx.data(); let net = env.net().clone(); let tasks = ctx.data().tasks().clone(); - wasi_try!(__sock_upgrade( - &mut ctx, + wasi_try_ok_ok!(__sock_upgrade( + ctx, sock, Rights::SOCK_BIND, move |socket| async move { socket.bind(tasks.deref(), net.deref(), addr).await } )); - Errno::Success + Ok(Ok(())) } diff --git a/lib/wasix/src/syscalls/wasix/sock_connect.rs b/lib/wasix/src/syscalls/wasix/sock_connect.rs index c9f00cc8e19..f0a01944674 100644 --- a/lib/wasix/src/syscalls/wasix/sock_connect.rs +++ b/lib/wasix/src/syscalls/wasix/sock_connect.rs @@ -18,21 +18,32 @@ pub fn sock_connect( mut ctx: FunctionEnvMut<'_, WasiEnv>, sock: WasiFd, addr: WasmPtr<__wasi_addr_port_t, M>, -) -> Errno { +) -> Result { let env = ctx.data(); - let net = env.net().clone(); let memory = unsafe { env.memory_view(&ctx) }; - let addr = wasi_try!(crate::net::read_ip_port(&memory, addr)); + let addr = wasi_try_ok!(crate::net::read_ip_port(&memory, addr)); let addr = SocketAddr::new(addr.0, addr.1); Span::current().record("addr", &format!("{:?}", addr)); + wasi_try_ok!(sock_connect_internal(&mut ctx, sock, addr)?); + + Ok(Errno::Success) +} + +pub(crate) fn sock_connect_internal( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + sock: WasiFd, + addr: SocketAddr, +) -> Result, WasiError> { + let env = ctx.data(); + let net = env.net().clone(); let tasks = ctx.data().tasks().clone(); - wasi_try!(__sock_upgrade( - &mut ctx, + wasi_try_ok_ok!(__sock_upgrade( + ctx, sock, Rights::SOCK_CONNECT, move |mut socket| async move { socket.connect(tasks.deref(), net.deref(), addr, None).await } )); - Errno::Success + Ok(Ok(())) } diff --git a/lib/wasix/src/syscalls/wasix/sock_join_multicast_v4.rs b/lib/wasix/src/syscalls/wasix/sock_join_multicast_v4.rs index e2c0d73351f..84ea760fde3 100644 --- a/lib/wasix/src/syscalls/wasix/sock_join_multicast_v4.rs +++ b/lib/wasix/src/syscalls/wasix/sock_join_multicast_v4.rs @@ -15,16 +15,28 @@ pub fn sock_join_multicast_v4( sock: WasiFd, multiaddr: WasmPtr<__wasi_addr_ip4_t, M>, iface: WasmPtr<__wasi_addr_ip4_t, M>, -) -> Errno { +) -> Result { let env = ctx.data(); let memory = unsafe { env.memory_view(&ctx) }; - let multiaddr = wasi_try!(crate::net::read_ip_v4(&memory, multiaddr)); - let iface = wasi_try!(crate::net::read_ip_v4(&memory, iface)); - wasi_try!(__sock_actor_mut( - &mut ctx, - sock, - Rights::empty(), - |socket, _| socket.join_multicast_v4(multiaddr, iface) - )); - Errno::Success + let multiaddr = wasi_try_ok!(crate::net::read_ip_v4(&memory, multiaddr)); + let iface = wasi_try_ok!(crate::net::read_ip_v4(&memory, iface)); + + wasi_try_ok!(sock_join_multicast_v4_internal( + &mut ctx, sock, multiaddr, iface + )?); + + Ok(Errno::Success) +} + +pub(crate) fn sock_join_multicast_v4_internal( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + sock: WasiFd, + multiaddr: Ipv4Addr, + iface: Ipv4Addr, +) -> Result, WasiError> { + let env = ctx.data(); + wasi_try_ok_ok!(__sock_actor_mut(ctx, sock, Rights::empty(), |socket, _| { + socket.join_multicast_v4(multiaddr, iface) + })); + Ok(Ok(())) } diff --git a/lib/wasix/src/syscalls/wasix/sock_join_multicast_v6.rs b/lib/wasix/src/syscalls/wasix/sock_join_multicast_v6.rs index 53d0417420f..9b141dfc8de 100644 --- a/lib/wasix/src/syscalls/wasix/sock_join_multicast_v6.rs +++ b/lib/wasix/src/syscalls/wasix/sock_join_multicast_v6.rs @@ -15,15 +15,27 @@ pub fn sock_join_multicast_v6( sock: WasiFd, multiaddr: WasmPtr<__wasi_addr_ip6_t, M>, iface: u32, -) -> Errno { +) -> Result { let env = ctx.data(); let memory = unsafe { env.memory_view(&ctx) }; - let multiaddr = wasi_try!(crate::net::read_ip_v6(&memory, multiaddr)); - wasi_try!(__sock_actor_mut( - &mut ctx, - sock, - Rights::empty(), - |socket, _| socket.join_multicast_v6(multiaddr, iface) - )); - Errno::Success + let multiaddr = wasi_try_ok!(crate::net::read_ip_v6(&memory, multiaddr)); + + wasi_try_ok!(sock_join_multicast_v6_internal( + &mut ctx, sock, multiaddr, iface + )?); + + Ok(Errno::Success) +} + +pub(crate) fn sock_join_multicast_v6_internal( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + sock: WasiFd, + multiaddr: Ipv6Addr, + iface: u32, +) -> Result, WasiError> { + let env = ctx.data(); + wasi_try_ok_ok!(__sock_actor_mut(ctx, sock, Rights::empty(), |socket, _| { + socket.join_multicast_v6(multiaddr, iface) + })); + Ok(Ok(())) } diff --git a/lib/wasix/src/syscalls/wasix/sock_leave_multicast_v4.rs b/lib/wasix/src/syscalls/wasix/sock_leave_multicast_v4.rs index f96526c5670..37306672b75 100644 --- a/lib/wasix/src/syscalls/wasix/sock_leave_multicast_v4.rs +++ b/lib/wasix/src/syscalls/wasix/sock_leave_multicast_v4.rs @@ -15,16 +15,28 @@ pub fn sock_leave_multicast_v4( sock: WasiFd, multiaddr: WasmPtr<__wasi_addr_ip4_t, M>, iface: WasmPtr<__wasi_addr_ip4_t, M>, -) -> Errno { +) -> Result { let env = ctx.data(); let memory = unsafe { env.memory_view(&ctx) }; - let multiaddr = wasi_try!(crate::net::read_ip_v4(&memory, multiaddr)); - let iface = wasi_try!(crate::net::read_ip_v4(&memory, iface)); - wasi_try!(__sock_actor_mut( - &mut ctx, - sock, - Rights::empty(), - |socket, _| socket.leave_multicast_v4(multiaddr, iface) - )); - Errno::Success + let multiaddr = wasi_try_ok!(crate::net::read_ip_v4(&memory, multiaddr)); + let iface = wasi_try_ok!(crate::net::read_ip_v4(&memory, iface)); + + wasi_try_ok!(sock_leave_multicast_v4_internal( + &mut ctx, sock, multiaddr, iface + )?); + + Ok(Errno::Success) +} + +pub(crate) fn sock_leave_multicast_v4_internal( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + sock: WasiFd, + multiaddr: Ipv4Addr, + iface: Ipv4Addr, +) -> Result, WasiError> { + let env = ctx.data(); + wasi_try_ok_ok!(__sock_actor_mut(ctx, sock, Rights::empty(), |socket, _| { + socket.leave_multicast_v4(multiaddr, iface) + })); + Ok(Ok(())) } diff --git a/lib/wasix/src/syscalls/wasix/sock_leave_multicast_v6.rs b/lib/wasix/src/syscalls/wasix/sock_leave_multicast_v6.rs index 7d990c7829a..70b7ec48088 100644 --- a/lib/wasix/src/syscalls/wasix/sock_leave_multicast_v6.rs +++ b/lib/wasix/src/syscalls/wasix/sock_leave_multicast_v6.rs @@ -15,15 +15,30 @@ pub fn sock_leave_multicast_v6( sock: WasiFd, multiaddr: WasmPtr<__wasi_addr_ip6_t, M>, iface: u32, -) -> Errno { +) -> Result { let env = ctx.data(); let memory = unsafe { env.memory_view(&ctx) }; - let multiaddr = wasi_try!(crate::net::read_ip_v6(&memory, multiaddr)); - wasi_try!(__sock_actor_mut( - &mut ctx, + let multiaddr = wasi_try_ok!(crate::net::read_ip_v6(&memory, multiaddr)); + + wasi_try_ok!(sock_leave_multicast_v6_internal( + &mut ctx, sock, multiaddr, iface + )?); + + Ok(Errno::Success) +} + +pub(crate) fn sock_leave_multicast_v6_internal( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + sock: WasiFd, + multiaddr: Ipv6Addr, + iface: u32, +) -> Result, WasiError> { + let env = ctx.data(); + wasi_try_ok_ok!(__sock_actor_mut( + ctx, sock, Rights::empty(), |mut socket, _| socket.leave_multicast_v6(multiaddr, iface) )); - Errno::Success + Ok(Ok(())) } diff --git a/lib/wasix/src/syscalls/wasix/sock_listen.rs b/lib/wasix/src/syscalls/wasix/sock_listen.rs index f1614789ca7..a746f2b90fd 100644 --- a/lib/wasix/src/syscalls/wasix/sock_listen.rs +++ b/lib/wasix/src/syscalls/wasix/sock_listen.rs @@ -22,16 +22,25 @@ pub fn sock_listen( ctx = wasi_try_ok!(maybe_snapshot_once::(ctx, SnapshotTrigger::FirstListen)?); let env = ctx.data(); - let net = env.net().clone(); let backlog: usize = wasi_try_ok!(backlog.try_into().map_err(|_| Errno::Inval)); + Ok(Errno::Success) +} + +pub(crate) fn sock_listen_internal( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + sock: WasiFd, + backlog: usize, +) -> Result, WasiError> { + let env = ctx.data(); + let net = env.net().clone(); let tasks = ctx.data().tasks().clone(); - wasi_try_ok!(__sock_upgrade( - &mut ctx, + wasi_try_ok_ok!(__sock_upgrade( + ctx, sock, Rights::SOCK_LISTEN, |socket| async move { socket.listen(tasks.deref(), net.deref(), backlog).await } )); - Ok(Errno::Success) + Ok(Ok(())) } diff --git a/lib/wasix/src/syscalls/wasix/sock_open.rs b/lib/wasix/src/syscalls/wasix/sock_open.rs index 52848d68317..0c255984044 100644 --- a/lib/wasix/src/syscalls/wasix/sock_open.rs +++ b/lib/wasix/src/syscalls/wasix/sock_open.rs @@ -22,33 +22,48 @@ use crate::syscalls::*; /// The file descriptor of the socket that has been opened. #[instrument(level = "debug", skip_all, fields(?af, ?ty, ?pt, sock = field::Empty), ret)] pub fn sock_open( - ctx: FunctionEnvMut<'_, WasiEnv>, + mut ctx: FunctionEnvMut<'_, WasiEnv>, af: Addressfamily, ty: Socktype, pt: SockProto, ro_sock: WasmPtr, -) -> Errno { - let env = ctx.data(); - let (memory, state, inodes) = unsafe { env.get_memory_and_wasi_state_and_inodes(&ctx, 0) }; - +) -> Result { // only certain combinations are supported match pt { SockProto::Tcp => { if ty != Socktype::Stream { - return Errno::Notsup; + return Ok(Errno::Notsup); } } SockProto::Udp => { if ty != Socktype::Dgram { - return Errno::Notsup; + return Ok(Errno::Notsup); } } _ => {} } + let fd = wasi_try_ok!(sock_open_internal(&mut ctx, af, ty, pt)?); + + let env = ctx.data(); + let (memory, state, inodes) = unsafe { env.get_memory_and_wasi_state_and_inodes(&ctx, 0) }; + wasi_try_mem_ok!(ro_sock.write(&memory, fd)); + + Ok(Errno::Success) +} + +pub(crate) fn sock_open_internal( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + af: Addressfamily, + ty: Socktype, + pt: SockProto, +) -> Result, WasiError> { + let env = ctx.data(); + let (memory, state, inodes) = unsafe { env.get_memory_and_wasi_state_and_inodes(&ctx, 0) }; + let kind = match ty { Socktype::Stream | Socktype::Dgram => Kind::Socket { - socket: wasi_try!(InodeSocket::new(InodeSocketKind::PreSocket { + socket: wasi_try_ok_ok!(InodeSocket::new(InodeSocketKind::PreSocket { family: af, ty, pt, @@ -69,7 +84,7 @@ pub fn sock_open( }) .map_err(net_error_into_wasi_err)), }, - _ => return Errno::Notsup, + _ => return Ok(Err(Errno::Notsup)), }; let inode = @@ -77,12 +92,10 @@ pub fn sock_open( .fs .create_inode_with_default_stat(inodes, kind, false, "socket".to_string().into()); let rights = Rights::all_socket(); - let fd = wasi_try!(state + let fd = wasi_try_ok_ok!(state .fs .create_fd(rights, rights, Fdflags::empty(), 0, inode)); Span::current().record("sock", fd); - wasi_try_mem!(ro_sock.write(&memory, fd)); - - Errno::Success + Ok(Ok(fd)) } diff --git a/lib/wasix/src/syscalls/wasix/sock_send.rs b/lib/wasix/src/syscalls/wasix/sock_send.rs index 4c1eb1de490..bab98c2cd36 100644 --- a/lib/wasix/src/syscalls/wasix/sock_send.rs +++ b/lib/wasix/src/syscalls/wasix/sock_send.rs @@ -16,46 +16,80 @@ use crate::{net::socket::TimeType, syscalls::*}; /// ## Return /// /// Number of bytes transmitted. -#[instrument(level = "trace", skip_all, fields(%sock, nsent = field::Empty), ret)] +#[instrument(level = "trace", skip_all, fields(%fd, nsent = field::Empty), ret)] pub fn sock_send( - ctx: FunctionEnvMut<'_, WasiEnv>, - sock: WasiFd, + mut ctx: FunctionEnvMut<'_, WasiEnv>, + fd: WasiFd, si_data: WasmPtr<__wasi_ciovec_t, M>, si_data_len: M::Offset, si_flags: SiFlags, ret_data_len: WasmPtr, ) -> Result { + wasi_try_ok!(WasiEnv::process_signals_and_exit(&mut ctx)?); + let env = ctx.data(); - let fd_entry = env.state.fs.get_fd(sock).unwrap(); + let fd_entry = env.state.fs.get_fd(fd).unwrap(); let guard = fd_entry.inode.read(); let use_write = matches!(guard.deref(), Kind::Pipe { .. }); drop(guard); - if use_write { - fd_write(ctx, sock, si_data, si_data_len, ret_data_len) + + let bytes_written = if use_write { + let offset = { + let state = env.state.clone(); + let inodes = state.inodes.clone(); + + let fd_entry = wasi_try_ok!(state.fs.get_fd(fd)); + fd_entry.offset.load(Ordering::Acquire) as usize + }; + + wasi_try_ok!(fd_write_internal::( + &ctx, + fd, + FdWriteSource::Iovs { + iovs: si_data, + iovs_len: si_data_len + }, + offset as u64, + true, + env.enable_journal, + )?) } else { - sock_send_internal(ctx, sock, si_data, si_data_len, si_flags, ret_data_len) - } + wasi_try_ok!(sock_send_internal::( + &mut ctx, + fd, + FdWriteSource::Iovs { + iovs: si_data, + iovs_len: si_data_len + }, + si_flags, + )?) + }; + Span::current().record("nsent", bytes_written); + + let env = ctx.data(); + let memory = unsafe { env.memory_view(&ctx) }; + let bytes_written: M::Offset = + wasi_try_ok!(bytes_written.try_into().map_err(|_| Errno::Overflow)); + wasi_try_mem_ok!(ret_data_len.write(&memory, bytes_written)); + + Ok(Errno::Success) } -pub(super) fn sock_send_internal( - mut ctx: FunctionEnvMut<'_, WasiEnv>, +pub(crate) fn sock_send_internal( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, sock: WasiFd, - si_data: WasmPtr<__wasi_ciovec_t, M>, - si_data_len: M::Offset, + si_data: FdWriteSource<'_, M>, si_flags: SiFlags, - ret_data_len: WasmPtr, -) -> Result { +) -> Result, WasiError> { let env = ctx.data(); let memory = unsafe { env.memory_view(&ctx) }; let runtime = env.runtime.clone(); - let res = { - __sock_asyncify(env, sock, Rights::SOCK_SEND, |socket, fd| async move { - let iovs_arr = si_data - .slice(&memory, si_data_len) - .map_err(mem_error_to_wasi)?; - let iovs_arr = iovs_arr.access().map_err(mem_error_to_wasi)?; - + let bytes_written = wasi_try_ok_ok!(__sock_asyncify( + env, + sock, + Rights::SOCK_SEND, + |socket, fd| async move { let nonblocking = fd.flags.contains(Fdflags::NONBLOCK); let timeout = socket .opt_time(TimeType::WriteTimeout) @@ -63,57 +97,54 @@ pub(super) fn sock_send_internal( .flatten() .unwrap_or(Duration::from_secs(30)); - let mut sent = 0usize; - for iovs in iovs_arr.iter() { - let buf = WasmPtr::::new(iovs.buf) - .slice(&memory, iovs.buf_len) - .map_err(mem_error_to_wasi)? - .access() - .map_err(mem_error_to_wasi)?; - let local_sent = match socket - .send( - env.tasks().deref(), - buf.as_ref(), - Some(timeout), - nonblocking, - ) - .await - { - Ok(s) => s, - Err(_) if sent > 0 => break, - Err(err) => return Err(err), - }; - sent += local_sent; - if local_sent != buf.len() { - break; + match si_data { + FdWriteSource::Iovs { iovs, iovs_len } => { + let iovs_arr = iovs.slice(&memory, iovs_len).map_err(mem_error_to_wasi)?; + let iovs_arr = iovs_arr.access().map_err(mem_error_to_wasi)?; + + let mut sent = 0usize; + for iovs in iovs_arr.iter() { + let buf = WasmPtr::::new(iovs.buf) + .slice(&memory, iovs.buf_len) + .map_err(mem_error_to_wasi)? + .access() + .map_err(mem_error_to_wasi)?; + let local_sent = match socket + .send( + env.tasks().deref(), + buf.as_ref(), + Some(timeout), + nonblocking, + ) + .await + { + Ok(s) => s, + Err(_) if sent > 0 => break, + Err(err) => return Err(err), + }; + sent += local_sent; + if local_sent != buf.len() { + break; + } + } + Ok(sent) + } + FdWriteSource::Buffer(data) => { + socket + .send( + env.tasks().deref(), + data.as_ref(), + Some(timeout), + nonblocking, + ) + .await } } - Ok(sent) - }) - }; - - let mut ret = Errno::Success; - let bytes_written = match res { - Ok(bytes_written) => { - trace!( - %bytes_written, - ); - bytes_written - } - Err(err) => { - let socket_err = err.name(); - trace!( - %socket_err, - ); - ret = err; - 0 } - }; - Span::current().record("nsent", bytes_written); + )); + trace!( + %bytes_written, + ); - let memory = unsafe { env.memory_view(&ctx) }; - let bytes_written: M::Offset = - wasi_try_ok!(bytes_written.try_into().map_err(|_| Errno::Overflow)); - wasi_try_mem_ok!(ret_data_len.write(&memory, bytes_written)); - Ok(ret) + Ok(Ok(bytes_written)) } diff --git a/lib/wasix/src/syscalls/wasix/sock_send_file.rs b/lib/wasix/src/syscalls/wasix/sock_send_file.rs index 253cc1f095b..076d36f06d4 100644 --- a/lib/wasix/src/syscalls/wasix/sock_send_file.rs +++ b/lib/wasix/src/syscalls/wasix/sock_send_file.rs @@ -21,201 +21,206 @@ pub fn sock_send_file( sock: WasiFd, in_fd: WasiFd, offset: Filesize, - mut count: Filesize, + count: Filesize, ret_sent: WasmPtr, ) -> Result { - wasi_try_ok!(WasiEnv::process_signals_and_exit(&mut ctx)?); + let total_written = wasi_try_ok!(sock_send_file_internal( + &mut ctx, sock, in_fd, offset, count + )?); + + Span::current().record("nsent", total_written); + + let env = ctx.data(); + let memory = unsafe { env.memory_view(&ctx) }; + wasi_try_mem_ok!(ret_sent.write(&memory, total_written as Filesize)); + + Ok(Errno::Success) +} + +pub(crate) fn sock_send_file_internal( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + sock: WasiFd, + in_fd: WasiFd, + offset: Filesize, + mut count: Filesize, +) -> Result, WasiError> { + wasi_try_ok_ok!(WasiEnv::process_signals_and_exit(ctx)?); let mut env = ctx.data(); let net = env.net(); let tasks = env.tasks().clone(); let state = env.state.clone(); - let ret = wasi_try_ok!({ - // Set the offset of the file - { - let mut fd_map = state.fs.fd_map.write().unwrap(); - let fd_entry = wasi_try_ok!(fd_map.get_mut(&in_fd).ok_or(Errno::Badf)); - fd_entry.offset.store(offset, Ordering::Release); - } - - // Enter a loop that will process all the data - let mut total_written: Filesize = 0; - while (count > 0) { - let sub_count = count.min(4096); - count -= sub_count; - - let fd_entry = wasi_try_ok!(state.fs.get_fd(in_fd)); - let fd_flags = fd_entry.flags; - - let data = { - match in_fd { - __WASI_STDIN_FILENO => { - let mut stdin = - wasi_try_ok!(WasiInodes::stdin_mut(&state.fs.fd_map) - .map_err(fs_error_into_wasi_err)); - let data = wasi_try_ok!(__asyncify(&mut ctx, None, async move { - // TODO: optimize with MaybeUninit - let mut buf = vec![0u8; sub_count as usize]; - let amt = stdin.read(&mut buf[..]).await.map_err(map_io_err)?; - buf.truncate(amt); - Ok(buf) - })?); - env = ctx.data(); - data + // Set the offset of the file + { + let mut fd_map = state.fs.fd_map.write().unwrap(); + let fd_entry = wasi_try_ok_ok!(fd_map.get_mut(&in_fd).ok_or(Errno::Badf)); + fd_entry.offset.store(offset, Ordering::Release); + } + + // Enter a loop that will process all the data + let mut total_written: Filesize = 0; + while (count > 0) { + let sub_count = count.min(4096); + count -= sub_count; + + let fd_entry = wasi_try_ok_ok!(state.fs.get_fd(in_fd)); + let fd_flags = fd_entry.flags; + + let data = { + match in_fd { + __WASI_STDIN_FILENO => { + let mut stdin = wasi_try_ok_ok!( + WasiInodes::stdin_mut(&state.fs.fd_map).map_err(fs_error_into_wasi_err) + ); + let data = wasi_try_ok_ok!(__asyncify(ctx, None, async move { + // TODO: optimize with MaybeUninit + let mut buf = vec![0u8; sub_count as usize]; + let amt = stdin.read(&mut buf[..]).await.map_err(map_io_err)?; + buf.truncate(amt); + Ok(buf) + })?); + env = ctx.data(); + data + } + __WASI_STDOUT_FILENO | __WASI_STDERR_FILENO => return Ok(Err(Errno::Inval)), + _ => { + if !fd_entry.rights.contains(Rights::FD_READ) { + // TODO: figure out the error to return when lacking rights + return Ok(Err(Errno::Access)); } - __WASI_STDOUT_FILENO | __WASI_STDERR_FILENO => return Ok(Errno::Inval), - _ => { - if !fd_entry.rights.contains(Rights::FD_READ) { - // TODO: figure out the error to return when lacking rights - return Ok(Errno::Access); - } - let offset = fd_entry.offset.load(Ordering::Acquire) as usize; - let inode = fd_entry.inode; - let data = { - let mut guard = inode.write(); - match guard.deref_mut() { - Kind::File { handle, .. } => { - if let Some(handle) = handle { - let data = - wasi_try_ok!(__asyncify(&mut ctx, None, async move { - let mut buf = vec![0u8; sub_count as usize]; - - let mut handle = handle.write().unwrap(); - handle - .seek(std::io::SeekFrom::Start(offset as u64)) - .await - .map_err(map_io_err)?; - let amt = handle - .read(&mut buf[..]) - .await - .map_err(map_io_err)?; - buf.truncate(amt); - Ok(buf) - })?); - env = ctx.data(); - data - } else { - return Ok(Errno::Inval); - } - } - Kind::Socket { socket, .. } => { - let socket = socket.clone(); - let tasks = tasks.clone(); - drop(guard); - - let read_timeout = socket - .opt_time(TimeType::WriteTimeout) - .ok() - .flatten() - .unwrap_or(Duration::from_secs(30)); - - let data = wasi_try_ok!(__asyncify(&mut ctx, None, async { - let mut buf = Vec::with_capacity(sub_count as usize); - unsafe { - buf.set_len(sub_count as usize); - } - socket - .recv( - tasks.deref(), - &mut buf, - Some(read_timeout), - false, - ) - .await - .map(|amt| { - unsafe { - buf.set_len(amt); - } - let buf: Vec = - unsafe { std::mem::transmute(buf) }; - buf - }) - })?); - env = ctx.data(); - data - } - Kind::Pipe { ref mut pipe, .. } => { + let offset = fd_entry.offset.load(Ordering::Acquire) as usize; + let inode = fd_entry.inode; + let data = { + let mut guard = inode.write(); + match guard.deref_mut() { + Kind::File { handle, .. } => { + if let Some(handle) = handle { let data = - wasi_try_ok!(__asyncify(&mut ctx, None, async move { - // TODO: optimize with MaybeUninit + wasi_try_ok_ok!(__asyncify(ctx, None, async move { let mut buf = vec![0u8; sub_count as usize]; - let amt = - virtual_fs::AsyncReadExt::read(pipe, &mut buf[..]) - .await - .map_err(map_io_err)?; + + let mut handle = handle.write().unwrap(); + handle + .seek(std::io::SeekFrom::Start(offset as u64)) + .await + .map_err(map_io_err)?; + let amt = handle + .read(&mut buf[..]) + .await + .map_err(map_io_err)?; buf.truncate(amt); Ok(buf) })?); env = ctx.data(); data + } else { + return Ok(Err(Errno::Inval)); } - Kind::Epoll { .. } => { - return Ok(Errno::Inval); - } - Kind::Dir { .. } | Kind::Root { .. } => { - return Ok(Errno::Isdir); - } - Kind::EventNotifications { .. } => { - return Ok(Errno::Inval); - } - Kind::Symlink { .. } => unimplemented!("Symlinks in wasi::fd_read"), - Kind::Buffer { buffer } => { + } + Kind::Socket { socket, .. } => { + let socket = socket.clone(); + let tasks = tasks.clone(); + drop(guard); + + let read_timeout = socket + .opt_time(TimeType::WriteTimeout) + .ok() + .flatten() + .unwrap_or(Duration::from_secs(30)); + + let data = wasi_try_ok_ok!(__asyncify(ctx, None, async { + let mut buf = Vec::with_capacity(sub_count as usize); + unsafe { + buf.set_len(sub_count as usize); + } + socket + .recv(tasks.deref(), &mut buf, Some(read_timeout), false) + .await + .map(|amt| { + unsafe { + buf.set_len(amt); + } + let buf: Vec = unsafe { std::mem::transmute(buf) }; + buf + }) + })?); + env = ctx.data(); + data + } + Kind::Pipe { ref mut pipe, .. } => { + let data = wasi_try_ok_ok!(__asyncify(ctx, None, async move { // TODO: optimize with MaybeUninit let mut buf = vec![0u8; sub_count as usize]; - - let mut buf_read = &buffer[offset..]; - let amt = wasi_try_ok!(std::io::Read::read( - &mut buf_read, - &mut buf[..] - ) - .map_err(map_io_err)); + let amt = virtual_fs::AsyncReadExt::read(pipe, &mut buf[..]) + .await + .map_err(map_io_err)?; buf.truncate(amt); - buf - } + Ok(buf) + })?); + env = ctx.data(); + data + } + Kind::Epoll { .. } => { + return Ok(Err(Errno::Inval)); + } + Kind::Dir { .. } | Kind::Root { .. } => { + return Ok(Err(Errno::Isdir)); } - }; + Kind::EventNotifications { .. } => { + return Ok(Err(Errno::Inval)); + } + Kind::Symlink { .. } => unimplemented!("Symlinks in wasi::fd_read"), + Kind::Buffer { buffer } => { + // TODO: optimize with MaybeUninit + let mut buf = vec![0u8; sub_count as usize]; + + let mut buf_read = &buffer[offset..]; + let amt = wasi_try_ok_ok!(std::io::Read::read( + &mut buf_read, + &mut buf[..] + ) + .map_err(map_io_err)); + buf.truncate(amt); + buf + } + } + }; - // reborrow - let mut fd_map = state.fs.fd_map.write().unwrap(); - let fd_entry = wasi_try_ok!(fd_map.get_mut(&in_fd).ok_or(Errno::Badf)); - fd_entry - .offset - .fetch_add(data.len() as u64, Ordering::AcqRel); + // reborrow + let mut fd_map = state.fs.fd_map.write().unwrap(); + let fd_entry = wasi_try_ok_ok!(fd_map.get_mut(&in_fd).ok_or(Errno::Badf)); + fd_entry + .offset + .fetch_add(data.len() as u64, Ordering::AcqRel); - data - } + data } - }; - - // Write it down to the socket - let tasks = ctx.data().tasks().clone(); - let bytes_written = wasi_try_ok!(__sock_asyncify_mut( - &mut ctx, - sock, - Rights::SOCK_SEND, - |socket, fd| async move { - let write_timeout = socket - .opt_time(TimeType::ReadTimeout) - .ok() - .flatten() - .unwrap_or(Duration::from_secs(30)); - socket - .send(tasks.deref(), &data, Some(write_timeout), true) - .await - }, - )); - env = ctx.data(); - - total_written += bytes_written as u64; - } - Span::current().record("nsent", total_written); - - let memory = unsafe { env.memory_view(&ctx) }; - wasi_try_mem_ok!(ret_sent.write(&memory, total_written as Filesize)); - - Ok(Errno::Success) - }); - Ok(ret) + } + }; + + // Write it down to the socket + let tasks = ctx.data().tasks().clone(); + let bytes_written = wasi_try_ok_ok!(__sock_asyncify_mut( + ctx, + sock, + Rights::SOCK_SEND, + |socket, fd| async move { + let write_timeout = socket + .opt_time(TimeType::ReadTimeout) + .ok() + .flatten() + .unwrap_or(Duration::from_secs(30)); + socket + .send(tasks.deref(), &data, Some(write_timeout), true) + .await + }, + )); + env = ctx.data(); + + total_written += bytes_written as u64; + } + + Ok(Ok(total_written)) } diff --git a/lib/wasix/src/syscalls/wasix/sock_send_to.rs b/lib/wasix/src/syscalls/wasix/sock_send_to.rs index 7a345bb8504..215c53325c0 100644 --- a/lib/wasix/src/syscalls/wasix/sock_send_to.rs +++ b/lib/wasix/src/syscalls/wasix/sock_send_to.rs @@ -27,7 +27,18 @@ pub fn sock_send_to( addr: WasmPtr<__wasi_addr_port_t, M>, ret_data_len: WasmPtr, ) -> Result { - sock_send_to_internal( + let env = ctx.data(); + let memory = unsafe { env.memory_view(&ctx) }; + let iovs_arr = wasi_try_mem_ok!(si_data.slice(&memory, si_data_len)); + + let (addr_ip, addr_port) = { + let memory = unsafe { env.memory_view(&ctx) }; + wasi_try_ok!(read_ip_port(&memory, addr)) + }; + let addr = SocketAddr::new(addr_ip, addr_port); + Span::current().record("addr", &format!("{:?}", addr)); + + wasi_try_ok!(sock_send_to_internal( ctx, sock, si_data, @@ -35,31 +46,24 @@ pub fn sock_send_to( si_flags, addr, ret_data_len, - ) + )?); + Ok(Errno::Success) } -pub(super) fn sock_send_to_internal( +pub(crate) fn sock_send_to_internal( mut ctx: FunctionEnvMut<'_, WasiEnv>, sock: WasiFd, si_data: WasmPtr<__wasi_ciovec_t, M>, si_data_len: M::Offset, _si_flags: SiFlags, - addr: WasmPtr<__wasi_addr_port_t, M>, + addr: SocketAddr, ret_data_len: WasmPtr, -) -> Result { +) -> Result, WasiError> { let env = ctx.data(); let memory = unsafe { env.memory_view(&ctx) }; - let iovs_arr = wasi_try_mem_ok!(si_data.slice(&memory, si_data_len)); - - let (addr_ip, addr_port) = { - let memory = unsafe { env.memory_view(&ctx) }; - wasi_try_ok!(read_ip_port(&memory, addr)) - }; - let addr = SocketAddr::new(addr_ip, addr_port); - Span::current().record("addr", &format!("{:?}", addr)); let bytes_written = { - wasi_try_ok!(__sock_asyncify( + wasi_try_ok_ok!(__sock_asyncify( env, sock, Rights::SOCK_SEND_TO, @@ -110,8 +114,8 @@ pub(super) fn sock_send_to_internal( let memory = unsafe { env.memory_view(&ctx) }; let bytes_written: M::Offset = - wasi_try_ok!(bytes_written.try_into().map_err(|_| Errno::Overflow)); - wasi_try_mem_ok!(ret_data_len.write(&memory, bytes_written)); + wasi_try_ok_ok!(bytes_written.try_into().map_err(|_| Errno::Overflow)); + wasi_try_mem_ok_ok!(ret_data_len.write(&memory, bytes_written)); - Ok(Errno::Success) + Ok(Ok(())) } diff --git a/lib/wasix/src/syscalls/wasix/sock_set_opt_flag.rs b/lib/wasix/src/syscalls/wasix/sock_set_opt_flag.rs index df6eb21bcfa..8ec9c0a70ab 100644 --- a/lib/wasix/src/syscalls/wasix/sock_set_opt_flag.rs +++ b/lib/wasix/src/syscalls/wasix/sock_set_opt_flag.rs @@ -16,19 +16,30 @@ pub fn sock_set_opt_flag( sock: WasiFd, opt: Sockoption, flag: Bool, -) -> Errno { +) -> Result { + wasi_try_ok!(sock_set_opt_flag_internal(&mut ctx, sock, opt, flag)?); + + Ok(Errno::Success) +} + +pub(crate) fn sock_set_opt_flag_internal( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + sock: WasiFd, + opt: Sockoption, + flag: Bool, +) -> Result, WasiError> { let flag = match flag { Bool::False => false, Bool::True => true, - _ => return Errno::Inval, + _ => return Ok(Err(Errno::Inval)), }; let option: crate::net::socket::WasiSocketOption = opt.into(); - wasi_try!(__sock_actor_mut( - &mut ctx, + wasi_try_ok_ok!(__sock_actor_mut( + ctx, sock, Rights::empty(), |mut socket, _| socket.set_opt_flag(option, flag) )); - Errno::Success + Ok(Ok(())) } diff --git a/lib/wasix/src/syscalls/wasix/sock_set_opt_size.rs b/lib/wasix/src/syscalls/wasix/sock_set_opt_size.rs index a66291b5be1..e5e1aa0b962 100644 --- a/lib/wasix/src/syscalls/wasix/sock_set_opt_size.rs +++ b/lib/wasix/src/syscalls/wasix/sock_set_opt_size.rs @@ -16,19 +16,30 @@ pub fn sock_set_opt_size( sock: WasiFd, opt: Sockoption, size: Filesize, -) -> Errno { +) -> Result { + wasi_try_ok!(sock_set_opt_size_internal(&mut ctx, sock, opt, size)?); + + Ok(Errno::Success) +} + +pub(crate) fn sock_set_opt_size_internal( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + sock: WasiFd, + opt: Sockoption, + size: Filesize, +) -> Result, WasiError> { let ty = match opt { Sockoption::RecvTimeout => TimeType::ReadTimeout, Sockoption::SendTimeout => TimeType::WriteTimeout, Sockoption::ConnectTimeout => TimeType::ConnectTimeout, Sockoption::AcceptTimeout => TimeType::AcceptTimeout, Sockoption::Linger => TimeType::Linger, - _ => return Errno::Inval, + _ => return Ok(Err(Errno::Inval)), }; let option: crate::net::socket::WasiSocketOption = opt.into(); - wasi_try!(__sock_actor_mut( - &mut ctx, + wasi_try_ok_ok!(__sock_actor_mut( + ctx, sock, Rights::empty(), |mut socket, _| match opt { @@ -39,5 +50,5 @@ pub fn sock_set_opt_size( _ => Err(Errno::Inval), } )); - Errno::Success + Ok(Ok(())) } diff --git a/lib/wasix/src/syscalls/wasix/sock_set_opt_time.rs b/lib/wasix/src/syscalls/wasix/sock_set_opt_time.rs index 49e9c037987..91d7ca93ce7 100644 --- a/lib/wasix/src/syscalls/wasix/sock_set_opt_time.rs +++ b/lib/wasix/src/syscalls/wasix/sock_set_opt_time.rs @@ -15,14 +15,14 @@ pub fn sock_set_opt_time( sock: WasiFd, opt: Sockoption, time: WasmPtr, -) -> Errno { +) -> Result { let env = ctx.data(); let memory = unsafe { env.memory_view(&ctx) }; - let time = wasi_try_mem!(time.read(&memory)); + let time = wasi_try_mem_ok!(time.read(&memory)); let time = match time.tag { OptionTag::None => None, OptionTag::Some => Some(Duration::from_nanos(time.u)), - _ => return Errno::Inval, + _ => return Ok(Errno::Inval), }; Span::current().record("time", &format!("{:?}", time)); @@ -32,15 +32,23 @@ pub fn sock_set_opt_time( Sockoption::ConnectTimeout => TimeType::ConnectTimeout, Sockoption::AcceptTimeout => TimeType::AcceptTimeout, Sockoption::Linger => TimeType::Linger, - _ => return Errno::Inval, + _ => return Ok(Errno::Inval), }; - let option: crate::net::socket::WasiSocketOption = opt.into(); - wasi_try!(__sock_actor_mut( - &mut ctx, - sock, - Rights::empty(), - |socket, _| socket.set_opt_time(ty, time) - )); - Errno::Success + wasi_try_ok!(sock_set_opt_time_internal(&mut ctx, sock, ty, time)?); + + Ok(Errno::Success) +} + +pub(crate) fn sock_set_opt_time_internal( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + sock: WasiFd, + ty: TimeType, + time: Option, +) -> Result, WasiError> { + wasi_try_ok_ok!(__sock_actor_mut(ctx, sock, Rights::empty(), |socket, _| { + socket.set_opt_time(ty, time) + })); + + Ok(Ok(())) } diff --git a/lib/wasix/src/syscalls/wasix/sock_shutdown.rs b/lib/wasix/src/syscalls/wasix/sock_shutdown.rs index 36cdd22cbcd..037b6b73024 100644 --- a/lib/wasix/src/syscalls/wasix/sock_shutdown.rs +++ b/lib/wasix/src/syscalls/wasix/sock_shutdown.rs @@ -1,3 +1,5 @@ +use std::net::Shutdown; + use super::*; use crate::syscalls::*; @@ -9,21 +11,35 @@ use crate::syscalls::*; /// /// * `how` - Which channels on the socket to shut down. #[instrument(level = "debug", skip_all, fields(%sock), ret)] -pub fn sock_shutdown(mut ctx: FunctionEnvMut<'_, WasiEnv>, sock: WasiFd, how: SdFlags) -> Errno { +pub fn sock_shutdown( + mut ctx: FunctionEnvMut<'_, WasiEnv>, + sock: WasiFd, + how: SdFlags, +) -> Result { let both = __WASI_SHUT_RD | __WASI_SHUT_WR; - let how = match how { + let shutdown = match how { __WASI_SHUT_RD => std::net::Shutdown::Read, __WASI_SHUT_WR => std::net::Shutdown::Write, a if a == both => std::net::Shutdown::Both, - _ => return Errno::Inval, + _ => return Ok(Errno::Inval), }; - wasi_try!(__sock_actor_mut( - &mut ctx, + wasi_try_ok!(sock_shutdown_internal(&mut ctx, sock, shutdown)?); + + Ok(Errno::Success) +} + +pub(crate) fn sock_shutdown_internal( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + sock: WasiFd, + shutdown: Shutdown, +) -> Result, WasiError> { + wasi_try_ok_ok!(__sock_actor_mut( + ctx, sock, Rights::SOCK_SHUTDOWN, - |mut socket, _| socket.shutdown(how) + |mut socket, _| socket.shutdown(shutdown) )); - Errno::Success + Ok(Ok(())) } From e53e11cc44bb5ef43c1394879608ad07dd8167af Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Tue, 21 Nov 2023 14:56:06 +1100 Subject: [PATCH 055/129] Added the socket syscall hooks --- lib/wasix/src/journaling/effector/mod.rs | 26 +++ .../journaling/effector/syscalls/fd_seek.rs | 5 +- .../effector/syscalls/fd_set_flags.rs | 17 +- .../effector/syscalls/fd_set_rights.rs | 2 +- .../effector/syscalls/fd_set_size.rs | 17 +- .../effector/syscalls/fd_set_times.rs | 2 +- .../effector/syscalls/port_addr_add.rs | 29 +++ .../effector/syscalls/port_addr_clear.rs | 20 ++ .../effector/syscalls/port_addr_remove.rs | 29 +++ .../effector/syscalls/port_bridge.rs | 41 ++++ .../effector/syscalls/port_dhcp_acquire.rs | 20 ++ .../effector/syscalls/port_gateway_set.rs | 29 +++ .../effector/syscalls/port_route_add.rs | 52 +++++ .../effector/syscalls/port_route_clear.rs | 20 ++ .../effector/syscalls/port_route_remove.rs | 29 +++ .../effector/syscalls/port_unbridge.rs | 20 ++ .../effector/syscalls/sock_accept.rs | 83 +++++++ .../journaling/effector/syscalls/sock_bind.rs | 32 +++ .../effector/syscalls/sock_connect.rs | 58 +++++ .../syscalls/sock_join_ipv4_multicast.rs | 42 ++++ .../syscalls/sock_join_ipv6_multicast.rs | 42 ++++ .../syscalls/sock_leave_ipv4_multicast.rs | 42 ++++ .../syscalls/sock_leave_ipv6_multicast.rs | 42 ++++ .../effector/syscalls/sock_listen.rs | 36 +++ .../journaling/effector/syscalls/sock_open.rs | 54 +++++ .../journaling/effector/syscalls/sock_send.rs | 71 ++++++ .../effector/syscalls/sock_send_file.rs | 43 ++++ .../effector/syscalls/sock_send_to.rs | 80 +++++++ .../effector/syscalls/sock_set_opt_flag.rs | 35 +++ .../effector/syscalls/sock_set_opt_size.rs | 35 +++ .../effector/syscalls/sock_set_opt_time.rs | 37 ++++ .../effector/syscalls/sock_shutdown.rs | 32 +++ lib/wasix/src/journaling/filter.rs | 4 +- lib/wasix/src/journaling/journal.rs | 31 ++- lib/wasix/src/journaling/log_file.rs | 209 ++++++++++++++---- lib/wasix/src/net/socket.rs | 21 +- lib/wasix/src/syscalls/mod.rs | 161 +++++++++++--- lib/wasix/src/syscalls/wasix/sock_accept.rs | 5 +- lib/wasix/src/syscalls/wasix/sock_open.rs | 5 +- lib/wasix/src/syscalls/wasix/sock_send.rs | 4 +- .../src/syscalls/wasix/sock_send_file.rs | 4 +- lib/wasix/src/syscalls/wasix/sock_send_to.rs | 112 ++++++---- .../src/syscalls/wasix/sock_set_opt_flag.rs | 14 +- 43 files changed, 1510 insertions(+), 182 deletions(-) create mode 100644 lib/wasix/src/journaling/effector/syscalls/port_addr_add.rs create mode 100644 lib/wasix/src/journaling/effector/syscalls/port_addr_clear.rs create mode 100644 lib/wasix/src/journaling/effector/syscalls/port_addr_remove.rs create mode 100644 lib/wasix/src/journaling/effector/syscalls/port_bridge.rs create mode 100644 lib/wasix/src/journaling/effector/syscalls/port_dhcp_acquire.rs create mode 100644 lib/wasix/src/journaling/effector/syscalls/port_gateway_set.rs create mode 100644 lib/wasix/src/journaling/effector/syscalls/port_route_add.rs create mode 100644 lib/wasix/src/journaling/effector/syscalls/port_route_clear.rs create mode 100644 lib/wasix/src/journaling/effector/syscalls/port_route_remove.rs create mode 100644 lib/wasix/src/journaling/effector/syscalls/port_unbridge.rs create mode 100644 lib/wasix/src/journaling/effector/syscalls/sock_accept.rs create mode 100644 lib/wasix/src/journaling/effector/syscalls/sock_bind.rs create mode 100644 lib/wasix/src/journaling/effector/syscalls/sock_connect.rs create mode 100644 lib/wasix/src/journaling/effector/syscalls/sock_join_ipv4_multicast.rs create mode 100644 lib/wasix/src/journaling/effector/syscalls/sock_join_ipv6_multicast.rs create mode 100644 lib/wasix/src/journaling/effector/syscalls/sock_leave_ipv4_multicast.rs create mode 100644 lib/wasix/src/journaling/effector/syscalls/sock_leave_ipv6_multicast.rs create mode 100644 lib/wasix/src/journaling/effector/syscalls/sock_listen.rs create mode 100644 lib/wasix/src/journaling/effector/syscalls/sock_open.rs create mode 100644 lib/wasix/src/journaling/effector/syscalls/sock_send.rs create mode 100644 lib/wasix/src/journaling/effector/syscalls/sock_send_file.rs create mode 100644 lib/wasix/src/journaling/effector/syscalls/sock_send_to.rs create mode 100644 lib/wasix/src/journaling/effector/syscalls/sock_set_opt_flag.rs create mode 100644 lib/wasix/src/journaling/effector/syscalls/sock_set_opt_size.rs create mode 100644 lib/wasix/src/journaling/effector/syscalls/sock_set_opt_time.rs create mode 100644 lib/wasix/src/journaling/effector/syscalls/sock_shutdown.rs diff --git a/lib/wasix/src/journaling/effector/mod.rs b/lib/wasix/src/journaling/effector/mod.rs index 7059855a76c..36de01e93b9 100644 --- a/lib/wasix/src/journaling/effector/mod.rs +++ b/lib/wasix/src/journaling/effector/mod.rs @@ -51,6 +51,32 @@ mod syscalls { mod path_set_times; mod path_symlink; mod path_unlink; + mod port_addr_add; + mod port_addr_clear; + mod port_addr_remove; + mod port_bridge; + mod port_dhcp_acquire; + mod port_gateway_set; + mod port_route_add; + mod port_route_clear; + mod port_route_remove; + mod port_unbridge; + mod sock_accept; + mod sock_bind; + mod sock_connect; + mod sock_join_ipv4_multicast; + mod sock_join_ipv6_multicast; + mod sock_leave_ipv4_multicast; + mod sock_leave_ipv6_multicast; + mod sock_listen; + mod sock_open; + mod sock_send; + mod sock_send_file; + mod sock_send_to; + mod sock_set_opt_flag; + mod sock_set_opt_size; + mod sock_set_opt_time; + mod sock_shutdown; mod tty_set; } #[cfg(feature = "journal")] diff --git a/lib/wasix/src/journaling/effector/syscalls/fd_seek.rs b/lib/wasix/src/journaling/effector/syscalls/fd_seek.rs index ecb19c1159b..039d4808b83 100644 --- a/lib/wasix/src/journaling/effector/syscalls/fd_seek.rs +++ b/lib/wasix/src/journaling/effector/syscalls/fd_seek.rs @@ -16,10 +16,9 @@ impl JournalEffector { offset: i64, whence: Whence, ) -> anyhow::Result<()> { - crate::syscalls::fd_seek_internal(ctx, fd, offset, whence)? - .map_err(|err| { + crate::syscalls::fd_seek_internal(ctx, fd, offset, whence)?.map_err(|err| { anyhow::format_err!( - "journal restore error: failed to seek descriptor (fd={}, offset={}, whence={:?}) - {}", + "journal restore error: failed to seek (fd={}, offset={}, whence={:?}) - {}", fd, offset, whence, diff --git a/lib/wasix/src/journaling/effector/syscalls/fd_set_flags.rs b/lib/wasix/src/journaling/effector/syscalls/fd_set_flags.rs index ba9e118b92e..293902275d3 100644 --- a/lib/wasix/src/journaling/effector/syscalls/fd_set_flags.rs +++ b/lib/wasix/src/journaling/effector/syscalls/fd_set_flags.rs @@ -14,15 +14,14 @@ impl JournalEffector { fd: Fd, flags: Fdflags, ) -> anyhow::Result<()> { - crate::syscalls::fd_fdstat_set_flags_internal(ctx, fd, flags) - .map_err(|err| { - anyhow::format_err!( - "journal restore error: failed to duplicate file descriptor (fd={}, flags={:?}) - {}", - fd, - flags, - err - ) - })?; + crate::syscalls::fd_fdstat_set_flags_internal(ctx, fd, flags).map_err(|err| { + anyhow::format_err!( + "journal restore error: failed to set file flags (fd={}, flags={:?}) - {}", + fd, + flags, + err + ) + })?; Ok(()) } } diff --git a/lib/wasix/src/journaling/effector/syscalls/fd_set_rights.rs b/lib/wasix/src/journaling/effector/syscalls/fd_set_rights.rs index b86539c0c14..a86a3f0a5e8 100644 --- a/lib/wasix/src/journaling/effector/syscalls/fd_set_rights.rs +++ b/lib/wasix/src/journaling/effector/syscalls/fd_set_rights.rs @@ -26,7 +26,7 @@ impl JournalEffector { crate::syscalls::fd_fdstat_set_rights_internal(ctx, fd, fs_rights_base, fs_rights_inheriting) .map_err(|err| { anyhow::format_err!( - "journal restore error: failed to duplicate file descriptor (fd={}, fs_rights_base={:?}, fs_rights_inheriting={:?}) - {}", + "journal restore error: failed to set file rights (fd={}, fs_rights_base={:?}, fs_rights_inheriting={:?}) - {}", fd, fs_rights_base, fs_rights_inheriting, diff --git a/lib/wasix/src/journaling/effector/syscalls/fd_set_size.rs b/lib/wasix/src/journaling/effector/syscalls/fd_set_size.rs index ec24f9ed0ad..2ad7531df88 100644 --- a/lib/wasix/src/journaling/effector/syscalls/fd_set_size.rs +++ b/lib/wasix/src/journaling/effector/syscalls/fd_set_size.rs @@ -14,15 +14,14 @@ impl JournalEffector { fd: Fd, st_size: Filesize, ) -> anyhow::Result<()> { - crate::syscalls::fd_filestat_set_size_internal(ctx, fd, st_size) - .map_err(|err| { - anyhow::format_err!( - "journal restore error: failed to duplicate file descriptor (fd={}, st_size={}) - {}", - fd, - st_size, - err - ) - })?; + crate::syscalls::fd_filestat_set_size_internal(ctx, fd, st_size).map_err(|err| { + anyhow::format_err!( + "journal restore error: failed to set file size (fd={}, st_size={}) - {}", + fd, + st_size, + err + ) + })?; Ok(()) } } diff --git a/lib/wasix/src/journaling/effector/syscalls/fd_set_times.rs b/lib/wasix/src/journaling/effector/syscalls/fd_set_times.rs index 6500378d392..26f17565b8a 100644 --- a/lib/wasix/src/journaling/effector/syscalls/fd_set_times.rs +++ b/lib/wasix/src/journaling/effector/syscalls/fd_set_times.rs @@ -29,7 +29,7 @@ impl JournalEffector { crate::syscalls::fd_filestat_set_times_internal(ctx, fd, st_atim, st_mtim, fst_flags) .map_err(|err| { anyhow::format_err!( - "journal restore error: failed to duplicate file descriptor (fd={}, st_atim={}, st_mtim={}, fst_flags={:?}) - {}", + "journal restore error: failed to set file times (fd={}, st_atim={}, st_mtim={}, fst_flags={:?}) - {}", fd, st_atim, st_mtim, diff --git a/lib/wasix/src/journaling/effector/syscalls/port_addr_add.rs b/lib/wasix/src/journaling/effector/syscalls/port_addr_add.rs new file mode 100644 index 00000000000..d3d3d51edfc --- /dev/null +++ b/lib/wasix/src/journaling/effector/syscalls/port_addr_add.rs @@ -0,0 +1,29 @@ +use virtual_net::IpCidr; + +use super::*; + +impl JournalEffector { + pub fn save_port_addr_add( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + cidr: IpCidr, + ) -> anyhow::Result<()> { + Self::save_event(ctx, JournalEntry::PortAddAddr { cidr }) + } + + pub fn apply_port_addr_add( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + cidr: IpCidr, + ) -> anyhow::Result<()> { + crate::syscalls::port_addr_add_internal(ctx, cidr) + .map(|r| r.map_err(|err| err.to_string())) + .unwrap_or_else(|err| Err(err.to_string())) + .map_err(|err| { + anyhow::format_err!( + "journal restore error: failed to add address to port file descriptor (cidr={:?}) - {}", + cidr, + err + ) + })?; + Ok(()) + } +} diff --git a/lib/wasix/src/journaling/effector/syscalls/port_addr_clear.rs b/lib/wasix/src/journaling/effector/syscalls/port_addr_clear.rs new file mode 100644 index 00000000000..078fab9157d --- /dev/null +++ b/lib/wasix/src/journaling/effector/syscalls/port_addr_clear.rs @@ -0,0 +1,20 @@ +use super::*; + +impl JournalEffector { + pub fn save_port_addr_clear(ctx: &mut FunctionEnvMut<'_, WasiEnv>) -> anyhow::Result<()> { + Self::save_event(ctx, JournalEntry::PortAddrClear) + } + + pub fn apply_port_addr_clear(ctx: &mut FunctionEnvMut<'_, WasiEnv>) -> anyhow::Result<()> { + crate::syscalls::port_addr_clear_internal(ctx) + .map(|r| r.map_err(|err| err.to_string())) + .unwrap_or_else(|err| Err(err.to_string())) + .map_err(|err| { + anyhow::format_err!( + "journal restore error: failed to clear port addresses - {}", + err + ) + })?; + Ok(()) + } +} diff --git a/lib/wasix/src/journaling/effector/syscalls/port_addr_remove.rs b/lib/wasix/src/journaling/effector/syscalls/port_addr_remove.rs new file mode 100644 index 00000000000..88d31ec2f60 --- /dev/null +++ b/lib/wasix/src/journaling/effector/syscalls/port_addr_remove.rs @@ -0,0 +1,29 @@ +use virtual_net::IpAddr; + +use super::*; + +impl JournalEffector { + pub fn save_port_addr_remove( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + addr: IpAddr, + ) -> anyhow::Result<()> { + Self::save_event(ctx, JournalEntry::PortDelAddr { addr }) + } + + pub fn apply_port_addr_remove( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + addr: IpAddr, + ) -> anyhow::Result<()> { + crate::syscalls::port_addr_remove_internal(ctx, addr) + .map(|r| r.map_err(|err| err.to_string())) + .unwrap_or_else(|err| Err(err.to_string())) + .map_err(|err| { + anyhow::format_err!( + "journal restore error: failed to remove address from port (ip={}) - {}", + addr, + err + ) + })?; + Ok(()) + } +} diff --git a/lib/wasix/src/journaling/effector/syscalls/port_bridge.rs b/lib/wasix/src/journaling/effector/syscalls/port_bridge.rs new file mode 100644 index 00000000000..66029849932 --- /dev/null +++ b/lib/wasix/src/journaling/effector/syscalls/port_bridge.rs @@ -0,0 +1,41 @@ +use virtual_net::StreamSecurity; + +use super::*; + +impl JournalEffector { + pub fn save_port_bridge( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + network: String, + token: String, + security: StreamSecurity, + ) -> anyhow::Result<()> { + Self::save_event( + ctx, + JournalEntry::PortBridge { + network, + token, + security, + }, + ) + } + + pub fn apply_port_bridge( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + network: &str, + token: &str, + security: StreamSecurity, + ) -> anyhow::Result<()> { + crate::syscalls::port_bridge_internal(ctx, network, token, security) + .map(|r| r.map_err(|err| err.to_string())) + .unwrap_or_else(|err| Err(err.to_string())) + .map_err(|err| { + anyhow::format_err!( + "journal restore error: failed to bridge the network file descriptor (network={}, security={:?}) - {}", + network, + security, + err + ) + })?; + Ok(()) + } +} diff --git a/lib/wasix/src/journaling/effector/syscalls/port_dhcp_acquire.rs b/lib/wasix/src/journaling/effector/syscalls/port_dhcp_acquire.rs new file mode 100644 index 00000000000..cfa6ea3e930 --- /dev/null +++ b/lib/wasix/src/journaling/effector/syscalls/port_dhcp_acquire.rs @@ -0,0 +1,20 @@ +use super::*; + +impl JournalEffector { + pub fn save_port_dhcp_acquire(ctx: &mut FunctionEnvMut<'_, WasiEnv>) -> anyhow::Result<()> { + Self::save_event(ctx, JournalEntry::PortDhcpAcquire) + } + + pub fn apply_port_dhcp_acquire(ctx: &mut FunctionEnvMut<'_, WasiEnv>) -> anyhow::Result<()> { + crate::syscalls::port_dhcp_acquire_internal(ctx) + .map(|r| r.map_err(|err| err.to_string())) + .unwrap_or_else(|err| Err(err.to_string())) + .map_err(|err| { + anyhow::format_err!( + "journal restore error: failed to acquire DHCP address - {}", + err + ) + })?; + Ok(()) + } +} diff --git a/lib/wasix/src/journaling/effector/syscalls/port_gateway_set.rs b/lib/wasix/src/journaling/effector/syscalls/port_gateway_set.rs new file mode 100644 index 00000000000..22247418c7f --- /dev/null +++ b/lib/wasix/src/journaling/effector/syscalls/port_gateway_set.rs @@ -0,0 +1,29 @@ +use virtual_net::IpAddr; + +use super::*; + +impl JournalEffector { + pub fn save_port_gateway_set( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + ip: IpAddr, + ) -> anyhow::Result<()> { + Self::save_event(ctx, JournalEntry::PortGatewaySet { ip }) + } + + pub fn apply_port_gateway_set( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + ip: IpAddr, + ) -> anyhow::Result<()> { + crate::syscalls::port_gateway_set_internal(ctx, ip) + .map(|r| r.map_err(|err| err.to_string())) + .unwrap_or_else(|err| Err(err.to_string())) + .map_err(|err| { + anyhow::format_err!( + "journal restore error: failed to set gateway address (ip={}) - {}", + ip, + err + ) + })?; + Ok(()) + } +} diff --git a/lib/wasix/src/journaling/effector/syscalls/port_route_add.rs b/lib/wasix/src/journaling/effector/syscalls/port_route_add.rs new file mode 100644 index 00000000000..489d440dc60 --- /dev/null +++ b/lib/wasix/src/journaling/effector/syscalls/port_route_add.rs @@ -0,0 +1,52 @@ +use virtual_net::{Duration, IpAddr, IpCidr}; + +use super::*; + +impl JournalEffector { + pub fn save_port_route_add( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + cidr: IpCidr, + via_router: IpAddr, + preferred_until: Option, + expires_at: Option, + ) -> anyhow::Result<()> { + Self::save_event( + ctx, + JournalEntry::PortRouteAdd { + cidr, + via_router, + preferred_until, + expires_at, + }, + ) + } + + pub fn apply_port_route_add( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + cidr: IpCidr, + via_router: IpAddr, + preferred_until: Option, + expires_at: Option, + ) -> anyhow::Result<()> { + crate::syscalls::port_route_add_internal( + ctx, + cidr, + via_router, + preferred_until, + expires_at, + ) + .map(|r| r.map_err(|err| err.to_string())) + .unwrap_or_else(|err| Err(err.to_string())) + .map_err(|err| { + anyhow::format_err!( + "journal restore error: failed to add route (cidr={:?}, via_router={}, preferred_until={:?}, expires_at={:?}) - {}", + cidr, + via_router, + preferred_until, + expires_at, + err + ) + })?; + Ok(()) + } +} diff --git a/lib/wasix/src/journaling/effector/syscalls/port_route_clear.rs b/lib/wasix/src/journaling/effector/syscalls/port_route_clear.rs new file mode 100644 index 00000000000..f02a3adad9e --- /dev/null +++ b/lib/wasix/src/journaling/effector/syscalls/port_route_clear.rs @@ -0,0 +1,20 @@ +use super::*; + +impl JournalEffector { + pub fn save_port_route_clear(ctx: &mut FunctionEnvMut<'_, WasiEnv>) -> anyhow::Result<()> { + Self::save_event(ctx, JournalEntry::PortRouteClear) + } + + pub fn apply_port_route_clear(ctx: &mut FunctionEnvMut<'_, WasiEnv>) -> anyhow::Result<()> { + crate::syscalls::port_route_clear_internal(ctx) + .map(|r| r.map_err(|err| err.to_string())) + .unwrap_or_else(|err| Err(err.to_string())) + .map_err(|err| { + anyhow::format_err!( + "journal restore error: failed to clear routing table - {}", + err + ) + })?; + Ok(()) + } +} diff --git a/lib/wasix/src/journaling/effector/syscalls/port_route_remove.rs b/lib/wasix/src/journaling/effector/syscalls/port_route_remove.rs new file mode 100644 index 00000000000..00af31c21da --- /dev/null +++ b/lib/wasix/src/journaling/effector/syscalls/port_route_remove.rs @@ -0,0 +1,29 @@ +use virtual_net::IpAddr; + +use super::*; + +impl JournalEffector { + pub fn save_port_route_remove( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + ip: IpAddr, + ) -> anyhow::Result<()> { + Self::save_event(ctx, JournalEntry::PortRouteDel { ip }) + } + + pub fn apply_port_route_remove( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + ip: IpAddr, + ) -> anyhow::Result<()> { + crate::syscalls::port_route_remove_internal(ctx, ip) + .map(|r| r.map_err(|err| err.to_string())) + .unwrap_or_else(|err| Err(err.to_string())) + .map_err(|err| { + anyhow::format_err!( + "journal restore error: failed to remove route (ip={}) - {}", + ip, + err + ) + })?; + Ok(()) + } +} diff --git a/lib/wasix/src/journaling/effector/syscalls/port_unbridge.rs b/lib/wasix/src/journaling/effector/syscalls/port_unbridge.rs new file mode 100644 index 00000000000..d5a4e25ec78 --- /dev/null +++ b/lib/wasix/src/journaling/effector/syscalls/port_unbridge.rs @@ -0,0 +1,20 @@ +use super::*; + +impl JournalEffector { + pub fn save_port_unbridge(ctx: &mut FunctionEnvMut<'_, WasiEnv>) -> anyhow::Result<()> { + Self::save_event(ctx, JournalEntry::PortUnbridge) + } + + pub fn apply_port_unbridge(ctx: &mut FunctionEnvMut<'_, WasiEnv>) -> anyhow::Result<()> { + crate::syscalls::port_unbridge_internal(ctx) + .map(|r| r.map_err(|err| err.to_string())) + .unwrap_or_else(|err| Err(err.to_string())) + .map_err(|err| { + anyhow::format_err!( + "journal restore error: failed to un-bridge network - {}", + err + ) + })?; + Ok(()) + } +} diff --git a/lib/wasix/src/journaling/effector/syscalls/sock_accept.rs b/lib/wasix/src/journaling/effector/syscalls/sock_accept.rs new file mode 100644 index 00000000000..0985509d05d --- /dev/null +++ b/lib/wasix/src/journaling/effector/syscalls/sock_accept.rs @@ -0,0 +1,83 @@ +use virtual_net::SocketAddr; + +use crate::{ + fs::Kind, + net::socket::{InodeSocket, InodeSocketKind}, +}; + +use super::*; + +impl JournalEffector { + pub fn save_sock_accepted( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + listen_fd: Fd, + fd: Fd, + peer_addr: SocketAddr, + fd_flags: Fdflags, + nonblocking: bool, + ) -> anyhow::Result<()> { + Self::save_event( + ctx, + JournalEntry::SocketAccepted { + listen_fd, + fd, + peer_addr, + fd_flags, + nonblocking, + }, + ) + } + + pub fn apply_sock_accepted( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + _listen_fd: Fd, + fd: Fd, + peer_addr: SocketAddr, + fd_flags: Fdflags, + nonblocking: bool, + ) -> anyhow::Result<()> { + let kind = Kind::Socket { + socket: InodeSocket::new(InodeSocketKind::RemoteTcpStream { peer_addr }), + }; + + let env = ctx.data(); + let state = env.state(); + let inodes = &state.inodes; + let inode = state + .fs + .create_inode_with_default_stat(inodes, kind, false, "socket".into()); + + let mut new_flags = Fdflags::empty(); + if nonblocking { + new_flags.set(Fdflags::NONBLOCK, true); + } + + let mut new_flags = Fdflags::empty(); + if fd_flags.contains(Fdflags::NONBLOCK) { + new_flags.set(Fdflags::NONBLOCK, true); + } + + let rights = Rights::all_socket(); + let ret_fd = state + .fs + .create_fd(rights, rights, new_flags, 0, inode) + .map_err(|err| { + anyhow::format_err!( + "journal restore error: failed to create remote accepted socket - {}", + err + ) + })?; + + let ret = crate::syscalls::fd_renumber_internal(ctx, ret_fd, fd); + if ret != Errno::Success { + bail!( + "journal restore error: failed renumber file descriptor after accepting socket (from={}, to={}) - {}", + ret_fd, + fd, + ret + ); + } + + Ok(()) + } +} diff --git a/lib/wasix/src/journaling/effector/syscalls/sock_bind.rs b/lib/wasix/src/journaling/effector/syscalls/sock_bind.rs new file mode 100644 index 00000000000..25e63641542 --- /dev/null +++ b/lib/wasix/src/journaling/effector/syscalls/sock_bind.rs @@ -0,0 +1,32 @@ +use virtual_net::SocketAddr; + +use super::*; + +impl JournalEffector { + pub fn save_sock_bind( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + fd: Fd, + addr: SocketAddr, + ) -> anyhow::Result<()> { + Self::save_event(ctx, JournalEntry::SocketBind { fd, addr }) + } + + pub fn apply_sock_bind( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + fd: Fd, + addr: SocketAddr, + ) -> anyhow::Result<()> { + crate::syscalls::sock_bind_internal(ctx, fd, addr) + .map(|r| r.map_err(|err| err.to_string())) + .unwrap_or_else(|err| Err(err.to_string())) + .map_err(|err| { + anyhow::format_err!( + "journal restore error: failed to bind socket to address (fd={}, addr={}) - {}", + fd, + addr, + err + ) + })?; + Ok(()) + } +} diff --git a/lib/wasix/src/journaling/effector/syscalls/sock_connect.rs b/lib/wasix/src/journaling/effector/syscalls/sock_connect.rs new file mode 100644 index 00000000000..9558b82af1e --- /dev/null +++ b/lib/wasix/src/journaling/effector/syscalls/sock_connect.rs @@ -0,0 +1,58 @@ +use virtual_net::SocketAddr; + +use crate::{ + fs::Kind, + net::socket::{InodeSocket, InodeSocketKind}, +}; + +use super::*; + +impl JournalEffector { + pub fn save_sock_connect( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + fd: Fd, + addr: SocketAddr, + ) -> anyhow::Result<()> { + Self::save_event(ctx, JournalEntry::SocketConnected { fd, addr }) + } + + pub fn apply_sock_connect( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + fd: Fd, + addr: SocketAddr, + ) -> anyhow::Result<()> { + let kind = Kind::Socket { + socket: InodeSocket::new(InodeSocketKind::RemoteTcpStream { peer_addr: addr }), + }; + + let env = ctx.data(); + let state = env.state(); + let inodes = &state.inodes; + let inode = state + .fs + .create_inode_with_default_stat(inodes, kind, false, "socket".into()); + + let rights = Rights::all_socket(); + let ret_fd = state + .fs + .create_fd(rights, rights, Fdflags::empty(), 0, inode) + .map_err(|err| { + anyhow::format_err!( + "journal restore error: failed to create remote connected socket - {}", + err + ) + })?; + + let ret = crate::syscalls::fd_renumber_internal(ctx, ret_fd, fd); + if ret != Errno::Success { + bail!( + "journal restore error: failed renumber file descriptor after connecting the socket (from={}, to={}) - {}", + ret_fd, + fd, + ret + ); + } + + Ok(()) + } +} diff --git a/lib/wasix/src/journaling/effector/syscalls/sock_join_ipv4_multicast.rs b/lib/wasix/src/journaling/effector/syscalls/sock_join_ipv4_multicast.rs new file mode 100644 index 00000000000..6768175d189 --- /dev/null +++ b/lib/wasix/src/journaling/effector/syscalls/sock_join_ipv4_multicast.rs @@ -0,0 +1,42 @@ +use virtual_net::Ipv4Addr; + +use super::*; + +impl JournalEffector { + pub fn save_sock_join_ipv4_multicast( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + fd: Fd, + multiaddr: Ipv4Addr, + iface: Ipv4Addr, + ) -> anyhow::Result<()> { + Self::save_event( + ctx, + JournalEntry::SocketJoinIpv4Multicast { + fd, + multiaddr, + iface, + }, + ) + } + + pub fn apply_sock_join_ipv4_multicast( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + fd: Fd, + multiaddr: Ipv4Addr, + iface: Ipv4Addr, + ) -> anyhow::Result<()> { + crate::syscalls::sock_join_multicast_v4_internal(ctx, fd, multiaddr, iface) + .map(|r| r.map_err(|err| err.to_string())) + .unwrap_or_else(|err| Err(err.to_string())) + .map_err(|err| { + anyhow::format_err!( + "journal restore error: failed to join ipv4 multicast (fd={}, multiaddr={:?}, iface={:?}) - {}", + fd, + multiaddr, + iface, + err + ) + })?; + Ok(()) + } +} diff --git a/lib/wasix/src/journaling/effector/syscalls/sock_join_ipv6_multicast.rs b/lib/wasix/src/journaling/effector/syscalls/sock_join_ipv6_multicast.rs new file mode 100644 index 00000000000..aa42495e8f4 --- /dev/null +++ b/lib/wasix/src/journaling/effector/syscalls/sock_join_ipv6_multicast.rs @@ -0,0 +1,42 @@ +use virtual_net::Ipv6Addr; + +use super::*; + +impl JournalEffector { + pub fn save_sock_join_ipv6_multicast( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + fd: Fd, + multiaddr: Ipv6Addr, + iface: u32, + ) -> anyhow::Result<()> { + Self::save_event( + ctx, + JournalEntry::SocketJoinIpv6Multicast { + fd, + multiaddr, + iface, + }, + ) + } + + pub fn apply_sock_join_ipv6_multicast( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + fd: Fd, + multiaddr: Ipv6Addr, + iface: u32, + ) -> anyhow::Result<()> { + crate::syscalls::sock_join_multicast_v6_internal(ctx, fd, multiaddr, iface) + .map(|r| r.map_err(|err| err.to_string())) + .unwrap_or_else(|err| Err(err.to_string())) + .map_err(|err| { + anyhow::format_err!( + "journal restore error: failed to join ipv6 multicast (fd={}, multiaddr={}, iface={}) - {}", + fd, + multiaddr, + iface, + err + ) + })?; + Ok(()) + } +} diff --git a/lib/wasix/src/journaling/effector/syscalls/sock_leave_ipv4_multicast.rs b/lib/wasix/src/journaling/effector/syscalls/sock_leave_ipv4_multicast.rs new file mode 100644 index 00000000000..df6d6cdf589 --- /dev/null +++ b/lib/wasix/src/journaling/effector/syscalls/sock_leave_ipv4_multicast.rs @@ -0,0 +1,42 @@ +use virtual_net::Ipv4Addr; + +use super::*; + +impl JournalEffector { + pub fn save_sock_leave_ipv4_multicast( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + fd: Fd, + multiaddr: Ipv4Addr, + iface: Ipv4Addr, + ) -> anyhow::Result<()> { + Self::save_event( + ctx, + JournalEntry::SocketLeaveIpv4Multicast { + fd, + multiaddr, + iface, + }, + ) + } + + pub fn apply_sock_leave_ipv4_multicast( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + fd: Fd, + multiaddr: Ipv4Addr, + iface: Ipv4Addr, + ) -> anyhow::Result<()> { + crate::syscalls::sock_leave_multicast_v4_internal(ctx, fd, multiaddr, iface) + .map(|r| r.map_err(|err| err.to_string())) + .unwrap_or_else(|err| Err(err.to_string())) + .map_err(|err| { + anyhow::format_err!( + "journal restore error: failed to leave ipv4 multicast (fd={}, multiaddr={}, iface={}) - {}", + fd, + multiaddr, + iface, + err + ) + })?; + Ok(()) + } +} diff --git a/lib/wasix/src/journaling/effector/syscalls/sock_leave_ipv6_multicast.rs b/lib/wasix/src/journaling/effector/syscalls/sock_leave_ipv6_multicast.rs new file mode 100644 index 00000000000..faf48b08a6e --- /dev/null +++ b/lib/wasix/src/journaling/effector/syscalls/sock_leave_ipv6_multicast.rs @@ -0,0 +1,42 @@ +use virtual_net::Ipv6Addr; + +use super::*; + +impl JournalEffector { + pub fn save_sock_leave_ipv6_multicast( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + fd: Fd, + multiaddr: Ipv6Addr, + iface: u32, + ) -> anyhow::Result<()> { + Self::save_event( + ctx, + JournalEntry::SocketLeaveIpv6Multicast { + fd, + multiaddr, + iface, + }, + ) + } + + pub fn apply_sock_leave_ipv6_multicast( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + fd: Fd, + multiaddr: Ipv6Addr, + iface: u32, + ) -> anyhow::Result<()> { + crate::syscalls::sock_leave_multicast_v6_internal(ctx, fd, multiaddr, iface) + .map(|r| r.map_err(|err| err.to_string())) + .unwrap_or_else(|err| Err(err.to_string())) + .map_err(|err| { + anyhow::format_err!( + "journal restore error: failed to leave ipv6 multicast (fd={}, multiaddr={}, iface={}) - {}", + fd, + multiaddr, + iface, + err + ) + })?; + Ok(()) + } +} diff --git a/lib/wasix/src/journaling/effector/syscalls/sock_listen.rs b/lib/wasix/src/journaling/effector/syscalls/sock_listen.rs new file mode 100644 index 00000000000..b404cf0e597 --- /dev/null +++ b/lib/wasix/src/journaling/effector/syscalls/sock_listen.rs @@ -0,0 +1,36 @@ +use super::*; + +impl JournalEffector { + pub fn save_sock_listen( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + fd: Fd, + backlog: usize, + ) -> anyhow::Result<()> { + Self::save_event( + ctx, + JournalEntry::SocketListen { + fd, + backlog: backlog as u32, + }, + ) + } + + pub fn apply_sock_listen( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + fd: Fd, + backlog: usize, + ) -> anyhow::Result<()> { + crate::syscalls::sock_listen_internal(ctx, fd, backlog) + .map(|r| r.map_err(|err| err.to_string())) + .unwrap_or_else(|err| Err(err.to_string())) + .map_err(|err| { + anyhow::format_err!( + "journal restore error: failed to listen on socket (fd={}, backlog={}) - {}", + fd, + backlog, + err + ) + })?; + Ok(()) + } +} diff --git a/lib/wasix/src/journaling/effector/syscalls/sock_open.rs b/lib/wasix/src/journaling/effector/syscalls/sock_open.rs new file mode 100644 index 00000000000..3dd3527cc08 --- /dev/null +++ b/lib/wasix/src/journaling/effector/syscalls/sock_open.rs @@ -0,0 +1,54 @@ +use wasmer_wasix_types::wasi::{Addressfamily, SockProto, Socktype}; + +use super::*; + +impl JournalEffector { + pub fn save_sock_open( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + af: Addressfamily, + ty: Socktype, + pt: SockProto, + fd: Fd, + ) -> anyhow::Result<()> { + Self::save_event(ctx, JournalEntry::SocketOpen { af, ty, pt, fd }) + } + + pub fn apply_sock_open( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + af: Addressfamily, + ty: Socktype, + pt: SockProto, + fd: Fd, + ) -> anyhow::Result<()> { + let ret_fd = crate::syscalls::sock_open_internal(ctx, af, ty, pt) + .map_err(|err| { + anyhow::format_err!( + "journal restore error: failed to open socket (af={:?}, ty={:?}, pt={:?}) - {}", + af, + ty, + pt, + err + ) + })? + .map_err(|err| { + anyhow::format_err!( + "journal restore error: failed to open socket (af={:?}, ty={:?}, pt={:?}) - {}", + af, + ty, + pt, + err + ) + })?; + + let ret = crate::syscalls::fd_renumber_internal(ctx, ret_fd, fd); + if ret != Errno::Success { + bail!( + "journal restore error: failed renumber file descriptor after opening socket (from={}, to={}) - {}", + ret_fd, + fd, + ret + ); + } + Ok(()) + } +} diff --git a/lib/wasix/src/journaling/effector/syscalls/sock_send.rs b/lib/wasix/src/journaling/effector/syscalls/sock_send.rs new file mode 100644 index 00000000000..a4ffec3470a --- /dev/null +++ b/lib/wasix/src/journaling/effector/syscalls/sock_send.rs @@ -0,0 +1,71 @@ +use wasmer_wasix_types::wasi::SiFlags; + +use crate::syscalls::sock_send_internal; + +use super::*; + +impl JournalEffector { + pub fn save_sock_send( + ctx: &FunctionEnvMut<'_, WasiEnv>, + fd: Fd, + sent: usize, + iovs: WasmPtr<__wasi_ciovec_t, M>, + iovs_len: M::Offset, + si_flags: SiFlags, + ) -> anyhow::Result<()> { + let env = ctx.data(); + let memory = unsafe { env.memory_view(&ctx) }; + let iovs_arr = iovs.slice(&memory, iovs_len)?; + + __asyncify_light(env, None, async { + let iovs_arr = iovs_arr.access().map_err(mem_error_to_wasi)?; + let mut remaining: M::Offset = TryFrom::::try_from(sent).unwrap_or_default(); + for iovs in iovs_arr.iter() { + let sub = iovs.buf_len.min(remaining); + if sub == M::ZERO { + continue; + } + remaining -= sub; + + let buf = WasmPtr::::new(iovs.buf) + .slice(&memory, sub) + .map_err(mem_error_to_wasi)? + .access() + .map_err(mem_error_to_wasi)?; + ctx.data() + .active_journal()? + .write(JournalEntry::SocketSend { + fd, + data: Cow::Borrowed(buf.as_ref()), + is_64bit: M::is_64bit(), + flags: si_flags as u16, + }) + .await + .map_err(map_snapshot_err)?; + } + Ok(()) + })? + .map_err(|err| WasiError::Exit(ExitCode::Errno(err)))?; + Ok(()) + } + + pub async fn apply_sock_send( + ctx: &FunctionEnvMut<'_, WasiEnv>, + sock: Fd, + si_data: Cow<'_, [u8]>, + si_flags: SiFlags, + ) -> anyhow::Result<()> { + let data_len = si_data.len(); + sock_send_internal(ctx, sock, FdWriteSource::<'_, M>::Buffer(si_data), si_flags)?.map_err( + |err| { + anyhow::format_err!( + "journal restore error: failed to send on socket (fd={}, data.len={}) - {}", + sock, + data_len, + err + ) + }, + )?; + Ok(()) + } +} diff --git a/lib/wasix/src/journaling/effector/syscalls/sock_send_file.rs b/lib/wasix/src/journaling/effector/syscalls/sock_send_file.rs new file mode 100644 index 00000000000..750c8d17075 --- /dev/null +++ b/lib/wasix/src/journaling/effector/syscalls/sock_send_file.rs @@ -0,0 +1,43 @@ +use crate::syscalls::sock_send_file_internal; + +use super::*; + +impl JournalEffector { + pub fn save_sock_send_file( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + socket_fd: Fd, + file_fd: Fd, + offset: Filesize, + count: Filesize, + ) -> anyhow::Result<()> { + Self::save_event( + ctx, + JournalEntry::SocketSendFile { + socket_fd, + file_fd, + offset, + count, + }, + ) + } + + pub async fn apply_sock_send_file( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + socket_fd: Fd, + file_fd: Fd, + offset: Filesize, + count: Filesize, + ) -> anyhow::Result<()> { + sock_send_file_internal(ctx, socket_fd, file_fd, offset, count)?.map_err(|err| { + anyhow::format_err!( + "journal restore error: failed to send_file on socket (sock={}, in_fd={}, offset={}, count={}) - {}", + socket_fd, + file_fd, + offset, + count, + err + ) + })?; + Ok(()) + } +} diff --git a/lib/wasix/src/journaling/effector/syscalls/sock_send_to.rs b/lib/wasix/src/journaling/effector/syscalls/sock_send_to.rs new file mode 100644 index 00000000000..6418e3270d0 --- /dev/null +++ b/lib/wasix/src/journaling/effector/syscalls/sock_send_to.rs @@ -0,0 +1,80 @@ +use virtual_net::SocketAddr; +use wasmer_wasix_types::wasi::SiFlags; + +use crate::syscalls::sock_send_to_internal; + +use super::*; + +impl JournalEffector { + pub fn save_sock_send_to( + ctx: &FunctionEnvMut<'_, WasiEnv>, + fd: Fd, + sent: usize, + iovs: WasmPtr<__wasi_ciovec_t, M>, + iovs_len: M::Offset, + addr: SocketAddr, + si_flags: SiFlags, + ) -> anyhow::Result<()> { + let env = ctx.data(); + let memory = unsafe { env.memory_view(&ctx) }; + let iovs_arr = iovs.slice(&memory, iovs_len)?; + + __asyncify_light(env, None, async { + let iovs_arr = iovs_arr.access().map_err(mem_error_to_wasi)?; + let mut remaining: M::Offset = TryFrom::::try_from(sent).unwrap_or_default(); + for iovs in iovs_arr.iter() { + let sub = iovs.buf_len.min(remaining); + if sub == M::ZERO { + continue; + } + remaining -= sub; + + let buf = WasmPtr::::new(iovs.buf) + .slice(&memory, sub) + .map_err(mem_error_to_wasi)? + .access() + .map_err(mem_error_to_wasi)?; + ctx.data() + .active_journal()? + .write(JournalEntry::SocketSendTo { + fd, + data: Cow::Borrowed(buf.as_ref()), + addr, + is_64bit: M::is_64bit(), + flags: si_flags as u16, + }) + .await + .map_err(map_snapshot_err)?; + } + Ok(()) + })? + .map_err(|err| WasiError::Exit(ExitCode::Errno(err)))?; + Ok(()) + } + + pub async fn apply_sock_send_to( + ctx: &FunctionEnvMut<'_, WasiEnv>, + sock: Fd, + si_data: Cow<'_, [u8]>, + si_flags: SiFlags, + addr: SocketAddr, + ) -> anyhow::Result<()> { + let data_len = si_data.len(); + sock_send_to_internal::( + ctx, + sock, + FdWriteSource::<'_, M>::Buffer(si_data), + si_flags, + addr, + )? + .map_err(|err| { + anyhow::format_err!( + "journal restore error: failed to send_to on socket (fd={}, data.len={}) - {}", + sock, + data_len, + err + ) + })?; + Ok(()) + } +} diff --git a/lib/wasix/src/journaling/effector/syscalls/sock_set_opt_flag.rs b/lib/wasix/src/journaling/effector/syscalls/sock_set_opt_flag.rs new file mode 100644 index 00000000000..8cfc5582cee --- /dev/null +++ b/lib/wasix/src/journaling/effector/syscalls/sock_set_opt_flag.rs @@ -0,0 +1,35 @@ +use wasmer_wasix_types::wasi::Sockoption; + +use super::*; + +impl JournalEffector { + pub fn save_sock_set_opt_flag( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + fd: Fd, + opt: Sockoption, + flag: bool, + ) -> anyhow::Result<()> { + Self::save_event(ctx, JournalEntry::SocketSetOptFlag { fd, opt, flag }) + } + + pub fn apply_sock_set_opt_flag( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + fd: Fd, + opt: Sockoption, + flag: bool, + ) -> anyhow::Result<()> { + crate::syscalls::sock_set_opt_flag_internal(ctx, fd, opt, flag) + .map(|r| r.map_err(|err| err.to_string())) + .unwrap_or_else(|err| Err(err.to_string())) + .map_err(|err| { + anyhow::format_err!( + "journal restore error: failed to set socket option (fd={}, opt={:?}, flag={}) - {}", + fd, + opt, + flag, + err + ) + })?; + Ok(()) + } +} diff --git a/lib/wasix/src/journaling/effector/syscalls/sock_set_opt_size.rs b/lib/wasix/src/journaling/effector/syscalls/sock_set_opt_size.rs new file mode 100644 index 00000000000..4db8c1d8937 --- /dev/null +++ b/lib/wasix/src/journaling/effector/syscalls/sock_set_opt_size.rs @@ -0,0 +1,35 @@ +use wasmer_wasix_types::wasi::Sockoption; + +use super::*; + +impl JournalEffector { + pub fn save_sock_set_opt_size( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + fd: Fd, + opt: Sockoption, + size: Filesize, + ) -> anyhow::Result<()> { + Self::save_event(ctx, JournalEntry::SocketSetOptSize { fd, opt, size }) + } + + pub fn apply_sock_set_opt_size( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + fd: Fd, + opt: Sockoption, + size: Filesize, + ) -> anyhow::Result<()> { + crate::syscalls::sock_set_opt_size_internal(ctx, fd, opt, size) + .map(|r| r.map_err(|err| err.to_string())) + .unwrap_or_else(|err| Err(err.to_string())) + .map_err(|err| { + anyhow::format_err!( + "journal restore error: failed to set socket option (fd={}, opt={:?}, size={}) - {}", + fd, + opt, + size, + err + ) + })?; + Ok(()) + } +} diff --git a/lib/wasix/src/journaling/effector/syscalls/sock_set_opt_time.rs b/lib/wasix/src/journaling/effector/syscalls/sock_set_opt_time.rs new file mode 100644 index 00000000000..d8aff0d15b2 --- /dev/null +++ b/lib/wasix/src/journaling/effector/syscalls/sock_set_opt_time.rs @@ -0,0 +1,37 @@ +use virtual_net::Duration; + +use crate::net::socket::TimeType; + +use super::*; + +impl JournalEffector { + pub fn save_sock_set_opt_time( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + fd: Fd, + ty: TimeType, + time: Option, + ) -> anyhow::Result<()> { + Self::save_event(ctx, JournalEntry::SocketSetOptTime { fd, ty, time }) + } + + pub fn apply_sock_set_opt_time( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + fd: Fd, + ty: TimeType, + time: Option, + ) -> anyhow::Result<()> { + crate::syscalls::sock_set_opt_time_internal(ctx, fd, ty, time) + .map(|r| r.map_err(|err| err.to_string())) + .unwrap_or_else(|err| Err(err.to_string())) + .map_err(|err| { + anyhow::format_err!( + "journal restore error: failed to set socket option (fd={}, opt={:?}, time={:?}) - {}", + fd, + ty, + time, + err + ) + })?; + Ok(()) + } +} diff --git a/lib/wasix/src/journaling/effector/syscalls/sock_shutdown.rs b/lib/wasix/src/journaling/effector/syscalls/sock_shutdown.rs new file mode 100644 index 00000000000..8d6d5e76000 --- /dev/null +++ b/lib/wasix/src/journaling/effector/syscalls/sock_shutdown.rs @@ -0,0 +1,32 @@ +use std::net::Shutdown; + +use super::*; + +impl JournalEffector { + pub fn save_sock_shutdown( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + fd: Fd, + shutdown: Shutdown, + ) -> anyhow::Result<()> { + Self::save_event(ctx, JournalEntry::SocketShutdown { fd, how: shutdown }) + } + + pub fn apply_sock_shutdown( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + fd: Fd, + shutdown: Shutdown, + ) -> anyhow::Result<()> { + crate::syscalls::sock_shutdown_internal(ctx, fd, shutdown) + .map(|r| r.map_err(|err| err.to_string())) + .unwrap_or_else(|err| Err(err.to_string())) + .map_err(|err| { + anyhow::format_err!( + "journal restore error: failed to shutdown socket (fd={}, how={:?}) - {}", + fd, + shutdown, + err + ) + })?; + Ok(()) + } +} diff --git a/lib/wasix/src/journaling/filter.rs b/lib/wasix/src/journaling/filter.rs index bdba9226596..33e28d63fd2 100644 --- a/lib/wasix/src/journaling/filter.rs +++ b/lib/wasix/src/journaling/filter.rs @@ -131,8 +131,8 @@ impl Journal for FilteredJournal { | JournalEntry::SocketOpen { .. } | JournalEntry::SocketListen { .. } | JournalEntry::SocketBind { .. } - | JournalEntry::SocketConnect { .. } - | JournalEntry::SocketAccept { .. } + | JournalEntry::SocketConnected { .. } + | JournalEntry::SocketAccepted { .. } | JournalEntry::SocketJoinIpv4Multicast { .. } | JournalEntry::SocketJoinIpv6Multicast { .. } | JournalEntry::SocketLeaveIpv4Multicast { .. } diff --git a/lib/wasix/src/journaling/journal.rs b/lib/wasix/src/journaling/journal.rs index 7d5bf1df265..5eb84b5d5a9 100644 --- a/lib/wasix/src/journaling/journal.rs +++ b/lib/wasix/src/journaling/journal.rs @@ -1,17 +1,18 @@ use serde::{Deserialize, Serialize}; -use std::net::SocketAddr; +use std::net::{Shutdown, SocketAddr}; use std::time::SystemTime; use std::{borrow::Cow, ops::Range}; -use virtual_net::{IpAddr, IpCidr, Ipv4Addr, Ipv6Addr}; +use virtual_net::{Duration, IpAddr, IpCidr, Ipv4Addr, Ipv6Addr, StreamSecurity}; use wasmer_wasix_types::wasi::{ Addressfamily, Advice, EpollCtl, EpollEventCtl, ExitCode, Fdflags, FileDelta, Filesize, - Fstflags, LookupFlags, Oflags, Rights, SdFlags, SiFlags, Snapshot0Clockid, SockProto, - Sockoption, Socktype, Streamsecurity, Timestamp, Tty, Whence, + Fstflags, LookupFlags, Oflags, Rights, SiFlags, Snapshot0Clockid, SockProto, Sockoption, + Socktype, Timestamp, Tty, Whence, }; use futures::future::LocalBoxFuture; use virtual_fs::Fd; +use crate::net::socket::TimeType; use crate::WasiThreadId; use super::SnapshotTrigger; @@ -198,7 +199,7 @@ pub enum JournalEntry<'a> { PortBridge { network: String, token: String, - security: Streamsecurity, + security: StreamSecurity, }, PortUnbridge, PortDhcpAcquire, @@ -208,8 +209,8 @@ pub enum JournalEntry<'a> { PortRouteAdd { cidr: IpCidr, via_router: IpAddr, - preferred_until: Option, - expires_at: Option, + preferred_until: Option, + expires_at: Option, }, PortRouteClear, PortRouteDel { @@ -229,14 +230,16 @@ pub enum JournalEntry<'a> { fd: Fd, addr: SocketAddr, }, - SocketConnect { + SocketConnected { fd: Fd, addr: SocketAddr, }, - SocketAccept { + SocketAccepted { listen_fd: Fd, fd: Fd, peer_addr: SocketAddr, + fd_flags: Fdflags, + nonblocking: bool, }, SocketJoinIpv4Multicast { fd: Fd, @@ -261,17 +264,21 @@ pub enum JournalEntry<'a> { SocketSendFile { socket_fd: Fd, file_fd: Fd, + offset: Filesize, + count: Filesize, }, SocketSendTo { fd: Fd, data: Cow<'a, [u8]>, flags: SiFlags, addr: SocketAddr, + is_64bit: bool, }, SocketSend { fd: Fd, data: Cow<'a, [u8]>, flags: SiFlags, + is_64bit: bool, }, SocketSetOptFlag { fd: Fd, @@ -285,12 +292,12 @@ pub enum JournalEntry<'a> { }, SocketSetOptTime { fd: Fd, - opt: Sockoption, - size: Option, + ty: TimeType, + time: Option, }, SocketShutdown { fd: Fd, - how: SdFlags, + how: Shutdown, }, /// Represents the marker for the end of a snapshot Snapshot { diff --git a/lib/wasix/src/journaling/log_file.rs b/lib/wasix/src/journaling/log_file.rs index 79324784431..1267a833f62 100644 --- a/lib/wasix/src/journaling/log_file.rs +++ b/lib/wasix/src/journaling/log_file.rs @@ -2,19 +2,18 @@ use serde::{Deserialize, Serialize}; use std::{ io::{self, ErrorKind, SeekFrom}, mem::MaybeUninit, + net::Shutdown, path::Path, time::SystemTime, }; use tokio::runtime::Handle; -use virtual_net::{IpAddr, IpCidr, Ipv4Addr, Ipv6Addr, SocketAddr}; -use wasmer_wasix_types::wasi::{ - self, Addressfamily, EpollEventCtl, Sockoption, Socktype, Streamsecurity, -}; +use virtual_net::{Duration, IpAddr, IpCidr, Ipv4Addr, Ipv6Addr, SocketAddr, StreamSecurity}; +use wasmer_wasix_types::wasi::{self, Addressfamily, EpollEventCtl, Sockoption, Socktype}; use futures::future::LocalBoxFuture; use virtual_fs::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt}; -use crate::WasiThreadId; +use crate::{net::socket::TimeType, WasiThreadId}; use super::*; @@ -201,8 +200,8 @@ pub(crate) enum LogFileJournalEntry { PortRouteAddV1 { cidr: IpCidr, via_router: IpAddr, - preferred_until: Option, - expires_at: Option, + preferred_until: Option, + expires_at: Option, }, PortRouteClearV1, PortRouteDelV1 { @@ -222,14 +221,16 @@ pub(crate) enum LogFileJournalEntry { fd: u32, addr: SocketAddr, }, - SocketConnectV1 { + SocketConnectedV1 { fd: u32, addr: SocketAddr, }, - SocketAcceptV1 { + SocketAcceptedV1 { listen_fd: u32, fd: u32, peer_addr: SocketAddr, + fd_flags: u16, + nonblocking: bool, }, SocketJoinIpv4MulticastV1 { fd: u32, @@ -254,17 +255,21 @@ pub(crate) enum LogFileJournalEntry { SocketSendFileV1 { socket_fd: u32, file_fd: u32, + offset: u64, + count: u64, }, SocketSendToV1 { fd: u32, data: Vec, flags: u16, addr: SocketAddr, + is_64bit: bool, }, SocketSendV1 { fd: u32, data: Vec, flags: u16, + is_64bit: bool, }, SocketSetOptFlagV1 { fd: u32, @@ -278,12 +283,12 @@ pub(crate) enum LogFileJournalEntry { }, SocketSetOptTimeV1 { fd: u32, - opt: JournalSockoptionV1, - size: Option, + ty: JournalTimeTypeV1, + time: Option, }, SocketShutdownV1 { fd: u32, - how: u8, + how: JournalSocketShutdownV1, }, SnapshotV1 { when: SystemTime, @@ -506,26 +511,25 @@ pub enum JournalStreamSecurityV1 { Unknown, } -impl From for JournalStreamSecurityV1 { - fn from(val: Streamsecurity) -> Self { +impl From for JournalStreamSecurityV1 { + fn from(val: StreamSecurity) -> Self { match val { - Streamsecurity::Unencrypted => JournalStreamSecurityV1::Unencrypted, - Streamsecurity::AnyEncryption => JournalStreamSecurityV1::AnyEncryption, - Streamsecurity::ClassicEncryption => JournalStreamSecurityV1::ClassicEncryption, - Streamsecurity::DoubleEncryption => JournalStreamSecurityV1::DoubleEncryption, - Streamsecurity::Unknown => JournalStreamSecurityV1::Unknown, + StreamSecurity::Unencrypted => JournalStreamSecurityV1::Unencrypted, + StreamSecurity::AnyEncyption => JournalStreamSecurityV1::AnyEncryption, + StreamSecurity::ClassicEncryption => JournalStreamSecurityV1::ClassicEncryption, + StreamSecurity::DoubleEncryption => JournalStreamSecurityV1::DoubleEncryption, } } } -impl From for Streamsecurity { +impl From for StreamSecurity { fn from(val: JournalStreamSecurityV1) -> Self { match val { - JournalStreamSecurityV1::Unencrypted => Streamsecurity::Unencrypted, - JournalStreamSecurityV1::AnyEncryption => Streamsecurity::AnyEncryption, - JournalStreamSecurityV1::ClassicEncryption => Streamsecurity::ClassicEncryption, - JournalStreamSecurityV1::DoubleEncryption => Streamsecurity::DoubleEncryption, - JournalStreamSecurityV1::Unknown => Streamsecurity::Unknown, + JournalStreamSecurityV1::Unencrypted => StreamSecurity::Unencrypted, + JournalStreamSecurityV1::AnyEncryption => StreamSecurity::AnyEncyption, + JournalStreamSecurityV1::ClassicEncryption => StreamSecurity::ClassicEncryption, + JournalStreamSecurityV1::DoubleEncryption => StreamSecurity::DoubleEncryption, + JournalStreamSecurityV1::Unknown => StreamSecurity::AnyEncyption, } } } @@ -692,6 +696,69 @@ impl From for Sockoption { } } +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +pub enum JournalTimeTypeV1 { + ReadTimeout, + WriteTimeout, + AcceptTimeout, + ConnectTimeout, + BindTimeout, + Linger, +} + +impl From for JournalTimeTypeV1 { + fn from(val: TimeType) -> Self { + match val { + TimeType::ReadTimeout => JournalTimeTypeV1::ReadTimeout, + TimeType::WriteTimeout => JournalTimeTypeV1::WriteTimeout, + TimeType::AcceptTimeout => JournalTimeTypeV1::AcceptTimeout, + TimeType::ConnectTimeout => JournalTimeTypeV1::ConnectTimeout, + TimeType::BindTimeout => JournalTimeTypeV1::BindTimeout, + TimeType::Linger => JournalTimeTypeV1::Linger, + } + } +} + +impl From for TimeType { + fn from(val: JournalTimeTypeV1) -> Self { + match val { + JournalTimeTypeV1::ReadTimeout => TimeType::ReadTimeout, + JournalTimeTypeV1::WriteTimeout => TimeType::WriteTimeout, + JournalTimeTypeV1::AcceptTimeout => TimeType::AcceptTimeout, + JournalTimeTypeV1::ConnectTimeout => TimeType::ConnectTimeout, + JournalTimeTypeV1::BindTimeout => TimeType::BindTimeout, + JournalTimeTypeV1::Linger => TimeType::Linger, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +pub enum JournalSocketShutdownV1 { + Read, + Write, + Both, +} + +impl From for JournalSocketShutdownV1 { + fn from(val: Shutdown) -> Self { + match val { + Shutdown::Read => JournalSocketShutdownV1::Read, + Shutdown::Write => JournalSocketShutdownV1::Write, + Shutdown::Both => JournalSocketShutdownV1::Both, + } + } +} + +impl From for Shutdown { + fn from(val: JournalSocketShutdownV1) -> Self { + match val { + JournalSocketShutdownV1::Read => Shutdown::Read, + JournalSocketShutdownV1::Write => Shutdown::Write, + JournalSocketShutdownV1::Both => Shutdown::Both, + } + } +} + impl<'a> From> for LogFileJournalEntry { fn from(value: JournalEntry<'a>) -> Self { match value { @@ -940,15 +1007,19 @@ impl<'a> From> for LogFileJournalEntry { }, JournalEntry::SocketListen { fd, backlog } => Self::SocketListenV1 { fd, backlog }, JournalEntry::SocketBind { fd, addr } => Self::SocketBindV1 { fd, addr }, - JournalEntry::SocketConnect { fd, addr } => Self::SocketConnectV1 { fd, addr }, - JournalEntry::SocketAccept { + JournalEntry::SocketConnected { fd, addr } => Self::SocketConnectedV1 { fd, addr }, + JournalEntry::SocketAccepted { listen_fd, fd, peer_addr, - } => Self::SocketAcceptV1 { + fd_flags, + nonblocking, + } => Self::SocketAcceptedV1 { listen_fd, fd, peer_addr, + fd_flags: fd_flags.bits(), + nonblocking, }, JournalEntry::SocketJoinIpv4Multicast { fd, @@ -986,24 +1057,40 @@ impl<'a> From> for LogFileJournalEntry { multiaddr, iface, }, - JournalEntry::SocketSendFile { socket_fd, file_fd } => { - Self::SocketSendFileV1 { socket_fd, file_fd } - } + JournalEntry::SocketSendFile { + socket_fd, + file_fd, + offset, + count, + } => Self::SocketSendFileV1 { + socket_fd, + file_fd, + offset, + count, + }, JournalEntry::SocketSendTo { fd, data, flags, addr, + is_64bit, } => Self::SocketSendToV1 { fd, data: data.into(), flags: flags as u16, addr, + is_64bit, }, - JournalEntry::SocketSend { fd, data, flags } => Self::SocketSendV1 { + JournalEntry::SocketSend { + fd, + data, + flags, + is_64bit, + } => Self::SocketSendV1 { fd, data: data.into(), flags: flags as u16, + is_64bit, }, JournalEntry::SocketSetOptFlag { fd, opt, flag } => Self::SocketSetOptFlagV1 { fd, @@ -1015,12 +1102,15 @@ impl<'a> From> for LogFileJournalEntry { opt: opt.into(), size, }, - JournalEntry::SocketSetOptTime { fd, opt, size } => Self::SocketSetOptTimeV1 { + JournalEntry::SocketSetOptTime { fd, ty, time } => Self::SocketSetOptTimeV1 { fd, - opt: opt.into(), - size, + ty: ty.into(), + time, + }, + JournalEntry::SocketShutdown { fd, how } => Self::SocketShutdownV1 { + fd, + how: how.into(), }, - JournalEntry::SocketShutdown { fd, how } => Self::SocketShutdownV1 { fd, how }, } } } @@ -1293,15 +1383,21 @@ impl<'a> From for JournalEntry<'a> { Self::SocketListen { fd, backlog } } LogFileJournalEntry::SocketBindV1 { fd, addr } => Self::SocketBind { fd, addr }, - LogFileJournalEntry::SocketConnectV1 { fd, addr } => Self::SocketConnect { fd, addr }, - LogFileJournalEntry::SocketAcceptV1 { + LogFileJournalEntry::SocketConnectedV1 { fd, addr } => { + Self::SocketConnected { fd, addr } + } + LogFileJournalEntry::SocketAcceptedV1 { listen_fd, fd, peer_addr, - } => Self::SocketAccept { + fd_flags, + nonblocking, + } => Self::SocketAccepted { listen_fd, fd, peer_addr, + fd_flags: Fdflags::from_bits_truncate(fd_flags), + nonblocking, }, LogFileJournalEntry::SocketJoinIpv4MulticastV1 { fd, @@ -1339,24 +1435,40 @@ impl<'a> From for JournalEntry<'a> { multiaddr, iface, }, - LogFileJournalEntry::SocketSendFileV1 { socket_fd, file_fd } => { - Self::SocketSendFile { socket_fd, file_fd } - } + LogFileJournalEntry::SocketSendFileV1 { + socket_fd, + file_fd, + offset, + count, + } => Self::SocketSendFile { + socket_fd, + file_fd, + offset, + count, + }, LogFileJournalEntry::SocketSendToV1 { fd, data, flags, addr, + is_64bit, } => Self::SocketSendTo { fd, data: data.into(), flags, addr, + is_64bit, }, - LogFileJournalEntry::SocketSendV1 { fd, data, flags } => Self::SocketSend { + LogFileJournalEntry::SocketSendV1 { + fd, + data, + flags, + is_64bit, + } => Self::SocketSend { fd, data: data.into(), flags, + is_64bit, }, LogFileJournalEntry::SocketSetOptFlagV1 { fd, opt, flag } => Self::SocketSetOptFlag { fd, @@ -1368,12 +1480,15 @@ impl<'a> From for JournalEntry<'a> { opt: opt.into(), size, }, - LogFileJournalEntry::SocketSetOptTimeV1 { fd, opt, size } => Self::SocketSetOptTime { + LogFileJournalEntry::SocketSetOptTimeV1 { fd, ty, time } => Self::SocketSetOptTime { fd, - opt: opt.into(), - size, + ty: ty.into(), + time, + }, + LogFileJournalEntry::SocketShutdownV1 { fd, how } => Self::SocketShutdown { + fd, + how: how.into(), }, - LogFileJournalEntry::SocketShutdownV1 { fd, how } => Self::SocketShutdown { fd, how }, } } } diff --git a/lib/wasix/src/net/socket.rs b/lib/wasix/src/net/socket.rs index ed7fb32819e..eabab412681 100644 --- a/lib/wasix/src/net/socket.rs +++ b/lib/wasix/src/net/socket.rs @@ -72,6 +72,9 @@ pub enum InodeSocketKind { socket: Box, peer: Option, }, + RemoteTcpStream { + peer_addr: SocketAddr, + }, } pub enum WasiSocketOption { @@ -176,13 +179,13 @@ pub struct InodeSocket { } impl InodeSocket { - pub fn new(kind: InodeSocketKind) -> virtual_net::Result { + pub fn new(kind: InodeSocketKind) -> Self { let protected = InodeSocketProtected { kind }; - Ok(Self { + Self { inner: Arc::new(InodeSocketInner { protected: RwLock::new(protected), }), - }) + } } pub fn poll_read_ready(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { @@ -266,7 +269,7 @@ impl InodeSocket { tokio::select! { socket = socket => { let socket = socket.map_err(net_error_into_wasi_err)?; - Ok(Some(InodeSocket::new(InodeSocketKind::UdpSocket { socket, peer: None }).map_err(net_error_into_wasi_err)?)) + Ok(Some(InodeSocket::new(InodeSocketKind::UdpSocket { socket, peer: None }))) }, _ = tasks.sleep_now(timeout) => Err(Errno::Timedout) } @@ -326,7 +329,7 @@ impl InodeSocket { Ok(Some(InodeSocket::new(InodeSocketKind::TcpListener { socket, accept_timeout: Some(timeout), - }).map_err(net_error_into_wasi_err)?)) + }))) }, _ = tasks.sleep_now(timeout) => Err(Errno::Timedout) } @@ -410,6 +413,7 @@ impl InodeSocket { InodeSocketKind::UdpSocket { .. } => {} InodeSocketKind::Raw(_) => {} InodeSocketKind::PreSocket { .. } => return Err(Errno::Notconn), + InodeSocketKind::RemoteTcpStream { .. } => {} }; Ok(()) } @@ -502,8 +506,7 @@ impl InodeSocket { socket, write_timeout: new_write_timeout, read_timeout: new_read_timeout, - }) - .map_err(net_error_into_wasi_err)?; + }); Ok(Some(socket)) } @@ -1305,6 +1308,7 @@ impl InodeSocketProtected { InodeSocketKind::PreSocket { handler, .. } => { handler.take(); } + InodeSocketKind::RemoteTcpStream { .. } => {} } } @@ -1316,6 +1320,7 @@ impl InodeSocketProtected { InodeSocketKind::Raw(socket) => socket.poll_read_ready(cx), InodeSocketKind::Icmp(socket) => socket.poll_read_ready(cx), InodeSocketKind::PreSocket { .. } => Poll::Pending, + InodeSocketKind::RemoteTcpStream { .. } => Poll::Pending, } .map_err(net_error_into_io_err) } @@ -1328,6 +1333,7 @@ impl InodeSocketProtected { InodeSocketKind::Raw(socket) => socket.poll_write_ready(cx), InodeSocketKind::Icmp(socket) => socket.poll_write_ready(cx), InodeSocketKind::PreSocket { .. } => Poll::Pending, + InodeSocketKind::RemoteTcpStream { .. } => Poll::Pending, } .map_err(net_error_into_io_err) } @@ -1346,6 +1352,7 @@ impl InodeSocketProtected { h.replace(handler); Ok(()) } + InodeSocketKind::RemoteTcpStream { .. } => Ok(()), } } } diff --git a/lib/wasix/src/syscalls/mod.rs b/lib/wasix/src/syscalls/mod.rs index 1e920c715c1..9c4d7f96b83 100644 --- a/lib/wasix/src/syscalls/mod.rs +++ b/lib/wasix/src/syscalls/mod.rs @@ -1296,6 +1296,8 @@ pub fn restore_snapshot( mut ctx: FunctionEnvMut<'_, WasiEnv>, journal: Arc, ) -> Result { + use crate::journaling::Journal; + InlineWaker::block_on(async { let mut is_same_module = false; let mut rewind = None; @@ -1560,66 +1562,171 @@ pub fn restore_snapshot( ) .map_err(anyhow_err_to_runtime_err)?; } - crate::journaling::JournalEntry::PortAddAddr { cidr } => todo!(), - crate::journaling::JournalEntry::PortDelAddr { addr } => todo!(), - crate::journaling::JournalEntry::PortAddrClear => todo!(), + crate::journaling::JournalEntry::PortAddAddr { cidr } => { + JournalEffector::apply_port_addr_add(&mut ctx, cidr) + .map_err(anyhow_err_to_runtime_err)? + } + crate::journaling::JournalEntry::PortDelAddr { addr } => { + JournalEffector::apply_port_addr_remove(&mut ctx, addr) + .map_err(anyhow_err_to_runtime_err)? + } + crate::journaling::JournalEntry::PortAddrClear => { + JournalEffector::apply_port_addr_clear(&mut ctx) + .map_err(anyhow_err_to_runtime_err)? + } crate::journaling::JournalEntry::PortBridge { network, token, security, - } => todo!(), - crate::journaling::JournalEntry::PortUnbridge => todo!(), - crate::journaling::JournalEntry::PortDhcpAcquire => todo!(), - crate::journaling::JournalEntry::PortGatewaySet { ip } => todo!(), + } => JournalEffector::apply_port_bridge(&mut ctx, &network, &token, security) + .map_err(anyhow_err_to_runtime_err)?, + crate::journaling::JournalEntry::PortUnbridge => { + JournalEffector::apply_port_unbridge(&mut ctx) + .map_err(anyhow_err_to_runtime_err)? + } + crate::journaling::JournalEntry::PortDhcpAcquire => { + JournalEffector::apply_port_dhcp_acquire(&mut ctx) + .map_err(anyhow_err_to_runtime_err)? + } + crate::journaling::JournalEntry::PortGatewaySet { ip } => { + JournalEffector::apply_port_gateway_set(&mut ctx, ip) + .map_err(anyhow_err_to_runtime_err)? + } crate::journaling::JournalEntry::PortRouteAdd { cidr, via_router, preferred_until, expires_at, - } => todo!(), - crate::journaling::JournalEntry::PortRouteClear => todo!(), - crate::journaling::JournalEntry::PortRouteDel { ip } => todo!(), - crate::journaling::JournalEntry::SocketOpen { af, ty, pt, fd } => todo!(), - crate::journaling::JournalEntry::SocketListen { fd, backlog } => todo!(), - crate::journaling::JournalEntry::SocketBind { fd, addr } => todo!(), - crate::journaling::JournalEntry::SocketConnect { fd, addr } => todo!(), - crate::journaling::JournalEntry::SocketAccept { + } => JournalEffector::apply_port_route_add( + &mut ctx, + cidr, + via_router, + preferred_until, + expires_at, + ) + .map_err(anyhow_err_to_runtime_err)?, + crate::journaling::JournalEntry::PortRouteClear => { + JournalEffector::apply_port_route_clear(&mut ctx) + .map_err(anyhow_err_to_runtime_err)? + } + crate::journaling::JournalEntry::PortRouteDel { ip } => { + JournalEffector::apply_port_route_remove(&mut ctx, ip) + .map_err(anyhow_err_to_runtime_err)? + } + crate::journaling::JournalEntry::SocketOpen { af, ty, pt, fd } => { + JournalEffector::apply_sock_open(&mut ctx, af, ty, pt, fd) + .map_err(anyhow_err_to_runtime_err)? + } + crate::journaling::JournalEntry::SocketListen { fd, backlog } => { + JournalEffector::apply_sock_listen(&mut ctx, fd, backlog as usize) + .map_err(anyhow_err_to_runtime_err)? + } + crate::journaling::JournalEntry::SocketBind { fd, addr } => { + JournalEffector::apply_sock_bind(&mut ctx, fd, addr) + .map_err(anyhow_err_to_runtime_err)? + } + crate::journaling::JournalEntry::SocketConnected { fd, addr } => { + JournalEffector::apply_sock_connect(&mut ctx, fd, addr) + .map_err(anyhow_err_to_runtime_err)? + } + crate::journaling::JournalEntry::SocketAccepted { listen_fd, fd, peer_addr, - } => todo!(), + fd_flags, + nonblocking, + } => JournalEffector::apply_sock_accepted( + &mut ctx, + listen_fd, + fd, + peer_addr, + fd_flags, + nonblocking, + ) + .map_err(anyhow_err_to_runtime_err)?, crate::journaling::JournalEntry::SocketJoinIpv4Multicast { fd, multiaddr, iface, - } => todo!(), + } => { + JournalEffector::apply_sock_join_ipv4_multicast(&mut ctx, fd, multiaddr, iface) + .map_err(anyhow_err_to_runtime_err)? + } crate::journaling::JournalEntry::SocketJoinIpv6Multicast { fd, multiaddr, iface, - } => todo!(), + } => { + JournalEffector::apply_sock_join_ipv6_multicast(&mut ctx, fd, multiaddr, iface) + .map_err(anyhow_err_to_runtime_err)? + } crate::journaling::JournalEntry::SocketLeaveIpv4Multicast { fd, multiaddr, iface, - } => todo!(), + } => { + JournalEffector::apply_sock_leave_ipv4_multicast(&mut ctx, fd, multiaddr, iface) + .map_err(anyhow_err_to_runtime_err)? + } crate::journaling::JournalEntry::SocketLeaveIpv6Multicast { fd, multiaddr, iface, - } => todo!(), - crate::journaling::JournalEntry::SocketSendFile { socket_fd, file_fd } => todo!(), + } => { + JournalEffector::apply_sock_leave_ipv6_multicast(&mut ctx, fd, multiaddr, iface) + .map_err(anyhow_err_to_runtime_err)? + } + crate::journaling::JournalEntry::SocketSendFile { + socket_fd, + file_fd, + offset, + count, + } => JournalEffector::apply_sock_send_file( + &mut ctx, socket_fd, file_fd, offset, count, + ) + .await + .map_err(anyhow_err_to_runtime_err)?, crate::journaling::JournalEntry::SocketSendTo { fd, data, flags, addr, - } => todo!(), - crate::journaling::JournalEntry::SocketSend { fd, data, flags } => todo!(), - crate::journaling::JournalEntry::SocketSetOptFlag { fd, opt, flag } => todo!(), - crate::journaling::JournalEntry::SocketSetOptSize { fd, opt, size } => todo!(), - crate::journaling::JournalEntry::SocketSetOptTime { fd, opt, size } => todo!(), - crate::journaling::JournalEntry::SocketShutdown { fd, how } => todo!(), + is_64bit, + } => if is_64bit { + JournalEffector::apply_sock_send_to::(&mut ctx, fd, data, flags, addr) + .await + } else { + JournalEffector::apply_sock_send_to::(&mut ctx, fd, data, flags, addr) + .await + } + .map_err(anyhow_err_to_runtime_err)?, + crate::journaling::JournalEntry::SocketSend { + fd, + data, + flags, + is_64bit, + } => if is_64bit { + JournalEffector::apply_sock_send::(&mut ctx, fd, data, flags).await + } else { + JournalEffector::apply_sock_send::(&mut ctx, fd, data, flags).await + } + .map_err(anyhow_err_to_runtime_err)?, + crate::journaling::JournalEntry::SocketSetOptFlag { fd, opt, flag } => { + JournalEffector::apply_sock_set_opt_flag(&mut ctx, fd, opt, flag) + .map_err(anyhow_err_to_runtime_err)? + } + crate::journaling::JournalEntry::SocketSetOptSize { fd, opt, size } => { + JournalEffector::apply_sock_set_opt_size(&mut ctx, fd, opt, size) + .map_err(anyhow_err_to_runtime_err)? + } + crate::journaling::JournalEntry::SocketSetOptTime { fd, ty, time } => { + JournalEffector::apply_sock_set_opt_time(&mut ctx, fd, ty, time) + .map_err(anyhow_err_to_runtime_err)? + } + crate::journaling::JournalEntry::SocketShutdown { fd, how } => { + JournalEffector::apply_sock_shutdown(&mut ctx, fd, how) + .map_err(anyhow_err_to_runtime_err)? + } } } // If we are not in the same module then we fire off an exit diff --git a/lib/wasix/src/syscalls/wasix/sock_accept.rs b/lib/wasix/src/syscalls/wasix/sock_accept.rs index 5c9555ca6c6..9b996adae07 100644 --- a/lib/wasix/src/syscalls/wasix/sock_accept.rs +++ b/lib/wasix/src/syscalls/wasix/sock_accept.rs @@ -111,12 +111,11 @@ pub(crate) fn sock_accept_internal( )); let kind = Kind::Socket { - socket: wasi_try_ok_ok!(InodeSocket::new(InodeSocketKind::TcpStream { + socket: InodeSocket::new(InodeSocketKind::TcpStream { socket: child, write_timeout: None, read_timeout: None, - }) - .map_err(net_error_into_wasi_err)), + }), }; let inode = state .fs diff --git a/lib/wasix/src/syscalls/wasix/sock_open.rs b/lib/wasix/src/syscalls/wasix/sock_open.rs index 0c255984044..5c068332214 100644 --- a/lib/wasix/src/syscalls/wasix/sock_open.rs +++ b/lib/wasix/src/syscalls/wasix/sock_open.rs @@ -63,7 +63,7 @@ pub(crate) fn sock_open_internal( let kind = match ty { Socktype::Stream | Socktype::Dgram => Kind::Socket { - socket: wasi_try_ok_ok!(InodeSocket::new(InodeSocketKind::PreSocket { + socket: InodeSocket::new(InodeSocketKind::PreSocket { family: af, ty, pt, @@ -81,8 +81,7 @@ pub(crate) fn sock_open_internal( accept_timeout: None, connect_timeout: None, handler: None, - }) - .map_err(net_error_into_wasi_err)), + }), }, _ => return Ok(Err(Errno::Notsup)), }; diff --git a/lib/wasix/src/syscalls/wasix/sock_send.rs b/lib/wasix/src/syscalls/wasix/sock_send.rs index bab98c2cd36..4c98973759b 100644 --- a/lib/wasix/src/syscalls/wasix/sock_send.rs +++ b/lib/wasix/src/syscalls/wasix/sock_send.rs @@ -55,7 +55,7 @@ pub fn sock_send( )?) } else { wasi_try_ok!(sock_send_internal::( - &mut ctx, + &ctx, fd, FdWriteSource::Iovs { iovs: si_data, @@ -76,7 +76,7 @@ pub fn sock_send( } pub(crate) fn sock_send_internal( - ctx: &mut FunctionEnvMut<'_, WasiEnv>, + ctx: &FunctionEnvMut<'_, WasiEnv>, sock: WasiFd, si_data: FdWriteSource<'_, M>, si_flags: SiFlags, diff --git a/lib/wasix/src/syscalls/wasix/sock_send_file.rs b/lib/wasix/src/syscalls/wasix/sock_send_file.rs index 076d36f06d4..85236beb24b 100644 --- a/lib/wasix/src/syscalls/wasix/sock_send_file.rs +++ b/lib/wasix/src/syscalls/wasix/sock_send_file.rs @@ -24,6 +24,8 @@ pub fn sock_send_file( count: Filesize, ret_sent: WasmPtr, ) -> Result { + wasi_try_ok!(WasiEnv::process_signals_and_exit(&mut ctx)?); + let total_written = wasi_try_ok!(sock_send_file_internal( &mut ctx, sock, in_fd, offset, count )?); @@ -44,8 +46,6 @@ pub(crate) fn sock_send_file_internal( offset: Filesize, mut count: Filesize, ) -> Result, WasiError> { - wasi_try_ok_ok!(WasiEnv::process_signals_and_exit(ctx)?); - let mut env = ctx.data(); let net = env.net(); let tasks = env.tasks().clone(); diff --git a/lib/wasix/src/syscalls/wasix/sock_send_to.rs b/lib/wasix/src/syscalls/wasix/sock_send_to.rs index 215c53325c0..2e3100d2a90 100644 --- a/lib/wasix/src/syscalls/wasix/sock_send_to.rs +++ b/lib/wasix/src/syscalls/wasix/sock_send_to.rs @@ -38,27 +38,35 @@ pub fn sock_send_to( let addr = SocketAddr::new(addr_ip, addr_port); Span::current().record("addr", &format!("{:?}", addr)); - wasi_try_ok!(sock_send_to_internal( - ctx, + let bytes_written = wasi_try_ok!(sock_send_to_internal( + &ctx, sock, - si_data, - si_data_len, + FdWriteSource::Iovs { + iovs: si_data, + iovs_len: si_data_len + }, si_flags, addr, - ret_data_len, )?); + + Span::current().record("nsent", bytes_written); + + let env = ctx.data(); + let memory = unsafe { env.memory_view(&ctx) }; + let bytes_written: M::Offset = + wasi_try_ok!(bytes_written.try_into().map_err(|_| Errno::Overflow)); + wasi_try_mem_ok!(ret_data_len.write(&memory, bytes_written)); + Ok(Errno::Success) } pub(crate) fn sock_send_to_internal( - mut ctx: FunctionEnvMut<'_, WasiEnv>, + ctx: &FunctionEnvMut<'_, WasiEnv>, sock: WasiFd, - si_data: WasmPtr<__wasi_ciovec_t, M>, - si_data_len: M::Offset, + si_data: FdWriteSource<'_, M>, _si_flags: SiFlags, addr: SocketAddr, - ret_data_len: WasmPtr, -) -> Result, WasiError> { +) -> Result, WasiError> { let env = ctx.data(); let memory = unsafe { env.memory_view(&ctx) }; @@ -68,11 +76,6 @@ pub(crate) fn sock_send_to_internal( sock, Rights::SOCK_SEND_TO, |socket, fd| async move { - let iovs_arr = si_data - .slice(&memory, si_data_len) - .map_err(mem_error_to_wasi)?; - let iovs_arr = iovs_arr.access().map_err(mem_error_to_wasi)?; - let nonblocking = fd.flags.contains(Fdflags::NONBLOCK); let timeout = socket .opt_time(TimeType::WriteTimeout) @@ -80,42 +83,57 @@ pub(crate) fn sock_send_to_internal( .flatten() .unwrap_or(Duration::from_secs(30)); - let mut sent = 0usize; - for iovs in iovs_arr.iter() { - let buf = WasmPtr::::new(iovs.buf) - .slice(&memory, iovs.buf_len) - .map_err(mem_error_to_wasi)? - .access() - .map_err(mem_error_to_wasi)?; - let local_sent = match socket - .send_to::( - env.tasks().deref(), - buf.as_ref(), - addr, - Some(timeout), - nonblocking, - ) - .await - { - Ok(s) => s, - Err(_) if sent > 0 => break, - Err(err) => return Err(err), - }; - sent += local_sent; - if local_sent != buf.len() { - break; + match si_data { + FdWriteSource::Iovs { iovs, iovs_len } => { + let iovs_arr = iovs.slice(&memory, iovs_len).map_err(mem_error_to_wasi)?; + let iovs_arr = iovs_arr.access().map_err(mem_error_to_wasi)?; + + let mut sent = 0usize; + for iovs in iovs_arr.iter() { + let buf = WasmPtr::::new(iovs.buf) + .slice(&memory, iovs.buf_len) + .map_err(mem_error_to_wasi)? + .access() + .map_err(mem_error_to_wasi)?; + let local_sent = match socket + .send_to::( + env.tasks().deref(), + buf.as_ref(), + addr, + Some(timeout), + nonblocking, + ) + .await + { + Ok(s) => s, + Err(_) if sent > 0 => break, + Err(err) => return Err(err), + }; + sent += local_sent; + if local_sent != buf.len() { + break; + } + } + Ok(sent) + } + FdWriteSource::Buffer(data) => { + socket + .send_to::( + env.tasks().deref(), + data.as_ref(), + addr, + Some(timeout), + nonblocking, + ) + .await } } - Ok(sent) }, )) }; - Span::current().record("nsent", bytes_written); - - let memory = unsafe { env.memory_view(&ctx) }; - let bytes_written: M::Offset = - wasi_try_ok_ok!(bytes_written.try_into().map_err(|_| Errno::Overflow)); - wasi_try_mem_ok_ok!(ret_data_len.write(&memory, bytes_written)); + trace!( + %bytes_written, + ); - Ok(Ok(())) + Ok(Ok(bytes_written)) } diff --git a/lib/wasix/src/syscalls/wasix/sock_set_opt_flag.rs b/lib/wasix/src/syscalls/wasix/sock_set_opt_flag.rs index 8ec9c0a70ab..15c5d6bb889 100644 --- a/lib/wasix/src/syscalls/wasix/sock_set_opt_flag.rs +++ b/lib/wasix/src/syscalls/wasix/sock_set_opt_flag.rs @@ -17,6 +17,12 @@ pub fn sock_set_opt_flag( opt: Sockoption, flag: Bool, ) -> Result { + let flag = match flag { + Bool::False => false, + Bool::True => true, + _ => return Ok(Errno::Inval), + }; + wasi_try_ok!(sock_set_opt_flag_internal(&mut ctx, sock, opt, flag)?); Ok(Errno::Success) @@ -26,14 +32,8 @@ pub(crate) fn sock_set_opt_flag_internal( ctx: &mut FunctionEnvMut<'_, WasiEnv>, sock: WasiFd, opt: Sockoption, - flag: Bool, + flag: bool, ) -> Result, WasiError> { - let flag = match flag { - Bool::False => false, - Bool::True => true, - _ => return Ok(Err(Errno::Inval)), - }; - let option: crate::net::socket::WasiSocketOption = opt.into(); wasi_try_ok_ok!(__sock_actor_mut( ctx, From 8af77a2b62c1d69dd06b50e11dc2de2438b532af Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Tue, 21 Nov 2023 15:47:05 +1100 Subject: [PATCH 056/129] Finished off the sockets for journaling --- lib/wasix/src/journaling/effector/mod.rs | 1 + .../journaling/effector/syscalls/fd_event.rs | 47 +++++++++++++++++++ lib/wasix/src/journaling/filter.rs | 3 +- lib/wasix/src/journaling/journal.rs | 11 +++-- lib/wasix/src/journaling/log_file.rs | 23 +++++++++ lib/wasix/src/net/socket.rs | 6 +++ lib/wasix/src/syscalls/mod.rs | 6 +++ lib/wasix/src/syscalls/wasi/fd_close.rs | 5 +- lib/wasix/src/syscalls/wasi/fd_dup.rs | 2 +- lib/wasix/src/syscalls/wasi/fd_event.rs | 34 +++++++++++--- lib/wasix/src/syscalls/wasi/fd_write.rs | 5 +- .../syscalls/wasi/path_create_directory.rs | 5 +- lib/wasix/src/syscalls/wasi/path_open.rs | 2 +- .../syscalls/wasi/path_remove_directory.rs | 2 +- lib/wasix/src/syscalls/wasi/path_rename.rs | 2 +- .../src/syscalls/wasi/path_unlink_file.rs | 2 +- lib/wasix/src/syscalls/wasix/port_addr_add.rs | 9 ++++ .../src/syscalls/wasix/port_addr_clear.rs | 9 ++++ .../src/syscalls/wasix/port_addr_remove.rs | 8 ++++ lib/wasix/src/syscalls/wasix/port_bridge.rs | 8 ++++ .../src/syscalls/wasix/port_dhcp_acquire.rs | 8 ++++ .../src/syscalls/wasix/port_gateway_set.rs | 8 ++++ .../src/syscalls/wasix/port_route_add.rs | 15 ++++++ .../src/syscalls/wasix/port_route_clear.rs | 8 ++++ .../src/syscalls/wasix/port_route_remove.rs | 8 ++++ lib/wasix/src/syscalls/wasix/port_unbridge.rs | 9 ++++ lib/wasix/src/syscalls/wasix/sock_accept.rs | 11 +++++ lib/wasix/src/syscalls/wasix/sock_bind.rs | 15 ++++-- lib/wasix/src/syscalls/wasix/sock_connect.rs | 8 ++++ .../syscalls/wasix/sock_join_multicast_v4.rs | 10 ++++ .../syscalls/wasix/sock_join_multicast_v6.rs | 10 ++++ .../syscalls/wasix/sock_leave_multicast_v4.rs | 10 ++++ .../syscalls/wasix/sock_leave_multicast_v6.rs | 10 ++++ lib/wasix/src/syscalls/wasix/sock_listen.rs | 10 ++++ lib/wasix/src/syscalls/wasix/sock_open.rs | 8 ++++ lib/wasix/src/syscalls/wasix/sock_send.rs | 17 +++++++ .../src/syscalls/wasix/sock_send_file.rs | 9 ++++ lib/wasix/src/syscalls/wasix/sock_send_to.rs | 19 +++++++- .../src/syscalls/wasix/sock_set_opt_flag.rs | 8 ++++ .../src/syscalls/wasix/sock_set_opt_size.rs | 8 ++++ .../src/syscalls/wasix/sock_set_opt_time.rs | 8 ++++ lib/wasix/src/syscalls/wasix/sock_shutdown.rs | 8 ++++ 42 files changed, 383 insertions(+), 32 deletions(-) create mode 100644 lib/wasix/src/journaling/effector/syscalls/fd_event.rs diff --git a/lib/wasix/src/journaling/effector/mod.rs b/lib/wasix/src/journaling/effector/mod.rs index 36de01e93b9..ae9b9fb57c6 100644 --- a/lib/wasix/src/journaling/effector/mod.rs +++ b/lib/wasix/src/journaling/effector/mod.rs @@ -35,6 +35,7 @@ mod syscalls { mod fd_allocate; mod fd_close; mod fd_duplicate; + mod fd_event; mod fd_pipe; mod fd_renumber; mod fd_seek; diff --git a/lib/wasix/src/journaling/effector/syscalls/fd_event.rs b/lib/wasix/src/journaling/effector/syscalls/fd_event.rs new file mode 100644 index 00000000000..7efd599a582 --- /dev/null +++ b/lib/wasix/src/journaling/effector/syscalls/fd_event.rs @@ -0,0 +1,47 @@ +use wasmer_wasix_types::wasi::EventFdFlags; + +use super::*; + +impl JournalEffector { + pub fn save_fd_event( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + initial_val: u64, + flags: EventFdFlags, + fd: Fd, + ) -> anyhow::Result<()> { + Self::save_event( + ctx, + JournalEntry::CreateEvent { + initial_val, + flags, + fd, + }, + ) + } + + pub fn apply_fd_event( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + initial_val: u64, + flags: EventFdFlags, + fd: Fd, + ) -> anyhow::Result<()> { + let ret_fd = crate::syscalls::fd_event_internal(ctx, initial_val, flags) + .map(|r| r.map_err(|err| err.to_string())) + .unwrap_or_else(|err| Err(err.to_string())) + .map_err(|err| { + anyhow::format_err!("journal restore error: failed to create event - {}", err) + })?; + + let ret = crate::syscalls::fd_renumber_internal(ctx, ret_fd, fd); + if ret != Errno::Success { + bail!( + "journal restore error: failed renumber file descriptor after create event (from={}, to={}) - {}", + ret_fd, + fd, + ret + ); + } + + Ok(()) + } +} diff --git a/lib/wasix/src/journaling/filter.rs b/lib/wasix/src/journaling/filter.rs index 33e28d63fd2..a91df18500e 100644 --- a/lib/wasix/src/journaling/filter.rs +++ b/lib/wasix/src/journaling/filter.rs @@ -106,7 +106,8 @@ impl Journal for FilteredJournal { | JournalEntry::CreateHardLink { .. } | JournalEntry::CreateSymbolicLink { .. } | JournalEntry::ChangeDirectory { .. } - | JournalEntry::CreatePipe { .. } => { + | JournalEntry::CreatePipe { .. } + | JournalEntry::CreateEvent { .. } => { if self.filter_fs { return Ok(()); } diff --git a/lib/wasix/src/journaling/journal.rs b/lib/wasix/src/journaling/journal.rs index 5eb84b5d5a9..e70932c4087 100644 --- a/lib/wasix/src/journaling/journal.rs +++ b/lib/wasix/src/journaling/journal.rs @@ -4,9 +4,9 @@ use std::time::SystemTime; use std::{borrow::Cow, ops::Range}; use virtual_net::{Duration, IpAddr, IpCidr, Ipv4Addr, Ipv6Addr, StreamSecurity}; use wasmer_wasix_types::wasi::{ - Addressfamily, Advice, EpollCtl, EpollEventCtl, ExitCode, Fdflags, FileDelta, Filesize, - Fstflags, LookupFlags, Oflags, Rights, SiFlags, Snapshot0Clockid, SockProto, Sockoption, - Socktype, Timestamp, Tty, Whence, + Addressfamily, Advice, EpollCtl, EpollEventCtl, EventFdFlags, ExitCode, Fdflags, FileDelta, + Filesize, Fstflags, LookupFlags, Oflags, Rights, SiFlags, Snapshot0Clockid, SockProto, + Sockoption, Socktype, Timestamp, Tty, Whence, }; use futures::future::LocalBoxFuture; @@ -189,6 +189,11 @@ pub enum JournalEntry<'a> { fd1: Fd, fd2: Fd, }, + CreateEvent { + initial_val: u64, + flags: EventFdFlags, + fd: Fd, + }, PortAddAddr { cidr: IpCidr, }, diff --git a/lib/wasix/src/journaling/log_file.rs b/lib/wasix/src/journaling/log_file.rs index 1267a833f62..6e01bdf160f 100644 --- a/lib/wasix/src/journaling/log_file.rs +++ b/lib/wasix/src/journaling/log_file.rs @@ -180,6 +180,11 @@ pub(crate) enum LogFileJournalEntry { fd1: u32, fd2: u32, }, + CreateEventV1 { + initial_val: u64, + flags: u16, + fd: u32, + }, PortAddAddrV1 { cidr: IpCidr, }, @@ -1111,6 +1116,15 @@ impl<'a> From> for LogFileJournalEntry { fd, how: how.into(), }, + JournalEntry::CreateEvent { + initial_val, + flags, + fd, + } => Self::CreateEventV1 { + initial_val, + flags, + fd, + }, } } } @@ -1489,6 +1503,15 @@ impl<'a> From for JournalEntry<'a> { fd, how: how.into(), }, + LogFileJournalEntry::CreateEventV1 { + initial_val, + flags, + fd, + } => Self::CreateEvent { + initial_val, + flags, + fd, + }, } } } diff --git a/lib/wasix/src/net/socket.rs b/lib/wasix/src/net/socket.rs index eabab412681..debf5fb78ee 100644 --- a/lib/wasix/src/net/socket.rs +++ b/lib/wasix/src/net/socket.rs @@ -992,6 +992,9 @@ impl InodeSocket { InodeSocketKind::PreSocket { .. } => { return Poll::Ready(Err(Errno::Notconn)) } + InodeSocketKind::RemoteTcpStream { .. } => { + return Poll::Ready(Ok(self.data.len())) + } _ => return Poll::Ready(Err(Errno::Notsup)), }; return match res { @@ -1069,6 +1072,9 @@ impl InodeSocket { InodeSocketKind::PreSocket { .. } => { return Poll::Ready(Err(Errno::Notconn)) } + InodeSocketKind::RemoteTcpStream { .. } => { + return Poll::Ready(Ok(self.data.len())) + } _ => return Poll::Ready(Err(Errno::Notsup)), }; return match res { diff --git a/lib/wasix/src/syscalls/mod.rs b/lib/wasix/src/syscalls/mod.rs index 9c4d7f96b83..e9e573ccfad 100644 --- a/lib/wasix/src/syscalls/mod.rs +++ b/lib/wasix/src/syscalls/mod.rs @@ -1727,6 +1727,12 @@ pub fn restore_snapshot( JournalEffector::apply_sock_shutdown(&mut ctx, fd, how) .map_err(anyhow_err_to_runtime_err)? } + crate::journaling::JournalEntry::CreateEvent { + initial_val, + flags, + fd, + } => JournalEffector::apply_fd_event(&mut ctx, initial_val, flags, fd) + .map_err(anyhow_err_to_runtime_err)?, } } // If we are not in the same module then we fire off an exit diff --git a/lib/wasix/src/syscalls/wasi/fd_close.rs b/lib/wasix/src/syscalls/wasi/fd_close.rs index c79b0023d4d..9196d2ade34 100644 --- a/lib/wasix/src/syscalls/wasi/fd_close.rs +++ b/lib/wasix/src/syscalls/wasi/fd_close.rs @@ -35,10 +35,7 @@ pub fn fd_close(mut ctx: FunctionEnvMut<'_, WasiEnv>, fd: WasiFd) -> Result( #[cfg(feature = "journal")] if env.enable_journal { JournalEffector::save_fd_duplicate(&mut ctx, fd, copied_fd).map_err(|err| { - tracing::error!("failed to save file descriptor renumber event - {}", err); + tracing::error!("failed to save file descriptor duplicate event - {}", err); WasiError::Exit(ExitCode::Errno(Errno::Fault)) })?; } diff --git a/lib/wasix/src/syscalls/wasi/fd_event.rs b/lib/wasix/src/syscalls/wasi/fd_event.rs index 00e0bb8a754..6f3a01f2942 100644 --- a/lib/wasix/src/syscalls/wasi/fd_event.rs +++ b/lib/wasix/src/syscalls/wasi/fd_event.rs @@ -5,11 +5,34 @@ use crate::{fs::NotificationInner, syscalls::*}; /// Creates a file handle for event notifications #[instrument(level = "debug", skip_all, fields(%initial_val, ret_fd = field::Empty), ret)] pub fn fd_event( - ctx: FunctionEnvMut<'_, WasiEnv>, + mut ctx: FunctionEnvMut<'_, WasiEnv>, initial_val: u64, flags: EventFdFlags, ret_fd: WasmPtr, -) -> Errno { +) -> Result { + let fd = wasi_try_ok!(fd_event_internal(&mut ctx, initial_val, flags)?); + + let env = ctx.data(); + let (memory, state, _) = unsafe { env.get_memory_and_wasi_state_and_inodes(&ctx, 0) }; + Span::current().record("ret_fd", fd); + wasi_try_mem_ok!(ret_fd.write(&memory, fd)); + + #[cfg(feature = "journal")] + if env.enable_journal { + JournalEffector::save_fd_event(&mut ctx, initial_val, flags, fd).map_err(|err| { + tracing::error!("failed to save fd_event event - {}", err); + WasiError::Exit(ExitCode::Errno(Errno::Fault)) + })?; + } + + Ok(Errno::Success) +} + +pub fn fd_event_internal( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + initial_val: u64, + flags: EventFdFlags, +) -> Result, WasiError> { let env = ctx.data(); let (memory, state, mut inodes) = unsafe { env.get_memory_and_wasi_state_and_inodes(&ctx, 0) }; @@ -26,12 +49,9 @@ pub fn fd_event( | Rights::FD_WRITE | Rights::POLL_FD_READWRITE | Rights::FD_FDSTAT_SET_FLAGS; - let fd = wasi_try!(state + let fd = wasi_try_ok_ok!(state .fs .create_fd(rights, rights, Fdflags::empty(), 0, inode)); - Span::current().record("ret_fd", fd); - wasi_try_mem!(ret_fd.write(&memory, fd)); - - Errno::Success + Ok(Ok(fd)) } diff --git a/lib/wasix/src/syscalls/wasi/fd_write.rs b/lib/wasix/src/syscalls/wasi/fd_write.rs index 220190ef16f..1f70b4ce67b 100644 --- a/lib/wasix/src/syscalls/wasi/fd_write.rs +++ b/lib/wasix/src/syscalls/wasi/fd_write.rs @@ -402,10 +402,7 @@ pub(crate) fn fd_write_internal( if let FdWriteSource::Iovs { iovs, iovs_len } = data { JournalEffector::save_fd_write(ctx, fd, offset, bytes_written, iovs, iovs_len) .map_err(|err| { - tracing::error!( - "failed to save terminal data to snapshot capturer - {}", - err - ); + tracing::error!("failed to save terminal data - {}", err); WasiError::Exit(ExitCode::Errno(Errno::Fault)) })?; } diff --git a/lib/wasix/src/syscalls/wasi/path_create_directory.rs b/lib/wasix/src/syscalls/wasi/path_create_directory.rs index 23f10c0ab95..8e514f43697 100644 --- a/lib/wasix/src/syscalls/wasi/path_create_directory.rs +++ b/lib/wasix/src/syscalls/wasi/path_create_directory.rs @@ -41,10 +41,7 @@ pub fn path_create_directory( #[cfg(feature = "journal")] if env.enable_journal { JournalEffector::save_path_create_directory(&mut ctx, fd, path_string).map_err(|err| { - tracing::error!( - "failed to save create directory event to snapshot capturer - {}", - err - ); + tracing::error!("failed to save create directory event - {}", err); WasiError::Exit(ExitCode::Errno(Errno::Fault)) })?; } diff --git a/lib/wasix/src/syscalls/wasi/path_open.rs b/lib/wasix/src/syscalls/wasi/path_open.rs index 803a4a0be24..58528347995 100644 --- a/lib/wasix/src/syscalls/wasi/path_open.rs +++ b/lib/wasix/src/syscalls/wasi/path_open.rs @@ -93,7 +93,7 @@ pub fn path_open( fs_flags, ) .map_err(|err| { - tracing::error!("failed to save unlink event to snapshot capturer - {}", err); + tracing::error!("failed to save unlink event - {}", err); WasiError::Exit(ExitCode::Errno(Errno::Fault)) })?; } diff --git a/lib/wasix/src/syscalls/wasi/path_remove_directory.rs b/lib/wasix/src/syscalls/wasi/path_remove_directory.rs index 24d9dc3b7d7..d6ac9e67f76 100644 --- a/lib/wasix/src/syscalls/wasi/path_remove_directory.rs +++ b/lib/wasix/src/syscalls/wasi/path_remove_directory.rs @@ -32,7 +32,7 @@ pub fn path_remove_directory( if env.enable_journal { wasi_try!( JournalEffector::save_path_remove_directory(&mut ctx, fd, path_str).map_err(|err| { - tracing::error!("failed to save unlink event to snapshot capturer - {}", err); + tracing::error!("failed to save remove directory event - {}", err); Errno::Fault }) ) diff --git a/lib/wasix/src/syscalls/wasi/path_rename.rs b/lib/wasix/src/syscalls/wasi/path_rename.rs index ca5084dc4f2..147c8f209ac 100644 --- a/lib/wasix/src/syscalls/wasi/path_rename.rs +++ b/lib/wasix/src/syscalls/wasi/path_rename.rs @@ -43,7 +43,7 @@ pub fn path_rename( if env.enable_journal { JournalEffector::save_path_rename(&mut ctx, old_fd, source_str, new_fd, target_str) .map_err(|err| { - tracing::error!("failed to save unlink event to snapshot capturer - {}", err); + tracing::error!("failed to save path rename event - {}", err); WasiError::Exit(ExitCode::Errno(Errno::Fault)) })?; } diff --git a/lib/wasix/src/syscalls/wasi/path_unlink_file.rs b/lib/wasix/src/syscalls/wasi/path_unlink_file.rs index 0159bf6eeac..a0f4d15ac83 100644 --- a/lib/wasix/src/syscalls/wasi/path_unlink_file.rs +++ b/lib/wasix/src/syscalls/wasi/path_unlink_file.rs @@ -40,7 +40,7 @@ pub fn path_unlink_file( if env.enable_journal { wasi_try_ok!( JournalEffector::save_path_unlink(&mut ctx, fd, path_str).map_err(|err| { - tracing::error!("failed to save unlink event to snapshot capturer - {}", err); + tracing::error!("failed to save unlink event - {}", err); Errno::Fault }) ) diff --git a/lib/wasix/src/syscalls/wasix/port_addr_add.rs b/lib/wasix/src/syscalls/wasix/port_addr_add.rs index 37b4f1d53a7..5a9d5909a66 100644 --- a/lib/wasix/src/syscalls/wasix/port_addr_add.rs +++ b/lib/wasix/src/syscalls/wasix/port_addr_add.rs @@ -21,6 +21,15 @@ pub fn port_addr_add( Span::current().record("ip", &format!("{:?}", cidr)); wasi_try_ok!(port_addr_add_internal(&mut ctx, cidr)?); + + #[cfg(feature = "journal")] + if ctx.data().enable_journal { + JournalEffector::save_port_addr_add(&mut ctx, cidr).map_err(|err| { + tracing::error!("failed to save port_addr_add event - {}", err); + WasiError::Exit(ExitCode::Errno(Errno::Fault)) + })?; + } + Ok(Errno::Success) } diff --git a/lib/wasix/src/syscalls/wasix/port_addr_clear.rs b/lib/wasix/src/syscalls/wasix/port_addr_clear.rs index e4bb6c28991..9ceb6c45c66 100644 --- a/lib/wasix/src/syscalls/wasix/port_addr_clear.rs +++ b/lib/wasix/src/syscalls/wasix/port_addr_clear.rs @@ -6,6 +6,15 @@ use crate::syscalls::*; #[instrument(level = "debug", skip_all, ret)] pub fn port_addr_clear(mut ctx: FunctionEnvMut<'_, WasiEnv>) -> Result { wasi_try_ok!(port_addr_clear_internal(&mut ctx)?); + + #[cfg(feature = "journal")] + if ctx.data().enable_journal { + JournalEffector::save_port_addr_clear(&mut ctx).map_err(|err| { + tracing::error!("failed to save port_addr_clear event - {}", err); + WasiError::Exit(ExitCode::Errno(Errno::Fault)) + })?; + } + Ok(Errno::Success) } diff --git a/lib/wasix/src/syscalls/wasix/port_addr_remove.rs b/lib/wasix/src/syscalls/wasix/port_addr_remove.rs index 9d91eec380b..eb689243576 100644 --- a/lib/wasix/src/syscalls/wasix/port_addr_remove.rs +++ b/lib/wasix/src/syscalls/wasix/port_addr_remove.rs @@ -20,6 +20,14 @@ pub fn port_addr_remove( wasi_try_ok!(port_addr_remove_internal(&mut ctx, ip)?); + #[cfg(feature = "journal")] + if ctx.data().enable_journal { + JournalEffector::save_port_addr_remove(&mut ctx, ip).map_err(|err| { + tracing::error!("failed to save port_addr_remove event - {}", err); + WasiError::Exit(ExitCode::Errno(Errno::Fault)) + })?; + } + Ok(Errno::Success) } diff --git a/lib/wasix/src/syscalls/wasix/port_bridge.rs b/lib/wasix/src/syscalls/wasix/port_bridge.rs index 47d8b739fac..e686ec5d078 100644 --- a/lib/wasix/src/syscalls/wasix/port_bridge.rs +++ b/lib/wasix/src/syscalls/wasix/port_bridge.rs @@ -40,6 +40,14 @@ pub fn port_bridge( security )?); + #[cfg(feature = "journal")] + if ctx.data().enable_journal { + JournalEffector::save_port_bridge(&mut ctx, network, token, security).map_err(|err| { + tracing::error!("failed to save port_bridge event - {}", err); + WasiError::Exit(ExitCode::Errno(Errno::Fault)) + })?; + } + Ok(Errno::Success) } diff --git a/lib/wasix/src/syscalls/wasix/port_dhcp_acquire.rs b/lib/wasix/src/syscalls/wasix/port_dhcp_acquire.rs index 80ba6e4aaba..1415f76c609 100644 --- a/lib/wasix/src/syscalls/wasix/port_dhcp_acquire.rs +++ b/lib/wasix/src/syscalls/wasix/port_dhcp_acquire.rs @@ -7,6 +7,14 @@ use crate::syscalls::*; pub fn port_dhcp_acquire(mut ctx: FunctionEnvMut<'_, WasiEnv>) -> Result { wasi_try_ok!(port_dhcp_acquire_internal(&mut ctx)?); + #[cfg(feature = "journal")] + if ctx.data().enable_journal { + JournalEffector::save_port_dhcp_acquire(&mut ctx).map_err(|err| { + tracing::error!("failed to save port_dhcp_acquire event - {}", err); + WasiError::Exit(ExitCode::Errno(Errno::Fault)) + })?; + } + Ok(Errno::Success) } diff --git a/lib/wasix/src/syscalls/wasix/port_gateway_set.rs b/lib/wasix/src/syscalls/wasix/port_gateway_set.rs index ef74389b492..fc0549ea2c1 100644 --- a/lib/wasix/src/syscalls/wasix/port_gateway_set.rs +++ b/lib/wasix/src/syscalls/wasix/port_gateway_set.rs @@ -20,6 +20,14 @@ pub fn port_gateway_set( wasi_try_ok!(port_gateway_set_internal(&mut ctx, ip)?); + #[cfg(feature = "journal")] + if ctx.data().enable_journal { + JournalEffector::save_port_gateway_set(&mut ctx, ip).map_err(|err| { + tracing::error!("failed to save port_gateway_set event - {}", err); + WasiError::Exit(ExitCode::Errno(Errno::Fault)) + })?; + } + Ok(Errno::Success) } diff --git a/lib/wasix/src/syscalls/wasix/port_route_add.rs b/lib/wasix/src/syscalls/wasix/port_route_add.rs index f2d94e95fea..00e9f781c34 100644 --- a/lib/wasix/src/syscalls/wasix/port_route_add.rs +++ b/lib/wasix/src/syscalls/wasix/port_route_add.rs @@ -43,6 +43,21 @@ pub fn port_route_add( expires_at )?); + #[cfg(feature = "journal")] + if ctx.data().enable_journal { + JournalEffector::save_port_route_add( + &mut ctx, + cidr, + via_router, + preferred_until, + expires_at, + ) + .map_err(|err| { + tracing::error!("failed to save port_route_add event - {}", err); + WasiError::Exit(ExitCode::Errno(Errno::Fault)) + })?; + } + Ok(Errno::Success) } diff --git a/lib/wasix/src/syscalls/wasix/port_route_clear.rs b/lib/wasix/src/syscalls/wasix/port_route_clear.rs index b1a4dc7b3db..3eca5de4208 100644 --- a/lib/wasix/src/syscalls/wasix/port_route_clear.rs +++ b/lib/wasix/src/syscalls/wasix/port_route_clear.rs @@ -7,6 +7,14 @@ use crate::syscalls::*; pub fn port_route_clear(mut ctx: FunctionEnvMut<'_, WasiEnv>) -> Result { wasi_try_ok!(port_route_clear_internal(&mut ctx)?); + #[cfg(feature = "journal")] + if ctx.data().enable_journal { + JournalEffector::save_port_route_clear(&mut ctx).map_err(|err| { + tracing::error!("failed to save port_route_clear event - {}", err); + WasiError::Exit(ExitCode::Errno(Errno::Fault)) + })?; + } + Ok(Errno::Success) } diff --git a/lib/wasix/src/syscalls/wasix/port_route_remove.rs b/lib/wasix/src/syscalls/wasix/port_route_remove.rs index 60e4301d4cc..0b1b5da2923 100644 --- a/lib/wasix/src/syscalls/wasix/port_route_remove.rs +++ b/lib/wasix/src/syscalls/wasix/port_route_remove.rs @@ -16,6 +16,14 @@ pub fn port_route_remove( wasi_try_ok!(port_route_remove_internal(&mut ctx, ip)?); + #[cfg(feature = "journal")] + if ctx.data().enable_journal { + JournalEffector::save_port_route_remove(&mut ctx, ip).map_err(|err| { + tracing::error!("failed to save port_route_remove event - {}", err); + WasiError::Exit(ExitCode::Errno(Errno::Fault)) + })?; + } + Ok(Errno::Success) } diff --git a/lib/wasix/src/syscalls/wasix/port_unbridge.rs b/lib/wasix/src/syscalls/wasix/port_unbridge.rs index 3d0cd749c8a..4495cd04862 100644 --- a/lib/wasix/src/syscalls/wasix/port_unbridge.rs +++ b/lib/wasix/src/syscalls/wasix/port_unbridge.rs @@ -6,6 +6,15 @@ use crate::syscalls::*; #[instrument(level = "debug", skip_all, ret)] pub fn port_unbridge(mut ctx: FunctionEnvMut<'_, WasiEnv>) -> Result { wasi_try_ok!(port_unbridge_internal(&mut ctx)?); + + #[cfg(feature = "journal")] + if ctx.data().enable_journal { + JournalEffector::save_port_unbridge(&mut ctx).map_err(|err| { + tracing::error!("failed to save port_unbridge event - {}", err); + WasiError::Exit(ExitCode::Errno(Errno::Fault)) + })?; + } + Ok(Errno::Success) } diff --git a/lib/wasix/src/syscalls/wasix/sock_accept.rs b/lib/wasix/src/syscalls/wasix/sock_accept.rs index 9b996adae07..9ce6b75a63c 100644 --- a/lib/wasix/src/syscalls/wasix/sock_accept.rs +++ b/lib/wasix/src/syscalls/wasix/sock_accept.rs @@ -68,6 +68,17 @@ pub fn sock_accept_v2( let (fd, addr) = wasi_try_ok!(sock_accept_internal(env, sock, fd_flags, nonblocking)?); + #[cfg(feature = "journal")] + if ctx.data().enable_journal { + JournalEffector::save_sock_accepted(&mut ctx, sock, fd, addr, fd_flags, nonblocking) + .map_err(|err| { + tracing::error!("failed to save sock_accepted event - {}", err); + WasiError::Exit(ExitCode::Errno(Errno::Fault)) + })?; + } + + let env = ctx.data(); + let (memory, state, _) = unsafe { env.get_memory_and_wasi_state_and_inodes(&ctx, 0) }; wasi_try_mem_ok!(ro_fd.write(&memory, fd)); wasi_try_ok!(crate::net::write_ip_port( &memory, diff --git a/lib/wasix/src/syscalls/wasix/sock_bind.rs b/lib/wasix/src/syscalls/wasix/sock_bind.rs index 7f72f98162d..db951f9e608 100644 --- a/lib/wasix/src/syscalls/wasix/sock_bind.rs +++ b/lib/wasix/src/syscalls/wasix/sock_bind.rs @@ -14,14 +14,23 @@ pub fn sock_bind( mut ctx: FunctionEnvMut<'_, WasiEnv>, sock: WasiFd, addr: WasmPtr<__wasi_addr_port_t, M>, -) -> Errno { +) -> Result { let env = ctx.data(); let memory = unsafe { env.memory_view(&ctx) }; - let addr = wasi_try!(crate::net::read_ip_port(&memory, addr)); + + let addr = wasi_try_ok!(crate::net::read_ip_port(&memory, addr)); let addr = SocketAddr::new(addr.0, addr.1); Span::current().record("addr", &format!("{:?}", addr)); - Errno::Success + #[cfg(feature = "journal")] + if ctx.data().enable_journal { + JournalEffector::save_sock_bind(&mut ctx, sock, addr).map_err(|err| { + tracing::error!("failed to save sock_bind event - {}", err); + WasiError::Exit(ExitCode::Errno(Errno::Fault)) + })?; + } + + Ok(Errno::Success) } pub(crate) fn sock_bind_internal( diff --git a/lib/wasix/src/syscalls/wasix/sock_connect.rs b/lib/wasix/src/syscalls/wasix/sock_connect.rs index f0a01944674..03e51ec28b4 100644 --- a/lib/wasix/src/syscalls/wasix/sock_connect.rs +++ b/lib/wasix/src/syscalls/wasix/sock_connect.rs @@ -27,6 +27,14 @@ pub fn sock_connect( wasi_try_ok!(sock_connect_internal(&mut ctx, sock, addr)?); + #[cfg(feature = "journal")] + if ctx.data().enable_journal { + JournalEffector::save_sock_connect(&mut ctx, sock, addr).map_err(|err| { + tracing::error!("failed to save sock_connected event - {}", err); + WasiError::Exit(ExitCode::Errno(Errno::Fault)) + })?; + } + Ok(Errno::Success) } diff --git a/lib/wasix/src/syscalls/wasix/sock_join_multicast_v4.rs b/lib/wasix/src/syscalls/wasix/sock_join_multicast_v4.rs index 84ea760fde3..56ef67e9299 100644 --- a/lib/wasix/src/syscalls/wasix/sock_join_multicast_v4.rs +++ b/lib/wasix/src/syscalls/wasix/sock_join_multicast_v4.rs @@ -25,6 +25,16 @@ pub fn sock_join_multicast_v4( &mut ctx, sock, multiaddr, iface )?); + #[cfg(feature = "journal")] + if ctx.data().enable_journal { + JournalEffector::save_sock_join_ipv4_multicast(&mut ctx, sock, multiaddr, iface).map_err( + |err| { + tracing::error!("failed to save sock_join_ipv4_multicast event - {}", err); + WasiError::Exit(ExitCode::Errno(Errno::Fault)) + }, + )?; + } + Ok(Errno::Success) } diff --git a/lib/wasix/src/syscalls/wasix/sock_join_multicast_v6.rs b/lib/wasix/src/syscalls/wasix/sock_join_multicast_v6.rs index 9b141dfc8de..420ddfbbd21 100644 --- a/lib/wasix/src/syscalls/wasix/sock_join_multicast_v6.rs +++ b/lib/wasix/src/syscalls/wasix/sock_join_multicast_v6.rs @@ -24,6 +24,16 @@ pub fn sock_join_multicast_v6( &mut ctx, sock, multiaddr, iface )?); + #[cfg(feature = "journal")] + if ctx.data().enable_journal { + JournalEffector::save_sock_join_ipv6_multicast(&mut ctx, sock, multiaddr, iface).map_err( + |err| { + tracing::error!("failed to save sock_join_ipv6_multicast event - {}", err); + WasiError::Exit(ExitCode::Errno(Errno::Fault)) + }, + )?; + } + Ok(Errno::Success) } diff --git a/lib/wasix/src/syscalls/wasix/sock_leave_multicast_v4.rs b/lib/wasix/src/syscalls/wasix/sock_leave_multicast_v4.rs index 37306672b75..b668c628aa4 100644 --- a/lib/wasix/src/syscalls/wasix/sock_leave_multicast_v4.rs +++ b/lib/wasix/src/syscalls/wasix/sock_leave_multicast_v4.rs @@ -25,6 +25,16 @@ pub fn sock_leave_multicast_v4( &mut ctx, sock, multiaddr, iface )?); + #[cfg(feature = "journal")] + if ctx.data().enable_journal { + JournalEffector::save_sock_leave_ipv4_multicast(&mut ctx, sock, multiaddr, iface).map_err( + |err| { + tracing::error!("failed to save sock_leave_ipv4_multicast event - {}", err); + WasiError::Exit(ExitCode::Errno(Errno::Fault)) + }, + )?; + } + Ok(Errno::Success) } diff --git a/lib/wasix/src/syscalls/wasix/sock_leave_multicast_v6.rs b/lib/wasix/src/syscalls/wasix/sock_leave_multicast_v6.rs index 70b7ec48088..886bfd563e5 100644 --- a/lib/wasix/src/syscalls/wasix/sock_leave_multicast_v6.rs +++ b/lib/wasix/src/syscalls/wasix/sock_leave_multicast_v6.rs @@ -24,6 +24,16 @@ pub fn sock_leave_multicast_v6( &mut ctx, sock, multiaddr, iface )?); + #[cfg(feature = "journal")] + if ctx.data().enable_journal { + JournalEffector::save_sock_leave_ipv6_multicast(&mut ctx, sock, multiaddr, iface).map_err( + |err| { + tracing::error!("failed to save sock_leave_ipv6_multicast event - {}", err); + WasiError::Exit(ExitCode::Errno(Errno::Fault)) + }, + )?; + } + Ok(Errno::Success) } diff --git a/lib/wasix/src/syscalls/wasix/sock_listen.rs b/lib/wasix/src/syscalls/wasix/sock_listen.rs index a746f2b90fd..5ec2836c69d 100644 --- a/lib/wasix/src/syscalls/wasix/sock_listen.rs +++ b/lib/wasix/src/syscalls/wasix/sock_listen.rs @@ -24,6 +24,16 @@ pub fn sock_listen( let env = ctx.data(); let backlog: usize = wasi_try_ok!(backlog.try_into().map_err(|_| Errno::Inval)); + wasi_try_ok!(sock_listen_internal(&mut ctx, sock, backlog)?); + + #[cfg(feature = "journal")] + if ctx.data().enable_journal { + JournalEffector::save_sock_listen(&mut ctx, sock, backlog).map_err(|err| { + tracing::error!("failed to save sock_listen event - {}", err); + WasiError::Exit(ExitCode::Errno(Errno::Fault)) + })?; + } + Ok(Errno::Success) } diff --git a/lib/wasix/src/syscalls/wasix/sock_open.rs b/lib/wasix/src/syscalls/wasix/sock_open.rs index 5c068332214..dfd0678d8fd 100644 --- a/lib/wasix/src/syscalls/wasix/sock_open.rs +++ b/lib/wasix/src/syscalls/wasix/sock_open.rs @@ -45,6 +45,14 @@ pub fn sock_open( let fd = wasi_try_ok!(sock_open_internal(&mut ctx, af, ty, pt)?); + #[cfg(feature = "journal")] + if ctx.data().enable_journal { + JournalEffector::save_sock_open(&mut ctx, af, ty, pt, fd).map_err(|err| { + tracing::error!("failed to save sock_open event - {}", err); + WasiError::Exit(ExitCode::Errno(Errno::Fault)) + })?; + } + let env = ctx.data(); let (memory, state, inodes) = unsafe { env.get_memory_and_wasi_state_and_inodes(&ctx, 0) }; wasi_try_mem_ok!(ro_sock.write(&memory, fd)); diff --git a/lib/wasix/src/syscalls/wasix/sock_send.rs b/lib/wasix/src/syscalls/wasix/sock_send.rs index 4c98973759b..ec61a9f6225 100644 --- a/lib/wasix/src/syscalls/wasix/sock_send.rs +++ b/lib/wasix/src/syscalls/wasix/sock_send.rs @@ -64,6 +64,23 @@ pub fn sock_send( si_flags, )?) }; + + #[cfg(feature = "journal")] + if ctx.data().enable_journal { + JournalEffector::save_sock_send( + &mut ctx, + fd, + bytes_written, + si_data, + si_data_len, + si_flags, + ) + .map_err(|err| { + tracing::error!("failed to save sock_send event - {}", err); + WasiError::Exit(ExitCode::Errno(Errno::Fault)) + })?; + } + Span::current().record("nsent", bytes_written); let env = ctx.data(); diff --git a/lib/wasix/src/syscalls/wasix/sock_send_file.rs b/lib/wasix/src/syscalls/wasix/sock_send_file.rs index 85236beb24b..70419d74cd5 100644 --- a/lib/wasix/src/syscalls/wasix/sock_send_file.rs +++ b/lib/wasix/src/syscalls/wasix/sock_send_file.rs @@ -30,6 +30,15 @@ pub fn sock_send_file( &mut ctx, sock, in_fd, offset, count )?); + #[cfg(feature = "journal")] + if ctx.data().enable_journal { + JournalEffector::save_sock_send_file::(&mut ctx, sock, in_fd, offset, total_written) + .map_err(|err| { + tracing::error!("failed to save sock_send_file event - {}", err); + WasiError::Exit(ExitCode::Errno(Errno::Fault)) + })?; + } + Span::current().record("nsent", total_written); let env = ctx.data(); diff --git a/lib/wasix/src/syscalls/wasix/sock_send_to.rs b/lib/wasix/src/syscalls/wasix/sock_send_to.rs index 2e3100d2a90..d2264907be0 100644 --- a/lib/wasix/src/syscalls/wasix/sock_send_to.rs +++ b/lib/wasix/src/syscalls/wasix/sock_send_to.rs @@ -19,7 +19,7 @@ use crate::{net::socket::TimeType, syscalls::*}; /// Number of bytes transmitted. #[instrument(level = "trace", skip_all, fields(%sock, ?addr, nsent = field::Empty), ret)] pub fn sock_send_to( - ctx: FunctionEnvMut<'_, WasiEnv>, + mut ctx: FunctionEnvMut<'_, WasiEnv>, sock: WasiFd, si_data: WasmPtr<__wasi_ciovec_t, M>, si_data_len: M::Offset, @@ -49,6 +49,23 @@ pub fn sock_send_to( addr, )?); + #[cfg(feature = "journal")] + if ctx.data().enable_journal { + JournalEffector::save_sock_send_to::( + &mut ctx, + sock, + bytes_written, + si_data, + si_data_len, + addr, + si_flags, + ) + .map_err(|err| { + tracing::error!("failed to save sock_send_to event - {}", err); + WasiError::Exit(ExitCode::Errno(Errno::Fault)) + })?; + } + Span::current().record("nsent", bytes_written); let env = ctx.data(); diff --git a/lib/wasix/src/syscalls/wasix/sock_set_opt_flag.rs b/lib/wasix/src/syscalls/wasix/sock_set_opt_flag.rs index 15c5d6bb889..993779520a1 100644 --- a/lib/wasix/src/syscalls/wasix/sock_set_opt_flag.rs +++ b/lib/wasix/src/syscalls/wasix/sock_set_opt_flag.rs @@ -25,6 +25,14 @@ pub fn sock_set_opt_flag( wasi_try_ok!(sock_set_opt_flag_internal(&mut ctx, sock, opt, flag)?); + #[cfg(feature = "journal")] + if ctx.data().enable_journal { + JournalEffector::save_sock_set_opt_flag(&mut ctx, sock, opt, flag).map_err(|err| { + tracing::error!("failed to save sock_set_opt_flag event - {}", err); + WasiError::Exit(ExitCode::Errno(Errno::Fault)) + })?; + } + Ok(Errno::Success) } diff --git a/lib/wasix/src/syscalls/wasix/sock_set_opt_size.rs b/lib/wasix/src/syscalls/wasix/sock_set_opt_size.rs index e5e1aa0b962..a9bdb842b49 100644 --- a/lib/wasix/src/syscalls/wasix/sock_set_opt_size.rs +++ b/lib/wasix/src/syscalls/wasix/sock_set_opt_size.rs @@ -19,6 +19,14 @@ pub fn sock_set_opt_size( ) -> Result { wasi_try_ok!(sock_set_opt_size_internal(&mut ctx, sock, opt, size)?); + #[cfg(feature = "journal")] + if ctx.data().enable_journal { + JournalEffector::save_sock_set_opt_size(&mut ctx, sock, opt, size).map_err(|err| { + tracing::error!("failed to save sock_set_opt_size event - {}", err); + WasiError::Exit(ExitCode::Errno(Errno::Fault)) + })?; + } + Ok(Errno::Success) } diff --git a/lib/wasix/src/syscalls/wasix/sock_set_opt_time.rs b/lib/wasix/src/syscalls/wasix/sock_set_opt_time.rs index 91d7ca93ce7..6196cc5e4a4 100644 --- a/lib/wasix/src/syscalls/wasix/sock_set_opt_time.rs +++ b/lib/wasix/src/syscalls/wasix/sock_set_opt_time.rs @@ -37,6 +37,14 @@ pub fn sock_set_opt_time( wasi_try_ok!(sock_set_opt_time_internal(&mut ctx, sock, ty, time)?); + #[cfg(feature = "journal")] + if ctx.data().enable_journal { + JournalEffector::save_sock_set_opt_time(&mut ctx, sock, ty, time).map_err(|err| { + tracing::error!("failed to save sock_set_opt_time event - {}", err); + WasiError::Exit(ExitCode::Errno(Errno::Fault)) + })?; + } + Ok(Errno::Success) } diff --git a/lib/wasix/src/syscalls/wasix/sock_shutdown.rs b/lib/wasix/src/syscalls/wasix/sock_shutdown.rs index 037b6b73024..8ef89634e2a 100644 --- a/lib/wasix/src/syscalls/wasix/sock_shutdown.rs +++ b/lib/wasix/src/syscalls/wasix/sock_shutdown.rs @@ -26,6 +26,14 @@ pub fn sock_shutdown( wasi_try_ok!(sock_shutdown_internal(&mut ctx, sock, shutdown)?); + #[cfg(feature = "journal")] + if ctx.data().enable_journal { + JournalEffector::save_sock_shutdown(&mut ctx, sock, shutdown).map_err(|err| { + tracing::error!("failed to save sock_shutdown event - {}", err); + WasiError::Exit(ExitCode::Errno(Errno::Fault)) + })?; + } + Ok(Errno::Success) } From 37d6c57c7cb356476ee8cc86bf3c0e3de4f830d2 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Tue, 21 Nov 2023 15:56:29 +1100 Subject: [PATCH 057/129] Fixed some linting issues --- .../src/journaling/effector/syscalls/sock_send.rs | 2 +- .../journaling/effector/syscalls/sock_send_to.rs | 2 +- lib/wasix/src/journaling/journal.rs | 4 ++-- lib/wasix/src/journaling/log_file.rs | 4 ++-- lib/wasix/src/syscalls/mod.rs | 14 ++++++-------- 5 files changed, 12 insertions(+), 14 deletions(-) diff --git a/lib/wasix/src/journaling/effector/syscalls/sock_send.rs b/lib/wasix/src/journaling/effector/syscalls/sock_send.rs index a4ffec3470a..ba19c75a4ac 100644 --- a/lib/wasix/src/journaling/effector/syscalls/sock_send.rs +++ b/lib/wasix/src/journaling/effector/syscalls/sock_send.rs @@ -38,7 +38,7 @@ impl JournalEffector { fd, data: Cow::Borrowed(buf.as_ref()), is_64bit: M::is_64bit(), - flags: si_flags as u16, + flags: si_flags, }) .await .map_err(map_snapshot_err)?; diff --git a/lib/wasix/src/journaling/effector/syscalls/sock_send_to.rs b/lib/wasix/src/journaling/effector/syscalls/sock_send_to.rs index 6418e3270d0..b5c84b69ebd 100644 --- a/lib/wasix/src/journaling/effector/syscalls/sock_send_to.rs +++ b/lib/wasix/src/journaling/effector/syscalls/sock_send_to.rs @@ -41,7 +41,7 @@ impl JournalEffector { data: Cow::Borrowed(buf.as_ref()), addr, is_64bit: M::is_64bit(), - flags: si_flags as u16, + flags: si_flags, }) .await .map_err(map_snapshot_err)?; diff --git a/lib/wasix/src/journaling/journal.rs b/lib/wasix/src/journaling/journal.rs index e70932c4087..f1699be9be3 100644 --- a/lib/wasix/src/journaling/journal.rs +++ b/lib/wasix/src/journaling/journal.rs @@ -202,8 +202,8 @@ pub enum JournalEntry<'a> { }, PortAddrClear, PortBridge { - network: String, - token: String, + network: Cow<'a, str>, + token: Cow<'a, str>, security: StreamSecurity, }, PortUnbridge, diff --git a/lib/wasix/src/journaling/log_file.rs b/lib/wasix/src/journaling/log_file.rs index 6e01bdf160f..ea7d9d1eac5 100644 --- a/lib/wasix/src/journaling/log_file.rs +++ b/lib/wasix/src/journaling/log_file.rs @@ -1082,7 +1082,7 @@ impl<'a> From> for LogFileJournalEntry { } => Self::SocketSendToV1 { fd, data: data.into(), - flags: flags as u16, + flags, addr, is_64bit, }, @@ -1094,7 +1094,7 @@ impl<'a> From> for LogFileJournalEntry { } => Self::SocketSendV1 { fd, data: data.into(), - flags: flags as u16, + flags, is_64bit, }, JournalEntry::SocketSetOptFlag { fd, opt, flag } => Self::SocketSetOptFlagV1 { diff --git a/lib/wasix/src/syscalls/mod.rs b/lib/wasix/src/syscalls/mod.rs index e9e573ccfad..71e96342f8c 100644 --- a/lib/wasix/src/syscalls/mod.rs +++ b/lib/wasix/src/syscalls/mod.rs @@ -1318,11 +1318,9 @@ pub fn restore_snapshot( is_64bit, } => { if is_64bit { - JournalEffector::apply_fd_write::(&mut ctx, fd, offset, data) - .await + JournalEffector::apply_fd_write::(&ctx, fd, offset, data).await } else { - JournalEffector::apply_fd_write::(&mut ctx, fd, offset, data) - .await + JournalEffector::apply_fd_write::(&ctx, fd, offset, data).await } .map_err(anyhow_err_to_runtime_err)?; } @@ -1693,10 +1691,10 @@ pub fn restore_snapshot( addr, is_64bit, } => if is_64bit { - JournalEffector::apply_sock_send_to::(&mut ctx, fd, data, flags, addr) + JournalEffector::apply_sock_send_to::(&ctx, fd, data, flags, addr) .await } else { - JournalEffector::apply_sock_send_to::(&mut ctx, fd, data, flags, addr) + JournalEffector::apply_sock_send_to::(&ctx, fd, data, flags, addr) .await } .map_err(anyhow_err_to_runtime_err)?, @@ -1706,9 +1704,9 @@ pub fn restore_snapshot( flags, is_64bit, } => if is_64bit { - JournalEffector::apply_sock_send::(&mut ctx, fd, data, flags).await + JournalEffector::apply_sock_send::(&ctx, fd, data, flags).await } else { - JournalEffector::apply_sock_send::(&mut ctx, fd, data, flags).await + JournalEffector::apply_sock_send::(&ctx, fd, data, flags).await } .map_err(anyhow_err_to_runtime_err)?, crate::journaling::JournalEntry::SocketSetOptFlag { fd, opt, flag } => { From 74c1d319af89bfeee8dbfea76edf079a3309c243 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Tue, 21 Nov 2023 16:03:03 +1100 Subject: [PATCH 058/129] Fixed a compile issue --- lib/wasix/src/journaling/effector/syscalls/port_bridge.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/wasix/src/journaling/effector/syscalls/port_bridge.rs b/lib/wasix/src/journaling/effector/syscalls/port_bridge.rs index 66029849932..ee69c557663 100644 --- a/lib/wasix/src/journaling/effector/syscalls/port_bridge.rs +++ b/lib/wasix/src/journaling/effector/syscalls/port_bridge.rs @@ -12,8 +12,8 @@ impl JournalEffector { Self::save_event( ctx, JournalEntry::PortBridge { - network, - token, + network: network.into(), + token: token.into(), security, }, ) From 68604f18b69df4a3ef081ddbec5c2b1e1079c8a2 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Tue, 21 Nov 2023 16:13:02 +1100 Subject: [PATCH 059/129] More linting fixes --- lib/wasix/src/syscalls/wasix/sock_send.rs | 17 +++++------------ lib/wasix/src/syscalls/wasix/sock_send_to.rs | 2 +- 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/lib/wasix/src/syscalls/wasix/sock_send.rs b/lib/wasix/src/syscalls/wasix/sock_send.rs index ec61a9f6225..c1ae7852a89 100644 --- a/lib/wasix/src/syscalls/wasix/sock_send.rs +++ b/lib/wasix/src/syscalls/wasix/sock_send.rs @@ -67,18 +67,11 @@ pub fn sock_send( #[cfg(feature = "journal")] if ctx.data().enable_journal { - JournalEffector::save_sock_send( - &mut ctx, - fd, - bytes_written, - si_data, - si_data_len, - si_flags, - ) - .map_err(|err| { - tracing::error!("failed to save sock_send event - {}", err); - WasiError::Exit(ExitCode::Errno(Errno::Fault)) - })?; + JournalEffector::save_sock_send(&ctx, fd, bytes_written, si_data, si_data_len, si_flags) + .map_err(|err| { + tracing::error!("failed to save sock_send event - {}", err); + WasiError::Exit(ExitCode::Errno(Errno::Fault)) + })?; } Span::current().record("nsent", bytes_written); diff --git a/lib/wasix/src/syscalls/wasix/sock_send_to.rs b/lib/wasix/src/syscalls/wasix/sock_send_to.rs index d2264907be0..48ec74675c9 100644 --- a/lib/wasix/src/syscalls/wasix/sock_send_to.rs +++ b/lib/wasix/src/syscalls/wasix/sock_send_to.rs @@ -52,7 +52,7 @@ pub fn sock_send_to( #[cfg(feature = "journal")] if ctx.data().enable_journal { JournalEffector::save_sock_send_to::( - &mut ctx, + &ctx, sock, bytes_written, si_data, From f8527224d6da15e32b789ed911511e842975fdb9 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Tue, 21 Nov 2023 17:02:30 +1100 Subject: [PATCH 060/129] Fixed a regression issue on the socket binding --- lib/wasix/src/syscalls/wasix/sock_bind.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/wasix/src/syscalls/wasix/sock_bind.rs b/lib/wasix/src/syscalls/wasix/sock_bind.rs index db951f9e608..ca030f41eda 100644 --- a/lib/wasix/src/syscalls/wasix/sock_bind.rs +++ b/lib/wasix/src/syscalls/wasix/sock_bind.rs @@ -22,6 +22,8 @@ pub fn sock_bind( let addr = SocketAddr::new(addr.0, addr.1); Span::current().record("addr", &format!("{:?}", addr)); + wasi_try_ok!(sock_bind_internal(&mut ctx, sock, addr)?); + #[cfg(feature = "journal")] if ctx.data().enable_journal { JournalEffector::save_sock_bind(&mut ctx, sock, addr).map_err(|err| { From 00a88732f45a481e6dc1b5ee2e39723693f975b4 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Tue, 21 Nov 2023 21:12:13 +1100 Subject: [PATCH 061/129] The journal entries can now support zero copy operations --- Cargo.lock | 4 + lib/virtual-net/Cargo.toml | 3 + lib/virtual-net/src/lib.rs | 6 + lib/wasix/Cargo.toml | 4 +- .../archived_journal.rs} | 480 ++++++++++-------- .../journaling/{ => concrete}/compactor.rs | 2 +- .../src/journaling/{ => concrete}/filter.rs | 2 +- lib/wasix/src/journaling/concrete/log_file.rs | 118 +++++ lib/wasix/src/journaling/concrete/mod.rs | 17 + lib/wasix/src/journaling/concrete/pipe.rs | 59 +++ .../journaling/{ => concrete}/unsupported.rs | 2 +- lib/wasix/src/journaling/journal.rs | 327 +++++++++++- lib/wasix/src/journaling/mod.rs | 12 +- 13 files changed, 813 insertions(+), 223 deletions(-) rename lib/wasix/src/journaling/{log_file.rs => concrete/archived_journal.rs} (82%) rename lib/wasix/src/journaling/{ => concrete}/compactor.rs (98%) rename lib/wasix/src/journaling/{ => concrete}/filter.rs (98%) create mode 100644 lib/wasix/src/journaling/concrete/log_file.rs create mode 100644 lib/wasix/src/journaling/concrete/mod.rs create mode 100644 lib/wasix/src/journaling/concrete/pipe.rs rename lib/wasix/src/journaling/{ => concrete}/unsupported.rs (87%) diff --git a/Cargo.lock b/Cargo.lock index 863abecadff..2f7e517c07d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5139,6 +5139,7 @@ dependencies = [ "async-trait", "base64", "bincode", + "bytecheck", "bytes", "derivative", "futures-util", @@ -5147,6 +5148,7 @@ dependencies = [ "libc", "mio", "pin-project-lite", + "rkyv", "serde", "socket2 0.4.10", "thiserror", @@ -6217,6 +6219,7 @@ dependencies = [ "anyhow", "async-trait", "bincode", + "bytecheck", "bytes", "cfg-if", "chrono", @@ -6240,6 +6243,7 @@ dependencies = [ "rand", "rayon", "reqwest", + "rkyv", "semver 1.0.20", "serde", "serde_cbor", diff --git a/lib/virtual-net/Cargo.toml b/lib/virtual-net/Cargo.toml index e0d51b1fe22..e4134f528f7 100644 --- a/lib/virtual-net/Cargo.toml +++ b/lib/virtual-net/Cargo.toml @@ -31,6 +31,8 @@ tokio-util = { version = "0.6", features = ["codec"], optional = true } hyper-tungstenite = { version = "0.11", optional = true } hyper = { version = "0.14", optional = true } tokio-tungstenite = { version = "0.20", optional = true } +rkyv = { version = "0.7.40", features = ["indexmap", "validation", "strict"], optional = true } +bytecheck = { version = "0.6.8", optional = true } [dev-dependencies] tokio = { version = "1", default_features = false, features = [ "macros" ] } @@ -45,6 +47,7 @@ messagepack = [ "tokio-serde/messagepack" ] cbor = [ "tokio-serde/cbor" ] hyper = [ "hyper-tungstenite", "dep:hyper" ] tokio-tungstenite = [ "dep:tokio-tungstenite" ] +rkyv = [ "dep:rkyv", "dep:bytecheck" ] [package.metadata.docs.rs] features = ["host-net", "remote"] diff --git a/lib/virtual-net/src/lib.rs b/lib/virtual-net/src/lib.rs index e8ba8d97e75..c59f613f083 100644 --- a/lib/virtual-net/src/lib.rs +++ b/lib/virtual-net/src/lib.rs @@ -16,6 +16,8 @@ mod tests; #[cfg(any(feature = "remote"))] pub use client::{RemoteNetworkingClient, RemoteNetworkingClientDriver}; use pin_project_lite::pin_project; +#[cfg(feature = "rkyv")] +use rkyv::{Archive, CheckBytes, Deserialize as RkyvDeserialize, Serialize as RkyvSerialize}; #[cfg(any(feature = "remote"))] pub use server::{RemoteNetworkingServer, RemoteNetworkingServerDriver}; use std::fmt; @@ -47,6 +49,8 @@ pub type Result = std::result::Result; /// Represents an IP address and its netmask #[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, Deserialize)] +#[cfg_attr(feature = "rkyv", derive(RkyvSerialize, RkyvDeserialize, Archive))] +#[cfg_attr(feature = "rkyv", archive_attr(derive(CheckBytes)))] pub struct IpCidr { pub ip: IpAddr, pub prefix: u8, @@ -54,6 +58,8 @@ pub struct IpCidr { /// Represents a routing entry in the routing table of the interface #[derive(Clone, Debug, Serialize, Deserialize)] +#[cfg_attr(feature = "rkyv", derive(RkyvSerialize, RkyvDeserialize, Archive))] +#[cfg_attr(feature = "rkyv", archive_attr(derive(CheckBytes)))] pub struct IpRoute { pub cidr: IpCidr, pub via_router: IpAddr, diff --git a/lib/wasix/Cargo.toml b/lib/wasix/Cargo.toml index 9cbc8388851..f2f564887d7 100644 --- a/lib/wasix/Cargo.toml +++ b/lib/wasix/Cargo.toml @@ -22,7 +22,7 @@ wasmer-types = { path = "../types", version = "=4.2.3", default-features = false wasmer = { path = "../api", version = "=4.2.3", default-features = false, features = ["wat", "js-serializable-module"] } virtual-mio = { path = "../virtual-io", version = "0.3.0", default-features = false } virtual-fs = { path = "../virtual-fs", version = "0.9.0", default-features = false, features = ["webc-fs"] } -virtual-net = { path = "../virtual-net", version = "0.6.1", default-features = false } +virtual-net = { path = "../virtual-net", version = "0.6.1", default-features = false, features = ["rkyv"] } wasmer-emscripten = { path = "../emscripten", version = "=4.2.3", optional = true } typetag = { version = "0.1", optional = true } serde = { version = "1.0", default-features = false, features = ["derive"] } @@ -75,6 +75,8 @@ tower-http = { version = "0.4.0", features = [ ], optional = true } tower = { version = "0.4.13", features = ["make", "util"], optional = true } url = "2.3.1" +rkyv = { version = "0.7.40", features = ["indexmap", "validation", "strict"] } +bytecheck = "0.6.8" petgraph = "0.6.3" rayon = { version = "1.7.0", optional = true } wasm-bindgen = { version = "0.2.87", optional = true } diff --git a/lib/wasix/src/journaling/log_file.rs b/lib/wasix/src/journaling/concrete/archived_journal.rs similarity index 82% rename from lib/wasix/src/journaling/log_file.rs rename to lib/wasix/src/journaling/concrete/archived_journal.rs index ea7d9d1eac5..ba5b33422af 100644 --- a/lib/wasix/src/journaling/log_file.rs +++ b/lib/wasix/src/journaling/concrete/archived_journal.rs @@ -1,19 +1,12 @@ -use serde::{Deserialize, Serialize}; -use std::{ - io::{self, ErrorKind, SeekFrom}, - mem::MaybeUninit, - net::Shutdown, - path::Path, - time::SystemTime, -}; -use tokio::runtime::Handle; +use rkyv::{Archive, CheckBytes, Deserialize as RkyvDeserialize, Serialize as RkyvSerialize}; +use serde; +use std::{net::Shutdown, time::SystemTime}; use virtual_net::{Duration, IpAddr, IpCidr, Ipv4Addr, Ipv6Addr, SocketAddr, StreamSecurity}; -use wasmer_wasix_types::wasi::{self, Addressfamily, EpollEventCtl, Sockoption, Socktype}; - -use futures::future::LocalBoxFuture; -use virtual_fs::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt}; +use wasmer_wasix_types::wasi::{ + self, Addressfamily, EpollEventCtl, EpollType, Sockoption, Socktype, +}; -use crate::{net::socket::TimeType, WasiThreadId}; +use crate::net::socket::TimeType; use super::*; @@ -24,8 +17,11 @@ use super::*; /// changes to the journal entry types without having to /// worry about backward and forward compatibility #[allow(clippy::large_enum_variant)] -#[derive(Debug, Clone, Serialize, Deserialize)] -pub(crate) enum LogFileJournalEntry { +#[derive( + Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, +)] +#[archive_attr(derive(CheckBytes))] +pub(crate) enum ArchivedJournalEntry { InitModuleV1 { wasm_hash: [u8; 32], }, @@ -33,14 +29,14 @@ pub(crate) enum LogFileJournalEntry { exit_code: Option, }, SetThreadV1 { - id: WasiThreadId, + id: u32, call_stack: Vec, memory_stack: Vec, store_data: Vec, is_64bit: bool, }, CloseThreadV1 { - id: WasiThreadId, + id: u32, exit_code: Option, }, FileDescriptorSeekV1 { @@ -162,7 +158,7 @@ pub(crate) enum LogFileJournalEntry { epfd: u32, op: JournalEpollCtlV1, fd: u32, - event: Option, + event: Option, }, TtySetV1 { cols: u32, @@ -296,12 +292,15 @@ pub(crate) enum LogFileJournalEntry { how: JournalSocketShutdownV1, }, SnapshotV1 { - when: SystemTime, + since_epoch: Duration, trigger: JournalSnapshotTriggerV1, }, } -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive( + Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, +)] +#[archive_attr(derive(CheckBytes))] pub(crate) enum JournalSnapshot0ClockidV1 { Realtime, Monotonic, @@ -334,7 +333,10 @@ impl From for wasi::Snapshot0Clockid { } } -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive( + Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, +)] +#[archive_attr(derive(CheckBytes))] pub(crate) enum JournalWhenceV1 { Set, Cur, @@ -364,7 +366,10 @@ impl From for wasi::Whence { } } -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive( + Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, +)] +#[archive_attr(derive(CheckBytes))] pub(crate) enum JournalAdviceV1 { Normal, Sequential, @@ -403,7 +408,10 @@ impl From for wasi::Advice { } } -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive( + Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, +)] +#[archive_attr(derive(CheckBytes))] pub(crate) enum JournalExitCodeV1 { Errno(u16), Other(i32), @@ -429,7 +437,22 @@ impl From for wasi::ExitCode { } } -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[derive( + Debug, + Clone, + Copy, + PartialEq, + Eq, + PartialOrd, + Ord, + Hash, + serde::Serialize, + serde::Deserialize, + RkyvSerialize, + RkyvDeserialize, + Archive, +)] +#[archive_attr(derive(CheckBytes))] pub(crate) enum JournalSnapshotTriggerV1 { Idle, Listen, @@ -477,7 +500,22 @@ impl From for SnapshotTrigger { } } -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[derive( + Debug, + Clone, + Copy, + PartialEq, + Eq, + PartialOrd, + Ord, + Hash, + serde::Serialize, + serde::Deserialize, + RkyvSerialize, + RkyvDeserialize, + Archive, +)] +#[archive_attr(derive(CheckBytes))] pub(crate) enum JournalEpollCtlV1 { Add, Mod, @@ -507,7 +545,58 @@ impl From for wasi::EpollCtl { } } -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[derive( + Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, +)] +#[archive_attr(derive(CheckBytes))] +pub struct JournalEpollEventCtlV1 { + pub events: u32, + pub ptr: u64, + pub fd: Fd, + pub data1: u32, + pub data2: u64, +} + +impl From for JournalEpollEventCtlV1 { + fn from(val: EpollEventCtl) -> Self { + JournalEpollEventCtlV1 { + events: val.events.bits(), + ptr: val.ptr, + fd: val.fd, + data1: val.data1, + data2: val.data2, + } + } +} + +impl From for EpollEventCtl { + fn from(val: JournalEpollEventCtlV1) -> Self { + Self { + events: EpollType::from_bits_truncate(val.events), + ptr: val.ptr, + fd: val.fd, + data1: val.data1, + data2: val.data2, + } + } +} + +#[derive( + Debug, + Clone, + Copy, + PartialEq, + Eq, + PartialOrd, + Ord, + Hash, + serde::Serialize, + serde::Deserialize, + RkyvSerialize, + RkyvDeserialize, + Archive, +)] +#[archive_attr(derive(CheckBytes))] pub enum JournalStreamSecurityV1 { Unencrypted, AnyEncryption, @@ -539,7 +628,22 @@ impl From for StreamSecurity { } } -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[derive( + Debug, + Clone, + Copy, + PartialEq, + Eq, + PartialOrd, + Ord, + Hash, + serde::Serialize, + serde::Deserialize, + RkyvSerialize, + RkyvDeserialize, + Archive, +)] +#[archive_attr(derive(CheckBytes))] pub enum JournalAddressfamilyV1 { Unspec, Inet4, @@ -569,7 +673,22 @@ impl From for Addressfamily { } } -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[derive( + Debug, + Clone, + Copy, + PartialEq, + Eq, + PartialOrd, + Ord, + Hash, + serde::Serialize, + serde::Deserialize, + RkyvSerialize, + RkyvDeserialize, + Archive, +)] +#[archive_attr(derive(CheckBytes))] pub enum JournalSocktypeV1 { Unknown, Stream, @@ -602,7 +721,22 @@ impl From for Socktype { } } -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[derive( + Debug, + Clone, + Copy, + PartialEq, + Eq, + PartialOrd, + Ord, + Hash, + serde::Serialize, + serde::Deserialize, + RkyvSerialize, + RkyvDeserialize, + Archive, +)] +#[archive_attr(derive(CheckBytes))] pub enum JournalSockoptionV1 { Noop, ReusePort, @@ -701,7 +835,22 @@ impl From for Sockoption { } } -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[derive( + Debug, + Clone, + Copy, + PartialEq, + Eq, + PartialOrd, + Ord, + Hash, + serde::Serialize, + serde::Deserialize, + RkyvSerialize, + RkyvDeserialize, + Archive, +)] +#[archive_attr(derive(CheckBytes))] pub enum JournalTimeTypeV1 { ReadTimeout, WriteTimeout, @@ -737,7 +886,22 @@ impl From for TimeType { } } -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[derive( + Debug, + Clone, + Copy, + PartialEq, + Eq, + PartialOrd, + Ord, + Hash, + serde::Serialize, + serde::Deserialize, + RkyvSerialize, + RkyvDeserialize, + Archive, +)] +#[archive_attr(derive(CheckBytes))] pub enum JournalSocketShutdownV1 { Read, Write, @@ -764,7 +928,7 @@ impl From for Shutdown { } } -impl<'a> From> for LogFileJournalEntry { +impl<'a> From> for ArchivedJournalEntry { fn from(value: JournalEntry<'a>) -> Self { match value { JournalEntry::InitModule { wasm_hash } => Self::InitModuleV1 { wasm_hash }, @@ -783,14 +947,14 @@ impl<'a> From> for LogFileJournalEntry { store_data, is_64bit, } => Self::SetThreadV1 { - id, + id: id.into(), call_stack: call_stack.into_owned(), memory_stack: memory_stack.into_owned(), store_data: store_data.into_owned(), is_64bit, }, JournalEntry::CloseThread { id, exit_code } => Self::CloseThreadV1 { - id, + id: id.into(), exit_code: exit_code.map(|code| code.into()), }, JournalEntry::FileDescriptorWrite { @@ -849,7 +1013,9 @@ impl<'a> From> for LogFileJournalEntry { new_path: new_path.into_owned(), }, JournalEntry::Snapshot { when, trigger } => Self::SnapshotV1 { - when, + since_epoch: when + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap_or(Duration::ZERO), trigger: trigger.into(), }, JournalEntry::SetClockTime { clock_id, time } => Self::SetClockTimeV1 { @@ -961,7 +1127,7 @@ impl<'a> From> for LogFileJournalEntry { epfd, op: op.into(), fd, - event, + event: event.map(|e| e.into()), }, JournalEntry::TtySet { tty, line_feeds } => Self::TtySetV1 { cols: tty.cols, @@ -1129,37 +1295,37 @@ impl<'a> From> for LogFileJournalEntry { } } -impl<'a> From for JournalEntry<'a> { - fn from(value: LogFileJournalEntry) -> Self { +impl<'a> From for JournalEntry<'a> { + fn from(value: ArchivedJournalEntry) -> Self { match value { - LogFileJournalEntry::InitModuleV1 { wasm_hash } => Self::InitModule { wasm_hash }, - LogFileJournalEntry::UpdateMemoryRegionV1 { start, end, data } => { + ArchivedJournalEntry::InitModuleV1 { wasm_hash } => Self::InitModule { wasm_hash }, + ArchivedJournalEntry::UpdateMemoryRegionV1 { start, end, data } => { Self::UpdateMemoryRegion { region: start..end, data: data.into(), } } - LogFileJournalEntry::ProcessExitV1 { exit_code } => Self::ProcessExit { + ArchivedJournalEntry::ProcessExitV1 { exit_code } => Self::ProcessExit { exit_code: exit_code.map(|code| code.into()), }, - LogFileJournalEntry::SetThreadV1 { + ArchivedJournalEntry::SetThreadV1 { id, call_stack, memory_stack, store_data, is_64bit, } => Self::SetThread { - id, + id: id.into(), call_stack: call_stack.into(), memory_stack: memory_stack.into(), store_data: store_data.into(), is_64bit, }, - LogFileJournalEntry::CloseThreadV1 { id, exit_code } => Self::CloseThread { - id, + ArchivedJournalEntry::CloseThreadV1 { id, exit_code } => Self::CloseThread { + id: id.into(), exit_code: exit_code.map(|code| code.into()), }, - LogFileJournalEntry::FileDescriptorWriteV1 { + ArchivedJournalEntry::FileDescriptorWriteV1 { data, fd, offset, @@ -1170,14 +1336,14 @@ impl<'a> From for JournalEntry<'a> { offset, is_64bit, }, - LogFileJournalEntry::FileDescriptorSeekV1 { fd, offset, whence } => { + ArchivedJournalEntry::FileDescriptorSeekV1 { fd, offset, whence } => { Self::FileDescriptorSeek { fd, offset, whence: whence.into(), } } - LogFileJournalEntry::OpenFileDescriptorV1 { + ArchivedJournalEntry::OpenFileDescriptorV1 { fd, dirfd, dirflags, @@ -1196,16 +1362,16 @@ impl<'a> From for JournalEntry<'a> { fs_rights_inheriting: wasi::Rights::from_bits_truncate(fs_rights_inheriting), fs_flags: wasi::Fdflags::from_bits_truncate(fs_flags), }, - LogFileJournalEntry::CloseFileDescriptorV1 { fd } => Self::CloseFileDescriptor { fd }, - LogFileJournalEntry::RemoveDirectoryV1 { fd, path } => Self::RemoveDirectory { + ArchivedJournalEntry::CloseFileDescriptorV1 { fd } => Self::CloseFileDescriptor { fd }, + ArchivedJournalEntry::RemoveDirectoryV1 { fd, path } => Self::RemoveDirectory { fd, path: path.into(), }, - LogFileJournalEntry::UnlinkFileV1 { fd, path } => Self::UnlinkFile { + ArchivedJournalEntry::UnlinkFileV1 { fd, path } => Self::UnlinkFile { fd, path: path.into(), }, - LogFileJournalEntry::PathRenameV1 { + ArchivedJournalEntry::PathRenameV1 { old_fd, old_path, new_fd, @@ -1216,29 +1382,34 @@ impl<'a> From for JournalEntry<'a> { new_fd, new_path: new_path.into(), }, - LogFileJournalEntry::SnapshotV1 { when, trigger } => Self::Snapshot { - when, + ArchivedJournalEntry::SnapshotV1 { + since_epoch, + trigger, + } => Self::Snapshot { + when: SystemTime::UNIX_EPOCH + .checked_add(since_epoch) + .unwrap_or(SystemTime::UNIX_EPOCH), trigger: trigger.into(), }, - LogFileJournalEntry::SetClockTimeV1 { clock_id, time } => Self::SetClockTime { + ArchivedJournalEntry::SetClockTimeV1 { clock_id, time } => Self::SetClockTime { clock_id: clock_id.into(), time, }, - LogFileJournalEntry::RenumberFileDescriptorV1 { old_fd, new_fd } => { + ArchivedJournalEntry::RenumberFileDescriptorV1 { old_fd, new_fd } => { Self::RenumberFileDescriptor { old_fd, new_fd } } - LogFileJournalEntry::DuplicateFileDescriptorV1 { + ArchivedJournalEntry::DuplicateFileDescriptorV1 { original_fd: old_fd, copied_fd: new_fd, } => Self::DuplicateFileDescriptor { original_fd: old_fd, copied_fd: new_fd, }, - LogFileJournalEntry::CreateDirectoryV1 { fd, path } => Self::CreateDirectory { + ArchivedJournalEntry::CreateDirectoryV1 { fd, path } => Self::CreateDirectory { fd, path: path.into(), }, - LogFileJournalEntry::PathSetTimesV1 { + ArchivedJournalEntry::PathSetTimesV1 { fd, path, flags, @@ -1253,7 +1424,7 @@ impl<'a> From for JournalEntry<'a> { st_mtim, fst_flags: wasi::Fstflags::from_bits_truncate(fst_flags), }, - LogFileJournalEntry::FileDescriptorSetTimesV1 { + ArchivedJournalEntry::FileDescriptorSetTimesV1 { fd, st_atim, st_mtim, @@ -1264,16 +1435,16 @@ impl<'a> From for JournalEntry<'a> { st_mtim, fst_flags: wasi::Fstflags::from_bits_truncate(fst_flags), }, - LogFileJournalEntry::FileDescriptorSetSizeV1 { fd, st_size } => { + ArchivedJournalEntry::FileDescriptorSetSizeV1 { fd, st_size } => { Self::FileDescriptorSetSize { fd, st_size } } - LogFileJournalEntry::FileDescriptorSetFlagsV1 { fd, flags } => { + ArchivedJournalEntry::FileDescriptorSetFlagsV1 { fd, flags } => { Self::FileDescriptorSetFlags { fd, flags: Fdflags::from_bits_truncate(flags), } } - LogFileJournalEntry::FileDescriptorSetRightsV1 { + ArchivedJournalEntry::FileDescriptorSetRightsV1 { fd, fs_rights_base, fs_rights_inheriting, @@ -1282,7 +1453,7 @@ impl<'a> From for JournalEntry<'a> { fs_rights_base: Rights::from_bits_truncate(fs_rights_base), fs_rights_inheriting: Rights::from_bits_truncate(fs_rights_inheriting), }, - LogFileJournalEntry::FileDescriptorAdviseV1 { + ArchivedJournalEntry::FileDescriptorAdviseV1 { fd, offset, len, @@ -1293,10 +1464,10 @@ impl<'a> From for JournalEntry<'a> { len, advice: advice.into(), }, - LogFileJournalEntry::FileDescriptorAllocateV1 { fd, offset, len } => { + ArchivedJournalEntry::FileDescriptorAllocateV1 { fd, offset, len } => { Self::FileDescriptorAllocate { fd, offset, len } } - LogFileJournalEntry::CreateHardLinkV1 { + ArchivedJournalEntry::CreateHardLinkV1 { old_fd, old_path, old_flags, @@ -1309,7 +1480,7 @@ impl<'a> From for JournalEntry<'a> { new_fd, new_path: new_path.into(), }, - LogFileJournalEntry::CreateSymbolicLinkV1 { + ArchivedJournalEntry::CreateSymbolicLinkV1 { old_path, fd, new_path, @@ -1318,11 +1489,11 @@ impl<'a> From for JournalEntry<'a> { fd, new_path: new_path.into(), }, - LogFileJournalEntry::ChangeDirectoryV1 { path } => { + ArchivedJournalEntry::ChangeDirectoryV1 { path } => { Self::ChangeDirectory { path: path.into() } } - LogFileJournalEntry::EpollCreateV1 { fd } => Self::EpollCreate { fd }, - LogFileJournalEntry::EpollCtlV1 { + ArchivedJournalEntry::EpollCreateV1 { fd } => Self::EpollCreate { fd }, + ArchivedJournalEntry::EpollCtlV1 { epfd, op, fd, @@ -1331,9 +1502,9 @@ impl<'a> From for JournalEntry<'a> { epfd, op: op.into(), fd, - event, + event: event.map(|e| e.into()), }, - LogFileJournalEntry::TtySetV1 { + ArchivedJournalEntry::TtySetV1 { cols, rows, width, @@ -1358,11 +1529,11 @@ impl<'a> From for JournalEntry<'a> { }, line_feeds, }, - LogFileJournalEntry::CreatePipeV1 { fd1, fd2 } => Self::CreatePipe { fd1, fd2 }, - LogFileJournalEntry::PortAddAddrV1 { cidr } => Self::PortAddAddr { cidr }, - LogFileJournalEntry::PortDelAddrV1 { addr } => Self::PortDelAddr { addr }, - LogFileJournalEntry::PortAddrClearV1 => Self::PortAddrClear, - LogFileJournalEntry::PortBridgeV1 { + ArchivedJournalEntry::CreatePipeV1 { fd1, fd2 } => Self::CreatePipe { fd1, fd2 }, + ArchivedJournalEntry::PortAddAddrV1 { cidr } => Self::PortAddAddr { cidr }, + ArchivedJournalEntry::PortDelAddrV1 { addr } => Self::PortDelAddr { addr }, + ArchivedJournalEntry::PortAddrClearV1 => Self::PortAddrClear, + ArchivedJournalEntry::PortBridgeV1 { network, token, security, @@ -1371,10 +1542,10 @@ impl<'a> From for JournalEntry<'a> { token: token.into(), security: security.into(), }, - LogFileJournalEntry::PortUnbridgeV1 => Self::PortUnbridge, - LogFileJournalEntry::PortDhcpAcquireV1 => Self::PortDhcpAcquire, - LogFileJournalEntry::PortGatewaySetV1 { ip } => Self::PortGatewaySet { ip }, - LogFileJournalEntry::PortRouteAddV1 { + ArchivedJournalEntry::PortUnbridgeV1 => Self::PortUnbridge, + ArchivedJournalEntry::PortDhcpAcquireV1 => Self::PortDhcpAcquire, + ArchivedJournalEntry::PortGatewaySetV1 { ip } => Self::PortGatewaySet { ip }, + ArchivedJournalEntry::PortRouteAddV1 { cidr, via_router, preferred_until, @@ -1385,22 +1556,22 @@ impl<'a> From for JournalEntry<'a> { preferred_until, expires_at, }, - LogFileJournalEntry::PortRouteClearV1 => Self::PortRouteClear, - LogFileJournalEntry::PortRouteDelV1 { ip } => Self::PortRouteDel { ip }, - LogFileJournalEntry::SocketOpenV1 { af, ty, pt, fd } => Self::SocketOpen { + ArchivedJournalEntry::PortRouteClearV1 => Self::PortRouteClear, + ArchivedJournalEntry::PortRouteDelV1 { ip } => Self::PortRouteDel { ip }, + ArchivedJournalEntry::SocketOpenV1 { af, ty, pt, fd } => Self::SocketOpen { af: af.into(), ty: ty.into(), pt: pt.try_into().unwrap_or(wasi::SockProto::Max), fd, }, - LogFileJournalEntry::SocketListenV1 { fd, backlog } => { + ArchivedJournalEntry::SocketListenV1 { fd, backlog } => { Self::SocketListen { fd, backlog } } - LogFileJournalEntry::SocketBindV1 { fd, addr } => Self::SocketBind { fd, addr }, - LogFileJournalEntry::SocketConnectedV1 { fd, addr } => { + ArchivedJournalEntry::SocketBindV1 { fd, addr } => Self::SocketBind { fd, addr }, + ArchivedJournalEntry::SocketConnectedV1 { fd, addr } => { Self::SocketConnected { fd, addr } } - LogFileJournalEntry::SocketAcceptedV1 { + ArchivedJournalEntry::SocketAcceptedV1 { listen_fd, fd, peer_addr, @@ -1413,7 +1584,7 @@ impl<'a> From for JournalEntry<'a> { fd_flags: Fdflags::from_bits_truncate(fd_flags), nonblocking, }, - LogFileJournalEntry::SocketJoinIpv4MulticastV1 { + ArchivedJournalEntry::SocketJoinIpv4MulticastV1 { fd, multiaddr, iface, @@ -1422,7 +1593,7 @@ impl<'a> From for JournalEntry<'a> { multiaddr, iface, }, - LogFileJournalEntry::SocketJoinIpv6MulticastV1 { + ArchivedJournalEntry::SocketJoinIpv6MulticastV1 { fd, multiaddr, iface, @@ -1431,7 +1602,7 @@ impl<'a> From for JournalEntry<'a> { multiaddr, iface, }, - LogFileJournalEntry::SocketLeaveIpv4MulticastV1 { + ArchivedJournalEntry::SocketLeaveIpv4MulticastV1 { fd, multiaddr, iface, @@ -1440,7 +1611,7 @@ impl<'a> From for JournalEntry<'a> { multiaddr, iface, }, - LogFileJournalEntry::SocketLeaveIpv6MulticastV1 { + ArchivedJournalEntry::SocketLeaveIpv6MulticastV1 { fd, multiaddr, iface, @@ -1449,7 +1620,7 @@ impl<'a> From for JournalEntry<'a> { multiaddr, iface, }, - LogFileJournalEntry::SocketSendFileV1 { + ArchivedJournalEntry::SocketSendFileV1 { socket_fd, file_fd, offset, @@ -1460,7 +1631,7 @@ impl<'a> From for JournalEntry<'a> { offset, count, }, - LogFileJournalEntry::SocketSendToV1 { + ArchivedJournalEntry::SocketSendToV1 { fd, data, flags, @@ -1473,7 +1644,7 @@ impl<'a> From for JournalEntry<'a> { addr, is_64bit, }, - LogFileJournalEntry::SocketSendV1 { + ArchivedJournalEntry::SocketSendV1 { fd, data, flags, @@ -1484,26 +1655,26 @@ impl<'a> From for JournalEntry<'a> { flags, is_64bit, }, - LogFileJournalEntry::SocketSetOptFlagV1 { fd, opt, flag } => Self::SocketSetOptFlag { + ArchivedJournalEntry::SocketSetOptFlagV1 { fd, opt, flag } => Self::SocketSetOptFlag { fd, opt: opt.into(), flag, }, - LogFileJournalEntry::SocketSetOptSizeV1 { fd, opt, size } => Self::SocketSetOptSize { + ArchivedJournalEntry::SocketSetOptSizeV1 { fd, opt, size } => Self::SocketSetOptSize { fd, opt: opt.into(), size, }, - LogFileJournalEntry::SocketSetOptTimeV1 { fd, ty, time } => Self::SocketSetOptTime { + ArchivedJournalEntry::SocketSetOptTimeV1 { fd, ty, time } => Self::SocketSetOptTime { fd, ty: ty.into(), time, }, - LogFileJournalEntry::SocketShutdownV1 { fd, how } => Self::SocketShutdown { + ArchivedJournalEntry::SocketShutdownV1 { fd, how } => Self::SocketShutdown { fd, how: how.into(), }, - LogFileJournalEntry::CreateEventV1 { + ArchivedJournalEntry::CreateEventV1 { initial_val, flags, fd, @@ -1515,110 +1686,3 @@ impl<'a> From for JournalEntry<'a> { } } } - -struct State { - file: tokio::fs::File, - at_end: bool, -} - -/// The LogFile snapshot capturer will write its snapshots to a linear journal -/// and read them when restoring. It uses the `bincode` serializer which -/// means that forwards and backwards compatibility must be dealt with -/// carefully. -/// -/// When opening an existing journal file that was previously saved -/// then new entries will be added to the end regardless of if -/// its been read. -/// -/// The logfile snapshot capturer uses a 64bit number as a entry encoding -/// delimiter. -pub struct LogFileJournal { - state: tokio::sync::Mutex, - handle: Handle, -} - -impl LogFileJournal { - pub async fn new(path: impl AsRef) -> io::Result { - let state = State { - file: tokio::fs::File::options() - .read(true) - .write(true) - .create(true) - .open(path) - .await?, - at_end: false, - }; - Ok(Self { - state: tokio::sync::Mutex::new(state), - handle: Handle::current(), - }) - } - - pub fn new_std(path: impl AsRef) -> io::Result { - let file = std::fs::File::options() - .read(true) - .write(true) - .create(true) - .open(path)?; - let state = State { - file: tokio::fs::File::from_std(file), - at_end: false, - }; - Ok(Self { - state: tokio::sync::Mutex::new(state), - handle: Handle::current(), - }) - } -} - -#[async_trait::async_trait] -impl Journal for LogFileJournal { - fn write<'a>(&'a self, entry: JournalEntry<'a>) -> LocalBoxFuture<'a, anyhow::Result<()>> { - tracing::debug!("journal event: {:?}", entry); - Box::pin(async { - let entry: LogFileJournalEntry = entry.into(); - - let _guard = Handle::try_current().map_err(|_| self.handle.enter()); - let mut state = self.state.lock().await; - if !state.at_end { - state.file.seek(SeekFrom::End(0)).await?; - state.at_end = true; - } - - let data = bincode::serialize(&entry)?; - let data_len = data.len() as u64; - let data_len = data_len.to_ne_bytes(); - - state.file.write_all(&data_len).await?; - state.file.write_all(&data).await?; - Ok(()) - }) - } - - /// UNSAFE: This method uses unsafe operations to remove the need to zero - /// the buffer before its read the log entries into it - fn read(&self) -> LocalBoxFuture<'_, anyhow::Result>>> { - Box::pin(async { - let mut state = self.state.lock().await; - - let mut data_len = [0u8; 8]; - match state.file.read_exact(&mut data_len).await { - Err(err) if err.kind() == ErrorKind::UnexpectedEof => return Ok(None), - Err(err) => return Err(err.into()), - Ok(_) => {} - } - - let data_len = u64::from_ne_bytes(data_len); - let mut data = Vec::with_capacity(data_len as usize); - let data_unsafe: &mut [MaybeUninit] = data.spare_capacity_mut(); - let data_unsafe: &mut [u8] = unsafe { std::mem::transmute(data_unsafe) }; - state.file.read_exact(data_unsafe).await?; - unsafe { - data.set_len(data_len as usize); - } - - let entry: LogFileJournalEntry = bincode::deserialize(&data)?; - Ok(Some(entry.into())) - }) - } -} diff --git a/lib/wasix/src/journaling/compactor.rs b/lib/wasix/src/journaling/concrete/compactor.rs similarity index 98% rename from lib/wasix/src/journaling/compactor.rs rename to lib/wasix/src/journaling/concrete/compactor.rs index 3c4d75c7681..4789f39f544 100644 --- a/lib/wasix/src/journaling/compactor.rs +++ b/lib/wasix/src/journaling/concrete/compactor.rs @@ -147,7 +147,7 @@ impl Journal for CompactingJournal { }) } - fn read(&self) -> LocalBoxFuture<'_, anyhow::Result>>> { + fn read<'a>(&'a self) -> LocalBoxFuture<'_, anyhow::Result>>> { Box::pin(async { Ok(match self.inner.read().await? { Some(JournalEntry::UpdateMemoryRegion { region, data }) => { diff --git a/lib/wasix/src/journaling/filter.rs b/lib/wasix/src/journaling/concrete/filter.rs similarity index 98% rename from lib/wasix/src/journaling/filter.rs rename to lib/wasix/src/journaling/concrete/filter.rs index a91df18500e..226d9d30249 100644 --- a/lib/wasix/src/journaling/filter.rs +++ b/lib/wasix/src/journaling/concrete/filter.rs @@ -155,7 +155,7 @@ impl Journal for FilteredJournal { }) } - fn read(&self) -> LocalBoxFuture<'_, anyhow::Result>>> { + fn read<'a>(&'a self) -> LocalBoxFuture<'_, anyhow::Result>>> { Box::pin(async { self.inner.read().await }) } } diff --git a/lib/wasix/src/journaling/concrete/log_file.rs b/lib/wasix/src/journaling/concrete/log_file.rs new file mode 100644 index 00000000000..67b8c52a1c8 --- /dev/null +++ b/lib/wasix/src/journaling/concrete/log_file.rs @@ -0,0 +1,118 @@ +use std::{ + io::{self, ErrorKind, SeekFrom}, + mem::MaybeUninit, + path::Path, +}; +use tokio::runtime::Handle; + +use futures::future::LocalBoxFuture; +use virtual_fs::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt}; + +use super::*; + +struct State { + file: tokio::fs::File, + at_end: bool, +} + +/// The LogFile snapshot capturer will write its snapshots to a linear journal +/// and read them when restoring. It uses the `bincode` serializer which +/// means that forwards and backwards compatibility must be dealt with +/// carefully. +/// +/// When opening an existing journal file that was previously saved +/// then new entries will be added to the end regardless of if +/// its been read. +/// +/// The logfile snapshot capturer uses a 64bit number as a entry encoding +/// delimiter. +pub struct LogFileJournal { + state: tokio::sync::Mutex, + handle: Handle, +} + +impl LogFileJournal { + pub async fn new(path: impl AsRef) -> io::Result { + let state = State { + file: tokio::fs::File::options() + .read(true) + .write(true) + .create(true) + .open(path) + .await?, + at_end: false, + }; + Ok(Self { + state: tokio::sync::Mutex::new(state), + handle: Handle::current(), + }) + } + + pub fn new_std(path: impl AsRef) -> io::Result { + let file = std::fs::File::options() + .read(true) + .write(true) + .create(true) + .open(path)?; + let state = State { + file: tokio::fs::File::from_std(file), + at_end: false, + }; + Ok(Self { + state: tokio::sync::Mutex::new(state), + handle: Handle::current(), + }) + } +} + +#[async_trait::async_trait] +impl Journal for LogFileJournal { + fn write<'a>(&'a self, entry: JournalEntry<'a>) -> LocalBoxFuture<'a, anyhow::Result<()>> { + tracing::debug!("journal event: {:?}", entry); + Box::pin(async { + let entry: ArchivedJournalEntry = entry.into(); + + let _guard = Handle::try_current().map_err(|_| self.handle.enter()); + let mut state = self.state.lock().await; + if !state.at_end { + state.file.seek(SeekFrom::End(0)).await?; + state.at_end = true; + } + + let data = bincode::serialize(&entry)?; + let data_len = data.len() as u64; + let data_len = data_len.to_ne_bytes(); + + state.file.write_all(&data_len).await?; + state.file.write_all(&data).await?; + Ok(()) + }) + } + + /// UNSAFE: This method uses unsafe operations to remove the need to zero + /// the buffer before its read the log entries into it + fn read<'a>(&'a self) -> LocalBoxFuture<'_, anyhow::Result>>> { + Box::pin(async { + let mut state = self.state.lock().await; + + let mut data_len = [0u8; 8]; + match state.file.read_exact(&mut data_len).await { + Err(err) if err.kind() == ErrorKind::UnexpectedEof => return Ok(None), + Err(err) => return Err(err.into()), + Ok(_) => {} + } + + let data_len = u64::from_ne_bytes(data_len); + let mut data = Vec::with_capacity(data_len as usize); + let data_unsafe: &mut [MaybeUninit] = data.spare_capacity_mut(); + let data_unsafe: &mut [u8] = unsafe { std::mem::transmute(data_unsafe) }; + state.file.read_exact(data_unsafe).await?; + unsafe { + data.set_len(data_len as usize); + } + + let entry: ArchivedJournalEntry = bincode::deserialize(&data)?; + Ok(Some(entry.into())) + }) + } +} diff --git a/lib/wasix/src/journaling/concrete/mod.rs b/lib/wasix/src/journaling/concrete/mod.rs new file mode 100644 index 00000000000..9d90f736012 --- /dev/null +++ b/lib/wasix/src/journaling/concrete/mod.rs @@ -0,0 +1,17 @@ +mod archived_journal; +mod compactor; +mod filter; +#[cfg(feature = "journal")] +mod log_file; +mod pipe; +mod unsupported; + +pub(super) use super::*; + +pub use archived_journal::*; +pub use compactor::*; +pub use filter::*; +#[cfg(feature = "journal")] +pub use log_file::*; +pub use pipe::*; +pub use unsupported::*; diff --git a/lib/wasix/src/journaling/concrete/pipe.rs b/lib/wasix/src/journaling/concrete/pipe.rs new file mode 100644 index 00000000000..b19f6e742cd --- /dev/null +++ b/lib/wasix/src/journaling/concrete/pipe.rs @@ -0,0 +1,59 @@ +use std::sync::Mutex; + +use futures::future::LocalBoxFuture; +use tokio::sync::mpsc::{self, error::TryRecvError}; + +use super::*; + +// The pipe journal will feed journal entries between two bi-directional ends +// of a pipe. +#[derive(Debug)] +pub struct PipeJournal { + tx: mpsc::Sender>, + rx: Mutex>>, +} + +impl PipeJournal { + pub fn channel(buffer: usize) -> (Self, Self) { + let (tx1, rx1) = mpsc::channel(buffer); + let (tx2, rx2) = mpsc::channel(buffer); + + let end1 = PipeJournal { + tx: tx1, + rx: Mutex::new(rx2), + }; + + let end2 = PipeJournal { + tx: tx2, + rx: Mutex::new(rx1), + }; + + (end1, end2) + } +} + +impl Journal for PipeJournal { + fn write<'a>(&'a self, entry: JournalEntry<'a>) -> LocalBoxFuture<'a, anyhow::Result<()>> { + let entry = entry.into_owned(); + Box::pin(async { + self.tx.send(entry).await.map_err(|err| { + anyhow::format_err!("failed to send journal event through the pipe - {}", err) + }) + }) + } + + fn read<'a>(&'a self) -> LocalBoxFuture<'_, anyhow::Result>>> { + Box::pin(async { + let mut rx = self.rx.lock().unwrap(); + match rx.try_recv() { + Ok(e) => Ok(Some(e.into())), + Err(TryRecvError::Empty) => Ok(None), + Err(TryRecvError::Disconnected) => { + return Err(anyhow::format_err!( + "failed to receive journal event from the pipe as its disconnected" + )) + } + } + }) + } +} diff --git a/lib/wasix/src/journaling/unsupported.rs b/lib/wasix/src/journaling/concrete/unsupported.rs similarity index 87% rename from lib/wasix/src/journaling/unsupported.rs rename to lib/wasix/src/journaling/concrete/unsupported.rs index 6256c18dd59..df58d934828 100644 --- a/lib/wasix/src/journaling/unsupported.rs +++ b/lib/wasix/src/journaling/concrete/unsupported.rs @@ -15,7 +15,7 @@ impl Journal for UnsupportedJournal { Box::pin(async { Err(anyhow::format_err!("unsupported")) }) } - fn read(&self) -> LocalBoxFuture<'_, anyhow::Result>>> { + fn read<'a>(&'a self) -> LocalBoxFuture<'_, anyhow::Result>>> { Box::pin(async { Ok(None) }) } } diff --git a/lib/wasix/src/journaling/journal.rs b/lib/wasix/src/journaling/journal.rs index f1699be9be3..8eefdf05d2a 100644 --- a/lib/wasix/src/journaling/journal.rs +++ b/lib/wasix/src/journaling/journal.rs @@ -311,6 +311,331 @@ pub enum JournalEntry<'a> { }, } +impl<'a> JournalEntry<'a> { + pub fn into_owned(self) -> JournalEntry<'static> { + match self { + Self::InitModule { wasm_hash } => JournalEntry::InitModule { wasm_hash }, + Self::UpdateMemoryRegion { region, data } => JournalEntry::UpdateMemoryRegion { + region, + data: data.into_owned().into(), + }, + Self::ProcessExit { exit_code } => JournalEntry::ProcessExit { exit_code }, + Self::SetThread { + id, + call_stack, + memory_stack, + store_data, + is_64bit, + } => JournalEntry::SetThread { + id, + call_stack: call_stack.into_owned().into(), + memory_stack: memory_stack.into_owned().into(), + store_data: store_data.into_owned().into(), + is_64bit, + }, + Self::CloseThread { id, exit_code } => JournalEntry::CloseThread { id, exit_code }, + Self::FileDescriptorSeek { fd, offset, whence } => { + JournalEntry::FileDescriptorSeek { fd, offset, whence } + } + Self::FileDescriptorWrite { + fd, + offset, + data, + is_64bit, + } => JournalEntry::FileDescriptorWrite { + fd, + offset, + data: data.into_owned().into(), + is_64bit, + }, + Self::SetClockTime { clock_id, time } => JournalEntry::SetClockTime { clock_id, time }, + Self::CloseFileDescriptor { fd } => JournalEntry::CloseFileDescriptor { fd }, + Self::OpenFileDescriptor { + fd, + dirfd, + dirflags, + path, + o_flags, + fs_rights_base, + fs_rights_inheriting, + fs_flags, + } => JournalEntry::OpenFileDescriptor { + fd, + dirfd, + dirflags, + path: path.into_owned().into(), + o_flags, + fs_rights_base, + fs_rights_inheriting, + fs_flags, + }, + Self::RenumberFileDescriptor { old_fd, new_fd } => { + JournalEntry::RenumberFileDescriptor { old_fd, new_fd } + } + Self::DuplicateFileDescriptor { + original_fd, + copied_fd, + } => JournalEntry::DuplicateFileDescriptor { + original_fd, + copied_fd, + }, + Self::CreateDirectory { fd, path } => JournalEntry::CreateDirectory { + fd, + path: path.into_owned().into(), + }, + Self::RemoveDirectory { fd, path } => JournalEntry::RemoveDirectory { + fd, + path: path.into_owned().into(), + }, + Self::PathSetTimes { + fd, + flags, + path, + st_atim, + st_mtim, + fst_flags, + } => JournalEntry::PathSetTimes { + fd, + flags, + path: path.into_owned().into(), + st_atim, + st_mtim, + fst_flags, + }, + Self::FileDescriptorSetTimes { + fd, + st_atim, + st_mtim, + fst_flags, + } => JournalEntry::FileDescriptorSetTimes { + fd, + st_atim, + st_mtim, + fst_flags, + }, + Self::FileDescriptorSetFlags { fd, flags } => { + JournalEntry::FileDescriptorSetFlags { fd, flags } + } + Self::FileDescriptorSetRights { + fd, + fs_rights_base, + fs_rights_inheriting, + } => JournalEntry::FileDescriptorSetRights { + fd, + fs_rights_base, + fs_rights_inheriting, + }, + Self::FileDescriptorSetSize { fd, st_size } => { + JournalEntry::FileDescriptorSetSize { fd, st_size } + } + Self::FileDescriptorAdvise { + fd, + offset, + len, + advice, + } => JournalEntry::FileDescriptorAdvise { + fd, + offset, + len, + advice, + }, + Self::FileDescriptorAllocate { fd, offset, len } => { + JournalEntry::FileDescriptorAllocate { fd, offset, len } + } + Self::CreateHardLink { + old_fd, + old_path, + old_flags, + new_fd, + new_path, + } => JournalEntry::CreateHardLink { + old_fd, + old_path: old_path.into_owned().into(), + old_flags, + new_fd, + new_path: new_path.into_owned().into(), + }, + Self::CreateSymbolicLink { + old_path, + fd, + new_path, + } => JournalEntry::CreateSymbolicLink { + old_path: old_path.into_owned().into(), + fd, + new_path: new_path.into_owned().into(), + }, + Self::UnlinkFile { fd, path } => JournalEntry::UnlinkFile { + fd, + path: path.into_owned().into(), + }, + Self::PathRename { + old_fd, + old_path, + new_fd, + new_path, + } => JournalEntry::PathRename { + old_fd, + old_path: old_path.into_owned().into(), + new_fd, + new_path: new_path.into_owned().into(), + }, + Self::ChangeDirectory { path } => JournalEntry::ChangeDirectory { + path: path.into_owned().into(), + }, + Self::EpollCreate { fd } => JournalEntry::EpollCreate { fd }, + Self::EpollCtl { + epfd, + op, + fd, + event, + } => JournalEntry::EpollCtl { + epfd, + op, + fd, + event, + }, + Self::TtySet { tty, line_feeds } => JournalEntry::TtySet { tty, line_feeds }, + Self::CreatePipe { fd1, fd2 } => JournalEntry::CreatePipe { fd1, fd2 }, + Self::CreateEvent { + initial_val, + flags, + fd, + } => JournalEntry::CreateEvent { + initial_val, + flags, + fd, + }, + Self::PortAddAddr { cidr } => JournalEntry::PortAddAddr { cidr }, + Self::PortDelAddr { addr } => JournalEntry::PortDelAddr { addr }, + Self::PortAddrClear => JournalEntry::PortAddrClear, + Self::PortBridge { + network, + token, + security, + } => JournalEntry::PortBridge { + network: network.into_owned().into(), + token: token.into_owned().into(), + security, + }, + Self::PortUnbridge => JournalEntry::PortUnbridge, + Self::PortDhcpAcquire => JournalEntry::PortDhcpAcquire, + Self::PortGatewaySet { ip } => JournalEntry::PortGatewaySet { ip }, + Self::PortRouteAdd { + cidr, + via_router, + preferred_until, + expires_at, + } => JournalEntry::PortRouteAdd { + cidr, + via_router, + preferred_until, + expires_at, + }, + Self::PortRouteClear => JournalEntry::PortRouteClear, + Self::PortRouteDel { ip } => JournalEntry::PortRouteDel { ip }, + Self::SocketOpen { af, ty, pt, fd } => JournalEntry::SocketOpen { af, ty, pt, fd }, + Self::SocketListen { fd, backlog } => JournalEntry::SocketListen { fd, backlog }, + Self::SocketBind { fd, addr } => JournalEntry::SocketBind { fd, addr }, + Self::SocketConnected { fd, addr } => JournalEntry::SocketConnected { fd, addr }, + Self::SocketAccepted { + listen_fd, + fd, + peer_addr, + fd_flags, + nonblocking, + } => JournalEntry::SocketAccepted { + listen_fd, + fd, + peer_addr, + fd_flags, + nonblocking, + }, + Self::SocketJoinIpv4Multicast { + fd, + multiaddr, + iface, + } => JournalEntry::SocketJoinIpv4Multicast { + fd, + multiaddr, + iface, + }, + Self::SocketJoinIpv6Multicast { + fd, + multiaddr, + iface, + } => JournalEntry::SocketJoinIpv6Multicast { + fd, + multiaddr, + iface, + }, + Self::SocketLeaveIpv4Multicast { + fd, + multiaddr, + iface, + } => JournalEntry::SocketLeaveIpv4Multicast { + fd, + multiaddr, + iface, + }, + Self::SocketLeaveIpv6Multicast { + fd, + multiaddr, + iface, + } => JournalEntry::SocketLeaveIpv6Multicast { + fd, + multiaddr, + iface, + }, + Self::SocketSendFile { + socket_fd, + file_fd, + offset, + count, + } => JournalEntry::SocketSendFile { + socket_fd, + file_fd, + offset, + count, + }, + Self::SocketSendTo { + fd, + data, + flags, + addr, + is_64bit, + } => JournalEntry::SocketSendTo { + fd, + data: data.into_owned().into(), + flags, + addr, + is_64bit, + }, + Self::SocketSend { + fd, + data, + flags, + is_64bit, + } => JournalEntry::SocketSend { + fd, + data: data.into_owned().into(), + flags, + is_64bit, + }, + Self::SocketSetOptFlag { fd, opt, flag } => { + JournalEntry::SocketSetOptFlag { fd, opt, flag } + } + Self::SocketSetOptSize { fd, opt, size } => { + JournalEntry::SocketSetOptSize { fd, opt, size } + } + Self::SocketSetOptTime { fd, ty, time } => { + JournalEntry::SocketSetOptTime { fd, ty, time } + } + Self::SocketShutdown { fd, how } => JournalEntry::SocketShutdown { fd, how }, + Self::Snapshot { when, trigger } => JournalEntry::Snapshot { when, trigger }, + } + } +} + /// The snapshot capturer will take a series of objects that represents the state of /// a WASM process at a point in time and saves it so that it can be restored. /// It also allows for the restoration of that state at a later moment @@ -322,7 +647,7 @@ pub trait Journal { /// Returns a stream of snapshot objects that the runtime will use /// to restore the state of a WASM process to a previous moment in time - fn read(&self) -> LocalBoxFuture<'_, anyhow::Result>>>; + fn read<'a>(&'a self) -> LocalBoxFuture<'_, anyhow::Result>>>; } pub type DynJournal = dyn Journal + Send + Sync; diff --git a/lib/wasix/src/journaling/mod.rs b/lib/wasix/src/journaling/mod.rs index 0dee6884595..aa704bca8db 100644 --- a/lib/wasix/src/journaling/mod.rs +++ b/lib/wasix/src/journaling/mod.rs @@ -1,22 +1,14 @@ -mod compactor; +mod concrete; #[cfg(feature = "journal")] mod effector; #[cfg(not(feature = "journal"))] #[path = "effector/unimplemented.rs"] mod effector; -mod filter; mod journal; -#[cfg(feature = "journal")] -mod log_file; -mod unsupported; -pub use compactor::*; +pub use concrete::*; pub use effector::*; -pub use filter::*; pub use journal::*; -#[cfg(feature = "journal")] -pub use log_file::*; -pub use unsupported::*; use serde::{Deserialize, Serialize}; use std::str::FromStr; From a57f30d8bae3d56de88c0f8e31ebe7d187c93571 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Wed, 22 Nov 2023 01:38:45 +1100 Subject: [PATCH 062/129] Switched to rkyv for the serialization of the journal so that reads are zero copy --- Cargo.lock | 1 + lib/cli/src/commands/run/mod.rs | 4 +- lib/cli/src/commands/run/wasi.rs | 4 +- lib/wasi-web/Cargo.lock | 5 + lib/wasix/Cargo.toml | 1 + .../journaling/concrete/archived_journal.rs | 988 ++++++++++++++---- .../src/journaling/concrete/compactor.rs | 26 +- lib/wasix/src/journaling/concrete/filter.rs | 4 +- lib/wasix/src/journaling/concrete/log_file.rs | 136 ++- lib/wasix/src/journaling/concrete/pipe.rs | 22 +- .../src/journaling/concrete/unsupported.rs | 4 +- lib/wasix/src/journaling/journal.rs | 2 +- lib/wasix/src/syscalls/mod.rs | 796 +++++++------- 13 files changed, 1314 insertions(+), 679 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2f7e517c07d..5757683ce80 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6251,6 +6251,7 @@ dependencies = [ "serde_json", "serde_yaml 0.8.26", "sha2", + "shared-buffer", "tempfile", "term_size", "termios", diff --git a/lib/cli/src/commands/run/mod.rs b/lib/cli/src/commands/run/mod.rs index 34d79105aba..2a1f9f1922f 100644 --- a/lib/cli/src/commands/run/mod.rs +++ b/lib/cli/src/commands/run/mod.rs @@ -267,7 +267,7 @@ impl Run { for journal in self.wasi.journals.clone() { runner .config() - .add_journal(Arc::new(LogFileJournal::new_std(journal)?)); + .add_journal(Arc::new(LogFileJournal::new(journal)?)); } } @@ -350,7 +350,7 @@ impl Run { runner.with_snapshot_interval(Duration::from_millis(period)); } for journal in self.wasi.journals.clone() { - runner.add_journal(Arc::new(LogFileJournal::new_std(journal)?)); + runner.add_journal(Arc::new(LogFileJournal::new(journal)?)); } } diff --git a/lib/cli/src/commands/run/wasi.rs b/lib/cli/src/commands/run/wasi.rs index 0ebbbbf8473..b04a71553cf 100644 --- a/lib/cli/src/commands/run/wasi.rs +++ b/lib/cli/src/commands/run/wasi.rs @@ -351,7 +351,7 @@ impl Wasi { builder.with_snapshot_interval(std::time::Duration::from_millis(interval)); } for journal in self.journals.iter() { - builder.add_journal(Arc::new(LogFileJournal::new_std(journal)?)); + builder.add_journal(Arc::new(LogFileJournal::new(journal)?)); } } @@ -510,7 +510,7 @@ impl Wasi { #[cfg(feature = "journal")] for journal in self.journals.clone() { - rt.add_journal(Arc::new(LogFileJournal::new_std(journal)?)); + rt.add_journal(Arc::new(LogFileJournal::new(journal)?)); } if !self.no_tty { diff --git a/lib/wasi-web/Cargo.lock b/lib/wasi-web/Cargo.lock index 290da77115b..39b59805ee2 100644 --- a/lib/wasi-web/Cargo.lock +++ b/lib/wasi-web/Cargo.lock @@ -2198,11 +2198,13 @@ dependencies = [ "async-trait", "base64 0.21.5", "bincode", + "bytecheck", "bytes", "derivative", "futures-util", "libc", "pin-project-lite", + "rkyv", "serde", "thiserror", "tokio", @@ -2565,6 +2567,7 @@ dependencies = [ "anyhow", "async-trait", "bincode", + "bytecheck", "bytes", "cfg-if", "chrono", @@ -2584,6 +2587,7 @@ dependencies = [ "petgraph", "pin-project", "rand", + "rkyv", "semver", "serde", "serde_cbor", @@ -2591,6 +2595,7 @@ dependencies = [ "serde_json", "serde_yaml 0.8.26", "sha2 0.10.8", + "shared-buffer", "tempfile", "term_size", "termios", diff --git a/lib/wasix/Cargo.toml b/lib/wasix/Cargo.toml index f2f564887d7..c080c63a27f 100644 --- a/lib/wasix/Cargo.toml +++ b/lib/wasix/Cargo.toml @@ -77,6 +77,7 @@ tower = { version = "0.4.13", features = ["make", "util"], optional = true } url = "2.3.1" rkyv = { version = "0.7.40", features = ["indexmap", "validation", "strict"] } bytecheck = "0.6.8" +shared-buffer = "0.1" petgraph = "0.6.3" rayon = { version = "1.7.0", optional = true } wasm-bindgen = { version = "0.2.87", optional = true } diff --git a/lib/wasix/src/journaling/concrete/archived_journal.rs b/lib/wasix/src/journaling/concrete/archived_journal.rs index ba5b33422af..43cebd300c3 100644 --- a/lib/wasix/src/journaling/concrete/archived_journal.rs +++ b/lib/wasix/src/journaling/concrete/archived_journal.rs @@ -1,15 +1,36 @@ use rkyv::{Archive, CheckBytes, Deserialize as RkyvDeserialize, Serialize as RkyvSerialize}; use serde; +use std::borrow::Cow; use std::{net::Shutdown, time::SystemTime}; use virtual_net::{Duration, IpAddr, IpCidr, Ipv4Addr, Ipv6Addr, SocketAddr, StreamSecurity}; -use wasmer_wasix_types::wasi::{ - self, Addressfamily, EpollEventCtl, EpollType, Sockoption, Socktype, -}; +use wasmer_wasix_types::wasi::{self, EpollEventCtl, EpollType, Fdflags, Rights, Sockoption}; use crate::net::socket::TimeType; use super::*; +pub const JOURNAL_MAGIC_NUMBER: u64 = 0x310d6dd027362979; + +/// Version of the archived journal +#[derive( + Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, +)] +#[archive_attr(derive(CheckBytes))] +pub(crate) enum JournalControlBatchType { + Abort, + Commit, +} + +/// Represents a batch of journal log entries +#[allow(clippy::large_enum_variant)] +#[derive( + Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, +)] +#[archive_attr(derive(CheckBytes))] +pub(crate) struct JournalBatch { + pub records: Vec, +} + /// The journal log entries are serializable which /// allows them to be written directly to a file /// @@ -21,7 +42,7 @@ use super::*; Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, )] #[archive_attr(derive(CheckBytes))] -pub(crate) enum ArchivedJournalEntry { +pub(crate) enum JournalBatchEntry { InitModuleV1 { wasm_hash: [u8; 32], }, @@ -40,7 +61,7 @@ pub(crate) enum ArchivedJournalEntry { exit_code: Option, }, FileDescriptorSeekV1 { - fd: Fd, + fd: u32, offset: i64, whence: JournalWhenceV1, }, @@ -89,7 +110,7 @@ pub(crate) enum ArchivedJournalEntry { path: String, }, PathSetTimesV1 { - fd: Fd, + fd: u32, flags: u32, path: String, st_atim: u64, @@ -312,11 +333,11 @@ pub(crate) enum JournalSnapshot0ClockidV1 { impl From for JournalSnapshot0ClockidV1 { fn from(val: wasi::Snapshot0Clockid) -> Self { match val { - Snapshot0Clockid::Realtime => JournalSnapshot0ClockidV1::Realtime, - Snapshot0Clockid::Monotonic => JournalSnapshot0ClockidV1::Monotonic, - Snapshot0Clockid::ProcessCputimeId => JournalSnapshot0ClockidV1::ProcessCputimeId, - Snapshot0Clockid::ThreadCputimeId => JournalSnapshot0ClockidV1::ThreadCputimeId, - Snapshot0Clockid::Unknown => JournalSnapshot0ClockidV1::Unknown, + wasi::Snapshot0Clockid::Realtime => JournalSnapshot0ClockidV1::Realtime, + wasi::Snapshot0Clockid::Monotonic => JournalSnapshot0ClockidV1::Monotonic, + wasi::Snapshot0Clockid::ProcessCputimeId => JournalSnapshot0ClockidV1::ProcessCputimeId, + wasi::Snapshot0Clockid::ThreadCputimeId => JournalSnapshot0ClockidV1::ThreadCputimeId, + wasi::Snapshot0Clockid::Unknown => JournalSnapshot0ClockidV1::Unknown, } } } @@ -324,11 +345,27 @@ impl From for JournalSnapshot0ClockidV1 { impl From for wasi::Snapshot0Clockid { fn from(val: JournalSnapshot0ClockidV1) -> Self { match val { - JournalSnapshot0ClockidV1::Realtime => Snapshot0Clockid::Realtime, - JournalSnapshot0ClockidV1::Monotonic => Snapshot0Clockid::Monotonic, - JournalSnapshot0ClockidV1::ProcessCputimeId => Snapshot0Clockid::ProcessCputimeId, - JournalSnapshot0ClockidV1::ThreadCputimeId => Snapshot0Clockid::ThreadCputimeId, - JournalSnapshot0ClockidV1::Unknown => Snapshot0Clockid::Unknown, + JournalSnapshot0ClockidV1::Realtime => wasi::Snapshot0Clockid::Realtime, + JournalSnapshot0ClockidV1::Monotonic => wasi::Snapshot0Clockid::Monotonic, + JournalSnapshot0ClockidV1::ProcessCputimeId => wasi::Snapshot0Clockid::ProcessCputimeId, + JournalSnapshot0ClockidV1::ThreadCputimeId => wasi::Snapshot0Clockid::ThreadCputimeId, + JournalSnapshot0ClockidV1::Unknown => wasi::Snapshot0Clockid::Unknown, + } + } +} + +impl From<&'_ ArchivedJournalSnapshot0ClockidV1> for wasi::Snapshot0Clockid { + fn from(val: &'_ ArchivedJournalSnapshot0ClockidV1) -> Self { + match val { + ArchivedJournalSnapshot0ClockidV1::Realtime => wasi::Snapshot0Clockid::Realtime, + ArchivedJournalSnapshot0ClockidV1::Monotonic => wasi::Snapshot0Clockid::Monotonic, + ArchivedJournalSnapshot0ClockidV1::ProcessCputimeId => { + wasi::Snapshot0Clockid::ProcessCputimeId + } + ArchivedJournalSnapshot0ClockidV1::ThreadCputimeId => { + wasi::Snapshot0Clockid::ThreadCputimeId + } + ArchivedJournalSnapshot0ClockidV1::Unknown => wasi::Snapshot0Clockid::Unknown, } } } @@ -358,10 +395,21 @@ impl From for JournalWhenceV1 { impl From for wasi::Whence { fn from(val: JournalWhenceV1) -> Self { match val { - JournalWhenceV1::Set => Whence::Set, - JournalWhenceV1::Cur => Whence::Cur, - JournalWhenceV1::End => Whence::End, - JournalWhenceV1::Unknown => Whence::Unknown, + JournalWhenceV1::Set => wasi::Whence::Set, + JournalWhenceV1::Cur => wasi::Whence::Cur, + JournalWhenceV1::End => wasi::Whence::End, + JournalWhenceV1::Unknown => wasi::Whence::Unknown, + } + } +} + +impl From<&'_ ArchivedJournalWhenceV1> for wasi::Whence { + fn from(val: &'_ ArchivedJournalWhenceV1) -> Self { + match val { + ArchivedJournalWhenceV1::Set => wasi::Whence::Set, + ArchivedJournalWhenceV1::Cur => wasi::Whence::Cur, + ArchivedJournalWhenceV1::End => wasi::Whence::End, + ArchivedJournalWhenceV1::Unknown => wasi::Whence::Unknown, } } } @@ -383,13 +431,13 @@ pub(crate) enum JournalAdviceV1 { impl From for JournalAdviceV1 { fn from(val: wasi::Advice) -> Self { match val { - Advice::Normal => JournalAdviceV1::Normal, - Advice::Sequential => JournalAdviceV1::Sequential, - Advice::Random => JournalAdviceV1::Random, - Advice::Willneed => JournalAdviceV1::Willneed, - Advice::Dontneed => JournalAdviceV1::Dontneed, - Advice::Noreuse => JournalAdviceV1::Noreuse, - Advice::Unknown => JournalAdviceV1::Unknown, + wasi::Advice::Normal => JournalAdviceV1::Normal, + wasi::Advice::Sequential => JournalAdviceV1::Sequential, + wasi::Advice::Random => JournalAdviceV1::Random, + wasi::Advice::Willneed => JournalAdviceV1::Willneed, + wasi::Advice::Dontneed => JournalAdviceV1::Dontneed, + wasi::Advice::Noreuse => JournalAdviceV1::Noreuse, + wasi::Advice::Unknown => JournalAdviceV1::Unknown, } } } @@ -397,13 +445,27 @@ impl From for JournalAdviceV1 { impl From for wasi::Advice { fn from(val: JournalAdviceV1) -> Self { match val { - JournalAdviceV1::Normal => Advice::Normal, - JournalAdviceV1::Sequential => Advice::Sequential, - JournalAdviceV1::Random => Advice::Random, - JournalAdviceV1::Willneed => Advice::Willneed, - JournalAdviceV1::Dontneed => Advice::Dontneed, - JournalAdviceV1::Noreuse => Advice::Noreuse, - JournalAdviceV1::Unknown => Advice::Unknown, + JournalAdviceV1::Normal => wasi::Advice::Normal, + JournalAdviceV1::Sequential => wasi::Advice::Sequential, + JournalAdviceV1::Random => wasi::Advice::Random, + JournalAdviceV1::Willneed => wasi::Advice::Willneed, + JournalAdviceV1::Dontneed => wasi::Advice::Dontneed, + JournalAdviceV1::Noreuse => wasi::Advice::Noreuse, + JournalAdviceV1::Unknown => wasi::Advice::Unknown, + } + } +} + +impl From<&'_ ArchivedJournalAdviceV1> for wasi::Advice { + fn from(val: &'_ ArchivedJournalAdviceV1) -> Self { + match val { + ArchivedJournalAdviceV1::Normal => wasi::Advice::Normal, + ArchivedJournalAdviceV1::Sequential => wasi::Advice::Sequential, + ArchivedJournalAdviceV1::Random => wasi::Advice::Random, + ArchivedJournalAdviceV1::Willneed => wasi::Advice::Willneed, + ArchivedJournalAdviceV1::Dontneed => wasi::Advice::Dontneed, + ArchivedJournalAdviceV1::Noreuse => wasi::Advice::Noreuse, + ArchivedJournalAdviceV1::Unknown => wasi::Advice::Unknown, } } } @@ -437,6 +499,17 @@ impl From for wasi::ExitCode { } } +impl From<&'_ ArchivedJournalExitCodeV1> for wasi::ExitCode { + fn from(val: &'_ ArchivedJournalExitCodeV1) -> Self { + match val { + ArchivedJournalExitCodeV1::Errno(errno) => { + wasi::ExitCode::Errno((*errno).try_into().unwrap_or(wasi::Errno::Unknown)) + } + ArchivedJournalExitCodeV1::Other(id) => wasi::ExitCode::Other(*id), + } + } +} + #[derive( Debug, Clone, @@ -500,6 +573,25 @@ impl From for SnapshotTrigger { } } +impl From<&'_ ArchivedJournalSnapshotTriggerV1> for SnapshotTrigger { + fn from(val: &'_ ArchivedJournalSnapshotTriggerV1) -> Self { + match val { + ArchivedJournalSnapshotTriggerV1::Idle => SnapshotTrigger::Idle, + ArchivedJournalSnapshotTriggerV1::Listen => SnapshotTrigger::FirstListen, + ArchivedJournalSnapshotTriggerV1::Environ => SnapshotTrigger::FirstEnviron, + ArchivedJournalSnapshotTriggerV1::Stdin => SnapshotTrigger::FirstStdin, + ArchivedJournalSnapshotTriggerV1::Timer => SnapshotTrigger::PeriodicInterval, + ArchivedJournalSnapshotTriggerV1::Sigint => SnapshotTrigger::Sigint, + ArchivedJournalSnapshotTriggerV1::Sigalrm => SnapshotTrigger::Sigalrm, + ArchivedJournalSnapshotTriggerV1::Sigtstp => SnapshotTrigger::Sigtstp, + ArchivedJournalSnapshotTriggerV1::Sigstop => SnapshotTrigger::Sigstop, + ArchivedJournalSnapshotTriggerV1::NonDeterministicCall => { + SnapshotTrigger::NonDeterministicCall + } + } + } +} + #[derive( Debug, Clone, @@ -537,10 +629,21 @@ impl From for JournalEpollCtlV1 { impl From for wasi::EpollCtl { fn from(val: JournalEpollCtlV1) -> Self { match val { - JournalEpollCtlV1::Add => EpollCtl::Add, - JournalEpollCtlV1::Mod => EpollCtl::Mod, - JournalEpollCtlV1::Del => EpollCtl::Del, - JournalEpollCtlV1::Unknown => EpollCtl::Unknown, + JournalEpollCtlV1::Add => wasi::EpollCtl::Add, + JournalEpollCtlV1::Mod => wasi::EpollCtl::Mod, + JournalEpollCtlV1::Del => wasi::EpollCtl::Del, + JournalEpollCtlV1::Unknown => wasi::EpollCtl::Unknown, + } + } +} + +impl From<&'_ ArchivedJournalEpollCtlV1> for wasi::EpollCtl { + fn from(val: &'_ ArchivedJournalEpollCtlV1) -> Self { + match val { + ArchivedJournalEpollCtlV1::Add => wasi::EpollCtl::Add, + ArchivedJournalEpollCtlV1::Mod => wasi::EpollCtl::Mod, + ArchivedJournalEpollCtlV1::Del => wasi::EpollCtl::Del, + ArchivedJournalEpollCtlV1::Unknown => wasi::EpollCtl::Unknown, } } } @@ -552,7 +655,7 @@ impl From for wasi::EpollCtl { pub struct JournalEpollEventCtlV1 { pub events: u32, pub ptr: u64, - pub fd: Fd, + pub fd: u32, pub data1: u32, pub data2: u64, } @@ -581,6 +684,18 @@ impl From for EpollEventCtl { } } +impl From<&'_ ArchivedJournalEpollEventCtlV1> for EpollEventCtl { + fn from(val: &'_ ArchivedJournalEpollEventCtlV1) -> Self { + Self { + events: EpollType::from_bits_truncate(val.events), + ptr: val.ptr, + fd: val.fd, + data1: val.data1, + data2: val.data2, + } + } +} + #[derive( Debug, Clone, @@ -628,6 +743,18 @@ impl From for StreamSecurity { } } +impl From<&'_ ArchivedJournalStreamSecurityV1> for StreamSecurity { + fn from(val: &'_ ArchivedJournalStreamSecurityV1) -> Self { + match val { + ArchivedJournalStreamSecurityV1::Unencrypted => StreamSecurity::Unencrypted, + ArchivedJournalStreamSecurityV1::AnyEncryption => StreamSecurity::AnyEncyption, + ArchivedJournalStreamSecurityV1::ClassicEncryption => StreamSecurity::ClassicEncryption, + ArchivedJournalStreamSecurityV1::DoubleEncryption => StreamSecurity::DoubleEncryption, + ArchivedJournalStreamSecurityV1::Unknown => StreamSecurity::AnyEncyption, + } + } +} + #[derive( Debug, Clone, @@ -651,24 +778,35 @@ pub enum JournalAddressfamilyV1 { Unix, } -impl From for JournalAddressfamilyV1 { - fn from(val: Addressfamily) -> Self { +impl From for JournalAddressfamilyV1 { + fn from(val: wasi::Addressfamily) -> Self { match val { - Addressfamily::Unspec => JournalAddressfamilyV1::Unspec, - Addressfamily::Inet4 => JournalAddressfamilyV1::Inet4, - Addressfamily::Inet6 => JournalAddressfamilyV1::Inet6, - Addressfamily::Unix => JournalAddressfamilyV1::Unix, + wasi::Addressfamily::Unspec => JournalAddressfamilyV1::Unspec, + wasi::Addressfamily::Inet4 => JournalAddressfamilyV1::Inet4, + wasi::Addressfamily::Inet6 => JournalAddressfamilyV1::Inet6, + wasi::Addressfamily::Unix => JournalAddressfamilyV1::Unix, } } } -impl From for Addressfamily { +impl From for wasi::Addressfamily { fn from(val: JournalAddressfamilyV1) -> Self { match val { - JournalAddressfamilyV1::Unspec => Addressfamily::Unspec, - JournalAddressfamilyV1::Inet4 => Addressfamily::Inet4, - JournalAddressfamilyV1::Inet6 => Addressfamily::Inet6, - JournalAddressfamilyV1::Unix => Addressfamily::Unix, + JournalAddressfamilyV1::Unspec => wasi::Addressfamily::Unspec, + JournalAddressfamilyV1::Inet4 => wasi::Addressfamily::Inet4, + JournalAddressfamilyV1::Inet6 => wasi::Addressfamily::Inet6, + JournalAddressfamilyV1::Unix => wasi::Addressfamily::Unix, + } + } +} + +impl From<&'_ ArchivedJournalAddressfamilyV1> for wasi::Addressfamily { + fn from(val: &'_ ArchivedJournalAddressfamilyV1) -> Self { + match val { + ArchivedJournalAddressfamilyV1::Unspec => wasi::Addressfamily::Unspec, + ArchivedJournalAddressfamilyV1::Inet4 => wasi::Addressfamily::Inet4, + ArchivedJournalAddressfamilyV1::Inet6 => wasi::Addressfamily::Inet6, + ArchivedJournalAddressfamilyV1::Unix => wasi::Addressfamily::Unix, } } } @@ -697,26 +835,38 @@ pub enum JournalSocktypeV1 { Seqpacket, } -impl From for JournalSocktypeV1 { - fn from(val: Socktype) -> Self { +impl From for JournalSocktypeV1 { + fn from(val: wasi::Socktype) -> Self { match val { - Socktype::Stream => JournalSocktypeV1::Stream, - Socktype::Dgram => JournalSocktypeV1::Dgram, - Socktype::Raw => JournalSocktypeV1::Raw, - Socktype::Seqpacket => JournalSocktypeV1::Seqpacket, - Socktype::Unknown => JournalSocktypeV1::Unknown, + wasi::Socktype::Stream => JournalSocktypeV1::Stream, + wasi::Socktype::Dgram => JournalSocktypeV1::Dgram, + wasi::Socktype::Raw => JournalSocktypeV1::Raw, + wasi::Socktype::Seqpacket => JournalSocktypeV1::Seqpacket, + wasi::Socktype::Unknown => JournalSocktypeV1::Unknown, } } } -impl From for Socktype { +impl From for wasi::Socktype { fn from(val: JournalSocktypeV1) -> Self { match val { - JournalSocktypeV1::Stream => Socktype::Stream, - JournalSocktypeV1::Dgram => Socktype::Dgram, - JournalSocktypeV1::Raw => Socktype::Raw, - JournalSocktypeV1::Seqpacket => Socktype::Seqpacket, - JournalSocktypeV1::Unknown => Socktype::Unknown, + JournalSocktypeV1::Stream => wasi::Socktype::Stream, + JournalSocktypeV1::Dgram => wasi::Socktype::Dgram, + JournalSocktypeV1::Raw => wasi::Socktype::Raw, + JournalSocktypeV1::Seqpacket => wasi::Socktype::Seqpacket, + JournalSocktypeV1::Unknown => wasi::Socktype::Unknown, + } + } +} + +impl From<&'_ ArchivedJournalSocktypeV1> for wasi::Socktype { + fn from(val: &'_ ArchivedJournalSocktypeV1) -> Self { + match val { + ArchivedJournalSocktypeV1::Stream => wasi::Socktype::Stream, + ArchivedJournalSocktypeV1::Dgram => wasi::Socktype::Dgram, + ArchivedJournalSocktypeV1::Raw => wasi::Socktype::Raw, + ArchivedJournalSocktypeV1::Seqpacket => wasi::Socktype::Seqpacket, + ArchivedJournalSocktypeV1::Unknown => wasi::Socktype::Unknown, } } } @@ -768,69 +918,103 @@ pub enum JournalSockoptionV1 { } impl From for JournalSockoptionV1 { - fn from(val: Sockoption) -> Self { + fn from(val: wasi::Sockoption) -> Self { match val { - Sockoption::Noop => JournalSockoptionV1::Noop, - Sockoption::ReusePort => JournalSockoptionV1::ReusePort, - Sockoption::ReuseAddr => JournalSockoptionV1::ReuseAddr, - Sockoption::NoDelay => JournalSockoptionV1::NoDelay, - Sockoption::DontRoute => JournalSockoptionV1::DontRoute, - Sockoption::OnlyV6 => JournalSockoptionV1::OnlyV6, - Sockoption::Broadcast => JournalSockoptionV1::Broadcast, - Sockoption::MulticastLoopV4 => JournalSockoptionV1::MulticastLoopV4, - Sockoption::MulticastLoopV6 => JournalSockoptionV1::MulticastLoopV6, - Sockoption::Promiscuous => JournalSockoptionV1::Promiscuous, - Sockoption::Listening => JournalSockoptionV1::Listening, - Sockoption::LastError => JournalSockoptionV1::LastError, - Sockoption::KeepAlive => JournalSockoptionV1::KeepAlive, - Sockoption::Linger => JournalSockoptionV1::Linger, - Sockoption::OobInline => JournalSockoptionV1::OobInline, - Sockoption::RecvBufSize => JournalSockoptionV1::RecvBufSize, - Sockoption::SendBufSize => JournalSockoptionV1::SendBufSize, - Sockoption::RecvLowat => JournalSockoptionV1::RecvLowat, - Sockoption::SendLowat => JournalSockoptionV1::SendLowat, - Sockoption::RecvTimeout => JournalSockoptionV1::RecvTimeout, - Sockoption::SendTimeout => JournalSockoptionV1::SendTimeout, - Sockoption::ConnectTimeout => JournalSockoptionV1::ConnectTimeout, - Sockoption::AcceptTimeout => JournalSockoptionV1::AcceptTimeout, - Sockoption::Ttl => JournalSockoptionV1::Ttl, - Sockoption::MulticastTtlV4 => JournalSockoptionV1::MulticastTtlV4, - Sockoption::Type => JournalSockoptionV1::Type, - Sockoption::Proto => JournalSockoptionV1::Proto, + wasi::Sockoption::Noop => JournalSockoptionV1::Noop, + wasi::Sockoption::ReusePort => JournalSockoptionV1::ReusePort, + wasi::Sockoption::ReuseAddr => JournalSockoptionV1::ReuseAddr, + wasi::Sockoption::NoDelay => JournalSockoptionV1::NoDelay, + wasi::Sockoption::DontRoute => JournalSockoptionV1::DontRoute, + wasi::Sockoption::OnlyV6 => JournalSockoptionV1::OnlyV6, + wasi::Sockoption::Broadcast => JournalSockoptionV1::Broadcast, + wasi::Sockoption::MulticastLoopV4 => JournalSockoptionV1::MulticastLoopV4, + wasi::Sockoption::MulticastLoopV6 => JournalSockoptionV1::MulticastLoopV6, + wasi::Sockoption::Promiscuous => JournalSockoptionV1::Promiscuous, + wasi::Sockoption::Listening => JournalSockoptionV1::Listening, + wasi::Sockoption::LastError => JournalSockoptionV1::LastError, + wasi::Sockoption::KeepAlive => JournalSockoptionV1::KeepAlive, + wasi::Sockoption::Linger => JournalSockoptionV1::Linger, + wasi::Sockoption::OobInline => JournalSockoptionV1::OobInline, + wasi::Sockoption::RecvBufSize => JournalSockoptionV1::RecvBufSize, + wasi::Sockoption::SendBufSize => JournalSockoptionV1::SendBufSize, + wasi::Sockoption::RecvLowat => JournalSockoptionV1::RecvLowat, + wasi::Sockoption::SendLowat => JournalSockoptionV1::SendLowat, + wasi::Sockoption::RecvTimeout => JournalSockoptionV1::RecvTimeout, + wasi::Sockoption::SendTimeout => JournalSockoptionV1::SendTimeout, + wasi::Sockoption::ConnectTimeout => JournalSockoptionV1::ConnectTimeout, + wasi::Sockoption::AcceptTimeout => JournalSockoptionV1::AcceptTimeout, + wasi::Sockoption::Ttl => JournalSockoptionV1::Ttl, + wasi::Sockoption::MulticastTtlV4 => JournalSockoptionV1::MulticastTtlV4, + wasi::Sockoption::Type => JournalSockoptionV1::Type, + wasi::Sockoption::Proto => JournalSockoptionV1::Proto, } } } -impl From for Sockoption { +impl From for wasi::Sockoption { fn from(val: JournalSockoptionV1) -> Self { match val { - JournalSockoptionV1::Noop => Sockoption::Noop, - JournalSockoptionV1::ReusePort => Sockoption::ReusePort, - JournalSockoptionV1::ReuseAddr => Sockoption::ReuseAddr, - JournalSockoptionV1::NoDelay => Sockoption::NoDelay, - JournalSockoptionV1::DontRoute => Sockoption::DontRoute, - JournalSockoptionV1::OnlyV6 => Sockoption::OnlyV6, - JournalSockoptionV1::Broadcast => Sockoption::Broadcast, - JournalSockoptionV1::MulticastLoopV4 => Sockoption::MulticastLoopV4, - JournalSockoptionV1::MulticastLoopV6 => Sockoption::MulticastLoopV6, - JournalSockoptionV1::Promiscuous => Sockoption::Promiscuous, - JournalSockoptionV1::Listening => Sockoption::Listening, - JournalSockoptionV1::LastError => Sockoption::LastError, - JournalSockoptionV1::KeepAlive => Sockoption::KeepAlive, - JournalSockoptionV1::Linger => Sockoption::Linger, - JournalSockoptionV1::OobInline => Sockoption::OobInline, - JournalSockoptionV1::RecvBufSize => Sockoption::RecvBufSize, - JournalSockoptionV1::SendBufSize => Sockoption::SendBufSize, - JournalSockoptionV1::RecvLowat => Sockoption::RecvLowat, - JournalSockoptionV1::SendLowat => Sockoption::SendLowat, - JournalSockoptionV1::RecvTimeout => Sockoption::RecvTimeout, - JournalSockoptionV1::SendTimeout => Sockoption::SendTimeout, - JournalSockoptionV1::ConnectTimeout => Sockoption::ConnectTimeout, - JournalSockoptionV1::AcceptTimeout => Sockoption::AcceptTimeout, - JournalSockoptionV1::Ttl => Sockoption::Ttl, - JournalSockoptionV1::MulticastTtlV4 => Sockoption::MulticastTtlV4, - JournalSockoptionV1::Type => Sockoption::Type, - JournalSockoptionV1::Proto => Sockoption::Proto, + JournalSockoptionV1::Noop => wasi::Sockoption::Noop, + JournalSockoptionV1::ReusePort => wasi::Sockoption::ReusePort, + JournalSockoptionV1::ReuseAddr => wasi::Sockoption::ReuseAddr, + JournalSockoptionV1::NoDelay => wasi::Sockoption::NoDelay, + JournalSockoptionV1::DontRoute => wasi::Sockoption::DontRoute, + JournalSockoptionV1::OnlyV6 => wasi::Sockoption::OnlyV6, + JournalSockoptionV1::Broadcast => wasi::Sockoption::Broadcast, + JournalSockoptionV1::MulticastLoopV4 => wasi::Sockoption::MulticastLoopV4, + JournalSockoptionV1::MulticastLoopV6 => wasi::Sockoption::MulticastLoopV6, + JournalSockoptionV1::Promiscuous => wasi::Sockoption::Promiscuous, + JournalSockoptionV1::Listening => wasi::Sockoption::Listening, + JournalSockoptionV1::LastError => wasi::Sockoption::LastError, + JournalSockoptionV1::KeepAlive => wasi::Sockoption::KeepAlive, + JournalSockoptionV1::Linger => wasi::Sockoption::Linger, + JournalSockoptionV1::OobInline => wasi::Sockoption::OobInline, + JournalSockoptionV1::RecvBufSize => wasi::Sockoption::RecvBufSize, + JournalSockoptionV1::SendBufSize => wasi::Sockoption::SendBufSize, + JournalSockoptionV1::RecvLowat => wasi::Sockoption::RecvLowat, + JournalSockoptionV1::SendLowat => wasi::Sockoption::SendLowat, + JournalSockoptionV1::RecvTimeout => wasi::Sockoption::RecvTimeout, + JournalSockoptionV1::SendTimeout => wasi::Sockoption::SendTimeout, + JournalSockoptionV1::ConnectTimeout => wasi::Sockoption::ConnectTimeout, + JournalSockoptionV1::AcceptTimeout => wasi::Sockoption::AcceptTimeout, + JournalSockoptionV1::Ttl => wasi::Sockoption::Ttl, + JournalSockoptionV1::MulticastTtlV4 => wasi::Sockoption::MulticastTtlV4, + JournalSockoptionV1::Type => wasi::Sockoption::Type, + JournalSockoptionV1::Proto => wasi::Sockoption::Proto, + } + } +} + +impl From<&'_ ArchivedJournalSockoptionV1> for wasi::Sockoption { + fn from(val: &'_ ArchivedJournalSockoptionV1) -> Self { + match val { + ArchivedJournalSockoptionV1::Noop => wasi::Sockoption::Noop, + ArchivedJournalSockoptionV1::ReusePort => wasi::Sockoption::ReusePort, + ArchivedJournalSockoptionV1::ReuseAddr => wasi::Sockoption::ReuseAddr, + ArchivedJournalSockoptionV1::NoDelay => wasi::Sockoption::NoDelay, + ArchivedJournalSockoptionV1::DontRoute => wasi::Sockoption::DontRoute, + ArchivedJournalSockoptionV1::OnlyV6 => wasi::Sockoption::OnlyV6, + ArchivedJournalSockoptionV1::Broadcast => wasi::Sockoption::Broadcast, + ArchivedJournalSockoptionV1::MulticastLoopV4 => wasi::Sockoption::MulticastLoopV4, + ArchivedJournalSockoptionV1::MulticastLoopV6 => wasi::Sockoption::MulticastLoopV6, + ArchivedJournalSockoptionV1::Promiscuous => wasi::Sockoption::Promiscuous, + ArchivedJournalSockoptionV1::Listening => wasi::Sockoption::Listening, + ArchivedJournalSockoptionV1::LastError => wasi::Sockoption::LastError, + ArchivedJournalSockoptionV1::KeepAlive => wasi::Sockoption::KeepAlive, + ArchivedJournalSockoptionV1::Linger => wasi::Sockoption::Linger, + ArchivedJournalSockoptionV1::OobInline => wasi::Sockoption::OobInline, + ArchivedJournalSockoptionV1::RecvBufSize => wasi::Sockoption::RecvBufSize, + ArchivedJournalSockoptionV1::SendBufSize => wasi::Sockoption::SendBufSize, + ArchivedJournalSockoptionV1::RecvLowat => wasi::Sockoption::RecvLowat, + ArchivedJournalSockoptionV1::SendLowat => wasi::Sockoption::SendLowat, + ArchivedJournalSockoptionV1::RecvTimeout => wasi::Sockoption::RecvTimeout, + ArchivedJournalSockoptionV1::SendTimeout => wasi::Sockoption::SendTimeout, + ArchivedJournalSockoptionV1::ConnectTimeout => wasi::Sockoption::ConnectTimeout, + ArchivedJournalSockoptionV1::AcceptTimeout => wasi::Sockoption::AcceptTimeout, + ArchivedJournalSockoptionV1::Ttl => wasi::Sockoption::Ttl, + ArchivedJournalSockoptionV1::MulticastTtlV4 => wasi::Sockoption::MulticastTtlV4, + ArchivedJournalSockoptionV1::Type => wasi::Sockoption::Type, + ArchivedJournalSockoptionV1::Proto => wasi::Sockoption::Proto, } } } @@ -886,6 +1070,19 @@ impl From for TimeType { } } +impl From<&'_ ArchivedJournalTimeTypeV1> for TimeType { + fn from(val: &'_ ArchivedJournalTimeTypeV1) -> Self { + match val { + ArchivedJournalTimeTypeV1::ReadTimeout => TimeType::ReadTimeout, + ArchivedJournalTimeTypeV1::WriteTimeout => TimeType::WriteTimeout, + ArchivedJournalTimeTypeV1::AcceptTimeout => TimeType::AcceptTimeout, + ArchivedJournalTimeTypeV1::ConnectTimeout => TimeType::ConnectTimeout, + ArchivedJournalTimeTypeV1::BindTimeout => TimeType::BindTimeout, + ArchivedJournalTimeTypeV1::Linger => TimeType::Linger, + } + } +} + #[derive( Debug, Clone, @@ -928,7 +1125,17 @@ impl From for Shutdown { } } -impl<'a> From> for ArchivedJournalEntry { +impl From<&'_ ArchivedJournalSocketShutdownV1> for Shutdown { + fn from(val: &'_ ArchivedJournalSocketShutdownV1) -> Self { + match val { + ArchivedJournalSocketShutdownV1::Read => Shutdown::Read, + ArchivedJournalSocketShutdownV1::Write => Shutdown::Write, + ArchivedJournalSocketShutdownV1::Both => Shutdown::Both, + } + } +} + +impl<'a> From> for JournalBatchEntry { fn from(value: JournalEntry<'a>) -> Self { match value { JournalEntry::InitModule { wasm_hash } => Self::InitModuleV1 { wasm_hash }, @@ -1295,20 +1502,20 @@ impl<'a> From> for ArchivedJournalEntry { } } -impl<'a> From for JournalEntry<'a> { - fn from(value: ArchivedJournalEntry) -> Self { +impl<'a> From for JournalEntry<'a> { + fn from(value: JournalBatchEntry) -> Self { match value { - ArchivedJournalEntry::InitModuleV1 { wasm_hash } => Self::InitModule { wasm_hash }, - ArchivedJournalEntry::UpdateMemoryRegionV1 { start, end, data } => { + JournalBatchEntry::InitModuleV1 { wasm_hash } => Self::InitModule { wasm_hash }, + JournalBatchEntry::UpdateMemoryRegionV1 { start, end, data } => { Self::UpdateMemoryRegion { region: start..end, data: data.into(), } } - ArchivedJournalEntry::ProcessExitV1 { exit_code } => Self::ProcessExit { + JournalBatchEntry::ProcessExitV1 { exit_code } => Self::ProcessExit { exit_code: exit_code.map(|code| code.into()), }, - ArchivedJournalEntry::SetThreadV1 { + JournalBatchEntry::SetThreadV1 { id, call_stack, memory_stack, @@ -1321,11 +1528,11 @@ impl<'a> From for JournalEntry<'a> { store_data: store_data.into(), is_64bit, }, - ArchivedJournalEntry::CloseThreadV1 { id, exit_code } => Self::CloseThread { + JournalBatchEntry::CloseThreadV1 { id, exit_code } => Self::CloseThread { id: id.into(), exit_code: exit_code.map(|code| code.into()), }, - ArchivedJournalEntry::FileDescriptorWriteV1 { + JournalBatchEntry::FileDescriptorWriteV1 { data, fd, offset, @@ -1336,14 +1543,14 @@ impl<'a> From for JournalEntry<'a> { offset, is_64bit, }, - ArchivedJournalEntry::FileDescriptorSeekV1 { fd, offset, whence } => { + JournalBatchEntry::FileDescriptorSeekV1 { fd, offset, whence } => { Self::FileDescriptorSeek { fd, offset, whence: whence.into(), } } - ArchivedJournalEntry::OpenFileDescriptorV1 { + JournalBatchEntry::OpenFileDescriptorV1 { fd, dirfd, dirflags, @@ -1362,16 +1569,16 @@ impl<'a> From for JournalEntry<'a> { fs_rights_inheriting: wasi::Rights::from_bits_truncate(fs_rights_inheriting), fs_flags: wasi::Fdflags::from_bits_truncate(fs_flags), }, - ArchivedJournalEntry::CloseFileDescriptorV1 { fd } => Self::CloseFileDescriptor { fd }, - ArchivedJournalEntry::RemoveDirectoryV1 { fd, path } => Self::RemoveDirectory { + JournalBatchEntry::CloseFileDescriptorV1 { fd } => Self::CloseFileDescriptor { fd }, + JournalBatchEntry::RemoveDirectoryV1 { fd, path } => Self::RemoveDirectory { fd, path: path.into(), }, - ArchivedJournalEntry::UnlinkFileV1 { fd, path } => Self::UnlinkFile { + JournalBatchEntry::UnlinkFileV1 { fd, path } => Self::UnlinkFile { fd, path: path.into(), }, - ArchivedJournalEntry::PathRenameV1 { + JournalBatchEntry::PathRenameV1 { old_fd, old_path, new_fd, @@ -1382,7 +1589,7 @@ impl<'a> From for JournalEntry<'a> { new_fd, new_path: new_path.into(), }, - ArchivedJournalEntry::SnapshotV1 { + JournalBatchEntry::SnapshotV1 { since_epoch, trigger, } => Self::Snapshot { @@ -1391,25 +1598,25 @@ impl<'a> From for JournalEntry<'a> { .unwrap_or(SystemTime::UNIX_EPOCH), trigger: trigger.into(), }, - ArchivedJournalEntry::SetClockTimeV1 { clock_id, time } => Self::SetClockTime { + JournalBatchEntry::SetClockTimeV1 { clock_id, time } => Self::SetClockTime { clock_id: clock_id.into(), time, }, - ArchivedJournalEntry::RenumberFileDescriptorV1 { old_fd, new_fd } => { + JournalBatchEntry::RenumberFileDescriptorV1 { old_fd, new_fd } => { Self::RenumberFileDescriptor { old_fd, new_fd } } - ArchivedJournalEntry::DuplicateFileDescriptorV1 { + JournalBatchEntry::DuplicateFileDescriptorV1 { original_fd: old_fd, copied_fd: new_fd, } => Self::DuplicateFileDescriptor { original_fd: old_fd, copied_fd: new_fd, }, - ArchivedJournalEntry::CreateDirectoryV1 { fd, path } => Self::CreateDirectory { + JournalBatchEntry::CreateDirectoryV1 { fd, path } => Self::CreateDirectory { fd, path: path.into(), }, - ArchivedJournalEntry::PathSetTimesV1 { + JournalBatchEntry::PathSetTimesV1 { fd, path, flags, @@ -1424,7 +1631,7 @@ impl<'a> From for JournalEntry<'a> { st_mtim, fst_flags: wasi::Fstflags::from_bits_truncate(fst_flags), }, - ArchivedJournalEntry::FileDescriptorSetTimesV1 { + JournalBatchEntry::FileDescriptorSetTimesV1 { fd, st_atim, st_mtim, @@ -1435,16 +1642,16 @@ impl<'a> From for JournalEntry<'a> { st_mtim, fst_flags: wasi::Fstflags::from_bits_truncate(fst_flags), }, - ArchivedJournalEntry::FileDescriptorSetSizeV1 { fd, st_size } => { + JournalBatchEntry::FileDescriptorSetSizeV1 { fd, st_size } => { Self::FileDescriptorSetSize { fd, st_size } } - ArchivedJournalEntry::FileDescriptorSetFlagsV1 { fd, flags } => { + JournalBatchEntry::FileDescriptorSetFlagsV1 { fd, flags } => { Self::FileDescriptorSetFlags { fd, flags: Fdflags::from_bits_truncate(flags), } } - ArchivedJournalEntry::FileDescriptorSetRightsV1 { + JournalBatchEntry::FileDescriptorSetRightsV1 { fd, fs_rights_base, fs_rights_inheriting, @@ -1453,7 +1660,7 @@ impl<'a> From for JournalEntry<'a> { fs_rights_base: Rights::from_bits_truncate(fs_rights_base), fs_rights_inheriting: Rights::from_bits_truncate(fs_rights_inheriting), }, - ArchivedJournalEntry::FileDescriptorAdviseV1 { + JournalBatchEntry::FileDescriptorAdviseV1 { fd, offset, len, @@ -1464,10 +1671,10 @@ impl<'a> From for JournalEntry<'a> { len, advice: advice.into(), }, - ArchivedJournalEntry::FileDescriptorAllocateV1 { fd, offset, len } => { + JournalBatchEntry::FileDescriptorAllocateV1 { fd, offset, len } => { Self::FileDescriptorAllocate { fd, offset, len } } - ArchivedJournalEntry::CreateHardLinkV1 { + JournalBatchEntry::CreateHardLinkV1 { old_fd, old_path, old_flags, @@ -1480,7 +1687,7 @@ impl<'a> From for JournalEntry<'a> { new_fd, new_path: new_path.into(), }, - ArchivedJournalEntry::CreateSymbolicLinkV1 { + JournalBatchEntry::CreateSymbolicLinkV1 { old_path, fd, new_path, @@ -1489,11 +1696,11 @@ impl<'a> From for JournalEntry<'a> { fd, new_path: new_path.into(), }, - ArchivedJournalEntry::ChangeDirectoryV1 { path } => { + JournalBatchEntry::ChangeDirectoryV1 { path } => { Self::ChangeDirectory { path: path.into() } } - ArchivedJournalEntry::EpollCreateV1 { fd } => Self::EpollCreate { fd }, - ArchivedJournalEntry::EpollCtlV1 { + JournalBatchEntry::EpollCreateV1 { fd } => Self::EpollCreate { fd }, + JournalBatchEntry::EpollCtlV1 { epfd, op, fd, @@ -1504,7 +1711,7 @@ impl<'a> From for JournalEntry<'a> { fd, event: event.map(|e| e.into()), }, - ArchivedJournalEntry::TtySetV1 { + JournalBatchEntry::TtySetV1 { cols, rows, width, @@ -1529,11 +1736,11 @@ impl<'a> From for JournalEntry<'a> { }, line_feeds, }, - ArchivedJournalEntry::CreatePipeV1 { fd1, fd2 } => Self::CreatePipe { fd1, fd2 }, - ArchivedJournalEntry::PortAddAddrV1 { cidr } => Self::PortAddAddr { cidr }, - ArchivedJournalEntry::PortDelAddrV1 { addr } => Self::PortDelAddr { addr }, - ArchivedJournalEntry::PortAddrClearV1 => Self::PortAddrClear, - ArchivedJournalEntry::PortBridgeV1 { + JournalBatchEntry::CreatePipeV1 { fd1, fd2 } => Self::CreatePipe { fd1, fd2 }, + JournalBatchEntry::PortAddAddrV1 { cidr } => Self::PortAddAddr { cidr }, + JournalBatchEntry::PortDelAddrV1 { addr } => Self::PortDelAddr { addr }, + JournalBatchEntry::PortAddrClearV1 => Self::PortAddrClear, + JournalBatchEntry::PortBridgeV1 { network, token, security, @@ -1542,10 +1749,10 @@ impl<'a> From for JournalEntry<'a> { token: token.into(), security: security.into(), }, - ArchivedJournalEntry::PortUnbridgeV1 => Self::PortUnbridge, - ArchivedJournalEntry::PortDhcpAcquireV1 => Self::PortDhcpAcquire, - ArchivedJournalEntry::PortGatewaySetV1 { ip } => Self::PortGatewaySet { ip }, - ArchivedJournalEntry::PortRouteAddV1 { + JournalBatchEntry::PortUnbridgeV1 => Self::PortUnbridge, + JournalBatchEntry::PortDhcpAcquireV1 => Self::PortDhcpAcquire, + JournalBatchEntry::PortGatewaySetV1 { ip } => Self::PortGatewaySet { ip }, + JournalBatchEntry::PortRouteAddV1 { cidr, via_router, preferred_until, @@ -1556,22 +1763,18 @@ impl<'a> From for JournalEntry<'a> { preferred_until, expires_at, }, - ArchivedJournalEntry::PortRouteClearV1 => Self::PortRouteClear, - ArchivedJournalEntry::PortRouteDelV1 { ip } => Self::PortRouteDel { ip }, - ArchivedJournalEntry::SocketOpenV1 { af, ty, pt, fd } => Self::SocketOpen { + JournalBatchEntry::PortRouteClearV1 => Self::PortRouteClear, + JournalBatchEntry::PortRouteDelV1 { ip } => Self::PortRouteDel { ip }, + JournalBatchEntry::SocketOpenV1 { af, ty, pt, fd } => Self::SocketOpen { af: af.into(), ty: ty.into(), pt: pt.try_into().unwrap_or(wasi::SockProto::Max), fd, }, - ArchivedJournalEntry::SocketListenV1 { fd, backlog } => { - Self::SocketListen { fd, backlog } - } - ArchivedJournalEntry::SocketBindV1 { fd, addr } => Self::SocketBind { fd, addr }, - ArchivedJournalEntry::SocketConnectedV1 { fd, addr } => { - Self::SocketConnected { fd, addr } - } - ArchivedJournalEntry::SocketAcceptedV1 { + JournalBatchEntry::SocketListenV1 { fd, backlog } => Self::SocketListen { fd, backlog }, + JournalBatchEntry::SocketBindV1 { fd, addr } => Self::SocketBind { fd, addr }, + JournalBatchEntry::SocketConnectedV1 { fd, addr } => Self::SocketConnected { fd, addr }, + JournalBatchEntry::SocketAcceptedV1 { listen_fd, fd, peer_addr, @@ -1584,7 +1787,7 @@ impl<'a> From for JournalEntry<'a> { fd_flags: Fdflags::from_bits_truncate(fd_flags), nonblocking, }, - ArchivedJournalEntry::SocketJoinIpv4MulticastV1 { + JournalBatchEntry::SocketJoinIpv4MulticastV1 { fd, multiaddr, iface, @@ -1593,7 +1796,7 @@ impl<'a> From for JournalEntry<'a> { multiaddr, iface, }, - ArchivedJournalEntry::SocketJoinIpv6MulticastV1 { + JournalBatchEntry::SocketJoinIpv6MulticastV1 { fd, multiaddr, iface, @@ -1602,7 +1805,7 @@ impl<'a> From for JournalEntry<'a> { multiaddr, iface, }, - ArchivedJournalEntry::SocketLeaveIpv4MulticastV1 { + JournalBatchEntry::SocketLeaveIpv4MulticastV1 { fd, multiaddr, iface, @@ -1611,7 +1814,7 @@ impl<'a> From for JournalEntry<'a> { multiaddr, iface, }, - ArchivedJournalEntry::SocketLeaveIpv6MulticastV1 { + JournalBatchEntry::SocketLeaveIpv6MulticastV1 { fd, multiaddr, iface, @@ -1620,7 +1823,7 @@ impl<'a> From for JournalEntry<'a> { multiaddr, iface, }, - ArchivedJournalEntry::SocketSendFileV1 { + JournalBatchEntry::SocketSendFileV1 { socket_fd, file_fd, offset, @@ -1631,7 +1834,7 @@ impl<'a> From for JournalEntry<'a> { offset, count, }, - ArchivedJournalEntry::SocketSendToV1 { + JournalBatchEntry::SocketSendToV1 { fd, data, flags, @@ -1644,7 +1847,7 @@ impl<'a> From for JournalEntry<'a> { addr, is_64bit, }, - ArchivedJournalEntry::SocketSendV1 { + JournalBatchEntry::SocketSendV1 { fd, data, flags, @@ -1655,26 +1858,26 @@ impl<'a> From for JournalEntry<'a> { flags, is_64bit, }, - ArchivedJournalEntry::SocketSetOptFlagV1 { fd, opt, flag } => Self::SocketSetOptFlag { + JournalBatchEntry::SocketSetOptFlagV1 { fd, opt, flag } => Self::SocketSetOptFlag { fd, opt: opt.into(), flag, }, - ArchivedJournalEntry::SocketSetOptSizeV1 { fd, opt, size } => Self::SocketSetOptSize { + JournalBatchEntry::SocketSetOptSizeV1 { fd, opt, size } => Self::SocketSetOptSize { fd, opt: opt.into(), size, }, - ArchivedJournalEntry::SocketSetOptTimeV1 { fd, ty, time } => Self::SocketSetOptTime { + JournalBatchEntry::SocketSetOptTimeV1 { fd, ty, time } => Self::SocketSetOptTime { fd, ty: ty.into(), time, }, - ArchivedJournalEntry::SocketShutdownV1 { fd, how } => Self::SocketShutdown { + JournalBatchEntry::SocketShutdownV1 { fd, how } => Self::SocketShutdown { fd, how: how.into(), }, - ArchivedJournalEntry::CreateEventV1 { + JournalBatchEntry::CreateEventV1 { initial_val, flags, fd, @@ -1686,3 +1889,420 @@ impl<'a> From for JournalEntry<'a> { } } } + +impl<'a> From<&'a ArchivedJournalBatchEntry> for JournalEntry<'a> { + fn from(value: &'a ArchivedJournalBatchEntry) -> Self { + type A = ArchivedJournalBatchEntry; + match value { + A::InitModuleV1 { wasm_hash } => Self::InitModule { + wasm_hash: *wasm_hash, + }, + A::UpdateMemoryRegionV1 { start, end, data } => Self::UpdateMemoryRegion { + region: *start..*end, + data: Cow::Borrowed(data.as_ref()), + }, + A::ProcessExitV1 { exit_code } => Self::ProcessExit { + exit_code: exit_code.as_ref().map(|code| code.into()), + }, + A::SetThreadV1 { + id, + call_stack, + memory_stack, + store_data, + is_64bit, + } => Self::SetThread { + id: (*id).into(), + call_stack: Cow::Borrowed(call_stack.as_ref()), + memory_stack: Cow::Borrowed(memory_stack.as_ref()), + store_data: Cow::Borrowed(store_data.as_ref()), + is_64bit: *is_64bit, + }, + A::CloseThreadV1 { id, exit_code } => Self::CloseThread { + id: (*id).into(), + exit_code: exit_code.as_ref().map(|code| code.into()), + }, + A::FileDescriptorWriteV1 { + data, + fd, + offset, + is_64bit, + } => Self::FileDescriptorWrite { + data: Cow::Borrowed(data.as_ref()), + fd: *fd, + offset: *offset, + is_64bit: *is_64bit, + }, + A::FileDescriptorSeekV1 { fd, offset, whence } => Self::FileDescriptorSeek { + fd: *fd, + offset: *offset, + whence: whence.into(), + }, + A::OpenFileDescriptorV1 { + fd, + dirfd, + dirflags, + path, + o_flags, + fs_rights_base, + fs_rights_inheriting, + fs_flags, + } => Self::OpenFileDescriptor { + fd: *fd, + dirfd: *dirfd, + dirflags: *dirflags, + path: Cow::Borrowed(path.as_str()), + o_flags: wasi::Oflags::from_bits_truncate(*o_flags), + fs_rights_base: wasi::Rights::from_bits_truncate(*fs_rights_base), + fs_rights_inheriting: wasi::Rights::from_bits_truncate(*fs_rights_inheriting), + fs_flags: wasi::Fdflags::from_bits_truncate(*fs_flags), + }, + A::CloseFileDescriptorV1 { fd } => Self::CloseFileDescriptor { fd: *fd }, + A::RemoveDirectoryV1 { fd, path } => Self::RemoveDirectory { + fd: *fd, + path: Cow::Borrowed(path.as_str()), + }, + A::UnlinkFileV1 { fd, path } => Self::UnlinkFile { + fd: *fd, + path: Cow::Borrowed(path.as_str()), + }, + A::PathRenameV1 { + old_fd, + old_path, + new_fd, + new_path, + } => Self::PathRename { + old_fd: *old_fd, + old_path: Cow::Borrowed(old_path.as_str()), + new_fd: *new_fd, + new_path: Cow::Borrowed(new_path.as_str()), + }, + A::SnapshotV1 { + since_epoch, + trigger, + } => Self::Snapshot { + when: SystemTime::UNIX_EPOCH + .checked_add(Duration::from_nanos(since_epoch.as_nanos() as u64)) + .unwrap_or(SystemTime::UNIX_EPOCH), + trigger: trigger.into(), + }, + A::SetClockTimeV1 { clock_id, time } => Self::SetClockTime { + clock_id: clock_id.into(), + time: *time, + }, + A::RenumberFileDescriptorV1 { old_fd, new_fd } => Self::RenumberFileDescriptor { + old_fd: *old_fd, + new_fd: *new_fd, + }, + A::DuplicateFileDescriptorV1 { + original_fd, + copied_fd, + } => Self::DuplicateFileDescriptor { + original_fd: *original_fd, + copied_fd: *copied_fd, + }, + A::CreateDirectoryV1 { fd, path } => Self::CreateDirectory { + fd: *fd, + path: Cow::Borrowed(path.as_str()), + }, + A::PathSetTimesV1 { + fd, + path, + flags, + st_atim, + st_mtim, + fst_flags, + } => Self::PathSetTimes { + fd: *fd, + path: Cow::Borrowed(path.as_str()), + flags: *flags, + st_atim: *st_atim, + st_mtim: *st_mtim, + fst_flags: wasi::Fstflags::from_bits_truncate(*fst_flags), + }, + A::FileDescriptorSetTimesV1 { + fd, + st_atim, + st_mtim, + fst_flags, + } => Self::FileDescriptorSetTimes { + fd: *fd, + st_atim: *st_atim, + st_mtim: *st_mtim, + fst_flags: wasi::Fstflags::from_bits_truncate(*fst_flags), + }, + A::FileDescriptorSetSizeV1 { fd, st_size } => Self::FileDescriptorSetSize { + fd: *fd, + st_size: *st_size, + }, + A::FileDescriptorSetFlagsV1 { fd, flags } => Self::FileDescriptorSetFlags { + fd: *fd, + flags: Fdflags::from_bits_truncate(*flags), + }, + A::FileDescriptorSetRightsV1 { + fd, + fs_rights_base, + fs_rights_inheriting, + } => Self::FileDescriptorSetRights { + fd: *fd, + fs_rights_base: Rights::from_bits_truncate(*fs_rights_base), + fs_rights_inheriting: Rights::from_bits_truncate(*fs_rights_inheriting), + }, + A::FileDescriptorAdviseV1 { + fd, + offset, + len, + advice, + } => Self::FileDescriptorAdvise { + fd: *fd, + offset: *offset, + len: *len, + advice: advice.into(), + }, + A::FileDescriptorAllocateV1 { fd, offset, len } => Self::FileDescriptorAllocate { + fd: *fd, + offset: *offset, + len: *len, + }, + A::CreateHardLinkV1 { + old_fd, + old_path, + old_flags, + new_fd, + new_path, + } => Self::CreateHardLink { + old_fd: *old_fd, + old_path: Cow::Borrowed(old_path.as_str()), + old_flags: *old_flags, + new_fd: *new_fd, + new_path: Cow::Borrowed(new_path.as_str()), + }, + A::CreateSymbolicLinkV1 { + old_path, + fd, + new_path, + } => Self::CreateSymbolicLink { + old_path: Cow::Borrowed(old_path.as_str()), + fd: *fd, + new_path: Cow::Borrowed(new_path.as_str()), + }, + A::ChangeDirectoryV1 { path } => Self::ChangeDirectory { + path: Cow::Borrowed(path.as_str()), + }, + A::EpollCreateV1 { fd } => Self::EpollCreate { fd: *fd }, + A::EpollCtlV1 { + epfd, + op, + fd, + event, + } => Self::EpollCtl { + epfd: *epfd, + op: op.into(), + fd: *fd, + event: event.as_ref().map(|e| e.into()), + }, + A::TtySetV1 { + cols, + rows, + width, + height, + stdin_tty, + stdout_tty, + stderr_tty, + echo, + line_buffered, + line_feeds, + } => Self::TtySet { + tty: wasi::Tty { + cols: *cols, + rows: *rows, + width: *width, + height: *height, + stdin_tty: *stdin_tty, + stdout_tty: *stdout_tty, + stderr_tty: *stderr_tty, + echo: *echo, + line_buffered: *line_buffered, + }, + line_feeds: *line_feeds, + }, + A::CreatePipeV1 { fd1, fd2 } => Self::CreatePipe { + fd1: *fd1, + fd2: *fd2, + }, + A::PortAddAddrV1 { cidr } => Self::PortAddAddr { + cidr: IpCidr { + ip: cidr.ip.as_ipaddr(), + prefix: cidr.prefix, + }, + }, + A::PortDelAddrV1 { addr } => Self::PortDelAddr { + addr: addr.as_ipaddr(), + }, + A::PortAddrClearV1 => Self::PortAddrClear, + A::PortBridgeV1 { + network, + token, + security, + } => Self::PortBridge { + network: Cow::Borrowed(network.as_str()), + token: Cow::Borrowed(token.as_str()), + security: security.into(), + }, + A::PortUnbridgeV1 => Self::PortUnbridge, + A::PortDhcpAcquireV1 => Self::PortDhcpAcquire, + A::PortGatewaySetV1 { ip } => Self::PortGatewaySet { ip: ip.as_ipaddr() }, + A::PortRouteAddV1 { + cidr, + via_router, + preferred_until, + expires_at, + } => Self::PortRouteAdd { + cidr: IpCidr { + ip: cidr.ip.as_ipaddr(), + prefix: cidr.prefix, + }, + via_router: via_router.as_ipaddr(), + preferred_until: preferred_until + .as_ref() + .map(|d| Duration::from_nanos(d.as_nanos() as u64)), + expires_at: expires_at + .as_ref() + .map(|d| Duration::from_nanos(d.as_nanos() as u64)), + }, + A::PortRouteClearV1 => Self::PortRouteClear, + A::PortRouteDelV1 { ip } => Self::PortRouteDel { ip: ip.as_ipaddr() }, + A::SocketOpenV1 { af, ty, pt, fd } => Self::SocketOpen { + af: af.into(), + ty: ty.into(), + pt: (*pt).try_into().unwrap_or(wasi::SockProto::Max), + fd: *fd, + }, + A::SocketListenV1 { fd, backlog } => Self::SocketListen { + fd: *fd, + backlog: *backlog, + }, + A::SocketBindV1 { fd, addr } => Self::SocketBind { + fd: *fd, + addr: addr.as_socket_addr(), + }, + A::SocketConnectedV1 { fd, addr } => Self::SocketConnected { + fd: *fd, + addr: addr.as_socket_addr(), + }, + A::SocketAcceptedV1 { + listen_fd, + fd, + peer_addr, + fd_flags, + nonblocking, + } => Self::SocketAccepted { + listen_fd: *listen_fd, + fd: *fd, + peer_addr: peer_addr.as_socket_addr(), + fd_flags: Fdflags::from_bits_truncate(*fd_flags), + nonblocking: *nonblocking, + }, + A::SocketJoinIpv4MulticastV1 { + fd, + multiaddr, + iface, + } => Self::SocketJoinIpv4Multicast { + fd: *fd, + multiaddr: multiaddr.as_ipv4(), + iface: iface.as_ipv4(), + }, + A::SocketJoinIpv6MulticastV1 { + fd, + multiaddr, + iface, + } => Self::SocketJoinIpv6Multicast { + fd: *fd, + multiaddr: multiaddr.as_ipv6(), + iface: *iface, + }, + A::SocketLeaveIpv4MulticastV1 { + fd, + multiaddr, + iface, + } => Self::SocketLeaveIpv4Multicast { + fd: *fd, + multiaddr: multiaddr.as_ipv4(), + iface: iface.as_ipv4(), + }, + A::SocketLeaveIpv6MulticastV1 { + fd, + multiaddr, + iface, + } => Self::SocketLeaveIpv6Multicast { + fd: *fd, + multiaddr: multiaddr.as_ipv6(), + iface: *iface, + }, + A::SocketSendFileV1 { + socket_fd, + file_fd, + offset, + count, + } => Self::SocketSendFile { + socket_fd: *socket_fd, + file_fd: *file_fd, + offset: *offset, + count: *count, + }, + A::SocketSendToV1 { + fd, + data, + flags, + addr, + is_64bit, + } => Self::SocketSendTo { + fd: *fd, + data: Cow::Borrowed(data.as_ref()), + flags: *flags, + addr: addr.as_socket_addr(), + is_64bit: *is_64bit, + }, + A::SocketSendV1 { + fd, + data, + flags, + is_64bit, + } => Self::SocketSend { + fd: *fd, + data: Cow::Borrowed(data.as_ref()), + flags: *flags, + is_64bit: *is_64bit, + }, + A::SocketSetOptFlagV1 { fd, opt, flag } => Self::SocketSetOptFlag { + fd: *fd, + opt: opt.into(), + flag: *flag, + }, + A::SocketSetOptSizeV1 { fd, opt, size } => Self::SocketSetOptSize { + fd: *fd, + opt: opt.into(), + size: *size, + }, + A::SocketSetOptTimeV1 { fd, ty, time } => Self::SocketSetOptTime { + fd: *fd, + ty: ty.into(), + time: time + .as_ref() + .map(|d| Duration::from_nanos(d.as_nanos() as u64)), + }, + A::SocketShutdownV1 { fd, how } => Self::SocketShutdown { + fd: *fd, + how: how.into(), + }, + A::CreateEventV1 { + initial_val, + flags, + fd, + } => Self::CreateEvent { + initial_val: *initial_val, + flags: *flags, + fd: *fd, + }, + } + } +} diff --git a/lib/wasix/src/journaling/concrete/compactor.rs b/lib/wasix/src/journaling/concrete/compactor.rs index 4789f39f544..4f8d203b423 100644 --- a/lib/wasix/src/journaling/concrete/compactor.rs +++ b/lib/wasix/src/journaling/concrete/compactor.rs @@ -147,22 +147,20 @@ impl Journal for CompactingJournal { }) } - fn read<'a>(&'a self) -> LocalBoxFuture<'_, anyhow::Result>>> { - Box::pin(async { - Ok(match self.inner.read().await? { - Some(JournalEntry::UpdateMemoryRegion { region, data }) => { - let mut hasher = Sha256::default(); - hasher.update(data.as_ref()); - let hash: [u8; 32] = hasher.finalize().try_into().unwrap(); + fn read<'a>(&'a self) -> anyhow::Result>> { + Ok(match self.inner.read()? { + Some(JournalEntry::UpdateMemoryRegion { region, data }) => { + let mut hasher = Sha256::default(); + hasher.update(data.as_ref()); + let hash: [u8; 32] = hasher.finalize().try_into().unwrap(); - let mut state = self.state.lock().unwrap(); - state.memory_map.insert(region.clone(), hash); + let mut state = self.state.lock().unwrap(); + state.memory_map.insert(region.clone(), hash); - Some(JournalEntry::UpdateMemoryRegion { region, data }) - } - Some(entry) => Some(entry), - None => None, - }) + Some(JournalEntry::UpdateMemoryRegion { region, data }) + } + Some(entry) => Some(entry), + None => None, }) } } diff --git a/lib/wasix/src/journaling/concrete/filter.rs b/lib/wasix/src/journaling/concrete/filter.rs index 226d9d30249..572a07d4f76 100644 --- a/lib/wasix/src/journaling/concrete/filter.rs +++ b/lib/wasix/src/journaling/concrete/filter.rs @@ -155,7 +155,7 @@ impl Journal for FilteredJournal { }) } - fn read<'a>(&'a self) -> LocalBoxFuture<'_, anyhow::Result>>> { - Box::pin(async { self.inner.read().await }) + fn read<'a>(&'a self) -> anyhow::Result>> { + self.inner.read() } } diff --git a/lib/wasix/src/journaling/concrete/log_file.rs b/lib/wasix/src/journaling/concrete/log_file.rs index 67b8c52a1c8..83972605a49 100644 --- a/lib/wasix/src/journaling/concrete/log_file.rs +++ b/lib/wasix/src/journaling/concrete/log_file.rs @@ -1,18 +1,20 @@ +use bytes::Buf; +use shared_buffer::OwnedBuffer; use std::{ - io::{self, ErrorKind, SeekFrom}, - mem::MaybeUninit, + io::{Seek, SeekFrom}, path::Path, }; use tokio::runtime::Handle; +use virtual_fs::AsyncWriteExt; use futures::future::LocalBoxFuture; -use virtual_fs::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt}; use super::*; struct State { file: tokio::fs::File, - at_end: bool, + buffer_pos: usize, + record_pos: usize, } /// The LogFile snapshot capturer will write its snapshots to a linear journal @@ -27,62 +29,53 @@ struct State { /// The logfile snapshot capturer uses a 64bit number as a entry encoding /// delimiter. pub struct LogFileJournal { - state: tokio::sync::Mutex, + state: std::sync::Mutex, handle: Handle, + buffer: OwnedBuffer, } impl LogFileJournal { - pub async fn new(path: impl AsRef) -> io::Result { - let state = State { - file: tokio::fs::File::options() - .read(true) - .write(true) - .create(true) - .open(path) - .await?, - at_end: false, - }; - Ok(Self { - state: tokio::sync::Mutex::new(state), - handle: Handle::current(), - }) - } - - pub fn new_std(path: impl AsRef) -> io::Result { - let file = std::fs::File::options() + pub fn new(path: impl AsRef) -> anyhow::Result { + let mut file = std::fs::File::options() .read(true) .write(true) .create(true) .open(path)?; - let state = State { - file: tokio::fs::File::from_std(file), - at_end: false, - }; + let buffer = OwnedBuffer::from_file(&file)?; + + // Move the file to the end + file.seek(SeekFrom::End(0))?; + Ok(Self { - state: tokio::sync::Mutex::new(state), + state: std::sync::Mutex::new(State { + file: tokio::fs::File::from_std(file), + buffer_pos: 0, + record_pos: 0, + }), handle: Handle::current(), + buffer, }) } } +impl JournalBatch {} + #[async_trait::async_trait] impl Journal for LogFileJournal { fn write<'a>(&'a self, entry: JournalEntry<'a>) -> LocalBoxFuture<'a, anyhow::Result<()>> { tracing::debug!("journal event: {:?}", entry); Box::pin(async { - let entry: ArchivedJournalEntry = entry.into(); + // Create the batch + let batch = JournalBatch { + records: vec![entry.into()], + }; - let _guard = Handle::try_current().map_err(|_| self.handle.enter()); - let mut state = self.state.lock().await; - if !state.at_end { - state.file.seek(SeekFrom::End(0)).await?; - state.at_end = true; - } - - let data = bincode::serialize(&entry)?; + let data = rkyv::to_bytes::<_, 128>(&batch).unwrap(); let data_len = data.len() as u64; - let data_len = data_len.to_ne_bytes(); + let data_len = data_len.to_be_bytes(); + let _guard = Handle::try_current().map_err(|_| self.handle.enter()); + let mut state = self.state.lock().unwrap(); state.file.write_all(&data_len).await?; state.file.write_all(&data).await?; Ok(()) @@ -91,28 +84,59 @@ impl Journal for LogFileJournal { /// UNSAFE: This method uses unsafe operations to remove the need to zero /// the buffer before its read the log entries into it - fn read<'a>(&'a self) -> LocalBoxFuture<'_, anyhow::Result>>> { - Box::pin(async { - let mut state = self.state.lock().await; + fn read<'a>(&'a self) -> anyhow::Result>> { + let mut state = self.state.lock().unwrap(); - let mut data_len = [0u8; 8]; - match state.file.read_exact(&mut data_len).await { - Err(err) if err.kind() == ErrorKind::UnexpectedEof => return Ok(None), - Err(err) => return Err(err.into()), - Ok(_) => {} + // Get a memory reference to the data on the disk at + // the current read location + let mut buffer_ptr = self.buffer.as_ref(); + + // First we read the magic number for the archive + if state.buffer_pos == 0 { + if buffer_ptr.len() >= 8 { + let magic = u64::from_be_bytes(buffer_ptr[0..8].try_into().unwrap()); + if magic != JOURNAL_MAGIC_NUMBER { + return Err(anyhow::format_err!( + "invalid magic number of journal ({} vs {})", + magic, + JOURNAL_MAGIC_NUMBER + )); + } + state.buffer_pos += 8; + } else { + return Ok(None); } + } + buffer_ptr.advance(state.buffer_pos); - let data_len = u64::from_ne_bytes(data_len); - let mut data = Vec::with_capacity(data_len as usize); - let data_unsafe: &mut [MaybeUninit] = data.spare_capacity_mut(); - let data_unsafe: &mut [u8] = unsafe { std::mem::transmute(data_unsafe) }; - state.file.read_exact(data_unsafe).await?; - unsafe { - data.set_len(data_len as usize); + loop { + // Next we read the length of the current batch + if buffer_ptr.len() < 8 { + return Ok(None); + } + let batch_len = u64::from_be_bytes(buffer_ptr[0..8].try_into().unwrap()) as usize; + buffer_ptr.advance(8); + if batch_len == 0 || batch_len > buffer_ptr.len() { + return Ok(None); } - let entry: ArchivedJournalEntry = bincode::deserialize(&data)?; - Ok(Some(entry.into())) - }) + // Read the batch data itself + let batch = &buffer_ptr[..batch_len]; + let batch: &ArchivedJournalBatch = + unsafe { rkyv::archived_unsized_root::(batch) }; + + // If we have reached the end then move onto the next batch + if state.record_pos >= batch.records.len() { + buffer_ptr.advance(batch_len); + state.buffer_pos += batch_len; + state.record_pos = 0; + continue; + } + let record = &batch.records[state.record_pos]; + + // Otherwise we return the record and advance + state.record_pos += 1; + return Ok(Some(record.into())); + } } } diff --git a/lib/wasix/src/journaling/concrete/pipe.rs b/lib/wasix/src/journaling/concrete/pipe.rs index b19f6e742cd..ca3f8ec59b3 100644 --- a/lib/wasix/src/journaling/concrete/pipe.rs +++ b/lib/wasix/src/journaling/concrete/pipe.rs @@ -42,18 +42,16 @@ impl Journal for PipeJournal { }) } - fn read<'a>(&'a self) -> LocalBoxFuture<'_, anyhow::Result>>> { - Box::pin(async { - let mut rx = self.rx.lock().unwrap(); - match rx.try_recv() { - Ok(e) => Ok(Some(e.into())), - Err(TryRecvError::Empty) => Ok(None), - Err(TryRecvError::Disconnected) => { - return Err(anyhow::format_err!( - "failed to receive journal event from the pipe as its disconnected" - )) - } + fn read<'a>(&'a self) -> anyhow::Result>> { + let mut rx = self.rx.lock().unwrap(); + match rx.try_recv() { + Ok(e) => Ok(Some(e.into())), + Err(TryRecvError::Empty) => Ok(None), + Err(TryRecvError::Disconnected) => { + return Err(anyhow::format_err!( + "failed to receive journal event from the pipe as its disconnected" + )) } - }) + } } } diff --git a/lib/wasix/src/journaling/concrete/unsupported.rs b/lib/wasix/src/journaling/concrete/unsupported.rs index df58d934828..fcfa5f2e6b0 100644 --- a/lib/wasix/src/journaling/concrete/unsupported.rs +++ b/lib/wasix/src/journaling/concrete/unsupported.rs @@ -15,7 +15,7 @@ impl Journal for UnsupportedJournal { Box::pin(async { Err(anyhow::format_err!("unsupported")) }) } - fn read<'a>(&'a self) -> LocalBoxFuture<'_, anyhow::Result>>> { - Box::pin(async { Ok(None) }) + fn read<'a>(&'a self) -> anyhow::Result>> { + Ok(None) } } diff --git a/lib/wasix/src/journaling/journal.rs b/lib/wasix/src/journaling/journal.rs index 8eefdf05d2a..11f5a345b38 100644 --- a/lib/wasix/src/journaling/journal.rs +++ b/lib/wasix/src/journaling/journal.rs @@ -647,7 +647,7 @@ pub trait Journal { /// Returns a stream of snapshot objects that the runtime will use /// to restore the state of a WASM process to a previous moment in time - fn read<'a>(&'a self) -> LocalBoxFuture<'_, anyhow::Result>>>; + fn read<'a>(&'a self) -> anyhow::Result>>; } pub type DynJournal = dyn Journal + Send + Sync; diff --git a/lib/wasix/src/syscalls/mod.rs b/lib/wasix/src/syscalls/mod.rs index 71e96342f8c..87d226911bc 100644 --- a/lib/wasix/src/syscalls/mod.rs +++ b/lib/wasix/src/syscalls/mod.rs @@ -1298,455 +1298,443 @@ pub fn restore_snapshot( ) -> Result { use crate::journaling::Journal; - InlineWaker::block_on(async { - let mut is_same_module = false; - let mut rewind = None; - while let Some(next) = journal.read().await.map_err(anyhow_err_to_runtime_err)? { - tracing::trace!("Restoring snapshot event - {next:?}"); - match next { - crate::journaling::JournalEntry::InitModule { wasm_hash } => { - is_same_module = ctx.data().process.module_hash.as_bytes() == wasm_hash; - } - crate::journaling::JournalEntry::ProcessExit { exit_code } => { - JournalEffector::apply_process_exit(ctx.data(), exit_code) - .map_err(anyhow_err_to_runtime_err)?; - } - crate::journaling::JournalEntry::FileDescriptorWrite { - fd, - offset, - data, - is_64bit, - } => { + let mut is_same_module = false; + let mut rewind = None; + while let Some(next) = journal.read().map_err(anyhow_err_to_runtime_err)? { + tracing::trace!("Restoring snapshot event - {next:?}"); + match next { + crate::journaling::JournalEntry::InitModule { wasm_hash } => { + is_same_module = ctx.data().process.module_hash.as_bytes() == wasm_hash; + } + crate::journaling::JournalEntry::ProcessExit { exit_code } => { + JournalEffector::apply_process_exit(ctx.data(), exit_code) + .map_err(anyhow_err_to_runtime_err)?; + } + crate::journaling::JournalEntry::FileDescriptorWrite { + fd, + offset, + data, + is_64bit, + } => { + InlineWaker::block_on(async { if is_64bit { JournalEffector::apply_fd_write::(&ctx, fd, offset, data).await } else { JournalEffector::apply_fd_write::(&ctx, fd, offset, data).await } + }) + .map_err(anyhow_err_to_runtime_err)?; + } + crate::journaling::JournalEntry::FileDescriptorSeek { fd, offset, whence } => { + InlineWaker::block_on(JournalEffector::apply_fd_seek(&mut ctx, fd, offset, whence)) .map_err(anyhow_err_to_runtime_err)?; + } + crate::journaling::JournalEntry::UpdateMemoryRegion { region, data } => { + if !is_same_module { + continue; } - crate::journaling::JournalEntry::FileDescriptorSeek { fd, offset, whence } => { - JournalEffector::apply_fd_seek(&mut ctx, fd, offset, whence) - .await - .map_err(anyhow_err_to_runtime_err)?; - } - crate::journaling::JournalEntry::UpdateMemoryRegion { region, data } => { - if !is_same_module { - continue; - } - JournalEffector::apply_memory(&mut ctx, region, &data) - .map_err(anyhow_err_to_runtime_err)?; - } - crate::journaling::JournalEntry::CloseThread { id, exit_code } => { - if id == ctx.data().tid() { - return Err(WasiRuntimeError::Runtime(RuntimeError::user( - anyhow::format_err!( - "Snapshot restoration has already closed the main thread." - ) - .into(), - ))); - } + JournalEffector::apply_memory(&mut ctx, region, &data) + .map_err(anyhow_err_to_runtime_err)?; + } + crate::journaling::JournalEntry::CloseThread { id, exit_code } => { + if id == ctx.data().tid() { + return Err(WasiRuntimeError::Runtime(RuntimeError::user( + anyhow::format_err!( + "Snapshot restoration has already closed the main thread." + ) + .into(), + ))); } - crate::journaling::JournalEntry::SetThread { - id, - call_stack, - memory_stack, - store_data, - is_64bit, - } => { - if !is_same_module { - continue; - } - if id == ctx.data().tid() { - rewind.replace(RewindState { - memory_stack: memory_stack.to_vec().into(), - rewind_stack: call_stack.to_vec().into(), - store_data: store_data.to_vec().into(), - is_64bit, - }); - } else { - return Err(WasiRuntimeError::Runtime(RuntimeError::user( - anyhow::format_err!( - "Snapshot restoration does not currently support multiple threads." - ) - .into(), - ))); - } + } + crate::journaling::JournalEntry::SetThread { + id, + call_stack, + memory_stack, + store_data, + is_64bit, + } => { + if !is_same_module { + continue; } - crate::journaling::JournalEntry::CloseFileDescriptor { fd } => { - JournalEffector::apply_fd_close(&mut ctx, fd) - .map_err(anyhow_err_to_runtime_err)?; + if id == ctx.data().tid() { + rewind.replace(RewindState { + memory_stack: memory_stack.to_vec().into(), + rewind_stack: call_stack.to_vec().into(), + store_data: store_data.to_vec().into(), + is_64bit, + }); + } else { + return Err(WasiRuntimeError::Runtime(RuntimeError::user( + anyhow::format_err!( + "Snapshot restoration does not currently support multiple threads." + ) + .into(), + ))); } - crate::journaling::JournalEntry::OpenFileDescriptor { + } + crate::journaling::JournalEntry::CloseFileDescriptor { fd } => { + JournalEffector::apply_fd_close(&mut ctx, fd).map_err(anyhow_err_to_runtime_err)?; + } + crate::journaling::JournalEntry::OpenFileDescriptor { + fd, + dirfd, + dirflags, + path, + o_flags, + fs_rights_base, + fs_rights_inheriting, + fs_flags, + } => { + JournalEffector::apply_path_open( + &mut ctx, fd, dirfd, dirflags, - path, + &path, o_flags, fs_rights_base, fs_rights_inheriting, fs_flags, - } => { - JournalEffector::apply_path_open( - &mut ctx, - fd, - dirfd, - dirflags, - &path, - o_flags, - fs_rights_base, - fs_rights_inheriting, - fs_flags, - ) + ) + .map_err(anyhow_err_to_runtime_err)?; + } + crate::journaling::JournalEntry::RemoveDirectory { fd, path } => { + JournalEffector::apply_path_remove_directory(&mut ctx, fd, &path) .map_err(anyhow_err_to_runtime_err)?; - } - crate::journaling::JournalEntry::RemoveDirectory { fd, path } => { - JournalEffector::apply_path_remove_directory(&mut ctx, fd, &path) - .map_err(anyhow_err_to_runtime_err)?; - } - crate::journaling::JournalEntry::UnlinkFile { fd, path } => { - JournalEffector::apply_path_unlink(&mut ctx, fd, &path) - .map_err(anyhow_err_to_runtime_err)?; - } - crate::journaling::JournalEntry::PathRename { - old_fd, - old_path, - new_fd, - new_path, - } => { - JournalEffector::apply_path_rename( - &mut ctx, old_fd, &old_path, new_fd, &new_path, - ) + } + crate::journaling::JournalEntry::UnlinkFile { fd, path } => { + JournalEffector::apply_path_unlink(&mut ctx, fd, &path) .map_err(anyhow_err_to_runtime_err)?; - } - crate::journaling::JournalEntry::Snapshot { - when: _, - trigger: _, - } => { - if !is_same_module { - continue; - } - } - crate::journaling::JournalEntry::SetClockTime { clock_id, time } => { - JournalEffector::apply_clock_time_set(&mut ctx, clock_id, time) - .map_err(anyhow_err_to_runtime_err)?; - } - crate::journaling::JournalEntry::RenumberFileDescriptor { old_fd, new_fd } => { - JournalEffector::apply_fd_renumber(&mut ctx, old_fd, new_fd) - .map_err(anyhow_err_to_runtime_err)?; - } - crate::journaling::JournalEntry::DuplicateFileDescriptor { - original_fd, - copied_fd, - } => { - JournalEffector::apply_fd_duplicate(&mut ctx, original_fd, copied_fd) - .map_err(anyhow_err_to_runtime_err)?; - } - crate::journaling::JournalEntry::CreateDirectory { fd, path } => { - JournalEffector::apply_path_create_directory(&mut ctx, fd, &path) - .map_err(anyhow_err_to_runtime_err)?; - } - crate::journaling::JournalEntry::PathSetTimes { - fd, - flags, - path, - st_atim, - st_mtim, - fst_flags, - } => { - JournalEffector::apply_path_set_times( - &mut ctx, fd, flags, &path, st_atim, st_mtim, fst_flags, - ) + } + crate::journaling::JournalEntry::PathRename { + old_fd, + old_path, + new_fd, + new_path, + } => { + JournalEffector::apply_path_rename(&mut ctx, old_fd, &old_path, new_fd, &new_path) .map_err(anyhow_err_to_runtime_err)?; + } + crate::journaling::JournalEntry::Snapshot { + when: _, + trigger: _, + } => { + if !is_same_module { + continue; } - crate::journaling::JournalEntry::FileDescriptorSetTimes { - fd, - st_atim, - st_mtim, - fst_flags, - } => { - JournalEffector::apply_fd_set_times(&mut ctx, fd, st_atim, st_mtim, fst_flags) - .map_err(anyhow_err_to_runtime_err)?; - } - crate::journaling::JournalEntry::FileDescriptorSetSize { fd, st_size } => { - JournalEffector::apply_fd_set_size(&mut ctx, fd, st_size) - .map_err(anyhow_err_to_runtime_err)?; - } - crate::journaling::JournalEntry::FileDescriptorSetFlags { fd, flags } => { - JournalEffector::apply_fd_set_flags(&mut ctx, fd, flags) - .map_err(anyhow_err_to_runtime_err)?; - } - crate::journaling::JournalEntry::FileDescriptorSetRights { + } + crate::journaling::JournalEntry::SetClockTime { clock_id, time } => { + JournalEffector::apply_clock_time_set(&mut ctx, clock_id, time) + .map_err(anyhow_err_to_runtime_err)?; + } + crate::journaling::JournalEntry::RenumberFileDescriptor { old_fd, new_fd } => { + JournalEffector::apply_fd_renumber(&mut ctx, old_fd, new_fd) + .map_err(anyhow_err_to_runtime_err)?; + } + crate::journaling::JournalEntry::DuplicateFileDescriptor { + original_fd, + copied_fd, + } => { + JournalEffector::apply_fd_duplicate(&mut ctx, original_fd, copied_fd) + .map_err(anyhow_err_to_runtime_err)?; + } + crate::journaling::JournalEntry::CreateDirectory { fd, path } => { + JournalEffector::apply_path_create_directory(&mut ctx, fd, &path) + .map_err(anyhow_err_to_runtime_err)?; + } + crate::journaling::JournalEntry::PathSetTimes { + fd, + flags, + path, + st_atim, + st_mtim, + fst_flags, + } => { + JournalEffector::apply_path_set_times( + &mut ctx, fd, flags, &path, st_atim, st_mtim, fst_flags, + ) + .map_err(anyhow_err_to_runtime_err)?; + } + crate::journaling::JournalEntry::FileDescriptorSetTimes { + fd, + st_atim, + st_mtim, + fst_flags, + } => { + JournalEffector::apply_fd_set_times(&mut ctx, fd, st_atim, st_mtim, fst_flags) + .map_err(anyhow_err_to_runtime_err)?; + } + crate::journaling::JournalEntry::FileDescriptorSetSize { fd, st_size } => { + JournalEffector::apply_fd_set_size(&mut ctx, fd, st_size) + .map_err(anyhow_err_to_runtime_err)?; + } + crate::journaling::JournalEntry::FileDescriptorSetFlags { fd, flags } => { + JournalEffector::apply_fd_set_flags(&mut ctx, fd, flags) + .map_err(anyhow_err_to_runtime_err)?; + } + crate::journaling::JournalEntry::FileDescriptorSetRights { + fd, + fs_rights_base, + fs_rights_inheriting, + } => { + JournalEffector::apply_fd_set_rights( + &mut ctx, fd, fs_rights_base, fs_rights_inheriting, - } => { - JournalEffector::apply_fd_set_rights( - &mut ctx, - fd, - fs_rights_base, - fs_rights_inheriting, - ) + ) + .map_err(anyhow_err_to_runtime_err)?; + } + crate::journaling::JournalEntry::FileDescriptorAdvise { + fd, + offset, + len, + advice, + } => { + JournalEffector::apply_fd_advise(&mut ctx, fd, offset, len, advice) .map_err(anyhow_err_to_runtime_err)?; - } - crate::journaling::JournalEntry::FileDescriptorAdvise { - fd, - offset, - len, - advice, - } => { - JournalEffector::apply_fd_advise(&mut ctx, fd, offset, len, advice) - .map_err(anyhow_err_to_runtime_err)?; - } - crate::journaling::JournalEntry::FileDescriptorAllocate { fd, offset, len } => { - JournalEffector::apply_fd_allocate(&mut ctx, fd, offset, len) - .map_err(anyhow_err_to_runtime_err)?; - } - crate::journaling::JournalEntry::CreateHardLink { - old_fd, - old_path, - old_flags, - new_fd, - new_path, - } => { - JournalEffector::apply_path_link( - &mut ctx, old_fd, old_flags, &old_path, new_fd, &new_path, - ) + } + crate::journaling::JournalEntry::FileDescriptorAllocate { fd, offset, len } => { + JournalEffector::apply_fd_allocate(&mut ctx, fd, offset, len) .map_err(anyhow_err_to_runtime_err)?; - } - crate::journaling::JournalEntry::CreateSymbolicLink { - old_path, - fd, - new_path, - } => { - JournalEffector::apply_path_symlink(&mut ctx, &old_path, fd, &new_path) - .map_err(anyhow_err_to_runtime_err)?; - } - crate::journaling::JournalEntry::ChangeDirectory { path } => { - JournalEffector::apply_chdir(&mut ctx, &path) - .map_err(anyhow_err_to_runtime_err)?; - } - crate::journaling::JournalEntry::CreatePipe { fd1, fd2 } => { - JournalEffector::apply_fd_pipe(&mut ctx, fd1, fd2) - .map_err(anyhow_err_to_runtime_err)?; - } - crate::journaling::JournalEntry::EpollCreate { fd } => { - JournalEffector::apply_epoll_create(&mut ctx, fd) - .map_err(anyhow_err_to_runtime_err)?; - } - crate::journaling::JournalEntry::EpollCtl { - epfd, - op, - fd, - event, - } => { - JournalEffector::apply_epoll_ctl(&mut ctx, epfd, op, fd, event) - .map_err(anyhow_err_to_runtime_err)?; - } - crate::journaling::JournalEntry::TtySet { tty, line_feeds } => { - JournalEffector::apply_tty_set( - &mut ctx, - crate::WasiTtyState { - cols: tty.cols, - rows: tty.rows, - width: tty.width, - height: tty.height, - stdin_tty: tty.stdin_tty, - stdout_tty: tty.stdout_tty, - stderr_tty: tty.stderr_tty, - echo: tty.echo, - line_buffered: tty.line_buffered, - line_feeds, - }, - ) + } + crate::journaling::JournalEntry::CreateHardLink { + old_fd, + old_path, + old_flags, + new_fd, + new_path, + } => { + JournalEffector::apply_path_link( + &mut ctx, old_fd, old_flags, &old_path, new_fd, &new_path, + ) + .map_err(anyhow_err_to_runtime_err)?; + } + crate::journaling::JournalEntry::CreateSymbolicLink { + old_path, + fd, + new_path, + } => { + JournalEffector::apply_path_symlink(&mut ctx, &old_path, fd, &new_path) .map_err(anyhow_err_to_runtime_err)?; - } - crate::journaling::JournalEntry::PortAddAddr { cidr } => { - JournalEffector::apply_port_addr_add(&mut ctx, cidr) - .map_err(anyhow_err_to_runtime_err)? - } - crate::journaling::JournalEntry::PortDelAddr { addr } => { - JournalEffector::apply_port_addr_remove(&mut ctx, addr) - .map_err(anyhow_err_to_runtime_err)? - } - crate::journaling::JournalEntry::PortAddrClear => { - JournalEffector::apply_port_addr_clear(&mut ctx) - .map_err(anyhow_err_to_runtime_err)? - } - crate::journaling::JournalEntry::PortBridge { - network, - token, - security, - } => JournalEffector::apply_port_bridge(&mut ctx, &network, &token, security) - .map_err(anyhow_err_to_runtime_err)?, - crate::journaling::JournalEntry::PortUnbridge => { - JournalEffector::apply_port_unbridge(&mut ctx) - .map_err(anyhow_err_to_runtime_err)? - } - crate::journaling::JournalEntry::PortDhcpAcquire => { - JournalEffector::apply_port_dhcp_acquire(&mut ctx) - .map_err(anyhow_err_to_runtime_err)? - } - crate::journaling::JournalEntry::PortGatewaySet { ip } => { - JournalEffector::apply_port_gateway_set(&mut ctx, ip) - .map_err(anyhow_err_to_runtime_err)? - } - crate::journaling::JournalEntry::PortRouteAdd { - cidr, - via_router, - preferred_until, - expires_at, - } => JournalEffector::apply_port_route_add( + } + crate::journaling::JournalEntry::ChangeDirectory { path } => { + JournalEffector::apply_chdir(&mut ctx, &path).map_err(anyhow_err_to_runtime_err)?; + } + crate::journaling::JournalEntry::CreatePipe { fd1, fd2 } => { + JournalEffector::apply_fd_pipe(&mut ctx, fd1, fd2) + .map_err(anyhow_err_to_runtime_err)?; + } + crate::journaling::JournalEntry::EpollCreate { fd } => { + JournalEffector::apply_epoll_create(&mut ctx, fd) + .map_err(anyhow_err_to_runtime_err)?; + } + crate::journaling::JournalEntry::EpollCtl { + epfd, + op, + fd, + event, + } => { + JournalEffector::apply_epoll_ctl(&mut ctx, epfd, op, fd, event) + .map_err(anyhow_err_to_runtime_err)?; + } + crate::journaling::JournalEntry::TtySet { tty, line_feeds } => { + JournalEffector::apply_tty_set( &mut ctx, - cidr, - via_router, - preferred_until, - expires_at, + crate::WasiTtyState { + cols: tty.cols, + rows: tty.rows, + width: tty.width, + height: tty.height, + stdin_tty: tty.stdin_tty, + stdout_tty: tty.stdout_tty, + stderr_tty: tty.stderr_tty, + echo: tty.echo, + line_buffered: tty.line_buffered, + line_feeds, + }, ) + .map_err(anyhow_err_to_runtime_err)?; + } + crate::journaling::JournalEntry::PortAddAddr { cidr } => { + JournalEffector::apply_port_addr_add(&mut ctx, cidr) + .map_err(anyhow_err_to_runtime_err)? + } + crate::journaling::JournalEntry::PortDelAddr { addr } => { + JournalEffector::apply_port_addr_remove(&mut ctx, addr) + .map_err(anyhow_err_to_runtime_err)? + } + crate::journaling::JournalEntry::PortAddrClear => { + JournalEffector::apply_port_addr_clear(&mut ctx) + .map_err(anyhow_err_to_runtime_err)? + } + crate::journaling::JournalEntry::PortBridge { + network, + token, + security, + } => JournalEffector::apply_port_bridge(&mut ctx, &network, &token, security) .map_err(anyhow_err_to_runtime_err)?, - crate::journaling::JournalEntry::PortRouteClear => { - JournalEffector::apply_port_route_clear(&mut ctx) - .map_err(anyhow_err_to_runtime_err)? - } - crate::journaling::JournalEntry::PortRouteDel { ip } => { - JournalEffector::apply_port_route_remove(&mut ctx, ip) - .map_err(anyhow_err_to_runtime_err)? - } - crate::journaling::JournalEntry::SocketOpen { af, ty, pt, fd } => { - JournalEffector::apply_sock_open(&mut ctx, af, ty, pt, fd) - .map_err(anyhow_err_to_runtime_err)? - } - crate::journaling::JournalEntry::SocketListen { fd, backlog } => { - JournalEffector::apply_sock_listen(&mut ctx, fd, backlog as usize) - .map_err(anyhow_err_to_runtime_err)? - } - crate::journaling::JournalEntry::SocketBind { fd, addr } => { - JournalEffector::apply_sock_bind(&mut ctx, fd, addr) - .map_err(anyhow_err_to_runtime_err)? - } - crate::journaling::JournalEntry::SocketConnected { fd, addr } => { - JournalEffector::apply_sock_connect(&mut ctx, fd, addr) - .map_err(anyhow_err_to_runtime_err)? - } - crate::journaling::JournalEntry::SocketAccepted { - listen_fd, - fd, - peer_addr, - fd_flags, - nonblocking, - } => JournalEffector::apply_sock_accepted( - &mut ctx, - listen_fd, - fd, - peer_addr, - fd_flags, - nonblocking, - ) + crate::journaling::JournalEntry::PortUnbridge => { + JournalEffector::apply_port_unbridge(&mut ctx).map_err(anyhow_err_to_runtime_err)? + } + crate::journaling::JournalEntry::PortDhcpAcquire => { + JournalEffector::apply_port_dhcp_acquire(&mut ctx) + .map_err(anyhow_err_to_runtime_err)? + } + crate::journaling::JournalEntry::PortGatewaySet { ip } => { + JournalEffector::apply_port_gateway_set(&mut ctx, ip) + .map_err(anyhow_err_to_runtime_err)? + } + crate::journaling::JournalEntry::PortRouteAdd { + cidr, + via_router, + preferred_until, + expires_at, + } => JournalEffector::apply_port_route_add( + &mut ctx, + cidr, + via_router, + preferred_until, + expires_at, + ) + .map_err(anyhow_err_to_runtime_err)?, + crate::journaling::JournalEntry::PortRouteClear => { + JournalEffector::apply_port_route_clear(&mut ctx) + .map_err(anyhow_err_to_runtime_err)? + } + crate::journaling::JournalEntry::PortRouteDel { ip } => { + JournalEffector::apply_port_route_remove(&mut ctx, ip) + .map_err(anyhow_err_to_runtime_err)? + } + crate::journaling::JournalEntry::SocketOpen { af, ty, pt, fd } => { + JournalEffector::apply_sock_open(&mut ctx, af, ty, pt, fd) + .map_err(anyhow_err_to_runtime_err)? + } + crate::journaling::JournalEntry::SocketListen { fd, backlog } => { + JournalEffector::apply_sock_listen(&mut ctx, fd, backlog as usize) + .map_err(anyhow_err_to_runtime_err)? + } + crate::journaling::JournalEntry::SocketBind { fd, addr } => { + JournalEffector::apply_sock_bind(&mut ctx, fd, addr) + .map_err(anyhow_err_to_runtime_err)? + } + crate::journaling::JournalEntry::SocketConnected { fd, addr } => { + JournalEffector::apply_sock_connect(&mut ctx, fd, addr) + .map_err(anyhow_err_to_runtime_err)? + } + crate::journaling::JournalEntry::SocketAccepted { + listen_fd, + fd, + peer_addr, + fd_flags, + nonblocking, + } => JournalEffector::apply_sock_accepted( + &mut ctx, + listen_fd, + fd, + peer_addr, + fd_flags, + nonblocking, + ) + .map_err(anyhow_err_to_runtime_err)?, + crate::journaling::JournalEntry::SocketJoinIpv4Multicast { + fd, + multiaddr, + iface, + } => JournalEffector::apply_sock_join_ipv4_multicast(&mut ctx, fd, multiaddr, iface) .map_err(anyhow_err_to_runtime_err)?, - crate::journaling::JournalEntry::SocketJoinIpv4Multicast { - fd, - multiaddr, - iface, - } => { - JournalEffector::apply_sock_join_ipv4_multicast(&mut ctx, fd, multiaddr, iface) - .map_err(anyhow_err_to_runtime_err)? - } - crate::journaling::JournalEntry::SocketJoinIpv6Multicast { - fd, - multiaddr, - iface, - } => { - JournalEffector::apply_sock_join_ipv6_multicast(&mut ctx, fd, multiaddr, iface) - .map_err(anyhow_err_to_runtime_err)? - } - crate::journaling::JournalEntry::SocketLeaveIpv4Multicast { - fd, - multiaddr, - iface, - } => { - JournalEffector::apply_sock_leave_ipv4_multicast(&mut ctx, fd, multiaddr, iface) - .map_err(anyhow_err_to_runtime_err)? - } - crate::journaling::JournalEntry::SocketLeaveIpv6Multicast { - fd, - multiaddr, - iface, - } => { - JournalEffector::apply_sock_leave_ipv6_multicast(&mut ctx, fd, multiaddr, iface) - .map_err(anyhow_err_to_runtime_err)? - } - crate::journaling::JournalEntry::SocketSendFile { - socket_fd, - file_fd, - offset, - count, - } => JournalEffector::apply_sock_send_file( - &mut ctx, socket_fd, file_fd, offset, count, - ) - .await + crate::journaling::JournalEntry::SocketJoinIpv6Multicast { + fd, + multiaddr, + iface, + } => JournalEffector::apply_sock_join_ipv6_multicast(&mut ctx, fd, multiaddr, iface) .map_err(anyhow_err_to_runtime_err)?, - crate::journaling::JournalEntry::SocketSendTo { - fd, - data, - flags, - addr, - is_64bit, - } => if is_64bit { + crate::journaling::JournalEntry::SocketLeaveIpv4Multicast { + fd, + multiaddr, + iface, + } => JournalEffector::apply_sock_leave_ipv4_multicast(&mut ctx, fd, multiaddr, iface) + .map_err(anyhow_err_to_runtime_err)?, + crate::journaling::JournalEntry::SocketLeaveIpv6Multicast { + fd, + multiaddr, + iface, + } => JournalEffector::apply_sock_leave_ipv6_multicast(&mut ctx, fd, multiaddr, iface) + .map_err(anyhow_err_to_runtime_err)?, + crate::journaling::JournalEntry::SocketSendFile { + socket_fd, + file_fd, + offset, + count, + } => InlineWaker::block_on(JournalEffector::apply_sock_send_file( + &mut ctx, socket_fd, file_fd, offset, count, + )) + .map_err(anyhow_err_to_runtime_err)?, + crate::journaling::JournalEntry::SocketSendTo { + fd, + data, + flags, + addr, + is_64bit, + } => InlineWaker::block_on(async { + if is_64bit { JournalEffector::apply_sock_send_to::(&ctx, fd, data, flags, addr) .await } else { JournalEffector::apply_sock_send_to::(&ctx, fd, data, flags, addr) .await } - .map_err(anyhow_err_to_runtime_err)?, - crate::journaling::JournalEntry::SocketSend { - fd, - data, - flags, - is_64bit, - } => if is_64bit { + }) + .map_err(anyhow_err_to_runtime_err)?, + crate::journaling::JournalEntry::SocketSend { + fd, + data, + flags, + is_64bit, + } => InlineWaker::block_on(async { + if is_64bit { JournalEffector::apply_sock_send::(&ctx, fd, data, flags).await } else { JournalEffector::apply_sock_send::(&ctx, fd, data, flags).await } - .map_err(anyhow_err_to_runtime_err)?, - crate::journaling::JournalEntry::SocketSetOptFlag { fd, opt, flag } => { - JournalEffector::apply_sock_set_opt_flag(&mut ctx, fd, opt, flag) - .map_err(anyhow_err_to_runtime_err)? - } - crate::journaling::JournalEntry::SocketSetOptSize { fd, opt, size } => { - JournalEffector::apply_sock_set_opt_size(&mut ctx, fd, opt, size) - .map_err(anyhow_err_to_runtime_err)? - } - crate::journaling::JournalEntry::SocketSetOptTime { fd, ty, time } => { - JournalEffector::apply_sock_set_opt_time(&mut ctx, fd, ty, time) - .map_err(anyhow_err_to_runtime_err)? - } - crate::journaling::JournalEntry::SocketShutdown { fd, how } => { - JournalEffector::apply_sock_shutdown(&mut ctx, fd, how) - .map_err(anyhow_err_to_runtime_err)? - } - crate::journaling::JournalEntry::CreateEvent { - initial_val, - flags, - fd, - } => JournalEffector::apply_fd_event(&mut ctx, initial_val, flags, fd) - .map_err(anyhow_err_to_runtime_err)?, + }) + .map_err(anyhow_err_to_runtime_err)?, + crate::journaling::JournalEntry::SocketSetOptFlag { fd, opt, flag } => { + JournalEffector::apply_sock_set_opt_flag(&mut ctx, fd, opt, flag) + .map_err(anyhow_err_to_runtime_err)? } + crate::journaling::JournalEntry::SocketSetOptSize { fd, opt, size } => { + JournalEffector::apply_sock_set_opt_size(&mut ctx, fd, opt, size) + .map_err(anyhow_err_to_runtime_err)? + } + crate::journaling::JournalEntry::SocketSetOptTime { fd, ty, time } => { + JournalEffector::apply_sock_set_opt_time(&mut ctx, fd, ty, time) + .map_err(anyhow_err_to_runtime_err)? + } + crate::journaling::JournalEntry::SocketShutdown { fd, how } => { + JournalEffector::apply_sock_shutdown(&mut ctx, fd, how) + .map_err(anyhow_err_to_runtime_err)? + } + crate::journaling::JournalEntry::CreateEvent { + initial_val, + flags, + fd, + } => JournalEffector::apply_fd_event(&mut ctx, initial_val, flags, fd) + .map_err(anyhow_err_to_runtime_err)?, } - // If we are not in the same module then we fire off an exit - // that simulates closing the process (hence keeps everything - // in a clean state) - if !is_same_module { - JournalEffector::apply_process_exit(ctx.data(), None) - .map_err(anyhow_err_to_runtime_err)?; - } else { - tracing::debug!( - "journal used on a different module - the process will simulate a restart." - ); - } + } + // If we are not in the same module then we fire off an exit + // that simulates closing the process (hence keeps everything + // in a clean state) + if !is_same_module { + JournalEffector::apply_process_exit(ctx.data(), None).map_err(anyhow_err_to_runtime_err)?; + } else { + tracing::debug!( + "journal used on a different module - the process will simulate a restart." + ); + } - rewind.ok_or(WasiRuntimeError::Runtime(RuntimeError::user(anyhow::format_err!("The restored snapshot journal does not have a thread stack events and hence we can not restore the state of the process.").into()))) - }) + rewind.ok_or(WasiRuntimeError::Runtime(RuntimeError::user(anyhow::format_err!("The restored snapshot journal does not have a thread stack events and hence we can not restore the state of the process.").into()))) } pub(crate) unsafe fn handle_rewind( From cbfd16706a448e235df16d70abc73cab106e8f9e Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Wed, 22 Nov 2023 14:13:56 +1100 Subject: [PATCH 063/129] Added unit tests for the journal and fixed the rkyv serializer --- Cargo.lock | 2 + lib/wasi-types/src/wasi/bindings.rs | 2 +- lib/wasix/Cargo.toml | 3 + .../journaling/concrete/archived_journal.rs | 5046 ++++++++++------- .../src/journaling/concrete/compactor.rs | 190 +- lib/wasix/src/journaling/concrete/filter.rs | 176 +- lib/wasix/src/journaling/concrete/log_file.rs | 288 +- lib/wasix/src/journaling/concrete/pipe.rs | 33 +- .../src/journaling/concrete/unsupported.rs | 8 +- .../effector/memory_and_snapshot.rs | 50 +- lib/wasix/src/journaling/effector/mod.rs | 4 +- .../src/journaling/effector/process_exit.rs | 11 +- .../src/journaling/effector/save_event.rs | 14 +- .../journaling/effector/syscalls/fd_write.rs | 51 +- .../journaling/effector/syscalls/sock_send.rs | 51 +- .../effector/syscalls/sock_send_to.rs | 53 +- .../src/journaling/effector/thread_exit.rs | 11 +- lib/wasix/src/journaling/journal.rs | 5 +- lib/wasix/src/syscalls/mod.rs | 2 +- 19 files changed, 3596 insertions(+), 2404 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5757683ce80..72e7aaa97d5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6236,6 +6236,7 @@ dependencies = [ "lazy_static", "libc", "linked_hash_set", + "num_enum", "once_cell", "petgraph", "pin-project", @@ -6261,6 +6262,7 @@ dependencies = [ "tower-http", "tracing", "tracing-subscriber", + "tracing-test", "tracing-wasm", "typetag", "url", diff --git a/lib/wasi-types/src/wasi/bindings.rs b/lib/wasi-types/src/wasi/bindings.rs index bbf707997c6..acb2ca6c4d2 100644 --- a/lib/wasi-types/src/wasi/bindings.rs +++ b/lib/wasi-types/src/wasi/bindings.rs @@ -1188,7 +1188,7 @@ impl core::fmt::Debug for Whence { } } #[repr(C)] -#[derive(Copy, Clone)] +#[derive(Copy, Clone, PartialEq)] #[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] pub struct Tty { pub cols: u32, diff --git a/lib/wasix/Cargo.toml b/lib/wasix/Cargo.toml index c080c63a27f..f0751dac2e9 100644 --- a/lib/wasix/Cargo.toml +++ b/lib/wasix/Cargo.toml @@ -63,6 +63,7 @@ pin-project = "1.0.12" semver = "1.0.17" dashmap = "5.4.0" tempfile = "3.6.0" +num_enum = "0.5.7" # Used by the WCGI runner hyper = { version = "0.14", features = ["server", "stream"], optional = true } wcgi = { version = "0.1.2", optional = true } @@ -118,6 +119,7 @@ winapi = "0.3" wasmer = { path = "../api", version = "=4.2.3", default-features = false, features = ["wat", "js-serializable-module"] } tokio = { version = "1", features = [ "sync", "macros", "rt" ], default_features = false } pretty_assertions = "1.3.0" +tracing-test = "0.2.4" wasm-bindgen-test = "0.3.0" [target.'cfg(target_arch = "wasm32")'.dev-dependencies] @@ -141,6 +143,7 @@ sys-default = [ "sys", "logging", "host-fs", + "journal", "sys-poll", "sys-thread", "host-vnet", diff --git a/lib/wasix/src/journaling/concrete/archived_journal.rs b/lib/wasix/src/journaling/concrete/archived_journal.rs index 43cebd300c3..9dbb704c7f0 100644 --- a/lib/wasix/src/journaling/concrete/archived_journal.rs +++ b/lib/wasix/src/journaling/concrete/archived_journal.rs @@ -1,3 +1,5 @@ +use num_enum::{IntoPrimitive, TryFromPrimitive}; +use rkyv::ser::{ScratchSpace, Serializer}; use rkyv::{Archive, CheckBytes, Deserialize as RkyvDeserialize, Serialize as RkyvSerialize}; use serde; use std::borrow::Cow; @@ -11,2096 +13,2530 @@ use super::*; pub const JOURNAL_MAGIC_NUMBER: u64 = 0x310d6dd027362979; -/// Version of the archived journal +#[repr(u16)] #[derive( - Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, -)] -#[archive_attr(derive(CheckBytes))] -pub(crate) enum JournalControlBatchType { - Abort, - Commit, -} - -/// Represents a batch of journal log entries -#[allow(clippy::large_enum_variant)] -#[derive( - Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, -)] -#[archive_attr(derive(CheckBytes))] -pub(crate) struct JournalBatch { - pub records: Vec, -} - -/// The journal log entries are serializable which -/// allows them to be written directly to a file -/// -/// Note: This structure is versioned which allows for -/// changes to the journal entry types without having to -/// worry about backward and forward compatibility -#[allow(clippy::large_enum_variant)] -#[derive( - Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, + Debug, + Clone, + Copy, + PartialEq, + Eq, + IntoPrimitive, + TryFromPrimitive, + serde::Serialize, + serde::Deserialize, + RkyvSerialize, + RkyvDeserialize, + Archive, )] #[archive_attr(derive(CheckBytes))] -pub(crate) enum JournalBatchEntry { - InitModuleV1 { - wasm_hash: [u8; 32], - }, - ProcessExitV1 { - exit_code: Option, - }, - SetThreadV1 { - id: u32, - call_stack: Vec, - memory_stack: Vec, - store_data: Vec, - is_64bit: bool, - }, - CloseThreadV1 { - id: u32, - exit_code: Option, - }, - FileDescriptorSeekV1 { - fd: u32, - offset: i64, - whence: JournalWhenceV1, - }, - FileDescriptorWriteV1 { - fd: u32, - offset: u64, - data: Vec, - is_64bit: bool, - }, - UpdateMemoryRegionV1 { - start: u64, - end: u64, - data: Vec, - }, - SetClockTimeV1 { - clock_id: JournalSnapshot0ClockidV1, - time: u64, - }, - OpenFileDescriptorV1 { - fd: u32, - dirfd: u32, - dirflags: u32, - path: String, - o_flags: u16, - fs_rights_base: u64, - fs_rights_inheriting: u64, - fs_flags: u16, - }, - CloseFileDescriptorV1 { - fd: u32, - }, - RenumberFileDescriptorV1 { - old_fd: u32, - new_fd: u32, - }, - DuplicateFileDescriptorV1 { - original_fd: u32, - copied_fd: u32, - }, - CreateDirectoryV1 { - fd: u32, - path: String, - }, - RemoveDirectoryV1 { - fd: u32, - path: String, - }, - PathSetTimesV1 { - fd: u32, - flags: u32, - path: String, - st_atim: u64, - st_mtim: u64, - fst_flags: u16, - }, - FileDescriptorSetTimesV1 { - fd: u32, - st_atim: u64, - st_mtim: u64, - fst_flags: u16, - }, - FileDescriptorSetSizeV1 { - fd: u32, - st_size: u64, - }, - FileDescriptorSetFlagsV1 { - fd: u32, - flags: u16, - }, - FileDescriptorSetRightsV1 { - fd: u32, - fs_rights_base: u64, - fs_rights_inheriting: u64, - }, - FileDescriptorAdviseV1 { - fd: u32, - offset: u64, - len: u64, - advice: JournalAdviceV1, - }, - FileDescriptorAllocateV1 { - fd: u32, - offset: u64, - len: u64, - }, - CreateHardLinkV1 { - old_fd: u32, - old_path: String, - old_flags: u32, - new_fd: u32, - new_path: String, - }, - CreateSymbolicLinkV1 { - old_path: String, - fd: u32, - new_path: String, - }, - UnlinkFileV1 { - fd: u32, - path: String, - }, - PathRenameV1 { - old_fd: u32, - old_path: String, - new_fd: u32, - new_path: String, - }, - ChangeDirectoryV1 { - path: String, - }, - EpollCreateV1 { - fd: u32, - }, - EpollCtlV1 { - epfd: u32, - op: JournalEpollCtlV1, - fd: u32, - event: Option, - }, - TtySetV1 { - cols: u32, - rows: u32, - width: u32, - height: u32, - stdin_tty: bool, - stdout_tty: bool, - stderr_tty: bool, - echo: bool, - line_buffered: bool, - line_feeds: bool, - }, - CreatePipeV1 { - fd1: u32, - fd2: u32, - }, - CreateEventV1 { - initial_val: u64, - flags: u16, - fd: u32, - }, - PortAddAddrV1 { - cidr: IpCidr, - }, - PortDelAddrV1 { - addr: IpAddr, - }, +pub enum JournalEntryRecordType { + InitModuleV1, + ProcessExitV1, + SetThreadV1, + CloseThreadV1, + FileDescriptorSeekV1, + FileDescriptorWriteV1, + UpdateMemoryRegionV1, + SetClockTimeV1, + OpenFileDescriptorV1, + CloseFileDescriptorV1, + RenumberFileDescriptorV1, + DuplicateFileDescriptorV1, + CreateDirectoryV1, + RemoveDirectoryV1, + PathSetTimesV1, + FileDescriptorSetTimesV1, + FileDescriptorSetSizeV1, + FileDescriptorSetFlagsV1, + FileDescriptorSetRightsV1, + FileDescriptorAdviseV1, + FileDescriptorAllocateV1, + CreateHardLinkV1, + CreateSymbolicLinkV1, + UnlinkFileV1, + PathRenameV1, + ChangeDirectoryV1, + EpollCreateV1, + EpollCtlV1, + TtySetV1, + CreatePipeV1, + CreateEventV1, + PortAddAddrV1, + PortDelAddrV1, PortAddrClearV1, - PortBridgeV1 { - network: String, - token: String, - security: JournalStreamSecurityV1, - }, + PortBridgeV1, PortUnbridgeV1, PortDhcpAcquireV1, - PortGatewaySetV1 { - ip: IpAddr, - }, - PortRouteAddV1 { - cidr: IpCidr, - via_router: IpAddr, - preferred_until: Option, - expires_at: Option, - }, + PortGatewaySetV1, + PortRouteAddV1, PortRouteClearV1, - PortRouteDelV1 { - ip: IpAddr, - }, - SocketOpenV1 { - af: JournalAddressfamilyV1, - ty: JournalSocktypeV1, - pt: u16, - fd: u32, - }, - SocketListenV1 { - fd: u32, - backlog: u32, - }, - SocketBindV1 { - fd: u32, - addr: SocketAddr, - }, - SocketConnectedV1 { - fd: u32, - addr: SocketAddr, - }, - SocketAcceptedV1 { - listen_fd: u32, - fd: u32, - peer_addr: SocketAddr, - fd_flags: u16, - nonblocking: bool, - }, - SocketJoinIpv4MulticastV1 { - fd: u32, - multiaddr: Ipv4Addr, - iface: Ipv4Addr, - }, - SocketJoinIpv6MulticastV1 { - fd: u32, - multiaddr: Ipv6Addr, - iface: u32, - }, - SocketLeaveIpv4MulticastV1 { - fd: u32, - multiaddr: Ipv4Addr, - iface: Ipv4Addr, - }, - SocketLeaveIpv6MulticastV1 { - fd: u32, - multiaddr: Ipv6Addr, - iface: u32, - }, - SocketSendFileV1 { - socket_fd: u32, - file_fd: u32, - offset: u64, - count: u64, - }, - SocketSendToV1 { - fd: u32, - data: Vec, - flags: u16, - addr: SocketAddr, - is_64bit: bool, - }, - SocketSendV1 { - fd: u32, - data: Vec, - flags: u16, - is_64bit: bool, - }, - SocketSetOptFlagV1 { - fd: u32, - opt: JournalSockoptionV1, - flag: bool, - }, - SocketSetOptSizeV1 { - fd: u32, - opt: JournalSockoptionV1, - size: u64, - }, - SocketSetOptTimeV1 { - fd: u32, - ty: JournalTimeTypeV1, - time: Option, - }, - SocketShutdownV1 { - fd: u32, - how: JournalSocketShutdownV1, - }, - SnapshotV1 { - since_epoch: Duration, - trigger: JournalSnapshotTriggerV1, - }, -} - -#[derive( - Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, -)] -#[archive_attr(derive(CheckBytes))] -pub(crate) enum JournalSnapshot0ClockidV1 { - Realtime, - Monotonic, - ProcessCputimeId, - ThreadCputimeId, - Unknown = 255, -} - -impl From for JournalSnapshot0ClockidV1 { - fn from(val: wasi::Snapshot0Clockid) -> Self { - match val { - wasi::Snapshot0Clockid::Realtime => JournalSnapshot0ClockidV1::Realtime, - wasi::Snapshot0Clockid::Monotonic => JournalSnapshot0ClockidV1::Monotonic, - wasi::Snapshot0Clockid::ProcessCputimeId => JournalSnapshot0ClockidV1::ProcessCputimeId, - wasi::Snapshot0Clockid::ThreadCputimeId => JournalSnapshot0ClockidV1::ThreadCputimeId, - wasi::Snapshot0Clockid::Unknown => JournalSnapshot0ClockidV1::Unknown, - } - } -} - -impl From for wasi::Snapshot0Clockid { - fn from(val: JournalSnapshot0ClockidV1) -> Self { - match val { - JournalSnapshot0ClockidV1::Realtime => wasi::Snapshot0Clockid::Realtime, - JournalSnapshot0ClockidV1::Monotonic => wasi::Snapshot0Clockid::Monotonic, - JournalSnapshot0ClockidV1::ProcessCputimeId => wasi::Snapshot0Clockid::ProcessCputimeId, - JournalSnapshot0ClockidV1::ThreadCputimeId => wasi::Snapshot0Clockid::ThreadCputimeId, - JournalSnapshot0ClockidV1::Unknown => wasi::Snapshot0Clockid::Unknown, - } - } + PortRouteDelV1, + SocketOpenV1, + SocketListenV1, + SocketBindV1, + SocketConnectedV1, + SocketAcceptedV1, + SocketJoinIpv4MulticastV1, + SocketJoinIpv6MulticastV1, + SocketLeaveIpv4MulticastV1, + SocketLeaveIpv6MulticastV1, + SocketSendFileV1, + SocketSendToV1, + SocketSendV1, + SocketSetOptFlagV1, + SocketSetOptSizeV1, + SocketSetOptTimeV1, + SocketShutdownV1, + SnapshotV1, } -impl From<&'_ ArchivedJournalSnapshot0ClockidV1> for wasi::Snapshot0Clockid { - fn from(val: &'_ ArchivedJournalSnapshot0ClockidV1) -> Self { - match val { - ArchivedJournalSnapshot0ClockidV1::Realtime => wasi::Snapshot0Clockid::Realtime, - ArchivedJournalSnapshot0ClockidV1::Monotonic => wasi::Snapshot0Clockid::Monotonic, - ArchivedJournalSnapshot0ClockidV1::ProcessCputimeId => { - wasi::Snapshot0Clockid::ProcessCputimeId +impl JournalEntryRecordType { + pub unsafe fn deserialize_archive<'a>(self, data: &'a [u8]) -> JournalEntry<'a> { + match self { + JournalEntryRecordType::InitModuleV1 => ArchivedJournalEntry::InitModuleV1( + rkyv::archived_root::(data), + ), + JournalEntryRecordType::ProcessExitV1 => ArchivedJournalEntry::ProcessExitV1( + rkyv::archived_root::(data), + ), + JournalEntryRecordType::SetThreadV1 => ArchivedJournalEntry::SetThreadV1( + rkyv::archived_root::(data), + ), + JournalEntryRecordType::CloseThreadV1 => ArchivedJournalEntry::CloseThreadV1( + rkyv::archived_root::(data), + ), + JournalEntryRecordType::FileDescriptorSeekV1 => { + ArchivedJournalEntry::FileDescriptorSeekV1(rkyv::archived_root::< + JournalEntryFileDescriptorSeekV1, + >(data)) } - ArchivedJournalSnapshot0ClockidV1::ThreadCputimeId => { - wasi::Snapshot0Clockid::ThreadCputimeId + JournalEntryRecordType::FileDescriptorWriteV1 => { + ArchivedJournalEntry::FileDescriptorWriteV1(rkyv::archived_root::< + JournalEntryFileDescriptorWriteV1, + >(data)) } - ArchivedJournalSnapshot0ClockidV1::Unknown => wasi::Snapshot0Clockid::Unknown, - } - } -} - -#[derive( - Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, -)] -#[archive_attr(derive(CheckBytes))] -pub(crate) enum JournalWhenceV1 { - Set, - Cur, - End, - Unknown = 255, -} - -impl From for JournalWhenceV1 { - fn from(val: wasi::Whence) -> Self { - match val { - wasi::Whence::Set => JournalWhenceV1::Set, - wasi::Whence::Cur => JournalWhenceV1::Cur, - wasi::Whence::End => JournalWhenceV1::End, - wasi::Whence::Unknown => JournalWhenceV1::Unknown, - } - } -} - -impl From for wasi::Whence { - fn from(val: JournalWhenceV1) -> Self { - match val { - JournalWhenceV1::Set => wasi::Whence::Set, - JournalWhenceV1::Cur => wasi::Whence::Cur, - JournalWhenceV1::End => wasi::Whence::End, - JournalWhenceV1::Unknown => wasi::Whence::Unknown, - } - } -} - -impl From<&'_ ArchivedJournalWhenceV1> for wasi::Whence { - fn from(val: &'_ ArchivedJournalWhenceV1) -> Self { - match val { - ArchivedJournalWhenceV1::Set => wasi::Whence::Set, - ArchivedJournalWhenceV1::Cur => wasi::Whence::Cur, - ArchivedJournalWhenceV1::End => wasi::Whence::End, - ArchivedJournalWhenceV1::Unknown => wasi::Whence::Unknown, - } - } -} - -#[derive( - Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, -)] -#[archive_attr(derive(CheckBytes))] -pub(crate) enum JournalAdviceV1 { - Normal, - Sequential, - Random, - Willneed, - Dontneed, - Noreuse, - Unknown = 255, -} - -impl From for JournalAdviceV1 { - fn from(val: wasi::Advice) -> Self { - match val { - wasi::Advice::Normal => JournalAdviceV1::Normal, - wasi::Advice::Sequential => JournalAdviceV1::Sequential, - wasi::Advice::Random => JournalAdviceV1::Random, - wasi::Advice::Willneed => JournalAdviceV1::Willneed, - wasi::Advice::Dontneed => JournalAdviceV1::Dontneed, - wasi::Advice::Noreuse => JournalAdviceV1::Noreuse, - wasi::Advice::Unknown => JournalAdviceV1::Unknown, - } - } -} - -impl From for wasi::Advice { - fn from(val: JournalAdviceV1) -> Self { - match val { - JournalAdviceV1::Normal => wasi::Advice::Normal, - JournalAdviceV1::Sequential => wasi::Advice::Sequential, - JournalAdviceV1::Random => wasi::Advice::Random, - JournalAdviceV1::Willneed => wasi::Advice::Willneed, - JournalAdviceV1::Dontneed => wasi::Advice::Dontneed, - JournalAdviceV1::Noreuse => wasi::Advice::Noreuse, - JournalAdviceV1::Unknown => wasi::Advice::Unknown, - } - } -} - -impl From<&'_ ArchivedJournalAdviceV1> for wasi::Advice { - fn from(val: &'_ ArchivedJournalAdviceV1) -> Self { - match val { - ArchivedJournalAdviceV1::Normal => wasi::Advice::Normal, - ArchivedJournalAdviceV1::Sequential => wasi::Advice::Sequential, - ArchivedJournalAdviceV1::Random => wasi::Advice::Random, - ArchivedJournalAdviceV1::Willneed => wasi::Advice::Willneed, - ArchivedJournalAdviceV1::Dontneed => wasi::Advice::Dontneed, - ArchivedJournalAdviceV1::Noreuse => wasi::Advice::Noreuse, - ArchivedJournalAdviceV1::Unknown => wasi::Advice::Unknown, - } - } -} - -#[derive( - Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, -)] -#[archive_attr(derive(CheckBytes))] -pub(crate) enum JournalExitCodeV1 { - Errno(u16), - Other(i32), -} - -impl From for JournalExitCodeV1 { - fn from(val: wasi::ExitCode) -> Self { - match val { - wasi::ExitCode::Errno(errno) => JournalExitCodeV1::Errno(errno as u16), - wasi::ExitCode::Other(id) => JournalExitCodeV1::Other(id), - } - } -} - -impl From for wasi::ExitCode { - fn from(val: JournalExitCodeV1) -> Self { - match val { - JournalExitCodeV1::Errno(errno) => { - wasi::ExitCode::Errno(errno.try_into().unwrap_or(wasi::Errno::Unknown)) + JournalEntryRecordType::UpdateMemoryRegionV1 => { + ArchivedJournalEntry::UpdateMemoryRegionV1(rkyv::archived_root::< + JournalEntryUpdateMemoryRegionV1, + >(data)) } - JournalExitCodeV1::Other(id) => wasi::ExitCode::Other(id), + JournalEntryRecordType::SetClockTimeV1 => { + ArchivedJournalEntry::SetClockTimeV1(rkyv::archived_root::< + JournalEntrySetClockTimeV1, + >(data)) + } + JournalEntryRecordType::OpenFileDescriptorV1 => { + ArchivedJournalEntry::OpenFileDescriptorV1(rkyv::archived_root::< + JournalEntryOpenFileDescriptorV1, + >(data)) + } + JournalEntryRecordType::CloseFileDescriptorV1 => { + ArchivedJournalEntry::CloseFileDescriptorV1(rkyv::archived_root::< + JournalEntryCloseFileDescriptorV1, + >(data)) + } + JournalEntryRecordType::RenumberFileDescriptorV1 => { + ArchivedJournalEntry::RenumberFileDescriptorV1(rkyv::archived_root::< + JournalEntryRenumberFileDescriptorV1, + >(data)) + } + JournalEntryRecordType::DuplicateFileDescriptorV1 => { + ArchivedJournalEntry::DuplicateFileDescriptorV1(rkyv::archived_root::< + JournalEntryDuplicateFileDescriptorV1, + >(data)) + } + JournalEntryRecordType::CreateDirectoryV1 => { + ArchivedJournalEntry::CreateDirectoryV1(rkyv::archived_root::< + JournalEntryCreateDirectoryV1, + >(data)) + } + JournalEntryRecordType::RemoveDirectoryV1 => { + ArchivedJournalEntry::RemoveDirectoryV1(rkyv::archived_root::< + JournalEntryRemoveDirectoryV1, + >(data)) + } + JournalEntryRecordType::PathSetTimesV1 => { + ArchivedJournalEntry::PathSetTimesV1(rkyv::archived_root::< + JournalEntryPathSetTimesV1, + >(data)) + } + JournalEntryRecordType::FileDescriptorSetTimesV1 => { + ArchivedJournalEntry::FileDescriptorSetTimesV1(rkyv::archived_root::< + JournalEntryFileDescriptorSetTimesV1, + >(data)) + } + JournalEntryRecordType::FileDescriptorSetSizeV1 => { + ArchivedJournalEntry::FileDescriptorSetSizeV1(rkyv::archived_root::< + JournalEntryFileDescriptorSetSizeV1, + >(data)) + } + JournalEntryRecordType::FileDescriptorSetFlagsV1 => { + ArchivedJournalEntry::FileDescriptorSetFlagsV1(rkyv::archived_root::< + JournalEntryFileDescriptorSetFlagsV1, + >(data)) + } + JournalEntryRecordType::FileDescriptorSetRightsV1 => { + ArchivedJournalEntry::FileDescriptorSetRightsV1(rkyv::archived_root::< + JournalEntryFileDescriptorSetRightsV1, + >(data)) + } + JournalEntryRecordType::FileDescriptorAdviseV1 => { + ArchivedJournalEntry::FileDescriptorAdviseV1(rkyv::archived_root::< + JournalEntryFileDescriptorAdviseV1, + >(data)) + } + JournalEntryRecordType::FileDescriptorAllocateV1 => { + ArchivedJournalEntry::FileDescriptorAllocateV1(rkyv::archived_root::< + JournalEntryFileDescriptorAllocateV1, + >(data)) + } + JournalEntryRecordType::CreateHardLinkV1 => { + ArchivedJournalEntry::CreateHardLinkV1(rkyv::archived_root::< + JournalEntryCreateHardLinkV1, + >(data)) + } + JournalEntryRecordType::CreateSymbolicLinkV1 => { + ArchivedJournalEntry::CreateSymbolicLinkV1(rkyv::archived_root::< + JournalEntryCreateSymbolicLinkV1, + >(data)) + } + JournalEntryRecordType::UnlinkFileV1 => ArchivedJournalEntry::UnlinkFileV1( + rkyv::archived_root::(data), + ), + JournalEntryRecordType::PathRenameV1 => ArchivedJournalEntry::PathRenameV1( + rkyv::archived_root::(data), + ), + JournalEntryRecordType::ChangeDirectoryV1 => { + ArchivedJournalEntry::ChangeDirectoryV1(rkyv::archived_root::< + JournalEntryChangeDirectoryV1, + >(data)) + } + JournalEntryRecordType::EpollCreateV1 => ArchivedJournalEntry::EpollCreateV1( + rkyv::archived_root::(data), + ), + JournalEntryRecordType::EpollCtlV1 => ArchivedJournalEntry::EpollCtlV1( + rkyv::archived_root::(data), + ), + JournalEntryRecordType::TtySetV1 => { + ArchivedJournalEntry::TtySetV1(rkyv::archived_root::(data)) + } + JournalEntryRecordType::CreatePipeV1 => ArchivedJournalEntry::CreatePipeV1( + rkyv::archived_root::(data), + ), + JournalEntryRecordType::CreateEventV1 => ArchivedJournalEntry::CreateEventV1( + rkyv::archived_root::(data), + ), + JournalEntryRecordType::PortAddAddrV1 => ArchivedJournalEntry::PortAddAddrV1( + rkyv::archived_root::(data), + ), + JournalEntryRecordType::PortDelAddrV1 => ArchivedJournalEntry::PortDelAddrV1( + rkyv::archived_root::(data), + ), + JournalEntryRecordType::PortAddrClearV1 => return JournalEntry::PortAddrClear, + JournalEntryRecordType::PortBridgeV1 => ArchivedJournalEntry::PortBridgeV1( + rkyv::archived_root::(data), + ), + JournalEntryRecordType::PortUnbridgeV1 => return JournalEntry::PortUnbridge, + JournalEntryRecordType::PortDhcpAcquireV1 => return JournalEntry::PortDhcpAcquire, + JournalEntryRecordType::PortGatewaySetV1 => { + ArchivedJournalEntry::PortGatewaySetV1(rkyv::archived_root::< + JournalEntryPortGatewaySetV1, + >(data)) + } + JournalEntryRecordType::PortRouteAddV1 => { + ArchivedJournalEntry::PortRouteAddV1(rkyv::archived_root::< + JournalEntryPortRouteAddV1, + >(data)) + } + JournalEntryRecordType::PortRouteClearV1 => return JournalEntry::PortRouteClear, + JournalEntryRecordType::PortRouteDelV1 => { + ArchivedJournalEntry::PortRouteDelV1(rkyv::archived_root::< + JournalEntryPortRouteDelV1, + >(data)) + } + JournalEntryRecordType::SocketOpenV1 => ArchivedJournalEntry::SocketOpenV1( + rkyv::archived_root::(data), + ), + JournalEntryRecordType::SocketListenV1 => { + ArchivedJournalEntry::SocketListenV1(rkyv::archived_root::< + JournalEntrySocketListenV1, + >(data)) + } + JournalEntryRecordType::SocketBindV1 => ArchivedJournalEntry::SocketBindV1( + rkyv::archived_root::(data), + ), + JournalEntryRecordType::SocketConnectedV1 => { + ArchivedJournalEntry::SocketConnectedV1(rkyv::archived_root::< + JournalEntrySocketConnectedV1, + >(data)) + } + JournalEntryRecordType::SocketAcceptedV1 => { + ArchivedJournalEntry::SocketAcceptedV1(rkyv::archived_root::< + JournalEntrySocketAcceptedV1, + >(data)) + } + JournalEntryRecordType::SocketJoinIpv4MulticastV1 => { + ArchivedJournalEntry::SocketJoinIpv4MulticastV1(rkyv::archived_root::< + JournalEntrySocketJoinIpv4MulticastV1, + >(data)) + } + JournalEntryRecordType::SocketJoinIpv6MulticastV1 => { + ArchivedJournalEntry::SocketJoinIpv6MulticastV1(rkyv::archived_root::< + JournalEntrySocketJoinIpv6MulticastV1, + >(data)) + } + JournalEntryRecordType::SocketLeaveIpv4MulticastV1 => { + ArchivedJournalEntry::SocketLeaveIpv4MulticastV1(rkyv::archived_root::< + JournalEntrySocketLeaveIpv4MulticastV1, + >(data)) + } + JournalEntryRecordType::SocketLeaveIpv6MulticastV1 => { + ArchivedJournalEntry::SocketLeaveIpv6MulticastV1(rkyv::archived_root::< + JournalEntrySocketLeaveIpv6MulticastV1, + >(data)) + } + JournalEntryRecordType::SocketSendFileV1 => { + ArchivedJournalEntry::SocketSendFileV1(rkyv::archived_root::< + JournalEntrySocketSendFileV1, + >(data)) + } + JournalEntryRecordType::SocketSendToV1 => { + ArchivedJournalEntry::SocketSendToV1(rkyv::archived_root::< + JournalEntrySocketSendToV1, + >(data)) + } + JournalEntryRecordType::SocketSendV1 => ArchivedJournalEntry::SocketSendV1( + rkyv::archived_root::(data), + ), + JournalEntryRecordType::SocketSetOptFlagV1 => { + ArchivedJournalEntry::SocketSetOptFlagV1(rkyv::archived_root::< + JournalEntrySocketSetOptFlagV1, + >(data)) + } + JournalEntryRecordType::SocketSetOptSizeV1 => { + ArchivedJournalEntry::SocketSetOptSizeV1(rkyv::archived_root::< + JournalEntrySocketSetOptSizeV1, + >(data)) + } + JournalEntryRecordType::SocketSetOptTimeV1 => { + ArchivedJournalEntry::SocketSetOptTimeV1(rkyv::archived_root::< + JournalEntrySocketSetOptTimeV1, + >(data)) + } + JournalEntryRecordType::SocketShutdownV1 => { + ArchivedJournalEntry::SocketShutdownV1(rkyv::archived_root::< + JournalEntrySocketShutdownV1, + >(data)) + } + JournalEntryRecordType::SnapshotV1 => ArchivedJournalEntry::SnapshotV1( + rkyv::archived_root::(data), + ), } + .into() } } -impl From<&'_ ArchivedJournalExitCodeV1> for wasi::ExitCode { - fn from(val: &'_ ArchivedJournalExitCodeV1) -> Self { - match val { - ArchivedJournalExitCodeV1::Errno(errno) => { - wasi::ExitCode::Errno((*errno).try_into().unwrap_or(wasi::Errno::Unknown)) +impl<'a> JournalEntry<'a> { + pub fn archive_record_type(&self) -> JournalEntryRecordType { + match self { + Self::InitModule { .. } => JournalEntryRecordType::InitModuleV1, + Self::UpdateMemoryRegion { .. } => JournalEntryRecordType::UpdateMemoryRegionV1, + Self::ProcessExit { .. } => JournalEntryRecordType::ProcessExitV1, + Self::SetThread { .. } => JournalEntryRecordType::SetThreadV1, + Self::CloseThread { .. } => JournalEntryRecordType::CloseThreadV1, + Self::FileDescriptorSeek { .. } => JournalEntryRecordType::FileDescriptorSeekV1, + Self::FileDescriptorWrite { .. } => JournalEntryRecordType::FileDescriptorWriteV1, + Self::SetClockTime { .. } => JournalEntryRecordType::SetClockTimeV1, + Self::CloseFileDescriptor { .. } => JournalEntryRecordType::CloseFileDescriptorV1, + Self::OpenFileDescriptor { .. } => JournalEntryRecordType::OpenFileDescriptorV1, + Self::RenumberFileDescriptor { .. } => JournalEntryRecordType::RenumberFileDescriptorV1, + Self::DuplicateFileDescriptor { .. } => { + JournalEntryRecordType::DuplicateFileDescriptorV1 } - ArchivedJournalExitCodeV1::Other(id) => wasi::ExitCode::Other(*id), + Self::CreateDirectory { .. } => JournalEntryRecordType::CreateDirectoryV1, + Self::RemoveDirectory { .. } => JournalEntryRecordType::RemoveDirectoryV1, + Self::PathSetTimes { .. } => JournalEntryRecordType::PathSetTimesV1, + Self::FileDescriptorSetTimes { .. } => JournalEntryRecordType::FileDescriptorSetTimesV1, + Self::FileDescriptorSetFlags { .. } => JournalEntryRecordType::FileDescriptorSetFlagsV1, + Self::FileDescriptorSetRights { .. } => { + JournalEntryRecordType::FileDescriptorSetRightsV1 + } + Self::FileDescriptorSetSize { .. } => JournalEntryRecordType::FileDescriptorSetSizeV1, + Self::FileDescriptorAdvise { .. } => JournalEntryRecordType::FileDescriptorAdviseV1, + Self::FileDescriptorAllocate { .. } => JournalEntryRecordType::FileDescriptorAllocateV1, + Self::CreateHardLink { .. } => JournalEntryRecordType::CreateHardLinkV1, + Self::CreateSymbolicLink { .. } => JournalEntryRecordType::CreateSymbolicLinkV1, + Self::UnlinkFile { .. } => JournalEntryRecordType::UnlinkFileV1, + Self::PathRename { .. } => JournalEntryRecordType::PathRenameV1, + Self::ChangeDirectory { .. } => JournalEntryRecordType::ChangeDirectoryV1, + Self::EpollCreate { .. } => JournalEntryRecordType::EpollCreateV1, + Self::EpollCtl { .. } => JournalEntryRecordType::EpollCtlV1, + Self::TtySet { .. } => JournalEntryRecordType::TtySetV1, + Self::CreatePipe { .. } => JournalEntryRecordType::CreatePipeV1, + Self::CreateEvent { .. } => JournalEntryRecordType::CreateEventV1, + Self::PortAddAddr { .. } => JournalEntryRecordType::PortAddAddrV1, + Self::PortDelAddr { .. } => JournalEntryRecordType::PortDelAddrV1, + Self::PortAddrClear => JournalEntryRecordType::PortAddrClearV1, + Self::PortBridge { .. } => JournalEntryRecordType::PortBridgeV1, + Self::PortUnbridge => JournalEntryRecordType::PortUnbridgeV1, + Self::PortDhcpAcquire => JournalEntryRecordType::PortDhcpAcquireV1, + Self::PortGatewaySet { .. } => JournalEntryRecordType::PortGatewaySetV1, + Self::PortRouteAdd { .. } => JournalEntryRecordType::PortRouteAddV1, + Self::PortRouteClear => JournalEntryRecordType::PortRouteClearV1, + Self::PortRouteDel { .. } => JournalEntryRecordType::PortRouteDelV1, + Self::SocketOpen { .. } => JournalEntryRecordType::SocketOpenV1, + Self::SocketListen { .. } => JournalEntryRecordType::SocketListenV1, + Self::SocketBind { .. } => JournalEntryRecordType::SocketBindV1, + Self::SocketConnected { .. } => JournalEntryRecordType::SocketConnectedV1, + Self::SocketAccepted { .. } => JournalEntryRecordType::SocketAcceptedV1, + Self::SocketJoinIpv4Multicast { .. } => { + JournalEntryRecordType::SocketJoinIpv4MulticastV1 + } + Self::SocketJoinIpv6Multicast { .. } => { + JournalEntryRecordType::SocketJoinIpv6MulticastV1 + } + Self::SocketLeaveIpv4Multicast { .. } => { + JournalEntryRecordType::SocketLeaveIpv4MulticastV1 + } + Self::SocketLeaveIpv6Multicast { .. } => { + JournalEntryRecordType::SocketLeaveIpv6MulticastV1 + } + Self::SocketSendFile { .. } => JournalEntryRecordType::SocketSendFileV1, + Self::SocketSendTo { .. } => JournalEntryRecordType::SocketSendToV1, + Self::SocketSend { .. } => JournalEntryRecordType::SocketSendV1, + Self::SocketSetOptFlag { .. } => JournalEntryRecordType::SocketSetOptFlagV1, + Self::SocketSetOptSize { .. } => JournalEntryRecordType::SocketSetOptSizeV1, + Self::SocketSetOptTime { .. } => JournalEntryRecordType::SocketSetOptTimeV1, + Self::SocketShutdown { .. } => JournalEntryRecordType::SocketShutdownV1, + Self::Snapshot { .. } => JournalEntryRecordType::SnapshotV1, } } -} -#[derive( - Debug, - Clone, - Copy, - PartialEq, - Eq, - PartialOrd, - Ord, - Hash, - serde::Serialize, - serde::Deserialize, - RkyvSerialize, - RkyvDeserialize, - Archive, -)] -#[archive_attr(derive(CheckBytes))] -pub(crate) enum JournalSnapshotTriggerV1 { - Idle, - Listen, - Environ, - Stdin, - Timer, - Sigint, - Sigalrm, - Sigtstp, - Sigstop, - NonDeterministicCall, -} - -impl From for JournalSnapshotTriggerV1 { - fn from(val: SnapshotTrigger) -> Self { - match val { - SnapshotTrigger::Idle => JournalSnapshotTriggerV1::Idle, - SnapshotTrigger::FirstListen => JournalSnapshotTriggerV1::Listen, - SnapshotTrigger::FirstEnviron => JournalSnapshotTriggerV1::Environ, - SnapshotTrigger::FirstStdin => JournalSnapshotTriggerV1::Stdin, - SnapshotTrigger::PeriodicInterval => JournalSnapshotTriggerV1::Timer, - SnapshotTrigger::Sigint => JournalSnapshotTriggerV1::Sigint, - SnapshotTrigger::Sigalrm => JournalSnapshotTriggerV1::Sigalrm, - SnapshotTrigger::Sigtstp => JournalSnapshotTriggerV1::Sigtstp, - SnapshotTrigger::Sigstop => JournalSnapshotTriggerV1::Sigstop, - SnapshotTrigger::NonDeterministicCall => JournalSnapshotTriggerV1::NonDeterministicCall, - } - } -} - -impl From for SnapshotTrigger { - fn from(val: JournalSnapshotTriggerV1) -> Self { - match val { - JournalSnapshotTriggerV1::Idle => SnapshotTrigger::Idle, - JournalSnapshotTriggerV1::Listen => SnapshotTrigger::FirstListen, - JournalSnapshotTriggerV1::Environ => SnapshotTrigger::FirstEnviron, - JournalSnapshotTriggerV1::Stdin => SnapshotTrigger::FirstStdin, - JournalSnapshotTriggerV1::Timer => SnapshotTrigger::PeriodicInterval, - JournalSnapshotTriggerV1::Sigint => SnapshotTrigger::Sigint, - JournalSnapshotTriggerV1::Sigalrm => SnapshotTrigger::Sigalrm, - JournalSnapshotTriggerV1::Sigtstp => SnapshotTrigger::Sigtstp, - JournalSnapshotTriggerV1::Sigstop => SnapshotTrigger::Sigstop, - JournalSnapshotTriggerV1::NonDeterministicCall => SnapshotTrigger::NonDeterministicCall, - } - } -} - -impl From<&'_ ArchivedJournalSnapshotTriggerV1> for SnapshotTrigger { - fn from(val: &'_ ArchivedJournalSnapshotTriggerV1) -> Self { - match val { - ArchivedJournalSnapshotTriggerV1::Idle => SnapshotTrigger::Idle, - ArchivedJournalSnapshotTriggerV1::Listen => SnapshotTrigger::FirstListen, - ArchivedJournalSnapshotTriggerV1::Environ => SnapshotTrigger::FirstEnviron, - ArchivedJournalSnapshotTriggerV1::Stdin => SnapshotTrigger::FirstStdin, - ArchivedJournalSnapshotTriggerV1::Timer => SnapshotTrigger::PeriodicInterval, - ArchivedJournalSnapshotTriggerV1::Sigint => SnapshotTrigger::Sigint, - ArchivedJournalSnapshotTriggerV1::Sigalrm => SnapshotTrigger::Sigalrm, - ArchivedJournalSnapshotTriggerV1::Sigtstp => SnapshotTrigger::Sigtstp, - ArchivedJournalSnapshotTriggerV1::Sigstop => SnapshotTrigger::Sigstop, - ArchivedJournalSnapshotTriggerV1::NonDeterministicCall => { - SnapshotTrigger::NonDeterministicCall + pub fn serialize_archive( + self, + serializer: &mut T, + ) -> anyhow::Result<()> + where + T::Error: std::fmt::Display, + { + match self { + JournalEntry::InitModule { wasm_hash } => { + serializer.serialize_value(&JournalEntryInitModuleV1 { wasm_hash }) + } + JournalEntry::UpdateMemoryRegion { region, data } => { + serializer.serialize_value(&JournalEntryUpdateMemoryRegionV1 { + start: region.start, + end: region.end, + data: data.into_owned(), + }) + } + JournalEntry::ProcessExit { exit_code } => { + serializer.serialize_value(&JournalEntryProcessExitV1 { + exit_code: exit_code.map(|e| e.into()), + }) + } + JournalEntry::SetThread { + id, + call_stack, + memory_stack, + store_data, + is_64bit, + } => serializer.serialize_value(&JournalEntrySetThreadV1 { + id: id.into(), + call_stack: call_stack.into_owned(), + memory_stack: memory_stack.into_owned(), + store_data: store_data.into_owned(), + is_64bit, + }), + JournalEntry::CloseThread { id, exit_code } => { + serializer.serialize_value(&JournalEntryCloseThreadV1 { + id: id.into(), + exit_code: exit_code.map(|e| e.into()), + }) + } + JournalEntry::FileDescriptorSeek { fd, offset, whence } => { + serializer.serialize_value(&JournalEntryFileDescriptorSeekV1 { + fd, + offset, + whence: whence.into(), + }) + } + JournalEntry::FileDescriptorWrite { + fd, + offset, + data, + is_64bit, + } => serializer.serialize_value(&JournalEntryFileDescriptorWriteV1 { + fd, + offset, + data: data.into_owned(), + is_64bit, + }), + JournalEntry::SetClockTime { clock_id, time } => { + serializer.serialize_value(&JournalEntrySetClockTimeV1 { + clock_id: clock_id.into(), + time, + }) + } + JournalEntry::CloseFileDescriptor { fd } => { + serializer.serialize_value(&JournalEntryCloseFileDescriptorV1 { fd }) + } + JournalEntry::OpenFileDescriptor { + fd, + dirfd, + dirflags, + path, + o_flags, + fs_rights_base, + fs_rights_inheriting, + fs_flags, + } => serializer.serialize_value(&JournalEntryOpenFileDescriptorV1 { + fd, + dirfd, + dirflags, + path: path.into_owned(), + o_flags: o_flags.bits(), + fs_rights_base: fs_rights_base.bits(), + fs_rights_inheriting: fs_rights_inheriting.bits(), + fs_flags: fs_flags.bits(), + }), + JournalEntry::RenumberFileDescriptor { old_fd, new_fd } => { + serializer.serialize_value(&JournalEntryRenumberFileDescriptorV1 { old_fd, new_fd }) + } + JournalEntry::DuplicateFileDescriptor { + original_fd, + copied_fd, + } => serializer.serialize_value(&JournalEntryDuplicateFileDescriptorV1 { + original_fd, + copied_fd, + }), + JournalEntry::CreateDirectory { fd, path } => { + serializer.serialize_value(&JournalEntryCreateDirectoryV1 { + fd, + path: path.into_owned(), + }) + } + JournalEntry::RemoveDirectory { fd, path } => { + serializer.serialize_value(&JournalEntryRemoveDirectoryV1 { + fd, + path: path.into_owned(), + }) + } + JournalEntry::PathSetTimes { + fd, + flags, + path, + st_atim, + st_mtim, + fst_flags, + } => serializer.serialize_value(&JournalEntryPathSetTimesV1 { + fd, + flags, + path: path.into_owned(), + st_atim, + st_mtim, + fst_flags: fst_flags.bits(), + }), + JournalEntry::FileDescriptorSetTimes { + fd, + st_atim, + st_mtim, + fst_flags, + } => serializer.serialize_value(&JournalEntryFileDescriptorSetTimesV1 { + fd, + st_atim, + st_mtim, + fst_flags: fst_flags.bits(), + }), + JournalEntry::FileDescriptorSetFlags { fd, flags } => { + serializer.serialize_value(&JournalEntryFileDescriptorSetFlagsV1 { + fd, + flags: flags.bits(), + }) + } + JournalEntry::FileDescriptorSetRights { + fd, + fs_rights_base, + fs_rights_inheriting, + } => serializer.serialize_value(&JournalEntryFileDescriptorSetRightsV1 { + fd, + fs_rights_base: fs_rights_base.bits(), + fs_rights_inheriting: fs_rights_inheriting.bits(), + }), + JournalEntry::FileDescriptorSetSize { fd, st_size } => { + serializer.serialize_value(&JournalEntryFileDescriptorSetSizeV1 { fd, st_size }) + } + JournalEntry::FileDescriptorAdvise { + fd, + offset, + len, + advice, + } => serializer.serialize_value(&JournalEntryFileDescriptorAdviseV1 { + fd, + offset, + len, + advice: advice.into(), + }), + JournalEntry::FileDescriptorAllocate { fd, offset, len } => serializer + .serialize_value(&JournalEntryFileDescriptorAllocateV1 { fd, offset, len }), + JournalEntry::CreateHardLink { + old_fd, + old_path, + old_flags, + new_fd, + new_path, + } => serializer.serialize_value(&JournalEntryCreateHardLinkV1 { + old_fd, + old_path: old_path.into_owned(), + old_flags, + new_fd, + new_path: new_path.into_owned(), + }), + JournalEntry::CreateSymbolicLink { + old_path, + fd, + new_path, + } => serializer.serialize_value(&JournalEntryCreateSymbolicLinkV1 { + old_path: old_path.into_owned(), + fd, + new_path: new_path.into_owned(), + }), + JournalEntry::UnlinkFile { fd, path } => { + serializer.serialize_value(&JournalEntryUnlinkFileV1 { + fd, + path: path.into_owned(), + }) + } + JournalEntry::PathRename { + old_fd, + old_path, + new_fd, + new_path, + } => serializer.serialize_value(&JournalEntryPathRenameV1 { + old_fd, + old_path: old_path.into_owned(), + new_fd, + new_path: new_path.into_owned(), + }), + JournalEntry::ChangeDirectory { path } => { + serializer.serialize_value(&JournalEntryChangeDirectoryV1 { + path: path.into_owned(), + }) + } + JournalEntry::EpollCreate { fd } => { + serializer.serialize_value(&JournalEntryEpollCreateV1 { fd }) + } + JournalEntry::EpollCtl { + epfd, + op, + fd, + event, + } => serializer.serialize_value(&JournalEntryEpollCtlV1 { + epfd, + op: op.into(), + fd, + event: event.map(|e| e.into()), + }), + JournalEntry::TtySet { tty, line_feeds } => { + serializer.serialize_value(&JournalEntryTtySetV1 { + cols: tty.cols, + rows: tty.rows, + width: tty.width, + height: tty.height, + stdin_tty: tty.stdin_tty, + stdout_tty: tty.stdout_tty, + stderr_tty: tty.stderr_tty, + echo: tty.echo, + line_buffered: tty.line_buffered, + line_feeds, + }) + } + JournalEntry::CreatePipe { fd1, fd2 } => { + serializer.serialize_value(&JournalEntryCreatePipeV1 { fd1, fd2 }) + } + JournalEntry::CreateEvent { + initial_val, + flags, + fd, + } => serializer.serialize_value(&JournalEntryCreateEventV1 { + initial_val, + flags, + fd, + }), + JournalEntry::PortAddAddr { cidr } => { + serializer.serialize_value(&JournalEntryPortAddAddrV1 { cidr }) + } + JournalEntry::PortDelAddr { addr } => { + serializer.serialize_value(&JournalEntryPortDelAddrV1 { addr }) + } + JournalEntry::PortAddrClear => return Ok(()), + JournalEntry::PortBridge { + network, + token, + security, + } => serializer.serialize_value(&JournalEntryPortBridgeV1 { + network: network.into_owned(), + token: token.into_owned(), + security: security.into(), + }), + JournalEntry::PortUnbridge => return Ok(()), + JournalEntry::PortDhcpAcquire => return Ok(()), + JournalEntry::PortGatewaySet { ip } => { + serializer.serialize_value(&JournalEntryPortGatewaySetV1 { ip }) + } + JournalEntry::PortRouteAdd { + cidr, + via_router, + preferred_until, + expires_at, + } => serializer.serialize_value(&JournalEntryPortRouteAddV1 { + cidr, + via_router, + preferred_until: preferred_until.clone(), + expires_at: expires_at.clone(), + }), + JournalEntry::PortRouteClear => return Ok(()), + JournalEntry::PortRouteDel { ip } => { + serializer.serialize_value(&JournalEntryPortRouteDelV1 { ip }) + } + JournalEntry::SocketOpen { af, ty, pt, fd } => { + serializer.serialize_value(&JournalEntrySocketOpenV1 { + af: af.into(), + ty: ty.into(), + pt: pt.into(), + fd, + }) + } + JournalEntry::SocketListen { fd, backlog } => { + serializer.serialize_value(&JournalEntrySocketListenV1 { fd, backlog }) + } + JournalEntry::SocketBind { fd, addr } => { + serializer.serialize_value(&JournalEntrySocketBindV1 { fd, addr }) + } + JournalEntry::SocketConnected { fd, addr } => { + serializer.serialize_value(&JournalEntrySocketConnectedV1 { fd, addr }) + } + JournalEntry::SocketAccepted { + listen_fd, + fd, + peer_addr, + fd_flags, + nonblocking, + } => serializer.serialize_value(&JournalEntrySocketAcceptedV1 { + listen_fd, + fd, + peer_addr, + fd_flags: fd_flags.bits(), + nonblocking, + }), + JournalEntry::SocketJoinIpv4Multicast { + fd, + multiaddr, + iface, + } => serializer.serialize_value(&JournalEntrySocketJoinIpv4MulticastV1 { + fd, + multiaddr, + iface, + }), + JournalEntry::SocketJoinIpv6Multicast { + fd, + multiaddr, + iface, + } => serializer.serialize_value(&JournalEntrySocketJoinIpv6MulticastV1 { + fd, + multiaddr, + iface, + }), + JournalEntry::SocketLeaveIpv4Multicast { + fd, + multiaddr, + iface, + } => serializer.serialize_value(&JournalEntrySocketLeaveIpv4MulticastV1 { + fd, + multiaddr, + iface, + }), + JournalEntry::SocketLeaveIpv6Multicast { + fd, + multiaddr, + iface, + } => serializer.serialize_value(&JournalEntrySocketLeaveIpv6MulticastV1 { + fd, + multiaddr, + iface, + }), + JournalEntry::SocketSendFile { + socket_fd, + file_fd, + offset, + count, + } => serializer.serialize_value(&JournalEntrySocketSendFileV1 { + socket_fd, + file_fd, + offset, + count, + }), + JournalEntry::SocketSendTo { + fd, + data, + flags, + addr, + is_64bit, + } => serializer.serialize_value(&JournalEntrySocketSendToV1 { + fd, + data: data.into_owned(), + flags, + addr, + is_64bit, + }), + JournalEntry::SocketSend { + fd, + data, + flags, + is_64bit, + } => serializer.serialize_value(&JournalEntrySocketSendV1 { + fd, + data: data.into_owned(), + flags, + is_64bit, + }), + JournalEntry::SocketSetOptFlag { fd, opt, flag } => { + serializer.serialize_value(&JournalEntrySocketSetOptFlagV1 { + fd, + opt: opt.into(), + flag, + }) + } + JournalEntry::SocketSetOptSize { fd, opt, size } => { + serializer.serialize_value(&JournalEntrySocketSetOptSizeV1 { + fd, + opt: opt.into(), + size, + }) + } + JournalEntry::SocketSetOptTime { fd, ty, time } => { + serializer.serialize_value(&JournalEntrySocketSetOptTimeV1 { + fd, + ty: ty.into(), + time: time.clone(), + }) + } + JournalEntry::SocketShutdown { fd, how } => { + serializer.serialize_value(&JournalEntrySocketShutdownV1 { + fd, + how: how.into(), + }) + } + JournalEntry::Snapshot { when, trigger } => { + serializer.serialize_value(&JournalEntrySnapshotV1 { + since_epoch: when + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap_or(Duration::ZERO), + trigger: trigger.into(), + }) } } + .map_err(|err| anyhow::format_err!("failed to serialize journal record - {}", err))?; + Ok(()) } } -#[derive( - Debug, - Clone, - Copy, - PartialEq, - Eq, - PartialOrd, - Ord, - Hash, - serde::Serialize, - serde::Deserialize, - RkyvSerialize, - RkyvDeserialize, - Archive, -)] -#[archive_attr(derive(CheckBytes))] -pub(crate) enum JournalEpollCtlV1 { - Add, - Mod, - Del, - Unknown, -} - -impl From for JournalEpollCtlV1 { - fn from(val: wasi::EpollCtl) -> Self { - match val { - wasi::EpollCtl::Add => JournalEpollCtlV1::Add, - wasi::EpollCtl::Mod => JournalEpollCtlV1::Mod, - wasi::EpollCtl::Del => JournalEpollCtlV1::Del, - wasi::EpollCtl::Unknown => JournalEpollCtlV1::Unknown, - } - } -} - -impl From for wasi::EpollCtl { - fn from(val: JournalEpollCtlV1) -> Self { - match val { - JournalEpollCtlV1::Add => wasi::EpollCtl::Add, - JournalEpollCtlV1::Mod => wasi::EpollCtl::Mod, - JournalEpollCtlV1::Del => wasi::EpollCtl::Del, - JournalEpollCtlV1::Unknown => wasi::EpollCtl::Unknown, - } - } -} - -impl From<&'_ ArchivedJournalEpollCtlV1> for wasi::EpollCtl { - fn from(val: &'_ ArchivedJournalEpollCtlV1) -> Self { - match val { - ArchivedJournalEpollCtlV1::Add => wasi::EpollCtl::Add, - ArchivedJournalEpollCtlV1::Mod => wasi::EpollCtl::Mod, - ArchivedJournalEpollCtlV1::Del => wasi::EpollCtl::Del, - ArchivedJournalEpollCtlV1::Unknown => wasi::EpollCtl::Unknown, - } - } -} - -#[derive( - Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, -)] -#[archive_attr(derive(CheckBytes))] -pub struct JournalEpollEventCtlV1 { - pub events: u32, - pub ptr: u64, - pub fd: u32, - pub data1: u32, - pub data2: u64, -} - -impl From for JournalEpollEventCtlV1 { - fn from(val: EpollEventCtl) -> Self { - JournalEpollEventCtlV1 { - events: val.events.bits(), - ptr: val.ptr, - fd: val.fd, - data1: val.data1, - data2: val.data2, - } - } -} - -impl From for EpollEventCtl { - fn from(val: JournalEpollEventCtlV1) -> Self { - Self { - events: EpollType::from_bits_truncate(val.events), - ptr: val.ptr, - fd: val.fd, - data1: val.data1, - data2: val.data2, - } - } -} - -impl From<&'_ ArchivedJournalEpollEventCtlV1> for EpollEventCtl { - fn from(val: &'_ ArchivedJournalEpollEventCtlV1) -> Self { - Self { - events: EpollType::from_bits_truncate(val.events), - ptr: val.ptr, - fd: val.fd, - data1: val.data1, - data2: val.data2, - } - } -} - -#[derive( - Debug, - Clone, - Copy, - PartialEq, - Eq, - PartialOrd, - Ord, - Hash, - serde::Serialize, - serde::Deserialize, - RkyvSerialize, - RkyvDeserialize, - Archive, -)] -#[archive_attr(derive(CheckBytes))] -pub enum JournalStreamSecurityV1 { - Unencrypted, - AnyEncryption, - ClassicEncryption, - DoubleEncryption, - Unknown, -} - -impl From for JournalStreamSecurityV1 { - fn from(val: StreamSecurity) -> Self { - match val { - StreamSecurity::Unencrypted => JournalStreamSecurityV1::Unencrypted, - StreamSecurity::AnyEncyption => JournalStreamSecurityV1::AnyEncryption, - StreamSecurity::ClassicEncryption => JournalStreamSecurityV1::ClassicEncryption, - StreamSecurity::DoubleEncryption => JournalStreamSecurityV1::DoubleEncryption, - } - } -} - -impl From for StreamSecurity { - fn from(val: JournalStreamSecurityV1) -> Self { - match val { - JournalStreamSecurityV1::Unencrypted => StreamSecurity::Unencrypted, - JournalStreamSecurityV1::AnyEncryption => StreamSecurity::AnyEncyption, - JournalStreamSecurityV1::ClassicEncryption => StreamSecurity::ClassicEncryption, - JournalStreamSecurityV1::DoubleEncryption => StreamSecurity::DoubleEncryption, - JournalStreamSecurityV1::Unknown => StreamSecurity::AnyEncyption, - } - } -} - -impl From<&'_ ArchivedJournalStreamSecurityV1> for StreamSecurity { - fn from(val: &'_ ArchivedJournalStreamSecurityV1) -> Self { - match val { - ArchivedJournalStreamSecurityV1::Unencrypted => StreamSecurity::Unencrypted, - ArchivedJournalStreamSecurityV1::AnyEncryption => StreamSecurity::AnyEncyption, - ArchivedJournalStreamSecurityV1::ClassicEncryption => StreamSecurity::ClassicEncryption, - ArchivedJournalStreamSecurityV1::DoubleEncryption => StreamSecurity::DoubleEncryption, - ArchivedJournalStreamSecurityV1::Unknown => StreamSecurity::AnyEncyption, - } - } -} - -#[derive( - Debug, - Clone, - Copy, - PartialEq, - Eq, - PartialOrd, - Ord, - Hash, - serde::Serialize, - serde::Deserialize, - RkyvSerialize, - RkyvDeserialize, - Archive, -)] -#[archive_attr(derive(CheckBytes))] -pub enum JournalAddressfamilyV1 { - Unspec, - Inet4, - Inet6, - Unix, -} - -impl From for JournalAddressfamilyV1 { - fn from(val: wasi::Addressfamily) -> Self { - match val { - wasi::Addressfamily::Unspec => JournalAddressfamilyV1::Unspec, - wasi::Addressfamily::Inet4 => JournalAddressfamilyV1::Inet4, - wasi::Addressfamily::Inet6 => JournalAddressfamilyV1::Inet6, - wasi::Addressfamily::Unix => JournalAddressfamilyV1::Unix, - } - } -} - -impl From for wasi::Addressfamily { - fn from(val: JournalAddressfamilyV1) -> Self { - match val { - JournalAddressfamilyV1::Unspec => wasi::Addressfamily::Unspec, - JournalAddressfamilyV1::Inet4 => wasi::Addressfamily::Inet4, - JournalAddressfamilyV1::Inet6 => wasi::Addressfamily::Inet6, - JournalAddressfamilyV1::Unix => wasi::Addressfamily::Unix, - } - } +/// The journal log entries are serializable which +/// allows them to be written directly to a file +/// +/// Note: This structure is versioned which allows for +/// changes to the journal entry types without having to +/// worry about backward and forward compatibility +#[derive( + Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, +)] +#[archive_attr(derive(CheckBytes))] +pub(crate) struct JournalEntryHeader { + pub record_type: u16, + pub record_size: u64, } -impl From<&'_ ArchivedJournalAddressfamilyV1> for wasi::Addressfamily { - fn from(val: &'_ ArchivedJournalAddressfamilyV1) -> Self { - match val { - ArchivedJournalAddressfamilyV1::Unspec => wasi::Addressfamily::Unspec, - ArchivedJournalAddressfamilyV1::Inet4 => wasi::Addressfamily::Inet4, - ArchivedJournalAddressfamilyV1::Inet6 => wasi::Addressfamily::Inet6, - ArchivedJournalAddressfamilyV1::Unix => wasi::Addressfamily::Unix, - } - } +pub enum ArchivedJournalEntry<'a> { + InitModuleV1(&'a ArchivedJournalEntryInitModuleV1), + ProcessExitV1(&'a ArchivedJournalEntryProcessExitV1), + SetThreadV1(&'a ArchivedJournalEntrySetThreadV1), + CloseThreadV1(&'a ArchivedJournalEntryCloseThreadV1), + FileDescriptorSeekV1(&'a ArchivedJournalEntryFileDescriptorSeekV1), + FileDescriptorWriteV1(&'a ArchivedJournalEntryFileDescriptorWriteV1), + UpdateMemoryRegionV1(&'a ArchivedJournalEntryUpdateMemoryRegionV1), + SetClockTimeV1(&'a ArchivedJournalEntrySetClockTimeV1), + OpenFileDescriptorV1(&'a ArchivedJournalEntryOpenFileDescriptorV1), + CloseFileDescriptorV1(&'a ArchivedJournalEntryCloseFileDescriptorV1), + RenumberFileDescriptorV1(&'a ArchivedJournalEntryRenumberFileDescriptorV1), + DuplicateFileDescriptorV1(&'a ArchivedJournalEntryDuplicateFileDescriptorV1), + CreateDirectoryV1(&'a ArchivedJournalEntryCreateDirectoryV1), + RemoveDirectoryV1(&'a ArchivedJournalEntryRemoveDirectoryV1), + PathSetTimesV1(&'a ArchivedJournalEntryPathSetTimesV1), + FileDescriptorSetTimesV1(&'a ArchivedJournalEntryFileDescriptorSetTimesV1), + FileDescriptorSetSizeV1(&'a ArchivedJournalEntryFileDescriptorSetSizeV1), + FileDescriptorSetFlagsV1(&'a ArchivedJournalEntryFileDescriptorSetFlagsV1), + FileDescriptorSetRightsV1(&'a ArchivedJournalEntryFileDescriptorSetRightsV1), + FileDescriptorAdviseV1(&'a ArchivedJournalEntryFileDescriptorAdviseV1), + FileDescriptorAllocateV1(&'a ArchivedJournalEntryFileDescriptorAllocateV1), + CreateHardLinkV1(&'a ArchivedJournalEntryCreateHardLinkV1), + CreateSymbolicLinkV1(&'a ArchivedJournalEntryCreateSymbolicLinkV1), + UnlinkFileV1(&'a ArchivedJournalEntryUnlinkFileV1), + PathRenameV1(&'a ArchivedJournalEntryPathRenameV1), + ChangeDirectoryV1(&'a ArchivedJournalEntryChangeDirectoryV1), + EpollCreateV1(&'a ArchivedJournalEntryEpollCreateV1), + EpollCtlV1(&'a ArchivedJournalEntryEpollCtlV1), + TtySetV1(&'a ArchivedJournalEntryTtySetV1), + CreatePipeV1(&'a ArchivedJournalEntryCreatePipeV1), + CreateEventV1(&'a ArchivedJournalEntryCreateEventV1), + PortAddAddrV1(&'a ArchivedJournalEntryPortAddAddrV1), + PortDelAddrV1(&'a ArchivedJournalEntryPortDelAddrV1), + PortAddrClearV1, + PortBridgeV1(&'a ArchivedJournalEntryPortBridgeV1), + PortUnbridgeV1, + PortDhcpAcquireV1, + PortGatewaySetV1(&'a ArchivedJournalEntryPortGatewaySetV1), + PortRouteAddV1(&'a ArchivedJournalEntryPortRouteAddV1), + PortRouteClearV1, + PortRouteDelV1(&'a ArchivedJournalEntryPortRouteDelV1), + SocketOpenV1(&'a ArchivedJournalEntrySocketOpenV1), + SocketListenV1(&'a ArchivedJournalEntrySocketListenV1), + SocketBindV1(&'a ArchivedJournalEntrySocketBindV1), + SocketConnectedV1(&'a ArchivedJournalEntrySocketConnectedV1), + SocketAcceptedV1(&'a ArchivedJournalEntrySocketAcceptedV1), + SocketJoinIpv4MulticastV1(&'a ArchivedJournalEntrySocketJoinIpv4MulticastV1), + SocketJoinIpv6MulticastV1(&'a ArchivedJournalEntrySocketJoinIpv6MulticastV1), + SocketLeaveIpv4MulticastV1(&'a ArchivedJournalEntrySocketLeaveIpv4MulticastV1), + SocketLeaveIpv6MulticastV1(&'a ArchivedJournalEntrySocketLeaveIpv6MulticastV1), + SocketSendFileV1(&'a ArchivedJournalEntrySocketSendFileV1), + SocketSendToV1(&'a ArchivedJournalEntrySocketSendToV1), + SocketSendV1(&'a ArchivedJournalEntrySocketSendV1), + SocketSetOptFlagV1(&'a ArchivedJournalEntrySocketSetOptFlagV1), + SocketSetOptSizeV1(&'a ArchivedJournalEntrySocketSetOptSizeV1), + SocketSetOptTimeV1(&'a ArchivedJournalEntrySocketSetOptTimeV1), + SocketShutdownV1(&'a ArchivedJournalEntrySocketShutdownV1), + SnapshotV1(&'a ArchivedJournalEntrySnapshotV1), } #[derive( - Debug, - Clone, - Copy, - PartialEq, - Eq, - PartialOrd, - Ord, - Hash, - serde::Serialize, - serde::Deserialize, - RkyvSerialize, - RkyvDeserialize, - Archive, + Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, )] #[archive_attr(derive(CheckBytes))] -pub enum JournalSocktypeV1 { - Unknown, - Stream, - Dgram, - Raw, - Seqpacket, +pub struct JournalEntryInitModuleV1 { + pub wasm_hash: [u8; 32], } -impl From for JournalSocktypeV1 { - fn from(val: wasi::Socktype) -> Self { - match val { - wasi::Socktype::Stream => JournalSocktypeV1::Stream, - wasi::Socktype::Dgram => JournalSocktypeV1::Dgram, - wasi::Socktype::Raw => JournalSocktypeV1::Raw, - wasi::Socktype::Seqpacket => JournalSocktypeV1::Seqpacket, - wasi::Socktype::Unknown => JournalSocktypeV1::Unknown, - } - } +#[derive( + Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, +)] +#[archive_attr(derive(CheckBytes))] +pub struct JournalEntryProcessExitV1 { + pub exit_code: Option, } -impl From for wasi::Socktype { - fn from(val: JournalSocktypeV1) -> Self { - match val { - JournalSocktypeV1::Stream => wasi::Socktype::Stream, - JournalSocktypeV1::Dgram => wasi::Socktype::Dgram, - JournalSocktypeV1::Raw => wasi::Socktype::Raw, - JournalSocktypeV1::Seqpacket => wasi::Socktype::Seqpacket, - JournalSocktypeV1::Unknown => wasi::Socktype::Unknown, - } - } +#[derive( + Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, +)] +#[archive_attr(derive(CheckBytes))] +pub struct JournalEntrySetThreadV1 { + pub id: u32, + pub call_stack: Vec, + pub memory_stack: Vec, + pub store_data: Vec, + pub is_64bit: bool, } -impl From<&'_ ArchivedJournalSocktypeV1> for wasi::Socktype { - fn from(val: &'_ ArchivedJournalSocktypeV1) -> Self { - match val { - ArchivedJournalSocktypeV1::Stream => wasi::Socktype::Stream, - ArchivedJournalSocktypeV1::Dgram => wasi::Socktype::Dgram, - ArchivedJournalSocktypeV1::Raw => wasi::Socktype::Raw, - ArchivedJournalSocktypeV1::Seqpacket => wasi::Socktype::Seqpacket, - ArchivedJournalSocktypeV1::Unknown => wasi::Socktype::Unknown, - } - } +#[derive( + Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, +)] +#[archive_attr(derive(CheckBytes))] +pub struct JournalEntryCloseThreadV1 { + pub id: u32, + pub exit_code: Option, } #[derive( - Debug, - Clone, - Copy, - PartialEq, - Eq, - PartialOrd, - Ord, - Hash, - serde::Serialize, - serde::Deserialize, - RkyvSerialize, - RkyvDeserialize, - Archive, + Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, )] #[archive_attr(derive(CheckBytes))] -pub enum JournalSockoptionV1 { - Noop, - ReusePort, - ReuseAddr, - NoDelay, - DontRoute, - OnlyV6, - Broadcast, - MulticastLoopV4, - MulticastLoopV6, - Promiscuous, - Listening, - LastError, - KeepAlive, - Linger, - OobInline, - RecvBufSize, - SendBufSize, - RecvLowat, - SendLowat, - RecvTimeout, - SendTimeout, - ConnectTimeout, - AcceptTimeout, - Ttl, - MulticastTtlV4, - Type, - Proto, +pub struct JournalEntryFileDescriptorSeekV1 { + pub fd: u32, + pub offset: i64, + pub whence: JournalWhenceV1, } -impl From for JournalSockoptionV1 { - fn from(val: wasi::Sockoption) -> Self { - match val { - wasi::Sockoption::Noop => JournalSockoptionV1::Noop, - wasi::Sockoption::ReusePort => JournalSockoptionV1::ReusePort, - wasi::Sockoption::ReuseAddr => JournalSockoptionV1::ReuseAddr, - wasi::Sockoption::NoDelay => JournalSockoptionV1::NoDelay, - wasi::Sockoption::DontRoute => JournalSockoptionV1::DontRoute, - wasi::Sockoption::OnlyV6 => JournalSockoptionV1::OnlyV6, - wasi::Sockoption::Broadcast => JournalSockoptionV1::Broadcast, - wasi::Sockoption::MulticastLoopV4 => JournalSockoptionV1::MulticastLoopV4, - wasi::Sockoption::MulticastLoopV6 => JournalSockoptionV1::MulticastLoopV6, - wasi::Sockoption::Promiscuous => JournalSockoptionV1::Promiscuous, - wasi::Sockoption::Listening => JournalSockoptionV1::Listening, - wasi::Sockoption::LastError => JournalSockoptionV1::LastError, - wasi::Sockoption::KeepAlive => JournalSockoptionV1::KeepAlive, - wasi::Sockoption::Linger => JournalSockoptionV1::Linger, - wasi::Sockoption::OobInline => JournalSockoptionV1::OobInline, - wasi::Sockoption::RecvBufSize => JournalSockoptionV1::RecvBufSize, - wasi::Sockoption::SendBufSize => JournalSockoptionV1::SendBufSize, - wasi::Sockoption::RecvLowat => JournalSockoptionV1::RecvLowat, - wasi::Sockoption::SendLowat => JournalSockoptionV1::SendLowat, - wasi::Sockoption::RecvTimeout => JournalSockoptionV1::RecvTimeout, - wasi::Sockoption::SendTimeout => JournalSockoptionV1::SendTimeout, - wasi::Sockoption::ConnectTimeout => JournalSockoptionV1::ConnectTimeout, - wasi::Sockoption::AcceptTimeout => JournalSockoptionV1::AcceptTimeout, - wasi::Sockoption::Ttl => JournalSockoptionV1::Ttl, - wasi::Sockoption::MulticastTtlV4 => JournalSockoptionV1::MulticastTtlV4, - wasi::Sockoption::Type => JournalSockoptionV1::Type, - wasi::Sockoption::Proto => JournalSockoptionV1::Proto, - } - } +#[derive( + Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, +)] +#[archive_attr(derive(CheckBytes))] +pub struct JournalEntryFileDescriptorWriteV1 { + pub fd: u32, + pub offset: u64, + pub data: Vec, + pub is_64bit: bool, } -impl From for wasi::Sockoption { - fn from(val: JournalSockoptionV1) -> Self { - match val { - JournalSockoptionV1::Noop => wasi::Sockoption::Noop, - JournalSockoptionV1::ReusePort => wasi::Sockoption::ReusePort, - JournalSockoptionV1::ReuseAddr => wasi::Sockoption::ReuseAddr, - JournalSockoptionV1::NoDelay => wasi::Sockoption::NoDelay, - JournalSockoptionV1::DontRoute => wasi::Sockoption::DontRoute, - JournalSockoptionV1::OnlyV6 => wasi::Sockoption::OnlyV6, - JournalSockoptionV1::Broadcast => wasi::Sockoption::Broadcast, - JournalSockoptionV1::MulticastLoopV4 => wasi::Sockoption::MulticastLoopV4, - JournalSockoptionV1::MulticastLoopV6 => wasi::Sockoption::MulticastLoopV6, - JournalSockoptionV1::Promiscuous => wasi::Sockoption::Promiscuous, - JournalSockoptionV1::Listening => wasi::Sockoption::Listening, - JournalSockoptionV1::LastError => wasi::Sockoption::LastError, - JournalSockoptionV1::KeepAlive => wasi::Sockoption::KeepAlive, - JournalSockoptionV1::Linger => wasi::Sockoption::Linger, - JournalSockoptionV1::OobInline => wasi::Sockoption::OobInline, - JournalSockoptionV1::RecvBufSize => wasi::Sockoption::RecvBufSize, - JournalSockoptionV1::SendBufSize => wasi::Sockoption::SendBufSize, - JournalSockoptionV1::RecvLowat => wasi::Sockoption::RecvLowat, - JournalSockoptionV1::SendLowat => wasi::Sockoption::SendLowat, - JournalSockoptionV1::RecvTimeout => wasi::Sockoption::RecvTimeout, - JournalSockoptionV1::SendTimeout => wasi::Sockoption::SendTimeout, - JournalSockoptionV1::ConnectTimeout => wasi::Sockoption::ConnectTimeout, - JournalSockoptionV1::AcceptTimeout => wasi::Sockoption::AcceptTimeout, - JournalSockoptionV1::Ttl => wasi::Sockoption::Ttl, - JournalSockoptionV1::MulticastTtlV4 => wasi::Sockoption::MulticastTtlV4, - JournalSockoptionV1::Type => wasi::Sockoption::Type, - JournalSockoptionV1::Proto => wasi::Sockoption::Proto, - } - } +#[derive( + Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, +)] +#[archive_attr(derive(CheckBytes))] +pub struct JournalEntryUpdateMemoryRegionV1 { + pub start: u64, + pub end: u64, + pub data: Vec, } -impl From<&'_ ArchivedJournalSockoptionV1> for wasi::Sockoption { - fn from(val: &'_ ArchivedJournalSockoptionV1) -> Self { - match val { - ArchivedJournalSockoptionV1::Noop => wasi::Sockoption::Noop, - ArchivedJournalSockoptionV1::ReusePort => wasi::Sockoption::ReusePort, - ArchivedJournalSockoptionV1::ReuseAddr => wasi::Sockoption::ReuseAddr, - ArchivedJournalSockoptionV1::NoDelay => wasi::Sockoption::NoDelay, - ArchivedJournalSockoptionV1::DontRoute => wasi::Sockoption::DontRoute, - ArchivedJournalSockoptionV1::OnlyV6 => wasi::Sockoption::OnlyV6, - ArchivedJournalSockoptionV1::Broadcast => wasi::Sockoption::Broadcast, - ArchivedJournalSockoptionV1::MulticastLoopV4 => wasi::Sockoption::MulticastLoopV4, - ArchivedJournalSockoptionV1::MulticastLoopV6 => wasi::Sockoption::MulticastLoopV6, - ArchivedJournalSockoptionV1::Promiscuous => wasi::Sockoption::Promiscuous, - ArchivedJournalSockoptionV1::Listening => wasi::Sockoption::Listening, - ArchivedJournalSockoptionV1::LastError => wasi::Sockoption::LastError, - ArchivedJournalSockoptionV1::KeepAlive => wasi::Sockoption::KeepAlive, - ArchivedJournalSockoptionV1::Linger => wasi::Sockoption::Linger, - ArchivedJournalSockoptionV1::OobInline => wasi::Sockoption::OobInline, - ArchivedJournalSockoptionV1::RecvBufSize => wasi::Sockoption::RecvBufSize, - ArchivedJournalSockoptionV1::SendBufSize => wasi::Sockoption::SendBufSize, - ArchivedJournalSockoptionV1::RecvLowat => wasi::Sockoption::RecvLowat, - ArchivedJournalSockoptionV1::SendLowat => wasi::Sockoption::SendLowat, - ArchivedJournalSockoptionV1::RecvTimeout => wasi::Sockoption::RecvTimeout, - ArchivedJournalSockoptionV1::SendTimeout => wasi::Sockoption::SendTimeout, - ArchivedJournalSockoptionV1::ConnectTimeout => wasi::Sockoption::ConnectTimeout, - ArchivedJournalSockoptionV1::AcceptTimeout => wasi::Sockoption::AcceptTimeout, - ArchivedJournalSockoptionV1::Ttl => wasi::Sockoption::Ttl, - ArchivedJournalSockoptionV1::MulticastTtlV4 => wasi::Sockoption::MulticastTtlV4, - ArchivedJournalSockoptionV1::Type => wasi::Sockoption::Type, - ArchivedJournalSockoptionV1::Proto => wasi::Sockoption::Proto, - } - } +#[derive( + Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, +)] +#[archive_attr(derive(CheckBytes))] +pub struct JournalEntrySetClockTimeV1 { + pub clock_id: JournalSnapshot0ClockidV1, + pub time: u64, } #[derive( - Debug, - Clone, - Copy, - PartialEq, - Eq, - PartialOrd, - Ord, - Hash, - serde::Serialize, - serde::Deserialize, - RkyvSerialize, - RkyvDeserialize, - Archive, + Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, )] #[archive_attr(derive(CheckBytes))] -pub enum JournalTimeTypeV1 { - ReadTimeout, - WriteTimeout, - AcceptTimeout, - ConnectTimeout, - BindTimeout, - Linger, +pub struct JournalEntryOpenFileDescriptorV1 { + pub fd: u32, + pub dirfd: u32, + pub dirflags: u32, + pub path: String, + pub o_flags: u16, + pub fs_rights_base: u64, + pub fs_rights_inheriting: u64, + pub fs_flags: u16, } -impl From for JournalTimeTypeV1 { - fn from(val: TimeType) -> Self { - match val { - TimeType::ReadTimeout => JournalTimeTypeV1::ReadTimeout, - TimeType::WriteTimeout => JournalTimeTypeV1::WriteTimeout, - TimeType::AcceptTimeout => JournalTimeTypeV1::AcceptTimeout, - TimeType::ConnectTimeout => JournalTimeTypeV1::ConnectTimeout, - TimeType::BindTimeout => JournalTimeTypeV1::BindTimeout, - TimeType::Linger => JournalTimeTypeV1::Linger, - } - } +#[derive( + Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, +)] +#[archive_attr(derive(CheckBytes))] +pub struct JournalEntryCloseFileDescriptorV1 { + pub fd: u32, } -impl From for TimeType { - fn from(val: JournalTimeTypeV1) -> Self { - match val { - JournalTimeTypeV1::ReadTimeout => TimeType::ReadTimeout, - JournalTimeTypeV1::WriteTimeout => TimeType::WriteTimeout, - JournalTimeTypeV1::AcceptTimeout => TimeType::AcceptTimeout, - JournalTimeTypeV1::ConnectTimeout => TimeType::ConnectTimeout, - JournalTimeTypeV1::BindTimeout => TimeType::BindTimeout, - JournalTimeTypeV1::Linger => TimeType::Linger, - } - } +#[derive( + Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, +)] +#[archive_attr(derive(CheckBytes))] +pub struct JournalEntryRenumberFileDescriptorV1 { + pub old_fd: u32, + pub new_fd: u32, } -impl From<&'_ ArchivedJournalTimeTypeV1> for TimeType { - fn from(val: &'_ ArchivedJournalTimeTypeV1) -> Self { - match val { - ArchivedJournalTimeTypeV1::ReadTimeout => TimeType::ReadTimeout, - ArchivedJournalTimeTypeV1::WriteTimeout => TimeType::WriteTimeout, - ArchivedJournalTimeTypeV1::AcceptTimeout => TimeType::AcceptTimeout, - ArchivedJournalTimeTypeV1::ConnectTimeout => TimeType::ConnectTimeout, - ArchivedJournalTimeTypeV1::BindTimeout => TimeType::BindTimeout, - ArchivedJournalTimeTypeV1::Linger => TimeType::Linger, - } - } +#[derive( + Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, +)] +#[archive_attr(derive(CheckBytes))] +pub struct JournalEntryDuplicateFileDescriptorV1 { + pub original_fd: u32, + pub copied_fd: u32, } #[derive( - Debug, - Clone, - Copy, - PartialEq, - Eq, - PartialOrd, - Ord, - Hash, - serde::Serialize, - serde::Deserialize, - RkyvSerialize, - RkyvDeserialize, - Archive, + Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, )] #[archive_attr(derive(CheckBytes))] -pub enum JournalSocketShutdownV1 { - Read, - Write, - Both, +pub struct JournalEntryCreateDirectoryV1 { + pub fd: u32, + pub path: String, } -impl From for JournalSocketShutdownV1 { - fn from(val: Shutdown) -> Self { - match val { - Shutdown::Read => JournalSocketShutdownV1::Read, - Shutdown::Write => JournalSocketShutdownV1::Write, - Shutdown::Both => JournalSocketShutdownV1::Both, - } - } +#[derive( + Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, +)] +#[archive_attr(derive(CheckBytes))] +pub struct JournalEntryRemoveDirectoryV1 { + pub fd: u32, + pub path: String, } -impl From for Shutdown { - fn from(val: JournalSocketShutdownV1) -> Self { - match val { - JournalSocketShutdownV1::Read => Shutdown::Read, - JournalSocketShutdownV1::Write => Shutdown::Write, - JournalSocketShutdownV1::Both => Shutdown::Both, - } - } +#[derive( + Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, +)] +#[archive_attr(derive(CheckBytes))] +pub struct JournalEntryPathSetTimesV1 { + pub fd: u32, + pub flags: u32, + pub path: String, + pub st_atim: u64, + pub st_mtim: u64, + pub fst_flags: u16, } -impl From<&'_ ArchivedJournalSocketShutdownV1> for Shutdown { - fn from(val: &'_ ArchivedJournalSocketShutdownV1) -> Self { - match val { - ArchivedJournalSocketShutdownV1::Read => Shutdown::Read, - ArchivedJournalSocketShutdownV1::Write => Shutdown::Write, - ArchivedJournalSocketShutdownV1::Both => Shutdown::Both, - } - } +#[derive( + Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, +)] +#[archive_attr(derive(CheckBytes))] +pub struct JournalEntryFileDescriptorSetTimesV1 { + pub fd: u32, + pub st_atim: u64, + pub st_mtim: u64, + pub fst_flags: u16, } -impl<'a> From> for JournalBatchEntry { - fn from(value: JournalEntry<'a>) -> Self { - match value { - JournalEntry::InitModule { wasm_hash } => Self::InitModuleV1 { wasm_hash }, - JournalEntry::UpdateMemoryRegion { region, data } => Self::UpdateMemoryRegionV1 { - start: region.start, - end: region.end, - data: data.into_owned(), - }, - JournalEntry::ProcessExit { exit_code } => Self::ProcessExitV1 { - exit_code: exit_code.map(|code| code.into()), - }, - JournalEntry::SetThread { - id, - call_stack, - memory_stack, - store_data, - is_64bit, - } => Self::SetThreadV1 { - id: id.into(), - call_stack: call_stack.into_owned(), - memory_stack: memory_stack.into_owned(), - store_data: store_data.into_owned(), - is_64bit, - }, - JournalEntry::CloseThread { id, exit_code } => Self::CloseThreadV1 { - id: id.into(), - exit_code: exit_code.map(|code| code.into()), - }, - JournalEntry::FileDescriptorWrite { - fd, - offset, - data, - is_64bit, - } => Self::FileDescriptorWriteV1 { - fd, - offset, - data: data.into_owned(), - is_64bit, - }, - JournalEntry::FileDescriptorSeek { fd, offset, whence } => Self::FileDescriptorSeekV1 { - fd, - offset, - whence: whence.into(), - }, - JournalEntry::OpenFileDescriptor { - fd, - dirfd, - dirflags, - path, - o_flags, - fs_rights_base, - fs_rights_inheriting, - fs_flags, - } => Self::OpenFileDescriptorV1 { - fd, - dirfd, - dirflags, - path: path.into_owned(), - o_flags: o_flags.bits(), - fs_rights_base: fs_rights_base.bits(), - fs_rights_inheriting: fs_rights_inheriting.bits(), - fs_flags: fs_flags.bits(), - }, - JournalEntry::CloseFileDescriptor { fd } => Self::CloseFileDescriptorV1 { fd }, - JournalEntry::RemoveDirectory { fd, path } => Self::RemoveDirectoryV1 { - fd, - path: path.into_owned(), - }, - JournalEntry::UnlinkFile { fd, path } => Self::UnlinkFileV1 { - fd, - path: path.into_owned(), - }, - JournalEntry::PathRename { - old_fd, - old_path, - new_fd, - new_path, - } => Self::PathRenameV1 { - old_fd, - old_path: old_path.into_owned(), - new_fd, - new_path: new_path.into_owned(), - }, - JournalEntry::Snapshot { when, trigger } => Self::SnapshotV1 { - since_epoch: when - .duration_since(SystemTime::UNIX_EPOCH) - .unwrap_or(Duration::ZERO), - trigger: trigger.into(), - }, - JournalEntry::SetClockTime { clock_id, time } => Self::SetClockTimeV1 { - clock_id: clock_id.into(), - time, - }, - JournalEntry::RenumberFileDescriptor { old_fd, new_fd } => { - Self::RenumberFileDescriptorV1 { old_fd, new_fd } - } - JournalEntry::DuplicateFileDescriptor { - original_fd, - copied_fd, - } => Self::DuplicateFileDescriptorV1 { - original_fd, - copied_fd, - }, - JournalEntry::CreateDirectory { fd, path } => Self::CreateDirectoryV1 { - fd, - path: path.into_owned(), - }, - JournalEntry::PathSetTimes { - fd, - path, - flags, - st_atim, - st_mtim, - fst_flags, - } => Self::PathSetTimesV1 { - fd, - path: path.into_owned(), - flags, - st_atim, - st_mtim, - fst_flags: fst_flags.bits(), - }, - JournalEntry::FileDescriptorSetTimes { - fd, - st_atim, - st_mtim, - fst_flags, - } => Self::FileDescriptorSetTimesV1 { - fd, - st_atim, - st_mtim, - fst_flags: fst_flags.bits(), - }, - JournalEntry::FileDescriptorSetSize { fd, st_size } => { - Self::FileDescriptorSetSizeV1 { fd, st_size } - } - JournalEntry::FileDescriptorSetFlags { fd, flags } => Self::FileDescriptorSetFlagsV1 { - fd, - flags: flags.bits(), - }, - JournalEntry::FileDescriptorSetRights { - fd, - fs_rights_base, - fs_rights_inheriting, - } => Self::FileDescriptorSetRightsV1 { - fd, - fs_rights_base: fs_rights_base.bits(), - fs_rights_inheriting: fs_rights_inheriting.bits(), - }, - JournalEntry::FileDescriptorAdvise { - fd, - offset, - len, - advice, - } => Self::FileDescriptorAdviseV1 { - fd, - offset, - len, - advice: advice.into(), - }, - JournalEntry::FileDescriptorAllocate { fd, offset, len } => { - Self::FileDescriptorAllocateV1 { fd, offset, len } - } - JournalEntry::CreateHardLink { - old_fd, - old_path, - old_flags, - new_fd, - new_path, - } => Self::CreateHardLinkV1 { - old_fd, - old_path: old_path.into_owned(), - old_flags, - new_fd, - new_path: new_path.into_owned(), - }, - JournalEntry::CreateSymbolicLink { - old_path, - fd, - new_path, - } => Self::CreateSymbolicLinkV1 { - old_path: old_path.into_owned(), - fd, - new_path: new_path.into_owned(), - }, - JournalEntry::ChangeDirectory { path } => Self::ChangeDirectoryV1 { - path: path.into_owned(), - }, - JournalEntry::EpollCreate { fd } => Self::EpollCreateV1 { fd }, - JournalEntry::EpollCtl { - epfd, - op, - fd, - event, - } => Self::EpollCtlV1 { - epfd, - op: op.into(), - fd, - event: event.map(|e| e.into()), - }, - JournalEntry::TtySet { tty, line_feeds } => Self::TtySetV1 { - cols: tty.cols, - rows: tty.rows, - width: tty.width, - height: tty.height, - stdin_tty: tty.stdin_tty, - stdout_tty: tty.stdout_tty, - stderr_tty: tty.stderr_tty, - echo: tty.echo, - line_buffered: tty.line_buffered, - line_feeds, - }, - JournalEntry::CreatePipe { fd1, fd2 } => Self::CreatePipeV1 { fd1, fd2 }, - JournalEntry::PortAddAddr { cidr } => Self::PortAddAddrV1 { cidr }, - JournalEntry::PortDelAddr { addr } => Self::PortDelAddrV1 { addr }, - JournalEntry::PortAddrClear => Self::PortAddrClearV1, - JournalEntry::PortBridge { - network, - token, - security, - } => Self::PortBridgeV1 { - network: network.into(), - token: token.into(), - security: security.into(), - }, - JournalEntry::PortUnbridge => Self::PortUnbridgeV1, - JournalEntry::PortDhcpAcquire => Self::PortDhcpAcquireV1, - JournalEntry::PortGatewaySet { ip } => Self::PortGatewaySetV1 { ip }, - JournalEntry::PortRouteAdd { - cidr, - via_router, - preferred_until, - expires_at, - } => Self::PortRouteAddV1 { - cidr, - via_router, - preferred_until, - expires_at, - }, - JournalEntry::PortRouteClear => Self::PortRouteClearV1, - JournalEntry::PortRouteDel { ip } => Self::PortRouteDelV1 { ip }, - JournalEntry::SocketOpen { af, ty, pt, fd } => Self::SocketOpenV1 { - af: af.into(), - ty: ty.into(), - pt: pt as u16, - fd, - }, - JournalEntry::SocketListen { fd, backlog } => Self::SocketListenV1 { fd, backlog }, - JournalEntry::SocketBind { fd, addr } => Self::SocketBindV1 { fd, addr }, - JournalEntry::SocketConnected { fd, addr } => Self::SocketConnectedV1 { fd, addr }, - JournalEntry::SocketAccepted { - listen_fd, - fd, - peer_addr, - fd_flags, - nonblocking, - } => Self::SocketAcceptedV1 { - listen_fd, - fd, - peer_addr, - fd_flags: fd_flags.bits(), - nonblocking, - }, - JournalEntry::SocketJoinIpv4Multicast { - fd, - multiaddr, - iface, - } => Self::SocketJoinIpv4MulticastV1 { - fd, - multiaddr, - iface, - }, - JournalEntry::SocketJoinIpv6Multicast { - fd, - multiaddr, - iface, - } => Self::SocketJoinIpv6MulticastV1 { - fd, - multiaddr, - iface, - }, - JournalEntry::SocketLeaveIpv4Multicast { - fd, - multiaddr, - iface, - } => Self::SocketLeaveIpv4MulticastV1 { - fd, - multiaddr, - iface, - }, - JournalEntry::SocketLeaveIpv6Multicast { - fd, - multiaddr, - iface, - } => Self::SocketLeaveIpv6MulticastV1 { - fd, - multiaddr, - iface, - }, - JournalEntry::SocketSendFile { - socket_fd, - file_fd, - offset, - count, - } => Self::SocketSendFileV1 { - socket_fd, - file_fd, - offset, - count, - }, - JournalEntry::SocketSendTo { - fd, - data, - flags, - addr, - is_64bit, - } => Self::SocketSendToV1 { - fd, - data: data.into(), - flags, - addr, - is_64bit, - }, - JournalEntry::SocketSend { - fd, - data, - flags, - is_64bit, - } => Self::SocketSendV1 { - fd, - data: data.into(), - flags, - is_64bit, - }, - JournalEntry::SocketSetOptFlag { fd, opt, flag } => Self::SocketSetOptFlagV1 { - fd, - opt: opt.into(), - flag, - }, - JournalEntry::SocketSetOptSize { fd, opt, size } => Self::SocketSetOptSizeV1 { - fd, - opt: opt.into(), - size, - }, - JournalEntry::SocketSetOptTime { fd, ty, time } => Self::SocketSetOptTimeV1 { - fd, - ty: ty.into(), - time, - }, - JournalEntry::SocketShutdown { fd, how } => Self::SocketShutdownV1 { - fd, - how: how.into(), - }, - JournalEntry::CreateEvent { - initial_val, - flags, - fd, - } => Self::CreateEventV1 { - initial_val, - flags, - fd, - }, +#[derive( + Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, +)] +#[archive_attr(derive(CheckBytes))] +pub struct JournalEntryFileDescriptorSetSizeV1 { + pub fd: u32, + pub st_size: u64, +} + +#[derive( + Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, +)] +#[archive_attr(derive(CheckBytes))] +pub struct JournalEntryFileDescriptorSetFlagsV1 { + pub fd: u32, + pub flags: u16, +} + +#[derive( + Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, +)] +#[archive_attr(derive(CheckBytes))] +pub struct JournalEntryFileDescriptorSetRightsV1 { + pub fd: u32, + pub fs_rights_base: u64, + pub fs_rights_inheriting: u64, +} + +#[derive( + Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, +)] +#[archive_attr(derive(CheckBytes))] +pub struct JournalEntryFileDescriptorAdviseV1 { + pub fd: u32, + pub offset: u64, + pub len: u64, + pub advice: JournalAdviceV1, +} + +#[derive( + Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, +)] +#[archive_attr(derive(CheckBytes))] +pub struct JournalEntryFileDescriptorAllocateV1 { + pub fd: u32, + pub offset: u64, + pub len: u64, +} + +#[derive( + Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, +)] +#[archive_attr(derive(CheckBytes))] +pub struct JournalEntryCreateHardLinkV1 { + pub old_fd: u32, + pub old_path: String, + pub old_flags: u32, + pub new_fd: u32, + pub new_path: String, +} + +#[derive( + Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, +)] +#[archive_attr(derive(CheckBytes))] +pub struct JournalEntryCreateSymbolicLinkV1 { + pub old_path: String, + pub fd: u32, + pub new_path: String, +} + +#[derive( + Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, +)] +#[archive_attr(derive(CheckBytes))] +pub struct JournalEntryUnlinkFileV1 { + pub fd: u32, + pub path: String, +} + +#[derive( + Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, +)] +#[archive_attr(derive(CheckBytes))] +pub struct JournalEntryPathRenameV1 { + pub old_fd: u32, + pub old_path: String, + pub new_fd: u32, + pub new_path: String, +} + +#[derive( + Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, +)] +#[archive_attr(derive(CheckBytes))] +pub struct JournalEntryChangeDirectoryV1 { + pub path: String, +} + +#[derive( + Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, +)] +#[archive_attr(derive(CheckBytes))] +pub struct JournalEntryEpollCreateV1 { + pub fd: u32, +} + +#[derive( + Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, +)] +#[archive_attr(derive(CheckBytes))] +pub struct JournalEntryEpollCtlV1 { + pub epfd: u32, + pub op: JournalEpollCtlV1, + pub fd: u32, + pub event: Option, +} + +#[derive( + Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, +)] +#[archive_attr(derive(CheckBytes))] +pub struct JournalEntryTtySetV1 { + pub cols: u32, + pub rows: u32, + pub width: u32, + pub height: u32, + pub stdin_tty: bool, + pub stdout_tty: bool, + pub stderr_tty: bool, + pub echo: bool, + pub line_buffered: bool, + pub line_feeds: bool, +} + +#[derive( + Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, +)] +#[archive_attr(derive(CheckBytes))] +pub struct JournalEntryCreatePipeV1 { + pub fd1: u32, + pub fd2: u32, +} + +#[derive( + Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, +)] +#[archive_attr(derive(CheckBytes))] +pub struct JournalEntryCreateEventV1 { + pub initial_val: u64, + pub flags: u16, + pub fd: u32, +} + +#[derive( + Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, +)] +#[archive_attr(derive(CheckBytes))] +pub struct JournalEntryPortAddAddrV1 { + pub cidr: IpCidr, +} + +#[derive( + Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, +)] +#[archive_attr(derive(CheckBytes))] +pub struct JournalEntryPortDelAddrV1 { + pub addr: IpAddr, +} + +#[derive( + Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, +)] +#[archive_attr(derive(CheckBytes))] +pub struct JournalEntryPortBridgeV1 { + pub network: String, + pub token: String, + pub security: JournalStreamSecurityV1, +} + +#[derive( + Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, +)] +#[archive_attr(derive(CheckBytes))] +pub struct JournalEntryPortGatewaySetV1 { + pub ip: IpAddr, +} + +#[derive( + Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, +)] +#[archive_attr(derive(CheckBytes))] +pub struct JournalEntryPortRouteAddV1 { + pub cidr: IpCidr, + pub via_router: IpAddr, + pub preferred_until: Option, + pub expires_at: Option, +} + +#[derive( + Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, +)] +#[archive_attr(derive(CheckBytes))] +pub struct JournalEntryPortRouteDelV1 { + pub ip: IpAddr, +} + +#[derive( + Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, +)] +#[archive_attr(derive(CheckBytes))] +pub struct JournalEntrySocketOpenV1 { + pub af: JournalAddressfamilyV1, + pub ty: JournalSocktypeV1, + pub pt: u16, + pub fd: u32, +} + +#[derive( + Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, +)] +#[archive_attr(derive(CheckBytes))] +pub struct JournalEntrySocketListenV1 { + pub fd: u32, + pub backlog: u32, +} + +#[derive( + Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, +)] +#[archive_attr(derive(CheckBytes))] +pub struct JournalEntrySocketBindV1 { + pub fd: u32, + pub addr: SocketAddr, +} + +#[derive( + Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, +)] +#[archive_attr(derive(CheckBytes))] +pub struct JournalEntrySocketConnectedV1 { + pub fd: u32, + pub addr: SocketAddr, +} + +#[derive( + Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, +)] +#[archive_attr(derive(CheckBytes))] +pub struct JournalEntrySocketAcceptedV1 { + pub listen_fd: u32, + pub fd: u32, + pub peer_addr: SocketAddr, + pub fd_flags: u16, + pub nonblocking: bool, +} + +#[derive( + Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, +)] +#[archive_attr(derive(CheckBytes))] +pub struct JournalEntrySocketJoinIpv4MulticastV1 { + pub fd: u32, + pub multiaddr: Ipv4Addr, + pub iface: Ipv4Addr, +} + +#[derive( + Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, +)] +#[archive_attr(derive(CheckBytes))] +pub struct JournalEntrySocketJoinIpv6MulticastV1 { + pub fd: u32, + pub multiaddr: Ipv6Addr, + pub iface: u32, +} + +#[derive( + Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, +)] +#[archive_attr(derive(CheckBytes))] +pub struct JournalEntrySocketLeaveIpv4MulticastV1 { + pub fd: u32, + pub multiaddr: Ipv4Addr, + pub iface: Ipv4Addr, +} + +#[derive( + Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, +)] +#[archive_attr(derive(CheckBytes))] +pub struct JournalEntrySocketLeaveIpv6MulticastV1 { + pub fd: u32, + pub multiaddr: Ipv6Addr, + pub iface: u32, +} + +#[derive( + Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, +)] +#[archive_attr(derive(CheckBytes))] +pub struct JournalEntrySocketSendFileV1 { + pub socket_fd: u32, + pub file_fd: u32, + pub offset: u64, + pub count: u64, +} + +#[derive( + Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, +)] +#[archive_attr(derive(CheckBytes))] +pub struct JournalEntrySocketSendToV1 { + pub fd: u32, + pub data: Vec, + pub flags: u16, + pub addr: SocketAddr, + pub is_64bit: bool, +} + +#[derive( + Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, +)] +#[archive_attr(derive(CheckBytes))] +pub struct JournalEntrySocketSendV1 { + pub fd: u32, + pub data: Vec, + pub flags: u16, + pub is_64bit: bool, +} + +#[derive( + Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, +)] +#[archive_attr(derive(CheckBytes))] +pub struct JournalEntrySocketSetOptFlagV1 { + pub fd: u32, + pub opt: JournalSockoptionV1, + pub flag: bool, +} + +#[derive( + Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, +)] +#[archive_attr(derive(CheckBytes))] +pub struct JournalEntrySocketSetOptSizeV1 { + pub fd: u32, + pub opt: JournalSockoptionV1, + pub size: u64, +} + +#[derive( + Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, +)] +#[archive_attr(derive(CheckBytes))] +pub struct JournalEntrySocketSetOptTimeV1 { + pub fd: u32, + pub ty: JournalTimeTypeV1, + pub time: Option, +} + +#[derive( + Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, +)] +#[archive_attr(derive(CheckBytes))] +pub struct JournalEntrySocketShutdownV1 { + pub fd: u32, + pub how: JournalSocketShutdownV1, +} + +#[derive( + Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, +)] +#[archive_attr(derive(CheckBytes))] +pub struct JournalEntrySnapshotV1 { + pub since_epoch: Duration, + pub trigger: JournalSnapshotTriggerV1, +} + +#[derive( + Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, +)] +#[archive_attr(derive(CheckBytes))] +pub enum JournalSnapshot0ClockidV1 { + Realtime, + Monotonic, + ProcessCputimeId, + ThreadCputimeId, + Unknown = 255, +} + +impl From for JournalSnapshot0ClockidV1 { + fn from(val: wasi::Snapshot0Clockid) -> Self { + match val { + wasi::Snapshot0Clockid::Realtime => JournalSnapshot0ClockidV1::Realtime, + wasi::Snapshot0Clockid::Monotonic => JournalSnapshot0ClockidV1::Monotonic, + wasi::Snapshot0Clockid::ProcessCputimeId => JournalSnapshot0ClockidV1::ProcessCputimeId, + wasi::Snapshot0Clockid::ThreadCputimeId => JournalSnapshot0ClockidV1::ThreadCputimeId, + wasi::Snapshot0Clockid::Unknown => JournalSnapshot0ClockidV1::Unknown, + } + } +} + +impl From for wasi::Snapshot0Clockid { + fn from(val: JournalSnapshot0ClockidV1) -> Self { + match val { + JournalSnapshot0ClockidV1::Realtime => wasi::Snapshot0Clockid::Realtime, + JournalSnapshot0ClockidV1::Monotonic => wasi::Snapshot0Clockid::Monotonic, + JournalSnapshot0ClockidV1::ProcessCputimeId => wasi::Snapshot0Clockid::ProcessCputimeId, + JournalSnapshot0ClockidV1::ThreadCputimeId => wasi::Snapshot0Clockid::ThreadCputimeId, + JournalSnapshot0ClockidV1::Unknown => wasi::Snapshot0Clockid::Unknown, + } + } +} + +impl From<&'_ ArchivedJournalSnapshot0ClockidV1> for wasi::Snapshot0Clockid { + fn from(val: &'_ ArchivedJournalSnapshot0ClockidV1) -> Self { + match val { + ArchivedJournalSnapshot0ClockidV1::Realtime => wasi::Snapshot0Clockid::Realtime, + ArchivedJournalSnapshot0ClockidV1::Monotonic => wasi::Snapshot0Clockid::Monotonic, + ArchivedJournalSnapshot0ClockidV1::ProcessCputimeId => { + wasi::Snapshot0Clockid::ProcessCputimeId + } + ArchivedJournalSnapshot0ClockidV1::ThreadCputimeId => { + wasi::Snapshot0Clockid::ThreadCputimeId + } + ArchivedJournalSnapshot0ClockidV1::Unknown => wasi::Snapshot0Clockid::Unknown, + } + } +} + +#[derive( + Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, +)] +#[archive_attr(derive(CheckBytes))] +pub enum JournalWhenceV1 { + Set, + Cur, + End, + Unknown = 255, +} + +impl From for JournalWhenceV1 { + fn from(val: wasi::Whence) -> Self { + match val { + wasi::Whence::Set => JournalWhenceV1::Set, + wasi::Whence::Cur => JournalWhenceV1::Cur, + wasi::Whence::End => JournalWhenceV1::End, + wasi::Whence::Unknown => JournalWhenceV1::Unknown, + } + } +} + +impl From for wasi::Whence { + fn from(val: JournalWhenceV1) -> Self { + match val { + JournalWhenceV1::Set => wasi::Whence::Set, + JournalWhenceV1::Cur => wasi::Whence::Cur, + JournalWhenceV1::End => wasi::Whence::End, + JournalWhenceV1::Unknown => wasi::Whence::Unknown, + } + } +} + +impl From<&'_ ArchivedJournalWhenceV1> for wasi::Whence { + fn from(val: &'_ ArchivedJournalWhenceV1) -> Self { + match val { + ArchivedJournalWhenceV1::Set => wasi::Whence::Set, + ArchivedJournalWhenceV1::Cur => wasi::Whence::Cur, + ArchivedJournalWhenceV1::End => wasi::Whence::End, + ArchivedJournalWhenceV1::Unknown => wasi::Whence::Unknown, + } + } +} + +#[derive( + Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, +)] +#[archive_attr(derive(CheckBytes))] +pub enum JournalAdviceV1 { + Normal, + Sequential, + Random, + Willneed, + Dontneed, + Noreuse, + Unknown = 255, +} + +impl From for JournalAdviceV1 { + fn from(val: wasi::Advice) -> Self { + match val { + wasi::Advice::Normal => JournalAdviceV1::Normal, + wasi::Advice::Sequential => JournalAdviceV1::Sequential, + wasi::Advice::Random => JournalAdviceV1::Random, + wasi::Advice::Willneed => JournalAdviceV1::Willneed, + wasi::Advice::Dontneed => JournalAdviceV1::Dontneed, + wasi::Advice::Noreuse => JournalAdviceV1::Noreuse, + wasi::Advice::Unknown => JournalAdviceV1::Unknown, + } + } +} + +impl From for wasi::Advice { + fn from(val: JournalAdviceV1) -> Self { + match val { + JournalAdviceV1::Normal => wasi::Advice::Normal, + JournalAdviceV1::Sequential => wasi::Advice::Sequential, + JournalAdviceV1::Random => wasi::Advice::Random, + JournalAdviceV1::Willneed => wasi::Advice::Willneed, + JournalAdviceV1::Dontneed => wasi::Advice::Dontneed, + JournalAdviceV1::Noreuse => wasi::Advice::Noreuse, + JournalAdviceV1::Unknown => wasi::Advice::Unknown, + } + } +} + +impl From<&'_ ArchivedJournalAdviceV1> for wasi::Advice { + fn from(val: &'_ ArchivedJournalAdviceV1) -> Self { + match val { + ArchivedJournalAdviceV1::Normal => wasi::Advice::Normal, + ArchivedJournalAdviceV1::Sequential => wasi::Advice::Sequential, + ArchivedJournalAdviceV1::Random => wasi::Advice::Random, + ArchivedJournalAdviceV1::Willneed => wasi::Advice::Willneed, + ArchivedJournalAdviceV1::Dontneed => wasi::Advice::Dontneed, + ArchivedJournalAdviceV1::Noreuse => wasi::Advice::Noreuse, + ArchivedJournalAdviceV1::Unknown => wasi::Advice::Unknown, + } + } +} + +#[derive( + Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, +)] +#[archive_attr(derive(CheckBytes))] +pub enum JournalExitCodeV1 { + Errno(u16), + Other(i32), +} + +impl From for JournalExitCodeV1 { + fn from(val: wasi::ExitCode) -> Self { + match val { + wasi::ExitCode::Errno(errno) => JournalExitCodeV1::Errno(errno as u16), + wasi::ExitCode::Other(id) => JournalExitCodeV1::Other(id), + } + } +} + +impl From for wasi::ExitCode { + fn from(val: JournalExitCodeV1) -> Self { + match val { + JournalExitCodeV1::Errno(errno) => { + wasi::ExitCode::Errno(errno.try_into().unwrap_or(wasi::Errno::Unknown)) + } + JournalExitCodeV1::Other(id) => wasi::ExitCode::Other(id), + } + } +} + +impl From<&'_ ArchivedJournalExitCodeV1> for wasi::ExitCode { + fn from(val: &'_ ArchivedJournalExitCodeV1) -> Self { + match val { + ArchivedJournalExitCodeV1::Errno(errno) => { + wasi::ExitCode::Errno((*errno).try_into().unwrap_or(wasi::Errno::Unknown)) + } + ArchivedJournalExitCodeV1::Other(id) => wasi::ExitCode::Other(*id), + } + } +} + +#[derive( + Debug, + Clone, + Copy, + PartialEq, + Eq, + PartialOrd, + Ord, + Hash, + serde::Serialize, + serde::Deserialize, + RkyvSerialize, + RkyvDeserialize, + Archive, +)] +#[archive_attr(derive(CheckBytes))] +pub enum JournalSnapshotTriggerV1 { + Idle, + Listen, + Environ, + Stdin, + Timer, + Sigint, + Sigalrm, + Sigtstp, + Sigstop, + NonDeterministicCall, +} + +impl From for JournalSnapshotTriggerV1 { + fn from(val: SnapshotTrigger) -> Self { + match val { + SnapshotTrigger::Idle => JournalSnapshotTriggerV1::Idle, + SnapshotTrigger::FirstListen => JournalSnapshotTriggerV1::Listen, + SnapshotTrigger::FirstEnviron => JournalSnapshotTriggerV1::Environ, + SnapshotTrigger::FirstStdin => JournalSnapshotTriggerV1::Stdin, + SnapshotTrigger::PeriodicInterval => JournalSnapshotTriggerV1::Timer, + SnapshotTrigger::Sigint => JournalSnapshotTriggerV1::Sigint, + SnapshotTrigger::Sigalrm => JournalSnapshotTriggerV1::Sigalrm, + SnapshotTrigger::Sigtstp => JournalSnapshotTriggerV1::Sigtstp, + SnapshotTrigger::Sigstop => JournalSnapshotTriggerV1::Sigstop, + SnapshotTrigger::NonDeterministicCall => JournalSnapshotTriggerV1::NonDeterministicCall, + } + } +} + +impl From for SnapshotTrigger { + fn from(val: JournalSnapshotTriggerV1) -> Self { + match val { + JournalSnapshotTriggerV1::Idle => SnapshotTrigger::Idle, + JournalSnapshotTriggerV1::Listen => SnapshotTrigger::FirstListen, + JournalSnapshotTriggerV1::Environ => SnapshotTrigger::FirstEnviron, + JournalSnapshotTriggerV1::Stdin => SnapshotTrigger::FirstStdin, + JournalSnapshotTriggerV1::Timer => SnapshotTrigger::PeriodicInterval, + JournalSnapshotTriggerV1::Sigint => SnapshotTrigger::Sigint, + JournalSnapshotTriggerV1::Sigalrm => SnapshotTrigger::Sigalrm, + JournalSnapshotTriggerV1::Sigtstp => SnapshotTrigger::Sigtstp, + JournalSnapshotTriggerV1::Sigstop => SnapshotTrigger::Sigstop, + JournalSnapshotTriggerV1::NonDeterministicCall => SnapshotTrigger::NonDeterministicCall, + } + } +} + +impl From<&'_ ArchivedJournalSnapshotTriggerV1> for SnapshotTrigger { + fn from(val: &'_ ArchivedJournalSnapshotTriggerV1) -> Self { + match val { + ArchivedJournalSnapshotTriggerV1::Idle => SnapshotTrigger::Idle, + ArchivedJournalSnapshotTriggerV1::Listen => SnapshotTrigger::FirstListen, + ArchivedJournalSnapshotTriggerV1::Environ => SnapshotTrigger::FirstEnviron, + ArchivedJournalSnapshotTriggerV1::Stdin => SnapshotTrigger::FirstStdin, + ArchivedJournalSnapshotTriggerV1::Timer => SnapshotTrigger::PeriodicInterval, + ArchivedJournalSnapshotTriggerV1::Sigint => SnapshotTrigger::Sigint, + ArchivedJournalSnapshotTriggerV1::Sigalrm => SnapshotTrigger::Sigalrm, + ArchivedJournalSnapshotTriggerV1::Sigtstp => SnapshotTrigger::Sigtstp, + ArchivedJournalSnapshotTriggerV1::Sigstop => SnapshotTrigger::Sigstop, + ArchivedJournalSnapshotTriggerV1::NonDeterministicCall => { + SnapshotTrigger::NonDeterministicCall + } + } + } +} + +#[derive( + Debug, + Clone, + Copy, + PartialEq, + Eq, + PartialOrd, + Ord, + Hash, + serde::Serialize, + serde::Deserialize, + RkyvSerialize, + RkyvDeserialize, + Archive, +)] +#[archive_attr(derive(CheckBytes))] +pub enum JournalEpollCtlV1 { + Add, + Mod, + Del, + Unknown, +} + +impl From for JournalEpollCtlV1 { + fn from(val: wasi::EpollCtl) -> Self { + match val { + wasi::EpollCtl::Add => JournalEpollCtlV1::Add, + wasi::EpollCtl::Mod => JournalEpollCtlV1::Mod, + wasi::EpollCtl::Del => JournalEpollCtlV1::Del, + wasi::EpollCtl::Unknown => JournalEpollCtlV1::Unknown, + } + } +} + +impl From for wasi::EpollCtl { + fn from(val: JournalEpollCtlV1) -> Self { + match val { + JournalEpollCtlV1::Add => wasi::EpollCtl::Add, + JournalEpollCtlV1::Mod => wasi::EpollCtl::Mod, + JournalEpollCtlV1::Del => wasi::EpollCtl::Del, + JournalEpollCtlV1::Unknown => wasi::EpollCtl::Unknown, + } + } +} + +impl From<&'_ ArchivedJournalEpollCtlV1> for wasi::EpollCtl { + fn from(val: &'_ ArchivedJournalEpollCtlV1) -> Self { + match val { + ArchivedJournalEpollCtlV1::Add => wasi::EpollCtl::Add, + ArchivedJournalEpollCtlV1::Mod => wasi::EpollCtl::Mod, + ArchivedJournalEpollCtlV1::Del => wasi::EpollCtl::Del, + ArchivedJournalEpollCtlV1::Unknown => wasi::EpollCtl::Unknown, + } + } +} + +#[derive( + Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, +)] +#[archive_attr(derive(CheckBytes))] +pub struct JournalEpollEventCtlV1 { + pub events: u32, + pub ptr: u64, + pub fd: u32, + pub data1: u32, + pub data2: u64, +} + +impl From for JournalEpollEventCtlV1 { + fn from(val: EpollEventCtl) -> Self { + JournalEpollEventCtlV1 { + events: val.events.bits(), + ptr: val.ptr, + fd: val.fd, + data1: val.data1, + data2: val.data2, + } + } +} + +impl From for EpollEventCtl { + fn from(val: JournalEpollEventCtlV1) -> Self { + Self { + events: EpollType::from_bits_truncate(val.events), + ptr: val.ptr, + fd: val.fd, + data1: val.data1, + data2: val.data2, + } + } +} + +impl From<&'_ ArchivedJournalEpollEventCtlV1> for EpollEventCtl { + fn from(val: &'_ ArchivedJournalEpollEventCtlV1) -> Self { + Self { + events: EpollType::from_bits_truncate(val.events), + ptr: val.ptr, + fd: val.fd, + data1: val.data1, + data2: val.data2, + } + } +} + +#[derive( + Debug, + Clone, + Copy, + PartialEq, + Eq, + PartialOrd, + Ord, + Hash, + serde::Serialize, + serde::Deserialize, + RkyvSerialize, + RkyvDeserialize, + Archive, +)] +#[archive_attr(derive(CheckBytes))] +pub enum JournalStreamSecurityV1 { + Unencrypted, + AnyEncryption, + ClassicEncryption, + DoubleEncryption, + Unknown, +} + +impl From for JournalStreamSecurityV1 { + fn from(val: StreamSecurity) -> Self { + match val { + StreamSecurity::Unencrypted => JournalStreamSecurityV1::Unencrypted, + StreamSecurity::AnyEncyption => JournalStreamSecurityV1::AnyEncryption, + StreamSecurity::ClassicEncryption => JournalStreamSecurityV1::ClassicEncryption, + StreamSecurity::DoubleEncryption => JournalStreamSecurityV1::DoubleEncryption, + } + } +} + +impl From for StreamSecurity { + fn from(val: JournalStreamSecurityV1) -> Self { + match val { + JournalStreamSecurityV1::Unencrypted => StreamSecurity::Unencrypted, + JournalStreamSecurityV1::AnyEncryption => StreamSecurity::AnyEncyption, + JournalStreamSecurityV1::ClassicEncryption => StreamSecurity::ClassicEncryption, + JournalStreamSecurityV1::DoubleEncryption => StreamSecurity::DoubleEncryption, + JournalStreamSecurityV1::Unknown => StreamSecurity::AnyEncyption, + } + } +} + +impl From<&'_ ArchivedJournalStreamSecurityV1> for StreamSecurity { + fn from(val: &'_ ArchivedJournalStreamSecurityV1) -> Self { + match val { + ArchivedJournalStreamSecurityV1::Unencrypted => StreamSecurity::Unencrypted, + ArchivedJournalStreamSecurityV1::AnyEncryption => StreamSecurity::AnyEncyption, + ArchivedJournalStreamSecurityV1::ClassicEncryption => StreamSecurity::ClassicEncryption, + ArchivedJournalStreamSecurityV1::DoubleEncryption => StreamSecurity::DoubleEncryption, + ArchivedJournalStreamSecurityV1::Unknown => StreamSecurity::AnyEncyption, + } + } +} + +#[derive( + Debug, + Clone, + Copy, + PartialEq, + Eq, + PartialOrd, + Ord, + Hash, + serde::Serialize, + serde::Deserialize, + RkyvSerialize, + RkyvDeserialize, + Archive, +)] +#[archive_attr(derive(CheckBytes))] +pub enum JournalAddressfamilyV1 { + Unspec, + Inet4, + Inet6, + Unix, +} + +impl From for JournalAddressfamilyV1 { + fn from(val: wasi::Addressfamily) -> Self { + match val { + wasi::Addressfamily::Unspec => JournalAddressfamilyV1::Unspec, + wasi::Addressfamily::Inet4 => JournalAddressfamilyV1::Inet4, + wasi::Addressfamily::Inet6 => JournalAddressfamilyV1::Inet6, + wasi::Addressfamily::Unix => JournalAddressfamilyV1::Unix, + } + } +} + +impl From for wasi::Addressfamily { + fn from(val: JournalAddressfamilyV1) -> Self { + match val { + JournalAddressfamilyV1::Unspec => wasi::Addressfamily::Unspec, + JournalAddressfamilyV1::Inet4 => wasi::Addressfamily::Inet4, + JournalAddressfamilyV1::Inet6 => wasi::Addressfamily::Inet6, + JournalAddressfamilyV1::Unix => wasi::Addressfamily::Unix, + } + } +} + +impl From<&'_ ArchivedJournalAddressfamilyV1> for wasi::Addressfamily { + fn from(val: &'_ ArchivedJournalAddressfamilyV1) -> Self { + match val { + ArchivedJournalAddressfamilyV1::Unspec => wasi::Addressfamily::Unspec, + ArchivedJournalAddressfamilyV1::Inet4 => wasi::Addressfamily::Inet4, + ArchivedJournalAddressfamilyV1::Inet6 => wasi::Addressfamily::Inet6, + ArchivedJournalAddressfamilyV1::Unix => wasi::Addressfamily::Unix, + } + } +} + +#[derive( + Debug, + Clone, + Copy, + PartialEq, + Eq, + PartialOrd, + Ord, + Hash, + serde::Serialize, + serde::Deserialize, + RkyvSerialize, + RkyvDeserialize, + Archive, +)] +#[archive_attr(derive(CheckBytes))] +pub enum JournalSocktypeV1 { + Unknown, + Stream, + Dgram, + Raw, + Seqpacket, +} + +impl From for JournalSocktypeV1 { + fn from(val: wasi::Socktype) -> Self { + match val { + wasi::Socktype::Stream => JournalSocktypeV1::Stream, + wasi::Socktype::Dgram => JournalSocktypeV1::Dgram, + wasi::Socktype::Raw => JournalSocktypeV1::Raw, + wasi::Socktype::Seqpacket => JournalSocktypeV1::Seqpacket, + wasi::Socktype::Unknown => JournalSocktypeV1::Unknown, + } + } +} + +impl From for wasi::Socktype { + fn from(val: JournalSocktypeV1) -> Self { + match val { + JournalSocktypeV1::Stream => wasi::Socktype::Stream, + JournalSocktypeV1::Dgram => wasi::Socktype::Dgram, + JournalSocktypeV1::Raw => wasi::Socktype::Raw, + JournalSocktypeV1::Seqpacket => wasi::Socktype::Seqpacket, + JournalSocktypeV1::Unknown => wasi::Socktype::Unknown, + } + } +} + +impl From<&'_ ArchivedJournalSocktypeV1> for wasi::Socktype { + fn from(val: &'_ ArchivedJournalSocktypeV1) -> Self { + match val { + ArchivedJournalSocktypeV1::Stream => wasi::Socktype::Stream, + ArchivedJournalSocktypeV1::Dgram => wasi::Socktype::Dgram, + ArchivedJournalSocktypeV1::Raw => wasi::Socktype::Raw, + ArchivedJournalSocktypeV1::Seqpacket => wasi::Socktype::Seqpacket, + ArchivedJournalSocktypeV1::Unknown => wasi::Socktype::Unknown, + } + } +} + +#[derive( + Debug, + Clone, + Copy, + PartialEq, + Eq, + PartialOrd, + Ord, + Hash, + serde::Serialize, + serde::Deserialize, + RkyvSerialize, + RkyvDeserialize, + Archive, +)] +#[archive_attr(derive(CheckBytes))] +pub enum JournalSockoptionV1 { + Noop, + ReusePort, + ReuseAddr, + NoDelay, + DontRoute, + OnlyV6, + Broadcast, + MulticastLoopV4, + MulticastLoopV6, + Promiscuous, + Listening, + LastError, + KeepAlive, + Linger, + OobInline, + RecvBufSize, + SendBufSize, + RecvLowat, + SendLowat, + RecvTimeout, + SendTimeout, + ConnectTimeout, + AcceptTimeout, + Ttl, + MulticastTtlV4, + Type, + Proto, +} + +impl From for JournalSockoptionV1 { + fn from(val: wasi::Sockoption) -> Self { + match val { + wasi::Sockoption::Noop => JournalSockoptionV1::Noop, + wasi::Sockoption::ReusePort => JournalSockoptionV1::ReusePort, + wasi::Sockoption::ReuseAddr => JournalSockoptionV1::ReuseAddr, + wasi::Sockoption::NoDelay => JournalSockoptionV1::NoDelay, + wasi::Sockoption::DontRoute => JournalSockoptionV1::DontRoute, + wasi::Sockoption::OnlyV6 => JournalSockoptionV1::OnlyV6, + wasi::Sockoption::Broadcast => JournalSockoptionV1::Broadcast, + wasi::Sockoption::MulticastLoopV4 => JournalSockoptionV1::MulticastLoopV4, + wasi::Sockoption::MulticastLoopV6 => JournalSockoptionV1::MulticastLoopV6, + wasi::Sockoption::Promiscuous => JournalSockoptionV1::Promiscuous, + wasi::Sockoption::Listening => JournalSockoptionV1::Listening, + wasi::Sockoption::LastError => JournalSockoptionV1::LastError, + wasi::Sockoption::KeepAlive => JournalSockoptionV1::KeepAlive, + wasi::Sockoption::Linger => JournalSockoptionV1::Linger, + wasi::Sockoption::OobInline => JournalSockoptionV1::OobInline, + wasi::Sockoption::RecvBufSize => JournalSockoptionV1::RecvBufSize, + wasi::Sockoption::SendBufSize => JournalSockoptionV1::SendBufSize, + wasi::Sockoption::RecvLowat => JournalSockoptionV1::RecvLowat, + wasi::Sockoption::SendLowat => JournalSockoptionV1::SendLowat, + wasi::Sockoption::RecvTimeout => JournalSockoptionV1::RecvTimeout, + wasi::Sockoption::SendTimeout => JournalSockoptionV1::SendTimeout, + wasi::Sockoption::ConnectTimeout => JournalSockoptionV1::ConnectTimeout, + wasi::Sockoption::AcceptTimeout => JournalSockoptionV1::AcceptTimeout, + wasi::Sockoption::Ttl => JournalSockoptionV1::Ttl, + wasi::Sockoption::MulticastTtlV4 => JournalSockoptionV1::MulticastTtlV4, + wasi::Sockoption::Type => JournalSockoptionV1::Type, + wasi::Sockoption::Proto => JournalSockoptionV1::Proto, + } + } +} + +impl From for wasi::Sockoption { + fn from(val: JournalSockoptionV1) -> Self { + match val { + JournalSockoptionV1::Noop => wasi::Sockoption::Noop, + JournalSockoptionV1::ReusePort => wasi::Sockoption::ReusePort, + JournalSockoptionV1::ReuseAddr => wasi::Sockoption::ReuseAddr, + JournalSockoptionV1::NoDelay => wasi::Sockoption::NoDelay, + JournalSockoptionV1::DontRoute => wasi::Sockoption::DontRoute, + JournalSockoptionV1::OnlyV6 => wasi::Sockoption::OnlyV6, + JournalSockoptionV1::Broadcast => wasi::Sockoption::Broadcast, + JournalSockoptionV1::MulticastLoopV4 => wasi::Sockoption::MulticastLoopV4, + JournalSockoptionV1::MulticastLoopV6 => wasi::Sockoption::MulticastLoopV6, + JournalSockoptionV1::Promiscuous => wasi::Sockoption::Promiscuous, + JournalSockoptionV1::Listening => wasi::Sockoption::Listening, + JournalSockoptionV1::LastError => wasi::Sockoption::LastError, + JournalSockoptionV1::KeepAlive => wasi::Sockoption::KeepAlive, + JournalSockoptionV1::Linger => wasi::Sockoption::Linger, + JournalSockoptionV1::OobInline => wasi::Sockoption::OobInline, + JournalSockoptionV1::RecvBufSize => wasi::Sockoption::RecvBufSize, + JournalSockoptionV1::SendBufSize => wasi::Sockoption::SendBufSize, + JournalSockoptionV1::RecvLowat => wasi::Sockoption::RecvLowat, + JournalSockoptionV1::SendLowat => wasi::Sockoption::SendLowat, + JournalSockoptionV1::RecvTimeout => wasi::Sockoption::RecvTimeout, + JournalSockoptionV1::SendTimeout => wasi::Sockoption::SendTimeout, + JournalSockoptionV1::ConnectTimeout => wasi::Sockoption::ConnectTimeout, + JournalSockoptionV1::AcceptTimeout => wasi::Sockoption::AcceptTimeout, + JournalSockoptionV1::Ttl => wasi::Sockoption::Ttl, + JournalSockoptionV1::MulticastTtlV4 => wasi::Sockoption::MulticastTtlV4, + JournalSockoptionV1::Type => wasi::Sockoption::Type, + JournalSockoptionV1::Proto => wasi::Sockoption::Proto, + } + } +} + +impl From<&'_ ArchivedJournalSockoptionV1> for wasi::Sockoption { + fn from(val: &'_ ArchivedJournalSockoptionV1) -> Self { + match val { + ArchivedJournalSockoptionV1::Noop => wasi::Sockoption::Noop, + ArchivedJournalSockoptionV1::ReusePort => wasi::Sockoption::ReusePort, + ArchivedJournalSockoptionV1::ReuseAddr => wasi::Sockoption::ReuseAddr, + ArchivedJournalSockoptionV1::NoDelay => wasi::Sockoption::NoDelay, + ArchivedJournalSockoptionV1::DontRoute => wasi::Sockoption::DontRoute, + ArchivedJournalSockoptionV1::OnlyV6 => wasi::Sockoption::OnlyV6, + ArchivedJournalSockoptionV1::Broadcast => wasi::Sockoption::Broadcast, + ArchivedJournalSockoptionV1::MulticastLoopV4 => wasi::Sockoption::MulticastLoopV4, + ArchivedJournalSockoptionV1::MulticastLoopV6 => wasi::Sockoption::MulticastLoopV6, + ArchivedJournalSockoptionV1::Promiscuous => wasi::Sockoption::Promiscuous, + ArchivedJournalSockoptionV1::Listening => wasi::Sockoption::Listening, + ArchivedJournalSockoptionV1::LastError => wasi::Sockoption::LastError, + ArchivedJournalSockoptionV1::KeepAlive => wasi::Sockoption::KeepAlive, + ArchivedJournalSockoptionV1::Linger => wasi::Sockoption::Linger, + ArchivedJournalSockoptionV1::OobInline => wasi::Sockoption::OobInline, + ArchivedJournalSockoptionV1::RecvBufSize => wasi::Sockoption::RecvBufSize, + ArchivedJournalSockoptionV1::SendBufSize => wasi::Sockoption::SendBufSize, + ArchivedJournalSockoptionV1::RecvLowat => wasi::Sockoption::RecvLowat, + ArchivedJournalSockoptionV1::SendLowat => wasi::Sockoption::SendLowat, + ArchivedJournalSockoptionV1::RecvTimeout => wasi::Sockoption::RecvTimeout, + ArchivedJournalSockoptionV1::SendTimeout => wasi::Sockoption::SendTimeout, + ArchivedJournalSockoptionV1::ConnectTimeout => wasi::Sockoption::ConnectTimeout, + ArchivedJournalSockoptionV1::AcceptTimeout => wasi::Sockoption::AcceptTimeout, + ArchivedJournalSockoptionV1::Ttl => wasi::Sockoption::Ttl, + ArchivedJournalSockoptionV1::MulticastTtlV4 => wasi::Sockoption::MulticastTtlV4, + ArchivedJournalSockoptionV1::Type => wasi::Sockoption::Type, + ArchivedJournalSockoptionV1::Proto => wasi::Sockoption::Proto, + } + } +} + +#[derive( + Debug, + Clone, + Copy, + PartialEq, + Eq, + PartialOrd, + Ord, + Hash, + serde::Serialize, + serde::Deserialize, + RkyvSerialize, + RkyvDeserialize, + Archive, +)] +#[archive_attr(derive(CheckBytes))] +pub enum JournalTimeTypeV1 { + ReadTimeout, + WriteTimeout, + AcceptTimeout, + ConnectTimeout, + BindTimeout, + Linger, +} + +impl From for JournalTimeTypeV1 { + fn from(val: TimeType) -> Self { + match val { + TimeType::ReadTimeout => JournalTimeTypeV1::ReadTimeout, + TimeType::WriteTimeout => JournalTimeTypeV1::WriteTimeout, + TimeType::AcceptTimeout => JournalTimeTypeV1::AcceptTimeout, + TimeType::ConnectTimeout => JournalTimeTypeV1::ConnectTimeout, + TimeType::BindTimeout => JournalTimeTypeV1::BindTimeout, + TimeType::Linger => JournalTimeTypeV1::Linger, + } + } +} + +impl From for TimeType { + fn from(val: JournalTimeTypeV1) -> Self { + match val { + JournalTimeTypeV1::ReadTimeout => TimeType::ReadTimeout, + JournalTimeTypeV1::WriteTimeout => TimeType::WriteTimeout, + JournalTimeTypeV1::AcceptTimeout => TimeType::AcceptTimeout, + JournalTimeTypeV1::ConnectTimeout => TimeType::ConnectTimeout, + JournalTimeTypeV1::BindTimeout => TimeType::BindTimeout, + JournalTimeTypeV1::Linger => TimeType::Linger, + } + } +} + +impl From<&'_ ArchivedJournalTimeTypeV1> for TimeType { + fn from(val: &'_ ArchivedJournalTimeTypeV1) -> Self { + match val { + ArchivedJournalTimeTypeV1::ReadTimeout => TimeType::ReadTimeout, + ArchivedJournalTimeTypeV1::WriteTimeout => TimeType::WriteTimeout, + ArchivedJournalTimeTypeV1::AcceptTimeout => TimeType::AcceptTimeout, + ArchivedJournalTimeTypeV1::ConnectTimeout => TimeType::ConnectTimeout, + ArchivedJournalTimeTypeV1::BindTimeout => TimeType::BindTimeout, + ArchivedJournalTimeTypeV1::Linger => TimeType::Linger, + } + } +} + +#[derive( + Debug, + Clone, + Copy, + PartialEq, + Eq, + PartialOrd, + Ord, + Hash, + serde::Serialize, + serde::Deserialize, + RkyvSerialize, + RkyvDeserialize, + Archive, +)] +#[archive_attr(derive(CheckBytes))] +pub enum JournalSocketShutdownV1 { + Read, + Write, + Both, +} + +impl From for JournalSocketShutdownV1 { + fn from(val: Shutdown) -> Self { + match val { + Shutdown::Read => JournalSocketShutdownV1::Read, + Shutdown::Write => JournalSocketShutdownV1::Write, + Shutdown::Both => JournalSocketShutdownV1::Both, } } } -impl<'a> From for JournalEntry<'a> { - fn from(value: JournalBatchEntry) -> Self { - match value { - JournalBatchEntry::InitModuleV1 { wasm_hash } => Self::InitModule { wasm_hash }, - JournalBatchEntry::UpdateMemoryRegionV1 { start, end, data } => { - Self::UpdateMemoryRegion { - region: start..end, - data: data.into(), - } - } - JournalBatchEntry::ProcessExitV1 { exit_code } => Self::ProcessExit { - exit_code: exit_code.map(|code| code.into()), - }, - JournalBatchEntry::SetThreadV1 { - id, - call_stack, - memory_stack, - store_data, - is_64bit, - } => Self::SetThread { - id: id.into(), - call_stack: call_stack.into(), - memory_stack: memory_stack.into(), - store_data: store_data.into(), - is_64bit, - }, - JournalBatchEntry::CloseThreadV1 { id, exit_code } => Self::CloseThread { - id: id.into(), - exit_code: exit_code.map(|code| code.into()), - }, - JournalBatchEntry::FileDescriptorWriteV1 { - data, - fd, - offset, - is_64bit, - } => Self::FileDescriptorWrite { - data: data.into(), - fd, - offset, - is_64bit, - }, - JournalBatchEntry::FileDescriptorSeekV1 { fd, offset, whence } => { - Self::FileDescriptorSeek { - fd, - offset, - whence: whence.into(), - } - } - JournalBatchEntry::OpenFileDescriptorV1 { - fd, - dirfd, - dirflags, - path, - o_flags, - fs_rights_base, - fs_rights_inheriting, - fs_flags, - } => Self::OpenFileDescriptor { - fd, - dirfd, - dirflags, - path: path.into(), - o_flags: wasi::Oflags::from_bits_truncate(o_flags), - fs_rights_base: wasi::Rights::from_bits_truncate(fs_rights_base), - fs_rights_inheriting: wasi::Rights::from_bits_truncate(fs_rights_inheriting), - fs_flags: wasi::Fdflags::from_bits_truncate(fs_flags), - }, - JournalBatchEntry::CloseFileDescriptorV1 { fd } => Self::CloseFileDescriptor { fd }, - JournalBatchEntry::RemoveDirectoryV1 { fd, path } => Self::RemoveDirectory { - fd, - path: path.into(), - }, - JournalBatchEntry::UnlinkFileV1 { fd, path } => Self::UnlinkFile { - fd, - path: path.into(), - }, - JournalBatchEntry::PathRenameV1 { - old_fd, - old_path, - new_fd, - new_path, - } => Self::PathRename { - old_fd, - old_path: old_path.into(), - new_fd, - new_path: new_path.into(), - }, - JournalBatchEntry::SnapshotV1 { - since_epoch, - trigger, - } => Self::Snapshot { - when: SystemTime::UNIX_EPOCH - .checked_add(since_epoch) - .unwrap_or(SystemTime::UNIX_EPOCH), - trigger: trigger.into(), - }, - JournalBatchEntry::SetClockTimeV1 { clock_id, time } => Self::SetClockTime { - clock_id: clock_id.into(), - time, - }, - JournalBatchEntry::RenumberFileDescriptorV1 { old_fd, new_fd } => { - Self::RenumberFileDescriptor { old_fd, new_fd } - } - JournalBatchEntry::DuplicateFileDescriptorV1 { - original_fd: old_fd, - copied_fd: new_fd, - } => Self::DuplicateFileDescriptor { - original_fd: old_fd, - copied_fd: new_fd, - }, - JournalBatchEntry::CreateDirectoryV1 { fd, path } => Self::CreateDirectory { - fd, - path: path.into(), - }, - JournalBatchEntry::PathSetTimesV1 { - fd, - path, - flags, - st_atim, - st_mtim, - fst_flags, - } => Self::PathSetTimes { - fd, - path: path.into(), - flags, - st_atim, - st_mtim, - fst_flags: wasi::Fstflags::from_bits_truncate(fst_flags), - }, - JournalBatchEntry::FileDescriptorSetTimesV1 { - fd, - st_atim, - st_mtim, - fst_flags, - } => Self::FileDescriptorSetTimes { - fd, - st_atim, - st_mtim, - fst_flags: wasi::Fstflags::from_bits_truncate(fst_flags), - }, - JournalBatchEntry::FileDescriptorSetSizeV1 { fd, st_size } => { - Self::FileDescriptorSetSize { fd, st_size } - } - JournalBatchEntry::FileDescriptorSetFlagsV1 { fd, flags } => { - Self::FileDescriptorSetFlags { - fd, - flags: Fdflags::from_bits_truncate(flags), - } - } - JournalBatchEntry::FileDescriptorSetRightsV1 { - fd, - fs_rights_base, - fs_rights_inheriting, - } => Self::FileDescriptorSetRights { - fd, - fs_rights_base: Rights::from_bits_truncate(fs_rights_base), - fs_rights_inheriting: Rights::from_bits_truncate(fs_rights_inheriting), - }, - JournalBatchEntry::FileDescriptorAdviseV1 { - fd, - offset, - len, - advice, - } => Self::FileDescriptorAdvise { - fd, - offset, - len, - advice: advice.into(), - }, - JournalBatchEntry::FileDescriptorAllocateV1 { fd, offset, len } => { - Self::FileDescriptorAllocate { fd, offset, len } - } - JournalBatchEntry::CreateHardLinkV1 { - old_fd, - old_path, - old_flags, - new_fd, - new_path, - } => Self::CreateHardLink { - old_fd, - old_path: old_path.into(), - old_flags, - new_fd, - new_path: new_path.into(), - }, - JournalBatchEntry::CreateSymbolicLinkV1 { - old_path, - fd, - new_path, - } => Self::CreateSymbolicLink { - old_path: old_path.into(), - fd, - new_path: new_path.into(), - }, - JournalBatchEntry::ChangeDirectoryV1 { path } => { - Self::ChangeDirectory { path: path.into() } - } - JournalBatchEntry::EpollCreateV1 { fd } => Self::EpollCreate { fd }, - JournalBatchEntry::EpollCtlV1 { - epfd, - op, - fd, - event, - } => Self::EpollCtl { - epfd, - op: op.into(), - fd, - event: event.map(|e| e.into()), - }, - JournalBatchEntry::TtySetV1 { - cols, - rows, - width, - height, - stdin_tty, - stdout_tty, - stderr_tty, - echo, - line_buffered, - line_feeds, - } => Self::TtySet { - tty: wasi::Tty { - cols, - rows, - width, - height, - stdin_tty, - stdout_tty, - stderr_tty, - echo, - line_buffered, - }, - line_feeds, - }, - JournalBatchEntry::CreatePipeV1 { fd1, fd2 } => Self::CreatePipe { fd1, fd2 }, - JournalBatchEntry::PortAddAddrV1 { cidr } => Self::PortAddAddr { cidr }, - JournalBatchEntry::PortDelAddrV1 { addr } => Self::PortDelAddr { addr }, - JournalBatchEntry::PortAddrClearV1 => Self::PortAddrClear, - JournalBatchEntry::PortBridgeV1 { - network, - token, - security, - } => Self::PortBridge { - network: network.into(), - token: token.into(), - security: security.into(), - }, - JournalBatchEntry::PortUnbridgeV1 => Self::PortUnbridge, - JournalBatchEntry::PortDhcpAcquireV1 => Self::PortDhcpAcquire, - JournalBatchEntry::PortGatewaySetV1 { ip } => Self::PortGatewaySet { ip }, - JournalBatchEntry::PortRouteAddV1 { - cidr, - via_router, - preferred_until, - expires_at, - } => Self::PortRouteAdd { - cidr, - via_router, - preferred_until, - expires_at, - }, - JournalBatchEntry::PortRouteClearV1 => Self::PortRouteClear, - JournalBatchEntry::PortRouteDelV1 { ip } => Self::PortRouteDel { ip }, - JournalBatchEntry::SocketOpenV1 { af, ty, pt, fd } => Self::SocketOpen { - af: af.into(), - ty: ty.into(), - pt: pt.try_into().unwrap_or(wasi::SockProto::Max), - fd, - }, - JournalBatchEntry::SocketListenV1 { fd, backlog } => Self::SocketListen { fd, backlog }, - JournalBatchEntry::SocketBindV1 { fd, addr } => Self::SocketBind { fd, addr }, - JournalBatchEntry::SocketConnectedV1 { fd, addr } => Self::SocketConnected { fd, addr }, - JournalBatchEntry::SocketAcceptedV1 { - listen_fd, - fd, - peer_addr, - fd_flags, - nonblocking, - } => Self::SocketAccepted { - listen_fd, - fd, - peer_addr, - fd_flags: Fdflags::from_bits_truncate(fd_flags), - nonblocking, - }, - JournalBatchEntry::SocketJoinIpv4MulticastV1 { - fd, - multiaddr, - iface, - } => Self::SocketJoinIpv4Multicast { - fd, - multiaddr, - iface, - }, - JournalBatchEntry::SocketJoinIpv6MulticastV1 { - fd, - multiaddr, - iface, - } => Self::SocketJoinIpv6Multicast { - fd, - multiaddr, - iface, - }, - JournalBatchEntry::SocketLeaveIpv4MulticastV1 { - fd, - multiaddr, - iface, - } => Self::SocketLeaveIpv4Multicast { - fd, - multiaddr, - iface, - }, - JournalBatchEntry::SocketLeaveIpv6MulticastV1 { - fd, - multiaddr, - iface, - } => Self::SocketLeaveIpv6Multicast { - fd, - multiaddr, - iface, - }, - JournalBatchEntry::SocketSendFileV1 { - socket_fd, - file_fd, - offset, - count, - } => Self::SocketSendFile { - socket_fd, - file_fd, - offset, - count, - }, - JournalBatchEntry::SocketSendToV1 { - fd, - data, - flags, - addr, - is_64bit, - } => Self::SocketSendTo { - fd, - data: data.into(), - flags, - addr, - is_64bit, - }, - JournalBatchEntry::SocketSendV1 { - fd, - data, - flags, - is_64bit, - } => Self::SocketSend { - fd, - data: data.into(), - flags, - is_64bit, - }, - JournalBatchEntry::SocketSetOptFlagV1 { fd, opt, flag } => Self::SocketSetOptFlag { - fd, - opt: opt.into(), - flag, - }, - JournalBatchEntry::SocketSetOptSizeV1 { fd, opt, size } => Self::SocketSetOptSize { - fd, - opt: opt.into(), - size, - }, - JournalBatchEntry::SocketSetOptTimeV1 { fd, ty, time } => Self::SocketSetOptTime { - fd, - ty: ty.into(), - time, - }, - JournalBatchEntry::SocketShutdownV1 { fd, how } => Self::SocketShutdown { - fd, - how: how.into(), - }, - JournalBatchEntry::CreateEventV1 { - initial_val, - flags, - fd, - } => Self::CreateEvent { - initial_val, - flags, - fd, - }, +impl From for Shutdown { + fn from(val: JournalSocketShutdownV1) -> Self { + match val { + JournalSocketShutdownV1::Read => Shutdown::Read, + JournalSocketShutdownV1::Write => Shutdown::Write, + JournalSocketShutdownV1::Both => Shutdown::Both, + } + } +} + +impl From<&'_ ArchivedJournalSocketShutdownV1> for Shutdown { + fn from(val: &'_ ArchivedJournalSocketShutdownV1) -> Self { + match val { + ArchivedJournalSocketShutdownV1::Read => Shutdown::Read, + ArchivedJournalSocketShutdownV1::Write => Shutdown::Write, + ArchivedJournalSocketShutdownV1::Both => Shutdown::Both, } } } -impl<'a> From<&'a ArchivedJournalBatchEntry> for JournalEntry<'a> { - fn from(value: &'a ArchivedJournalBatchEntry) -> Self { - type A = ArchivedJournalBatchEntry; +impl<'a> From> for JournalEntry<'a> { + fn from(value: ArchivedJournalEntry<'a>) -> Self { match value { - A::InitModuleV1 { wasm_hash } => Self::InitModule { - wasm_hash: *wasm_hash, - }, - A::UpdateMemoryRegionV1 { start, end, data } => Self::UpdateMemoryRegion { - region: *start..*end, + ArchivedJournalEntry::InitModuleV1(ArchivedJournalEntryInitModuleV1 { wasm_hash }) => { + Self::InitModule { + wasm_hash: *wasm_hash, + } + } + ArchivedJournalEntry::UpdateMemoryRegionV1( + ArchivedJournalEntryUpdateMemoryRegionV1 { start, end, data }, + ) => Self::UpdateMemoryRegion { + region: (*start)..(*end), data: Cow::Borrowed(data.as_ref()), }, - A::ProcessExitV1 { exit_code } => Self::ProcessExit { + ArchivedJournalEntry::ProcessExitV1(ArchivedJournalEntryProcessExitV1 { + exit_code, + }) => Self::ProcessExit { exit_code: exit_code.as_ref().map(|code| code.into()), }, - A::SetThreadV1 { + ArchivedJournalEntry::SetThreadV1(ArchivedJournalEntrySetThreadV1 { id, call_stack, memory_stack, store_data, is_64bit, - } => Self::SetThread { + }) => Self::SetThread { id: (*id).into(), - call_stack: Cow::Borrowed(call_stack.as_ref()), - memory_stack: Cow::Borrowed(memory_stack.as_ref()), - store_data: Cow::Borrowed(store_data.as_ref()), + call_stack: call_stack.as_ref().into(), + memory_stack: memory_stack.as_ref().into(), + store_data: store_data.as_ref().into(), is_64bit: *is_64bit, }, - A::CloseThreadV1 { id, exit_code } => Self::CloseThread { + ArchivedJournalEntry::CloseThreadV1(ArchivedJournalEntryCloseThreadV1 { + id, + exit_code, + }) => Self::CloseThread { id: (*id).into(), exit_code: exit_code.as_ref().map(|code| code.into()), }, - A::FileDescriptorWriteV1 { - data, - fd, - offset, - is_64bit, - } => Self::FileDescriptorWrite { - data: Cow::Borrowed(data.as_ref()), + ArchivedJournalEntry::FileDescriptorWriteV1( + ArchivedJournalEntryFileDescriptorWriteV1 { + data, + fd, + offset, + is_64bit, + }, + ) => Self::FileDescriptorWrite { + data: data.as_ref().into(), fd: *fd, offset: *offset, is_64bit: *is_64bit, }, - A::FileDescriptorSeekV1 { fd, offset, whence } => Self::FileDescriptorSeek { + ArchivedJournalEntry::FileDescriptorSeekV1( + ArchivedJournalEntryFileDescriptorSeekV1 { + fd, + offset, + ref whence, + }, + ) => Self::FileDescriptorSeek { fd: *fd, offset: *offset, whence: whence.into(), }, - A::OpenFileDescriptorV1 { - fd, - dirfd, - dirflags, - path, - o_flags, - fs_rights_base, - fs_rights_inheriting, - fs_flags, - } => Self::OpenFileDescriptor { + ArchivedJournalEntry::OpenFileDescriptorV1( + ArchivedJournalEntryOpenFileDescriptorV1 { + fd, + dirfd, + dirflags, + path, + o_flags, + fs_rights_base, + fs_rights_inheriting, + fs_flags, + }, + ) => Self::OpenFileDescriptor { fd: *fd, dirfd: *dirfd, dirflags: *dirflags, - path: Cow::Borrowed(path.as_str()), + path: path.as_ref().into(), o_flags: wasi::Oflags::from_bits_truncate(*o_flags), fs_rights_base: wasi::Rights::from_bits_truncate(*fs_rights_base), fs_rights_inheriting: wasi::Rights::from_bits_truncate(*fs_rights_inheriting), fs_flags: wasi::Fdflags::from_bits_truncate(*fs_flags), }, - A::CloseFileDescriptorV1 { fd } => Self::CloseFileDescriptor { fd: *fd }, - A::RemoveDirectoryV1 { fd, path } => Self::RemoveDirectory { - fd: *fd, - path: Cow::Borrowed(path.as_str()), - }, - A::UnlinkFileV1 { fd, path } => Self::UnlinkFile { + ArchivedJournalEntry::CloseFileDescriptorV1( + ArchivedJournalEntryCloseFileDescriptorV1 { fd }, + ) => Self::CloseFileDescriptor { fd: *fd }, + ArchivedJournalEntry::RemoveDirectoryV1(ArchivedJournalEntryRemoveDirectoryV1 { + fd, + path, + }) => Self::RemoveDirectory { fd: *fd, - path: Cow::Borrowed(path.as_str()), + path: path.as_ref().into(), }, - A::PathRenameV1 { + ArchivedJournalEntry::UnlinkFileV1(ArchivedJournalEntryUnlinkFileV1 { fd, path }) => { + Self::UnlinkFile { + fd: *fd, + path: path.as_ref().into(), + } + } + ArchivedJournalEntry::PathRenameV1(ArchivedJournalEntryPathRenameV1 { old_fd, old_path, new_fd, new_path, - } => Self::PathRename { + }) => Self::PathRename { old_fd: *old_fd, - old_path: Cow::Borrowed(old_path.as_str()), + old_path: old_path.as_ref().into(), new_fd: *new_fd, - new_path: Cow::Borrowed(new_path.as_str()), + new_path: new_path.as_ref().into(), }, - A::SnapshotV1 { + ArchivedJournalEntry::SnapshotV1(ArchivedJournalEntrySnapshotV1 { since_epoch, - trigger, - } => Self::Snapshot { + ref trigger, + }) => Self::Snapshot { when: SystemTime::UNIX_EPOCH - .checked_add(Duration::from_nanos(since_epoch.as_nanos() as u64)) + .checked_add(since_epoch.clone().try_into().unwrap()) .unwrap_or(SystemTime::UNIX_EPOCH), trigger: trigger.into(), }, - A::SetClockTimeV1 { clock_id, time } => Self::SetClockTime { + ArchivedJournalEntry::SetClockTimeV1(ArchivedJournalEntrySetClockTimeV1 { + ref clock_id, + time, + }) => Self::SetClockTime { clock_id: clock_id.into(), time: *time, }, - A::RenumberFileDescriptorV1 { old_fd, new_fd } => Self::RenumberFileDescriptor { + ArchivedJournalEntry::RenumberFileDescriptorV1( + ArchivedJournalEntryRenumberFileDescriptorV1 { old_fd, new_fd }, + ) => Self::RenumberFileDescriptor { old_fd: *old_fd, new_fd: *new_fd, }, - A::DuplicateFileDescriptorV1 { - original_fd, - copied_fd, - } => Self::DuplicateFileDescriptor { - original_fd: *original_fd, - copied_fd: *copied_fd, + ArchivedJournalEntry::DuplicateFileDescriptorV1( + ArchivedJournalEntryDuplicateFileDescriptorV1 { + original_fd: old_fd, + copied_fd: new_fd, + }, + ) => Self::DuplicateFileDescriptor { + original_fd: *old_fd, + copied_fd: *new_fd, }, - A::CreateDirectoryV1 { fd, path } => Self::CreateDirectory { + ArchivedJournalEntry::CreateDirectoryV1(ArchivedJournalEntryCreateDirectoryV1 { + fd, + path, + }) => Self::CreateDirectory { fd: *fd, - path: Cow::Borrowed(path.as_str()), + path: path.as_ref().into(), }, - A::PathSetTimesV1 { + ArchivedJournalEntry::PathSetTimesV1(ArchivedJournalEntryPathSetTimesV1 { fd, path, flags, st_atim, st_mtim, fst_flags, - } => Self::PathSetTimes { + }) => Self::PathSetTimes { fd: *fd, - path: Cow::Borrowed(path.as_str()), + path: path.as_ref().into(), flags: *flags, st_atim: *st_atim, st_mtim: *st_mtim, fst_flags: wasi::Fstflags::from_bits_truncate(*fst_flags), }, - A::FileDescriptorSetTimesV1 { - fd, - st_atim, - st_mtim, - fst_flags, - } => Self::FileDescriptorSetTimes { + ArchivedJournalEntry::FileDescriptorSetTimesV1( + ArchivedJournalEntryFileDescriptorSetTimesV1 { + fd, + st_atim, + st_mtim, + fst_flags, + }, + ) => Self::FileDescriptorSetTimes { fd: *fd, st_atim: *st_atim, st_mtim: *st_mtim, fst_flags: wasi::Fstflags::from_bits_truncate(*fst_flags), }, - A::FileDescriptorSetSizeV1 { fd, st_size } => Self::FileDescriptorSetSize { + ArchivedJournalEntry::FileDescriptorSetSizeV1( + ArchivedJournalEntryFileDescriptorSetSizeV1 { fd, st_size }, + ) => Self::FileDescriptorSetSize { fd: *fd, st_size: *st_size, }, - A::FileDescriptorSetFlagsV1 { fd, flags } => Self::FileDescriptorSetFlags { + ArchivedJournalEntry::FileDescriptorSetFlagsV1( + ArchivedJournalEntryFileDescriptorSetFlagsV1 { fd, flags }, + ) => Self::FileDescriptorSetFlags { fd: *fd, flags: Fdflags::from_bits_truncate(*flags), }, - A::FileDescriptorSetRightsV1 { - fd, - fs_rights_base, - fs_rights_inheriting, - } => Self::FileDescriptorSetRights { + ArchivedJournalEntry::FileDescriptorSetRightsV1( + ArchivedJournalEntryFileDescriptorSetRightsV1 { + fd, + fs_rights_base, + fs_rights_inheriting, + }, + ) => Self::FileDescriptorSetRights { fd: *fd, fs_rights_base: Rights::from_bits_truncate(*fs_rights_base), fs_rights_inheriting: Rights::from_bits_truncate(*fs_rights_inheriting), }, - A::FileDescriptorAdviseV1 { - fd, - offset, - len, - advice, - } => Self::FileDescriptorAdvise { + ArchivedJournalEntry::FileDescriptorAdviseV1( + ArchivedJournalEntryFileDescriptorAdviseV1 { + fd, + offset, + len, + ref advice, + }, + ) => Self::FileDescriptorAdvise { fd: *fd, offset: *offset, len: *len, advice: advice.into(), }, - A::FileDescriptorAllocateV1 { fd, offset, len } => Self::FileDescriptorAllocate { + ArchivedJournalEntry::FileDescriptorAllocateV1( + ArchivedJournalEntryFileDescriptorAllocateV1 { fd, offset, len }, + ) => Self::FileDescriptorAllocate { fd: *fd, offset: *offset, len: *len, }, - A::CreateHardLinkV1 { + ArchivedJournalEntry::CreateHardLinkV1(ArchivedJournalEntryCreateHardLinkV1 { old_fd, old_path, old_flags, new_fd, new_path, - } => Self::CreateHardLink { + }) => Self::CreateHardLink { old_fd: *old_fd, - old_path: Cow::Borrowed(old_path.as_str()), + old_path: old_path.as_ref().into(), old_flags: *old_flags, new_fd: *new_fd, - new_path: Cow::Borrowed(new_path.as_str()), + new_path: new_path.as_ref().into(), }, - A::CreateSymbolicLinkV1 { - old_path, - fd, - new_path, - } => Self::CreateSymbolicLink { - old_path: Cow::Borrowed(old_path.as_str()), + ArchivedJournalEntry::CreateSymbolicLinkV1( + ArchivedJournalEntryCreateSymbolicLinkV1 { + old_path, + fd, + new_path, + }, + ) => Self::CreateSymbolicLink { + old_path: old_path.as_ref().into(), fd: *fd, - new_path: Cow::Borrowed(new_path.as_str()), + new_path: new_path.as_ref().into(), }, - A::ChangeDirectoryV1 { path } => Self::ChangeDirectory { - path: Cow::Borrowed(path.as_str()), + ArchivedJournalEntry::ChangeDirectoryV1(ArchivedJournalEntryChangeDirectoryV1 { + path, + }) => Self::ChangeDirectory { + path: path.as_ref().into(), }, - A::EpollCreateV1 { fd } => Self::EpollCreate { fd: *fd }, - A::EpollCtlV1 { + ArchivedJournalEntry::EpollCreateV1(ArchivedJournalEntryEpollCreateV1 { fd }) => { + Self::EpollCreate { fd: *fd } + } + ArchivedJournalEntry::EpollCtlV1(ArchivedJournalEntryEpollCtlV1 { epfd, - op, + ref op, fd, - event, - } => Self::EpollCtl { + ref event, + }) => Self::EpollCtl { epfd: *epfd, op: op.into(), fd: *fd, event: event.as_ref().map(|e| e.into()), }, - A::TtySetV1 { + ArchivedJournalEntry::TtySetV1(ArchivedJournalEntryTtySetV1 { cols, rows, width, @@ -2111,7 +2547,7 @@ impl<'a> From<&'a ArchivedJournalBatchEntry> for JournalEntry<'a> { echo, line_buffered, line_feeds, - } => Self::TtySet { + }) => Self::TtySet { tty: wasi::Tty { cols: *cols, rows: *rows, @@ -2125,38 +2561,46 @@ impl<'a> From<&'a ArchivedJournalBatchEntry> for JournalEntry<'a> { }, line_feeds: *line_feeds, }, - A::CreatePipeV1 { fd1, fd2 } => Self::CreatePipe { - fd1: *fd1, - fd2: *fd2, - }, - A::PortAddAddrV1 { cidr } => Self::PortAddAddr { - cidr: IpCidr { - ip: cidr.ip.as_ipaddr(), - prefix: cidr.prefix, - }, - }, - A::PortDelAddrV1 { addr } => Self::PortDelAddr { - addr: addr.as_ipaddr(), - }, - A::PortAddrClearV1 => Self::PortAddrClear, - A::PortBridgeV1 { + ArchivedJournalEntry::CreatePipeV1(ArchivedJournalEntryCreatePipeV1 { fd1, fd2 }) => { + Self::CreatePipe { + fd1: *fd1, + fd2: *fd2, + } + } + ArchivedJournalEntry::PortAddAddrV1(ArchivedJournalEntryPortAddAddrV1 { cidr }) => { + Self::PortAddAddr { + cidr: IpCidr { + ip: cidr.ip.as_ipaddr(), + prefix: cidr.prefix, + }, + } + } + ArchivedJournalEntry::PortDelAddrV1(ArchivedJournalEntryPortDelAddrV1 { addr }) => { + Self::PortDelAddr { + addr: addr.as_ipaddr(), + } + } + ArchivedJournalEntry::PortAddrClearV1 => Self::PortAddrClear, + ArchivedJournalEntry::PortBridgeV1(ArchivedJournalEntryPortBridgeV1 { network, token, - security, - } => Self::PortBridge { - network: Cow::Borrowed(network.as_str()), - token: Cow::Borrowed(token.as_str()), + ref security, + }) => Self::PortBridge { + network: network.as_ref().into(), + token: token.as_ref().into(), security: security.into(), }, - A::PortUnbridgeV1 => Self::PortUnbridge, - A::PortDhcpAcquireV1 => Self::PortDhcpAcquire, - A::PortGatewaySetV1 { ip } => Self::PortGatewaySet { ip: ip.as_ipaddr() }, - A::PortRouteAddV1 { + ArchivedJournalEntry::PortUnbridgeV1 => Self::PortUnbridge, + ArchivedJournalEntry::PortDhcpAcquireV1 => Self::PortDhcpAcquire, + ArchivedJournalEntry::PortGatewaySetV1(ArchivedJournalEntryPortGatewaySetV1 { ip }) => { + Self::PortGatewaySet { ip: ip.as_ipaddr() } + } + ArchivedJournalEntry::PortRouteAddV1(ArchivedJournalEntryPortRouteAddV1 { cidr, via_router, preferred_until, expires_at, - } => Self::PortRouteAdd { + }) => Self::PortRouteAdd { cidr: IpCidr { ip: cidr.ip.as_ipaddr(), prefix: cidr.prefix, @@ -2164,141 +2608,177 @@ impl<'a> From<&'a ArchivedJournalBatchEntry> for JournalEntry<'a> { via_router: via_router.as_ipaddr(), preferred_until: preferred_until .as_ref() - .map(|d| Duration::from_nanos(d.as_nanos() as u64)), + .map(|time| time.clone().try_into().unwrap()), expires_at: expires_at .as_ref() - .map(|d| Duration::from_nanos(d.as_nanos() as u64)), + .map(|time| time.clone().try_into().unwrap()), }, - A::PortRouteClearV1 => Self::PortRouteClear, - A::PortRouteDelV1 { ip } => Self::PortRouteDel { ip: ip.as_ipaddr() }, - A::SocketOpenV1 { af, ty, pt, fd } => Self::SocketOpen { + ArchivedJournalEntry::PortRouteClearV1 => Self::PortRouteClear, + ArchivedJournalEntry::PortRouteDelV1(ArchivedJournalEntryPortRouteDelV1 { ip }) => { + Self::PortRouteDel { ip: ip.as_ipaddr() } + } + ArchivedJournalEntry::SocketOpenV1(ArchivedJournalEntrySocketOpenV1 { + ref af, + ref ty, + pt, + fd, + }) => Self::SocketOpen { af: af.into(), ty: ty.into(), pt: (*pt).try_into().unwrap_or(wasi::SockProto::Max), fd: *fd, }, - A::SocketListenV1 { fd, backlog } => Self::SocketListen { + ArchivedJournalEntry::SocketListenV1(ArchivedJournalEntrySocketListenV1 { + fd, + backlog, + }) => Self::SocketListen { fd: *fd, backlog: *backlog, }, - A::SocketBindV1 { fd, addr } => Self::SocketBind { - fd: *fd, - addr: addr.as_socket_addr(), - }, - A::SocketConnectedV1 { fd, addr } => Self::SocketConnected { + ArchivedJournalEntry::SocketBindV1(ArchivedJournalEntrySocketBindV1 { fd, addr }) => { + Self::SocketBind { + fd: *fd, + addr: addr.as_socket_addr(), + } + } + ArchivedJournalEntry::SocketConnectedV1(ArchivedJournalEntrySocketConnectedV1 { + fd, + addr, + }) => Self::SocketConnected { fd: *fd, addr: addr.as_socket_addr(), }, - A::SocketAcceptedV1 { + ArchivedJournalEntry::SocketAcceptedV1(ArchivedJournalEntrySocketAcceptedV1 { listen_fd, fd, peer_addr, fd_flags, nonblocking, - } => Self::SocketAccepted { + }) => Self::SocketAccepted { listen_fd: *listen_fd, fd: *fd, peer_addr: peer_addr.as_socket_addr(), fd_flags: Fdflags::from_bits_truncate(*fd_flags), nonblocking: *nonblocking, }, - A::SocketJoinIpv4MulticastV1 { - fd, - multiaddr, - iface, - } => Self::SocketJoinIpv4Multicast { + ArchivedJournalEntry::SocketJoinIpv4MulticastV1( + ArchivedJournalEntrySocketJoinIpv4MulticastV1 { + fd, + multiaddr, + iface, + }, + ) => Self::SocketJoinIpv4Multicast { fd: *fd, multiaddr: multiaddr.as_ipv4(), iface: iface.as_ipv4(), }, - A::SocketJoinIpv6MulticastV1 { - fd, - multiaddr, - iface, - } => Self::SocketJoinIpv6Multicast { + ArchivedJournalEntry::SocketJoinIpv6MulticastV1( + ArchivedJournalEntrySocketJoinIpv6MulticastV1 { + fd, + multiaddr, + iface, + }, + ) => Self::SocketJoinIpv6Multicast { fd: *fd, multiaddr: multiaddr.as_ipv6(), iface: *iface, }, - A::SocketLeaveIpv4MulticastV1 { - fd, - multiaddr, - iface, - } => Self::SocketLeaveIpv4Multicast { + ArchivedJournalEntry::SocketLeaveIpv4MulticastV1( + ArchivedJournalEntrySocketLeaveIpv4MulticastV1 { + fd, + multiaddr, + iface, + }, + ) => Self::SocketLeaveIpv4Multicast { fd: *fd, multiaddr: multiaddr.as_ipv4(), iface: iface.as_ipv4(), }, - A::SocketLeaveIpv6MulticastV1 { - fd, - multiaddr, - iface, - } => Self::SocketLeaveIpv6Multicast { + ArchivedJournalEntry::SocketLeaveIpv6MulticastV1( + ArchivedJournalEntrySocketLeaveIpv6MulticastV1 { + fd, + multiaddr, + iface, + }, + ) => Self::SocketLeaveIpv6Multicast { fd: *fd, multiaddr: multiaddr.as_ipv6(), iface: *iface, }, - A::SocketSendFileV1 { + ArchivedJournalEntry::SocketSendFileV1(ArchivedJournalEntrySocketSendFileV1 { socket_fd, file_fd, offset, count, - } => Self::SocketSendFile { + }) => Self::SocketSendFile { socket_fd: *socket_fd, file_fd: *file_fd, offset: *offset, count: *count, }, - A::SocketSendToV1 { + ArchivedJournalEntry::SocketSendToV1(ArchivedJournalEntrySocketSendToV1 { fd, data, flags, addr, is_64bit, - } => Self::SocketSendTo { + }) => Self::SocketSendTo { fd: *fd, - data: Cow::Borrowed(data.as_ref()), + data: data.as_ref().into(), flags: *flags, addr: addr.as_socket_addr(), is_64bit: *is_64bit, }, - A::SocketSendV1 { + ArchivedJournalEntry::SocketSendV1(ArchivedJournalEntrySocketSendV1 { fd, data, flags, is_64bit, - } => Self::SocketSend { + }) => Self::SocketSend { fd: *fd, - data: Cow::Borrowed(data.as_ref()), + data: data.as_ref().into(), flags: *flags, is_64bit: *is_64bit, }, - A::SocketSetOptFlagV1 { fd, opt, flag } => Self::SocketSetOptFlag { + ArchivedJournalEntry::SocketSetOptFlagV1(ArchivedJournalEntrySocketSetOptFlagV1 { + fd, + ref opt, + flag, + }) => Self::SocketSetOptFlag { fd: *fd, opt: opt.into(), flag: *flag, }, - A::SocketSetOptSizeV1 { fd, opt, size } => Self::SocketSetOptSize { + ArchivedJournalEntry::SocketSetOptSizeV1(ArchivedJournalEntrySocketSetOptSizeV1 { + fd, + ref opt, + size, + }) => Self::SocketSetOptSize { fd: *fd, opt: opt.into(), size: *size, }, - A::SocketSetOptTimeV1 { fd, ty, time } => Self::SocketSetOptTime { + ArchivedJournalEntry::SocketSetOptTimeV1(ArchivedJournalEntrySocketSetOptTimeV1 { + fd, + ref ty, + time, + }) => Self::SocketSetOptTime { fd: *fd, ty: ty.into(), - time: time - .as_ref() - .map(|d| Duration::from_nanos(d.as_nanos() as u64)), + time: time.as_ref().map(|time| time.clone().try_into().unwrap()), }, - A::SocketShutdownV1 { fd, how } => Self::SocketShutdown { + ArchivedJournalEntry::SocketShutdownV1(ArchivedJournalEntrySocketShutdownV1 { + fd, + ref how, + }) => Self::SocketShutdown { fd: *fd, how: how.into(), }, - A::CreateEventV1 { + ArchivedJournalEntry::CreateEventV1(ArchivedJournalEntryCreateEventV1 { initial_val, flags, fd, - } => Self::CreateEvent { + }) => Self::CreateEvent { initial_val: *initial_val, flags: *flags, fd: *fd, @@ -2306,3 +2786,621 @@ impl<'a> From<&'a ArchivedJournalBatchEntry> for JournalEntry<'a> { } } } + +#[cfg(test)] +mod tests { + use rkyv::ser::serializers::{ + AllocScratch, CompositeSerializer, SharedSerializeMap, WriteSerializer, + }; + use wasmer_wasix_types::wasi::{Addressfamily, SockProto, Socktype, Tty}; + + use super::*; + + pub fn run_test<'a>(record: JournalEntry<'a>) { + tracing::info!("record2: {:?}", record); + + // Determine the record type + let record_type = record.archive_record_type(); + tracing::info!("record_type: {:?}", record_type); + + // Serialize it + let mut buffer = Vec::new(); + let mut serializer = CompositeSerializer::new( + WriteSerializer::new(&mut buffer), + AllocScratch::default(), + SharedSerializeMap::default(), + ); + + record.clone().serialize_archive(&mut serializer).unwrap(); + let buffer = &buffer[..]; + if buffer.len() < 20 { + tracing::info!("buffer: {:x?}", buffer); + } else { + tracing::info!("buffer_len: {}", buffer.len()); + } + + // Deserialize it + let record2 = unsafe { record_type.deserialize_archive(buffer) }; + tracing::info!("record2: {:?}", record2); + + // Check it + assert_eq!(record, record2); + + // Now make it static and check it again + let record3 = record2.into_owned(); + tracing::info!("record3: {:?}", record3); + assert_eq!(record, record3); + } + + #[tracing_test::traced_test] + #[test] + pub fn test_record_init_module() { + run_test(JournalEntry::InitModule { + wasm_hash: [13u8; 32], + }); + } + + #[tracing_test::traced_test] + #[test] + pub fn test_record_process_exit() { + run_test(JournalEntry::ProcessExit { + exit_code: Some(ExitCode::Errno(Errno::Fault)), + }); + } + + #[tracing_test::traced_test] + #[test] + pub fn test_record_set_thread() { + run_test(JournalEntry::SetThread { + id: 1234u32.into(), + call_stack: vec![1, 2, 3].into(), + memory_stack: vec![4, 5, 6, 7].into(), + store_data: vec![10, 11].into(), + is_64bit: true, + }); + } + + #[tracing_test::traced_test] + #[test] + pub fn test_record_close_thread() { + run_test(JournalEntry::CloseThread { + id: 987u32.into(), + exit_code: Some(ExitCode::Errno(Errno::Fault)), + }); + } + + #[tracing_test::traced_test] + #[test] + pub fn test_record_descriptor_seek() { + run_test(JournalEntry::FileDescriptorSeek { + fd: 765u32, + offset: 9183722450971234i64, + whence: Whence::End, + }); + } + + #[tracing_test::traced_test] + #[test] + pub fn test_record_descriptor_write() { + run_test(JournalEntry::FileDescriptorWrite { + fd: 54321u32, + offset: 13897412934u64, + data: vec![74u8, 98u8, 36u8].into(), + is_64bit: true, + }); + } + + #[tracing_test::traced_test] + #[test] + pub fn test_record_update_memory() { + run_test(JournalEntry::UpdateMemoryRegion { + region: 76u64..8237453u64, + data: [74u8; 40960].to_vec().into(), + }); + } + + #[tracing_test::traced_test] + #[test] + pub fn test_record_set_clock_time() { + run_test(JournalEntry::SetClockTime { + clock_id: Snapshot0Clockid::Realtime, + time: 7912837412934u64, + }); + } + + #[tracing_test::traced_test] + #[test] + pub fn test_record_open_file_descriptor() { + run_test(JournalEntry::OpenFileDescriptor { + fd: 298745u32, + dirfd: 23458922u32, + dirflags: 134512345, + path: "/blah".into(), + o_flags: Oflags::all(), + fs_rights_base: Rights::all(), + fs_rights_inheriting: Rights::all(), + fs_flags: Fdflags::all(), + }); + } + + #[tracing_test::traced_test] + #[test] + pub fn test_record_close_descriptor() { + run_test(JournalEntry::CloseFileDescriptor { fd: 23845732u32 }); + } + + #[tracing_test::traced_test] + #[test] + pub fn test_record_renumber_file_descriptor() { + run_test(JournalEntry::RenumberFileDescriptor { + old_fd: 27834u32, + new_fd: 398452345u32, + }); + } + + #[tracing_test::traced_test] + #[test] + pub fn test_record_duplicate_file_descriptor() { + run_test(JournalEntry::DuplicateFileDescriptor { + original_fd: 23482934u32, + copied_fd: 9384529u32, + }); + } + + #[tracing_test::traced_test] + #[test] + pub fn test_record_create_directory() { + run_test(JournalEntry::CreateDirectory { + fd: 238472u32, + path: "/joasjdf/asdfn".into(), + }); + } + + #[tracing_test::traced_test] + #[test] + pub fn test_record_remove_directory() { + run_test(JournalEntry::RemoveDirectory { + fd: 23894952u32, + path: "/blahblah".into(), + }); + } + + #[tracing_test::traced_test] + #[test] + pub fn test_record_path_set_times() { + run_test(JournalEntry::PathSetTimes { + fd: 1238934u32, + flags: 234523, + path: "/".into(), + st_atim: 923452345, + st_mtim: 350, + fst_flags: Fstflags::all(), + }); + } + + #[tracing_test::traced_test] + #[test] + pub fn test_record_file_descriptor_set_times() { + run_test(JournalEntry::FileDescriptorSetTimes { + fd: 898785u32, + st_atim: 29834952345, + st_mtim: 239845892345, + fst_flags: Fstflags::all(), + }); + } + + #[tracing_test::traced_test] + #[test] + pub fn test_record_file_descriptor_set_size() { + run_test(JournalEntry::FileDescriptorSetSize { + fd: 34958234u32, + st_size: 234958293845u64, + }); + } + + #[tracing_test::traced_test] + #[test] + pub fn test_record_file_descriptor_set_flags() { + run_test(JournalEntry::FileDescriptorSetFlags { + fd: 982348752u32, + flags: Fdflags::all(), + }); + } + + #[tracing_test::traced_test] + #[test] + pub fn test_record_file_descriptor_set_rights() { + run_test(JournalEntry::FileDescriptorSetRights { + fd: 872345u32, + fs_rights_base: Rights::all(), + fs_rights_inheriting: Rights::all(), + }); + } + + #[tracing_test::traced_test] + #[test] + pub fn test_record_file_descriptor_advise() { + run_test(JournalEntry::FileDescriptorAdvise { + fd: 298434u32, + offset: 92834529092345, + len: 23485928345, + advice: Advice::Random, + }); + } + + #[tracing_test::traced_test] + #[test] + pub fn test_record_file_descriptor_allocate() { + run_test(JournalEntry::FileDescriptorAllocate { + fd: 2934852, + offset: 23489582934523, + len: 9845982345, + }); + } + + #[tracing_test::traced_test] + #[test] + pub fn test_record_create_hard_link() { + run_test(JournalEntry::CreateHardLink { + old_fd: 324983845, + old_path: "/asjdfiasidfasdf".into(), + old_flags: 234857, + new_fd: 34958345, + new_path: "/ashdufnasd".into(), + }); + } + + #[tracing_test::traced_test] + #[test] + pub fn test_record_create_symbolic_link() { + run_test(JournalEntry::CreateSymbolicLink { + old_path: "/asjbndfjasdf/asdafasdf".into(), + fd: 235422345, + new_path: "/asdf".into(), + }); + } + + #[tracing_test::traced_test] + #[test] + pub fn test_record_unlink_file() { + run_test(JournalEntry::UnlinkFile { + fd: 32452345, + path: "/asdfasd".into(), + }); + } + + #[tracing_test::traced_test] + #[test] + pub fn test_record_path_rename() { + run_test(JournalEntry::PathRename { + old_fd: 32451345, + old_path: "/asdfasdfas/asdfasdf".into(), + new_fd: 23452345, + new_path: "/ahgfdfghdfghdfgh".into(), + }); + } + + #[tracing_test::traced_test] + #[test] + pub fn test_record_change_directory() { + run_test(JournalEntry::ChangeDirectory { + path: "/etc".to_string().into(), + }); + } + + #[tracing_test::traced_test] + #[test] + pub fn test_record_epoll_create() { + run_test(JournalEntry::EpollCreate { fd: 45384752 }); + } + + #[tracing_test::traced_test] + #[test] + pub fn test_record_epoll_ctl() { + run_test(JournalEntry::EpollCtl { + epfd: 34523455, + op: EpollCtl::Unknown, + fd: 23452345, + event: Some(EpollEventCtl { + events: EpollType::all(), + ptr: 32452345, + fd: 23452345, + data1: 1235245756, + data2: 23452345, + }), + }); + } + + #[tracing_test::traced_test] + #[test] + pub fn test_record_tty_set() { + run_test(JournalEntry::TtySet { + tty: Tty { + cols: 1234, + rows: 6754, + width: 4563456, + height: 345, + stdin_tty: true, + stdout_tty: false, + stderr_tty: true, + echo: true, + line_buffered: true, + }, + line_feeds: true, + }); + } + + #[tracing_test::traced_test] + #[test] + pub fn test_record_create_pipe() { + run_test(JournalEntry::CreatePipe { + fd1: 3452345, + fd2: 2345163, + }); + } + + #[tracing_test::traced_test] + #[test] + pub fn test_record_create_event() { + run_test(JournalEntry::CreateEvent { + initial_val: 13451345, + flags: 2343, + fd: 5836544, + }); + } + + #[tracing_test::traced_test] + #[test] + pub fn test_record_port_add_addr() { + run_test(JournalEntry::PortAddAddr { + cidr: IpCidr { + ip: Ipv4Addr::LOCALHOST.into(), + prefix: 24, + }, + }); + } + + #[tracing_test::traced_test] + #[test] + pub fn test_record_del_addr() { + run_test(JournalEntry::PortDelAddr { + addr: Ipv6Addr::LOCALHOST.into(), + }); + } + + #[tracing_test::traced_test] + #[test] + pub fn test_record_addr_clear() { + run_test(JournalEntry::PortAddrClear); + } + + #[tracing_test::traced_test] + #[test] + pub fn test_record_port_bridge() { + run_test(JournalEntry::PortBridge { + network: "mynetwork".into(), + token: format!("blh blah").into(), + security: StreamSecurity::ClassicEncryption, + }); + } + + #[tracing_test::traced_test] + #[test] + pub fn test_record_unbridge() { + run_test(JournalEntry::PortUnbridge); + } + + #[tracing_test::traced_test] + #[test] + pub fn test_record_dhcp_acquire() { + run_test(JournalEntry::PortDhcpAcquire); + } + + #[tracing_test::traced_test] + #[test] + pub fn test_record_gateway_set() { + run_test(JournalEntry::PortGatewaySet { + ip: Ipv4Addr::new(12, 34, 136, 220).into(), + }); + } + + #[tracing_test::traced_test] + #[test] + pub fn test_record_route_add() { + run_test(JournalEntry::PortRouteAdd { + cidr: IpCidr { + ip: Ipv4Addr::LOCALHOST.into(), + prefix: 24, + }, + via_router: Ipv4Addr::LOCALHOST.into(), + preferred_until: Some(Duration::MAX), + expires_at: Some(Duration::ZERO), + }); + } + + #[tracing_test::traced_test] + #[test] + pub fn test_record_route_clear() { + run_test(JournalEntry::PortRouteClear); + } + + #[tracing_test::traced_test] + #[test] + pub fn test_record_route_del() { + run_test(JournalEntry::PortRouteDel { + ip: Ipv4Addr::BROADCAST.into(), + }); + } + + #[tracing_test::traced_test] + #[test] + pub fn test_record_socket_open() { + run_test(JournalEntry::SocketOpen { + af: Addressfamily::Inet6, + ty: Socktype::Stream, + pt: SockProto::Tcp, + fd: 23452345, + }); + } + + #[tracing_test::traced_test] + #[test] + pub fn test_record_socket_listen() { + run_test(JournalEntry::SocketListen { + fd: 12341234, + backlog: 123, + }); + } + + #[tracing_test::traced_test] + #[test] + pub fn test_record_socket_bind() { + run_test(JournalEntry::SocketBind { + fd: 2341234, + addr: SocketAddr::new(Ipv6Addr::UNSPECIFIED.into(), 1234), + }); + } + + #[tracing_test::traced_test] + #[test] + pub fn test_record_socket_connected() { + run_test(JournalEntry::SocketConnected { + fd: 12341, + addr: SocketAddr::new(Ipv6Addr::UNSPECIFIED.into(), 1234), + }); + } + + #[tracing_test::traced_test] + #[test] + pub fn test_record_socket_accepted() { + run_test(JournalEntry::SocketAccepted { + listen_fd: 21234, + fd: 1, + peer_addr: SocketAddr::new(Ipv6Addr::UNSPECIFIED.into(), 3452), + fd_flags: Fdflags::all(), + nonblocking: true, + }); + } + + #[tracing_test::traced_test] + #[test] + pub fn test_record_socket_join_ipv4_multicast() { + run_test(JournalEntry::SocketJoinIpv4Multicast { + fd: 12, + multiaddr: Ipv4Addr::new(123, 123, 123, 123).into(), + iface: Ipv4Addr::new(128, 0, 0, 1).into(), + }); + } + + #[tracing_test::traced_test] + #[test] + pub fn test_record_socket_join_ipv6_multicast() { + run_test(JournalEntry::SocketJoinIpv6Multicast { + fd: 12, + multiaddr: Ipv6Addr::new(123, 123, 123, 123, 1234, 12663, 31, 1324).into(), + iface: 23541, + }); + } + + #[tracing_test::traced_test] + #[test] + pub fn test_record_socket_leave_ipv4_multicast() { + run_test(JournalEntry::SocketLeaveIpv4Multicast { + fd: 12, + multiaddr: Ipv4Addr::new(123, 123, 123, 123).into(), + iface: Ipv4Addr::new(128, 0, 0, 1).into(), + }); + } + + #[tracing_test::traced_test] + #[test] + pub fn test_record_socket_leave_ipv6_multicast() { + run_test(JournalEntry::SocketLeaveIpv6Multicast { + fd: 12, + multiaddr: Ipv6Addr::new(123, 123, 123, 123, 1234, 12663, 31, 1324).into(), + iface: 23541, + }); + } + + #[tracing_test::traced_test] + #[test] + pub fn test_record_socket_send_file() { + run_test(JournalEntry::SocketSendFile { + socket_fd: 22234, + file_fd: 989, + offset: 124, + count: 345673456234651234, + }); + } + + #[tracing_test::traced_test] + #[test] + pub fn test_record_socket_send_to() { + run_test(JournalEntry::SocketSendTo { + fd: 123, + data: [98u8; 102400].to_vec().into(), + flags: 1234, + addr: SocketAddr::new(Ipv6Addr::UNSPECIFIED.into(), 3452), + is_64bit: true, + }); + } + + #[tracing_test::traced_test] + #[test] + pub fn test_record_socket_send() { + run_test(JournalEntry::SocketSend { + fd: 123, + data: [98u8; 102400].to_vec().into(), + flags: 1234, + is_64bit: true, + }); + } + + #[tracing_test::traced_test] + #[test] + pub fn test_record_socket_set_opt_flag() { + run_test(JournalEntry::SocketSetOptFlag { + fd: 0, + opt: Sockoption::Linger, + flag: true, + }); + } + + #[tracing_test::traced_test] + #[test] + pub fn test_record_socket_set_opt_size() { + run_test(JournalEntry::SocketSetOptSize { + fd: 15, + opt: Sockoption::Linger, + size: 234234, + }); + } + + #[tracing_test::traced_test] + #[test] + pub fn test_record_socket_set_opt_time() { + run_test(JournalEntry::SocketSetOptTime { + fd: 0, + ty: TimeType::AcceptTimeout, + time: Some(Duration::ZERO), + }); + } + + #[tracing_test::traced_test] + #[test] + pub fn test_record_socket_shutdown() { + run_test(JournalEntry::SocketShutdown { + fd: 123, + how: Shutdown::Both, + }); + } + + #[tracing_test::traced_test] + #[test] + pub fn test_record_snapshot() { + run_test(JournalEntry::Snapshot { + when: SystemTime::now(), + trigger: SnapshotTrigger::Idle, + }); + } +} diff --git a/lib/wasix/src/journaling/concrete/compactor.rs b/lib/wasix/src/journaling/concrete/compactor.rs index 4f8d203b423..26dc5b980f4 100644 --- a/lib/wasix/src/journaling/concrete/compactor.rs +++ b/lib/wasix/src/journaling/concrete/compactor.rs @@ -4,7 +4,6 @@ use std::{ sync::Mutex, }; -use futures::future::LocalBoxFuture; use sha2::{Digest, Sha256}; use virtual_fs::Fd; @@ -38,116 +37,111 @@ impl CompactingJournal { } impl Journal for CompactingJournal { - fn write<'a>(&'a self, entry: JournalEntry<'a>) -> LocalBoxFuture<'a, anyhow::Result<()>> { - Box::pin(async { - match entry { - JournalEntry::UpdateMemoryRegion { region, data } => { - let mut hasher = Sha256::default(); - hasher.update(data.as_ref()); - let hash: [u8; 32] = hasher.finalize().try_into().unwrap(); + fn write<'a>(&'a self, entry: JournalEntry<'a>) -> anyhow::Result<()> { + match entry { + JournalEntry::UpdateMemoryRegion { region, data } => { + let mut hasher = Sha256::default(); + hasher.update(data.as_ref()); + let hash: [u8; 32] = hasher.finalize().try_into().unwrap(); - { - let mut state = self.state.lock().unwrap(); - if let Some(other) = state.memory_map.get_mut(®ion) { - if *other == hash { - return Ok(()); - } else { - *other = hash; - } + { + let mut state = self.state.lock().unwrap(); + if let Some(other) = state.memory_map.get_mut(®ion) { + if *other == hash { + return Ok(()); } else { - #[allow(clippy::nonminimal_bool)] - let to_remove = state - .memory_map - .keys() - .filter(|r| { - // Covers the whole range - ( - region.start <= r.start && - region.end >= r.end - ) || - // Clips the left side - ( - region.start <= r.start && - region.end > r.start - ) || - // Clips the right side + *other = hash; + } + } else { + #[allow(clippy::nonminimal_bool)] + let to_remove = state + .memory_map + .keys() + .filter(|r| { + // Covers the whole range ( - region.start < r.end && - region.end >= r.end - ) - }) - .cloned() - .collect::>(); - for r in to_remove { - state.memory_map.remove(&r); - } - - state.memory_map.insert(region.clone(), hash); + region.start <= r.start && + region.end >= r.end + ) || + // Clips the left side + ( + region.start <= r.start && + region.end > r.start + ) || + // Clips the right side + ( + region.start < r.end && + region.end >= r.end + ) + }) + .cloned() + .collect::>(); + for r in to_remove { + state.memory_map.remove(&r); } + + state.memory_map.insert(region.clone(), hash); } - return self - .inner - .write(JournalEntry::UpdateMemoryRegion { region, data }) - .await; - } - JournalEntry::CloseFileDescriptor { fd } => { - let mut state = self.state.lock().unwrap(); - state.open_file.remove(&fd); - state.close_file.insert(fd); } - JournalEntry::OpenFileDescriptor { + return self + .inner + .write(JournalEntry::UpdateMemoryRegion { region, data }); + } + JournalEntry::CloseFileDescriptor { fd } => { + let mut state = self.state.lock().unwrap(); + state.open_file.remove(&fd); + state.close_file.insert(fd); + } + JournalEntry::OpenFileDescriptor { + fd, + dirfd, + dirflags, + path, + o_flags, + fs_rights_base, + fs_rights_inheriting, + fs_flags, + } => { + let mut state = self.state.lock().unwrap(); + state.close_file.remove(&fd); + state.open_file.insert( fd, - dirfd, - dirflags, - path, - o_flags, - fs_rights_base, - fs_rights_inheriting, - fs_flags, - } => { - let mut state = self.state.lock().unwrap(); - state.close_file.remove(&fd); - state.open_file.insert( + JournalEntry::OpenFileDescriptor { fd, - JournalEntry::OpenFileDescriptor { - fd, - dirfd, - dirflags, - path: path.into_owned().into(), - o_flags, - fs_rights_base, - fs_rights_inheriting, - fs_flags, - }, - ); - } - JournalEntry::Snapshot { .. } => { - let (to_close, to_open) = { - let mut state = self.state.lock().unwrap(); - ( - state.close_file.drain().collect::>(), - state.open_file.drain().collect::>(), - ) - }; - for fd in to_close { - self.inner - .write(JournalEntry::CloseFileDescriptor { fd }) - .await?; - } - for (_, entry) in to_open { - self.inner.write(entry).await?; - } - return self.inner.write(entry).await; + dirfd, + dirflags, + path: path.into_owned().into(), + o_flags, + fs_rights_base, + fs_rights_inheriting, + fs_flags, + }, + ); + } + JournalEntry::Snapshot { .. } => { + let (to_close, to_open) = { + let mut state = self.state.lock().unwrap(); + ( + state.close_file.drain().collect::>(), + state.open_file.drain().collect::>(), + ) + }; + for fd in to_close { + self.inner.write(JournalEntry::CloseFileDescriptor { fd })?; } - entry => { - return self.inner.write(entry).await; + for (_, entry) in to_open { + self.inner.write(entry)?; } + return self.inner.write(entry); } - Ok(()) - }) + entry => { + return self.inner.write(entry); + } + } + Ok(()) } - fn read<'a>(&'a self) -> anyhow::Result>> { + fn read(&self) -> anyhow::Result>> { Ok(match self.inner.read()? { Some(JournalEntry::UpdateMemoryRegion { region, data }) => { let mut hasher = Sha256::default(); diff --git a/lib/wasix/src/journaling/concrete/filter.rs b/lib/wasix/src/journaling/concrete/filter.rs index 572a07d4f76..5ad88bddf49 100644 --- a/lib/wasix/src/journaling/concrete/filter.rs +++ b/lib/wasix/src/journaling/concrete/filter.rs @@ -1,5 +1,3 @@ -use futures::future::LocalBoxFuture; - use super::*; /// Filters out a specific set of journal events and drops the rest, this @@ -60,102 +58,100 @@ impl FilteredJournal { } impl Journal for FilteredJournal { - fn write<'a>(&'a self, entry: JournalEntry<'a>) -> LocalBoxFuture<'a, anyhow::Result<()>> { - Box::pin(async { - let evt = match entry { - JournalEntry::SetClockTime { .. } - | JournalEntry::InitModule { .. } - | JournalEntry::ProcessExit { .. } - | JournalEntry::EpollCreate { .. } - | JournalEntry::EpollCtl { .. } - | JournalEntry::TtySet { .. } => { - if self.filter_core { - return Ok(()); - } - entry + fn write<'a>(&'a self, entry: JournalEntry<'a>) -> anyhow::Result<()> { + let evt = match entry { + JournalEntry::SetClockTime { .. } + | JournalEntry::InitModule { .. } + | JournalEntry::ProcessExit { .. } + | JournalEntry::EpollCreate { .. } + | JournalEntry::EpollCtl { .. } + | JournalEntry::TtySet { .. } => { + if self.filter_core { + return Ok(()); } - JournalEntry::SetThread { .. } | JournalEntry::CloseThread { .. } => { - if self.filter_threads { - return Ok(()); - } - entry + entry + } + JournalEntry::SetThread { .. } | JournalEntry::CloseThread { .. } => { + if self.filter_threads { + return Ok(()); } - JournalEntry::UpdateMemoryRegion { .. } => { - if self.filter_memory { - return Ok(()); - } - entry + entry + } + JournalEntry::UpdateMemoryRegion { .. } => { + if self.filter_memory { + return Ok(()); } - JournalEntry::FileDescriptorSeek { .. } - | JournalEntry::FileDescriptorWrite { .. } - | JournalEntry::OpenFileDescriptor { .. } - | JournalEntry::CloseFileDescriptor { .. } - | JournalEntry::RemoveDirectory { .. } - | JournalEntry::UnlinkFile { .. } - | JournalEntry::PathRename { .. } - | JournalEntry::RenumberFileDescriptor { .. } - | JournalEntry::DuplicateFileDescriptor { .. } - | JournalEntry::CreateDirectory { .. } - | JournalEntry::PathSetTimes { .. } - | JournalEntry::FileDescriptorSetFlags { .. } - | JournalEntry::FileDescriptorAdvise { .. } - | JournalEntry::FileDescriptorAllocate { .. } - | JournalEntry::FileDescriptorSetRights { .. } - | JournalEntry::FileDescriptorSetTimes { .. } - | JournalEntry::FileDescriptorSetSize { .. } - | JournalEntry::CreateHardLink { .. } - | JournalEntry::CreateSymbolicLink { .. } - | JournalEntry::ChangeDirectory { .. } - | JournalEntry::CreatePipe { .. } - | JournalEntry::CreateEvent { .. } => { - if self.filter_fs { - return Ok(()); - } - entry + entry + } + JournalEntry::FileDescriptorSeek { .. } + | JournalEntry::FileDescriptorWrite { .. } + | JournalEntry::OpenFileDescriptor { .. } + | JournalEntry::CloseFileDescriptor { .. } + | JournalEntry::RemoveDirectory { .. } + | JournalEntry::UnlinkFile { .. } + | JournalEntry::PathRename { .. } + | JournalEntry::RenumberFileDescriptor { .. } + | JournalEntry::DuplicateFileDescriptor { .. } + | JournalEntry::CreateDirectory { .. } + | JournalEntry::PathSetTimes { .. } + | JournalEntry::FileDescriptorSetFlags { .. } + | JournalEntry::FileDescriptorAdvise { .. } + | JournalEntry::FileDescriptorAllocate { .. } + | JournalEntry::FileDescriptorSetRights { .. } + | JournalEntry::FileDescriptorSetTimes { .. } + | JournalEntry::FileDescriptorSetSize { .. } + | JournalEntry::CreateHardLink { .. } + | JournalEntry::CreateSymbolicLink { .. } + | JournalEntry::ChangeDirectory { .. } + | JournalEntry::CreatePipe { .. } + | JournalEntry::CreateEvent { .. } => { + if self.filter_fs { + return Ok(()); } - JournalEntry::Snapshot { .. } => { - if self.filter_snapshots { - return Ok(()); - } - entry + entry + } + JournalEntry::Snapshot { .. } => { + if self.filter_snapshots { + return Ok(()); } - JournalEntry::PortAddAddr { .. } - | JournalEntry::PortDelAddr { .. } - | JournalEntry::PortAddrClear - | JournalEntry::PortBridge { .. } - | JournalEntry::PortUnbridge - | JournalEntry::PortDhcpAcquire - | JournalEntry::PortGatewaySet { .. } - | JournalEntry::PortRouteAdd { .. } - | JournalEntry::PortRouteClear - | JournalEntry::PortRouteDel { .. } - | JournalEntry::SocketOpen { .. } - | JournalEntry::SocketListen { .. } - | JournalEntry::SocketBind { .. } - | JournalEntry::SocketConnected { .. } - | JournalEntry::SocketAccepted { .. } - | JournalEntry::SocketJoinIpv4Multicast { .. } - | JournalEntry::SocketJoinIpv6Multicast { .. } - | JournalEntry::SocketLeaveIpv4Multicast { .. } - | JournalEntry::SocketLeaveIpv6Multicast { .. } - | JournalEntry::SocketSendFile { .. } - | JournalEntry::SocketSendTo { .. } - | JournalEntry::SocketSend { .. } - | JournalEntry::SocketSetOptFlag { .. } - | JournalEntry::SocketSetOptSize { .. } - | JournalEntry::SocketSetOptTime { .. } - | JournalEntry::SocketShutdown { .. } => { - if self.filter_net { - return Ok(()); - } - entry + entry + } + JournalEntry::PortAddAddr { .. } + | JournalEntry::PortDelAddr { .. } + | JournalEntry::PortAddrClear + | JournalEntry::PortBridge { .. } + | JournalEntry::PortUnbridge + | JournalEntry::PortDhcpAcquire + | JournalEntry::PortGatewaySet { .. } + | JournalEntry::PortRouteAdd { .. } + | JournalEntry::PortRouteClear + | JournalEntry::PortRouteDel { .. } + | JournalEntry::SocketOpen { .. } + | JournalEntry::SocketListen { .. } + | JournalEntry::SocketBind { .. } + | JournalEntry::SocketConnected { .. } + | JournalEntry::SocketAccepted { .. } + | JournalEntry::SocketJoinIpv4Multicast { .. } + | JournalEntry::SocketJoinIpv6Multicast { .. } + | JournalEntry::SocketLeaveIpv4Multicast { .. } + | JournalEntry::SocketLeaveIpv6Multicast { .. } + | JournalEntry::SocketSendFile { .. } + | JournalEntry::SocketSendTo { .. } + | JournalEntry::SocketSend { .. } + | JournalEntry::SocketSetOptFlag { .. } + | JournalEntry::SocketSetOptSize { .. } + | JournalEntry::SocketSetOptTime { .. } + | JournalEntry::SocketShutdown { .. } => { + if self.filter_net { + return Ok(()); } - }; - self.inner.write(evt).await - }) + entry + } + }; + self.inner.write(evt) } - fn read<'a>(&'a self) -> anyhow::Result>> { + fn read(&self) -> anyhow::Result>> { self.inner.read() } } diff --git a/lib/wasix/src/journaling/concrete/log_file.rs b/lib/wasix/src/journaling/concrete/log_file.rs index 83972605a49..acef42767cf 100644 --- a/lib/wasix/src/journaling/concrete/log_file.rs +++ b/lib/wasix/src/journaling/concrete/log_file.rs @@ -1,20 +1,20 @@ use bytes::Buf; +use rkyv::ser::serializers::{ + AllocScratch, CompositeSerializer, SharedSerializeMap, WriteSerializer, +}; use shared_buffer::OwnedBuffer; use std::{ - io::{Seek, SeekFrom}, + fs::File, + io::{Seek, SeekFrom, Write}, path::Path, }; -use tokio::runtime::Handle; -use virtual_fs::AsyncWriteExt; - -use futures::future::LocalBoxFuture; use super::*; struct State { - file: tokio::fs::File, + file: File, + serializer: CompositeSerializer, AllocScratch, SharedSerializeMap>, buffer_pos: usize, - record_pos: usize, } /// The LogFile snapshot capturer will write its snapshots to a linear journal @@ -30,56 +30,93 @@ struct State { /// delimiter. pub struct LogFileJournal { state: std::sync::Mutex, - handle: Handle, buffer: OwnedBuffer, } impl LogFileJournal { pub fn new(path: impl AsRef) -> anyhow::Result { - let mut file = std::fs::File::options() + let file = std::fs::File::options() .read(true) .write(true) .create(true) .open(path)?; + Self::from_file(file) + } + + pub fn from_file(mut file: std::fs::File) -> anyhow::Result { let buffer = OwnedBuffer::from_file(&file)?; - // Move the file to the end - file.seek(SeekFrom::End(0))?; + // If the buffer exists we valid the magic number + let mut buffer_pos = 0; + let mut buffer_ptr = buffer.as_ref(); + if buffer_ptr.len() >= 8 { + let magic = u64::from_be_bytes(buffer_ptr[0..8].try_into().unwrap()); + if magic != JOURNAL_MAGIC_NUMBER { + return Err(anyhow::format_err!( + "invalid magic number of journal ({} vs {})", + magic, + JOURNAL_MAGIC_NUMBER + )); + } + buffer_ptr.advance(8); + buffer_pos += 8; + } else { + tracing::trace!("journal has no magic (could be empty?)"); + } + + // Move to the end of the file and write the + // magic if one is needed + if file.seek(SeekFrom::End(0)).unwrap() == 0 { + let magic = JOURNAL_MAGIC_NUMBER as u64; + let magic = magic.to_be_bytes(); + file.write_all(&magic)?; + } + // We create various serializers and state that is + // used when writing. Plus we store the owned buffer + // thats used for zero copy deserialization Ok(Self { state: std::sync::Mutex::new(State { - file: tokio::fs::File::from_std(file), - buffer_pos: 0, - record_pos: 0, + file: file.try_clone()?, + serializer: CompositeSerializer::new( + WriteSerializer::new(file), + AllocScratch::default(), + SharedSerializeMap::default(), + ), + buffer_pos, }), - handle: Handle::current(), buffer, }) } } -impl JournalBatch {} - -#[async_trait::async_trait] impl Journal for LogFileJournal { - fn write<'a>(&'a self, entry: JournalEntry<'a>) -> LocalBoxFuture<'a, anyhow::Result<()>> { + fn write<'a>(&'a self, entry: JournalEntry<'a>) -> anyhow::Result<()> { tracing::debug!("journal event: {:?}", entry); - Box::pin(async { - // Create the batch - let batch = JournalBatch { - records: vec![entry.into()], - }; - let data = rkyv::to_bytes::<_, 128>(&batch).unwrap(); - let data_len = data.len() as u64; - let data_len = data_len.to_be_bytes(); + let mut state = self.state.lock().unwrap(); - let _guard = Handle::try_current().map_err(|_| self.handle.enter()); - let mut state = self.state.lock().unwrap(); - state.file.write_all(&data_len).await?; - state.file.write_all(&data).await?; - Ok(()) - }) + // Write the header (with a record size of zero) + let record_type: JournalEntryRecordType = entry.archive_record_type(); + state.file.write_all(&(record_type as u16).to_be_bytes())?; + let offset_size = state.file.seek(SeekFrom::Current(0))?; + state.file.write_all(&[0u8; 6])?; // record size (48 bits) + + // Now serialize the actual data to the log + let offset_start = state.file.seek(SeekFrom::Current(0))?; + entry.serialize_archive(&mut state.serializer)?; + let offset_end = state.file.seek(SeekFrom::Current(0))?; + let record_size = offset_end - offset_start; + + // Write the record and then move back to the end again + state.file.seek(SeekFrom::Start(offset_size))?; + state + .file + .write_all(&(record_size as u64).to_be_bytes()[2..8])?; + state.file.seek(SeekFrom::Start(offset_end))?; + + // Now write the actual data and update the offsets + Ok(()) } /// UNSAFE: This method uses unsafe operations to remove the need to zero @@ -90,53 +127,160 @@ impl Journal for LogFileJournal { // Get a memory reference to the data on the disk at // the current read location let mut buffer_ptr = self.buffer.as_ref(); - - // First we read the magic number for the archive - if state.buffer_pos == 0 { - if buffer_ptr.len() >= 8 { - let magic = u64::from_be_bytes(buffer_ptr[0..8].try_into().unwrap()); - if magic != JOURNAL_MAGIC_NUMBER { - return Err(anyhow::format_err!( - "invalid magic number of journal ({} vs {})", - magic, - JOURNAL_MAGIC_NUMBER - )); - } - state.buffer_pos += 8; - } else { - return Ok(None); - } - } buffer_ptr.advance(state.buffer_pos); - loop { - // Next we read the length of the current batch - if buffer_ptr.len() < 8 { + // Read the headers and advance + let header_size = 8; + if buffer_ptr.len() < header_size { return Ok(None); } - let batch_len = u64::from_be_bytes(buffer_ptr[0..8].try_into().unwrap()) as usize; - buffer_ptr.advance(8); - if batch_len == 0 || batch_len > buffer_ptr.len() { + let header = { + let b = buffer_ptr; + let header = JournalEntryHeader { + record_type: u16::from_be_bytes([b[0], b[1]]), + record_size: u64::from_be_bytes([0u8, 0u8, b[2], b[3], b[4], b[5], b[6], b[7]]), + }; + buffer_ptr.advance(8); + header + }; + + if header.record_size as usize > buffer_ptr.len() { + state.buffer_pos += buffer_ptr.len(); + tracing::trace!( + "journal is corrupt (record_size={} vs remaining={})", + header.record_size, + buffer_ptr.len() + ); return Ok(None); } - // Read the batch data itself - let batch = &buffer_ptr[..batch_len]; - let batch: &ArchivedJournalBatch = - unsafe { rkyv::archived_unsized_root::(batch) }; - - // If we have reached the end then move onto the next batch - if state.record_pos >= batch.records.len() { - buffer_ptr.advance(batch_len); - state.buffer_pos += batch_len; - state.record_pos = 0; - continue; - } - let record = &batch.records[state.record_pos]; + // Move the buffer position forward + let entry = &buffer_ptr[..(header.record_size as usize)]; + buffer_ptr.advance(header.record_size as usize); + state.buffer_pos += header_size; + state.buffer_pos += header.record_size as usize; + + // Now we read the entry + let record_type: JournalEntryRecordType = match header.record_type.try_into() { + Ok(t) => t, + Err(_) => { + tracing::debug!( + "unknown journal entry type ({}) - skipping", + header.record_type + ); + continue; + } + }; - // Otherwise we return the record and advance - state.record_pos += 1; - return Ok(Some(record.into())); + let record = unsafe { record_type.deserialize_archive(entry) }; + return Ok(Some(record)); } } } + +#[cfg(test)] +mod tests { + use super::*; + + #[tracing_test::traced_test] + #[test] + pub fn test_save_and_load_journal_events() { + // Get a random file path + let file = tempfile::NamedTempFile::new().unwrap(); + + // Write some events to it + let journal = LogFileJournal::from_file(file.as_file().try_clone().unwrap()).unwrap(); + journal + .write(JournalEntry::CreatePipe { fd1: 1, fd2: 2 }) + .unwrap(); + journal.write(JournalEntry::PortAddrClear).unwrap(); + drop(journal); + + // Read the events and validate + let journal = LogFileJournal::new(file.path()).unwrap(); + let event1 = journal.read().unwrap(); + let event2 = journal.read().unwrap(); + let event3 = journal.read().unwrap(); + + // Check the events + assert_eq!(event1, Some(JournalEntry::CreatePipe { fd1: 1, fd2: 2 })); + assert_eq!(event2, Some(JournalEntry::PortAddrClear)); + assert_eq!(event3, None); + + // Now write another event + journal + .write(JournalEntry::SocketSend { + fd: 1234, + data: [12; 1024].to_vec().into(), + flags: 123, + is_64bit: true, + }) + .unwrap(); + + // The event should not be visible yet unless we reload the log file + assert_eq!(journal.read().unwrap(), None); + + // Reload the load file + let journal = LogFileJournal::new(file.path()).unwrap(); + + // Before we read it, we will throw in another event + journal + .write(JournalEntry::CreatePipe { + fd1: 1234, + fd2: 5432, + }) + .unwrap(); + + let event1 = journal.read().unwrap(); + let event2 = journal.read().unwrap(); + let event3 = journal.read().unwrap(); + let event4 = journal.read().unwrap(); + assert_eq!(event1, Some(JournalEntry::CreatePipe { fd1: 1, fd2: 2 })); + assert_eq!(event2, Some(JournalEntry::PortAddrClear)); + assert_eq!( + event3, + Some(JournalEntry::SocketSend { + fd: 1234, + data: [12; 1024].to_vec().into(), + flags: 123, + is_64bit: true, + }) + ); + assert_eq!(event4, None); + + // Load it again + let journal = LogFileJournal::new(file.path()).unwrap(); + + let event1 = journal.read().unwrap(); + let event2 = journal.read().unwrap(); + let event3 = journal.read().unwrap(); + let event4 = journal.read().unwrap(); + let event5 = journal.read().unwrap(); + + tracing::info!("event1 {:?}", event1); + tracing::info!("event2 {:?}", event2); + tracing::info!("event3 {:?}", event3); + tracing::info!("event4 {:?}", event4); + tracing::info!("event5 {:?}", event5); + + assert_eq!(event1, Some(JournalEntry::CreatePipe { fd1: 1, fd2: 2 })); + assert_eq!(event2, Some(JournalEntry::PortAddrClear)); + assert_eq!( + event3, + Some(JournalEntry::SocketSend { + fd: 1234, + data: [12; 1024].to_vec().into(), + flags: 123, + is_64bit: true, + }) + ); + assert_eq!( + event4, + Some(JournalEntry::CreatePipe { + fd1: 1234, + fd2: 5432, + }) + ); + assert_eq!(event5, None); + } +} diff --git a/lib/wasix/src/journaling/concrete/pipe.rs b/lib/wasix/src/journaling/concrete/pipe.rs index ca3f8ec59b3..78d1c07639f 100644 --- a/lib/wasix/src/journaling/concrete/pipe.rs +++ b/lib/wasix/src/journaling/concrete/pipe.rs @@ -1,8 +1,7 @@ +use std::sync::mpsc; +use std::sync::mpsc::TryRecvError; use std::sync::Mutex; -use futures::future::LocalBoxFuture; -use tokio::sync::mpsc::{self, error::TryRecvError}; - use super::*; // The pipe journal will feed journal entries between two bi-directional ends @@ -14,9 +13,9 @@ pub struct PipeJournal { } impl PipeJournal { - pub fn channel(buffer: usize) -> (Self, Self) { - let (tx1, rx1) = mpsc::channel(buffer); - let (tx2, rx2) = mpsc::channel(buffer); + pub fn channel() -> (Self, Self) { + let (tx1, rx1) = mpsc::channel(); + let (tx2, rx2) = mpsc::channel(); let end1 = PipeJournal { tx: tx1, @@ -33,25 +32,21 @@ impl PipeJournal { } impl Journal for PipeJournal { - fn write<'a>(&'a self, entry: JournalEntry<'a>) -> LocalBoxFuture<'a, anyhow::Result<()>> { + fn write<'a>(&'a self, entry: JournalEntry<'a>) -> anyhow::Result<()> { let entry = entry.into_owned(); - Box::pin(async { - self.tx.send(entry).await.map_err(|err| { - anyhow::format_err!("failed to send journal event through the pipe - {}", err) - }) + self.tx.send(entry).map_err(|err| { + anyhow::format_err!("failed to send journal event through the pipe - {}", err) }) } - fn read<'a>(&'a self) -> anyhow::Result>> { - let mut rx = self.rx.lock().unwrap(); + fn read(&self) -> anyhow::Result>> { + let rx = self.rx.lock().unwrap(); match rx.try_recv() { - Ok(e) => Ok(Some(e.into())), + Ok(e) => Ok(Some(e)), Err(TryRecvError::Empty) => Ok(None), - Err(TryRecvError::Disconnected) => { - return Err(anyhow::format_err!( - "failed to receive journal event from the pipe as its disconnected" - )) - } + Err(TryRecvError::Disconnected) => Err(anyhow::format_err!( + "failed to receive journal event from the pipe as its disconnected" + )), } } } diff --git a/lib/wasix/src/journaling/concrete/unsupported.rs b/lib/wasix/src/journaling/concrete/unsupported.rs index fcfa5f2e6b0..fa5e3d1a452 100644 --- a/lib/wasix/src/journaling/concrete/unsupported.rs +++ b/lib/wasix/src/journaling/concrete/unsupported.rs @@ -1,5 +1,3 @@ -use futures::future::LocalBoxFuture; - use super::*; pub static UNSUPPORTED_SNAPSHOT_CAPTURER: UnsupportedJournal = UnsupportedJournal {}; @@ -10,12 +8,12 @@ pub static UNSUPPORTED_SNAPSHOT_CAPTURER: UnsupportedJournal = UnsupportedJourna pub struct UnsupportedJournal {} impl Journal for UnsupportedJournal { - fn write<'a>(&'a self, entry: JournalEntry<'a>) -> LocalBoxFuture<'a, anyhow::Result<()>> { + fn write<'a>(&'a self, entry: JournalEntry<'a>) -> anyhow::Result<()> { tracing::debug!("journal event: {:?}", entry); - Box::pin(async { Err(anyhow::format_err!("unsupported")) }) + Err(anyhow::format_err!("unsupported")) } - fn read<'a>(&'a self) -> anyhow::Result>> { + fn read(&self) -> anyhow::Result>> { Ok(None) } } diff --git a/lib/wasix/src/journaling/effector/memory_and_snapshot.rs b/lib/wasix/src/journaling/effector/memory_and_snapshot.rs index 46c8898274a..7659a8b8814 100644 --- a/lib/wasix/src/journaling/effector/memory_and_snapshot.rs +++ b/lib/wasix/src/journaling/effector/memory_and_snapshot.rs @@ -41,38 +41,32 @@ impl JournalEffector { // Now that we know all the regions that need to be saved we // enter a processing loop that dumps all the data to the log // file in an orderly manner. - __asyncify_light(env, None, async { - let memory = unsafe { env.memory_view(ctx) }; - let journal = ctx.data().active_journal()?; - - for region in regions { - // We grab this region of memory as a vector and hash - // it, which allows us to make some logging efficiency - // gains. - let data = memory - .copy_range_to_vec(region.clone()) - .map_err(mem_error_to_wasi)?; + let memory = unsafe { env.memory_view(ctx) }; + let journal = ctx.data().active_journal()?; - // Now we write it to the snap snapshot capturer - journal - .write(JournalEntry::UpdateMemoryRegion { - region, - data: data.into(), - }) - .await - .map_err(map_snapshot_err)?; - } + for region in regions { + // We grab this region of memory as a vector and hash + // it, which allows us to make some logging efficiency + // gains. + let data = memory + .copy_range_to_vec(region.clone()) + .map_err(mem_error_to_wasi)?; - // Finally we mark the end of the snapshot so that - // it can act as a restoration point - let when = SystemTime::now(); + // Now we write it to the snap snapshot capturer journal - .write(JournalEntry::Snapshot { when, trigger }) - .await + .write(JournalEntry::UpdateMemoryRegion { + region, + data: data.into(), + }) .map_err(map_snapshot_err)?; - Ok(()) - })? - .map_err(|err| WasiError::Exit(ExitCode::Errno(err)))?; + } + + // Finally we mark the end of the snapshot so that + // it can act as a restoration point + let when = SystemTime::now(); + journal + .write(JournalEntry::Snapshot { when, trigger }) + .map_err(map_snapshot_err)?; Ok(()) } diff --git a/lib/wasix/src/journaling/effector/mod.rs b/lib/wasix/src/journaling/effector/mod.rs index ae9b9fb57c6..f730c73d120 100644 --- a/lib/wasix/src/journaling/effector/mod.rs +++ b/lib/wasix/src/journaling/effector/mod.rs @@ -17,9 +17,9 @@ pub(super) use wasmer_wasix_types::{ pub(super) use crate::{ mem_error_to_wasi, os::task::process::WasiProcessInner, - syscalls::{__asyncify_light, fd_write_internal, FdWriteSource}, + syscalls::{fd_write_internal, FdWriteSource}, utils::map_snapshot_err, - WasiEnv, WasiError, WasiRuntimeError, WasiThreadId, + WasiEnv, WasiRuntimeError, WasiThreadId, }; use super::*; diff --git a/lib/wasix/src/journaling/effector/process_exit.rs b/lib/wasix/src/journaling/effector/process_exit.rs index 45f32eb4498..e181a6640c0 100644 --- a/lib/wasix/src/journaling/effector/process_exit.rs +++ b/lib/wasix/src/journaling/effector/process_exit.rs @@ -2,14 +2,9 @@ use super::*; impl JournalEffector { pub fn save_process_exit(env: &WasiEnv, exit_code: Option) -> anyhow::Result<()> { - __asyncify_light(env, None, async { - env.active_journal()? - .write(JournalEntry::ProcessExit { exit_code }) - .await - .map_err(map_snapshot_err)?; - Ok(()) - })? - .map_err(|err| WasiError::Exit(ExitCode::Errno(err)))?; + env.active_journal()? + .write(JournalEntry::ProcessExit { exit_code }) + .map_err(map_snapshot_err)?; Ok(()) } diff --git a/lib/wasix/src/journaling/effector/save_event.rs b/lib/wasix/src/journaling/effector/save_event.rs index d214d973263..415b61c434f 100644 --- a/lib/wasix/src/journaling/effector/save_event.rs +++ b/lib/wasix/src/journaling/effector/save_event.rs @@ -10,16 +10,10 @@ impl JournalEffector { return Ok(()); } - __asyncify_light(env, None, async { - ctx.data() - .active_journal()? - .write(event) - .await - .map_err(map_snapshot_err)?; - Ok(()) - })? - .map_err(|err| WasiError::Exit(ExitCode::Errno(err)))?; - + ctx.data() + .active_journal()? + .write(event) + .map_err(map_snapshot_err)?; Ok(()) } } diff --git a/lib/wasix/src/journaling/effector/syscalls/fd_write.rs b/lib/wasix/src/journaling/effector/syscalls/fd_write.rs index f9a71c625e2..8fc90d26814 100644 --- a/lib/wasix/src/journaling/effector/syscalls/fd_write.rs +++ b/lib/wasix/src/journaling/effector/syscalls/fd_write.rs @@ -13,35 +13,30 @@ impl JournalEffector { let memory = unsafe { env.memory_view(&ctx) }; let iovs_arr = iovs.slice(&memory, iovs_len)?; - __asyncify_light(env, None, async { - let iovs_arr = iovs_arr.access().map_err(mem_error_to_wasi)?; - let mut remaining: M::Offset = TryFrom::::try_from(written).unwrap_or_default(); - for iovs in iovs_arr.iter() { - let sub = iovs.buf_len.min(remaining); - if sub == M::ZERO { - continue; - } - remaining -= sub; - - let buf = WasmPtr::::new(iovs.buf) - .slice(&memory, sub) - .map_err(mem_error_to_wasi)? - .access() - .map_err(mem_error_to_wasi)?; - ctx.data() - .active_journal()? - .write(JournalEntry::FileDescriptorWrite { - fd, - offset, - data: Cow::Borrowed(buf.as_ref()), - is_64bit: M::is_64bit(), - }) - .await - .map_err(map_snapshot_err)?; + let iovs_arr = iovs_arr.access().map_err(mem_error_to_wasi)?; + let mut remaining: M::Offset = TryFrom::::try_from(written).unwrap_or_default(); + for iovs in iovs_arr.iter() { + let sub = iovs.buf_len.min(remaining); + if sub == M::ZERO { + continue; } - Ok(()) - })? - .map_err(|err| WasiError::Exit(ExitCode::Errno(err)))?; + remaining -= sub; + + let buf = WasmPtr::::new(iovs.buf) + .slice(&memory, sub) + .map_err(mem_error_to_wasi)? + .access() + .map_err(mem_error_to_wasi)?; + ctx.data() + .active_journal()? + .write(JournalEntry::FileDescriptorWrite { + fd, + offset, + data: Cow::Borrowed(buf.as_ref()), + is_64bit: M::is_64bit(), + }) + .map_err(map_snapshot_err)?; + } Ok(()) } diff --git a/lib/wasix/src/journaling/effector/syscalls/sock_send.rs b/lib/wasix/src/journaling/effector/syscalls/sock_send.rs index ba19c75a4ac..67f106bd8c1 100644 --- a/lib/wasix/src/journaling/effector/syscalls/sock_send.rs +++ b/lib/wasix/src/journaling/effector/syscalls/sock_send.rs @@ -17,35 +17,30 @@ impl JournalEffector { let memory = unsafe { env.memory_view(&ctx) }; let iovs_arr = iovs.slice(&memory, iovs_len)?; - __asyncify_light(env, None, async { - let iovs_arr = iovs_arr.access().map_err(mem_error_to_wasi)?; - let mut remaining: M::Offset = TryFrom::::try_from(sent).unwrap_or_default(); - for iovs in iovs_arr.iter() { - let sub = iovs.buf_len.min(remaining); - if sub == M::ZERO { - continue; - } - remaining -= sub; - - let buf = WasmPtr::::new(iovs.buf) - .slice(&memory, sub) - .map_err(mem_error_to_wasi)? - .access() - .map_err(mem_error_to_wasi)?; - ctx.data() - .active_journal()? - .write(JournalEntry::SocketSend { - fd, - data: Cow::Borrowed(buf.as_ref()), - is_64bit: M::is_64bit(), - flags: si_flags, - }) - .await - .map_err(map_snapshot_err)?; + let iovs_arr = iovs_arr.access().map_err(mem_error_to_wasi)?; + let mut remaining: M::Offset = TryFrom::::try_from(sent).unwrap_or_default(); + for iovs in iovs_arr.iter() { + let sub = iovs.buf_len.min(remaining); + if sub == M::ZERO { + continue; } - Ok(()) - })? - .map_err(|err| WasiError::Exit(ExitCode::Errno(err)))?; + remaining -= sub; + + let buf = WasmPtr::::new(iovs.buf) + .slice(&memory, sub) + .map_err(mem_error_to_wasi)? + .access() + .map_err(mem_error_to_wasi)?; + ctx.data() + .active_journal()? + .write(JournalEntry::SocketSend { + fd, + data: Cow::Borrowed(buf.as_ref()), + is_64bit: M::is_64bit(), + flags: si_flags, + }) + .map_err(map_snapshot_err)?; + } Ok(()) } diff --git a/lib/wasix/src/journaling/effector/syscalls/sock_send_to.rs b/lib/wasix/src/journaling/effector/syscalls/sock_send_to.rs index b5c84b69ebd..47d9b38c780 100644 --- a/lib/wasix/src/journaling/effector/syscalls/sock_send_to.rs +++ b/lib/wasix/src/journaling/effector/syscalls/sock_send_to.rs @@ -19,36 +19,31 @@ impl JournalEffector { let memory = unsafe { env.memory_view(&ctx) }; let iovs_arr = iovs.slice(&memory, iovs_len)?; - __asyncify_light(env, None, async { - let iovs_arr = iovs_arr.access().map_err(mem_error_to_wasi)?; - let mut remaining: M::Offset = TryFrom::::try_from(sent).unwrap_or_default(); - for iovs in iovs_arr.iter() { - let sub = iovs.buf_len.min(remaining); - if sub == M::ZERO { - continue; - } - remaining -= sub; - - let buf = WasmPtr::::new(iovs.buf) - .slice(&memory, sub) - .map_err(mem_error_to_wasi)? - .access() - .map_err(mem_error_to_wasi)?; - ctx.data() - .active_journal()? - .write(JournalEntry::SocketSendTo { - fd, - data: Cow::Borrowed(buf.as_ref()), - addr, - is_64bit: M::is_64bit(), - flags: si_flags, - }) - .await - .map_err(map_snapshot_err)?; + let iovs_arr = iovs_arr.access().map_err(mem_error_to_wasi)?; + let mut remaining: M::Offset = TryFrom::::try_from(sent).unwrap_or_default(); + for iovs in iovs_arr.iter() { + let sub = iovs.buf_len.min(remaining); + if sub == M::ZERO { + continue; } - Ok(()) - })? - .map_err(|err| WasiError::Exit(ExitCode::Errno(err)))?; + remaining -= sub; + + let buf = WasmPtr::::new(iovs.buf) + .slice(&memory, sub) + .map_err(mem_error_to_wasi)? + .access() + .map_err(mem_error_to_wasi)?; + ctx.data() + .active_journal()? + .write(JournalEntry::SocketSendTo { + fd, + data: Cow::Borrowed(buf.as_ref()), + addr, + is_64bit: M::is_64bit(), + flags: si_flags, + }) + .map_err(map_snapshot_err)?; + } Ok(()) } diff --git a/lib/wasix/src/journaling/effector/thread_exit.rs b/lib/wasix/src/journaling/effector/thread_exit.rs index 30c05ea972f..359cbfe13f1 100644 --- a/lib/wasix/src/journaling/effector/thread_exit.rs +++ b/lib/wasix/src/journaling/effector/thread_exit.rs @@ -6,14 +6,9 @@ impl JournalEffector { id: WasiThreadId, exit_code: Option, ) -> anyhow::Result<()> { - __asyncify_light(env, None, async { - env.active_journal()? - .write(JournalEntry::CloseThread { id, exit_code }) - .await - .map_err(map_snapshot_err)?; - Ok(()) - })? - .map_err(|err| WasiError::Exit(ExitCode::Errno(err)))?; + env.active_journal()? + .write(JournalEntry::CloseThread { id, exit_code }) + .map_err(map_snapshot_err)?; Ok(()) } } diff --git a/lib/wasix/src/journaling/journal.rs b/lib/wasix/src/journaling/journal.rs index 11f5a345b38..418b8c53117 100644 --- a/lib/wasix/src/journaling/journal.rs +++ b/lib/wasix/src/journaling/journal.rs @@ -9,7 +9,6 @@ use wasmer_wasix_types::wasi::{ Sockoption, Socktype, Timestamp, Tty, Whence, }; -use futures::future::LocalBoxFuture; use virtual_fs::Fd; use crate::net::socket::TimeType; @@ -42,7 +41,7 @@ pub enum SocketJournalEvent { /// Represents a log entry in a snapshot log stream that represents the total /// state of a WASM process at a point in time. #[allow(clippy::large_enum_variant)] -#[derive(Debug)] +#[derive(Debug, Clone, PartialEq)] pub enum JournalEntry<'a> { InitModule { wasm_hash: [u8; 32], @@ -643,7 +642,7 @@ impl<'a> JournalEntry<'a> { pub trait Journal { /// Takes in a stream of snapshot log entries and saves them so that they /// may be restored at a later moment - fn write<'a>(&'a self, entry: JournalEntry<'a>) -> LocalBoxFuture<'a, anyhow::Result<()>>; + fn write<'a>(&'a self, entry: JournalEntry<'a>) -> anyhow::Result<()>; /// Returns a stream of snapshot objects that the runtime will use /// to restore the state of a WASM process to a previous moment in time diff --git a/lib/wasix/src/syscalls/mod.rs b/lib/wasix/src/syscalls/mod.rs index 87d226911bc..32d2d91d5bc 100644 --- a/lib/wasix/src/syscalls/mod.rs +++ b/lib/wasix/src/syscalls/mod.rs @@ -1304,7 +1304,7 @@ pub fn restore_snapshot( tracing::trace!("Restoring snapshot event - {next:?}"); match next { crate::journaling::JournalEntry::InitModule { wasm_hash } => { - is_same_module = ctx.data().process.module_hash.as_bytes() == wasm_hash; + is_same_module = ctx.data().process.module_hash.as_bytes()[0..16] == wasm_hash; } crate::journaling::JournalEntry::ProcessExit { exit_code } => { JournalEffector::apply_process_exit(ctx.data(), exit_code) From 41f4a92c4fa486fabffcce9852b8b9784d8f340c Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Wed, 22 Nov 2023 14:18:33 +1100 Subject: [PATCH 064/129] cargo update --- lib/wasi-web/Cargo.lock | 129 ++++++++++++++++++++-------------------- 1 file changed, 65 insertions(+), 64 deletions(-) diff --git a/lib/wasi-web/Cargo.lock b/lib/wasi-web/Cargo.lock index 39b59805ee2..0e5c18b149e 100644 --- a/lib/wasi-web/Cargo.lock +++ b/lib/wasi-web/Cargo.lock @@ -72,7 +72,7 @@ checksum = "a66537f1bb974b254c98ed142ff995236e81b9d0fe4db0575f46612cb15eb0f9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.39", ] [[package]] @@ -381,7 +381,7 @@ dependencies = [ "ident_case", "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.39", ] [[package]] @@ -403,7 +403,7 @@ checksum = "836a9bbc7ad63342d6d6e7b815ccab164bc77a2d95d84bc3117a8c0d5c98e2d5" dependencies = [ "darling_core 0.20.3", "quote", - "syn 2.0.38", + "syn 2.0.39", ] [[package]] @@ -537,7 +537,7 @@ dependencies = [ "num-traits", "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.39", ] [[package]] @@ -558,7 +558,7 @@ dependencies = [ "darling 0.20.3", "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.39", ] [[package]] @@ -569,9 +569,9 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "errno" -version = "0.3.5" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3e13f66a2f95e32a39eaa81f6b95d42878ca0e1db0c7543723dfe12557e860" +checksum = "f258a7194e7f7c2a7837a8913aeab7fd8c383457034fa20ce4dd3dcb813e8eb8" dependencies = [ "libc", "windows-sys 0.48.0", @@ -628,9 +628,9 @@ checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" [[package]] name = "form_urlencoded" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a62bc1cf6f830c2ec14a513a9fb124d0a213a629668a4186f329db21fe045652" +checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" dependencies = [ "percent-encoding", ] @@ -697,7 +697,7 @@ checksum = "53b153fd91e4b0147f4aced87be237c98248656bb01050b96bf3ee89220a8ddb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.39", ] [[package]] @@ -742,9 +742,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.10" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" +checksum = "fe9006bed769170c11f845cf00c7c1e9092aeb3f268e007c3e760ac68008070f" dependencies = [ "cfg-if", "js-sys", @@ -825,9 +825,9 @@ checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] name = "http" -version = "0.2.9" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" +checksum = "8947b1a6fad4393052c7ba1f4cd97bed3e953a95c79c92ad9b051a04611d9fbb" dependencies = [ "bytes", "fnv", @@ -871,9 +871,9 @@ checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" [[package]] name = "idna" -version = "0.4.0" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d20d6b07bfbc108882d88ed8e37d39636dcc260e15e30c45e6ba089610b917c" +checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" dependencies = [ "unicode-bidi", "unicode-normalization", @@ -951,9 +951,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.149" +version = "0.2.150" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a08173bc88b7955d1b3145aa561539096c421ac8debde8cbc3612ec635fee29b" +checksum = "89d92a4743f9a61002fae18374ed11e7973f530cb3a3255fb354818118b2203c" [[package]] name = "linked-hash-map" @@ -972,9 +972,9 @@ dependencies = [ [[package]] name = "linux-raw-sys" -version = "0.4.10" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da2479e8c062e40bf0066ffa0bc823de0a9368974af99c9f6df941d2c231e03f" +checksum = "969488b55f8ac402214f3f5fd243ebb7206cf82de60d3172994707a4bcc2b829" [[package]] name = "lock_api" @@ -1183,9 +1183,9 @@ checksum = "17359afc20d7ab31fdb42bb844c8b3bb1dabd7dcf7e68428492da7f16966fcef" [[package]] name = "percent-encoding" -version = "2.3.0" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "petgraph" @@ -1214,7 +1214,7 @@ checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.39", ] [[package]] @@ -1489,9 +1489,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.21" +version = "0.38.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b426b0506e5d50a7d8dafcf2e81471400deb602392c7dd110815afb4eaf02a3" +checksum = "dc99bc2d4f1fed22595588a013687477aedf3cdcfb26558c559edb67b4d9b22e" dependencies = [ "bitflags 2.4.1", "errno", @@ -1535,9 +1535,9 @@ checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" [[package]] name = "self_cell" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c309e515543e67811222dbc9e3dd7e1056279b782e1dacffe4242b718734fb6" +checksum = "e388332cd64eb80cd595a00941baf513caffae8dce9cfd0467fc9c66397dade6" [[package]] name = "semver" @@ -1550,9 +1550,9 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.190" +version = "1.0.193" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91d3c334ca1ee894a2c6f6ad698fe8c435b76d504b13d436f0685d648d6d96f7" +checksum = "25dd9975e68d0cb5aa1120c288333fc98731bd1dd12f561e468ea4728c042b89" dependencies = [ "serde_derive", ] @@ -1580,13 +1580,13 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.190" +version = "1.0.193" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67c5609f394e5c2bd7fc51efda478004ea80ef42fee983d5c67a65e34f32c0e3" +checksum = "43576ca501357b9b071ac53cdc7da8ef0cbd9493d8df094cd821777ea6e894d3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.39", ] [[package]] @@ -1694,9 +1694,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.11.1" +version = "1.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "942b4a808e05215192e39f4ab80813e599068285906cc91aa64f923db842bd5a" +checksum = "4dccd0940a2dcdf68d092b8cbab7dc0ad8fa938bf95787e1b916b0e3d0e8e970" [[package]] name = "spin" @@ -1732,9 +1732,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.38" +version = "2.0.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e96b79aaa137db8f61e26363a0c9b47d8b4ec75da28b7d1d614c2303e232408b" +checksum = "23e78b90f2fcf45d3e842032ce32e3f2d1545ba6636271dcbf24fa306d87be7a" dependencies = [ "proc-macro2", "quote", @@ -1813,7 +1813,7 @@ checksum = "266b2e40bc00e5a6c09c3584011e08b06f123c00362c92b975ba9843aaaa14b8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.39", ] [[package]] @@ -1872,9 +1872,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.33.0" +version = "1.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f38200e3ef7995e5ef13baec2f432a6da0aa9ac495b2c0e8f3b7eec2c92d653" +checksum = "d0c014766411e834f7af5b8f4cf46257aab4036ca95e9d2c144a10f59ad6f5b9" dependencies = [ "backtrace", "bytes", @@ -1884,13 +1884,13 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.1.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" +checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.39", ] [[package]] @@ -1936,14 +1936,14 @@ dependencies = [ [[package]] name = "toml" -version = "0.8.6" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ff9e3abce27ee2c9a37f9ad37238c1bdd4e789c84ba37df76aa4d528f5072cc" +checksum = "a1a195ec8c9da26928f773888e0742ca3ca1040c6cd859c919c9f59c1954ab35" dependencies = [ "serde", "serde_spanned", "toml_datetime", - "toml_edit 0.20.7", + "toml_edit 0.21.0", ] [[package]] @@ -1970,9 +1970,9 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.20.7" +version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70f427fce4d84c72b5b732388bf4a9f4531b53f74e2887e3ecb2481f68f66d81" +checksum = "d34d383cd00a163b4a5b85053df514d45bc330f6de7737edfe0a93311d1eaa03" dependencies = [ "indexmap 2.1.0", "serde", @@ -2001,7 +2001,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.39", ] [[package]] @@ -2026,9 +2026,9 @@ dependencies = [ [[package]] name = "tracing-log" -version = "0.1.4" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f751112709b4e791d8ce53e32c4ed2d353565a795ce84da2285393f41557bdf2" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" dependencies = [ "log", "once_cell", @@ -2037,9 +2037,9 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.3.17" +version = "0.3.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30a651bc37f915e81f087d86e62a18eec5f79550c7faff886f7090b4ea757c77" +checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b" dependencies = [ "nu-ansi-term", "sharded-slab", @@ -2122,9 +2122,9 @@ checksum = "f28467d3e1d3c6586d8f25fa243f544f5800fec42d97032474e17222c2b75cfa" [[package]] name = "url" -version = "2.4.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "143b538f18257fac9cad154828a57c6bf5157e1aa604d4816b5995bf6de87ae5" +checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633" dependencies = [ "form_urlencoded", "idna", @@ -2140,9 +2140,9 @@ checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" [[package]] name = "uuid" -version = "1.5.0" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88ad59a7560b41a70d191093a945f0b87bc1deeda46fb237479708a1d6b6cdfc" +checksum = "5e395fcf16a7a3d8127ec99782007af141946b4795001f876d54fb0d55978560" [[package]] name = "valuable" @@ -2361,7 +2361,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.39", "wasm-bindgen-shared", ] @@ -2395,7 +2395,7 @@ checksum = "c5353b8dab669f5e10f5bd76df26a9360c748f054f862ff5f3f8aae0c7fb3907" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.39", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -2428,7 +2428,7 @@ checksum = "493fcbab756bb764fa37e6bee8cec2dd709eb4273d06d0c282a5e74275ded735" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.39", ] [[package]] @@ -2516,7 +2516,7 @@ dependencies = [ "serde_json", "serde_yaml 0.9.27", "thiserror", - "toml 0.8.6", + "toml 0.8.8", ] [[package]] @@ -2583,6 +2583,7 @@ dependencies = [ "lazy_static", "libc", "linked_hash_set", + "num_enum", "once_cell", "petgraph", "pin-project", @@ -2733,9 +2734,9 @@ dependencies = [ [[package]] name = "webc" -version = "5.8.0" +version = "5.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac815d472f09ed064ef70a3046843972646727cbe55db795dd8a9925b76ed0c4" +checksum = "973ca5a91b4fb3e4bb37cfebe03ef9364d0aff2765256abefdb7e79dc9188483" dependencies = [ "anyhow", "base64 0.21.5", @@ -2920,9 +2921,9 @@ checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" [[package]] name = "winnow" -version = "0.5.18" +version = "0.5.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "176b6138793677221d420fd2f0aeeced263f197688b36484660da767bca2fa32" +checksum = "829846f3e3db426d4cee4510841b71a8e58aa2a76b1132579487ae430ccd9c7b" dependencies = [ "memchr", ] From 09efd351e4d3655c23dbf3811cf7253398e45caf Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Wed, 22 Nov 2023 22:37:31 +1100 Subject: [PATCH 065/129] Added the compacting journal with some basic test cases --- .../journaling/concrete/archived_journal.rs | 2 +- .../src/journaling/concrete/boxed_journal.rs | 35 ++ .../src/journaling/concrete/compactor.rs | 574 ++++++++++++++---- .../src/journaling/concrete/composite.rs | 34 ++ lib/wasix/src/journaling/concrete/filter.rs | 118 +++- lib/wasix/src/journaling/concrete/log_file.rs | 120 +++- lib/wasix/src/journaling/concrete/mod.rs | 4 + lib/wasix/src/journaling/concrete/pipe.rs | 70 ++- .../src/journaling/concrete/unsupported.rs | 21 +- lib/wasix/src/journaling/journal.rs | 23 +- 10 files changed, 817 insertions(+), 184 deletions(-) create mode 100644 lib/wasix/src/journaling/concrete/boxed_journal.rs create mode 100644 lib/wasix/src/journaling/concrete/composite.rs diff --git a/lib/wasix/src/journaling/concrete/archived_journal.rs b/lib/wasix/src/journaling/concrete/archived_journal.rs index 9dbb704c7f0..6fe7697f83d 100644 --- a/lib/wasix/src/journaling/concrete/archived_journal.rs +++ b/lib/wasix/src/journaling/concrete/archived_journal.rs @@ -2797,7 +2797,7 @@ mod tests { use super::*; pub fn run_test<'a>(record: JournalEntry<'a>) { - tracing::info!("record2: {:?}", record); + tracing::info!("record: {:?}", record); // Determine the record type let record_type = record.archive_record_type(); diff --git a/lib/wasix/src/journaling/concrete/boxed_journal.rs b/lib/wasix/src/journaling/concrete/boxed_journal.rs new file mode 100644 index 00000000000..70b29d131e1 --- /dev/null +++ b/lib/wasix/src/journaling/concrete/boxed_journal.rs @@ -0,0 +1,35 @@ +use std::ops::Deref; + +use super::*; + +impl ReadableJournal for Box { + fn read<'a>(&'a self) -> anyhow::Result>> { + self.deref().read() + } + + fn as_restarted(&self) -> anyhow::Result> { + self.deref().as_restarted() + } +} + +impl WritableJournal for Box { + fn write<'a>(&'a self, entry: JournalEntry<'a>) -> anyhow::Result<()> { + self.deref().write(entry) + } +} + +impl ReadableJournal for Box { + fn read<'a>(&'a self) -> anyhow::Result>> { + self.deref().read() + } + + fn as_restarted(&self) -> anyhow::Result> { + self.deref().as_restarted() + } +} + +impl WritableJournal for Box { + fn write<'a>(&'a self, entry: JournalEntry<'a>) -> anyhow::Result<()> { + self.deref().write(entry) + } +} diff --git a/lib/wasix/src/journaling/concrete/compactor.rs b/lib/wasix/src/journaling/concrete/compactor.rs index 26dc5b980f4..639241d71ce 100644 --- a/lib/wasix/src/journaling/concrete/compactor.rs +++ b/lib/wasix/src/journaling/concrete/compactor.rs @@ -1,160 +1,492 @@ +use derivative::Derivative; use std::{ collections::{HashMap, HashSet}, ops::Range, - sync::Mutex, + sync::{Arc, Mutex}, }; - -use sha2::{Digest, Sha256}; use virtual_fs::Fd; use super::*; +#[derive(Debug)] +struct StateDescriptor { + events: Vec, + write_map: HashMap, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +struct MemoryRange { + start: u64, + end: u64, +} +impl From> for MemoryRange { + fn from(value: Range) -> Self { + Self { + start: value.start, + end: value.end, + } + } +} + +#[derive(Derivative)] +#[derivative(Debug)] struct State { - memory_map: HashMap, [u8; 32]>, - open_file: HashMap>, - close_file: HashSet, + // We maintain a memory map of the events that are significant + memory_map: HashMap, + // Thread events are only maintained while the thread and the + // process are still running + thread_map: HashMap, + // Any descriptors are assumed to be read only operations until + // they actually do something that changes the system + suspect_descriptors: HashMap>, + // Any descriptors are assumed to be read only operations until + // they actually do something that changes the system + keep_descriptors: HashMap, + // Everything that will be retained during the next compact + whitelist: HashSet, + // We use an event index to track what to keep + event_index: usize, + // The delta list is used for all the events that happened + // after a compact started + delta_list: Option>, + // The inner journal that we will write to + #[derivative(Debug = "ignore")] + inner_tx: Box, + // The inner journal that we read from + #[derivative(Debug = "ignore")] + inner_rx: Box, +} + +impl State { + fn create_filter(&self, inner: J) -> FilteredJournal + where + J: Journal, + { + let mut filter = FilteredJournal::new(inner) + .with_filter_events(self.whitelist.clone().into_iter().collect()); + for (_, e) in self.memory_map.iter() { + filter.add_event_to_whitelist(*e); + } + for t in self.thread_map.iter() { + filter.add_event_to_whitelist(*t.1); + } + for (_, d) in self.suspect_descriptors.iter() { + for e in d.iter() { + filter.add_event_to_whitelist(*e); + } + } + for (_, d) in self.keep_descriptors.iter() { + for e in d.events.iter() { + filter.add_event_to_whitelist(*e); + } + for e in d.write_map.values() { + filter.add_event_to_whitelist(*e); + } + } + filter + } } /// Deduplicates memory and stacks to reduce the number of volume of /// log events sent to its inner capturer. Compacting the events occurs /// in line as the events are generated +#[derive(Debug, Clone)] +pub struct CompactingJournalTx { + state: Arc>, + compacting: Arc>, +} + +#[derive(Derivative)] +#[derivative(Debug)] +pub struct CompactingJournalRx { + #[derivative(Debug = "ignore")] + inner: Box, +} + +#[derive(Debug)] pub struct CompactingJournal { - inner: Box, - state: Mutex, + tx: CompactingJournalTx, + rx: CompactingJournalRx, } impl CompactingJournal { - pub fn new(inner: Box) -> Self { - Self { - inner, - state: Mutex::new(State { - memory_map: Default::default(), - open_file: Default::default(), - close_file: Default::default(), - }), + pub fn new(inner: J) -> anyhow::Result + where + J: Journal, + { + let (tx, rx) = inner.split(); + Ok(Self { + tx: CompactingJournalTx { + state: Arc::new(Mutex::new(State { + inner_tx: tx, + inner_rx: rx.as_restarted()?, + memory_map: Default::default(), + thread_map: Default::default(), + suspect_descriptors: Default::default(), + keep_descriptors: Default::default(), + whitelist: Default::default(), + delta_list: None, + event_index: 0, + })), + compacting: Arc::new(Mutex::new(())), + }, + rx: CompactingJournalRx { inner: rx }, + }) + } +} + +impl CompactingJournalTx { + pub fn create_filter(&self, inner: J) -> FilteredJournal + where + J: Journal, + { + let state = self.state.lock().unwrap(); + state.create_filter(inner) + } + + /// Compacts the inner journal into a new journal + pub fn compact(&mut self, new_journal: J) -> anyhow::Result<()> + where + J: Journal, + { + // Enter a compacting lock + let _guard = self.compacting.lock().unwrap(); + + // The first thing we do is create a filter that we + // place around the new journal so that it only receives new events + let (new_journal, replay_rx) = { + let mut state = self.state.lock().unwrap(); + state.delta_list.replace(Default::default()); + ( + state.create_filter(new_journal), + state.inner_rx.as_restarted()?, + ) + }; + + // Read all the events and feed them into the filtered journal + while let Some(entry) = replay_rx.read()? { + new_journal.write(entry)?; + } + + // We now go into a blocking situation which will freeze the journals + let mut state = self.state.lock().unwrap(); + + // Now we build a filtered journal which will pick up any events that were + // added which we did the compacting + let new_journal = FilteredJournal::new(new_journal.into_inner()).with_filter_events( + state + .delta_list + .take() + .unwrap_or_default() + .into_iter() + .collect(), + ); + + // Now we feed all the events into the new journal using the delta filter + let replay_rx = state.inner_rx.as_restarted()?; + while let Some(entry) = replay_rx.read()? { + new_journal.write(entry)?; } + + // Now we install the new journal + let (mut tx, mut rx) = new_journal.into_inner().split(); + std::mem::swap(&mut state.inner_tx, &mut tx); + std::mem::swap(&mut state.inner_rx, &mut rx); + + Ok(()) } } -impl Journal for CompactingJournal { +impl WritableJournal for CompactingJournalTx { fn write<'a>(&'a self, entry: JournalEntry<'a>) -> anyhow::Result<()> { - match entry { - JournalEntry::UpdateMemoryRegion { region, data } => { - let mut hasher = Sha256::default(); - hasher.update(data.as_ref()); - let hash: [u8; 32] = hasher.finalize().try_into().unwrap(); - - { - let mut state = self.state.lock().unwrap(); - if let Some(other) = state.memory_map.get_mut(®ion) { - if *other == hash { - return Ok(()); - } else { - *other = hash; - } - } else { - #[allow(clippy::nonminimal_bool)] - let to_remove = state - .memory_map - .keys() - .filter(|r| { - // Covers the whole range - ( - region.start <= r.start && - region.end >= r.end - ) || - // Clips the left side - ( - region.start <= r.start && - region.end > r.start - ) || - // Clips the right side - ( - region.start < r.end && - region.end >= r.end - ) - }) - .cloned() - .collect::>(); - for r in to_remove { - state.memory_map.remove(&r); - } - - state.memory_map.insert(region.clone(), hash); - } - } - return self - .inner - .write(JournalEntry::UpdateMemoryRegion { region, data }); + let mut state = self.state.lock().unwrap(); + let event_index = state.event_index; + state.event_index += 1; + + if let Some(delta) = state.delta_list.as_mut() { + delta.push(event_index); + } + + match &entry { + JournalEntry::UpdateMemoryRegion { region, .. } => { + state.memory_map.insert(region.clone().into(), event_index); + } + JournalEntry::SetThread { id, .. } => { + state.thread_map.insert(*id, event_index); + } + JournalEntry::CloseThread { id, .. } => { + state.thread_map.remove(&id); + } + JournalEntry::ProcessExit { .. } => { + state.thread_map.clear(); + state.memory_map.clear(); + state.suspect_descriptors.clear(); + state.whitelist.insert(event_index); } JournalEntry::CloseFileDescriptor { fd } => { - let mut state = self.state.lock().unwrap(); - state.open_file.remove(&fd); - state.close_file.insert(fd); + // If its not suspect we need to record this event + if state.suspect_descriptors.remove(&fd).is_some() { + // suspect descriptors that are closed are dropped + // as they made no material difference to the state + } else if let Some(e) = state.keep_descriptors.get_mut(fd) { + e.events.push(event_index); + } else { + state.whitelist.insert(event_index); + } } - JournalEntry::OpenFileDescriptor { - fd, - dirfd, - dirflags, - path, - o_flags, - fs_rights_base, - fs_rights_inheriting, - fs_flags, - } => { - let mut state = self.state.lock().unwrap(); - state.close_file.remove(&fd); - state.open_file.insert( - fd, - JournalEntry::OpenFileDescriptor { - fd, - dirfd, - dirflags, - path: path.into_owned().into(), - o_flags, - fs_rights_base, - fs_rights_inheriting, - fs_flags, - }, - ); + JournalEntry::OpenFileDescriptor { fd, .. } => { + // All file descriptors are opened in a suspect state which + // means if they are closed without modifying the file system + // then the events will be ignored. + state.suspect_descriptors.insert(*fd, vec![event_index]); } - JournalEntry::Snapshot { .. } => { - let (to_close, to_open) = { - let mut state = self.state.lock().unwrap(); - ( - state.close_file.drain().collect::>(), - state.open_file.drain().collect::>(), - ) - }; - for fd in to_close { - self.inner.write(JournalEntry::CloseFileDescriptor { fd })?; + // We keep non-mutable events for file descriptors that are suspect + JournalEntry::FileDescriptorSeek { fd, .. } => { + if let Some(events) = state.suspect_descriptors.get_mut(&fd) { + events.push(event_index); + } else if let Some(s) = state.keep_descriptors.get_mut(&fd) { + s.events.push(event_index); + } else { + state.whitelist.insert(event_index); + } + } + // Things that modify a file descriptor mean that it is + // no longer suspect and thus it needs to be kept + JournalEntry::FileDescriptorAdvise { fd, .. } + | JournalEntry::FileDescriptorAllocate { fd, .. } + | JournalEntry::FileDescriptorSetFlags { fd, .. } + | JournalEntry::FileDescriptorSetTimes { fd, .. } + | JournalEntry::FileDescriptorWrite { fd, .. } + | JournalEntry::DuplicateFileDescriptor { + original_fd: fd, .. + } => { + // Its no longer suspect + if let Some(events) = state.suspect_descriptors.remove(&fd) { + state.keep_descriptors.insert( + *fd, + StateDescriptor { + events, + write_map: Default::default(), + }, + ); } - for (_, entry) in to_open { - self.inner.write(entry)?; + if let Some(state) = state.keep_descriptors.get_mut(fd) { + if let JournalEntry::FileDescriptorWrite { offset, data, .. } = &entry { + state.write_map.insert( + MemoryRange { + start: *offset, + end: *offset + data.len() as u64, + }, + event_index, + ); + } else { + state.events.push(event_index); + } + } else { + state.whitelist.insert(event_index); + } + } + // Renumbered file descriptors will retain their suspect status + JournalEntry::RenumberFileDescriptor { old_fd, new_fd } => { + if let Some(mut events) = state.suspect_descriptors.remove(old_fd) { + events.push(event_index); + state.suspect_descriptors.insert(*new_fd, events); + } else if let Some(mut s) = state.keep_descriptors.remove(old_fd) { + s.events.push(event_index); + state.keep_descriptors.insert(*new_fd, s); + } else { + state.whitelist.insert(event_index); } - return self.inner.write(entry); } - entry => { - return self.inner.write(entry); + _ => { + // The fallthrough is to whitelist the event so that it will + // be reflected in the next compaction event + state.whitelist.insert(event_index); } } - Ok(()) + state.inner_tx.write(entry) + } +} + +impl CompactingJournal { + /// Compacts the inner journal into a new journal + pub fn compact(&mut self, new_journal: J) -> anyhow::Result<()> + where + J: Journal, + { + self.tx.compact(new_journal) } +} +impl ReadableJournal for CompactingJournalRx { fn read(&self) -> anyhow::Result>> { - Ok(match self.inner.read()? { - Some(JournalEntry::UpdateMemoryRegion { region, data }) => { - let mut hasher = Sha256::default(); - hasher.update(data.as_ref()); - let hash: [u8; 32] = hasher.finalize().try_into().unwrap(); + self.inner.read() + } - let mut state = self.state.lock().unwrap(); - state.memory_map.insert(region.clone(), hash); + fn as_restarted(&self) -> anyhow::Result> { + self.inner.as_restarted() + } +} - Some(JournalEntry::UpdateMemoryRegion { region, data }) - } - Some(entry) => Some(entry), - None => None, - }) +impl WritableJournal for CompactingJournal { + fn write<'a>(&'a self, entry: JournalEntry<'a>) -> anyhow::Result<()> { + self.tx.write(entry) + } +} + +impl ReadableJournal for CompactingJournal { + fn read(&self) -> anyhow::Result>> { + self.rx.read() + } + + fn as_restarted(&self) -> anyhow::Result> { + let state = self.tx.state.lock().unwrap(); + state.inner_rx.as_restarted() + } +} + +impl Journal for CompactingJournal { + fn split(self) -> (Box, Box) { + (Box::new(self.tx), Box::new(self.rx)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + pub fn run_test<'a>( + in_records: Vec>, + out_records: Vec>, + ) -> anyhow::Result<()> { + // Build a journal that will store the records before compacting + let in_file = tempfile::NamedTempFile::new()?; + let mut compacting_journal = CompactingJournal::new(LogFileJournal::from_file( + in_file.as_file().try_clone().unwrap(), + )?)?; + for record in in_records { + compacting_journal.write(record)?; + } + + // Now we build a new one using the compactor + let new_file = tempfile::NamedTempFile::new()?; + let new_journal = LogFileJournal::from_file(new_file.as_file().try_clone()?)?; + compacting_journal.compact(new_journal)?; + + // Read the records + let new_records = compacting_journal.as_restarted()?; + for record1 in out_records { + let record2 = new_records.read()?; + assert_eq!(Some(record1), record2); + } + assert!(new_records.read()?.is_none()); + + Ok(()) + } + + #[tracing_test::traced_test] + #[test] + pub fn test_compact_purge_duplicate_memory_writes() { + run_test( + vec![ + JournalEntry::UpdateMemoryRegion { + region: 0..16, + data: [11u8; 16].to_vec().into(), + }, + JournalEntry::UpdateMemoryRegion { + region: 0..16, + data: [22u8; 16].to_vec().into(), + }, + ], + vec![JournalEntry::UpdateMemoryRegion { + region: 0..16, + data: [22u8; 16].to_vec().into(), + }], + ) + .unwrap() + } + + #[tracing_test::traced_test] + #[test] + pub fn test_compact_thread_stacks() { + run_test( + vec![ + JournalEntry::SetThread { + id: 4321.into(), + call_stack: [44u8; 87].to_vec().into(), + memory_stack: [55u8; 34].to_vec().into(), + store_data: [66u8; 70].to_vec().into(), + is_64bit: true, + }, + JournalEntry::SetThread { + id: 1234.into(), + call_stack: [11u8; 124].to_vec().into(), + memory_stack: [22u8; 51].to_vec().into(), + store_data: [33u8; 87].to_vec().into(), + is_64bit: true, + }, + JournalEntry::SetThread { + id: 65.into(), + call_stack: [77u8; 34].to_vec().into(), + memory_stack: [88u8; 51].to_vec().into(), + store_data: [99u8; 12].to_vec().into(), + is_64bit: true, + }, + JournalEntry::CloseThread { + id: 1234.into(), + exit_code: None, + }, + ], + vec![ + JournalEntry::SetThread { + id: 4321.into(), + call_stack: [44u8; 87].to_vec().into(), + memory_stack: [55u8; 34].to_vec().into(), + store_data: [66u8; 70].to_vec().into(), + is_64bit: true, + }, + JournalEntry::SetThread { + id: 65.into(), + call_stack: [77u8; 34].to_vec().into(), + memory_stack: [88u8; 51].to_vec().into(), + store_data: [99u8; 12].to_vec().into(), + is_64bit: true, + }, + ], + ) + .unwrap() + } + + #[tracing_test::traced_test] + #[test] + pub fn test_compact_processed_exited() { + run_test( + vec![ + JournalEntry::UpdateMemoryRegion { + region: 0..16, + data: [11u8; 16].to_vec().into(), + }, + JournalEntry::SetThread { + id: 4321.into(), + call_stack: [44u8; 87].to_vec().into(), + memory_stack: [55u8; 34].to_vec().into(), + store_data: [66u8; 70].to_vec().into(), + is_64bit: true, + }, + JournalEntry::OpenFileDescriptor { + fd: 1234, + dirfd: 3452345, + dirflags: 0, + path: "/blah".into(), + o_flags: Oflags::all(), + fs_rights_base: Rights::all(), + fs_rights_inheriting: Rights::all(), + fs_flags: Fdflags::all(), + }, + JournalEntry::ProcessExit { exit_code: None }, + ], + vec![JournalEntry::ProcessExit { exit_code: None }], + ) + .unwrap() } } diff --git a/lib/wasix/src/journaling/concrete/composite.rs b/lib/wasix/src/journaling/concrete/composite.rs new file mode 100644 index 00000000000..59884a99af3 --- /dev/null +++ b/lib/wasix/src/journaling/concrete/composite.rs @@ -0,0 +1,34 @@ +use super::*; + +pub struct CompositeJournal { + tx: Box, + rx: Box, +} + +impl CompositeJournal { + pub fn new(tx: Box, rx: Box) -> Self { + Self { tx, rx } + } +} + +impl WritableJournal for CompositeJournal { + fn write<'a>(&'a self, entry: JournalEntry<'a>) -> anyhow::Result<()> { + self.tx.write(entry) + } +} + +impl ReadableJournal for CompositeJournal { + fn read(&self) -> anyhow::Result>> { + self.rx.read() + } + + fn as_restarted(&self) -> anyhow::Result> { + self.rx.as_restarted() + } +} + +impl Journal for CompositeJournal { + fn split(self) -> (Box, Box) { + (self.tx, self.rx) + } +} diff --git a/lib/wasix/src/journaling/concrete/filter.rs b/lib/wasix/src/journaling/concrete/filter.rs index 5ad88bddf49..b598a366a9b 100644 --- a/lib/wasix/src/journaling/concrete/filter.rs +++ b/lib/wasix/src/journaling/concrete/filter.rs @@ -1,64 +1,120 @@ +use std::{ + collections::HashSet, + sync::atomic::{AtomicUsize, Ordering}, +}; + +use derivative::Derivative; + use super::*; /// Filters out a specific set of journal events and drops the rest, this /// journal can be useful for restoring to a previous call point but /// retaining the memory changes (e.g. WCGI runner). +#[derive(Debug)] pub struct FilteredJournal { - inner: Box, + tx: FilteredJournalTx, + rx: FilteredJournalRx, +} + +#[derive(Derivative)] +#[derivative(Debug)] +pub struct FilteredJournalTx { + #[derivative(Debug = "ignore")] + inner: Box, filter_memory: bool, filter_threads: bool, filter_fs: bool, filter_core: bool, filter_snapshots: bool, filter_net: bool, + filter_events: Option>, + event_index: AtomicUsize, +} + +#[derive(Derivative)] +#[derivative(Debug)] +pub struct FilteredJournalRx { + #[derivative(Debug = "ignore")] + inner: Box, } impl FilteredJournal { - pub fn new(inner: Box) -> Self { + pub fn new(inner: J) -> Self + where + J: Journal, + { + let (tx, rx) = inner.split(); Self { - inner, - filter_memory: false, - filter_threads: false, - filter_fs: false, - filter_core: false, - filter_snapshots: false, - filter_net: false, + tx: FilteredJournalTx { + inner: tx, + filter_memory: false, + filter_threads: false, + filter_fs: false, + filter_core: false, + filter_snapshots: false, + filter_net: false, + filter_events: None, + event_index: AtomicUsize::new(0), + }, + rx: FilteredJournalRx { inner: rx }, } } pub fn with_ignore_memory(mut self, val: bool) -> Self { - self.filter_memory = val; + self.tx.filter_memory = val; self } pub fn with_ignore_threads(mut self, val: bool) -> Self { - self.filter_threads = val; + self.tx.filter_threads = val; self } pub fn with_ignore_fs(mut self, val: bool) -> Self { - self.filter_fs = val; + self.tx.filter_fs = val; self } pub fn with_ignore_core(mut self, val: bool) -> Self { - self.filter_core = val; + self.tx.filter_core = val; self } pub fn with_ignore_snapshots(mut self, val: bool) -> Self { - self.filter_snapshots = val; + self.tx.filter_snapshots = val; self } pub fn with_ignore_networking(mut self, val: bool) -> Self { - self.filter_net = val; + self.tx.filter_net = val; + self + } + + pub fn with_filter_events(mut self, events: HashSet) -> Self { + self.tx.filter_events = Some(events); self } + + pub fn add_event_to_whitelist(&mut self, event_index: usize) { + if let Some(filter) = self.tx.filter_events.as_mut() { + filter.insert(event_index); + } + } + + pub fn into_inner(self) -> CompositeJournal { + CompositeJournal::new(self.tx.inner, self.rx.inner) + } } -impl Journal for FilteredJournal { +impl WritableJournal for FilteredJournalTx { fn write<'a>(&'a self, entry: JournalEntry<'a>) -> anyhow::Result<()> { + let event_index = self.event_index.fetch_add(1, Ordering::SeqCst); + if let Some(events) = self.filter_events.as_ref() { + if !events.contains(&event_index) { + return Ok(()); + } + } + let evt = match entry { JournalEntry::SetClockTime { .. } | JournalEntry::InitModule { .. } @@ -150,8 +206,38 @@ impl Journal for FilteredJournal { }; self.inner.write(evt) } +} +impl ReadableJournal for FilteredJournalRx { fn read(&self) -> anyhow::Result>> { self.inner.read() } + + fn as_restarted(&self) -> anyhow::Result> { + Ok(Box::new(FilteredJournalRx { + inner: self.inner.as_restarted()?, + })) + } +} + +impl WritableJournal for FilteredJournal { + fn write<'a>(&'a self, entry: JournalEntry<'a>) -> anyhow::Result<()> { + self.tx.write(entry) + } +} + +impl ReadableJournal for FilteredJournal { + fn read(&self) -> anyhow::Result>> { + self.rx.read() + } + + fn as_restarted(&self) -> anyhow::Result> { + self.rx.as_restarted() + } +} + +impl Journal for FilteredJournal { + fn split(self) -> (Box, Box) { + (Box::new(self.tx), Box::new(self.rx)) + } } diff --git a/lib/wasix/src/journaling/concrete/log_file.rs b/lib/wasix/src/journaling/concrete/log_file.rs index acef42767cf..a9a92cae218 100644 --- a/lib/wasix/src/journaling/concrete/log_file.rs +++ b/lib/wasix/src/journaling/concrete/log_file.rs @@ -7,16 +7,11 @@ use std::{ fs::File, io::{Seek, SeekFrom, Write}, path::Path, + sync::{Arc, Mutex}, }; use super::*; -struct State { - file: File, - serializer: CompositeSerializer, AllocScratch, SharedSerializeMap>, - buffer_pos: usize, -} - /// The LogFile snapshot capturer will write its snapshots to a linear journal /// and read them when restoring. It uses the `bincode` serializer which /// means that forwards and backwards compatibility must be dealt with @@ -29,21 +24,33 @@ struct State { /// The logfile snapshot capturer uses a 64bit number as a entry encoding /// delimiter. pub struct LogFileJournal { - state: std::sync::Mutex, + tx: LogFileJournalTx, + rx: LogFileJournalRx, +} + +#[derive(Debug)] +struct TxState { + file: File, + serializer: CompositeSerializer, AllocScratch, SharedSerializeMap>, +} + +#[derive(Debug, Clone)] +pub struct LogFileJournalTx { + state: Arc>, +} + +#[derive(Debug)] +pub struct LogFileJournalRx { + tx: LogFileJournalTx, + buffer_pos: Mutex, buffer: OwnedBuffer, } -impl LogFileJournal { - pub fn new(path: impl AsRef) -> anyhow::Result { - let file = std::fs::File::options() - .read(true) - .write(true) - .create(true) - .open(path)?; - Self::from_file(file) - } +impl LogFileJournalTx { + pub fn as_rx(&self) -> anyhow::Result { + let state = self.state.lock().unwrap(); + let file = state.file.try_clone()?; - pub fn from_file(mut file: std::fs::File) -> anyhow::Result { let buffer = OwnedBuffer::from_file(&file)?; // If the buffer exists we valid the magic number @@ -64,6 +71,25 @@ impl LogFileJournal { tracing::trace!("journal has no magic (could be empty?)"); } + Ok(LogFileJournalRx { + tx: self.clone(), + buffer_pos: Mutex::new(buffer_pos), + buffer, + }) + } +} + +impl LogFileJournal { + pub fn new(path: impl AsRef) -> anyhow::Result { + let file = std::fs::File::options() + .read(true) + .write(true) + .create(true) + .open(path)?; + Self::from_file(file) + } + + pub fn from_file(mut file: std::fs::File) -> anyhow::Result { // Move to the end of the file and write the // magic if one is needed if file.seek(SeekFrom::End(0)).unwrap() == 0 { @@ -72,25 +98,26 @@ impl LogFileJournal { file.write_all(&magic)?; } - // We create various serializers and state that is - // used when writing. Plus we store the owned buffer - // thats used for zero copy deserialization - Ok(Self { - state: std::sync::Mutex::new(State { + // Create the tx + let tx = LogFileJournalTx { + state: Arc::new(Mutex::new(TxState { file: file.try_clone()?, serializer: CompositeSerializer::new( WriteSerializer::new(file), AllocScratch::default(), SharedSerializeMap::default(), ), - buffer_pos, - }), - buffer, - }) + })), + }; + + // First we create the readable journal + let rx = tx.as_rx()?; + + Ok(Self { rx, tx }) } } -impl Journal for LogFileJournal { +impl WritableJournal for LogFileJournalTx { fn write<'a>(&'a self, entry: JournalEntry<'a>) -> anyhow::Result<()> { tracing::debug!("journal event: {:?}", entry); @@ -118,16 +145,18 @@ impl Journal for LogFileJournal { // Now write the actual data and update the offsets Ok(()) } +} +impl ReadableJournal for LogFileJournalRx { /// UNSAFE: This method uses unsafe operations to remove the need to zero /// the buffer before its read the log entries into it fn read<'a>(&'a self) -> anyhow::Result>> { - let mut state = self.state.lock().unwrap(); + let mut buffer_pos = self.buffer_pos.lock().unwrap(); // Get a memory reference to the data on the disk at // the current read location let mut buffer_ptr = self.buffer.as_ref(); - buffer_ptr.advance(state.buffer_pos); + buffer_ptr.advance(*buffer_pos); loop { // Read the headers and advance let header_size = 8; @@ -145,7 +174,7 @@ impl Journal for LogFileJournal { }; if header.record_size as usize > buffer_ptr.len() { - state.buffer_pos += buffer_ptr.len(); + *buffer_pos += buffer_ptr.len(); tracing::trace!( "journal is corrupt (record_size={} vs remaining={})", header.record_size, @@ -157,8 +186,8 @@ impl Journal for LogFileJournal { // Move the buffer position forward let entry = &buffer_ptr[..(header.record_size as usize)]; buffer_ptr.advance(header.record_size as usize); - state.buffer_pos += header_size; - state.buffer_pos += header.record_size as usize; + *buffer_pos += header_size; + *buffer_pos += header.record_size as usize; // Now we read the entry let record_type: JournalEntryRecordType = match header.record_type.try_into() { @@ -176,6 +205,33 @@ impl Journal for LogFileJournal { return Ok(Some(record)); } } + + fn as_restarted(&self) -> anyhow::Result> { + let ret = self.tx.as_rx()?; + Ok(Box::new(ret)) + } +} + +impl WritableJournal for LogFileJournal { + fn write<'a>(&'a self, entry: JournalEntry<'a>) -> anyhow::Result<()> { + self.tx.write(entry) + } +} + +impl ReadableJournal for LogFileJournal { + fn read(&self) -> anyhow::Result>> { + self.rx.read() + } + + fn as_restarted(&self) -> anyhow::Result> { + self.rx.as_restarted() + } +} + +impl Journal for LogFileJournal { + fn split(self) -> (Box, Box) { + (Box::new(self.tx), Box::new(self.rx)) + } } #[cfg(test)] diff --git a/lib/wasix/src/journaling/concrete/mod.rs b/lib/wasix/src/journaling/concrete/mod.rs index 9d90f736012..d6f73994f47 100644 --- a/lib/wasix/src/journaling/concrete/mod.rs +++ b/lib/wasix/src/journaling/concrete/mod.rs @@ -1,5 +1,7 @@ mod archived_journal; +mod boxed_journal; mod compactor; +mod composite; mod filter; #[cfg(feature = "journal")] mod log_file; @@ -9,7 +11,9 @@ mod unsupported; pub(super) use super::*; pub use archived_journal::*; +pub use boxed_journal::*; pub use compactor::*; +pub use composite::*; pub use filter::*; #[cfg(feature = "journal")] pub use log_file::*; diff --git a/lib/wasix/src/journaling/concrete/pipe.rs b/lib/wasix/src/journaling/concrete/pipe.rs index 78d1c07639f..3d708a2f20c 100644 --- a/lib/wasix/src/journaling/concrete/pipe.rs +++ b/lib/wasix/src/journaling/concrete/pipe.rs @@ -1,6 +1,6 @@ -use std::sync::mpsc; use std::sync::mpsc::TryRecvError; use std::sync::Mutex; +use std::sync::{mpsc, Arc}; use super::*; @@ -8,8 +8,18 @@ use super::*; // of a pipe. #[derive(Debug)] pub struct PipeJournal { - tx: mpsc::Sender>, - rx: Mutex>>, + tx: PipeJournalTx, + rx: PipeJournalRx, +} + +#[derive(Debug)] +pub struct PipeJournalRx { + receiver: Arc>>>, +} + +#[derive(Debug)] +pub struct PipeJournalTx { + sender: Arc>>>, } impl PipeJournal { @@ -18,29 +28,41 @@ impl PipeJournal { let (tx2, rx2) = mpsc::channel(); let end1 = PipeJournal { - tx: tx1, - rx: Mutex::new(rx2), + tx: PipeJournalTx { + sender: Arc::new(Mutex::new(tx1)), + }, + rx: PipeJournalRx { + receiver: Arc::new(Mutex::new(rx2)), + }, }; let end2 = PipeJournal { - tx: tx2, - rx: Mutex::new(rx1), + tx: PipeJournalTx { + sender: Arc::new(Mutex::new(tx2)), + }, + rx: PipeJournalRx { + receiver: Arc::new(Mutex::new(rx1)), + }, }; (end1, end2) } } -impl Journal for PipeJournal { +impl WritableJournal for PipeJournalTx { fn write<'a>(&'a self, entry: JournalEntry<'a>) -> anyhow::Result<()> { let entry = entry.into_owned(); - self.tx.send(entry).map_err(|err| { + + let sender = self.sender.lock().unwrap(); + sender.send(entry).map_err(|err| { anyhow::format_err!("failed to send journal event through the pipe - {}", err) }) } +} +impl ReadableJournal for PipeJournalRx { fn read(&self) -> anyhow::Result>> { - let rx = self.rx.lock().unwrap(); + let rx = self.receiver.lock().unwrap(); match rx.try_recv() { Ok(e) => Ok(Some(e)), Err(TryRecvError::Empty) => Ok(None), @@ -49,4 +71,32 @@ impl Journal for PipeJournal { )), } } + + fn as_restarted(&self) -> anyhow::Result> { + Ok(Box::new(PipeJournalRx { + receiver: self.receiver.clone(), + })) + } +} + +impl WritableJournal for PipeJournal { + fn write<'a>(&'a self, entry: JournalEntry<'a>) -> anyhow::Result<()> { + self.tx.write(entry) + } +} + +impl ReadableJournal for PipeJournal { + fn read(&self) -> anyhow::Result>> { + self.rx.read() + } + + fn as_restarted(&self) -> anyhow::Result> { + self.rx.as_restarted() + } +} + +impl Journal for PipeJournal { + fn split(self) -> (Box, Box) { + (Box::new(self.tx), Box::new(self.rx)) + } } diff --git a/lib/wasix/src/journaling/concrete/unsupported.rs b/lib/wasix/src/journaling/concrete/unsupported.rs index fa5e3d1a452..952c7f989a1 100644 --- a/lib/wasix/src/journaling/concrete/unsupported.rs +++ b/lib/wasix/src/journaling/concrete/unsupported.rs @@ -7,13 +7,28 @@ pub static UNSUPPORTED_SNAPSHOT_CAPTURER: UnsupportedJournal = UnsupportedJourna #[derive(Debug, Default)] pub struct UnsupportedJournal {} -impl Journal for UnsupportedJournal { +impl ReadableJournal for UnsupportedJournal { + fn read(&self) -> anyhow::Result>> { + Ok(None) + } + + fn as_restarted(&self) -> anyhow::Result> { + Ok(Box::new(UnsupportedJournal::default())) + } +} + +impl WritableJournal for UnsupportedJournal { fn write<'a>(&'a self, entry: JournalEntry<'a>) -> anyhow::Result<()> { tracing::debug!("journal event: {:?}", entry); Err(anyhow::format_err!("unsupported")) } +} - fn read(&self) -> anyhow::Result>> { - Ok(None) +impl Journal for UnsupportedJournal { + fn split(self) -> (Box, Box) { + ( + Box::new(UnsupportedJournal::default()), + Box::new(UnsupportedJournal::default()), + ) } } diff --git a/lib/wasix/src/journaling/journal.rs b/lib/wasix/src/journaling/journal.rs index 418b8c53117..dfcbd5bd688 100644 --- a/lib/wasix/src/journaling/journal.rs +++ b/lib/wasix/src/journaling/journal.rs @@ -639,14 +639,35 @@ impl<'a> JournalEntry<'a> { /// a WASM process at a point in time and saves it so that it can be restored. /// It also allows for the restoration of that state at a later moment #[allow(unused_variables)] -pub trait Journal { +pub trait WritableJournal { /// Takes in a stream of snapshot log entries and saves them so that they /// may be restored at a later moment fn write<'a>(&'a self, entry: JournalEntry<'a>) -> anyhow::Result<()>; +} +/// The snapshot capturer will take a series of objects that represents the state of +/// a WASM process at a point in time and saves it so that it can be restored. +/// It also allows for the restoration of that state at a later moment +#[allow(unused_variables)] +pub trait ReadableJournal { /// Returns a stream of snapshot objects that the runtime will use /// to restore the state of a WASM process to a previous moment in time fn read<'a>(&'a self) -> anyhow::Result>>; + + /// Resets the journal so that reads will start from the + /// beginning again + fn as_restarted(&self) -> anyhow::Result>; +} + +/// The snapshot capturer will take a series of objects that represents the state of +/// a WASM process at a point in time and saves it so that it can be restored. +/// It also allows for the restoration of that state at a later moment +#[allow(unused_variables)] +pub trait Journal: WritableJournal + ReadableJournal { + /// Splits the journal into a read and write side + fn split(self) -> (Box, Box); } pub type DynJournal = dyn Journal + Send + Sync; +pub type DynWritableJournal = dyn WritableJournal + Send + Sync; +pub type DynReadableJournal = dyn ReadableJournal + Send + Sync; From c73b5a802eceed270ad0053816a5692346227486 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Wed, 22 Nov 2023 23:04:20 +1100 Subject: [PATCH 066/129] Fixed some linting issues --- .../journaling/concrete/archived_journal.rs | 69 ++++++++++--------- lib/wasix/src/journaling/concrete/log_file.rs | 14 ++-- lib/wasix/src/journaling/journal.rs | 2 +- 3 files changed, 43 insertions(+), 42 deletions(-) diff --git a/lib/wasix/src/journaling/concrete/archived_journal.rs b/lib/wasix/src/journaling/concrete/archived_journal.rs index 6fe7697f83d..254acda392d 100644 --- a/lib/wasix/src/journaling/concrete/archived_journal.rs +++ b/lib/wasix/src/journaling/concrete/archived_journal.rs @@ -91,7 +91,12 @@ pub enum JournalEntryRecordType { } impl JournalEntryRecordType { - pub unsafe fn deserialize_archive<'a>(self, data: &'a [u8]) -> JournalEntry<'a> { + /// # Safety + /// + /// `rykv` makes direct memory references to achieve high performance + /// however this does mean care must be taken that the data itself + /// can not be manipulated or corrupted. + pub unsafe fn deserialize_archive(self, data: &[u8]) -> JournalEntry<'_> { match self { JournalEntryRecordType::InitModuleV1 => ArchivedJournalEntry::InitModuleV1( rkyv::archived_root::(data), @@ -692,8 +697,8 @@ impl<'a> JournalEntry<'a> { } => serializer.serialize_value(&JournalEntryPortRouteAddV1 { cidr, via_router, - preferred_until: preferred_until.clone(), - expires_at: expires_at.clone(), + preferred_until, + expires_at, }), JournalEntry::PortRouteClear => return Ok(()), JournalEntry::PortRouteDel { ip } => { @@ -818,7 +823,7 @@ impl<'a> JournalEntry<'a> { serializer.serialize_value(&JournalEntrySocketSetOptTimeV1 { fd, ty: ty.into(), - time: time.clone(), + time, }) } JournalEntry::SocketShutdown { fd, how } => { @@ -2389,7 +2394,7 @@ impl<'a> From> for JournalEntry<'a> { ref trigger, }) => Self::Snapshot { when: SystemTime::UNIX_EPOCH - .checked_add(since_epoch.clone().try_into().unwrap()) + .checked_add((*since_epoch).try_into().unwrap()) .unwrap_or(SystemTime::UNIX_EPOCH), trigger: trigger.into(), }, @@ -2608,10 +2613,8 @@ impl<'a> From> for JournalEntry<'a> { via_router: via_router.as_ipaddr(), preferred_until: preferred_until .as_ref() - .map(|time| time.clone().try_into().unwrap()), - expires_at: expires_at - .as_ref() - .map(|time| time.clone().try_into().unwrap()), + .map(|time| (*time).try_into().unwrap()), + expires_at: expires_at.as_ref().map(|time| (*time).try_into().unwrap()), }, ArchivedJournalEntry::PortRouteClearV1 => Self::PortRouteClear, ArchivedJournalEntry::PortRouteDelV1(ArchivedJournalEntryPortRouteDelV1 { ip }) => { @@ -2765,7 +2768,7 @@ impl<'a> From> for JournalEntry<'a> { }) => Self::SocketSetOptTime { fd: *fd, ty: ty.into(), - time: time.as_ref().map(|time| time.clone().try_into().unwrap()), + time: time.as_ref().map(|time| (*time).try_into().unwrap()), }, ArchivedJournalEntry::SocketShutdownV1(ArchivedJournalEntrySocketShutdownV1 { fd, @@ -2844,7 +2847,7 @@ mod tests { #[test] pub fn test_record_process_exit() { run_test(JournalEntry::ProcessExit { - exit_code: Some(ExitCode::Errno(Errno::Fault)), + exit_code: Some(ExitCode::Errno(wasi::Errno::Fault)), }); } @@ -2865,7 +2868,7 @@ mod tests { pub fn test_record_close_thread() { run_test(JournalEntry::CloseThread { id: 987u32.into(), - exit_code: Some(ExitCode::Errno(Errno::Fault)), + exit_code: Some(ExitCode::Errno(wasi::Errno::Fault)), }); } @@ -2875,7 +2878,7 @@ mod tests { run_test(JournalEntry::FileDescriptorSeek { fd: 765u32, offset: 9183722450971234i64, - whence: Whence::End, + whence: wasi::Whence::End, }); } @@ -2903,7 +2906,7 @@ mod tests { #[test] pub fn test_record_set_clock_time() { run_test(JournalEntry::SetClockTime { - clock_id: Snapshot0Clockid::Realtime, + clock_id: wasi::Snapshot0Clockid::Realtime, time: 7912837412934u64, }); } @@ -2916,10 +2919,10 @@ mod tests { dirfd: 23458922u32, dirflags: 134512345, path: "/blah".into(), - o_flags: Oflags::all(), - fs_rights_base: Rights::all(), - fs_rights_inheriting: Rights::all(), - fs_flags: Fdflags::all(), + o_flags: wasi::Oflags::all(), + fs_rights_base: wasi::Rights::all(), + fs_rights_inheriting: wasi::Rights::all(), + fs_flags: wasi::Fdflags::all(), }); } @@ -2974,7 +2977,7 @@ mod tests { path: "/".into(), st_atim: 923452345, st_mtim: 350, - fst_flags: Fstflags::all(), + fst_flags: wasi::Fstflags::all(), }); } @@ -2985,7 +2988,7 @@ mod tests { fd: 898785u32, st_atim: 29834952345, st_mtim: 239845892345, - fst_flags: Fstflags::all(), + fst_flags: wasi::Fstflags::all(), }); } @@ -3003,7 +3006,7 @@ mod tests { pub fn test_record_file_descriptor_set_flags() { run_test(JournalEntry::FileDescriptorSetFlags { fd: 982348752u32, - flags: Fdflags::all(), + flags: wasi::Fdflags::all(), }); } @@ -3012,8 +3015,8 @@ mod tests { pub fn test_record_file_descriptor_set_rights() { run_test(JournalEntry::FileDescriptorSetRights { fd: 872345u32, - fs_rights_base: Rights::all(), - fs_rights_inheriting: Rights::all(), + fs_rights_base: wasi::Rights::all(), + fs_rights_inheriting: wasi::Rights::all(), }); } @@ -3024,7 +3027,7 @@ mod tests { fd: 298434u32, offset: 92834529092345, len: 23485928345, - advice: Advice::Random, + advice: wasi::Advice::Random, }); } @@ -3099,10 +3102,10 @@ mod tests { pub fn test_record_epoll_ctl() { run_test(JournalEntry::EpollCtl { epfd: 34523455, - op: EpollCtl::Unknown, + op: wasi::EpollCtl::Unknown, fd: 23452345, - event: Some(EpollEventCtl { - events: EpollType::all(), + event: Some(wasi::EpollEventCtl { + events: wasi::EpollType::all(), ptr: 32452345, fd: 23452345, data1: 1235245756, @@ -3115,7 +3118,7 @@ mod tests { #[test] pub fn test_record_tty_set() { run_test(JournalEntry::TtySet { - tty: Tty { + tty: wasi::Tty { cols: 1234, rows: 6754, width: 4563456, @@ -3237,8 +3240,8 @@ mod tests { pub fn test_record_socket_open() { run_test(JournalEntry::SocketOpen { af: Addressfamily::Inet6, - ty: Socktype::Stream, - pt: SockProto::Tcp, + ty: wasi::Socktype::Stream, + pt: wasi::SockProto::Tcp, fd: 23452345, }); } @@ -3277,7 +3280,7 @@ mod tests { listen_fd: 21234, fd: 1, peer_addr: SocketAddr::new(Ipv6Addr::UNSPECIFIED.into(), 3452), - fd_flags: Fdflags::all(), + fd_flags: wasi::Fdflags::all(), nonblocking: true, }); } @@ -3361,7 +3364,7 @@ mod tests { pub fn test_record_socket_set_opt_flag() { run_test(JournalEntry::SocketSetOptFlag { fd: 0, - opt: Sockoption::Linger, + opt: wasi::Sockoption::Linger, flag: true, }); } @@ -3371,7 +3374,7 @@ mod tests { pub fn test_record_socket_set_opt_size() { run_test(JournalEntry::SocketSetOptSize { fd: 15, - opt: Sockoption::Linger, + opt: wasi::Sockoption::Linger, size: 234234, }); } diff --git a/lib/wasix/src/journaling/concrete/log_file.rs b/lib/wasix/src/journaling/concrete/log_file.rs index a9a92cae218..25944fc21ce 100644 --- a/lib/wasix/src/journaling/concrete/log_file.rs +++ b/lib/wasix/src/journaling/concrete/log_file.rs @@ -93,7 +93,7 @@ impl LogFileJournal { // Move to the end of the file and write the // magic if one is needed if file.seek(SeekFrom::End(0)).unwrap() == 0 { - let magic = JOURNAL_MAGIC_NUMBER as u64; + let magic = JOURNAL_MAGIC_NUMBER; let magic = magic.to_be_bytes(); file.write_all(&magic)?; } @@ -126,20 +126,18 @@ impl WritableJournal for LogFileJournalTx { // Write the header (with a record size of zero) let record_type: JournalEntryRecordType = entry.archive_record_type(); state.file.write_all(&(record_type as u16).to_be_bytes())?; - let offset_size = state.file.seek(SeekFrom::Current(0))?; + let offset_size = state.file.stream_position()?; state.file.write_all(&[0u8; 6])?; // record size (48 bits) // Now serialize the actual data to the log - let offset_start = state.file.seek(SeekFrom::Current(0))?; + let offset_start = state.file.stream_position()?; entry.serialize_archive(&mut state.serializer)?; - let offset_end = state.file.seek(SeekFrom::Current(0))?; + let offset_end = state.file.stream_position()?; let record_size = offset_end - offset_start; // Write the record and then move back to the end again state.file.seek(SeekFrom::Start(offset_size))?; - state - .file - .write_all(&(record_size as u64).to_be_bytes()[2..8])?; + state.file.write_all(&record_size.to_be_bytes()[2..8])?; state.file.seek(SeekFrom::Start(offset_end))?; // Now write the actual data and update the offsets @@ -150,7 +148,7 @@ impl WritableJournal for LogFileJournalTx { impl ReadableJournal for LogFileJournalRx { /// UNSAFE: This method uses unsafe operations to remove the need to zero /// the buffer before its read the log entries into it - fn read<'a>(&'a self) -> anyhow::Result>> { + fn read(&self) -> anyhow::Result>> { let mut buffer_pos = self.buffer_pos.lock().unwrap(); // Get a memory reference to the data on the disk at diff --git a/lib/wasix/src/journaling/journal.rs b/lib/wasix/src/journaling/journal.rs index dfcbd5bd688..d28e86142d6 100644 --- a/lib/wasix/src/journaling/journal.rs +++ b/lib/wasix/src/journaling/journal.rs @@ -652,7 +652,7 @@ pub trait WritableJournal { pub trait ReadableJournal { /// Returns a stream of snapshot objects that the runtime will use /// to restore the state of a WASM process to a previous moment in time - fn read<'a>(&'a self) -> anyhow::Result>>; + fn read(&self) -> anyhow::Result>>; /// Resets the journal so that reads will start from the /// beginning again From d4fdbc514bd076ed102835fb8c4584c8bbc429b4 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Wed, 22 Nov 2023 23:08:11 +1100 Subject: [PATCH 067/129] Another fix for the linting --- lib/wasix/src/journaling/concrete/archived_journal.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/wasix/src/journaling/concrete/archived_journal.rs b/lib/wasix/src/journaling/concrete/archived_journal.rs index 254acda392d..13f1ee64f64 100644 --- a/lib/wasix/src/journaling/concrete/archived_journal.rs +++ b/lib/wasix/src/journaling/concrete/archived_journal.rs @@ -2795,7 +2795,6 @@ mod tests { use rkyv::ser::serializers::{ AllocScratch, CompositeSerializer, SharedSerializeMap, WriteSerializer, }; - use wasmer_wasix_types::wasi::{Addressfamily, SockProto, Socktype, Tty}; use super::*; @@ -3239,7 +3238,7 @@ mod tests { #[test] pub fn test_record_socket_open() { run_test(JournalEntry::SocketOpen { - af: Addressfamily::Inet6, + af: wasi::Addressfamily::Inet6, ty: wasi::Socktype::Stream, pt: wasi::SockProto::Tcp, fd: 23452345, From 787f9e96c209e661d340d21f646307a66f673af0 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Thu, 23 Nov 2023 05:03:33 +1100 Subject: [PATCH 068/129] More linting fixes and additional tests --- .../journaling/concrete/archived_journal.rs | 4 +- .../src/journaling/concrete/boxed_journal.rs | 4 +- .../src/journaling/concrete/compactor.rs | 330 +++++++++++++++++- .../src/journaling/concrete/unsupported.rs | 6 +- 4 files changed, 328 insertions(+), 16 deletions(-) diff --git a/lib/wasix/src/journaling/concrete/archived_journal.rs b/lib/wasix/src/journaling/concrete/archived_journal.rs index 13f1ee64f64..f94580881c3 100644 --- a/lib/wasix/src/journaling/concrete/archived_journal.rs +++ b/lib/wasix/src/journaling/concrete/archived_journal.rs @@ -2846,7 +2846,7 @@ mod tests { #[test] pub fn test_record_process_exit() { run_test(JournalEntry::ProcessExit { - exit_code: Some(ExitCode::Errno(wasi::Errno::Fault)), + exit_code: Some(wasi::ExitCode::Errno(wasi::Errno::Fault)), }); } @@ -2867,7 +2867,7 @@ mod tests { pub fn test_record_close_thread() { run_test(JournalEntry::CloseThread { id: 987u32.into(), - exit_code: Some(ExitCode::Errno(wasi::Errno::Fault)), + exit_code: Some(wasi::ExitCode::Errno(wasi::Errno::Fault)), }); } diff --git a/lib/wasix/src/journaling/concrete/boxed_journal.rs b/lib/wasix/src/journaling/concrete/boxed_journal.rs index 70b29d131e1..b2d0be4665c 100644 --- a/lib/wasix/src/journaling/concrete/boxed_journal.rs +++ b/lib/wasix/src/journaling/concrete/boxed_journal.rs @@ -3,7 +3,7 @@ use std::ops::Deref; use super::*; impl ReadableJournal for Box { - fn read<'a>(&'a self) -> anyhow::Result>> { + fn read(&self) -> anyhow::Result>> { self.deref().read() } @@ -19,7 +19,7 @@ impl WritableJournal for Box { } impl ReadableJournal for Box { - fn read<'a>(&'a self) -> anyhow::Result>> { + fn read(&self) -> anyhow::Result>> { self.deref().read() } diff --git a/lib/wasix/src/journaling/concrete/compactor.rs b/lib/wasix/src/journaling/concrete/compactor.rs index 639241d71ce..538008a7c87 100644 --- a/lib/wasix/src/journaling/concrete/compactor.rs +++ b/lib/wasix/src/journaling/concrete/compactor.rs @@ -215,7 +215,7 @@ impl WritableJournal for CompactingJournalTx { state.thread_map.insert(*id, event_index); } JournalEntry::CloseThread { id, .. } => { - state.thread_map.remove(&id); + state.thread_map.remove(id); } JournalEntry::ProcessExit { .. } => { state.thread_map.clear(); @@ -225,7 +225,7 @@ impl WritableJournal for CompactingJournalTx { } JournalEntry::CloseFileDescriptor { fd } => { // If its not suspect we need to record this event - if state.suspect_descriptors.remove(&fd).is_some() { + if state.suspect_descriptors.remove(fd).is_some() { // suspect descriptors that are closed are dropped // as they made no material difference to the state } else if let Some(e) = state.keep_descriptors.get_mut(fd) { @@ -242,9 +242,9 @@ impl WritableJournal for CompactingJournalTx { } // We keep non-mutable events for file descriptors that are suspect JournalEntry::FileDescriptorSeek { fd, .. } => { - if let Some(events) = state.suspect_descriptors.get_mut(&fd) { + if let Some(events) = state.suspect_descriptors.get_mut(fd) { events.push(event_index); - } else if let Some(s) = state.keep_descriptors.get_mut(&fd) { + } else if let Some(s) = state.keep_descriptors.get_mut(fd) { s.events.push(event_index); } else { state.whitelist.insert(event_index); @@ -261,7 +261,7 @@ impl WritableJournal for CompactingJournalTx { original_fd: fd, .. } => { // Its no longer suspect - if let Some(events) = state.suspect_descriptors.remove(&fd) { + if let Some(events) = state.suspect_descriptors.remove(fd) { state.keep_descriptors.insert( *fd, StateDescriptor { @@ -351,10 +351,13 @@ impl Journal for CompactingJournal { } } +#[cfg(feature = "journal")] #[cfg(test)] mod tests { use super::*; + use wasmer_wasix_types::wasi; + pub fn run_test<'a>( in_records: Vec>, out_records: Vec>, @@ -406,6 +409,84 @@ mod tests { .unwrap() } + #[tracing_test::traced_test] + #[test] + pub fn test_compact_keep_overlapping_memory() { + run_test( + vec![ + JournalEntry::UpdateMemoryRegion { + region: 0..16, + data: [11u8; 16].to_vec().into(), + }, + JournalEntry::UpdateMemoryRegion { + region: 20..36, + data: [22u8; 16].to_vec().into(), + }, + ], + vec![ + JournalEntry::UpdateMemoryRegion { + region: 0..16, + data: [11u8; 16].to_vec().into(), + }, + JournalEntry::UpdateMemoryRegion { + region: 20..36, + data: [22u8; 16].to_vec().into(), + }, + ], + ) + .unwrap() + } + + #[tracing_test::traced_test] + #[test] + pub fn test_compact_keep_adjacent_memory_writes() { + run_test( + vec![ + JournalEntry::UpdateMemoryRegion { + region: 0..16, + data: [11u8; 16].to_vec().into(), + }, + JournalEntry::UpdateMemoryRegion { + region: 16..32, + data: [22u8; 16].to_vec().into(), + }, + ], + vec![ + JournalEntry::UpdateMemoryRegion { + region: 0..16, + data: [11u8; 16].to_vec().into(), + }, + JournalEntry::UpdateMemoryRegion { + region: 16..32, + data: [22u8; 16].to_vec().into(), + }, + ], + ) + .unwrap() + } + + #[tracing_test::traced_test] + #[test] + pub fn test_compact_purge_identical_memory_writes() { + run_test( + vec![ + JournalEntry::UpdateMemoryRegion { + region: 0..16, + data: [11u8; 16].to_vec().into(), + }, + JournalEntry::UpdateMemoryRegion { + region: 0..16, + data: [11u8; 16].to_vec().into(), + }, + ], + vec![JournalEntry::UpdateMemoryRegion { + region: 0..16, + data: [11u8; 16].to_vec().into(), + }], + ) + .unwrap() + } + #[tracing_test::traced_test] #[test] pub fn test_compact_thread_stacks() { @@ -478,10 +559,10 @@ mod tests { dirfd: 3452345, dirflags: 0, path: "/blah".into(), - o_flags: Oflags::all(), - fs_rights_base: Rights::all(), - fs_rights_inheriting: Rights::all(), - fs_flags: Fdflags::all(), + o_flags: wasi::Oflags::all(), + fs_rights_base: wasi::Rights::all(), + fs_rights_inheriting: wasi::Rights::all(), + fs_flags: wasi::Fdflags::all(), }, JournalEntry::ProcessExit { exit_code: None }, ], @@ -489,4 +570,235 @@ mod tests { ) .unwrap() } + + #[tracing_test::traced_test] + #[test] + pub fn test_compact_file_system_partial_write_survives() { + run_test( + vec![ + JournalEntry::OpenFileDescriptor { + fd: 1234, + dirfd: 3452345, + dirflags: 0, + path: "/blah".into(), + o_flags: wasi::Oflags::all(), + fs_rights_base: wasi::Rights::all(), + fs_rights_inheriting: wasi::Rights::all(), + fs_flags: wasi::Fdflags::all(), + }, + JournalEntry::FileDescriptorWrite { + fd: 1234, + offset: 1234, + data: [1u8; 16].to_vec().into(), + is_64bit: true, + }, + ], + vec![ + JournalEntry::OpenFileDescriptor { + fd: 1234, + dirfd: 3452345, + dirflags: 0, + path: "/blah".into(), + o_flags: wasi::Oflags::all(), + fs_rights_base: wasi::Rights::all(), + fs_rights_inheriting: wasi::Rights::all(), + fs_flags: wasi::Fdflags::all(), + }, + JournalEntry::FileDescriptorWrite { + fd: 1234, + offset: 1234, + data: [1u8; 16].to_vec().into(), + is_64bit: true, + }, + ], + ) + .unwrap() + } + + #[tracing_test::traced_test] + #[test] + pub fn test_compact_file_system_write_survives_close() { + run_test( + vec![ + JournalEntry::OpenFileDescriptor { + fd: 1234, + dirfd: 3452345, + dirflags: 0, + path: "/blah".into(), + o_flags: wasi::Oflags::all(), + fs_rights_base: wasi::Rights::all(), + fs_rights_inheriting: wasi::Rights::all(), + fs_flags: wasi::Fdflags::all(), + }, + JournalEntry::FileDescriptorWrite { + fd: 1234, + offset: 1234, + data: [1u8; 16].to_vec().into(), + is_64bit: true, + }, + JournalEntry::CloseFileDescriptor { fd: 1234 }, + ], + vec![ + JournalEntry::OpenFileDescriptor { + fd: 1234, + dirfd: 3452345, + dirflags: 0, + path: "/blah".into(), + o_flags: wasi::Oflags::all(), + fs_rights_base: wasi::Rights::all(), + fs_rights_inheriting: wasi::Rights::all(), + fs_flags: wasi::Fdflags::all(), + }, + JournalEntry::FileDescriptorWrite { + fd: 1234, + offset: 1234, + data: [1u8; 16].to_vec().into(), + is_64bit: true, + }, + JournalEntry::CloseFileDescriptor { fd: 1234 }, + ], + ) + .unwrap() + } + + #[tracing_test::traced_test] + #[test] + pub fn test_compact_file_system_write_survives_exit() { + run_test( + vec![ + JournalEntry::OpenFileDescriptor { + fd: 1234, + dirfd: 3452345, + dirflags: 0, + path: "/blah".into(), + o_flags: wasi::Oflags::all(), + fs_rights_base: wasi::Rights::all(), + fs_rights_inheriting: wasi::Rights::all(), + fs_flags: wasi::Fdflags::all(), + }, + JournalEntry::FileDescriptorWrite { + fd: 1234, + offset: 1234, + data: [1u8; 16].to_vec().into(), + is_64bit: true, + }, + JournalEntry::ProcessExit { exit_code: None }, + ], + vec![ + JournalEntry::OpenFileDescriptor { + fd: 1234, + dirfd: 3452345, + dirflags: 0, + path: "/blah".into(), + o_flags: wasi::Oflags::all(), + fs_rights_base: wasi::Rights::all(), + fs_rights_inheriting: wasi::Rights::all(), + fs_flags: wasi::Fdflags::all(), + }, + JournalEntry::FileDescriptorWrite { + fd: 1234, + offset: 1234, + data: [1u8; 16].to_vec().into(), + is_64bit: true, + }, + JournalEntry::ProcessExit { exit_code: None }, + ], + ) + .unwrap() + } + + #[tracing_test::traced_test] + #[test] + pub fn test_compact_file_system_read_is_ignored() { + run_test( + vec![ + JournalEntry::OpenFileDescriptor { + fd: 1234, + dirfd: 3452345, + dirflags: 0, + path: "/blah".into(), + o_flags: wasi::Oflags::all(), + fs_rights_base: wasi::Rights::all(), + fs_rights_inheriting: wasi::Rights::all(), + fs_flags: wasi::Fdflags::all(), + }, + JournalEntry::FileDescriptorSeek { + fd: 1234, + offset: 1234, + whence: wasi::Whence::End, + }, + JournalEntry::CloseFileDescriptor { fd: 1234 }, + ], + Vec::new(), + ) + .unwrap() + } + + #[tracing_test::traced_test] + #[test] + pub fn test_compact_file_system_ignore_double_writes() { + run_test( + vec![ + JournalEntry::OpenFileDescriptor { + fd: 1234, + dirfd: 3452345, + dirflags: 0, + path: "/blah".into(), + o_flags: wasi::Oflags::all(), + fs_rights_base: wasi::Rights::all(), + fs_rights_inheriting: wasi::Rights::all(), + fs_flags: wasi::Fdflags::all(), + }, + JournalEntry::FileDescriptorWrite { + fd: 1234, + offset: 1234, + data: [1u8; 16].to_vec().into(), + is_64bit: true, + }, + JournalEntry::FileDescriptorWrite { + fd: 1234, + offset: 1234, + data: [5u8; 16].to_vec().into(), + is_64bit: true, + }, + JournalEntry::CloseFileDescriptor { fd: 1234 }, + ], + vec![ + JournalEntry::OpenFileDescriptor { + fd: 1234, + dirfd: 3452345, + dirflags: 0, + path: "/blah".into(), + o_flags: wasi::Oflags::all(), + fs_rights_base: wasi::Rights::all(), + fs_rights_inheriting: wasi::Rights::all(), + fs_flags: wasi::Fdflags::all(), + }, + JournalEntry::FileDescriptorWrite { + fd: 1234, + offset: 1234, + data: [5u8; 16].to_vec().into(), + is_64bit: true, + }, + JournalEntry::CloseFileDescriptor { fd: 1234 }, + ], + ) + .unwrap() + } + + #[tracing_test::traced_test] + #[test] + pub fn test_compact_file_system_create_directory() { + run_test( + vec![JournalEntry::CreateDirectory { + fd: 1234, + path: "/blah".into(), + }], + vec![JournalEntry::CreateDirectory { + fd: 1234, + path: "/blah".into(), + }], + ) + .unwrap() + } } diff --git a/lib/wasix/src/journaling/concrete/unsupported.rs b/lib/wasix/src/journaling/concrete/unsupported.rs index 952c7f989a1..9f0dbc90116 100644 --- a/lib/wasix/src/journaling/concrete/unsupported.rs +++ b/lib/wasix/src/journaling/concrete/unsupported.rs @@ -13,7 +13,7 @@ impl ReadableJournal for UnsupportedJournal { } fn as_restarted(&self) -> anyhow::Result> { - Ok(Box::new(UnsupportedJournal::default())) + Ok(Box::::default()) } } @@ -27,8 +27,8 @@ impl WritableJournal for UnsupportedJournal { impl Journal for UnsupportedJournal { fn split(self) -> (Box, Box) { ( - Box::new(UnsupportedJournal::default()), - Box::new(UnsupportedJournal::default()), + Box::::default(), + Box::::default(), ) } } From e4fc594cc23eb5c0a089aa6ab7e5bb8ea0fa9431 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Thu, 23 Nov 2023 17:23:48 +1100 Subject: [PATCH 069/129] Finished the compactor and added CLI --- Cargo.lock | 2 + lib/cli/src/cli.rs | 9 +- lib/cli/src/commands/journal/compact.rs | 34 + lib/cli/src/commands/journal/export.rs | 28 + lib/cli/src/commands/journal/filter.rs | 108 +++ lib/cli/src/commands/journal/import.rs | 57 ++ lib/cli/src/commands/journal/inspect.rs | 28 + lib/cli/src/commands/journal/mod.rs | 40 ++ lib/cli/src/commands/mod.rs | 6 +- lib/cli/src/commands/run/mod.rs | 2 +- lib/cli/src/commands/run/wasi.rs | 2 +- lib/virtual-fs/Cargo.toml | 3 +- lib/virtual-fs/src/host_fs.rs | 46 +- lib/wasi-types/src/wasi/bindings.rs | 10 + lib/wasix/Cargo.toml | 1 + lib/wasix/src/journal/base64.rs | 18 + .../concrete/archived.rs} | 657 +++++++++++------- .../concrete/boxed.rs} | 4 +- lib/wasix/src/journal/concrete/buffered.rs | 91 +++ .../concrete/compacting.rs} | 54 +- .../journal/concrete/compacting_log_file.rs | 201 ++++++ .../concrete/filter.rs | 22 +- .../concrete/log_file.rs | 99 ++- lib/wasix/src/journal/concrete/mod.rs | 29 + lib/wasix/src/journal/concrete/null.rs | 30 + .../{journaling => journal}/concrete/pipe.rs | 8 +- lib/wasix/src/journal/concrete/printing.rs | 322 +++++++++ .../concrete/recombined.rs} | 12 +- .../concrete/unsupported.rs | 4 +- .../effector/memory_and_snapshot.rs | 0 .../{journaling => journal}/effector/mod.rs | 0 .../effector/process_exit.rs | 0 .../effector/save_event.rs | 0 .../effector/syscalls/chdir.rs | 0 .../effector/syscalls/clock_time.rs | 0 .../effector/syscalls/epoll_create.rs | 0 .../effector/syscalls/epoll_ctl.rs | 0 .../effector/syscalls/fd_advise.rs | 0 .../effector/syscalls/fd_allocate.rs | 0 .../effector/syscalls/fd_close.rs | 0 .../effector/syscalls/fd_duplicate.rs | 0 .../effector/syscalls/fd_event.rs | 0 .../effector/syscalls/fd_pipe.rs | 0 .../effector/syscalls/fd_renumber.rs | 0 .../effector/syscalls/fd_seek.rs | 0 .../effector/syscalls/fd_set_flags.rs | 0 .../effector/syscalls/fd_set_rights.rs | 0 .../effector/syscalls/fd_set_size.rs | 0 .../effector/syscalls/fd_set_times.rs | 0 .../effector/syscalls/fd_write.rs | 0 .../syscalls/path_create_directory.rs | 0 .../effector/syscalls/path_link.rs | 0 .../effector/syscalls/path_open.rs | 0 .../syscalls/path_remove_directory.rs | 0 .../effector/syscalls/path_rename.rs | 0 .../effector/syscalls/path_set_times.rs | 0 .../effector/syscalls/path_symlink.rs | 0 .../effector/syscalls/path_unlink.rs | 0 .../effector/syscalls/port_addr_add.rs | 0 .../effector/syscalls/port_addr_clear.rs | 0 .../effector/syscalls/port_addr_remove.rs | 0 .../effector/syscalls/port_bridge.rs | 0 .../effector/syscalls/port_dhcp_acquire.rs | 0 .../effector/syscalls/port_gateway_set.rs | 0 .../effector/syscalls/port_route_add.rs | 0 .../effector/syscalls/port_route_clear.rs | 0 .../effector/syscalls/port_route_remove.rs | 0 .../effector/syscalls/port_unbridge.rs | 0 .../effector/syscalls/sock_accept.rs | 0 .../effector/syscalls/sock_bind.rs | 0 .../effector/syscalls/sock_connect.rs | 0 .../syscalls/sock_join_ipv4_multicast.rs | 0 .../syscalls/sock_join_ipv6_multicast.rs | 0 .../syscalls/sock_leave_ipv4_multicast.rs | 0 .../syscalls/sock_leave_ipv6_multicast.rs | 0 .../effector/syscalls/sock_listen.rs | 0 .../effector/syscalls/sock_open.rs | 0 .../effector/syscalls/sock_send.rs | 0 .../effector/syscalls/sock_send_file.rs | 0 .../effector/syscalls/sock_send_to.rs | 0 .../effector/syscalls/sock_set_opt_flag.rs | 0 .../effector/syscalls/sock_set_opt_size.rs | 0 .../effector/syscalls/sock_set_opt_time.rs | 9 +- .../effector/syscalls/sock_shutdown.rs | 8 +- .../effector/syscalls/tty_set.rs | 0 .../effector/thread_exit.rs | 0 .../effector/thread_state.rs | 0 .../effector/unimplemented.rs | 0 .../journal.rs => journal/entry.rs} | 185 +++-- lib/wasix/src/journal/mod.rs | 56 ++ .../mod.rs => journal/snapshot.rs} | 15 +- lib/wasix/src/journal/util.rs | 11 + lib/wasix/src/journaling/concrete/mod.rs | 21 - lib/wasix/src/lib.rs | 4 +- lib/wasix/src/net/socket.rs | 1 + lib/wasix/src/os/task/process.rs | 4 +- lib/wasix/src/runners/wasi.rs | 4 +- lib/wasix/src/runners/wasi_common.rs | 2 +- lib/wasix/src/runners/wcgi/runner.rs | 12 +- lib/wasix/src/runtime/mod.rs | 2 +- lib/wasix/src/state/builder.rs | 6 +- lib/wasix/src/state/env.rs | 2 +- lib/wasix/src/syscalls/mod.rs | 128 ++-- lib/wasix/src/syscalls/wasi/environ_get.rs | 2 +- .../src/syscalls/wasi/environ_sizes_get.rs | 2 +- lib/wasix/src/syscalls/wasi/fd_read.rs | 2 +- lib/wasix/src/syscalls/wasi/fd_write.rs | 2 +- lib/wasix/src/syscalls/wasix/sock_listen.rs | 2 +- lib/wasix/src/syscalls/wasix/thread_spawn.rs | 2 +- 109 files changed, 1921 insertions(+), 488 deletions(-) create mode 100644 lib/cli/src/commands/journal/compact.rs create mode 100644 lib/cli/src/commands/journal/export.rs create mode 100644 lib/cli/src/commands/journal/filter.rs create mode 100644 lib/cli/src/commands/journal/import.rs create mode 100644 lib/cli/src/commands/journal/inspect.rs create mode 100644 lib/cli/src/commands/journal/mod.rs create mode 100644 lib/wasix/src/journal/base64.rs rename lib/wasix/src/{journaling/concrete/archived_journal.rs => journal/concrete/archived.rs} (87%) rename lib/wasix/src/{journaling/concrete/boxed_journal.rs => journal/concrete/boxed.rs} (97%) create mode 100644 lib/wasix/src/journal/concrete/buffered.rs rename lib/wasix/src/{journaling/concrete/compactor.rs => journal/concrete/compacting.rs} (94%) create mode 100644 lib/wasix/src/journal/concrete/compacting_log_file.rs rename lib/wasix/src/{journaling => journal}/concrete/filter.rs (95%) rename lib/wasix/src/{journaling => journal}/concrete/log_file.rs (78%) create mode 100644 lib/wasix/src/journal/concrete/mod.rs create mode 100644 lib/wasix/src/journal/concrete/null.rs rename lib/wasix/src/{journaling => journal}/concrete/pipe.rs (95%) create mode 100644 lib/wasix/src/journal/concrete/printing.rs rename lib/wasix/src/{journaling/concrete/composite.rs => journal/concrete/recombined.rs} (75%) rename lib/wasix/src/{journaling => journal}/concrete/unsupported.rs (90%) rename lib/wasix/src/{journaling => journal}/effector/memory_and_snapshot.rs (100%) rename lib/wasix/src/{journaling => journal}/effector/mod.rs (100%) rename lib/wasix/src/{journaling => journal}/effector/process_exit.rs (100%) rename lib/wasix/src/{journaling => journal}/effector/save_event.rs (100%) rename lib/wasix/src/{journaling => journal}/effector/syscalls/chdir.rs (100%) rename lib/wasix/src/{journaling => journal}/effector/syscalls/clock_time.rs (100%) rename lib/wasix/src/{journaling => journal}/effector/syscalls/epoll_create.rs (100%) rename lib/wasix/src/{journaling => journal}/effector/syscalls/epoll_ctl.rs (100%) rename lib/wasix/src/{journaling => journal}/effector/syscalls/fd_advise.rs (100%) rename lib/wasix/src/{journaling => journal}/effector/syscalls/fd_allocate.rs (100%) rename lib/wasix/src/{journaling => journal}/effector/syscalls/fd_close.rs (100%) rename lib/wasix/src/{journaling => journal}/effector/syscalls/fd_duplicate.rs (100%) rename lib/wasix/src/{journaling => journal}/effector/syscalls/fd_event.rs (100%) rename lib/wasix/src/{journaling => journal}/effector/syscalls/fd_pipe.rs (100%) rename lib/wasix/src/{journaling => journal}/effector/syscalls/fd_renumber.rs (100%) rename lib/wasix/src/{journaling => journal}/effector/syscalls/fd_seek.rs (100%) rename lib/wasix/src/{journaling => journal}/effector/syscalls/fd_set_flags.rs (100%) rename lib/wasix/src/{journaling => journal}/effector/syscalls/fd_set_rights.rs (100%) rename lib/wasix/src/{journaling => journal}/effector/syscalls/fd_set_size.rs (100%) rename lib/wasix/src/{journaling => journal}/effector/syscalls/fd_set_times.rs (100%) rename lib/wasix/src/{journaling => journal}/effector/syscalls/fd_write.rs (100%) rename lib/wasix/src/{journaling => journal}/effector/syscalls/path_create_directory.rs (100%) rename lib/wasix/src/{journaling => journal}/effector/syscalls/path_link.rs (100%) rename lib/wasix/src/{journaling => journal}/effector/syscalls/path_open.rs (100%) rename lib/wasix/src/{journaling => journal}/effector/syscalls/path_remove_directory.rs (100%) rename lib/wasix/src/{journaling => journal}/effector/syscalls/path_rename.rs (100%) rename lib/wasix/src/{journaling => journal}/effector/syscalls/path_set_times.rs (100%) rename lib/wasix/src/{journaling => journal}/effector/syscalls/path_symlink.rs (100%) rename lib/wasix/src/{journaling => journal}/effector/syscalls/path_unlink.rs (100%) rename lib/wasix/src/{journaling => journal}/effector/syscalls/port_addr_add.rs (100%) rename lib/wasix/src/{journaling => journal}/effector/syscalls/port_addr_clear.rs (100%) rename lib/wasix/src/{journaling => journal}/effector/syscalls/port_addr_remove.rs (100%) rename lib/wasix/src/{journaling => journal}/effector/syscalls/port_bridge.rs (100%) rename lib/wasix/src/{journaling => journal}/effector/syscalls/port_dhcp_acquire.rs (100%) rename lib/wasix/src/{journaling => journal}/effector/syscalls/port_gateway_set.rs (100%) rename lib/wasix/src/{journaling => journal}/effector/syscalls/port_route_add.rs (100%) rename lib/wasix/src/{journaling => journal}/effector/syscalls/port_route_clear.rs (100%) rename lib/wasix/src/{journaling => journal}/effector/syscalls/port_route_remove.rs (100%) rename lib/wasix/src/{journaling => journal}/effector/syscalls/port_unbridge.rs (100%) rename lib/wasix/src/{journaling => journal}/effector/syscalls/sock_accept.rs (100%) rename lib/wasix/src/{journaling => journal}/effector/syscalls/sock_bind.rs (100%) rename lib/wasix/src/{journaling => journal}/effector/syscalls/sock_connect.rs (100%) rename lib/wasix/src/{journaling => journal}/effector/syscalls/sock_join_ipv4_multicast.rs (100%) rename lib/wasix/src/{journaling => journal}/effector/syscalls/sock_join_ipv6_multicast.rs (100%) rename lib/wasix/src/{journaling => journal}/effector/syscalls/sock_leave_ipv4_multicast.rs (100%) rename lib/wasix/src/{journaling => journal}/effector/syscalls/sock_leave_ipv6_multicast.rs (100%) rename lib/wasix/src/{journaling => journal}/effector/syscalls/sock_listen.rs (100%) rename lib/wasix/src/{journaling => journal}/effector/syscalls/sock_open.rs (100%) rename lib/wasix/src/{journaling => journal}/effector/syscalls/sock_send.rs (100%) rename lib/wasix/src/{journaling => journal}/effector/syscalls/sock_send_file.rs (100%) rename lib/wasix/src/{journaling => journal}/effector/syscalls/sock_send_to.rs (100%) rename lib/wasix/src/{journaling => journal}/effector/syscalls/sock_set_opt_flag.rs (100%) rename lib/wasix/src/{journaling => journal}/effector/syscalls/sock_set_opt_size.rs (100%) rename lib/wasix/src/{journaling => journal}/effector/syscalls/sock_set_opt_time.rs (84%) rename lib/wasix/src/{journaling => journal}/effector/syscalls/sock_shutdown.rs (83%) rename lib/wasix/src/{journaling => journal}/effector/syscalls/tty_set.rs (100%) rename lib/wasix/src/{journaling => journal}/effector/thread_exit.rs (100%) rename lib/wasix/src/{journaling => journal}/effector/thread_state.rs (100%) rename lib/wasix/src/{journaling => journal}/effector/unimplemented.rs (100%) rename lib/wasix/src/{journaling/journal.rs => journal/entry.rs} (72%) create mode 100644 lib/wasix/src/journal/mod.rs rename lib/wasix/src/{journaling/mod.rs => journal/snapshot.rs} (89%) create mode 100644 lib/wasix/src/journal/util.rs delete mode 100644 lib/wasix/src/journaling/concrete/mod.rs diff --git a/Cargo.lock b/Cargo.lock index 72e7aaa97d5..3962e83b5e6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5090,6 +5090,7 @@ dependencies = [ "pin-project-lite", "pretty_assertions", "replace_with", + "serde", "slab", "tempfile", "thiserror", @@ -6218,6 +6219,7 @@ version = "0.16.0" dependencies = [ "anyhow", "async-trait", + "base64", "bincode", "bytecheck", "bytes", diff --git a/lib/cli/src/cli.rs b/lib/cli/src/cli.rs index 7c38362a4f1..7fb022f6798 100644 --- a/lib/cli/src/cli.rs +++ b/lib/cli/src/cli.rs @@ -9,7 +9,8 @@ use crate::commands::CreateExe; #[cfg(feature = "wast")] use crate::commands::Wast; use crate::commands::{ - Add, Cache, Config, Init, Inspect, Login, Package, Publish, Run, SelfUpdate, Validate, Whoami, + Add, Cache, CmdJournal, Config, Init, Inspect, Login, Package, Publish, Run, SelfUpdate, + Validate, Whoami, }; #[cfg(feature = "static-artifact-create")] use crate::commands::{CreateObj, GenCHeader}; @@ -130,6 +131,7 @@ impl Args { // Deploy commands. Some(Cmd::Deploy(c)) => c.run(), Some(Cmd::App(apps)) => apps.run(), + Some(Cmd::Journal(journal)) => journal.run(), Some(Cmd::Ssh(ssh)) => ssh.run(), Some(Cmd::Namespace(namespace)) => namespace.run(), None => { @@ -266,6 +268,11 @@ enum Cmd { #[clap(alias = "run-unstable")] Run(Run), + /// Manage journals (compacting, inspecting, filtering, ...) + #[clap(subcommand)] + #[cfg(feature = "journal")] + Journal(CmdJournal), + #[clap(subcommand)] Package(crate::commands::Package), diff --git a/lib/cli/src/commands/journal/compact.rs b/lib/cli/src/commands/journal/compact.rs new file mode 100644 index 00000000000..218e47d05e3 --- /dev/null +++ b/lib/cli/src/commands/journal/compact.rs @@ -0,0 +1,34 @@ +use std::path::PathBuf; + +use clap::Parser; +use wasmer_deploy_cli::cmd::AsyncCliCommand; +use wasmer_wasix::journal::{ + copy_journal, CompactingLogFileJournal, LogFileJournal, PrintingJournal, +}; + +/// Compacts a journal by removing duplicate or redundant +/// events and rewriting the log +#[derive(Debug, Parser)] +pub struct CmdJournalCompact { + /// Path to the journal that will be compacted + #[clap(index = 1)] + journal_path: PathBuf, +} + +impl AsyncCliCommand for CmdJournalCompact { + fn run_async(self) -> futures::future::BoxFuture<'static, Result<(), anyhow::Error>> { + Box::pin(self.run()) + } +} + +impl CmdJournalCompact { + async fn run(self) -> Result<(), anyhow::Error> { + let compactor = CompactingLogFileJournal::new(&self.journal_path)?.with_compact_on_drop(); + drop(compactor); + + let journal = LogFileJournal::new(&self.journal_path)?; + let printer = PrintingJournal::default(); + copy_journal(&journal, &printer)?; + Ok(()) + } +} diff --git a/lib/cli/src/commands/journal/export.rs b/lib/cli/src/commands/journal/export.rs new file mode 100644 index 00000000000..469bb853910 --- /dev/null +++ b/lib/cli/src/commands/journal/export.rs @@ -0,0 +1,28 @@ +use std::path::PathBuf; + +use clap::Parser; +use wasmer_deploy_cli::cmd::AsyncCliCommand; +use wasmer_wasix::journal::{copy_journal, JournalPrintingMode, LogFileJournal, PrintingJournal}; + +/// Exports all the events in a journal to STDOUT as JSON data +#[derive(Debug, Parser)] +pub struct CmdJournalExport { + /// Path to the journal that will be printed + #[clap(index = 1)] + journal_path: PathBuf, +} + +impl AsyncCliCommand for CmdJournalExport { + fn run_async(self) -> futures::future::BoxFuture<'static, Result<(), anyhow::Error>> { + Box::pin(self.run()) + } +} + +impl CmdJournalExport { + async fn run(self) -> Result<(), anyhow::Error> { + let journal = LogFileJournal::new(self.journal_path)?; + let printer = PrintingJournal::new(JournalPrintingMode::Json); + copy_journal(&journal, &printer)?; + Ok(()) + } +} diff --git a/lib/cli/src/commands/journal/filter.rs b/lib/cli/src/commands/journal/filter.rs new file mode 100644 index 00000000000..9a94854150f --- /dev/null +++ b/lib/cli/src/commands/journal/filter.rs @@ -0,0 +1,108 @@ +use std::{path::PathBuf, str::FromStr}; + +use clap::Parser; +use wasmer_deploy_cli::cmd::AsyncCliCommand; +use wasmer_wasix::journal::{copy_journal, FilteredJournal, LogFileJournal, PrintingJournal}; + +/// Flags that specify what should be filtered out +#[derive(Debug, Clone, Copy, PartialOrd, Ord, PartialEq, Eq, Hash)] +pub enum FilterOut { + /// Filters out all the memory events + Memory, + /// Filters out all the thread stacks + Threads, + /// Filters out all the file system operations + FileSystem, + /// Filters out all core syscalls + Core, + /// Filters out the snapshots + Snapshots, + /// Filters out the networking + Networking, +} + +impl FromStr for FilterOut { + type Err = String; + + fn from_str(s: &str) -> Result { + Ok(match s.to_lowercase().as_str() { + "mem" | "memory" => Self::Memory, + "thread" | "threads" => Self::Threads, + "fs" | "file" | "filesystem" | "file-system" => Self::FileSystem, + "core" => Self::Core, + "snap" | "snapshot" | "snapshots" => Self::Snapshots, + "net" | "network" | "networking" => Self::Networking, + t => return Err(format!("unknown filter type - {}", t)), + }) + } +} + +/// Rewrites a journal log removing events that match the +/// filter parameters. +#[derive(Debug, Parser)] +pub struct CmdJournalFilter { + /// Path to the journal that will be read + #[clap(index = 1)] + source_path: PathBuf, + /// Path to the journal that will be the output of the filter + #[clap(index = 2)] + target_path: PathBuf, + /// Filters to be applied to the output journal + #[clap(short, long = "filter")] + filters: Vec, +} + +impl AsyncCliCommand for CmdJournalFilter { + fn run_async(self) -> futures::future::BoxFuture<'static, Result<(), anyhow::Error>> { + Box::pin(self.run()) + } +} + +impl CmdJournalFilter { + async fn run(self) -> Result<(), anyhow::Error> { + // Create a new file name that will be the temp new file + // while its written + let mut temp_filename = self + .target_path + .file_name() + .ok_or_else(|| { + anyhow::format_err!( + "The path is not a valid filename - {}", + self.target_path.to_string_lossy() + ) + })? + .to_string_lossy() + .to_string(); + temp_filename.insert_str(0, ".staging."); + let temp_path = self.target_path.clone().with_file_name(&temp_filename); + std::fs::remove_file(&temp_path).ok(); + + // Load the source journal and the target journal (in the temp location) + let source = LogFileJournal::new(self.source_path)?; + let target = LogFileJournal::new(temp_path.clone())?; + + // Put a filter on the farget + let mut target = FilteredJournal::new(target); + for filter in self.filters { + target = match filter { + FilterOut::Memory => target.with_ignore_memory(true), + FilterOut::Threads => target.with_ignore_threads(true), + FilterOut::FileSystem => target.with_ignore_fs(true), + FilterOut::Core => target.with_ignore_core(true), + FilterOut::Snapshots => target.with_ignore_snapshots(true), + FilterOut::Networking => target.with_ignore_networking(true), + } + } + + // Copy the journal over and rename the temp file to the target file + copy_journal(&source, &target)?; + drop(target); + std::fs::rename(temp_path, self.target_path.clone())?; + + // Now print the outcome + let journal = LogFileJournal::new(self.target_path.clone())?; + let printer = PrintingJournal::default(); + copy_journal(&journal, &printer)?; + Ok(()) + } +} diff --git a/lib/cli/src/commands/journal/import.rs b/lib/cli/src/commands/journal/import.rs new file mode 100644 index 00000000000..85e726ba9e3 --- /dev/null +++ b/lib/cli/src/commands/journal/import.rs @@ -0,0 +1,57 @@ +use std::{io::ErrorKind, path::PathBuf}; + +use clap::Parser; +use wasmer_deploy_cli::cmd::AsyncCliCommand; +use wasmer_wasix::journal::{JournalEntry, LogFileJournal, WritableJournal}; + +/// Imports events into a journal file. Events are streamed as JSON +/// objects into `stdin` +#[derive(Debug, Parser)] +pub struct CmdJournaImport { + /// Path to the journal that will be printed + #[clap(index = 1)] + journal_path: PathBuf, +} + +impl AsyncCliCommand for CmdJournaImport { + fn run_async(self) -> futures::future::BoxFuture<'static, Result<(), anyhow::Error>> { + Box::pin(self.run()) + } +} + +impl CmdJournaImport { + async fn run(self) -> Result<(), anyhow::Error> { + // Erase the journal file at the path and reopen it + if self.journal_path.exists() { + std::fs::remove_file(&self.journal_path)?; + } + let journal = LogFileJournal::new(self.journal_path)?; + + // Read all the events from `stdin`, deserialize them and save them to the journal + let stdin = std::io::stdin(); + let mut lines = String::new(); + let mut line = String::new(); + loop { + // Read until the end marker + loop { + line.clear(); + match stdin.read_line(&mut line) { + Ok(0) => return Ok(()), + Ok(_) => { + lines.push_str(&line); + if line.trim_end() == "}" { + break; + } + } + Err(err) if err.kind() == ErrorKind::UnexpectedEof => return Ok(()), + Err(err) => return Err(err.into()), + } + } + + let record = serde_json::from_str::>(&lines)?; + println!("{}", record); + journal.write(record)?; + lines.clear(); + } + } +} diff --git a/lib/cli/src/commands/journal/inspect.rs b/lib/cli/src/commands/journal/inspect.rs new file mode 100644 index 00000000000..bafac9bf480 --- /dev/null +++ b/lib/cli/src/commands/journal/inspect.rs @@ -0,0 +1,28 @@ +use std::path::PathBuf; + +use clap::Parser; +use wasmer_deploy_cli::cmd::AsyncCliCommand; +use wasmer_wasix::journal::{copy_journal, LogFileJournal, PrintingJournal}; + +/// Prints a summarized version of contents of a journal to stdout +#[derive(Debug, Parser)] +pub struct CmdJournaInspect { + /// Path to the journal that will be printed + #[clap(index = 1)] + journal_path: PathBuf, +} + +impl AsyncCliCommand for CmdJournaInspect { + fn run_async(self) -> futures::future::BoxFuture<'static, Result<(), anyhow::Error>> { + Box::pin(self.run()) + } +} + +impl CmdJournaInspect { + async fn run(self) -> Result<(), anyhow::Error> { + let journal = LogFileJournal::new(self.journal_path)?; + let printer = PrintingJournal::default(); + copy_journal(&journal, &printer)?; + Ok(()) + } +} diff --git a/lib/cli/src/commands/journal/mod.rs b/lib/cli/src/commands/journal/mod.rs new file mode 100644 index 00000000000..81394e1c0a7 --- /dev/null +++ b/lib/cli/src/commands/journal/mod.rs @@ -0,0 +1,40 @@ +use wasmer_deploy_cli::cmd::AsyncCliCommand; + +mod compact; +mod export; +mod filter; +mod import; +mod inspect; + +pub use compact::*; +pub use export::*; +pub use filter::*; +pub use import::*; +pub use inspect::*; + +/// Manage Journal files. +#[derive(clap::Subcommand, Debug)] +pub enum CmdJournal { + /// Compacts a journal into a smaller size by removed redundant or duplicate events + Compact(CmdJournalCompact), + /// Exports the contents of a journal to stdout as JSON objects + Export(CmdJournalExport), + /// Imports the events into a journal as JSON objects + Import(CmdJournaImport), + /// Inspects the contents of a journal and summarizes it to `stdout` + Inspect(CmdJournaInspect), + /// Filters out certain events from a journal + Filter(CmdJournalFilter), +} + +impl AsyncCliCommand for CmdJournal { + fn run_async(self) -> futures::future::BoxFuture<'static, Result<(), anyhow::Error>> { + match self { + Self::Compact(cmd) => cmd.run_async(), + Self::Import(cmd) => cmd.run_async(), + Self::Export(cmd) => cmd.run_async(), + Self::Inspect(cmd) => cmd.run_async(), + Self::Filter(cmd) => cmd.run_async(), + } + } +} diff --git a/lib/cli/src/commands/mod.rs b/lib/cli/src/commands/mod.rs index 337faa19c8c..cfe6a439153 100644 --- a/lib/cli/src/commands/mod.rs +++ b/lib/cli/src/commands/mod.rs @@ -15,6 +15,8 @@ mod create_obj; mod gen_c_header; mod init; mod inspect; +#[cfg(feature = "journal")] +mod journal; mod login; mod package; mod publish; @@ -26,8 +28,8 @@ mod wast; mod whoami; pub use self::{ - add::*, cache::*, config::*, container::*, init::*, inspect::*, login::*, package::*, - publish::*, run::Run, self_update::*, validate::*, whoami::*, + add::*, cache::*, config::*, container::*, init::*, inspect::*, journal::*, login::*, + package::*, publish::*, run::Run, self_update::*, validate::*, whoami::*, }; #[cfg(target_os = "linux")] diff --git a/lib/cli/src/commands/run/mod.rs b/lib/cli/src/commands/run/mod.rs index 2a1f9f1922f..7b3a97f8176 100644 --- a/lib/cli/src/commands/run/mod.rs +++ b/lib/cli/src/commands/run/mod.rs @@ -29,7 +29,7 @@ use wasmer::{ use wasmer_compiler::ArtifactBuild; use wasmer_registry::{wasmer_env::WasmerEnv, Package}; #[cfg(feature = "journal")] -use wasmer_wasix::journaling::{LogFileJournal, SnapshotTrigger}; +use wasmer_wasix::journal::{LogFileJournal, SnapshotTrigger}; use wasmer_wasix::{ bin_factory::BinaryPackage, runners::{MappedCommand, MappedDirectory, Runner}, diff --git a/lib/cli/src/commands/run/wasi.rs b/lib/cli/src/commands/run/wasi.rs index b04a71553cf..fd1c2a46052 100644 --- a/lib/cli/src/commands/run/wasi.rs +++ b/lib/cli/src/commands/run/wasi.rs @@ -15,7 +15,7 @@ use virtual_fs::{DeviceFile, FileSystem, PassthruFileSystem, RootFileSystemBuild use wasmer::{Engine, Function, Instance, Memory32, Memory64, Module, RuntimeError, Store, Value}; use wasmer_registry::wasmer_env::WasmerEnv; #[cfg(feature = "journal")] -use wasmer_wasix::journaling::{LogFileJournal, SnapshotTrigger}; +use wasmer_wasix::journal::{LogFileJournal, SnapshotTrigger}; use wasmer_wasix::{ bin_factory::BinaryPackage, capabilities::Capabilities, diff --git a/lib/virtual-fs/Cargo.toml b/lib/virtual-fs/Cargo.toml index c22527bb988..83614423ca4 100644 --- a/lib/virtual-fs/Cargo.toml +++ b/lib/virtual-fs/Cargo.toml @@ -23,6 +23,7 @@ async-trait = { version = "^0.1" } lazy_static = "1.4" fs_extra = { version = "1.2.0", optional = true } filetime = { version = "0.2.18", optional = true } +serde = { version = "1.0", default-features = false, features = ["derive"], optional = true } bytes = "1" tokio = { version = "1", features = ["io-util", "sync", "macros"], default_features = false } pin-project-lite = "0.2.9" @@ -45,7 +46,7 @@ default = ["host-fs", "webc-fs", "static-fs"] host-fs = ["libc", "fs_extra", "filetime", "tokio/fs", "tokio/io-std", "tokio/rt"] webc-fs = ["webc", "anyhow"] static-fs = ["webc", "anyhow"] -enable-serde = ["typetag"] +enable-serde = ["typetag", "serde"] no-time = [] # Enables memory tracking/limiting functionality for the in-memory filesystem. tracking = [] diff --git a/lib/virtual-fs/src/host_fs.rs b/lib/virtual-fs/src/host_fs.rs index da7b845eb1b..5ccd9b5f9ee 100644 --- a/lib/virtual-fs/src/host_fs.rs +++ b/lib/virtual-fs/src/host_fs.rs @@ -20,16 +20,25 @@ use tokio::runtime::Handle; #[derive(Debug, Clone)] #[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] -pub struct FileSystem(Handle); +pub struct FileSystem { + #[cfg_attr(feature = "enable-serde", serde(skip, default = "default_handle"))] + handle: Handle, +} +#[allow(dead_code)] +fn default_handle() -> Handle { + Handle::current() +} impl Default for FileSystem { fn default() -> Self { - Self(Handle::current()) + Self { + handle: Handle::current(), + } } } impl FileSystem { pub fn new(handle: Handle) -> Self { - FileSystem(handle) + FileSystem { handle } } } @@ -225,7 +234,7 @@ impl crate::FileOpener for FileSystem { .map_err(Into::into) .map(|file| { Box::new(File::new( - self.0.clone(), + self.handle.clone(), file, path.to_owned(), read, @@ -240,9 +249,11 @@ impl crate::FileOpener for FileSystem { #[derive(Debug)] #[cfg_attr(feature = "enable-serde", derive(Serialize))] pub struct File { + #[cfg_attr(feature = "enable-serde", serde(skip, default = "default_handle"))] handle: Handle, #[cfg_attr(feature = "enable-serde", serde(skip_serializing))] inner_std: fs::File, + #[cfg_attr(feature = "enable-serde", serde(skip))] inner: tfs::File, pub host_path: PathBuf, #[cfg(feature = "enable-serde")] @@ -288,7 +299,9 @@ impl<'de> Deserialize<'de> for File { .open(&host_path) .map_err(|_| de::Error::custom("Could not open file on this system"))?; Ok(File { - inner, + handle: Handle::current(), + inner: tokio::fs::File::from_std(inner.try_clone().unwrap()), + inner_std: inner, host_path, flags, }) @@ -325,7 +338,9 @@ impl<'de> Deserialize<'de> for File { .open(&host_path) .map_err(|_| de::Error::custom("Could not open file on this system"))?; Ok(File { - inner, + handle: Handle::current(), + inner: tokio::fs::File::from_std(inner.try_clone().unwrap()), + inner_std: inner, host_path, flags, }) @@ -517,10 +532,15 @@ impl AsyncSeek for File { #[derive(Debug)] #[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] pub struct Stdout { + #[cfg_attr(feature = "enable-serde", serde(skip, default = "default_handle"))] handle: Handle, + #[cfg_attr(feature = "enable-serde", serde(skip, default = "default_stdout"))] inner: tokio::io::Stdout, } - +#[allow(dead_code)] +fn default_stdout() -> tokio::io::Stdout { + tokio::io::stdout() +} impl Default for Stdout { fn default() -> Self { Self { @@ -647,9 +667,15 @@ impl AsyncSeek for Stdout { #[derive(Debug)] #[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] pub struct Stderr { + #[cfg_attr(feature = "enable-serde", serde(skip, default = "default_handle"))] handle: Handle, + #[cfg_attr(feature = "enable-serde", serde(skip, default = "default_stderr"))] inner: tokio::io::Stderr, } +#[allow(dead_code)] +fn default_stderr() -> tokio::io::Stderr { + tokio::io::stderr() +} impl Default for Stderr { fn default() -> Self { Self { @@ -768,9 +794,15 @@ impl VirtualFile for Stderr { #[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] pub struct Stdin { read_buffer: Arc>>, + #[cfg_attr(feature = "enable-serde", serde(skip, default = "default_handle"))] handle: Handle, + #[cfg_attr(feature = "enable-serde", serde(skip, default = "default_stdin"))] inner: tokio::io::Stdin, } +#[allow(dead_code)] +fn default_stdin() -> tokio::io::Stdin { + tokio::io::stdin() +} impl Default for Stdin { fn default() -> Self { Self { diff --git a/lib/wasi-types/src/wasi/bindings.rs b/lib/wasi-types/src/wasi/bindings.rs index acb2ca6c4d2..8b07961d637 100644 --- a/lib/wasi-types/src/wasi/bindings.rs +++ b/lib/wasi-types/src/wasi/bindings.rs @@ -35,6 +35,7 @@ pub type Pid = u32; #[doc = " Identifiers for clocks, snapshot0 version."] #[repr(u32)] #[derive(Clone, Copy, PartialEq, Eq, num_enum :: TryFromPrimitive, Hash)] +#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] pub enum Snapshot0Clockid { #[doc = " The clock measuring real time. Time value zero corresponds with"] #[doc = " 1970-01-01T00:00:00Z."] @@ -448,6 +449,7 @@ impl core::fmt::Display for Errno { impl std::error::Error for Errno {} wai_bindgen_rust::bitflags::bitflags! { #[doc = " File descriptor rights, determining which actions may be performed."] + #[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] pub struct Rights : u64 { #[doc = " The right to invoke `fd_datasync`."] #[doc = " "] @@ -673,6 +675,7 @@ impl core::fmt::Debug for Advice { } wai_bindgen_rust::bitflags::bitflags! { #[doc = " File descriptor flags."] + #[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] pub struct Fdflags : u16 { #[doc = " Append mode: Data written to the file is always appended to the file's end."] const APPEND = 1 << 0; @@ -723,6 +726,7 @@ wai_bindgen_rust::bitflags::bitflags! { #[doc = " Which file time attributes to adjust."] #[doc = " TODO: wit appears to not have support for flags repr"] #[doc = " (@witx repr u16)"] + #[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] pub struct Fstflags : u16 { #[doc = " Adjust the last data access timestamp to the value stored in `filestat::atim`."] const SET_ATIM = 1 << 0; @@ -761,6 +765,7 @@ wai_bindgen_rust::bitflags::bitflags! { #[doc = " Open flags used by `path_open`."] #[doc = " TODO: wit appears to not have support for flags repr"] #[doc = " (@witx repr u16)"] + #[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] pub struct Oflags : u16 { #[doc = " Create file if it does not exist."] const CREATE = 1 << 0; @@ -953,6 +958,7 @@ impl core::fmt::Debug for SubscriptionFsReadwrite { } #[repr(u8)] #[derive(Clone, Copy, PartialEq, Eq)] +#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] pub enum Socktype { Unknown, Stream, @@ -993,6 +999,7 @@ impl core::fmt::Debug for Sockstatus { } #[repr(u8)] #[derive(Clone, Copy, PartialEq, Eq)] +#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] pub enum Sockoption { Noop, ReusePort, @@ -1083,6 +1090,7 @@ impl core::fmt::Debug for Streamsecurity { } #[repr(u8)] #[derive(Clone, Copy, PartialEq, Eq)] +#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] pub enum Addressfamily { Unspec, Inet4, @@ -1171,6 +1179,7 @@ impl core::fmt::Debug for Snapshot0Whence { } #[repr(u8)] #[derive(Clone, Copy, PartialEq, Eq)] +#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] pub enum Whence { Set, Cur, @@ -1352,6 +1361,7 @@ impl core::fmt::Debug for StdioMode { } #[repr(u16)] #[derive(Clone, Copy, PartialEq, Eq, IntoPrimitive, TryFromPrimitive)] +#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] pub enum SockProto { Ip, Icmp, diff --git a/lib/wasix/Cargo.toml b/lib/wasix/Cargo.toml index f0751dac2e9..ff0110ad0a6 100644 --- a/lib/wasix/Cargo.toml +++ b/lib/wasix/Cargo.toml @@ -80,6 +80,7 @@ rkyv = { version = "0.7.40", features = ["indexmap", "validation", "strict"] } bytecheck = "0.6.8" shared-buffer = "0.1" petgraph = "0.6.3" +base64 = "0.21" rayon = { version = "1.7.0", optional = true } wasm-bindgen = { version = "0.2.87", optional = true } js-sys = { version = "0.3.64", optional = true } diff --git a/lib/wasix/src/journal/base64.rs b/lib/wasix/src/journal/base64.rs new file mode 100644 index 00000000000..c6ade6192ef --- /dev/null +++ b/lib/wasix/src/journal/base64.rs @@ -0,0 +1,18 @@ +use std::borrow::Cow; + +use serde::{Deserialize, Serialize}; +use serde::{Deserializer, Serializer}; + +pub fn serialize(v: &Cow<'_, [u8]>, s: S) -> Result { + #[allow(deprecated)] + let base64 = base64::encode(v); + String::serialize(&base64, s) +} + +pub fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result, D::Error> { + let base64 = String::deserialize(d)?; + #[allow(deprecated)] + base64::decode(base64.as_bytes()) + .map_err(|e| serde::de::Error::custom(e)) + .map(|d| d.into()) +} diff --git a/lib/wasix/src/journaling/concrete/archived_journal.rs b/lib/wasix/src/journal/concrete/archived.rs similarity index 87% rename from lib/wasix/src/journaling/concrete/archived_journal.rs rename to lib/wasix/src/journal/concrete/archived.rs index f94580881c3..31689ae41ef 100644 --- a/lib/wasix/src/journaling/concrete/archived_journal.rs +++ b/lib/wasix/src/journal/concrete/archived.rs @@ -1,17 +1,15 @@ use num_enum::{IntoPrimitive, TryFromPrimitive}; use rkyv::ser::{ScratchSpace, Serializer}; use rkyv::{Archive, CheckBytes, Deserialize as RkyvDeserialize, Serialize as RkyvSerialize}; -use serde; use std::borrow::Cow; -use std::{net::Shutdown, time::SystemTime}; +use std::time::SystemTime; use virtual_net::{Duration, IpAddr, IpCidr, Ipv4Addr, Ipv6Addr, SocketAddr, StreamSecurity}; use wasmer_wasix_types::wasi::{self, EpollEventCtl, EpollType, Fdflags, Rights, Sockoption}; -use crate::net::socket::TimeType; - use super::*; pub const JOURNAL_MAGIC_NUMBER: u64 = 0x310d6dd027362979; +pub const JOURNAL_MAGIC_NUMBER_BYTES: [u8; 8] = JOURNAL_MAGIC_NUMBER.to_be_bytes(); #[repr(u16)] #[derive( @@ -22,8 +20,6 @@ pub const JOURNAL_MAGIC_NUMBER: u64 = 0x310d6dd027362979; Eq, IntoPrimitive, TryFromPrimitive, - serde::Serialize, - serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, @@ -424,6 +420,14 @@ impl<'a> JournalEntry<'a> { where T::Error: std::fmt::Display, { + let padding = |size: usize| { + let padding = size % 16; + let padding = match padding { + 0 => 0, + a => 16 - a, + }; + vec![0u8; padding] + }; match self { JournalEntry::InitModule { wasm_hash } => { serializer.serialize_value(&JournalEntryInitModuleV1 { wasm_hash }) @@ -432,12 +436,14 @@ impl<'a> JournalEntry<'a> { serializer.serialize_value(&JournalEntryUpdateMemoryRegionV1 { start: region.start, end: region.end, + _padding: padding(data.len()), data: data.into_owned(), }) } JournalEntry::ProcessExit { exit_code } => { serializer.serialize_value(&JournalEntryProcessExitV1 { exit_code: exit_code.map(|e| e.into()), + _padding: 0, }) } JournalEntry::SetThread { @@ -448,6 +454,7 @@ impl<'a> JournalEntry<'a> { is_64bit, } => serializer.serialize_value(&JournalEntrySetThreadV1 { id: id.into(), + _padding: padding(call_stack.len() + memory_stack.len() + store_data.len()), call_stack: call_stack.into_owned(), memory_stack: memory_stack.into_owned(), store_data: store_data.into_owned(), @@ -474,6 +481,7 @@ impl<'a> JournalEntry<'a> { } => serializer.serialize_value(&JournalEntryFileDescriptorWriteV1 { fd, offset, + _padding: padding(data.len()), data: data.into_owned(), is_64bit, }), @@ -484,7 +492,7 @@ impl<'a> JournalEntry<'a> { }) } JournalEntry::CloseFileDescriptor { fd } => { - serializer.serialize_value(&JournalEntryCloseFileDescriptorV1 { fd }) + serializer.serialize_value(&JournalEntryCloseFileDescriptorV1 { fd, _padding: 0 }) } JournalEntry::OpenFileDescriptor { fd, @@ -499,6 +507,7 @@ impl<'a> JournalEntry<'a> { fd, dirfd, dirflags, + _padding: padding(path.as_bytes().len()), path: path.into_owned(), o_flags: o_flags.bits(), fs_rights_base: fs_rights_base.bits(), @@ -518,12 +527,14 @@ impl<'a> JournalEntry<'a> { JournalEntry::CreateDirectory { fd, path } => { serializer.serialize_value(&JournalEntryCreateDirectoryV1 { fd, + _padding: padding(path.as_bytes().len()), path: path.into_owned(), }) } JournalEntry::RemoveDirectory { fd, path } => { serializer.serialize_value(&JournalEntryRemoveDirectoryV1 { fd, + _padding: padding(path.as_bytes().len()), path: path.into_owned(), }) } @@ -537,6 +548,7 @@ impl<'a> JournalEntry<'a> { } => serializer.serialize_value(&JournalEntryPathSetTimesV1 { fd, flags, + _padding: padding(path.as_bytes().len()), path: path.into_owned(), st_atim, st_mtim, @@ -592,6 +604,7 @@ impl<'a> JournalEntry<'a> { new_path, } => serializer.serialize_value(&JournalEntryCreateHardLinkV1 { old_fd, + _padding: padding(old_path.as_bytes().len() + new_path.as_bytes().len()), old_path: old_path.into_owned(), old_flags, new_fd, @@ -602,6 +615,7 @@ impl<'a> JournalEntry<'a> { fd, new_path, } => serializer.serialize_value(&JournalEntryCreateSymbolicLinkV1 { + _padding: padding(old_path.as_bytes().len() + new_path.as_bytes().len()), old_path: old_path.into_owned(), fd, new_path: new_path.into_owned(), @@ -609,6 +623,7 @@ impl<'a> JournalEntry<'a> { JournalEntry::UnlinkFile { fd, path } => { serializer.serialize_value(&JournalEntryUnlinkFileV1 { fd, + _padding: padding(path.as_bytes().len()), path: path.into_owned(), }) } @@ -619,17 +634,19 @@ impl<'a> JournalEntry<'a> { new_path, } => serializer.serialize_value(&JournalEntryPathRenameV1 { old_fd, + _padding: padding(old_path.as_bytes().len() + new_path.as_bytes().len()), old_path: old_path.into_owned(), new_fd, new_path: new_path.into_owned(), }), JournalEntry::ChangeDirectory { path } => { serializer.serialize_value(&JournalEntryChangeDirectoryV1 { + _padding: padding(path.as_bytes().len()), path: path.into_owned(), }) } JournalEntry::EpollCreate { fd } => { - serializer.serialize_value(&JournalEntryEpollCreateV1 { fd }) + serializer.serialize_value(&JournalEntryEpollCreateV1 { fd, _padding: 0 }) } JournalEntry::EpollCtl { epfd, @@ -648,12 +665,12 @@ impl<'a> JournalEntry<'a> { rows: tty.rows, width: tty.width, height: tty.height, - stdin_tty: tty.stdin_tty, - stdout_tty: tty.stdout_tty, - stderr_tty: tty.stderr_tty, - echo: tty.echo, - line_buffered: tty.line_buffered, - line_feeds, + stdin_tty: tty.stdin_tty.into(), + stdout_tty: tty.stdout_tty.into(), + stderr_tty: tty.stderr_tty.into(), + echo: tty.echo.into(), + line_buffered: tty.line_buffered.into(), + line_feeds: line_feeds.into(), }) } JournalEntry::CreatePipe { fd1, fd2 } => { @@ -680,6 +697,7 @@ impl<'a> JournalEntry<'a> { token, security, } => serializer.serialize_value(&JournalEntryPortBridgeV1 { + _padding: padding(network.as_bytes().len() + token.as_bytes().len()), network: network.into_owned(), token: token.into_owned(), security: security.into(), @@ -789,6 +807,7 @@ impl<'a> JournalEntry<'a> { is_64bit, } => serializer.serialize_value(&JournalEntrySocketSendToV1 { fd, + _padding: padding(data.len()), data: data.into_owned(), flags, addr, @@ -801,6 +820,7 @@ impl<'a> JournalEntry<'a> { is_64bit, } => serializer.serialize_value(&JournalEntrySocketSendV1 { fd, + _padding: padding(data.len()), data: data.into_owned(), flags, is_64bit, @@ -852,9 +872,7 @@ impl<'a> JournalEntry<'a> { /// Note: This structure is versioned which allows for /// changes to the journal entry types without having to /// worry about backward and forward compatibility -#[derive( - Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, -)] +#[derive(Debug, Clone, RkyvSerialize, RkyvDeserialize, Archive)] #[archive_attr(derive(CheckBytes))] pub(crate) struct JournalEntryHeader { pub record_type: u16, @@ -922,46 +940,48 @@ pub enum ArchivedJournalEntry<'a> { SnapshotV1(&'a ArchivedJournalEntrySnapshotV1), } -#[derive( - Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, -)] +#[repr(C)] +#[repr(align(8))] +#[derive(Debug, Clone, RkyvSerialize, RkyvDeserialize, Archive)] #[archive_attr(derive(CheckBytes))] pub struct JournalEntryInitModuleV1 { pub wasm_hash: [u8; 32], } -#[derive( - Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, -)] +#[repr(C)] +#[repr(align(8))] +#[derive(Debug, Clone, RkyvSerialize, RkyvDeserialize, Archive)] #[archive_attr(derive(CheckBytes))] pub struct JournalEntryProcessExitV1 { pub exit_code: Option, + pub _padding: u32, } -#[derive( - Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, -)] +#[repr(C)] +#[repr(align(8))] +#[derive(Debug, Clone, RkyvSerialize, RkyvDeserialize, Archive)] #[archive_attr(derive(CheckBytes))] pub struct JournalEntrySetThreadV1 { pub id: u32, pub call_stack: Vec, pub memory_stack: Vec, pub store_data: Vec, + pub _padding: Vec, pub is_64bit: bool, } -#[derive( - Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, -)] +#[repr(C)] +#[repr(align(8))] +#[derive(Debug, Clone, RkyvSerialize, RkyvDeserialize, Archive)] #[archive_attr(derive(CheckBytes))] pub struct JournalEntryCloseThreadV1 { pub id: u32, pub exit_code: Option, } -#[derive( - Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, -)] +#[repr(C)] +#[repr(align(8))] +#[derive(Debug, Clone, RkyvSerialize, RkyvDeserialize, Archive)] #[archive_attr(derive(CheckBytes))] pub struct JournalEntryFileDescriptorSeekV1 { pub fd: u32, @@ -969,111 +989,118 @@ pub struct JournalEntryFileDescriptorSeekV1 { pub whence: JournalWhenceV1, } -#[derive( - Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, -)] +#[repr(C)] +#[repr(align(8))] +#[derive(Debug, Clone, RkyvSerialize, RkyvDeserialize, Archive)] #[archive_attr(derive(CheckBytes))] pub struct JournalEntryFileDescriptorWriteV1 { pub fd: u32, pub offset: u64, pub data: Vec, + pub _padding: Vec, pub is_64bit: bool, } -#[derive( - Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, -)] +#[repr(C)] +#[repr(align(8))] +#[derive(Debug, Clone, RkyvSerialize, RkyvDeserialize, Archive)] #[archive_attr(derive(CheckBytes))] pub struct JournalEntryUpdateMemoryRegionV1 { pub start: u64, pub end: u64, pub data: Vec, + pub _padding: Vec, } -#[derive( - Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, -)] +#[repr(C)] +#[repr(align(8))] +#[derive(Debug, Clone, RkyvSerialize, RkyvDeserialize, Archive)] #[archive_attr(derive(CheckBytes))] pub struct JournalEntrySetClockTimeV1 { pub clock_id: JournalSnapshot0ClockidV1, pub time: u64, } -#[derive( - Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, -)] +#[repr(C)] +#[repr(align(8))] +#[derive(Debug, Clone, RkyvSerialize, RkyvDeserialize, Archive)] #[archive_attr(derive(CheckBytes))] pub struct JournalEntryOpenFileDescriptorV1 { pub fd: u32, pub dirfd: u32, pub dirflags: u32, pub path: String, + pub _padding: Vec, pub o_flags: u16, pub fs_rights_base: u64, pub fs_rights_inheriting: u64, pub fs_flags: u16, } -#[derive( - Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, -)] +#[repr(C)] +#[repr(align(8))] +#[derive(Debug, Clone, RkyvSerialize, RkyvDeserialize, Archive)] #[archive_attr(derive(CheckBytes))] pub struct JournalEntryCloseFileDescriptorV1 { pub fd: u32, + pub _padding: u32, } -#[derive( - Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, -)] +#[repr(C)] +#[repr(align(8))] +#[derive(Debug, Clone, RkyvSerialize, RkyvDeserialize, Archive)] #[archive_attr(derive(CheckBytes))] pub struct JournalEntryRenumberFileDescriptorV1 { pub old_fd: u32, pub new_fd: u32, } -#[derive( - Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, -)] +#[repr(C)] +#[repr(align(8))] +#[derive(Debug, Clone, RkyvSerialize, RkyvDeserialize, Archive)] #[archive_attr(derive(CheckBytes))] pub struct JournalEntryDuplicateFileDescriptorV1 { pub original_fd: u32, pub copied_fd: u32, } -#[derive( - Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, -)] +#[repr(C)] +#[repr(align(8))] +#[derive(Debug, Clone, RkyvSerialize, RkyvDeserialize, Archive)] #[archive_attr(derive(CheckBytes))] pub struct JournalEntryCreateDirectoryV1 { pub fd: u32, pub path: String, + pub _padding: Vec, } -#[derive( - Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, -)] +#[repr(C)] +#[repr(align(8))] +#[derive(Debug, Clone, RkyvSerialize, RkyvDeserialize, Archive)] #[archive_attr(derive(CheckBytes))] pub struct JournalEntryRemoveDirectoryV1 { pub fd: u32, pub path: String, + pub _padding: Vec, } -#[derive( - Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, -)] +#[repr(C)] +#[repr(align(8))] +#[derive(Debug, Clone, RkyvSerialize, RkyvDeserialize, Archive)] #[archive_attr(derive(CheckBytes))] pub struct JournalEntryPathSetTimesV1 { pub fd: u32, pub flags: u32, pub path: String, + pub _padding: Vec, pub st_atim: u64, pub st_mtim: u64, pub fst_flags: u16, } -#[derive( - Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, -)] +#[repr(C)] +#[repr(align(8))] +#[derive(Debug, Clone, RkyvSerialize, RkyvDeserialize, Archive)] #[archive_attr(derive(CheckBytes))] pub struct JournalEntryFileDescriptorSetTimesV1 { pub fd: u32, @@ -1082,27 +1109,27 @@ pub struct JournalEntryFileDescriptorSetTimesV1 { pub fst_flags: u16, } -#[derive( - Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, -)] +#[repr(C)] +#[repr(align(8))] +#[derive(Debug, Clone, RkyvSerialize, RkyvDeserialize, Archive)] #[archive_attr(derive(CheckBytes))] pub struct JournalEntryFileDescriptorSetSizeV1 { pub fd: u32, pub st_size: u64, } -#[derive( - Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, -)] +#[repr(C)] +#[repr(align(8))] +#[derive(Debug, Clone, RkyvSerialize, RkyvDeserialize, Archive)] #[archive_attr(derive(CheckBytes))] pub struct JournalEntryFileDescriptorSetFlagsV1 { pub fd: u32, pub flags: u16, } -#[derive( - Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, -)] +#[repr(C)] +#[repr(align(8))] +#[derive(Debug, Clone, RkyvSerialize, RkyvDeserialize, Archive)] #[archive_attr(derive(CheckBytes))] pub struct JournalEntryFileDescriptorSetRightsV1 { pub fd: u32, @@ -1110,9 +1137,9 @@ pub struct JournalEntryFileDescriptorSetRightsV1 { pub fs_rights_inheriting: u64, } -#[derive( - Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, -)] +#[repr(C)] +#[repr(align(8))] +#[derive(Debug, Clone, RkyvSerialize, RkyvDeserialize, Archive)] #[archive_attr(derive(CheckBytes))] pub struct JournalEntryFileDescriptorAdviseV1 { pub fd: u32, @@ -1121,9 +1148,9 @@ pub struct JournalEntryFileDescriptorAdviseV1 { pub advice: JournalAdviceV1, } -#[derive( - Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, -)] +#[repr(C)] +#[repr(align(8))] +#[derive(Debug, Clone, RkyvSerialize, RkyvDeserialize, Archive)] #[archive_attr(derive(CheckBytes))] pub struct JournalEntryFileDescriptorAllocateV1 { pub fd: u32, @@ -1131,9 +1158,9 @@ pub struct JournalEntryFileDescriptorAllocateV1 { pub len: u64, } -#[derive( - Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, -)] +#[repr(C)] +#[repr(align(8))] +#[derive(Debug, Clone, RkyvSerialize, RkyvDeserialize, Archive)] #[archive_attr(derive(CheckBytes))] pub struct JournalEntryCreateHardLinkV1 { pub old_fd: u32, @@ -1141,57 +1168,63 @@ pub struct JournalEntryCreateHardLinkV1 { pub old_flags: u32, pub new_fd: u32, pub new_path: String, + pub _padding: Vec, } -#[derive( - Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, -)] +#[repr(C)] +#[repr(align(8))] +#[derive(Debug, Clone, RkyvSerialize, RkyvDeserialize, Archive)] #[archive_attr(derive(CheckBytes))] pub struct JournalEntryCreateSymbolicLinkV1 { pub old_path: String, pub fd: u32, pub new_path: String, + pub _padding: Vec, } -#[derive( - Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, -)] +#[repr(C)] +#[repr(align(8))] +#[derive(Debug, Clone, RkyvSerialize, RkyvDeserialize, Archive)] #[archive_attr(derive(CheckBytes))] pub struct JournalEntryUnlinkFileV1 { pub fd: u32, pub path: String, + pub _padding: Vec, } -#[derive( - Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, -)] +#[repr(C)] +#[repr(align(8))] +#[derive(Debug, Clone, RkyvSerialize, RkyvDeserialize, Archive)] #[archive_attr(derive(CheckBytes))] pub struct JournalEntryPathRenameV1 { pub old_fd: u32, pub old_path: String, pub new_fd: u32, pub new_path: String, + pub _padding: Vec, } -#[derive( - Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, -)] +#[repr(C)] +#[repr(align(8))] +#[derive(Debug, Clone, RkyvSerialize, RkyvDeserialize, Archive)] #[archive_attr(derive(CheckBytes))] pub struct JournalEntryChangeDirectoryV1 { pub path: String, + pub _padding: Vec, } -#[derive( - Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, -)] +#[repr(C)] +#[repr(align(8))] +#[derive(Debug, Clone, RkyvSerialize, RkyvDeserialize, Archive)] #[archive_attr(derive(CheckBytes))] pub struct JournalEntryEpollCreateV1 { pub fd: u32, + pub _padding: u32, } -#[derive( - Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, -)] +#[repr(C)] +#[repr(align(8))] +#[derive(Debug, Clone, RkyvSerialize, RkyvDeserialize, Archive)] #[archive_attr(derive(CheckBytes))] pub struct JournalEntryEpollCtlV1 { pub epfd: u32, @@ -1200,9 +1233,9 @@ pub struct JournalEntryEpollCtlV1 { pub event: Option, } -#[derive( - Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, -)] +#[repr(C)] +#[repr(align(8))] +#[derive(Debug, Clone, RkyvSerialize, RkyvDeserialize, Archive)] #[archive_attr(derive(CheckBytes))] pub struct JournalEntryTtySetV1 { pub cols: u32, @@ -1217,18 +1250,18 @@ pub struct JournalEntryTtySetV1 { pub line_feeds: bool, } -#[derive( - Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, -)] +#[repr(C)] +#[repr(align(8))] +#[derive(Debug, Clone, RkyvSerialize, RkyvDeserialize, Archive)] #[archive_attr(derive(CheckBytes))] pub struct JournalEntryCreatePipeV1 { pub fd1: u32, pub fd2: u32, } -#[derive( - Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, -)] +#[repr(C)] +#[repr(align(8))] +#[derive(Debug, Clone, RkyvSerialize, RkyvDeserialize, Archive)] #[archive_attr(derive(CheckBytes))] pub struct JournalEntryCreateEventV1 { pub initial_val: u64, @@ -1236,43 +1269,44 @@ pub struct JournalEntryCreateEventV1 { pub fd: u32, } -#[derive( - Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, -)] +#[repr(C)] +#[repr(align(8))] +#[derive(Debug, Clone, RkyvSerialize, RkyvDeserialize, Archive)] #[archive_attr(derive(CheckBytes))] pub struct JournalEntryPortAddAddrV1 { pub cidr: IpCidr, } -#[derive( - Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, -)] +#[repr(C)] +#[repr(align(8))] +#[derive(Debug, Clone, RkyvSerialize, RkyvDeserialize, Archive)] #[archive_attr(derive(CheckBytes))] pub struct JournalEntryPortDelAddrV1 { pub addr: IpAddr, } -#[derive( - Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, -)] +#[repr(C)] +#[repr(align(8))] +#[derive(Debug, Clone, RkyvSerialize, RkyvDeserialize, Archive)] #[archive_attr(derive(CheckBytes))] pub struct JournalEntryPortBridgeV1 { pub network: String, pub token: String, + pub _padding: Vec, pub security: JournalStreamSecurityV1, } -#[derive( - Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, -)] +#[repr(C)] +#[repr(align(8))] +#[derive(Debug, Clone, RkyvSerialize, RkyvDeserialize, Archive)] #[archive_attr(derive(CheckBytes))] pub struct JournalEntryPortGatewaySetV1 { pub ip: IpAddr, } -#[derive( - Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, -)] +#[repr(C)] +#[repr(align(8))] +#[derive(Debug, Clone, RkyvSerialize, RkyvDeserialize, Archive)] #[archive_attr(derive(CheckBytes))] pub struct JournalEntryPortRouteAddV1 { pub cidr: IpCidr, @@ -1281,17 +1315,17 @@ pub struct JournalEntryPortRouteAddV1 { pub expires_at: Option, } -#[derive( - Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, -)] +#[repr(C)] +#[repr(align(8))] +#[derive(Debug, Clone, RkyvSerialize, RkyvDeserialize, Archive)] #[archive_attr(derive(CheckBytes))] pub struct JournalEntryPortRouteDelV1 { pub ip: IpAddr, } -#[derive( - Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, -)] +#[repr(C)] +#[repr(align(8))] +#[derive(Debug, Clone, RkyvSerialize, RkyvDeserialize, Archive)] #[archive_attr(derive(CheckBytes))] pub struct JournalEntrySocketOpenV1 { pub af: JournalAddressfamilyV1, @@ -1300,36 +1334,36 @@ pub struct JournalEntrySocketOpenV1 { pub fd: u32, } -#[derive( - Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, -)] +#[repr(C)] +#[repr(align(8))] +#[derive(Debug, Clone, RkyvSerialize, RkyvDeserialize, Archive)] #[archive_attr(derive(CheckBytes))] pub struct JournalEntrySocketListenV1 { pub fd: u32, pub backlog: u32, } -#[derive( - Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, -)] +#[repr(C)] +#[repr(align(8))] +#[derive(Debug, Clone, RkyvSerialize, RkyvDeserialize, Archive)] #[archive_attr(derive(CheckBytes))] pub struct JournalEntrySocketBindV1 { pub fd: u32, pub addr: SocketAddr, } -#[derive( - Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, -)] +#[repr(C)] +#[repr(align(8))] +#[derive(Debug, Clone, RkyvSerialize, RkyvDeserialize, Archive)] #[archive_attr(derive(CheckBytes))] pub struct JournalEntrySocketConnectedV1 { pub fd: u32, pub addr: SocketAddr, } -#[derive( - Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, -)] +#[repr(C)] +#[repr(align(8))] +#[derive(Debug, Clone, RkyvSerialize, RkyvDeserialize, Archive)] #[archive_attr(derive(CheckBytes))] pub struct JournalEntrySocketAcceptedV1 { pub listen_fd: u32, @@ -1339,9 +1373,9 @@ pub struct JournalEntrySocketAcceptedV1 { pub nonblocking: bool, } -#[derive( - Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, -)] +#[repr(C)] +#[repr(align(8))] +#[derive(Debug, Clone, RkyvSerialize, RkyvDeserialize, Archive)] #[archive_attr(derive(CheckBytes))] pub struct JournalEntrySocketJoinIpv4MulticastV1 { pub fd: u32, @@ -1349,9 +1383,9 @@ pub struct JournalEntrySocketJoinIpv4MulticastV1 { pub iface: Ipv4Addr, } -#[derive( - Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, -)] +#[repr(C)] +#[repr(align(8))] +#[derive(Debug, Clone, RkyvSerialize, RkyvDeserialize, Archive)] #[archive_attr(derive(CheckBytes))] pub struct JournalEntrySocketJoinIpv6MulticastV1 { pub fd: u32, @@ -1359,9 +1393,9 @@ pub struct JournalEntrySocketJoinIpv6MulticastV1 { pub iface: u32, } -#[derive( - Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, -)] +#[repr(C)] +#[repr(align(8))] +#[derive(Debug, Clone, RkyvSerialize, RkyvDeserialize, Archive)] #[archive_attr(derive(CheckBytes))] pub struct JournalEntrySocketLeaveIpv4MulticastV1 { pub fd: u32, @@ -1369,9 +1403,9 @@ pub struct JournalEntrySocketLeaveIpv4MulticastV1 { pub iface: Ipv4Addr, } -#[derive( - Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, -)] +#[repr(C)] +#[repr(align(8))] +#[derive(Debug, Clone, RkyvSerialize, RkyvDeserialize, Archive)] #[archive_attr(derive(CheckBytes))] pub struct JournalEntrySocketLeaveIpv6MulticastV1 { pub fd: u32, @@ -1379,9 +1413,9 @@ pub struct JournalEntrySocketLeaveIpv6MulticastV1 { pub iface: u32, } -#[derive( - Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, -)] +#[repr(C)] +#[repr(align(8))] +#[derive(Debug, Clone, RkyvSerialize, RkyvDeserialize, Archive)] #[archive_attr(derive(CheckBytes))] pub struct JournalEntrySocketSendFileV1 { pub socket_fd: u32, @@ -1390,32 +1424,34 @@ pub struct JournalEntrySocketSendFileV1 { pub count: u64, } -#[derive( - Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, -)] +#[repr(C)] +#[repr(align(8))] +#[derive(Debug, Clone, RkyvSerialize, RkyvDeserialize, Archive)] #[archive_attr(derive(CheckBytes))] pub struct JournalEntrySocketSendToV1 { pub fd: u32, pub data: Vec, + pub _padding: Vec, pub flags: u16, pub addr: SocketAddr, pub is_64bit: bool, } -#[derive( - Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, -)] +#[repr(C)] +#[repr(align(8))] +#[derive(Debug, Clone, RkyvSerialize, RkyvDeserialize, Archive)] #[archive_attr(derive(CheckBytes))] pub struct JournalEntrySocketSendV1 { pub fd: u32, pub data: Vec, + pub _padding: Vec, pub flags: u16, pub is_64bit: bool, } -#[derive( - Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, -)] +#[repr(C)] +#[repr(align(8))] +#[derive(Debug, Clone, RkyvSerialize, RkyvDeserialize, Archive)] #[archive_attr(derive(CheckBytes))] pub struct JournalEntrySocketSetOptFlagV1 { pub fd: u32, @@ -1423,9 +1459,9 @@ pub struct JournalEntrySocketSetOptFlagV1 { pub flag: bool, } -#[derive( - Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, -)] +#[repr(C)] +#[repr(align(8))] +#[derive(Debug, Clone, RkyvSerialize, RkyvDeserialize, Archive)] #[archive_attr(derive(CheckBytes))] pub struct JournalEntrySocketSetOptSizeV1 { pub fd: u32, @@ -1433,9 +1469,9 @@ pub struct JournalEntrySocketSetOptSizeV1 { pub size: u64, } -#[derive( - Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, -)] +#[repr(C)] +#[repr(align(8))] +#[derive(Debug, Clone, RkyvSerialize, RkyvDeserialize, Archive)] #[archive_attr(derive(CheckBytes))] pub struct JournalEntrySocketSetOptTimeV1 { pub fd: u32, @@ -1443,27 +1479,26 @@ pub struct JournalEntrySocketSetOptTimeV1 { pub time: Option, } -#[derive( - Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, -)] +#[repr(C)] +#[repr(align(8))] +#[derive(Debug, Clone, RkyvSerialize, RkyvDeserialize, Archive)] #[archive_attr(derive(CheckBytes))] pub struct JournalEntrySocketShutdownV1 { pub fd: u32, pub how: JournalSocketShutdownV1, } -#[derive( - Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, -)] +#[repr(C)] +#[repr(align(8))] +#[derive(Debug, Clone, RkyvSerialize, RkyvDeserialize, Archive)] #[archive_attr(derive(CheckBytes))] pub struct JournalEntrySnapshotV1 { pub since_epoch: Duration, pub trigger: JournalSnapshotTriggerV1, } -#[derive( - Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, -)] +#[repr(C)] +#[derive(Debug, Clone, RkyvSerialize, RkyvDeserialize, Archive)] #[archive_attr(derive(CheckBytes))] pub enum JournalSnapshot0ClockidV1 { Realtime, @@ -1513,9 +1548,8 @@ impl From<&'_ ArchivedJournalSnapshot0ClockidV1> for wasi::Snapshot0Clockid { } } -#[derive( - Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, -)] +#[repr(C)] +#[derive(Debug, Clone, RkyvSerialize, RkyvDeserialize, Archive)] #[archive_attr(derive(CheckBytes))] pub enum JournalWhenceV1 { Set, @@ -1557,9 +1591,8 @@ impl From<&'_ ArchivedJournalWhenceV1> for wasi::Whence { } } -#[derive( - Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, -)] +#[repr(C)] +#[derive(Debug, Clone, RkyvSerialize, RkyvDeserialize, Archive)] #[archive_attr(derive(CheckBytes))] pub enum JournalAdviceV1 { Normal, @@ -1613,9 +1646,8 @@ impl From<&'_ ArchivedJournalAdviceV1> for wasi::Advice { } } -#[derive( - Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, -)] +#[repr(C)] +#[derive(Debug, Clone, RkyvSerialize, RkyvDeserialize, Archive)] #[archive_attr(derive(CheckBytes))] pub enum JournalExitCodeV1 { Errno(u16), @@ -1653,6 +1685,7 @@ impl From<&'_ ArchivedJournalExitCodeV1> for wasi::ExitCode { } } +#[repr(C)] #[derive( Debug, Clone, @@ -1662,8 +1695,6 @@ impl From<&'_ ArchivedJournalExitCodeV1> for wasi::ExitCode { PartialOrd, Ord, Hash, - serde::Serialize, - serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, @@ -1735,6 +1766,7 @@ impl From<&'_ ArchivedJournalSnapshotTriggerV1> for SnapshotTrigger { } } +#[repr(C)] #[derive( Debug, Clone, @@ -1744,8 +1776,6 @@ impl From<&'_ ArchivedJournalSnapshotTriggerV1> for SnapshotTrigger { PartialOrd, Ord, Hash, - serde::Serialize, - serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, @@ -1791,9 +1821,9 @@ impl From<&'_ ArchivedJournalEpollCtlV1> for wasi::EpollCtl { } } -#[derive( - Debug, Clone, serde::Serialize, serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, -)] +#[repr(C)] +#[repr(align(8))] +#[derive(Debug, Clone, RkyvSerialize, RkyvDeserialize, Archive)] #[archive_attr(derive(CheckBytes))] pub struct JournalEpollEventCtlV1 { pub events: u32, @@ -1839,6 +1869,7 @@ impl From<&'_ ArchivedJournalEpollEventCtlV1> for EpollEventCtl { } } +#[repr(C)] #[derive( Debug, Clone, @@ -1848,8 +1879,6 @@ impl From<&'_ ArchivedJournalEpollEventCtlV1> for EpollEventCtl { PartialOrd, Ord, Hash, - serde::Serialize, - serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, @@ -1898,6 +1927,7 @@ impl From<&'_ ArchivedJournalStreamSecurityV1> for StreamSecurity { } } +#[repr(C)] #[derive( Debug, Clone, @@ -1907,8 +1937,6 @@ impl From<&'_ ArchivedJournalStreamSecurityV1> for StreamSecurity { PartialOrd, Ord, Hash, - serde::Serialize, - serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, @@ -1954,6 +1982,7 @@ impl From<&'_ ArchivedJournalAddressfamilyV1> for wasi::Addressfamily { } } +#[repr(C)] #[derive( Debug, Clone, @@ -1963,8 +1992,6 @@ impl From<&'_ ArchivedJournalAddressfamilyV1> for wasi::Addressfamily { PartialOrd, Ord, Hash, - serde::Serialize, - serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, @@ -2014,6 +2041,7 @@ impl From<&'_ ArchivedJournalSocktypeV1> for wasi::Socktype { } } +#[repr(C)] #[derive( Debug, Clone, @@ -2023,8 +2051,6 @@ impl From<&'_ ArchivedJournalSocktypeV1> for wasi::Socktype { PartialOrd, Ord, Hash, - serde::Serialize, - serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, @@ -2162,6 +2188,7 @@ impl From<&'_ ArchivedJournalSockoptionV1> for wasi::Sockoption { } } +#[repr(C)] #[derive( Debug, Clone, @@ -2171,8 +2198,6 @@ impl From<&'_ ArchivedJournalSockoptionV1> for wasi::Sockoption { PartialOrd, Ord, Hash, - serde::Serialize, - serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, @@ -2187,45 +2212,46 @@ pub enum JournalTimeTypeV1 { Linger, } -impl From for JournalTimeTypeV1 { - fn from(val: TimeType) -> Self { +impl From for JournalTimeTypeV1 { + fn from(val: SocketOptTimeType) -> Self { match val { - TimeType::ReadTimeout => JournalTimeTypeV1::ReadTimeout, - TimeType::WriteTimeout => JournalTimeTypeV1::WriteTimeout, - TimeType::AcceptTimeout => JournalTimeTypeV1::AcceptTimeout, - TimeType::ConnectTimeout => JournalTimeTypeV1::ConnectTimeout, - TimeType::BindTimeout => JournalTimeTypeV1::BindTimeout, - TimeType::Linger => JournalTimeTypeV1::Linger, + SocketOptTimeType::ReadTimeout => JournalTimeTypeV1::ReadTimeout, + SocketOptTimeType::WriteTimeout => JournalTimeTypeV1::WriteTimeout, + SocketOptTimeType::AcceptTimeout => JournalTimeTypeV1::AcceptTimeout, + SocketOptTimeType::ConnectTimeout => JournalTimeTypeV1::ConnectTimeout, + SocketOptTimeType::BindTimeout => JournalTimeTypeV1::BindTimeout, + SocketOptTimeType::Linger => JournalTimeTypeV1::Linger, } } } -impl From for TimeType { +impl From for SocketOptTimeType { fn from(val: JournalTimeTypeV1) -> Self { match val { - JournalTimeTypeV1::ReadTimeout => TimeType::ReadTimeout, - JournalTimeTypeV1::WriteTimeout => TimeType::WriteTimeout, - JournalTimeTypeV1::AcceptTimeout => TimeType::AcceptTimeout, - JournalTimeTypeV1::ConnectTimeout => TimeType::ConnectTimeout, - JournalTimeTypeV1::BindTimeout => TimeType::BindTimeout, - JournalTimeTypeV1::Linger => TimeType::Linger, + JournalTimeTypeV1::ReadTimeout => SocketOptTimeType::ReadTimeout, + JournalTimeTypeV1::WriteTimeout => SocketOptTimeType::WriteTimeout, + JournalTimeTypeV1::AcceptTimeout => SocketOptTimeType::AcceptTimeout, + JournalTimeTypeV1::ConnectTimeout => SocketOptTimeType::ConnectTimeout, + JournalTimeTypeV1::BindTimeout => SocketOptTimeType::BindTimeout, + JournalTimeTypeV1::Linger => SocketOptTimeType::Linger, } } } -impl From<&'_ ArchivedJournalTimeTypeV1> for TimeType { +impl From<&'_ ArchivedJournalTimeTypeV1> for SocketOptTimeType { fn from(val: &'_ ArchivedJournalTimeTypeV1) -> Self { match val { - ArchivedJournalTimeTypeV1::ReadTimeout => TimeType::ReadTimeout, - ArchivedJournalTimeTypeV1::WriteTimeout => TimeType::WriteTimeout, - ArchivedJournalTimeTypeV1::AcceptTimeout => TimeType::AcceptTimeout, - ArchivedJournalTimeTypeV1::ConnectTimeout => TimeType::ConnectTimeout, - ArchivedJournalTimeTypeV1::BindTimeout => TimeType::BindTimeout, - ArchivedJournalTimeTypeV1::Linger => TimeType::Linger, + ArchivedJournalTimeTypeV1::ReadTimeout => SocketOptTimeType::ReadTimeout, + ArchivedJournalTimeTypeV1::WriteTimeout => SocketOptTimeType::WriteTimeout, + ArchivedJournalTimeTypeV1::AcceptTimeout => SocketOptTimeType::AcceptTimeout, + ArchivedJournalTimeTypeV1::ConnectTimeout => SocketOptTimeType::ConnectTimeout, + ArchivedJournalTimeTypeV1::BindTimeout => SocketOptTimeType::BindTimeout, + ArchivedJournalTimeTypeV1::Linger => SocketOptTimeType::Linger, } } } +#[repr(C)] #[derive( Debug, Clone, @@ -2235,8 +2261,6 @@ impl From<&'_ ArchivedJournalTimeTypeV1> for TimeType { PartialOrd, Ord, Hash, - serde::Serialize, - serde::Deserialize, RkyvSerialize, RkyvDeserialize, Archive, @@ -2248,32 +2272,32 @@ pub enum JournalSocketShutdownV1 { Both, } -impl From for JournalSocketShutdownV1 { - fn from(val: Shutdown) -> Self { +impl From for JournalSocketShutdownV1 { + fn from(val: SocketShutdownHow) -> Self { match val { - Shutdown::Read => JournalSocketShutdownV1::Read, - Shutdown::Write => JournalSocketShutdownV1::Write, - Shutdown::Both => JournalSocketShutdownV1::Both, + SocketShutdownHow::Read => JournalSocketShutdownV1::Read, + SocketShutdownHow::Write => JournalSocketShutdownV1::Write, + SocketShutdownHow::Both => JournalSocketShutdownV1::Both, } } } -impl From for Shutdown { +impl From for SocketShutdownHow { fn from(val: JournalSocketShutdownV1) -> Self { match val { - JournalSocketShutdownV1::Read => Shutdown::Read, - JournalSocketShutdownV1::Write => Shutdown::Write, - JournalSocketShutdownV1::Both => Shutdown::Both, + JournalSocketShutdownV1::Read => SocketShutdownHow::Read, + JournalSocketShutdownV1::Write => SocketShutdownHow::Write, + JournalSocketShutdownV1::Both => SocketShutdownHow::Both, } } } -impl From<&'_ ArchivedJournalSocketShutdownV1> for Shutdown { +impl From<&'_ ArchivedJournalSocketShutdownV1> for SocketShutdownHow { fn from(val: &'_ ArchivedJournalSocketShutdownV1) -> Self { match val { - ArchivedJournalSocketShutdownV1::Read => Shutdown::Read, - ArchivedJournalSocketShutdownV1::Write => Shutdown::Write, - ArchivedJournalSocketShutdownV1::Both => Shutdown::Both, + ArchivedJournalSocketShutdownV1::Read => SocketShutdownHow::Read, + ArchivedJournalSocketShutdownV1::Write => SocketShutdownHow::Write, + ArchivedJournalSocketShutdownV1::Both => SocketShutdownHow::Both, } } } @@ -2287,13 +2311,19 @@ impl<'a> From> for JournalEntry<'a> { } } ArchivedJournalEntry::UpdateMemoryRegionV1( - ArchivedJournalEntryUpdateMemoryRegionV1 { start, end, data }, + ArchivedJournalEntryUpdateMemoryRegionV1 { + start, + end, + data, + _padding: _, + }, ) => Self::UpdateMemoryRegion { region: (*start)..(*end), data: Cow::Borrowed(data.as_ref()), }, ArchivedJournalEntry::ProcessExitV1(ArchivedJournalEntryProcessExitV1 { exit_code, + _padding: _, }) => Self::ProcessExit { exit_code: exit_code.as_ref().map(|code| code.into()), }, @@ -2302,6 +2332,7 @@ impl<'a> From> for JournalEntry<'a> { call_stack, memory_stack, store_data, + _padding: _, is_64bit, }) => Self::SetThread { id: (*id).into(), @@ -2323,6 +2354,7 @@ impl<'a> From> for JournalEntry<'a> { fd, offset, is_64bit, + _padding: _, }, ) => Self::FileDescriptorWrite { data: data.as_ref().into(), @@ -2351,6 +2383,7 @@ impl<'a> From> for JournalEntry<'a> { fs_rights_base, fs_rights_inheriting, fs_flags, + _padding: _, }, ) => Self::OpenFileDescriptor { fd: *fd, @@ -2363,26 +2396,30 @@ impl<'a> From> for JournalEntry<'a> { fs_flags: wasi::Fdflags::from_bits_truncate(*fs_flags), }, ArchivedJournalEntry::CloseFileDescriptorV1( - ArchivedJournalEntryCloseFileDescriptorV1 { fd }, + ArchivedJournalEntryCloseFileDescriptorV1 { fd, _padding: _ }, ) => Self::CloseFileDescriptor { fd: *fd }, ArchivedJournalEntry::RemoveDirectoryV1(ArchivedJournalEntryRemoveDirectoryV1 { fd, path, + _padding: _, }) => Self::RemoveDirectory { fd: *fd, path: path.as_ref().into(), }, - ArchivedJournalEntry::UnlinkFileV1(ArchivedJournalEntryUnlinkFileV1 { fd, path }) => { - Self::UnlinkFile { - fd: *fd, - path: path.as_ref().into(), - } - } + ArchivedJournalEntry::UnlinkFileV1(ArchivedJournalEntryUnlinkFileV1 { + fd, + path, + _padding: _, + }) => Self::UnlinkFile { + fd: *fd, + path: path.as_ref().into(), + }, ArchivedJournalEntry::PathRenameV1(ArchivedJournalEntryPathRenameV1 { old_fd, old_path, new_fd, new_path, + _padding: _, }) => Self::PathRename { old_fd: *old_fd, old_path: old_path.as_ref().into(), @@ -2423,6 +2460,7 @@ impl<'a> From> for JournalEntry<'a> { ArchivedJournalEntry::CreateDirectoryV1(ArchivedJournalEntryCreateDirectoryV1 { fd, path, + _padding: _, }) => Self::CreateDirectory { fd: *fd, path: path.as_ref().into(), @@ -2434,6 +2472,7 @@ impl<'a> From> for JournalEntry<'a> { st_atim, st_mtim, fst_flags, + _padding: _, }) => Self::PathSetTimes { fd: *fd, path: path.as_ref().into(), @@ -2504,6 +2543,7 @@ impl<'a> From> for JournalEntry<'a> { old_flags, new_fd, new_path, + _padding: _, }) => Self::CreateHardLink { old_fd: *old_fd, old_path: old_path.as_ref().into(), @@ -2516,6 +2556,7 @@ impl<'a> From> for JournalEntry<'a> { old_path, fd, new_path, + _padding: _, }, ) => Self::CreateSymbolicLink { old_path: old_path.as_ref().into(), @@ -2524,12 +2565,14 @@ impl<'a> From> for JournalEntry<'a> { }, ArchivedJournalEntry::ChangeDirectoryV1(ArchivedJournalEntryChangeDirectoryV1 { path, + _padding: _, }) => Self::ChangeDirectory { path: path.as_ref().into(), }, - ArchivedJournalEntry::EpollCreateV1(ArchivedJournalEntryEpollCreateV1 { fd }) => { - Self::EpollCreate { fd: *fd } - } + ArchivedJournalEntry::EpollCreateV1(ArchivedJournalEntryEpollCreateV1 { + fd, + _padding: _, + }) => Self::EpollCreate { fd: *fd }, ArchivedJournalEntry::EpollCtlV1(ArchivedJournalEntryEpollCtlV1 { epfd, ref op, @@ -2590,6 +2633,7 @@ impl<'a> From> for JournalEntry<'a> { network, token, ref security, + _padding: _, }) => Self::PortBridge { network: network.as_ref().into(), token: token.as_ref().into(), @@ -2725,6 +2769,7 @@ impl<'a> From> for JournalEntry<'a> { flags, addr, is_64bit, + _padding: _, }) => Self::SocketSendTo { fd: *fd, data: data.as_ref().into(), @@ -2737,6 +2782,7 @@ impl<'a> From> for JournalEntry<'a> { data, flags, is_64bit, + _padding: _, }) => Self::SocketSend { fd: *fd, data: data.as_ref().into(), @@ -3383,7 +3429,7 @@ mod tests { pub fn test_record_socket_set_opt_time() { run_test(JournalEntry::SocketSetOptTime { fd: 0, - ty: TimeType::AcceptTimeout, + ty: SocketOptTimeType::AcceptTimeout, time: Some(Duration::ZERO), }); } @@ -3393,7 +3439,7 @@ mod tests { pub fn test_record_socket_shutdown() { run_test(JournalEntry::SocketShutdown { fd: 123, - how: Shutdown::Both, + how: SocketShutdownHow::Both, }); } @@ -3405,4 +3451,99 @@ mod tests { trigger: SnapshotTrigger::Idle, }); } + + #[tracing_test::traced_test] + #[test] + pub fn test_record_alignment() { + assert_eq!(std::mem::align_of::(), 8); + assert_eq!(std::mem::align_of::(), 8); + assert_eq!(std::mem::align_of::(), 8); + assert_eq!(std::mem::align_of::(), 8); + assert_eq!(std::mem::align_of::(), 8); + assert_eq!(std::mem::align_of::(), 8); + assert_eq!(std::mem::align_of::(), 8); + assert_eq!(std::mem::align_of::(), 8); + assert_eq!(std::mem::align_of::(), 8); + assert_eq!(std::mem::align_of::(), 8); + assert_eq!( + std::mem::align_of::(), + 8 + ); + assert_eq!( + std::mem::align_of::(), + 8 + ); + assert_eq!(std::mem::align_of::(), 8); + assert_eq!(std::mem::align_of::(), 8); + assert_eq!(std::mem::align_of::(), 8); + assert_eq!( + std::mem::align_of::(), + 8 + ); + assert_eq!( + std::mem::align_of::(), + 8 + ); + assert_eq!( + std::mem::align_of::(), + 8 + ); + assert_eq!( + std::mem::align_of::(), + 8 + ); + assert_eq!( + std::mem::align_of::(), + 8 + ); + assert_eq!( + std::mem::align_of::(), + 8 + ); + assert_eq!(std::mem::align_of::(), 8); + assert_eq!(std::mem::align_of::(), 8); + assert_eq!(std::mem::align_of::(), 8); + assert_eq!(std::mem::align_of::(), 8); + assert_eq!(std::mem::align_of::(), 8); + assert_eq!(std::mem::align_of::(), 8); + assert_eq!(std::mem::align_of::(), 8); + assert_eq!(std::mem::align_of::(), 8); + assert_eq!(std::mem::align_of::(), 8); + assert_eq!(std::mem::align_of::(), 8); + assert_eq!(std::mem::align_of::(), 8); + assert_eq!(std::mem::align_of::(), 8); + assert_eq!(std::mem::align_of::(), 8); + assert_eq!(std::mem::align_of::(), 8); + assert_eq!(std::mem::align_of::(), 8); + assert_eq!(std::mem::align_of::(), 8); + assert_eq!(std::mem::align_of::(), 8); + assert_eq!(std::mem::align_of::(), 8); + assert_eq!(std::mem::align_of::(), 8); + assert_eq!(std::mem::align_of::(), 8); + assert_eq!(std::mem::align_of::(), 8); + assert_eq!( + std::mem::align_of::(), + 8 + ); + assert_eq!( + std::mem::align_of::(), + 8 + ); + assert_eq!( + std::mem::align_of::(), + 8 + ); + assert_eq!( + std::mem::align_of::(), + 8 + ); + assert_eq!(std::mem::align_of::(), 8); + assert_eq!(std::mem::align_of::(), 8); + assert_eq!(std::mem::align_of::(), 8); + assert_eq!(std::mem::align_of::(), 8); + assert_eq!(std::mem::align_of::(), 8); + assert_eq!(std::mem::align_of::(), 8); + assert_eq!(std::mem::align_of::(), 8); + assert_eq!(std::mem::align_of::(), 8); + } } diff --git a/lib/wasix/src/journaling/concrete/boxed_journal.rs b/lib/wasix/src/journal/concrete/boxed.rs similarity index 97% rename from lib/wasix/src/journaling/concrete/boxed_journal.rs rename to lib/wasix/src/journal/concrete/boxed.rs index b2d0be4665c..75cb3de356a 100644 --- a/lib/wasix/src/journaling/concrete/boxed_journal.rs +++ b/lib/wasix/src/journal/concrete/boxed.rs @@ -13,7 +13,7 @@ impl ReadableJournal for Box { } impl WritableJournal for Box { - fn write<'a>(&'a self, entry: JournalEntry<'a>) -> anyhow::Result<()> { + fn write<'a>(&'a self, entry: JournalEntry<'a>) -> anyhow::Result { self.deref().write(entry) } } @@ -29,7 +29,7 @@ impl ReadableJournal for Box { } impl WritableJournal for Box { - fn write<'a>(&'a self, entry: JournalEntry<'a>) -> anyhow::Result<()> { + fn write<'a>(&'a self, entry: JournalEntry<'a>) -> anyhow::Result { self.deref().write(entry) } } diff --git a/lib/wasix/src/journal/concrete/buffered.rs b/lib/wasix/src/journal/concrete/buffered.rs new file mode 100644 index 00000000000..76c905c5821 --- /dev/null +++ b/lib/wasix/src/journal/concrete/buffered.rs @@ -0,0 +1,91 @@ +use std::sync::Arc; +use std::sync::Mutex; + +use super::*; + +// The buffered journal will keep all the events in memory until it +// is either reset or dropped. +#[derive(Debug)] +pub struct BufferedJournal { + tx: BufferedJournalTx, + rx: BufferedJournalRx, +} + +#[derive(Debug, Default, Clone)] +struct State { + records: Arc>>>, + offset: usize, +} + +#[derive(Debug)] +pub struct BufferedJournalRx { + state: Arc>, +} + +#[derive(Debug)] +pub struct BufferedJournalTx { + state: Arc>, +} + +impl BufferedJournal { + pub fn new() -> Self { + let state = Arc::new(Mutex::new(State::default())); + Self { + tx: BufferedJournalTx { + state: state.clone(), + }, + rx: BufferedJournalRx { state: state }, + } + } +} + +impl WritableJournal for BufferedJournalTx { + fn write<'a>(&'a self, entry: JournalEntry<'a>) -> anyhow::Result { + let entry = entry.into_owned(); + let state = self.state.lock().unwrap(); + let estimate_size = entry.estimate_size(); + state.records.lock().unwrap().push(entry); + Ok(estimate_size as u64) + } +} + +impl ReadableJournal for BufferedJournalRx { + fn read(&self) -> anyhow::Result>> { + let mut state = self.state.lock().unwrap(); + let ret = state.records.lock().unwrap().get(state.offset).cloned(); + if ret.is_some() { + state.offset += 1; + } + Ok(ret) + } + + fn as_restarted(&self) -> anyhow::Result> { + let mut state = self.state.lock().unwrap().clone(); + state.offset = 0; + Ok(Box::new(BufferedJournalRx { + state: Arc::new(Mutex::new(state)), + })) + } +} + +impl WritableJournal for BufferedJournal { + fn write<'a>(&'a self, entry: JournalEntry<'a>) -> anyhow::Result { + self.tx.write(entry) + } +} + +impl ReadableJournal for BufferedJournal { + fn read(&self) -> anyhow::Result>> { + self.rx.read() + } + + fn as_restarted(&self) -> anyhow::Result> { + self.rx.as_restarted() + } +} + +impl Journal for BufferedJournal { + fn split(self) -> (Box, Box) { + (Box::new(self.tx), Box::new(self.rx)) + } +} diff --git a/lib/wasix/src/journaling/concrete/compactor.rs b/lib/wasix/src/journal/concrete/compacting.rs similarity index 94% rename from lib/wasix/src/journaling/concrete/compactor.rs rename to lib/wasix/src/journal/concrete/compacting.rs index 538008a7c87..935603247de 100644 --- a/lib/wasix/src/journaling/concrete/compactor.rs +++ b/lib/wasix/src/journal/concrete/compacting.rs @@ -33,6 +33,10 @@ impl From> for MemoryRange { struct State { // We maintain a memory map of the events that are significant memory_map: HashMap, + // List of all the snapshots + snapshots: Vec, + // Last tty event thats been set + tty: Option, // Thread events are only maintained while the thread and the // process are still running thread_map: HashMap, @@ -64,6 +68,12 @@ impl State { { let mut filter = FilteredJournal::new(inner) .with_filter_events(self.whitelist.clone().into_iter().collect()); + if let Some(tty) = self.tty.as_ref() { + filter.add_event_to_whitelist(*tty); + } + for e in self.snapshots.iter() { + filter.add_event_to_whitelist(*e); + } for (_, e) in self.memory_map.iter() { filter.add_event_to_whitelist(*e); } @@ -120,6 +130,8 @@ impl CompactingJournal { state: Arc::new(Mutex::new(State { inner_tx: tx, inner_rx: rx.as_restarted()?, + tty: None, + snapshots: Default::default(), memory_map: Default::default(), thread_map: Default::default(), suspect_descriptors: Default::default(), @@ -145,7 +157,7 @@ impl CompactingJournalTx { } /// Compacts the inner journal into a new journal - pub fn compact(&mut self, new_journal: J) -> anyhow::Result<()> + pub fn compact_to(&self, new_journal: J) -> anyhow::Result<()> where J: Journal, { @@ -195,10 +207,17 @@ impl CompactingJournalTx { Ok(()) } + + pub fn replace_inner(&self, inner: J) { + let mut state = self.state.lock().unwrap(); + let (mut tx, mut rx) = inner.split(); + std::mem::swap(&mut state.inner_tx, &mut tx); + std::mem::swap(&mut state.inner_rx, &mut rx); + } } impl WritableJournal for CompactingJournalTx { - fn write<'a>(&'a self, entry: JournalEntry<'a>) -> anyhow::Result<()> { + fn write<'a>(&'a self, entry: JournalEntry<'a>) -> anyhow::Result { let mut state = self.state.lock().unwrap(); let event_index = state.event_index; state.event_index += 1; @@ -217,11 +236,15 @@ impl WritableJournal for CompactingJournalTx { JournalEntry::CloseThread { id, .. } => { state.thread_map.remove(id); } + JournalEntry::Snapshot { .. } => { + state.snapshots.push(event_index); + } JournalEntry::ProcessExit { .. } => { state.thread_map.clear(); state.memory_map.clear(); state.suspect_descriptors.clear(); state.whitelist.insert(event_index); + state.snapshots.clear(); } JournalEntry::CloseFileDescriptor { fd } => { // If its not suspect we need to record this event @@ -234,6 +257,9 @@ impl WritableJournal for CompactingJournalTx { state.whitelist.insert(event_index); } } + JournalEntry::TtySet { .. } => { + state.tty.replace(event_index); + } JournalEntry::OpenFileDescriptor { fd, .. } => { // All file descriptors are opened in a suspect state which // means if they are closed without modifying the file system @@ -310,11 +336,19 @@ impl WritableJournal for CompactingJournalTx { impl CompactingJournal { /// Compacts the inner journal into a new journal - pub fn compact(&mut self, new_journal: J) -> anyhow::Result<()> + pub fn compact_to(&mut self, new_journal: J) -> anyhow::Result<()> where J: Journal, { - self.tx.compact(new_journal) + self.tx.compact_to(new_journal) + } + + pub fn into_split(self) -> (CompactingJournalTx, CompactingJournalRx) { + (self.tx, self.rx) + } + + pub fn replace_inner(&self, inner: J) { + self.tx.replace_inner(inner) } } @@ -329,7 +363,7 @@ impl ReadableJournal for CompactingJournalRx { } impl WritableJournal for CompactingJournal { - fn write<'a>(&'a self, entry: JournalEntry<'a>) -> anyhow::Result<()> { + fn write<'a>(&'a self, entry: JournalEntry<'a>) -> anyhow::Result { self.tx.write(entry) } } @@ -363,18 +397,14 @@ mod tests { out_records: Vec>, ) -> anyhow::Result<()> { // Build a journal that will store the records before compacting - let in_file = tempfile::NamedTempFile::new()?; - let mut compacting_journal = CompactingJournal::new(LogFileJournal::from_file( - in_file.as_file().try_clone().unwrap(), - )?)?; + let mut compacting_journal = CompactingJournal::new(BufferedJournal::new())?; for record in in_records { compacting_journal.write(record)?; } // Now we build a new one using the compactor - let new_file = tempfile::NamedTempFile::new()?; - let new_journal = LogFileJournal::from_file(new_file.as_file().try_clone()?)?; - compacting_journal.compact(new_journal)?; + let new_journal = BufferedJournal::new(); + compacting_journal.compact_to(new_journal)?; // Read the records let new_records = compacting_journal.as_restarted()?; diff --git a/lib/wasix/src/journal/concrete/compacting_log_file.rs b/lib/wasix/src/journal/concrete/compacting_log_file.rs new file mode 100644 index 00000000000..18de2ea0f86 --- /dev/null +++ b/lib/wasix/src/journal/concrete/compacting_log_file.rs @@ -0,0 +1,201 @@ +use std::{ + path::{Path, PathBuf}, + sync::{Arc, Mutex}, +}; + +use super::*; + +#[derive(Debug)] +struct State { + on_n_records: Option, + on_n_size: Option, + on_drop: bool, + cnt_records: u64, + cnt_size: u64, +} + +#[derive(Debug)] +pub struct CompactingLogFileJournal { + tx: CompactingLogFileJournalTx, + rx: CompactingLogFileJournalRx, +} + +#[derive(Debug)] +pub struct CompactingLogFileJournalTx { + state: Arc>, + inner: CompactingJournalTx, + main_path: PathBuf, + temp_path: PathBuf, +} + +#[derive(Debug)] +pub struct CompactingLogFileJournalRx { + #[allow(dead_code)] + state: Arc>, + inner: CompactingJournalRx, +} + +impl CompactingLogFileJournal { + pub fn new(path: impl AsRef) -> anyhow::Result { + // We prepare a compacting journal which does nothing + // with the events other than learn from them + let compacting = CompactingJournal::new(NullJournal::default())?; + + // We first feed all the entries into the compactor so that + // it learns all the records + let log_file = LogFileJournal::new(path.as_ref())?; + copy_journal(&log_file, &compacting)?; + + // Now everything is learned its time to attach the + // log file to the compacting journal + compacting.replace_inner(log_file); + let (tx, rx) = compacting.into_split(); + + let mut temp_filename = path + .as_ref() + .file_name() + .ok_or_else(|| { + anyhow::format_err!( + "The path is not a valid filename - {}", + path.as_ref().to_string_lossy() + ) + })? + .to_string_lossy() + .to_string(); + temp_filename.insert_str(0, ".compacting."); + let temp_path = path.as_ref().clone().with_file_name(&temp_filename); + + let state = Arc::new(Mutex::new(State { + on_drop: false, + on_n_records: None, + on_n_size: None, + cnt_records: 0, + cnt_size: 0, + })); + let tx = CompactingLogFileJournalTx { + state: state.clone(), + inner: tx, + main_path: path.as_ref().to_path_buf(), + temp_path, + }; + let rx = CompactingLogFileJournalRx { state, inner: rx }; + + Ok(Self { tx, rx }) + } + + pub fn compact_now(&mut self) -> anyhow::Result<()> { + self.tx.compact_now() + } + + pub fn with_compact_on_drop(self) -> Self { + self.tx.state.lock().unwrap().on_drop = true; + self + } + + pub fn with_compact_on_n_records(self, n_records: u64) -> Self { + self.tx + .state + .lock() + .unwrap() + .on_n_records + .replace(n_records); + self + } + + pub fn with_compact_on_n_size(self, n_size: u64) -> Self { + self.tx.state.lock().unwrap().on_n_size.replace(n_size); + self + } +} + +impl CompactingLogFileJournalTx { + pub fn compact_now(&self) -> anyhow::Result<()> { + // Reset the counters + self.reset_counters(); + + // Create the staging file and open it + std::fs::remove_file(&self.temp_path).ok(); + let target = LogFileJournal::new(self.temp_path.clone())?; + + // Compact the data into the new target and rename it over the last one + self.inner.compact_to(target)?; + std::fs::rename(&self.temp_path, &self.main_path)?; + Ok(()) + } + + pub fn reset_counters(&self) { + let mut state = self.state.lock().unwrap(); + state.cnt_records = 0; + state.cnt_size = 0; + } +} + +impl Drop for CompactingLogFileJournalTx { + fn drop(&mut self) { + let triggered = self.state.lock().unwrap().on_drop; + if triggered { + if let Err(err) = self.compact_now() { + tracing::error!("failed to compact log - {}", err); + } + } + } +} + +impl ReadableJournal for CompactingLogFileJournalRx { + fn read(&self) -> anyhow::Result>> { + self.inner.read() + } + + fn as_restarted(&self) -> anyhow::Result> { + self.inner.as_restarted() + } +} + +impl WritableJournal for CompactingLogFileJournalTx { + fn write<'a>(&'a self, entry: JournalEntry<'a>) -> anyhow::Result { + let amt = self.inner.write(entry)?; + + let triggered = { + let mut state = self.state.lock().unwrap(); + if amt > 0 { + state.cnt_records += 1; + state.cnt_size += amt; + } + + let mut triggered = false; + if let Some(on) = state.on_n_records.as_ref() { + if state.cnt_records >= *on { + triggered = true; + } + } + if let Some(on) = state.on_n_size.as_ref() { + if state.cnt_size >= *on { + triggered = true; + } + } + triggered + }; + + if triggered { + self.compact_now()?; + } + + Ok(amt) + } +} + +impl ReadableJournal for CompactingLogFileJournal { + fn read(&self) -> anyhow::Result>> { + self.rx.read() + } + + fn as_restarted(&self) -> anyhow::Result> { + self.rx.as_restarted() + } +} + +impl WritableJournal for CompactingLogFileJournal { + fn write<'a>(&'a self, entry: JournalEntry<'a>) -> anyhow::Result { + self.tx.write(entry) + } +} diff --git a/lib/wasix/src/journaling/concrete/filter.rs b/lib/wasix/src/journal/concrete/filter.rs similarity index 95% rename from lib/wasix/src/journaling/concrete/filter.rs rename to lib/wasix/src/journal/concrete/filter.rs index b598a366a9b..59decac6fa4 100644 --- a/lib/wasix/src/journaling/concrete/filter.rs +++ b/lib/wasix/src/journal/concrete/filter.rs @@ -101,17 +101,17 @@ impl FilteredJournal { } } - pub fn into_inner(self) -> CompositeJournal { - CompositeJournal::new(self.tx.inner, self.rx.inner) + pub fn into_inner(self) -> RecombinedJournal { + RecombinedJournal::new(self.tx.inner, self.rx.inner) } } impl WritableJournal for FilteredJournalTx { - fn write<'a>(&'a self, entry: JournalEntry<'a>) -> anyhow::Result<()> { + fn write<'a>(&'a self, entry: JournalEntry<'a>) -> anyhow::Result { let event_index = self.event_index.fetch_add(1, Ordering::SeqCst); if let Some(events) = self.filter_events.as_ref() { if !events.contains(&event_index) { - return Ok(()); + return Ok(0); } } @@ -123,19 +123,19 @@ impl WritableJournal for FilteredJournalTx { | JournalEntry::EpollCtl { .. } | JournalEntry::TtySet { .. } => { if self.filter_core { - return Ok(()); + return Ok(0); } entry } JournalEntry::SetThread { .. } | JournalEntry::CloseThread { .. } => { if self.filter_threads { - return Ok(()); + return Ok(0); } entry } JournalEntry::UpdateMemoryRegion { .. } => { if self.filter_memory { - return Ok(()); + return Ok(0); } entry } @@ -162,13 +162,13 @@ impl WritableJournal for FilteredJournalTx { | JournalEntry::CreatePipe { .. } | JournalEntry::CreateEvent { .. } => { if self.filter_fs { - return Ok(()); + return Ok(0); } entry } JournalEntry::Snapshot { .. } => { if self.filter_snapshots { - return Ok(()); + return Ok(0); } entry } @@ -199,7 +199,7 @@ impl WritableJournal for FilteredJournalTx { | JournalEntry::SocketSetOptTime { .. } | JournalEntry::SocketShutdown { .. } => { if self.filter_net { - return Ok(()); + return Ok(0); } entry } @@ -221,7 +221,7 @@ impl ReadableJournal for FilteredJournalRx { } impl WritableJournal for FilteredJournal { - fn write<'a>(&'a self, entry: JournalEntry<'a>) -> anyhow::Result<()> { + fn write<'a>(&'a self, entry: JournalEntry<'a>) -> anyhow::Result { self.tx.write(entry) } } diff --git a/lib/wasix/src/journaling/concrete/log_file.rs b/lib/wasix/src/journal/concrete/log_file.rs similarity index 78% rename from lib/wasix/src/journaling/concrete/log_file.rs rename to lib/wasix/src/journal/concrete/log_file.rs index 25944fc21ce..dbbeaca25e9 100644 --- a/lib/wasix/src/journaling/concrete/log_file.rs +++ b/lib/wasix/src/journal/concrete/log_file.rs @@ -118,7 +118,7 @@ impl LogFileJournal { } impl WritableJournal for LogFileJournalTx { - fn write<'a>(&'a self, entry: JournalEntry<'a>) -> anyhow::Result<()> { + fn write<'a>(&'a self, entry: JournalEntry<'a>) -> anyhow::Result { tracing::debug!("journal event: {:?}", entry); let mut state = self.state.lock().unwrap(); @@ -127,7 +127,7 @@ impl WritableJournal for LogFileJournalTx { let record_type: JournalEntryRecordType = entry.archive_record_type(); state.file.write_all(&(record_type as u16).to_be_bytes())?; let offset_size = state.file.stream_position()?; - state.file.write_all(&[0u8; 6])?; // record size (48 bits) + state.file.write_all(&[0u8; 6])?; // record and pad size (48 bits) // Now serialize the actual data to the log let offset_start = state.file.stream_position()?; @@ -135,13 +135,23 @@ impl WritableJournal for LogFileJournalTx { let offset_end = state.file.stream_position()?; let record_size = offset_end - offset_start; + // If the alightment is out then fail + if record_size % 8 != 0 { + tracing::error!( + "alignment is out for journal event (type={:?}, record_size={}, alignment={})", + record_type, + record_size, + record_size % 8 + ); + } + // Write the record and then move back to the end again state.file.seek(SeekFrom::Start(offset_size))?; state.file.write_all(&record_size.to_be_bytes()[2..8])?; state.file.seek(SeekFrom::Start(offset_end))?; // Now write the actual data and update the offsets - Ok(()) + Ok(record_size) } } @@ -157,17 +167,30 @@ impl ReadableJournal for LogFileJournalRx { buffer_ptr.advance(*buffer_pos); loop { // Read the headers and advance - let header_size = 8; - if buffer_ptr.len() < header_size { + if buffer_ptr.len() < 8 { return Ok(None); } let header = { let b = buffer_ptr; + + // If the next header is the magic itself then skip it. + // You may be wondering how a magic could appear later + // in the journal itself. This can happen if someone + // concat's multiple journals together to make a combined + // journal + if b[0..8] == JOURNAL_MAGIC_NUMBER_BYTES[0..8] { + buffer_ptr.advance(8); + *buffer_pos += 8; + continue; + } + + // Otherwise we decode the header let header = JournalEntryHeader { record_type: u16::from_be_bytes([b[0], b[1]]), record_size: u64::from_be_bytes([0u8, 0u8, b[2], b[3], b[4], b[5], b[6], b[7]]), }; buffer_ptr.advance(8); + *buffer_pos += 8; header }; @@ -181,10 +204,9 @@ impl ReadableJournal for LogFileJournalRx { return Ok(None); } - // Move the buffer position forward + // Move the buffer position forward past the record let entry = &buffer_ptr[..(header.record_size as usize)]; buffer_ptr.advance(header.record_size as usize); - *buffer_pos += header_size; *buffer_pos += header.record_size as usize; // Now we read the entry @@ -211,7 +233,7 @@ impl ReadableJournal for LogFileJournalRx { } impl WritableJournal for LogFileJournal { - fn write<'a>(&'a self, entry: JournalEntry<'a>) -> anyhow::Result<()> { + fn write<'a>(&'a self, entry: JournalEntry<'a>) -> anyhow::Result { self.tx.write(entry) } } @@ -247,6 +269,15 @@ mod tests { journal .write(JournalEntry::CreatePipe { fd1: 1, fd2: 2 }) .unwrap(); + journal + .write(JournalEntry::SetThread { + id: 1.into(), + call_stack: vec![11; 116].into(), + memory_stack: vec![22; 16].into(), + store_data: vec![33; 136].into(), + is_64bit: false, + }) + .unwrap(); journal.write(JournalEntry::PortAddrClear).unwrap(); drop(journal); @@ -255,11 +286,22 @@ mod tests { let event1 = journal.read().unwrap(); let event2 = journal.read().unwrap(); let event3 = journal.read().unwrap(); + let event4 = journal.read().unwrap(); // Check the events assert_eq!(event1, Some(JournalEntry::CreatePipe { fd1: 1, fd2: 2 })); - assert_eq!(event2, Some(JournalEntry::PortAddrClear)); - assert_eq!(event3, None); + assert_eq!( + event2, + Some(JournalEntry::SetThread { + id: 1.into(), + call_stack: vec![11; 116].into(), + memory_stack: vec![22; 16].into(), + store_data: vec![33; 136].into(), + is_64bit: false, + }) + ); + assert_eq!(event3, Some(JournalEntry::PortAddrClear)); + assert_eq!(event4, None); // Now write another event journal @@ -289,10 +331,21 @@ mod tests { let event2 = journal.read().unwrap(); let event3 = journal.read().unwrap(); let event4 = journal.read().unwrap(); + let event5 = journal.read().unwrap(); assert_eq!(event1, Some(JournalEntry::CreatePipe { fd1: 1, fd2: 2 })); - assert_eq!(event2, Some(JournalEntry::PortAddrClear)); assert_eq!( - event3, + event2, + Some(JournalEntry::SetThread { + id: 1.into(), + call_stack: vec![11; 116].into(), + memory_stack: vec![22; 16].into(), + store_data: vec![33; 136].into(), + is_64bit: false, + }) + ); + assert_eq!(event3, Some(JournalEntry::PortAddrClear)); + assert_eq!( + event4, Some(JournalEntry::SocketSend { fd: 1234, data: [12; 1024].to_vec().into(), @@ -300,7 +353,7 @@ mod tests { is_64bit: true, }) ); - assert_eq!(event4, None); + assert_eq!(event5, None); // Load it again let journal = LogFileJournal::new(file.path()).unwrap(); @@ -310,17 +363,29 @@ mod tests { let event3 = journal.read().unwrap(); let event4 = journal.read().unwrap(); let event5 = journal.read().unwrap(); + let event6 = journal.read().unwrap(); tracing::info!("event1 {:?}", event1); tracing::info!("event2 {:?}", event2); tracing::info!("event3 {:?}", event3); tracing::info!("event4 {:?}", event4); tracing::info!("event5 {:?}", event5); + tracing::info!("event6 {:?}", event6); assert_eq!(event1, Some(JournalEntry::CreatePipe { fd1: 1, fd2: 2 })); - assert_eq!(event2, Some(JournalEntry::PortAddrClear)); assert_eq!( - event3, + event2, + Some(JournalEntry::SetThread { + id: 1.into(), + call_stack: vec![11; 116].into(), + memory_stack: vec![22; 16].into(), + store_data: vec![33; 136].into(), + is_64bit: false, + }) + ); + assert_eq!(event3, Some(JournalEntry::PortAddrClear)); + assert_eq!( + event4, Some(JournalEntry::SocketSend { fd: 1234, data: [12; 1024].to_vec().into(), @@ -329,12 +394,12 @@ mod tests { }) ); assert_eq!( - event4, + event5, Some(JournalEntry::CreatePipe { fd1: 1234, fd2: 5432, }) ); - assert_eq!(event5, None); + assert_eq!(event6, None); } } diff --git a/lib/wasix/src/journal/concrete/mod.rs b/lib/wasix/src/journal/concrete/mod.rs new file mode 100644 index 00000000000..a1a3108afac --- /dev/null +++ b/lib/wasix/src/journal/concrete/mod.rs @@ -0,0 +1,29 @@ +mod archived; +mod boxed; +mod buffered; +mod compacting; +mod compacting_log_file; +mod filter; +#[cfg(feature = "journal")] +mod log_file; +mod null; +mod pipe; +mod printing; +mod recombined; +mod unsupported; + +pub(super) use super::*; + +pub use archived::*; +pub use boxed::*; +pub use buffered::*; +pub use compacting::*; +pub use compacting_log_file::*; +pub use filter::*; +#[cfg(feature = "journal")] +pub use log_file::*; +pub use null::*; +pub use pipe::*; +pub use printing::*; +pub use recombined::*; +pub use unsupported::*; diff --git a/lib/wasix/src/journal/concrete/null.rs b/lib/wasix/src/journal/concrete/null.rs new file mode 100644 index 00000000000..72b54f005c4 --- /dev/null +++ b/lib/wasix/src/journal/concrete/null.rs @@ -0,0 +1,30 @@ +use super::*; + +pub static NULL_JOURNAL: NullJournal = NullJournal {}; + +/// The null journal sends all the records into the abyss +#[derive(Debug, Default)] +pub struct NullJournal {} + +impl ReadableJournal for NullJournal { + fn read(&self) -> anyhow::Result>> { + Ok(None) + } + + fn as_restarted(&self) -> anyhow::Result> { + Ok(Box::::default()) + } +} + +impl WritableJournal for NullJournal { + fn write<'a>(&'a self, entry: JournalEntry<'a>) -> anyhow::Result { + tracing::debug!("journal event: {:?}", entry); + Ok(entry.estimate_size() as u64) + } +} + +impl Journal for NullJournal { + fn split(self) -> (Box, Box) { + (Box::::default(), Box::::default()) + } +} diff --git a/lib/wasix/src/journaling/concrete/pipe.rs b/lib/wasix/src/journal/concrete/pipe.rs similarity index 95% rename from lib/wasix/src/journaling/concrete/pipe.rs rename to lib/wasix/src/journal/concrete/pipe.rs index 3d708a2f20c..1f9f7d15a31 100644 --- a/lib/wasix/src/journaling/concrete/pipe.rs +++ b/lib/wasix/src/journal/concrete/pipe.rs @@ -50,13 +50,15 @@ impl PipeJournal { } impl WritableJournal for PipeJournalTx { - fn write<'a>(&'a self, entry: JournalEntry<'a>) -> anyhow::Result<()> { + fn write<'a>(&'a self, entry: JournalEntry<'a>) -> anyhow::Result { let entry = entry.into_owned(); + let entry_size = entry.estimate_size(); let sender = self.sender.lock().unwrap(); sender.send(entry).map_err(|err| { anyhow::format_err!("failed to send journal event through the pipe - {}", err) - }) + })?; + Ok(entry_size as u64) } } @@ -80,7 +82,7 @@ impl ReadableJournal for PipeJournalRx { } impl WritableJournal for PipeJournal { - fn write<'a>(&'a self, entry: JournalEntry<'a>) -> anyhow::Result<()> { + fn write<'a>(&'a self, entry: JournalEntry<'a>) -> anyhow::Result { self.tx.write(entry) } } diff --git a/lib/wasix/src/journal/concrete/printing.rs b/lib/wasix/src/journal/concrete/printing.rs new file mode 100644 index 00000000000..5a59a1f485d --- /dev/null +++ b/lib/wasix/src/journal/concrete/printing.rs @@ -0,0 +1,322 @@ +use std::fmt; + +use super::*; + +/// Type of printing mode to use +#[derive(Debug)] +pub enum JournalPrintingMode { + Text, + Json, +} +impl Default for JournalPrintingMode { + fn default() -> Self { + Self::Text + } +} + +/// The default for runtime is to use the unsupported journal +/// which will fail to write journal entries if one attempts to do so. +#[derive(Debug, Default)] +pub struct PrintingJournal { + mode: JournalPrintingMode, +} + +impl PrintingJournal { + pub fn new(mode: JournalPrintingMode) -> Self { + Self { mode } + } +} + +impl ReadableJournal for PrintingJournal { + fn read(&self) -> anyhow::Result>> { + Ok(None) + } + + fn as_restarted(&self) -> anyhow::Result> { + Ok(Box::::default()) + } +} + +impl WritableJournal for PrintingJournal { + fn write<'a>(&'a self, entry: JournalEntry<'a>) -> anyhow::Result { + match self.mode { + JournalPrintingMode::Text => println!("{}", entry), + JournalPrintingMode::Json => { + println!("{}", serde_json::to_string_pretty(&entry)?) + } + } + Ok(entry.estimate_size() as u64) + } +} + +impl Journal for PrintingJournal { + fn split(self) -> (Box, Box) { + ( + Box::::default(), + Box::::default(), + ) + } +} + +impl<'a> fmt::Display for JournalEntry<'a> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + JournalEntry::InitModule { wasm_hash } => { + write!(f, "init-module (hash={:x?})", wasm_hash) + } + JournalEntry::UpdateMemoryRegion { region, data } => write!( + f, + "memory-update (start={}, end={}, data.len={})", + region.start, + region.end, + data.len() + ), + JournalEntry::ProcessExit { exit_code } => { + write!(f, "process-exit (code={:?})", exit_code) + } + JournalEntry::SetThread { + id, + call_stack, + memory_stack, + store_data, + .. + } => write!( + f, + "thread-update (id={}, call-stack.len={}, mem-stack.len={}, store-size={}", + id, + call_stack.len(), + memory_stack.len(), + store_data.len(), + ), + JournalEntry::CloseThread { id, exit_code } => { + write!(f, "thread-close (id={}, code={:?})", id, exit_code) + } + JournalEntry::FileDescriptorSeek { fd, offset, whence } => write!( + f, + "fd-seek (fd={}, offset={}, whence={:?})", + fd, offset, whence + ), + JournalEntry::FileDescriptorWrite { + fd, offset, data, .. + } => write!( + f, + "fd-write (fd={}, offset={}, data.len={})", + fd, + offset, + data.len() + ), + JournalEntry::SetClockTime { clock_id, time } => { + write!(f, "set-clock-time (id={:?}, time={})", clock_id, time) + } + JournalEntry::CloseFileDescriptor { fd } => write!(f, "fd-close (fd={})", fd), + JournalEntry::OpenFileDescriptor { fd, path, .. } => { + write!(f, "fd-open (path={}, fd={})", fd, path) + } + JournalEntry::RenumberFileDescriptor { old_fd, new_fd } => { + write!(f, "fd-renumber (old={}, new={})", old_fd, new_fd) + } + JournalEntry::DuplicateFileDescriptor { + original_fd, + copied_fd, + } => write!( + f, + "fd-duplicate (original={}, copied={})", + original_fd, copied_fd + ), + JournalEntry::CreateDirectory { path, .. } => { + write!(f, "path-create-dir (path={})", path) + } + JournalEntry::RemoveDirectory { path, .. } => { + write!(f, "path-remove-dir (path={})", path) + } + JournalEntry::PathSetTimes { + path, + st_atim, + st_mtim, + .. + } => write!( + f, + "path-set-times (path={}, atime={}, mtime={}))", + path, st_atim, st_mtim + ), + JournalEntry::FileDescriptorSetTimes { + fd, + st_atim, + st_mtim, + .. + } => write!( + f, + "fd-set-times (fd={}, atime={}, mtime={})", + fd, st_atim, st_mtim + ), + JournalEntry::FileDescriptorSetFlags { fd, flags } => { + write!(f, "fd-set-flags (fd={}, flags={:?})", fd, flags) + } + JournalEntry::FileDescriptorSetRights { + fd, + fs_rights_base, + fs_rights_inheriting, + } => write!( + f, + "fd-set-rights (fd={}, base={:?}, inherited={:?})", + fd, fs_rights_base, fs_rights_inheriting + ), + JournalEntry::FileDescriptorSetSize { fd, st_size } => { + write!(f, "fd-set-size (fd={}, size={})", fd, st_size) + } + JournalEntry::FileDescriptorAdvise { + fd, offset, len, .. + } => write!(f, "fd-advise (fd={}, offset={}, len={})", fd, offset, len), + JournalEntry::FileDescriptorAllocate { fd, offset, len } => { + write!(f, "fd-allocate (fd={}, offset={}, len={})", fd, offset, len) + } + JournalEntry::CreateHardLink { + old_path, new_path, .. + } => write!(f, "path-link (from={}, to={})", old_path, new_path), + JournalEntry::CreateSymbolicLink { + old_path, new_path, .. + } => write!(f, "path-symlink (from={}, to={})", old_path, new_path), + JournalEntry::UnlinkFile { path, .. } => write!(f, "path-unlink (path={})", path), + JournalEntry::PathRename { + old_path, new_path, .. + } => write!( + f, + "path-rename (old-path={}, new-path={})", + old_path, new_path + ), + JournalEntry::ChangeDirectory { path } => write!(f, "chdir (path={})", path), + JournalEntry::EpollCreate { fd } => write!(f, "epoll-create (fd={})", fd), + JournalEntry::EpollCtl { epfd, op, fd, .. } => { + write!(f, "epoll-ctl (epfd={}, op={:?}, fd={})", epfd, op, fd) + } + JournalEntry::TtySet { tty, line_feeds } => write!( + f, + "tty-set (echo={}, buffering={}, feeds={})", + tty.echo, tty.line_buffered, line_feeds + ), + JournalEntry::CreatePipe { fd1, fd2 } => { + write!(f, "fd-pipe (fd1={}, fd2={})", fd1, fd2) + } + JournalEntry::CreateEvent { + initial_val, fd, .. + } => write!(f, "fd-event (fd={}, initial={})", fd, initial_val), + JournalEntry::PortAddAddr { cidr } => { + write!(f, "port-addr-add (ip={}, prefix={})", cidr.ip, cidr.prefix) + } + JournalEntry::PortDelAddr { addr } => write!(f, "port-addr-del (addr={})", addr), + JournalEntry::PortAddrClear => write!(f, "port-addr-clear"), + JournalEntry::PortBridge { network, .. } => { + write!(f, "port-bridge (network={})", network) + } + JournalEntry::PortUnbridge => write!(f, "port-unbridge"), + JournalEntry::PortDhcpAcquire => write!(f, "port-dhcp-acquire"), + JournalEntry::PortGatewaySet { ip } => write!(f, "port-gateway-set (ip={})", ip), + JournalEntry::PortRouteAdd { + cidr, via_router, .. + } => write!( + f, + "port-route-add (ip={}, prefix={}, via_router={})", + cidr.ip, cidr.prefix, via_router + ), + JournalEntry::PortRouteClear => write!(f, "port-route-clear"), + JournalEntry::PortRouteDel { ip } => write!(f, "port-route-del (ip={})", ip), + JournalEntry::SocketOpen { af, ty, pt, fd } => { + write!( + f, + "sock-open (fd={}, af={:?}, ty={:?}, pt={:?})", + fd, af, ty, pt + ) + } + JournalEntry::SocketListen { fd, backlog } => { + write!(f, "sock-listen (fd={}, backlog={})", fd, backlog) + } + JournalEntry::SocketBind { fd, addr } => { + write!(f, "sock-bind (fd={}, addr={})", fd, addr) + } + JournalEntry::SocketConnected { fd, addr } => { + write!(f, "sock-connect (fd={}, addr={})", fd, addr) + } + JournalEntry::SocketAccepted { + listen_fd, + fd, + peer_addr, + .. + } => write!( + f, + "sock-accept (listen-fd={}, sock_fd={}, peer={})", + listen_fd, fd, peer_addr + ), + JournalEntry::SocketJoinIpv4Multicast { + fd, + multiaddr, + iface, + } => write!( + f, + "sock-join-mcast-ipv4 (fd={}, addr={}, iface={})", + fd, multiaddr, iface + ), + JournalEntry::SocketJoinIpv6Multicast { + fd, + multiaddr, + iface, + } => write!( + f, + "sock-join-mcast-ipv6 (fd={}, addr={}, iface={})", + fd, multiaddr, iface + ), + JournalEntry::SocketLeaveIpv4Multicast { + fd, + multiaddr, + iface, + } => write!( + f, + "sock-leave-mcast-ipv4 (fd={}, addr={}, iface={})", + fd, multiaddr, iface + ), + JournalEntry::SocketLeaveIpv6Multicast { + fd, + multiaddr, + iface, + } => write!( + f, + "sock-leave-mcast-ipv6 (fd={}, addr={}, iface={})", + fd, multiaddr, iface + ), + JournalEntry::SocketSendFile { + socket_fd, + file_fd, + offset, + count, + } => write!( + f, + "sock-send-file (sock-fd={}, file-fd={}, offset={}, count={})", + socket_fd, file_fd, offset, count + ), + JournalEntry::SocketSendTo { fd, data, addr, .. } => write!( + f, + "sock-send-to (fd={}, data.len={}, addr={})", + fd, + data.len(), + addr + ), + JournalEntry::SocketSend { fd, data, .. } => { + write!(f, "sock-send (fd={}, data.len={}", fd, data.len()) + } + JournalEntry::SocketSetOptFlag { fd, opt, flag } => { + write!(f, "sock-set-opt (fd={}, opt={:?}, flag={})", fd, opt, flag) + } + JournalEntry::SocketSetOptSize { fd, opt, size } => { + write!(f, "sock-set-opt (fd={}, opt={:?}, size={})", fd, opt, size) + } + JournalEntry::SocketSetOptTime { fd, ty, time } => { + write!(f, "sock-set-opt (fd={}, opt={:?}, time={:?})", fd, ty, time) + } + JournalEntry::SocketShutdown { fd, how } => { + write!(f, "sock-shutdown (fd={}, how={:?})", fd, how) + } + JournalEntry::Snapshot { when, trigger } => { + write!(f, "snapshot (when={:?}, trigger={:?})", when, trigger) + } + } + } +} diff --git a/lib/wasix/src/journaling/concrete/composite.rs b/lib/wasix/src/journal/concrete/recombined.rs similarity index 75% rename from lib/wasix/src/journaling/concrete/composite.rs rename to lib/wasix/src/journal/concrete/recombined.rs index 59884a99af3..05ae9e53cb9 100644 --- a/lib/wasix/src/journaling/concrete/composite.rs +++ b/lib/wasix/src/journal/concrete/recombined.rs @@ -1,23 +1,23 @@ use super::*; -pub struct CompositeJournal { +pub struct RecombinedJournal { tx: Box, rx: Box, } -impl CompositeJournal { +impl RecombinedJournal { pub fn new(tx: Box, rx: Box) -> Self { Self { tx, rx } } } -impl WritableJournal for CompositeJournal { - fn write<'a>(&'a self, entry: JournalEntry<'a>) -> anyhow::Result<()> { +impl WritableJournal for RecombinedJournal { + fn write<'a>(&'a self, entry: JournalEntry<'a>) -> anyhow::Result { self.tx.write(entry) } } -impl ReadableJournal for CompositeJournal { +impl ReadableJournal for RecombinedJournal { fn read(&self) -> anyhow::Result>> { self.rx.read() } @@ -27,7 +27,7 @@ impl ReadableJournal for CompositeJournal { } } -impl Journal for CompositeJournal { +impl Journal for RecombinedJournal { fn split(self) -> (Box, Box) { (self.tx, self.rx) } diff --git a/lib/wasix/src/journaling/concrete/unsupported.rs b/lib/wasix/src/journal/concrete/unsupported.rs similarity index 90% rename from lib/wasix/src/journaling/concrete/unsupported.rs rename to lib/wasix/src/journal/concrete/unsupported.rs index 9f0dbc90116..a7e52a78f21 100644 --- a/lib/wasix/src/journaling/concrete/unsupported.rs +++ b/lib/wasix/src/journal/concrete/unsupported.rs @@ -1,6 +1,6 @@ use super::*; -pub static UNSUPPORTED_SNAPSHOT_CAPTURER: UnsupportedJournal = UnsupportedJournal {}; +pub static UNSUPPORTED_JOURNAL: UnsupportedJournal = UnsupportedJournal {}; /// The default for runtime is to use the unsupported journal /// which will fail to write journal entries if one attempts to do so. @@ -18,7 +18,7 @@ impl ReadableJournal for UnsupportedJournal { } impl WritableJournal for UnsupportedJournal { - fn write<'a>(&'a self, entry: JournalEntry<'a>) -> anyhow::Result<()> { + fn write<'a>(&'a self, entry: JournalEntry<'a>) -> anyhow::Result { tracing::debug!("journal event: {:?}", entry); Err(anyhow::format_err!("unsupported")) } diff --git a/lib/wasix/src/journaling/effector/memory_and_snapshot.rs b/lib/wasix/src/journal/effector/memory_and_snapshot.rs similarity index 100% rename from lib/wasix/src/journaling/effector/memory_and_snapshot.rs rename to lib/wasix/src/journal/effector/memory_and_snapshot.rs diff --git a/lib/wasix/src/journaling/effector/mod.rs b/lib/wasix/src/journal/effector/mod.rs similarity index 100% rename from lib/wasix/src/journaling/effector/mod.rs rename to lib/wasix/src/journal/effector/mod.rs diff --git a/lib/wasix/src/journaling/effector/process_exit.rs b/lib/wasix/src/journal/effector/process_exit.rs similarity index 100% rename from lib/wasix/src/journaling/effector/process_exit.rs rename to lib/wasix/src/journal/effector/process_exit.rs diff --git a/lib/wasix/src/journaling/effector/save_event.rs b/lib/wasix/src/journal/effector/save_event.rs similarity index 100% rename from lib/wasix/src/journaling/effector/save_event.rs rename to lib/wasix/src/journal/effector/save_event.rs diff --git a/lib/wasix/src/journaling/effector/syscalls/chdir.rs b/lib/wasix/src/journal/effector/syscalls/chdir.rs similarity index 100% rename from lib/wasix/src/journaling/effector/syscalls/chdir.rs rename to lib/wasix/src/journal/effector/syscalls/chdir.rs diff --git a/lib/wasix/src/journaling/effector/syscalls/clock_time.rs b/lib/wasix/src/journal/effector/syscalls/clock_time.rs similarity index 100% rename from lib/wasix/src/journaling/effector/syscalls/clock_time.rs rename to lib/wasix/src/journal/effector/syscalls/clock_time.rs diff --git a/lib/wasix/src/journaling/effector/syscalls/epoll_create.rs b/lib/wasix/src/journal/effector/syscalls/epoll_create.rs similarity index 100% rename from lib/wasix/src/journaling/effector/syscalls/epoll_create.rs rename to lib/wasix/src/journal/effector/syscalls/epoll_create.rs diff --git a/lib/wasix/src/journaling/effector/syscalls/epoll_ctl.rs b/lib/wasix/src/journal/effector/syscalls/epoll_ctl.rs similarity index 100% rename from lib/wasix/src/journaling/effector/syscalls/epoll_ctl.rs rename to lib/wasix/src/journal/effector/syscalls/epoll_ctl.rs diff --git a/lib/wasix/src/journaling/effector/syscalls/fd_advise.rs b/lib/wasix/src/journal/effector/syscalls/fd_advise.rs similarity index 100% rename from lib/wasix/src/journaling/effector/syscalls/fd_advise.rs rename to lib/wasix/src/journal/effector/syscalls/fd_advise.rs diff --git a/lib/wasix/src/journaling/effector/syscalls/fd_allocate.rs b/lib/wasix/src/journal/effector/syscalls/fd_allocate.rs similarity index 100% rename from lib/wasix/src/journaling/effector/syscalls/fd_allocate.rs rename to lib/wasix/src/journal/effector/syscalls/fd_allocate.rs diff --git a/lib/wasix/src/journaling/effector/syscalls/fd_close.rs b/lib/wasix/src/journal/effector/syscalls/fd_close.rs similarity index 100% rename from lib/wasix/src/journaling/effector/syscalls/fd_close.rs rename to lib/wasix/src/journal/effector/syscalls/fd_close.rs diff --git a/lib/wasix/src/journaling/effector/syscalls/fd_duplicate.rs b/lib/wasix/src/journal/effector/syscalls/fd_duplicate.rs similarity index 100% rename from lib/wasix/src/journaling/effector/syscalls/fd_duplicate.rs rename to lib/wasix/src/journal/effector/syscalls/fd_duplicate.rs diff --git a/lib/wasix/src/journaling/effector/syscalls/fd_event.rs b/lib/wasix/src/journal/effector/syscalls/fd_event.rs similarity index 100% rename from lib/wasix/src/journaling/effector/syscalls/fd_event.rs rename to lib/wasix/src/journal/effector/syscalls/fd_event.rs diff --git a/lib/wasix/src/journaling/effector/syscalls/fd_pipe.rs b/lib/wasix/src/journal/effector/syscalls/fd_pipe.rs similarity index 100% rename from lib/wasix/src/journaling/effector/syscalls/fd_pipe.rs rename to lib/wasix/src/journal/effector/syscalls/fd_pipe.rs diff --git a/lib/wasix/src/journaling/effector/syscalls/fd_renumber.rs b/lib/wasix/src/journal/effector/syscalls/fd_renumber.rs similarity index 100% rename from lib/wasix/src/journaling/effector/syscalls/fd_renumber.rs rename to lib/wasix/src/journal/effector/syscalls/fd_renumber.rs diff --git a/lib/wasix/src/journaling/effector/syscalls/fd_seek.rs b/lib/wasix/src/journal/effector/syscalls/fd_seek.rs similarity index 100% rename from lib/wasix/src/journaling/effector/syscalls/fd_seek.rs rename to lib/wasix/src/journal/effector/syscalls/fd_seek.rs diff --git a/lib/wasix/src/journaling/effector/syscalls/fd_set_flags.rs b/lib/wasix/src/journal/effector/syscalls/fd_set_flags.rs similarity index 100% rename from lib/wasix/src/journaling/effector/syscalls/fd_set_flags.rs rename to lib/wasix/src/journal/effector/syscalls/fd_set_flags.rs diff --git a/lib/wasix/src/journaling/effector/syscalls/fd_set_rights.rs b/lib/wasix/src/journal/effector/syscalls/fd_set_rights.rs similarity index 100% rename from lib/wasix/src/journaling/effector/syscalls/fd_set_rights.rs rename to lib/wasix/src/journal/effector/syscalls/fd_set_rights.rs diff --git a/lib/wasix/src/journaling/effector/syscalls/fd_set_size.rs b/lib/wasix/src/journal/effector/syscalls/fd_set_size.rs similarity index 100% rename from lib/wasix/src/journaling/effector/syscalls/fd_set_size.rs rename to lib/wasix/src/journal/effector/syscalls/fd_set_size.rs diff --git a/lib/wasix/src/journaling/effector/syscalls/fd_set_times.rs b/lib/wasix/src/journal/effector/syscalls/fd_set_times.rs similarity index 100% rename from lib/wasix/src/journaling/effector/syscalls/fd_set_times.rs rename to lib/wasix/src/journal/effector/syscalls/fd_set_times.rs diff --git a/lib/wasix/src/journaling/effector/syscalls/fd_write.rs b/lib/wasix/src/journal/effector/syscalls/fd_write.rs similarity index 100% rename from lib/wasix/src/journaling/effector/syscalls/fd_write.rs rename to lib/wasix/src/journal/effector/syscalls/fd_write.rs diff --git a/lib/wasix/src/journaling/effector/syscalls/path_create_directory.rs b/lib/wasix/src/journal/effector/syscalls/path_create_directory.rs similarity index 100% rename from lib/wasix/src/journaling/effector/syscalls/path_create_directory.rs rename to lib/wasix/src/journal/effector/syscalls/path_create_directory.rs diff --git a/lib/wasix/src/journaling/effector/syscalls/path_link.rs b/lib/wasix/src/journal/effector/syscalls/path_link.rs similarity index 100% rename from lib/wasix/src/journaling/effector/syscalls/path_link.rs rename to lib/wasix/src/journal/effector/syscalls/path_link.rs diff --git a/lib/wasix/src/journaling/effector/syscalls/path_open.rs b/lib/wasix/src/journal/effector/syscalls/path_open.rs similarity index 100% rename from lib/wasix/src/journaling/effector/syscalls/path_open.rs rename to lib/wasix/src/journal/effector/syscalls/path_open.rs diff --git a/lib/wasix/src/journaling/effector/syscalls/path_remove_directory.rs b/lib/wasix/src/journal/effector/syscalls/path_remove_directory.rs similarity index 100% rename from lib/wasix/src/journaling/effector/syscalls/path_remove_directory.rs rename to lib/wasix/src/journal/effector/syscalls/path_remove_directory.rs diff --git a/lib/wasix/src/journaling/effector/syscalls/path_rename.rs b/lib/wasix/src/journal/effector/syscalls/path_rename.rs similarity index 100% rename from lib/wasix/src/journaling/effector/syscalls/path_rename.rs rename to lib/wasix/src/journal/effector/syscalls/path_rename.rs diff --git a/lib/wasix/src/journaling/effector/syscalls/path_set_times.rs b/lib/wasix/src/journal/effector/syscalls/path_set_times.rs similarity index 100% rename from lib/wasix/src/journaling/effector/syscalls/path_set_times.rs rename to lib/wasix/src/journal/effector/syscalls/path_set_times.rs diff --git a/lib/wasix/src/journaling/effector/syscalls/path_symlink.rs b/lib/wasix/src/journal/effector/syscalls/path_symlink.rs similarity index 100% rename from lib/wasix/src/journaling/effector/syscalls/path_symlink.rs rename to lib/wasix/src/journal/effector/syscalls/path_symlink.rs diff --git a/lib/wasix/src/journaling/effector/syscalls/path_unlink.rs b/lib/wasix/src/journal/effector/syscalls/path_unlink.rs similarity index 100% rename from lib/wasix/src/journaling/effector/syscalls/path_unlink.rs rename to lib/wasix/src/journal/effector/syscalls/path_unlink.rs diff --git a/lib/wasix/src/journaling/effector/syscalls/port_addr_add.rs b/lib/wasix/src/journal/effector/syscalls/port_addr_add.rs similarity index 100% rename from lib/wasix/src/journaling/effector/syscalls/port_addr_add.rs rename to lib/wasix/src/journal/effector/syscalls/port_addr_add.rs diff --git a/lib/wasix/src/journaling/effector/syscalls/port_addr_clear.rs b/lib/wasix/src/journal/effector/syscalls/port_addr_clear.rs similarity index 100% rename from lib/wasix/src/journaling/effector/syscalls/port_addr_clear.rs rename to lib/wasix/src/journal/effector/syscalls/port_addr_clear.rs diff --git a/lib/wasix/src/journaling/effector/syscalls/port_addr_remove.rs b/lib/wasix/src/journal/effector/syscalls/port_addr_remove.rs similarity index 100% rename from lib/wasix/src/journaling/effector/syscalls/port_addr_remove.rs rename to lib/wasix/src/journal/effector/syscalls/port_addr_remove.rs diff --git a/lib/wasix/src/journaling/effector/syscalls/port_bridge.rs b/lib/wasix/src/journal/effector/syscalls/port_bridge.rs similarity index 100% rename from lib/wasix/src/journaling/effector/syscalls/port_bridge.rs rename to lib/wasix/src/journal/effector/syscalls/port_bridge.rs diff --git a/lib/wasix/src/journaling/effector/syscalls/port_dhcp_acquire.rs b/lib/wasix/src/journal/effector/syscalls/port_dhcp_acquire.rs similarity index 100% rename from lib/wasix/src/journaling/effector/syscalls/port_dhcp_acquire.rs rename to lib/wasix/src/journal/effector/syscalls/port_dhcp_acquire.rs diff --git a/lib/wasix/src/journaling/effector/syscalls/port_gateway_set.rs b/lib/wasix/src/journal/effector/syscalls/port_gateway_set.rs similarity index 100% rename from lib/wasix/src/journaling/effector/syscalls/port_gateway_set.rs rename to lib/wasix/src/journal/effector/syscalls/port_gateway_set.rs diff --git a/lib/wasix/src/journaling/effector/syscalls/port_route_add.rs b/lib/wasix/src/journal/effector/syscalls/port_route_add.rs similarity index 100% rename from lib/wasix/src/journaling/effector/syscalls/port_route_add.rs rename to lib/wasix/src/journal/effector/syscalls/port_route_add.rs diff --git a/lib/wasix/src/journaling/effector/syscalls/port_route_clear.rs b/lib/wasix/src/journal/effector/syscalls/port_route_clear.rs similarity index 100% rename from lib/wasix/src/journaling/effector/syscalls/port_route_clear.rs rename to lib/wasix/src/journal/effector/syscalls/port_route_clear.rs diff --git a/lib/wasix/src/journaling/effector/syscalls/port_route_remove.rs b/lib/wasix/src/journal/effector/syscalls/port_route_remove.rs similarity index 100% rename from lib/wasix/src/journaling/effector/syscalls/port_route_remove.rs rename to lib/wasix/src/journal/effector/syscalls/port_route_remove.rs diff --git a/lib/wasix/src/journaling/effector/syscalls/port_unbridge.rs b/lib/wasix/src/journal/effector/syscalls/port_unbridge.rs similarity index 100% rename from lib/wasix/src/journaling/effector/syscalls/port_unbridge.rs rename to lib/wasix/src/journal/effector/syscalls/port_unbridge.rs diff --git a/lib/wasix/src/journaling/effector/syscalls/sock_accept.rs b/lib/wasix/src/journal/effector/syscalls/sock_accept.rs similarity index 100% rename from lib/wasix/src/journaling/effector/syscalls/sock_accept.rs rename to lib/wasix/src/journal/effector/syscalls/sock_accept.rs diff --git a/lib/wasix/src/journaling/effector/syscalls/sock_bind.rs b/lib/wasix/src/journal/effector/syscalls/sock_bind.rs similarity index 100% rename from lib/wasix/src/journaling/effector/syscalls/sock_bind.rs rename to lib/wasix/src/journal/effector/syscalls/sock_bind.rs diff --git a/lib/wasix/src/journaling/effector/syscalls/sock_connect.rs b/lib/wasix/src/journal/effector/syscalls/sock_connect.rs similarity index 100% rename from lib/wasix/src/journaling/effector/syscalls/sock_connect.rs rename to lib/wasix/src/journal/effector/syscalls/sock_connect.rs diff --git a/lib/wasix/src/journaling/effector/syscalls/sock_join_ipv4_multicast.rs b/lib/wasix/src/journal/effector/syscalls/sock_join_ipv4_multicast.rs similarity index 100% rename from lib/wasix/src/journaling/effector/syscalls/sock_join_ipv4_multicast.rs rename to lib/wasix/src/journal/effector/syscalls/sock_join_ipv4_multicast.rs diff --git a/lib/wasix/src/journaling/effector/syscalls/sock_join_ipv6_multicast.rs b/lib/wasix/src/journal/effector/syscalls/sock_join_ipv6_multicast.rs similarity index 100% rename from lib/wasix/src/journaling/effector/syscalls/sock_join_ipv6_multicast.rs rename to lib/wasix/src/journal/effector/syscalls/sock_join_ipv6_multicast.rs diff --git a/lib/wasix/src/journaling/effector/syscalls/sock_leave_ipv4_multicast.rs b/lib/wasix/src/journal/effector/syscalls/sock_leave_ipv4_multicast.rs similarity index 100% rename from lib/wasix/src/journaling/effector/syscalls/sock_leave_ipv4_multicast.rs rename to lib/wasix/src/journal/effector/syscalls/sock_leave_ipv4_multicast.rs diff --git a/lib/wasix/src/journaling/effector/syscalls/sock_leave_ipv6_multicast.rs b/lib/wasix/src/journal/effector/syscalls/sock_leave_ipv6_multicast.rs similarity index 100% rename from lib/wasix/src/journaling/effector/syscalls/sock_leave_ipv6_multicast.rs rename to lib/wasix/src/journal/effector/syscalls/sock_leave_ipv6_multicast.rs diff --git a/lib/wasix/src/journaling/effector/syscalls/sock_listen.rs b/lib/wasix/src/journal/effector/syscalls/sock_listen.rs similarity index 100% rename from lib/wasix/src/journaling/effector/syscalls/sock_listen.rs rename to lib/wasix/src/journal/effector/syscalls/sock_listen.rs diff --git a/lib/wasix/src/journaling/effector/syscalls/sock_open.rs b/lib/wasix/src/journal/effector/syscalls/sock_open.rs similarity index 100% rename from lib/wasix/src/journaling/effector/syscalls/sock_open.rs rename to lib/wasix/src/journal/effector/syscalls/sock_open.rs diff --git a/lib/wasix/src/journaling/effector/syscalls/sock_send.rs b/lib/wasix/src/journal/effector/syscalls/sock_send.rs similarity index 100% rename from lib/wasix/src/journaling/effector/syscalls/sock_send.rs rename to lib/wasix/src/journal/effector/syscalls/sock_send.rs diff --git a/lib/wasix/src/journaling/effector/syscalls/sock_send_file.rs b/lib/wasix/src/journal/effector/syscalls/sock_send_file.rs similarity index 100% rename from lib/wasix/src/journaling/effector/syscalls/sock_send_file.rs rename to lib/wasix/src/journal/effector/syscalls/sock_send_file.rs diff --git a/lib/wasix/src/journaling/effector/syscalls/sock_send_to.rs b/lib/wasix/src/journal/effector/syscalls/sock_send_to.rs similarity index 100% rename from lib/wasix/src/journaling/effector/syscalls/sock_send_to.rs rename to lib/wasix/src/journal/effector/syscalls/sock_send_to.rs diff --git a/lib/wasix/src/journaling/effector/syscalls/sock_set_opt_flag.rs b/lib/wasix/src/journal/effector/syscalls/sock_set_opt_flag.rs similarity index 100% rename from lib/wasix/src/journaling/effector/syscalls/sock_set_opt_flag.rs rename to lib/wasix/src/journal/effector/syscalls/sock_set_opt_flag.rs diff --git a/lib/wasix/src/journaling/effector/syscalls/sock_set_opt_size.rs b/lib/wasix/src/journal/effector/syscalls/sock_set_opt_size.rs similarity index 100% rename from lib/wasix/src/journaling/effector/syscalls/sock_set_opt_size.rs rename to lib/wasix/src/journal/effector/syscalls/sock_set_opt_size.rs diff --git a/lib/wasix/src/journaling/effector/syscalls/sock_set_opt_time.rs b/lib/wasix/src/journal/effector/syscalls/sock_set_opt_time.rs similarity index 84% rename from lib/wasix/src/journaling/effector/syscalls/sock_set_opt_time.rs rename to lib/wasix/src/journal/effector/syscalls/sock_set_opt_time.rs index d8aff0d15b2..b88f74d73a2 100644 --- a/lib/wasix/src/journaling/effector/syscalls/sock_set_opt_time.rs +++ b/lib/wasix/src/journal/effector/syscalls/sock_set_opt_time.rs @@ -11,7 +11,14 @@ impl JournalEffector { ty: TimeType, time: Option, ) -> anyhow::Result<()> { - Self::save_event(ctx, JournalEntry::SocketSetOptTime { fd, ty, time }) + Self::save_event( + ctx, + JournalEntry::SocketSetOptTime { + fd, + ty: ty.into(), + time, + }, + ) } pub fn apply_sock_set_opt_time( diff --git a/lib/wasix/src/journaling/effector/syscalls/sock_shutdown.rs b/lib/wasix/src/journal/effector/syscalls/sock_shutdown.rs similarity index 83% rename from lib/wasix/src/journaling/effector/syscalls/sock_shutdown.rs rename to lib/wasix/src/journal/effector/syscalls/sock_shutdown.rs index 8d6d5e76000..1d22351ad43 100644 --- a/lib/wasix/src/journaling/effector/syscalls/sock_shutdown.rs +++ b/lib/wasix/src/journal/effector/syscalls/sock_shutdown.rs @@ -8,7 +8,13 @@ impl JournalEffector { fd: Fd, shutdown: Shutdown, ) -> anyhow::Result<()> { - Self::save_event(ctx, JournalEntry::SocketShutdown { fd, how: shutdown }) + Self::save_event( + ctx, + JournalEntry::SocketShutdown { + fd, + how: shutdown.into(), + }, + ) } pub fn apply_sock_shutdown( diff --git a/lib/wasix/src/journaling/effector/syscalls/tty_set.rs b/lib/wasix/src/journal/effector/syscalls/tty_set.rs similarity index 100% rename from lib/wasix/src/journaling/effector/syscalls/tty_set.rs rename to lib/wasix/src/journal/effector/syscalls/tty_set.rs diff --git a/lib/wasix/src/journaling/effector/thread_exit.rs b/lib/wasix/src/journal/effector/thread_exit.rs similarity index 100% rename from lib/wasix/src/journaling/effector/thread_exit.rs rename to lib/wasix/src/journal/effector/thread_exit.rs diff --git a/lib/wasix/src/journaling/effector/thread_state.rs b/lib/wasix/src/journal/effector/thread_state.rs similarity index 100% rename from lib/wasix/src/journaling/effector/thread_state.rs rename to lib/wasix/src/journal/effector/thread_state.rs diff --git a/lib/wasix/src/journaling/effector/unimplemented.rs b/lib/wasix/src/journal/effector/unimplemented.rs similarity index 100% rename from lib/wasix/src/journaling/effector/unimplemented.rs rename to lib/wasix/src/journal/effector/unimplemented.rs diff --git a/lib/wasix/src/journaling/journal.rs b/lib/wasix/src/journal/entry.rs similarity index 72% rename from lib/wasix/src/journaling/journal.rs rename to lib/wasix/src/journal/entry.rs index d28e86142d6..a096ef2a920 100644 --- a/lib/wasix/src/journaling/journal.rs +++ b/lib/wasix/src/journal/entry.rs @@ -1,3 +1,4 @@ +use super::base64; use serde::{Deserialize, Serialize}; use std::net::{Shutdown, SocketAddr}; use std::time::SystemTime; @@ -38,16 +39,76 @@ pub enum SocketJournalEvent { }, } +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialOrd, Ord, PartialEq, Eq, Hash)] +pub enum SocketShutdownHow { + Read, + Write, + Both, +} +impl From for SocketShutdownHow { + fn from(value: Shutdown) -> Self { + match value { + Shutdown::Read => Self::Read, + Shutdown::Write => Self::Write, + Shutdown::Both => Self::Both, + } + } +} +impl From for Shutdown { + fn from(value: SocketShutdownHow) -> Self { + match value { + SocketShutdownHow::Read => Self::Read, + SocketShutdownHow::Write => Self::Write, + SocketShutdownHow::Both => Self::Both, + } + } +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialOrd, Ord, PartialEq, Eq, Hash)] +pub enum SocketOptTimeType { + ReadTimeout, + WriteTimeout, + AcceptTimeout, + ConnectTimeout, + BindTimeout, + Linger, +} +impl From for SocketOptTimeType { + fn from(value: TimeType) -> Self { + match value { + TimeType::ReadTimeout => Self::ReadTimeout, + TimeType::WriteTimeout => Self::WriteTimeout, + TimeType::AcceptTimeout => Self::AcceptTimeout, + TimeType::ConnectTimeout => Self::ConnectTimeout, + TimeType::BindTimeout => Self::BindTimeout, + TimeType::Linger => Self::Linger, + } + } +} +impl From for TimeType { + fn from(value: SocketOptTimeType) -> Self { + match value { + SocketOptTimeType::ReadTimeout => Self::ReadTimeout, + SocketOptTimeType::WriteTimeout => Self::WriteTimeout, + SocketOptTimeType::AcceptTimeout => Self::AcceptTimeout, + SocketOptTimeType::ConnectTimeout => Self::ConnectTimeout, + SocketOptTimeType::BindTimeout => Self::BindTimeout, + SocketOptTimeType::Linger => Self::Linger, + } + } +} + /// Represents a log entry in a snapshot log stream that represents the total /// state of a WASM process at a point in time. #[allow(clippy::large_enum_variant)] -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] pub enum JournalEntry<'a> { InitModule { wasm_hash: [u8; 32], }, UpdateMemoryRegion { region: Range, + #[serde(with = "base64")] data: Cow<'a, [u8]>, }, ProcessExit { @@ -55,8 +116,11 @@ pub enum JournalEntry<'a> { }, SetThread { id: WasiThreadId, + #[serde(with = "base64")] call_stack: Cow<'a, [u8]>, + #[serde(with = "base64")] memory_stack: Cow<'a, [u8]>, + #[serde(with = "base64")] store_data: Cow<'a, [u8]>, is_64bit: bool, }, @@ -72,6 +136,7 @@ pub enum JournalEntry<'a> { FileDescriptorWrite { fd: Fd, offset: u64, + #[serde(with = "base64")] data: Cow<'a, [u8]>, is_64bit: bool, }, @@ -273,6 +338,7 @@ pub enum JournalEntry<'a> { }, SocketSendTo { fd: Fd, + #[serde(with = "base64")] data: Cow<'a, [u8]>, flags: SiFlags, addr: SocketAddr, @@ -280,6 +346,7 @@ pub enum JournalEntry<'a> { }, SocketSend { fd: Fd, + #[serde(with = "base64")] data: Cow<'a, [u8]>, flags: SiFlags, is_64bit: bool, @@ -296,12 +363,12 @@ pub enum JournalEntry<'a> { }, SocketSetOptTime { fd: Fd, - ty: TimeType, + ty: SocketOptTimeType, time: Option, }, SocketShutdown { fd: Fd, - how: Shutdown, + how: SocketShutdownHow, }, /// Represents the marker for the end of a snapshot Snapshot { @@ -633,41 +700,81 @@ impl<'a> JournalEntry<'a> { Self::Snapshot { when, trigger } => JournalEntry::Snapshot { when, trigger }, } } -} - -/// The snapshot capturer will take a series of objects that represents the state of -/// a WASM process at a point in time and saves it so that it can be restored. -/// It also allows for the restoration of that state at a later moment -#[allow(unused_variables)] -pub trait WritableJournal { - /// Takes in a stream of snapshot log entries and saves them so that they - /// may be restored at a later moment - fn write<'a>(&'a self, entry: JournalEntry<'a>) -> anyhow::Result<()>; -} -/// The snapshot capturer will take a series of objects that represents the state of -/// a WASM process at a point in time and saves it so that it can be restored. -/// It also allows for the restoration of that state at a later moment -#[allow(unused_variables)] -pub trait ReadableJournal { - /// Returns a stream of snapshot objects that the runtime will use - /// to restore the state of a WASM process to a previous moment in time - fn read(&self) -> anyhow::Result>>; - - /// Resets the journal so that reads will start from the - /// beginning again - fn as_restarted(&self) -> anyhow::Result>; -} - -/// The snapshot capturer will take a series of objects that represents the state of -/// a WASM process at a point in time and saves it so that it can be restored. -/// It also allows for the restoration of that state at a later moment -#[allow(unused_variables)] -pub trait Journal: WritableJournal + ReadableJournal { - /// Splits the journal into a read and write side - fn split(self) -> (Box, Box); + pub fn estimate_size(&self) -> usize { + let base_size = std::mem::size_of_val(self); + match self { + JournalEntry::InitModule { .. } => base_size, + JournalEntry::UpdateMemoryRegion { data, .. } => base_size + data.len(), + JournalEntry::ProcessExit { .. } => base_size, + JournalEntry::SetThread { + call_stack, + memory_stack, + store_data, + .. + } => base_size + call_stack.len() + memory_stack.len() + store_data.len(), + JournalEntry::CloseThread { .. } => base_size, + JournalEntry::FileDescriptorSeek { .. } => base_size, + JournalEntry::FileDescriptorWrite { data, .. } => base_size + data.len(), + JournalEntry::SetClockTime { .. } => base_size, + JournalEntry::CloseFileDescriptor { .. } => base_size, + JournalEntry::OpenFileDescriptor { path, .. } => base_size + path.as_bytes().len(), + JournalEntry::RenumberFileDescriptor { .. } => base_size, + JournalEntry::DuplicateFileDescriptor { .. } => base_size, + JournalEntry::CreateDirectory { path, .. } => base_size + path.as_bytes().len(), + JournalEntry::RemoveDirectory { path, .. } => base_size + path.as_bytes().len(), + JournalEntry::PathSetTimes { path, .. } => base_size + path.as_bytes().len(), + JournalEntry::FileDescriptorSetTimes { .. } => base_size, + JournalEntry::FileDescriptorSetFlags { .. } => base_size, + JournalEntry::FileDescriptorSetRights { .. } => base_size, + JournalEntry::FileDescriptorSetSize { .. } => base_size, + JournalEntry::FileDescriptorAdvise { .. } => base_size, + JournalEntry::FileDescriptorAllocate { .. } => base_size, + JournalEntry::CreateHardLink { + old_path, new_path, .. + } => base_size + old_path.as_bytes().len() + new_path.as_bytes().len(), + JournalEntry::CreateSymbolicLink { + old_path, new_path, .. + } => base_size + old_path.as_bytes().len() + new_path.as_bytes().len(), + JournalEntry::UnlinkFile { path, .. } => base_size + path.as_bytes().len(), + JournalEntry::PathRename { + old_path, new_path, .. + } => base_size + old_path.as_bytes().len() + new_path.as_bytes().len(), + JournalEntry::ChangeDirectory { path } => base_size + path.as_bytes().len(), + JournalEntry::EpollCreate { .. } => base_size, + JournalEntry::EpollCtl { .. } => base_size, + JournalEntry::TtySet { .. } => base_size, + JournalEntry::CreatePipe { .. } => base_size, + JournalEntry::CreateEvent { .. } => base_size, + JournalEntry::PortAddAddr { .. } => base_size, + JournalEntry::PortDelAddr { .. } => base_size, + JournalEntry::PortAddrClear => base_size, + JournalEntry::PortBridge { network, token, .. } => { + base_size + network.as_bytes().len() + token.as_bytes().len() + } + JournalEntry::PortUnbridge => base_size, + JournalEntry::PortDhcpAcquire => base_size, + JournalEntry::PortGatewaySet { .. } => base_size, + JournalEntry::PortRouteAdd { .. } => base_size, + JournalEntry::PortRouteClear => base_size, + JournalEntry::PortRouteDel { .. } => base_size, + JournalEntry::SocketOpen { .. } => base_size, + JournalEntry::SocketListen { .. } => base_size, + JournalEntry::SocketBind { .. } => base_size, + JournalEntry::SocketConnected { .. } => base_size, + JournalEntry::SocketAccepted { .. } => base_size, + JournalEntry::SocketJoinIpv4Multicast { .. } => base_size, + JournalEntry::SocketJoinIpv6Multicast { .. } => base_size, + JournalEntry::SocketLeaveIpv4Multicast { .. } => base_size, + JournalEntry::SocketLeaveIpv6Multicast { .. } => base_size, + JournalEntry::SocketSendFile { .. } => base_size, + JournalEntry::SocketSendTo { data, .. } => base_size + data.len(), + JournalEntry::SocketSend { data, .. } => base_size + data.len(), + JournalEntry::SocketSetOptFlag { .. } => base_size, + JournalEntry::SocketSetOptSize { .. } => base_size, + JournalEntry::SocketSetOptTime { .. } => base_size, + JournalEntry::SocketShutdown { .. } => base_size, + JournalEntry::Snapshot { .. } => base_size, + } + } } - -pub type DynJournal = dyn Journal + Send + Sync; -pub type DynWritableJournal = dyn WritableJournal + Send + Sync; -pub type DynReadableJournal = dyn ReadableJournal + Send + Sync; diff --git a/lib/wasix/src/journal/mod.rs b/lib/wasix/src/journal/mod.rs new file mode 100644 index 00000000000..14b5cb5984b --- /dev/null +++ b/lib/wasix/src/journal/mod.rs @@ -0,0 +1,56 @@ +mod base64; +mod concrete; +#[cfg(feature = "journal")] +mod effector; +#[cfg(not(feature = "journal"))] +#[path = "effector/unimplemented.rs"] +mod effector; +mod entry; +mod snapshot; +mod util; + +pub use concrete::*; +pub use effector::*; +pub use entry::*; +pub use snapshot::*; +pub use util::*; + +use serde::{Deserialize, Serialize}; +use std::str::FromStr; + +/// The snapshot capturer will take a series of objects that represents the state of +/// a WASM process at a point in time and saves it so that it can be restored. +/// It also allows for the restoration of that state at a later moment +#[allow(unused_variables)] +pub trait WritableJournal { + /// Takes in a stream of snapshot log entries and saves them so that they + /// may be restored at a later moment + fn write<'a>(&'a self, entry: JournalEntry<'a>) -> anyhow::Result; +} + +/// The snapshot capturer will take a series of objects that represents the state of +/// a WASM process at a point in time and saves it so that it can be restored. +/// It also allows for the restoration of that state at a later moment +#[allow(unused_variables)] +pub trait ReadableJournal { + /// Returns a stream of snapshot objects that the runtime will use + /// to restore the state of a WASM process to a previous moment in time + fn read(&self) -> anyhow::Result>>; + + /// Resets the journal so that reads will start from the + /// beginning again + fn as_restarted(&self) -> anyhow::Result>; +} + +/// The snapshot capturer will take a series of objects that represents the state of +/// a WASM process at a point in time and saves it so that it can be restored. +/// It also allows for the restoration of that state at a later moment +#[allow(unused_variables)] +pub trait Journal: WritableJournal + ReadableJournal { + /// Splits the journal into a read and write side + fn split(self) -> (Box, Box); +} + +pub type DynJournal = dyn Journal + Send + Sync; +pub type DynWritableJournal = dyn WritableJournal + Send + Sync; +pub type DynReadableJournal = dyn ReadableJournal + Send + Sync; diff --git a/lib/wasix/src/journaling/mod.rs b/lib/wasix/src/journal/snapshot.rs similarity index 89% rename from lib/wasix/src/journaling/mod.rs rename to lib/wasix/src/journal/snapshot.rs index aa704bca8db..2d7d9d09ab6 100644 --- a/lib/wasix/src/journaling/mod.rs +++ b/lib/wasix/src/journal/snapshot.rs @@ -1,17 +1,4 @@ -mod concrete; -#[cfg(feature = "journal")] -mod effector; -#[cfg(not(feature = "journal"))] -#[path = "effector/unimplemented.rs"] -mod effector; -mod journal; - -pub use concrete::*; -pub use effector::*; -pub use journal::*; - -use serde::{Deserialize, Serialize}; -use std::str::FromStr; +use super::*; /// Various triggers that will cause the runtime to take snapshot /// of the WASM state and store it in the snapshot file. diff --git a/lib/wasix/src/journal/util.rs b/lib/wasix/src/journal/util.rs new file mode 100644 index 00000000000..528c131fb2c --- /dev/null +++ b/lib/wasix/src/journal/util.rs @@ -0,0 +1,11 @@ +use super::*; + +pub fn copy_journal( + from: &R, + to: &W, +) -> anyhow::Result<()> { + while let Some(record) = from.read()? { + to.write(record)?; + } + Ok(()) +} diff --git a/lib/wasix/src/journaling/concrete/mod.rs b/lib/wasix/src/journaling/concrete/mod.rs deleted file mode 100644 index d6f73994f47..00000000000 --- a/lib/wasix/src/journaling/concrete/mod.rs +++ /dev/null @@ -1,21 +0,0 @@ -mod archived_journal; -mod boxed_journal; -mod compactor; -mod composite; -mod filter; -#[cfg(feature = "journal")] -mod log_file; -mod pipe; -mod unsupported; - -pub(super) use super::*; - -pub use archived_journal::*; -pub use boxed_journal::*; -pub use compactor::*; -pub use composite::*; -pub use filter::*; -#[cfg(feature = "journal")] -pub use log_file::*; -pub use pipe::*; -pub use unsupported::*; diff --git a/lib/wasix/src/lib.rs b/lib/wasix/src/lib.rs index 9bc3716632c..06f8f646778 100644 --- a/lib/wasix/src/lib.rs +++ b/lib/wasix/src/lib.rs @@ -47,7 +47,7 @@ pub mod net; pub mod capabilities; pub mod fs; pub mod http; -pub mod journaling; +pub mod journal; mod rewind; pub mod runners; pub mod runtime; @@ -62,7 +62,7 @@ use std::sync::Arc; #[allow(unused_imports)] use bytes::{Bytes, BytesMut}; -use journaling::DynJournal; +use journal::DynJournal; use os::task::control_plane::ControlPlaneError; use thiserror::Error; use tracing::error; diff --git a/lib/wasix/src/net/socket.rs b/lib/wasix/src/net/socket.rs index debf5fb78ee..6b372bc1c3a 100644 --- a/lib/wasix/src/net/socket.rs +++ b/lib/wasix/src/net/socket.rs @@ -151,6 +151,7 @@ pub enum WasiSocketStatus { } #[derive(Debug, Copy, Clone, PartialEq, Eq)] +#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] pub enum TimeType { ReadTimeout, WriteTimeout, diff --git a/lib/wasix/src/os/task/process.rs b/lib/wasix/src/os/task/process.rs index 0b1ad096e32..b8d476d58ee 100644 --- a/lib/wasix/src/os/task/process.rs +++ b/lib/wasix/src/os/task/process.rs @@ -1,7 +1,7 @@ #[cfg(feature = "journal")] -use crate::{journaling::JournalEffector, unwind, WasiResult}; +use crate::{journal::JournalEffector, unwind, WasiResult}; use crate::{ - journaling::SnapshotTrigger, runtime::module_cache::ModuleHash, WasiEnv, WasiRuntimeError, + journal::SnapshotTrigger, runtime::module_cache::ModuleHash, WasiEnv, WasiRuntimeError, }; use serde::{Deserialize, Serialize}; use std::{ diff --git a/lib/wasix/src/runners/wasi.rs b/lib/wasix/src/runners/wasi.rs index b351f38db18..9329ea9237d 100644 --- a/lib/wasix/src/runners/wasi.rs +++ b/lib/wasix/src/runners/wasi.rs @@ -11,7 +11,7 @@ use webc::metadata::{annotations::Wasi, Command}; use crate::{ bin_factory::BinaryPackage, capabilities::Capabilities, - journaling::{DynJournal, SnapshotTrigger}, + journal::{DynJournal, SnapshotTrigger}, runners::{wasi_common::CommonWasiOptions, MappedDirectory}, runtime::{module_cache::ModuleHash, task_manager::VirtualTaskManagerExt}, Runtime, WasiEnvBuilder, WasiRuntimeError, @@ -195,7 +195,7 @@ impl WasiRunner { } pub fn add_default_snapshot_triggers(&mut self) -> &mut Self { - for on in crate::journaling::DEFAULT_SNAPSHOT_TRIGGERS { + for on in crate::journal::DEFAULT_SNAPSHOT_TRIGGERS { if !self.has_snapshot_trigger(on) { self.add_snapshot_trigger(on); } diff --git a/lib/wasix/src/runners/wasi_common.rs b/lib/wasix/src/runners/wasi_common.rs index 6df106be212..f037c3d7f1e 100644 --- a/lib/wasix/src/runners/wasi_common.rs +++ b/lib/wasix/src/runners/wasi_common.rs @@ -13,7 +13,7 @@ use webc::metadata::annotations::Wasi as WasiAnnotation; use crate::{ bin_factory::BinaryPackage, capabilities::Capabilities, - journaling::{DynJournal, SnapshotTrigger}, + journal::{DynJournal, SnapshotTrigger}, runners::MappedDirectory, WasiEnvBuilder, }; diff --git a/lib/wasix/src/runners/wcgi/runner.rs b/lib/wasix/src/runners/wcgi/runner.rs index eff7b1b402d..918fc4157e2 100644 --- a/lib/wasix/src/runners/wcgi/runner.rs +++ b/lib/wasix/src/runners/wcgi/runner.rs @@ -244,13 +244,13 @@ impl Config { } #[cfg(feature = "journal")] - pub fn add_snapshot_trigger(&mut self, on: crate::journaling::SnapshotTrigger) { + pub fn add_snapshot_trigger(&mut self, on: crate::journal::SnapshotTrigger) { self.wasi.snapshot_on.push(on); } #[cfg(feature = "journal")] pub fn add_default_snapshot_triggers(&mut self) -> &mut Self { - for on in crate::journaling::DEFAULT_SNAPSHOT_TRIGGERS { + for on in crate::journal::DEFAULT_SNAPSHOT_TRIGGERS { if !self.has_snapshot_trigger(on) { self.add_snapshot_trigger(on); } @@ -259,21 +259,21 @@ impl Config { } #[cfg(feature = "journal")] - pub fn has_snapshot_trigger(&self, on: crate::journaling::SnapshotTrigger) -> bool { + pub fn has_snapshot_trigger(&self, on: crate::journal::SnapshotTrigger) -> bool { self.wasi.snapshot_on.iter().any(|t| *t == on) } #[cfg(feature = "journal")] pub fn with_snapshot_interval(&mut self, period: std::time::Duration) -> &mut Self { - if !self.has_snapshot_trigger(crate::journaling::SnapshotTrigger::PeriodicInterval) { - self.add_snapshot_trigger(crate::journaling::SnapshotTrigger::PeriodicInterval); + if !self.has_snapshot_trigger(crate::journal::SnapshotTrigger::PeriodicInterval) { + self.add_snapshot_trigger(crate::journal::SnapshotTrigger::PeriodicInterval); } self.wasi.snapshot_interval.replace(period); self } #[cfg(feature = "journal")] - pub fn add_journal(&mut self, journal: Arc) -> &mut Self { + pub fn add_journal(&mut self, journal: Arc) -> &mut Self { self.wasi.journals.push(journal); self } diff --git a/lib/wasix/src/runtime/mod.rs b/lib/wasix/src/runtime/mod.rs index 9dd8b7d8751..76cc56db94b 100644 --- a/lib/wasix/src/runtime/mod.rs +++ b/lib/wasix/src/runtime/mod.rs @@ -20,7 +20,7 @@ use virtual_net::{DynVirtualNetworking, VirtualNetworking}; use wasmer::Module; #[cfg(feature = "journal")] -use crate::journaling::DynJournal; +use crate::journal::DynJournal; use crate::{ http::{DynHttpClient, HttpClient}, os::TtyBridge, diff --git a/lib/wasix/src/state/builder.rs b/lib/wasix/src/state/builder.rs index 827c52ba936..b93ad047f21 100644 --- a/lib/wasix/src/state/builder.rs +++ b/lib/wasix/src/state/builder.rs @@ -27,7 +27,7 @@ use crate::{ }; #[cfg(feature = "journal")] use crate::{ - journaling::{DynJournal, SnapshotTrigger}, + journal::{DynJournal, SnapshotTrigger}, syscalls::restore_snapshot, }; @@ -1023,9 +1023,9 @@ impl WasiEnvBuilder { // process or if a process has been recompiled let wasm_hash = env.data(&store).process.module_hash.as_bytes(); let mut ctx = env.env.clone().into_mut(&mut store); - crate::journaling::JournalEffector::save_event( + crate::journal::JournalEffector::save_event( &mut ctx, - crate::journaling::JournalEntry::InitModule { wasm_hash }, + crate::journal::JournalEntry::InitModule { wasm_hash }, ) .map_err(|err| { WasiRuntimeError::Runtime(RuntimeError::new(format!( diff --git a/lib/wasix/src/state/env.rs b/lib/wasix/src/state/env.rs index 43b52c6b968..52959201c5e 100644 --- a/lib/wasix/src/state/env.rs +++ b/lib/wasix/src/state/env.rs @@ -23,7 +23,7 @@ use wasmer_wasix_types::{ }; #[cfg(feature = "journal")] -use crate::journaling::{DynJournal, JournalEffector, SnapshotTrigger}; +use crate::journal::{DynJournal, JournalEffector, SnapshotTrigger}; use crate::{ bin_factory::{BinFactory, BinaryPackage}, capabilities::Capabilities, diff --git a/lib/wasix/src/syscalls/mod.rs b/lib/wasix/src/syscalls/mod.rs index 32d2d91d5bc..9b8288e77a0 100644 --- a/lib/wasix/src/syscalls/mod.rs +++ b/lib/wasix/src/syscalls/mod.rs @@ -117,7 +117,7 @@ use crate::{ fs_error_into_wasi_err, virtual_file_type_to_wasi_file_type, Fd, InodeVal, Kind, MAX_SYMLINKS, }, - journaling::{DynJournal, JournalEffector}, + journal::{DynJournal, JournalEffector}, os::task::{process::MaybeCheckpointResult, thread::RewindResult}, runtime::task_manager::InlineWaker, utils::store::InstanceSnapshot, @@ -1222,7 +1222,7 @@ pub fn rewind_ext( #[cfg(not(feature = "journal"))] pub fn maybe_snapshot_once( ctx: FunctionEnvMut<'_, WasiEnv>, - _trigger: crate::journaling::SnapshotTrigger, + _trigger: crate::journal::SnapshotTrigger, ) -> WasiResult> { Ok(Ok(ctx)) } @@ -1230,7 +1230,7 @@ pub fn maybe_snapshot_once( #[cfg(feature = "journal")] pub fn maybe_snapshot_once( mut ctx: FunctionEnvMut<'_, WasiEnv>, - trigger: crate::journaling::SnapshotTrigger, + trigger: crate::journal::SnapshotTrigger, ) -> WasiResult> { use crate::os::task::process::{WasiProcessCheckpoint, WasiProcessInner}; @@ -1296,21 +1296,21 @@ pub fn restore_snapshot( mut ctx: FunctionEnvMut<'_, WasiEnv>, journal: Arc, ) -> Result { - use crate::journaling::Journal; + use crate::journal::Journal; let mut is_same_module = false; let mut rewind = None; while let Some(next) = journal.read().map_err(anyhow_err_to_runtime_err)? { tracing::trace!("Restoring snapshot event - {next:?}"); match next { - crate::journaling::JournalEntry::InitModule { wasm_hash } => { + crate::journal::JournalEntry::InitModule { wasm_hash } => { is_same_module = ctx.data().process.module_hash.as_bytes()[0..16] == wasm_hash; } - crate::journaling::JournalEntry::ProcessExit { exit_code } => { + crate::journal::JournalEntry::ProcessExit { exit_code } => { JournalEffector::apply_process_exit(ctx.data(), exit_code) .map_err(anyhow_err_to_runtime_err)?; } - crate::journaling::JournalEntry::FileDescriptorWrite { + crate::journal::JournalEntry::FileDescriptorWrite { fd, offset, data, @@ -1325,18 +1325,18 @@ pub fn restore_snapshot( }) .map_err(anyhow_err_to_runtime_err)?; } - crate::journaling::JournalEntry::FileDescriptorSeek { fd, offset, whence } => { + crate::journal::JournalEntry::FileDescriptorSeek { fd, offset, whence } => { InlineWaker::block_on(JournalEffector::apply_fd_seek(&mut ctx, fd, offset, whence)) .map_err(anyhow_err_to_runtime_err)?; } - crate::journaling::JournalEntry::UpdateMemoryRegion { region, data } => { + crate::journal::JournalEntry::UpdateMemoryRegion { region, data } => { if !is_same_module { continue; } JournalEffector::apply_memory(&mut ctx, region, &data) .map_err(anyhow_err_to_runtime_err)?; } - crate::journaling::JournalEntry::CloseThread { id, exit_code } => { + crate::journal::JournalEntry::CloseThread { id, exit_code } => { if id == ctx.data().tid() { return Err(WasiRuntimeError::Runtime(RuntimeError::user( anyhow::format_err!( @@ -1346,7 +1346,7 @@ pub fn restore_snapshot( ))); } } - crate::journaling::JournalEntry::SetThread { + crate::journal::JournalEntry::SetThread { id, call_stack, memory_stack, @@ -1372,10 +1372,10 @@ pub fn restore_snapshot( ))); } } - crate::journaling::JournalEntry::CloseFileDescriptor { fd } => { + crate::journal::JournalEntry::CloseFileDescriptor { fd } => { JournalEffector::apply_fd_close(&mut ctx, fd).map_err(anyhow_err_to_runtime_err)?; } - crate::journaling::JournalEntry::OpenFileDescriptor { + crate::journal::JournalEntry::OpenFileDescriptor { fd, dirfd, dirflags, @@ -1398,15 +1398,15 @@ pub fn restore_snapshot( ) .map_err(anyhow_err_to_runtime_err)?; } - crate::journaling::JournalEntry::RemoveDirectory { fd, path } => { + crate::journal::JournalEntry::RemoveDirectory { fd, path } => { JournalEffector::apply_path_remove_directory(&mut ctx, fd, &path) .map_err(anyhow_err_to_runtime_err)?; } - crate::journaling::JournalEntry::UnlinkFile { fd, path } => { + crate::journal::JournalEntry::UnlinkFile { fd, path } => { JournalEffector::apply_path_unlink(&mut ctx, fd, &path) .map_err(anyhow_err_to_runtime_err)?; } - crate::journaling::JournalEntry::PathRename { + crate::journal::JournalEntry::PathRename { old_fd, old_path, new_fd, @@ -1415,7 +1415,7 @@ pub fn restore_snapshot( JournalEffector::apply_path_rename(&mut ctx, old_fd, &old_path, new_fd, &new_path) .map_err(anyhow_err_to_runtime_err)?; } - crate::journaling::JournalEntry::Snapshot { + crate::journal::JournalEntry::Snapshot { when: _, trigger: _, } => { @@ -1423,26 +1423,26 @@ pub fn restore_snapshot( continue; } } - crate::journaling::JournalEntry::SetClockTime { clock_id, time } => { + crate::journal::JournalEntry::SetClockTime { clock_id, time } => { JournalEffector::apply_clock_time_set(&mut ctx, clock_id, time) .map_err(anyhow_err_to_runtime_err)?; } - crate::journaling::JournalEntry::RenumberFileDescriptor { old_fd, new_fd } => { + crate::journal::JournalEntry::RenumberFileDescriptor { old_fd, new_fd } => { JournalEffector::apply_fd_renumber(&mut ctx, old_fd, new_fd) .map_err(anyhow_err_to_runtime_err)?; } - crate::journaling::JournalEntry::DuplicateFileDescriptor { + crate::journal::JournalEntry::DuplicateFileDescriptor { original_fd, copied_fd, } => { JournalEffector::apply_fd_duplicate(&mut ctx, original_fd, copied_fd) .map_err(anyhow_err_to_runtime_err)?; } - crate::journaling::JournalEntry::CreateDirectory { fd, path } => { + crate::journal::JournalEntry::CreateDirectory { fd, path } => { JournalEffector::apply_path_create_directory(&mut ctx, fd, &path) .map_err(anyhow_err_to_runtime_err)?; } - crate::journaling::JournalEntry::PathSetTimes { + crate::journal::JournalEntry::PathSetTimes { fd, flags, path, @@ -1455,7 +1455,7 @@ pub fn restore_snapshot( ) .map_err(anyhow_err_to_runtime_err)?; } - crate::journaling::JournalEntry::FileDescriptorSetTimes { + crate::journal::JournalEntry::FileDescriptorSetTimes { fd, st_atim, st_mtim, @@ -1464,15 +1464,15 @@ pub fn restore_snapshot( JournalEffector::apply_fd_set_times(&mut ctx, fd, st_atim, st_mtim, fst_flags) .map_err(anyhow_err_to_runtime_err)?; } - crate::journaling::JournalEntry::FileDescriptorSetSize { fd, st_size } => { + crate::journal::JournalEntry::FileDescriptorSetSize { fd, st_size } => { JournalEffector::apply_fd_set_size(&mut ctx, fd, st_size) .map_err(anyhow_err_to_runtime_err)?; } - crate::journaling::JournalEntry::FileDescriptorSetFlags { fd, flags } => { + crate::journal::JournalEntry::FileDescriptorSetFlags { fd, flags } => { JournalEffector::apply_fd_set_flags(&mut ctx, fd, flags) .map_err(anyhow_err_to_runtime_err)?; } - crate::journaling::JournalEntry::FileDescriptorSetRights { + crate::journal::JournalEntry::FileDescriptorSetRights { fd, fs_rights_base, fs_rights_inheriting, @@ -1485,7 +1485,7 @@ pub fn restore_snapshot( ) .map_err(anyhow_err_to_runtime_err)?; } - crate::journaling::JournalEntry::FileDescriptorAdvise { + crate::journal::JournalEntry::FileDescriptorAdvise { fd, offset, len, @@ -1494,11 +1494,11 @@ pub fn restore_snapshot( JournalEffector::apply_fd_advise(&mut ctx, fd, offset, len, advice) .map_err(anyhow_err_to_runtime_err)?; } - crate::journaling::JournalEntry::FileDescriptorAllocate { fd, offset, len } => { + crate::journal::JournalEntry::FileDescriptorAllocate { fd, offset, len } => { JournalEffector::apply_fd_allocate(&mut ctx, fd, offset, len) .map_err(anyhow_err_to_runtime_err)?; } - crate::journaling::JournalEntry::CreateHardLink { + crate::journal::JournalEntry::CreateHardLink { old_fd, old_path, old_flags, @@ -1510,7 +1510,7 @@ pub fn restore_snapshot( ) .map_err(anyhow_err_to_runtime_err)?; } - crate::journaling::JournalEntry::CreateSymbolicLink { + crate::journal::JournalEntry::CreateSymbolicLink { old_path, fd, new_path, @@ -1518,18 +1518,18 @@ pub fn restore_snapshot( JournalEffector::apply_path_symlink(&mut ctx, &old_path, fd, &new_path) .map_err(anyhow_err_to_runtime_err)?; } - crate::journaling::JournalEntry::ChangeDirectory { path } => { + crate::journal::JournalEntry::ChangeDirectory { path } => { JournalEffector::apply_chdir(&mut ctx, &path).map_err(anyhow_err_to_runtime_err)?; } - crate::journaling::JournalEntry::CreatePipe { fd1, fd2 } => { + crate::journal::JournalEntry::CreatePipe { fd1, fd2 } => { JournalEffector::apply_fd_pipe(&mut ctx, fd1, fd2) .map_err(anyhow_err_to_runtime_err)?; } - crate::journaling::JournalEntry::EpollCreate { fd } => { + crate::journal::JournalEntry::EpollCreate { fd } => { JournalEffector::apply_epoll_create(&mut ctx, fd) .map_err(anyhow_err_to_runtime_err)?; } - crate::journaling::JournalEntry::EpollCtl { + crate::journal::JournalEntry::EpollCtl { epfd, op, fd, @@ -1538,7 +1538,7 @@ pub fn restore_snapshot( JournalEffector::apply_epoll_ctl(&mut ctx, epfd, op, fd, event) .map_err(anyhow_err_to_runtime_err)?; } - crate::journaling::JournalEntry::TtySet { tty, line_feeds } => { + crate::journal::JournalEntry::TtySet { tty, line_feeds } => { JournalEffector::apply_tty_set( &mut ctx, crate::WasiTtyState { @@ -1556,36 +1556,36 @@ pub fn restore_snapshot( ) .map_err(anyhow_err_to_runtime_err)?; } - crate::journaling::JournalEntry::PortAddAddr { cidr } => { + crate::journal::JournalEntry::PortAddAddr { cidr } => { JournalEffector::apply_port_addr_add(&mut ctx, cidr) .map_err(anyhow_err_to_runtime_err)? } - crate::journaling::JournalEntry::PortDelAddr { addr } => { + crate::journal::JournalEntry::PortDelAddr { addr } => { JournalEffector::apply_port_addr_remove(&mut ctx, addr) .map_err(anyhow_err_to_runtime_err)? } - crate::journaling::JournalEntry::PortAddrClear => { + crate::journal::JournalEntry::PortAddrClear => { JournalEffector::apply_port_addr_clear(&mut ctx) .map_err(anyhow_err_to_runtime_err)? } - crate::journaling::JournalEntry::PortBridge { + crate::journal::JournalEntry::PortBridge { network, token, security, } => JournalEffector::apply_port_bridge(&mut ctx, &network, &token, security) .map_err(anyhow_err_to_runtime_err)?, - crate::journaling::JournalEntry::PortUnbridge => { + crate::journal::JournalEntry::PortUnbridge => { JournalEffector::apply_port_unbridge(&mut ctx).map_err(anyhow_err_to_runtime_err)? } - crate::journaling::JournalEntry::PortDhcpAcquire => { + crate::journal::JournalEntry::PortDhcpAcquire => { JournalEffector::apply_port_dhcp_acquire(&mut ctx) .map_err(anyhow_err_to_runtime_err)? } - crate::journaling::JournalEntry::PortGatewaySet { ip } => { + crate::journal::JournalEntry::PortGatewaySet { ip } => { JournalEffector::apply_port_gateway_set(&mut ctx, ip) .map_err(anyhow_err_to_runtime_err)? } - crate::journaling::JournalEntry::PortRouteAdd { + crate::journal::JournalEntry::PortRouteAdd { cidr, via_router, preferred_until, @@ -1598,31 +1598,31 @@ pub fn restore_snapshot( expires_at, ) .map_err(anyhow_err_to_runtime_err)?, - crate::journaling::JournalEntry::PortRouteClear => { + crate::journal::JournalEntry::PortRouteClear => { JournalEffector::apply_port_route_clear(&mut ctx) .map_err(anyhow_err_to_runtime_err)? } - crate::journaling::JournalEntry::PortRouteDel { ip } => { + crate::journal::JournalEntry::PortRouteDel { ip } => { JournalEffector::apply_port_route_remove(&mut ctx, ip) .map_err(anyhow_err_to_runtime_err)? } - crate::journaling::JournalEntry::SocketOpen { af, ty, pt, fd } => { + crate::journal::JournalEntry::SocketOpen { af, ty, pt, fd } => { JournalEffector::apply_sock_open(&mut ctx, af, ty, pt, fd) .map_err(anyhow_err_to_runtime_err)? } - crate::journaling::JournalEntry::SocketListen { fd, backlog } => { + crate::journal::JournalEntry::SocketListen { fd, backlog } => { JournalEffector::apply_sock_listen(&mut ctx, fd, backlog as usize) .map_err(anyhow_err_to_runtime_err)? } - crate::journaling::JournalEntry::SocketBind { fd, addr } => { + crate::journal::JournalEntry::SocketBind { fd, addr } => { JournalEffector::apply_sock_bind(&mut ctx, fd, addr) .map_err(anyhow_err_to_runtime_err)? } - crate::journaling::JournalEntry::SocketConnected { fd, addr } => { + crate::journal::JournalEntry::SocketConnected { fd, addr } => { JournalEffector::apply_sock_connect(&mut ctx, fd, addr) .map_err(anyhow_err_to_runtime_err)? } - crate::journaling::JournalEntry::SocketAccepted { + crate::journal::JournalEntry::SocketAccepted { listen_fd, fd, peer_addr, @@ -1637,31 +1637,31 @@ pub fn restore_snapshot( nonblocking, ) .map_err(anyhow_err_to_runtime_err)?, - crate::journaling::JournalEntry::SocketJoinIpv4Multicast { + crate::journal::JournalEntry::SocketJoinIpv4Multicast { fd, multiaddr, iface, } => JournalEffector::apply_sock_join_ipv4_multicast(&mut ctx, fd, multiaddr, iface) .map_err(anyhow_err_to_runtime_err)?, - crate::journaling::JournalEntry::SocketJoinIpv6Multicast { + crate::journal::JournalEntry::SocketJoinIpv6Multicast { fd, multiaddr, iface, } => JournalEffector::apply_sock_join_ipv6_multicast(&mut ctx, fd, multiaddr, iface) .map_err(anyhow_err_to_runtime_err)?, - crate::journaling::JournalEntry::SocketLeaveIpv4Multicast { + crate::journal::JournalEntry::SocketLeaveIpv4Multicast { fd, multiaddr, iface, } => JournalEffector::apply_sock_leave_ipv4_multicast(&mut ctx, fd, multiaddr, iface) .map_err(anyhow_err_to_runtime_err)?, - crate::journaling::JournalEntry::SocketLeaveIpv6Multicast { + crate::journal::JournalEntry::SocketLeaveIpv6Multicast { fd, multiaddr, iface, } => JournalEffector::apply_sock_leave_ipv6_multicast(&mut ctx, fd, multiaddr, iface) .map_err(anyhow_err_to_runtime_err)?, - crate::journaling::JournalEntry::SocketSendFile { + crate::journal::JournalEntry::SocketSendFile { socket_fd, file_fd, offset, @@ -1670,7 +1670,7 @@ pub fn restore_snapshot( &mut ctx, socket_fd, file_fd, offset, count, )) .map_err(anyhow_err_to_runtime_err)?, - crate::journaling::JournalEntry::SocketSendTo { + crate::journal::JournalEntry::SocketSendTo { fd, data, flags, @@ -1686,7 +1686,7 @@ pub fn restore_snapshot( } }) .map_err(anyhow_err_to_runtime_err)?, - crate::journaling::JournalEntry::SocketSend { + crate::journal::JournalEntry::SocketSend { fd, data, flags, @@ -1699,23 +1699,23 @@ pub fn restore_snapshot( } }) .map_err(anyhow_err_to_runtime_err)?, - crate::journaling::JournalEntry::SocketSetOptFlag { fd, opt, flag } => { + crate::journal::JournalEntry::SocketSetOptFlag { fd, opt, flag } => { JournalEffector::apply_sock_set_opt_flag(&mut ctx, fd, opt, flag) .map_err(anyhow_err_to_runtime_err)? } - crate::journaling::JournalEntry::SocketSetOptSize { fd, opt, size } => { + crate::journal::JournalEntry::SocketSetOptSize { fd, opt, size } => { JournalEffector::apply_sock_set_opt_size(&mut ctx, fd, opt, size) .map_err(anyhow_err_to_runtime_err)? } - crate::journaling::JournalEntry::SocketSetOptTime { fd, ty, time } => { - JournalEffector::apply_sock_set_opt_time(&mut ctx, fd, ty, time) + crate::journal::JournalEntry::SocketSetOptTime { fd, ty, time } => { + JournalEffector::apply_sock_set_opt_time(&mut ctx, fd, ty.into(), time) .map_err(anyhow_err_to_runtime_err)? } - crate::journaling::JournalEntry::SocketShutdown { fd, how } => { - JournalEffector::apply_sock_shutdown(&mut ctx, fd, how) + crate::journal::JournalEntry::SocketShutdown { fd, how } => { + JournalEffector::apply_sock_shutdown(&mut ctx, fd, how.into()) .map_err(anyhow_err_to_runtime_err)? } - crate::journaling::JournalEntry::CreateEvent { + crate::journal::JournalEntry::CreateEvent { initial_val, flags, fd, diff --git a/lib/wasix/src/syscalls/wasi/environ_get.rs b/lib/wasix/src/syscalls/wasi/environ_get.rs index 201c73440b2..a39a5faa5f6 100644 --- a/lib/wasix/src/syscalls/wasi/environ_get.rs +++ b/lib/wasix/src/syscalls/wasi/environ_get.rs @@ -1,5 +1,5 @@ use super::*; -use crate::{journaling::SnapshotTrigger, syscalls::*}; +use crate::{journal::SnapshotTrigger, syscalls::*}; /// ### `environ_get()` /// Read environment variable data. diff --git a/lib/wasix/src/syscalls/wasi/environ_sizes_get.rs b/lib/wasix/src/syscalls/wasi/environ_sizes_get.rs index 9a68b784d4f..846b8e92839 100644 --- a/lib/wasix/src/syscalls/wasi/environ_sizes_get.rs +++ b/lib/wasix/src/syscalls/wasi/environ_sizes_get.rs @@ -1,5 +1,5 @@ use super::*; -use crate::{journaling::SnapshotTrigger, syscalls::*}; +use crate::{journal::SnapshotTrigger, syscalls::*}; /// ### `environ_sizes_get()` /// Return command-line argument data sizes. diff --git a/lib/wasix/src/syscalls/wasi/fd_read.rs b/lib/wasix/src/syscalls/wasi/fd_read.rs index cc26a529543..2d16eda58b4 100644 --- a/lib/wasix/src/syscalls/wasi/fd_read.rs +++ b/lib/wasix/src/syscalls/wasi/fd_read.rs @@ -5,7 +5,7 @@ use virtual_fs::{AsyncReadExt, DeviceFile, ReadBuf}; use super::*; use crate::{ fs::NotificationInner, - journaling::SnapshotTrigger, + journal::SnapshotTrigger, net::socket::TimeType, os::task::process::{MaybeCheckpointResult, WasiProcessCheckpoint, WasiProcessInner}, syscalls::*, diff --git a/lib/wasix/src/syscalls/wasi/fd_write.rs b/lib/wasix/src/syscalls/wasi/fd_write.rs index 1f70b4ce67b..e586d897163 100644 --- a/lib/wasix/src/syscalls/wasi/fd_write.rs +++ b/lib/wasix/src/syscalls/wasi/fd_write.rs @@ -3,7 +3,7 @@ use std::task::Waker; use super::*; #[cfg(feature = "journal")] use crate::{ - journaling::{JournalEffector, JournalEntry}, + journal::{JournalEffector, JournalEntry}, utils::map_snapshot_err, }; use crate::{net::socket::TimeType, syscalls::*}; diff --git a/lib/wasix/src/syscalls/wasix/sock_listen.rs b/lib/wasix/src/syscalls/wasix/sock_listen.rs index 5ec2836c69d..1bc969ebc28 100644 --- a/lib/wasix/src/syscalls/wasix/sock_listen.rs +++ b/lib/wasix/src/syscalls/wasix/sock_listen.rs @@ -1,5 +1,5 @@ use super::*; -use crate::{journaling::SnapshotTrigger, syscalls::*}; +use crate::{journal::SnapshotTrigger, syscalls::*}; /// ### `sock_listen()` /// Listen for connections on a socket diff --git a/lib/wasix/src/syscalls/wasix/thread_spawn.rs b/lib/wasix/src/syscalls/wasix/thread_spawn.rs index 048c35c8514..ee7264b3009 100644 --- a/lib/wasix/src/syscalls/wasix/thread_spawn.rs +++ b/lib/wasix/src/syscalls/wasix/thread_spawn.rs @@ -2,7 +2,7 @@ use std::f32::consts::E; use super::*; #[cfg(feature = "journal")] -use crate::journaling::JournalEffector; +use crate::journal::JournalEffector; use crate::{ capture_instance_snapshot, os::task::thread::WasiMemoryLayout, From 0f1a9fac99a4b330250f7da6351f58dc44b1db5e Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Thu, 23 Nov 2023 17:39:30 +1100 Subject: [PATCH 070/129] Added a couple more unit tests --- lib/wasix/src/journal/concrete/compacting.rs | 58 +++++++++++++++++++- 1 file changed, 57 insertions(+), 1 deletion(-) diff --git a/lib/wasix/src/journal/concrete/compacting.rs b/lib/wasix/src/journal/concrete/compacting.rs index 935603247de..dd5d25eb302 100644 --- a/lib/wasix/src/journal/concrete/compacting.rs +++ b/lib/wasix/src/journal/concrete/compacting.rs @@ -390,7 +390,7 @@ impl Journal for CompactingJournal { mod tests { use super::*; - use wasmer_wasix_types::wasi; + use wasmer_wasix_types::wasi::{self, Tty}; pub fn run_test<'a>( in_records: Vec>, @@ -584,6 +584,10 @@ mod tests { store_data: [66u8; 70].to_vec().into(), is_64bit: true, }, + JournalEntry::Snapshot { + when: SystemTime::now(), + trigger: SnapshotTrigger::FirstListen, + }, JournalEntry::OpenFileDescriptor { fd: 1234, dirfd: 3452345, @@ -831,4 +835,56 @@ mod tests { ) .unwrap() } + + #[tracing_test::traced_test] + #[test] + pub fn test_compact_duplicate_tty() { + run_test( + vec![ + JournalEntry::TtySet { + tty: Tty { + cols: 123, + rows: 65, + width: 2341, + height: 573457, + stdin_tty: true, + stdout_tty: true, + stderr_tty: true, + echo: true, + line_buffered: true, + }, + line_feeds: true, + }, + JournalEntry::TtySet { + tty: Tty { + cols: 12, + rows: 65, + width: 2341, + height: 573457, + stdin_tty: true, + stdout_tty: false, + stderr_tty: true, + echo: true, + line_buffered: true, + }, + line_feeds: true, + }, + ], + vec![JournalEntry::TtySet { + tty: Tty { + cols: 12, + rows: 65, + width: 2341, + height: 573457, + stdin_tty: true, + stdout_tty: false, + stderr_tty: true, + echo: true, + line_buffered: true, + }, + line_feeds: true, + }], + ) + .unwrap() + } } From 38d74beb1ea254dcc6047d0604d303a90b46ada2 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Thu, 23 Nov 2023 17:47:44 +1100 Subject: [PATCH 071/129] Some linting and compile fixes --- lib/cli/src/cli.rs | 8 +++++--- lib/cli/src/commands/mod.rs | 6 ++++-- lib/wasi-web/Cargo.lock | 5 +++-- lib/wasix/src/journal/concrete/mod.rs | 2 ++ 4 files changed, 14 insertions(+), 7 deletions(-) diff --git a/lib/cli/src/cli.rs b/lib/cli/src/cli.rs index 7fb022f6798..c21155e1365 100644 --- a/lib/cli/src/cli.rs +++ b/lib/cli/src/cli.rs @@ -2,6 +2,8 @@ #[cfg(target_os = "linux")] use crate::commands::Binfmt; +#[cfg(feature = "journal")] +use crate::commands::CmdJournal; #[cfg(feature = "compiler")] use crate::commands::Compile; #[cfg(any(feature = "static-artifact-create", feature = "wasmer-artifact-create"))] @@ -9,8 +11,7 @@ use crate::commands::CreateExe; #[cfg(feature = "wast")] use crate::commands::Wast; use crate::commands::{ - Add, Cache, CmdJournal, Config, Init, Inspect, Login, Package, Publish, Run, SelfUpdate, - Validate, Whoami, + Add, Cache, Config, Init, Inspect, Login, Package, Publish, Run, SelfUpdate, Validate, Whoami, }; #[cfg(feature = "static-artifact-create")] use crate::commands::{CreateObj, GenCHeader}; @@ -131,6 +132,7 @@ impl Args { // Deploy commands. Some(Cmd::Deploy(c)) => c.run(), Some(Cmd::App(apps)) => apps.run(), + #[cfg(feature = "journal")] Some(Cmd::Journal(journal)) => journal.run(), Some(Cmd::Ssh(ssh)) => ssh.run(), Some(Cmd::Namespace(namespace)) => namespace.run(), @@ -269,8 +271,8 @@ enum Cmd { Run(Run), /// Manage journals (compacting, inspecting, filtering, ...) - #[clap(subcommand)] #[cfg(feature = "journal")] + #[clap(subcommand)] Journal(CmdJournal), #[clap(subcommand)] diff --git a/lib/cli/src/commands/mod.rs b/lib/cli/src/commands/mod.rs index cfe6a439153..9db38db12e7 100644 --- a/lib/cli/src/commands/mod.rs +++ b/lib/cli/src/commands/mod.rs @@ -27,9 +27,11 @@ mod validate; mod wast; mod whoami; +#[cfg(feature = "journal")] +pub use self::journal::*; pub use self::{ - add::*, cache::*, config::*, container::*, init::*, inspect::*, journal::*, login::*, - package::*, publish::*, run::Run, self_update::*, validate::*, whoami::*, + add::*, cache::*, config::*, container::*, init::*, inspect::*, login::*, package::*, + publish::*, run::Run, self_update::*, validate::*, whoami::*, }; #[cfg(target_os = "linux")] diff --git a/lib/wasi-web/Cargo.lock b/lib/wasi-web/Cargo.lock index 0e5c18b149e..ae34bf9dcb0 100644 --- a/lib/wasi-web/Cargo.lock +++ b/lib/wasi-web/Cargo.lock @@ -755,9 +755,9 @@ dependencies = [ [[package]] name = "gimli" -version = "0.28.0" +version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0" +checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" [[package]] name = "glob" @@ -2566,6 +2566,7 @@ version = "0.16.0" dependencies = [ "anyhow", "async-trait", + "base64 0.21.5", "bincode", "bytecheck", "bytes", diff --git a/lib/wasix/src/journal/concrete/mod.rs b/lib/wasix/src/journal/concrete/mod.rs index a1a3108afac..1e142ab02eb 100644 --- a/lib/wasix/src/journal/concrete/mod.rs +++ b/lib/wasix/src/journal/concrete/mod.rs @@ -2,6 +2,7 @@ mod archived; mod boxed; mod buffered; mod compacting; +#[cfg(feature = "journal")] mod compacting_log_file; mod filter; #[cfg(feature = "journal")] @@ -18,6 +19,7 @@ pub use archived::*; pub use boxed::*; pub use buffered::*; pub use compacting::*; +#[cfg(feature = "journal")] pub use compacting_log_file::*; pub use filter::*; #[cfg(feature = "journal")] From 0d728c3e24ccd767efd10c9f31163d85c25ddd78 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Thu, 23 Nov 2023 19:29:03 +1100 Subject: [PATCH 072/129] Fix for lints --- lib/wasix/src/journal/base64.rs | 4 ++-- lib/wasix/src/journal/concrete/archived.rs | 12 ++++++------ lib/wasix/src/journal/concrete/buffered.rs | 6 +++--- lib/wasix/src/journal/concrete/compacting.rs | 4 ++-- .../src/journal/concrete/compacting_log_file.rs | 2 +- 5 files changed, 14 insertions(+), 14 deletions(-) diff --git a/lib/wasix/src/journal/base64.rs b/lib/wasix/src/journal/base64.rs index c6ade6192ef..cc4cfb702c7 100644 --- a/lib/wasix/src/journal/base64.rs +++ b/lib/wasix/src/journal/base64.rs @@ -3,7 +3,7 @@ use std::borrow::Cow; use serde::{Deserialize, Serialize}; use serde::{Deserializer, Serializer}; -pub fn serialize(v: &Cow<'_, [u8]>, s: S) -> Result { +pub fn serialize(v: &[u8], s: S) -> Result { #[allow(deprecated)] let base64 = base64::encode(v); String::serialize(&base64, s) @@ -13,6 +13,6 @@ pub fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result let base64 = String::deserialize(d)?; #[allow(deprecated)] base64::decode(base64.as_bytes()) - .map_err(|e| serde::de::Error::custom(e)) + .map_err(serde::de::Error::custom) .map(|d| d.into()) } diff --git a/lib/wasix/src/journal/concrete/archived.rs b/lib/wasix/src/journal/concrete/archived.rs index 31689ae41ef..ee4598db55c 100644 --- a/lib/wasix/src/journal/concrete/archived.rs +++ b/lib/wasix/src/journal/concrete/archived.rs @@ -665,12 +665,12 @@ impl<'a> JournalEntry<'a> { rows: tty.rows, width: tty.width, height: tty.height, - stdin_tty: tty.stdin_tty.into(), - stdout_tty: tty.stdout_tty.into(), - stderr_tty: tty.stderr_tty.into(), - echo: tty.echo.into(), - line_buffered: tty.line_buffered.into(), - line_feeds: line_feeds.into(), + stdin_tty: tty.stdin_tty, + stdout_tty: tty.stdout_tty, + stderr_tty: tty.stderr_tty, + echo: tty.echo, + line_buffered: tty.line_buffered, + line_feeds: line_feeds, }) } JournalEntry::CreatePipe { fd1, fd2 } => { diff --git a/lib/wasix/src/journal/concrete/buffered.rs b/lib/wasix/src/journal/concrete/buffered.rs index 76c905c5821..67b16631120 100644 --- a/lib/wasix/src/journal/concrete/buffered.rs +++ b/lib/wasix/src/journal/concrete/buffered.rs @@ -27,14 +27,14 @@ pub struct BufferedJournalTx { state: Arc>, } -impl BufferedJournal { - pub fn new() -> Self { +impl Default for BufferedJournal { + fn default() -> Self { let state = Arc::new(Mutex::new(State::default())); Self { tx: BufferedJournalTx { state: state.clone(), }, - rx: BufferedJournalRx { state: state }, + rx: BufferedJournalRx { state }, } } } diff --git a/lib/wasix/src/journal/concrete/compacting.rs b/lib/wasix/src/journal/concrete/compacting.rs index dd5d25eb302..9c1f0a7d639 100644 --- a/lib/wasix/src/journal/concrete/compacting.rs +++ b/lib/wasix/src/journal/concrete/compacting.rs @@ -397,13 +397,13 @@ mod tests { out_records: Vec>, ) -> anyhow::Result<()> { // Build a journal that will store the records before compacting - let mut compacting_journal = CompactingJournal::new(BufferedJournal::new())?; + let mut compacting_journal = CompactingJournal::new(BufferedJournal::default())?; for record in in_records { compacting_journal.write(record)?; } // Now we build a new one using the compactor - let new_journal = BufferedJournal::new(); + let new_journal = BufferedJournal::default(); compacting_journal.compact_to(new_journal)?; // Read the records diff --git a/lib/wasix/src/journal/concrete/compacting_log_file.rs b/lib/wasix/src/journal/concrete/compacting_log_file.rs index 18de2ea0f86..3ee8078f694 100644 --- a/lib/wasix/src/journal/concrete/compacting_log_file.rs +++ b/lib/wasix/src/journal/concrete/compacting_log_file.rs @@ -63,7 +63,7 @@ impl CompactingLogFileJournal { .to_string_lossy() .to_string(); temp_filename.insert_str(0, ".compacting."); - let temp_path = path.as_ref().clone().with_file_name(&temp_filename); + let temp_path = path.as_ref().with_file_name(&temp_filename); let state = Arc::new(Mutex::new(State { on_drop: false, From 7875068547b771f8157bdfa628bc590f959aaa59 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Thu, 23 Nov 2023 19:55:41 +1100 Subject: [PATCH 073/129] Another linting fix --- lib/wasix/src/journal/concrete/archived.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/wasix/src/journal/concrete/archived.rs b/lib/wasix/src/journal/concrete/archived.rs index ee4598db55c..d63207796e1 100644 --- a/lib/wasix/src/journal/concrete/archived.rs +++ b/lib/wasix/src/journal/concrete/archived.rs @@ -670,7 +670,7 @@ impl<'a> JournalEntry<'a> { stderr_tty: tty.stderr_tty, echo: tty.echo, line_buffered: tty.line_buffered, - line_feeds: line_feeds, + line_feeds, }) } JournalEntry::CreatePipe { fd1, fd2 } => { From bce86dde4cc9fca201c3e55832880e1d49cc0b0a Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Fri, 24 Nov 2023 00:41:19 +1100 Subject: [PATCH 074/129] Added better error handling that reduces a panic --- lib/api/src/errors.rs | 4 +-- lib/api/src/exports.rs | 2 +- lib/types/src/error.rs | 2 +- lib/wasix/src/lib.rs | 2 ++ lib/wasix/src/os/task/thread.rs | 4 +-- lib/wasix/src/runners/wasi.rs | 48 +++++++++++++++++++++++++++++++-- lib/wasix/src/state/builder.rs | 2 +- lib/wasix/src/state/func_env.rs | 4 ++- 8 files changed, 58 insertions(+), 10 deletions(-) diff --git a/lib/api/src/errors.rs b/lib/api/src/errors.rs index 8dff0864a24..8c3509598d3 100644 --- a/lib/api/src/errors.rs +++ b/lib/api/src/errors.rs @@ -17,7 +17,7 @@ use wasmer_types::ImportError; /// This is based on the [link error][link-error] API. /// /// [link-error]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/WebAssembly/LinkError -#[derive(Debug)] +#[derive(Debug, Clone)] #[cfg_attr(feature = "std", derive(Error))] #[cfg_attr(feature = "std", error("Link error: {0}"))] pub enum LinkError { @@ -41,7 +41,7 @@ pub enum LinkError { /// Trap that occurs when calling the WebAssembly module /// start function, and an error when initializing the user's /// host environments. -#[derive(Debug)] +#[derive(Debug, Clone)] #[cfg_attr(feature = "std", derive(Error))] pub enum InstantiationError { /// A linking ocurred during instantiation. diff --git a/lib/api/src/exports.rs b/lib/api/src/exports.rs index e6eecf7e301..d2f821145f2 100644 --- a/lib/api/src/exports.rs +++ b/lib/api/src/exports.rs @@ -42,7 +42,7 @@ use thiserror::Error; /// // This results with an error: `ExportError::Missing`. /// let export = instance.exports.get_function("unknown").unwrap(); /// ``` -#[derive(Error, Debug)] +#[derive(Error, Debug, Clone)] pub enum ExportError { /// An error than occurs when the exported type and the expected type /// are incompatible. diff --git a/lib/types/src/error.rs b/lib/types/src/error.rs index 1ae02dda022..36b2c93ba30 100644 --- a/lib/types/src/error.rs +++ b/lib/types/src/error.rs @@ -95,7 +95,7 @@ pub enum MemoryError { /// /// Note: this error is not standard to WebAssembly, but it's /// useful to determine the import issue on the API side. -#[derive(Error, Debug)] +#[derive(Error, Debug, Clone)] pub enum ImportError { /// Incompatible Import Type. /// This error occurs when the import types mismatch. diff --git a/lib/wasix/src/lib.rs b/lib/wasix/src/lib.rs index 06f8f646778..3e1fc5f3d57 100644 --- a/lib/wasix/src/lib.rs +++ b/lib/wasix/src/lib.rs @@ -209,6 +209,8 @@ pub enum WasiRuntimeError { Runtime(#[from] RuntimeError), #[error("Memory access error")] Thread(#[from] WasiThreadError), + #[error("{0}")] + Anyhow(#[from] Arc), } impl WasiRuntimeError { diff --git a/lib/wasix/src/os/task/thread.rs b/lib/wasix/src/os/task/thread.rs index 5dfe7f4f11f..94ee92a4ec4 100644 --- a/lib/wasix/src/os/task/thread.rs +++ b/lib/wasix/src/os/task/thread.rs @@ -545,7 +545,7 @@ impl std::ops::Deref for WasiThreadHandle { } } -#[derive(thiserror::Error, Debug)] +#[derive(thiserror::Error, Debug, Clone)] pub enum WasiThreadError { #[error("Multithreading is not supported")] Unsupported, @@ -559,7 +559,7 @@ pub enum WasiThreadError { // Note: Boxed so we can keep the error size down InstanceCreateFailed(Box), #[error("Initialization function failed - {0}")] - InitFailed(anyhow::Error), + InitFailed(Arc), /// This will happen if WASM is running in a thread has not been created by the spawn_wasm call #[error("WASM context is invalid")] InvalidWasmContext, diff --git a/lib/wasix/src/runners/wasi.rs b/lib/wasix/src/runners/wasi.rs index 9329ea9237d..c4b429a731a 100644 --- a/lib/wasix/src/runners/wasi.rs +++ b/lib/wasix/src/runners/wasi.rs @@ -14,7 +14,7 @@ use crate::{ journal::{DynJournal, SnapshotTrigger}, runners::{wasi_common::CommonWasiOptions, MappedDirectory}, runtime::{module_cache::ModuleHash, task_manager::VirtualTaskManagerExt}, - Runtime, WasiEnvBuilder, WasiRuntimeError, + Runtime, WasiEnvBuilder, WasiError, WasiRuntimeError, }; use super::wasi_common::MappedCommand; @@ -361,7 +361,51 @@ impl crate::runners::Runner for WasiRunner { task_handle .wait_finished() .await - .map_err(|err| Arc::into_inner(err).expect("Error shouldn't be shared")) + .map_err(|err| { + // We do our best to recover the error + let msg = err.to_string(); + let weak = Arc::downgrade(&err); + Arc::into_inner(err).unwrap_or_else(|| { + weak.upgrade() + .map(|err| match err.as_ref() { + WasiRuntimeError::Init(a) => WasiRuntimeError::Init(a.clone()), + WasiRuntimeError::Export(a) => { + WasiRuntimeError::Export(a.clone()) + } + WasiRuntimeError::Instantiation(a) => { + WasiRuntimeError::Instantiation(a.clone()) + } + WasiRuntimeError::Wasi(WasiError::Exit(a)) => { + WasiRuntimeError::Wasi(WasiError::Exit(a.clone())) + } + WasiRuntimeError::Wasi(WasiError::UnknownWasiVersion) => { + WasiRuntimeError::Wasi(WasiError::UnknownWasiVersion) + } + WasiRuntimeError::Wasi(WasiError::DeepSleep(_)) => { + WasiRuntimeError::Anyhow(Arc::new(anyhow::format_err!( + "deep-sleep" + ))) + } + WasiRuntimeError::ControlPlane(a) => { + WasiRuntimeError::ControlPlane(a.clone()) + } + WasiRuntimeError::Runtime(a) => { + WasiRuntimeError::Runtime(a.clone()) + } + WasiRuntimeError::Thread(a) => { + WasiRuntimeError::Thread(a.clone()) + } + WasiRuntimeError::Anyhow(a) => { + WasiRuntimeError::Anyhow(a.clone()) + } + }) + .unwrap_or_else(|| { + WasiRuntimeError::Anyhow(Arc::new(anyhow::format_err!( + "{}", msg + ))) + }) + }) + }) .context("Unable to wait for the process to exit") } .in_current_span(), diff --git a/lib/wasix/src/state/builder.rs b/lib/wasix/src/state/builder.rs index b93ad047f21..b0acd9fcbcd 100644 --- a/lib/wasix/src/state/builder.rs +++ b/lib/wasix/src/state/builder.rs @@ -105,7 +105,7 @@ impl std::fmt::Debug for WasiEnvBuilder { } /// Error type returned when bad data is given to [`WasiEnvBuilder`]. -#[derive(Error, Debug, PartialEq, Eq)] +#[derive(Error, Debug, Clone, PartialEq, Eq)] pub enum WasiStateCreationError { #[error("bad environment variable format: `{0}`")] EnvironmentVariableFormatError(String), diff --git a/lib/wasix/src/state/func_env.rs b/lib/wasix/src/state/func_env.rs index 0a3983e78f5..35ed4f1b7ef 100644 --- a/lib/wasix/src/state/func_env.rs +++ b/lib/wasix/src/state/func_env.rs @@ -1,3 +1,5 @@ +use std::sync::Arc; + use tracing::trace; use wasmer::{ AsStoreMut, AsStoreRef, ExportError, FunctionEnv, Imports, Instance, Memory, Module, Store, @@ -61,7 +63,7 @@ impl WasiFunctionEnv { init(&instance, &store).map_err(|err| { tracing::warn!("failed to init instance - {}", err); - WasiThreadError::InitFailed(err) + WasiThreadError::InitFailed(Arc::new(err)) })?; // Initialize the WASI environment From b6eb4f65b4765ed46c0ea4fea20a784b39a75716 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Fri, 24 Nov 2023 00:45:01 +1100 Subject: [PATCH 075/129] Fixed a runtime in runtime bug on journal replays --- .../src/journal/effector/syscalls/fd_seek.rs | 2 +- .../src/journal/effector/syscalls/fd_write.rs | 2 +- .../journal/effector/syscalls/sock_send.rs | 2 +- .../effector/syscalls/sock_send_file.rs | 2 +- .../journal/effector/syscalls/sock_send_to.rs | 2 +- lib/wasix/src/syscalls/mod.rs | 46 ++++++++----------- 6 files changed, 23 insertions(+), 33 deletions(-) diff --git a/lib/wasix/src/journal/effector/syscalls/fd_seek.rs b/lib/wasix/src/journal/effector/syscalls/fd_seek.rs index 039d4808b83..16baa705eea 100644 --- a/lib/wasix/src/journal/effector/syscalls/fd_seek.rs +++ b/lib/wasix/src/journal/effector/syscalls/fd_seek.rs @@ -10,7 +10,7 @@ impl JournalEffector { Self::save_event(ctx, JournalEntry::FileDescriptorSeek { fd, offset, whence }) } - pub async fn apply_fd_seek( + pub fn apply_fd_seek( ctx: &mut FunctionEnvMut<'_, WasiEnv>, fd: Fd, offset: i64, diff --git a/lib/wasix/src/journal/effector/syscalls/fd_write.rs b/lib/wasix/src/journal/effector/syscalls/fd_write.rs index 8fc90d26814..a9f449457fc 100644 --- a/lib/wasix/src/journal/effector/syscalls/fd_write.rs +++ b/lib/wasix/src/journal/effector/syscalls/fd_write.rs @@ -40,7 +40,7 @@ impl JournalEffector { Ok(()) } - pub async fn apply_fd_write( + pub fn apply_fd_write( ctx: &FunctionEnvMut<'_, WasiEnv>, fd: Fd, offset: u64, diff --git a/lib/wasix/src/journal/effector/syscalls/sock_send.rs b/lib/wasix/src/journal/effector/syscalls/sock_send.rs index 67f106bd8c1..8b5ed27e6b3 100644 --- a/lib/wasix/src/journal/effector/syscalls/sock_send.rs +++ b/lib/wasix/src/journal/effector/syscalls/sock_send.rs @@ -44,7 +44,7 @@ impl JournalEffector { Ok(()) } - pub async fn apply_sock_send( + pub fn apply_sock_send( ctx: &FunctionEnvMut<'_, WasiEnv>, sock: Fd, si_data: Cow<'_, [u8]>, diff --git a/lib/wasix/src/journal/effector/syscalls/sock_send_file.rs b/lib/wasix/src/journal/effector/syscalls/sock_send_file.rs index 750c8d17075..e3abe981dbc 100644 --- a/lib/wasix/src/journal/effector/syscalls/sock_send_file.rs +++ b/lib/wasix/src/journal/effector/syscalls/sock_send_file.rs @@ -21,7 +21,7 @@ impl JournalEffector { ) } - pub async fn apply_sock_send_file( + pub fn apply_sock_send_file( ctx: &mut FunctionEnvMut<'_, WasiEnv>, socket_fd: Fd, file_fd: Fd, diff --git a/lib/wasix/src/journal/effector/syscalls/sock_send_to.rs b/lib/wasix/src/journal/effector/syscalls/sock_send_to.rs index 47d9b38c780..7bf0fd671e2 100644 --- a/lib/wasix/src/journal/effector/syscalls/sock_send_to.rs +++ b/lib/wasix/src/journal/effector/syscalls/sock_send_to.rs @@ -47,7 +47,7 @@ impl JournalEffector { Ok(()) } - pub async fn apply_sock_send_to( + pub fn apply_sock_send_to( ctx: &FunctionEnvMut<'_, WasiEnv>, sock: Fd, si_data: Cow<'_, [u8]>, diff --git a/lib/wasix/src/syscalls/mod.rs b/lib/wasix/src/syscalls/mod.rs index 9b8288e77a0..baf194f78aa 100644 --- a/lib/wasix/src/syscalls/mod.rs +++ b/lib/wasix/src/syscalls/mod.rs @@ -1316,17 +1316,15 @@ pub fn restore_snapshot( data, is_64bit, } => { - InlineWaker::block_on(async { - if is_64bit { - JournalEffector::apply_fd_write::(&ctx, fd, offset, data).await - } else { - JournalEffector::apply_fd_write::(&ctx, fd, offset, data).await - } - }) + if is_64bit { + JournalEffector::apply_fd_write::(&ctx, fd, offset, data) + } else { + JournalEffector::apply_fd_write::(&ctx, fd, offset, data) + } .map_err(anyhow_err_to_runtime_err)?; } crate::journal::JournalEntry::FileDescriptorSeek { fd, offset, whence } => { - InlineWaker::block_on(JournalEffector::apply_fd_seek(&mut ctx, fd, offset, whence)) + JournalEffector::apply_fd_seek(&mut ctx, fd, offset, whence) .map_err(anyhow_err_to_runtime_err)?; } crate::journal::JournalEntry::UpdateMemoryRegion { region, data } => { @@ -1666,38 +1664,30 @@ pub fn restore_snapshot( file_fd, offset, count, - } => InlineWaker::block_on(JournalEffector::apply_sock_send_file( - &mut ctx, socket_fd, file_fd, offset, count, - )) - .map_err(anyhow_err_to_runtime_err)?, + } => JournalEffector::apply_sock_send_file(&mut ctx, socket_fd, file_fd, offset, count) + .map_err(anyhow_err_to_runtime_err)?, crate::journal::JournalEntry::SocketSendTo { fd, data, flags, addr, is_64bit, - } => InlineWaker::block_on(async { - if is_64bit { - JournalEffector::apply_sock_send_to::(&ctx, fd, data, flags, addr) - .await - } else { - JournalEffector::apply_sock_send_to::(&ctx, fd, data, flags, addr) - .await - } - }) + } => if is_64bit { + JournalEffector::apply_sock_send_to::(&ctx, fd, data, flags, addr) + } else { + JournalEffector::apply_sock_send_to::(&ctx, fd, data, flags, addr) + } .map_err(anyhow_err_to_runtime_err)?, crate::journal::JournalEntry::SocketSend { fd, data, flags, is_64bit, - } => InlineWaker::block_on(async { - if is_64bit { - JournalEffector::apply_sock_send::(&ctx, fd, data, flags).await - } else { - JournalEffector::apply_sock_send::(&ctx, fd, data, flags).await - } - }) + } => if is_64bit { + JournalEffector::apply_sock_send::(&ctx, fd, data, flags) + } else { + JournalEffector::apply_sock_send::(&ctx, fd, data, flags) + } .map_err(anyhow_err_to_runtime_err)?, crate::journal::JournalEntry::SocketSetOptFlag { fd, opt, flag } => { JournalEffector::apply_sock_set_opt_flag(&mut ctx, fd, opt, flag) From 4f6a193d4587f9f4c7a3bf231e9942618a7c9bf1 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Fri, 24 Nov 2023 00:47:21 +1100 Subject: [PATCH 076/129] Missing rewinds no long cause the process to fail --- lib/wasix/src/state/builder.rs | 2 +- lib/wasix/src/syscalls/mod.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/wasix/src/state/builder.rs b/lib/wasix/src/state/builder.rs index b0acd9fcbcd..12daa769d16 100644 --- a/lib/wasix/src/state/builder.rs +++ b/lib/wasix/src/state/builder.rs @@ -1013,7 +1013,7 @@ impl WasiEnvBuilder { for journal in journals { let ctx = env.env.clone().into_mut(&mut store); let rewind = restore_snapshot(ctx, journal)?; - rewind_state = Some((rewind, None)); + rewind_state = rewind.map(|rewind| (rewind, None)); } env.data_mut(&mut store).replaying_journal = false; diff --git a/lib/wasix/src/syscalls/mod.rs b/lib/wasix/src/syscalls/mod.rs index baf194f78aa..6603d0e8972 100644 --- a/lib/wasix/src/syscalls/mod.rs +++ b/lib/wasix/src/syscalls/mod.rs @@ -1295,7 +1295,7 @@ pub fn anyhow_err_to_runtime_err(err: anyhow::Error) -> WasiRuntimeError { pub fn restore_snapshot( mut ctx: FunctionEnvMut<'_, WasiEnv>, journal: Arc, -) -> Result { +) -> Result, WasiRuntimeError> { use crate::journal::Journal; let mut is_same_module = false; @@ -1724,7 +1724,7 @@ pub fn restore_snapshot( ); } - rewind.ok_or(WasiRuntimeError::Runtime(RuntimeError::user(anyhow::format_err!("The restored snapshot journal does not have a thread stack events and hence we can not restore the state of the process.").into()))) + Ok(rewind) } pub(crate) unsafe fn handle_rewind( From 74edba8cced1195965314bf9d9d2adec36ab0e6a Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Fri, 24 Nov 2023 00:58:51 +1100 Subject: [PATCH 077/129] Added additional method for rewinding --- lib/wasix/src/bin_factory/exec.rs | 9 ++-- lib/wasix/src/os/task/process.rs | 3 +- lib/wasix/src/rewind.rs | 2 + lib/wasix/src/state/builder.rs | 10 ++-- lib/wasix/src/syscalls/mod.rs | 48 +++++++++++++++++--- lib/wasix/src/syscalls/wasix/proc_fork.rs | 3 +- lib/wasix/src/syscalls/wasix/thread_spawn.rs | 3 +- 7 files changed, 59 insertions(+), 19 deletions(-) diff --git a/lib/wasix/src/bin_factory/exec.rs b/lib/wasix/src/bin_factory/exec.rs index d6bc8b7762d..fda008bc2eb 100644 --- a/lib/wasix/src/bin_factory/exec.rs +++ b/lib/wasix/src/bin_factory/exec.rs @@ -162,28 +162,29 @@ fn call_module( // If we need to rewind then do so if let Some((rewind_state, rewind_result)) = rewind_state { + let mut ctx = ctx.env.clone().into_mut(&mut store); if rewind_state.is_64bit { let res = rewind_ext::( - ctx.env.clone().into_mut(&mut store), + &mut ctx, rewind_state.memory_stack, rewind_state.rewind_stack, rewind_state.store_data, Some(rewind_result), ); if res != Errno::Success { - ctx.data(&store).blocking_on_exit(Some(res.into())); + ctx.data().blocking_on_exit(Some(res.into())); return; } } else { let res = rewind_ext::( - ctx.env.clone().into_mut(&mut store), + &mut ctx, rewind_state.memory_stack, rewind_state.rewind_stack, rewind_state.store_data, Some(rewind_result), ); if res != Errno::Success { - ctx.data(&store).blocking_on_exit(Some(res.into())); + ctx.data().blocking_on_exit(Some(res.into())); return; } }; diff --git a/lib/wasix/src/os/task/process.rs b/lib/wasix/src/os/task/process.rs index b8d476d58ee..d4501248f0e 100644 --- a/lib/wasix/src/os/task/process.rs +++ b/lib/wasix/src/os/task/process.rs @@ -243,7 +243,8 @@ impl WasiProcessInner { trace!("checkpoint finished"); // Rewind the stack and carry on - return match rewind_ext::(ctx, memory_stack, rewind_stack, store_data, None) { + return match rewind_ext::(&mut ctx, memory_stack, rewind_stack, store_data, None) + { Errno::Success => OnCalledAction::InvokeAgain, err => { tracing::warn!( diff --git a/lib/wasix/src/rewind.rs b/lib/wasix/src/rewind.rs index 2ad1fc2f4b4..92f72a6edc3 100644 --- a/lib/wasix/src/rewind.rs +++ b/lib/wasix/src/rewind.rs @@ -28,6 +28,8 @@ pub struct RewindState { pub is_64bit: bool, } +pub type RewindStateOption = Option<(RewindState, Option)>; + /// Represents the work that will be done when a thread goes to deep sleep and /// includes the things needed to restore it again pub struct DeepSleepWork { diff --git a/lib/wasix/src/state/builder.rs b/lib/wasix/src/state/builder.rs index 12daa769d16..64038d7c38d 100644 --- a/lib/wasix/src/state/builder.rs +++ b/lib/wasix/src/state/builder.rs @@ -6,7 +6,6 @@ use std::{ sync::Arc, }; -use bytes::Bytes; use rand::Rng; use thiserror::Error; use virtual_fs::{ArcFile, FileSystem, FsError, TmpFileSystem, VirtualFile}; @@ -23,7 +22,7 @@ use crate::{ runtime::{module_cache::ModuleHash, task_manager::InlineWaker}, state::WasiState, syscalls::types::{__WASI_STDERR_FILENO, __WASI_STDIN_FILENO, __WASI_STDOUT_FILENO}, - RewindState, Runtime, WasiEnv, WasiError, WasiFunctionEnv, WasiRuntimeError, + RewindStateOption, Runtime, WasiEnv, WasiError, WasiFunctionEnv, WasiRuntimeError, }; #[cfg(feature = "journal")] use crate::{ @@ -1095,15 +1094,16 @@ fn wasi_exit_code( fn run_with_deep_sleep( mut store: Store, - rewind_state: Option<(RewindState, Option)>, + rewind_state: RewindStateOption, env: WasiFunctionEnv, sender: tokio::sync::mpsc::UnboundedSender>, ) { if let Some((rewind_state, rewind_result)) = rewind_state { tracing::trace!("Rewinding"); + let mut ctx = env.env.clone().into_mut(&mut store); let errno = if rewind_state.is_64bit { crate::rewind_ext::( - env.env.clone().into_mut(&mut store), + &mut ctx, rewind_state.memory_stack, rewind_state.rewind_stack, rewind_state.store_data, @@ -1111,7 +1111,7 @@ fn run_with_deep_sleep( ) } else { crate::rewind_ext::( - env.env.clone().into_mut(&mut store), + &mut ctx, rewind_state.memory_stack, rewind_state.rewind_stack, rewind_state.store_data, diff --git a/lib/wasix/src/syscalls/mod.rs b/lib/wasix/src/syscalls/mod.rs index 6603d0e8972..b86b998a9a7 100644 --- a/lib/wasix/src/syscalls/mod.rs +++ b/lib/wasix/src/syscalls/mod.rs @@ -121,8 +121,8 @@ use crate::{ os::task::{process::MaybeCheckpointResult, thread::RewindResult}, runtime::task_manager::InlineWaker, utils::store::InstanceSnapshot, - DeepSleepWork, RewindPostProcess, RewindState, SpawnError, WasiInodes, WasiResult, - WasiRuntimeError, + DeepSleepWork, RewindPostProcess, RewindState, RewindStateOption, SpawnError, WasiInodes, + WasiResult, WasiRuntimeError, }; pub(crate) use crate::{net::net_error_into_wasi_err, utils::WasiParkingLot}; @@ -1112,7 +1112,7 @@ where #[instrument(level = "debug", skip_all, fields(memory_stack_len = memory_stack.len(), rewind_stack_len = rewind_stack.len(), store_data_len = store_data.len()))] #[must_use = "the action must be passed to the call loop"] pub fn rewind( - ctx: FunctionEnvMut, + mut ctx: FunctionEnvMut, memory_stack: Bytes, rewind_stack: Bytes, store_data: Bytes, @@ -1123,7 +1123,7 @@ where { let rewind_result = bincode::serialize(&result).unwrap().into(); rewind_ext::( - ctx, + &mut ctx, memory_stack, rewind_stack, store_data, @@ -1134,7 +1134,7 @@ where #[instrument(level = "debug", skip_all, fields(memory_stack_len = memory_stack.len(), rewind_stack_len = rewind_stack.len(), store_data_len = store_data.len()))] #[must_use = "the action must be passed to the call loop"] pub fn rewind_ext( - mut ctx: FunctionEnvMut, + ctx: &mut FunctionEnvMut, memory_stack: Bytes, rewind_stack: Bytes, store_data: Bytes, @@ -1154,7 +1154,7 @@ pub fn rewind_ext( return Errno::Unknown; } }; - crate::utils::store::restore_instance_snapshot(&mut ctx, &store_snapshot); + crate::utils::store::restore_instance_snapshot(ctx, &store_snapshot); let env = ctx.data(); let memory = match env.try_memory_view(&ctx) { Some(v) => v, @@ -1209,7 +1209,7 @@ pub fn rewind_ext( .filter_map(|a| a.asyncify_start_rewind.clone()) .next() { - asyncify_start_rewind.call(&mut ctx, asyncify_data); + asyncify_start_rewind.call(ctx, asyncify_data); } else { warn!("failed to rewind the stack because the asyncify_start_rewind export is missing or inaccessible"); return Errno::Noexec; @@ -1218,6 +1218,40 @@ pub fn rewind_ext( Errno::Success } +pub fn rewind_ext2( + ctx: &mut FunctionEnvMut, + rewind_state: RewindStateOption, +) -> Result<(), ExitCode> { + if let Some((rewind_state, rewind_result)) = rewind_state { + tracing::trace!("Rewinding"); + let errno = if rewind_state.is_64bit { + crate::rewind_ext::( + ctx, + rewind_state.memory_stack, + rewind_state.rewind_stack, + rewind_state.store_data, + rewind_result, + ) + } else { + crate::rewind_ext::( + ctx, + rewind_state.memory_stack, + rewind_state.rewind_stack, + rewind_state.store_data, + rewind_result, + ) + }; + + if errno != Errno::Success { + let exit_code = ExitCode::from(errno); + ctx.data().on_exit(Some(exit_code)); + return Err(exit_code); + } + } + + Ok(()) +} + #[allow(clippy::extra_unused_type_parameters)] #[cfg(not(feature = "journal"))] pub fn maybe_snapshot_once( diff --git a/lib/wasix/src/syscalls/wasix/proc_fork.rs b/lib/wasix/src/syscalls/wasix/proc_fork.rs index 9b87ee9b978..9bcdc337312 100644 --- a/lib/wasix/src/syscalls/wasix/proc_fork.rs +++ b/lib/wasix/src/syscalls/wasix/proc_fork.rs @@ -247,8 +247,9 @@ fn run( // If we need to rewind then do so if let Some((rewind_state, rewind_result)) = rewind_state { + let mut ctx = ctx.env.clone().into_mut(&mut store); let res = rewind_ext::( - ctx.env.clone().into_mut(&mut store), + &mut ctx, rewind_state.memory_stack, rewind_state.rewind_stack, rewind_state.store_data, diff --git a/lib/wasix/src/syscalls/wasix/thread_spawn.rs b/lib/wasix/src/syscalls/wasix/thread_spawn.rs index ee7264b3009..eb6ffa382c2 100644 --- a/lib/wasix/src/syscalls/wasix/thread_spawn.rs +++ b/lib/wasix/src/syscalls/wasix/thread_spawn.rs @@ -205,8 +205,9 @@ fn call_module( // If we need to rewind then do so if let Some((rewind_state, rewind_result)) = rewind_state { + let mut ctx = ctx.env.clone().into_mut(&mut store); let res = rewind_ext::( - ctx.env.clone().into_mut(&mut store), + &mut ctx, rewind_state.memory_stack, rewind_state.rewind_stack, rewind_state.store_data, From d1d5b8e7f4d460eb81728ca71dfbdbd5c1fcd4dc Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Fri, 24 Nov 2023 01:03:45 +1100 Subject: [PATCH 078/129] Removed the journals parameter from the API --- lib/wasix/src/lib.rs | 14 +------------- lib/wasix/src/state/builder.rs | 7 +------ lib/wasix/src/state/env.rs | 2 +- 3 files changed, 3 insertions(+), 20 deletions(-) diff --git a/lib/wasix/src/lib.rs b/lib/wasix/src/lib.rs index 3e1fc5f3d57..e039ea7b0bd 100644 --- a/lib/wasix/src/lib.rs +++ b/lib/wasix/src/lib.rs @@ -62,7 +62,6 @@ use std::sync::Arc; #[allow(unused_imports)] use bytes::{Bytes, BytesMut}; -use journal::DynJournal; use os::task::control_plane::ControlPlaneError; use thiserror::Error; use tracing::error; @@ -236,18 +235,8 @@ impl WasiRuntimeError { pub(crate) fn run_wasi_func( func: &wasmer::Function, store: &mut impl AsStoreMut, - journals: Vec>, params: &[wasmer::Value], ) -> Result, WasiRuntimeError> { - if !journals.is_empty() { - return Err(WasiRuntimeError::Runtime(RuntimeError::user( - anyhow::format_err!( - "journal restoration is not currently supported when running specific functions" - ) - .into(), - ))); - } - func.call(store, params).map_err(|err| { if let Some(_werr) = err.downcast_ref::() { let werr = err.downcast::().unwrap(); @@ -268,9 +257,8 @@ pub(crate) fn run_wasi_func( pub(crate) fn run_wasi_func_start( func: &wasmer::Function, store: &mut impl AsStoreMut, - journals: Vec>, ) -> Result<(), WasiRuntimeError> { - run_wasi_func(func, store, journals, &[])?; + run_wasi_func(func, store, &[])?; Ok(()) } diff --git a/lib/wasix/src/state/builder.rs b/lib/wasix/src/state/builder.rs index 64038d7c38d..6fab4a7a681 100644 --- a/lib/wasix/src/state/builder.rs +++ b/lib/wasix/src/state/builder.rs @@ -945,17 +945,12 @@ impl WasiEnvBuilder { ); } - #[cfg(feature = "journal")] - let journals: Vec> = self.journals.clone(); - #[cfg(not(feature = "journal"))] - let journals = Vec::new(); - let (instance, env) = self.instantiate_ext(module, module_hash, store)?; let start = instance.exports.get_function("_start")?; env.data(&store).thread.set_status_running(); - let result = crate::run_wasi_func_start(start, store, journals); + let result = crate::run_wasi_func_start(start, store); let (result, exit_code) = wasi_exit_code(result); let pid = env.data(&store).pid(); diff --git a/lib/wasix/src/state/env.rs b/lib/wasix/src/state/env.rs index 52959201c5e..d3498e801d7 100644 --- a/lib/wasix/src/state/env.rs +++ b/lib/wasix/src/state/env.rs @@ -568,7 +568,7 @@ impl WasiEnv { // If this module exports an _initialize function, run that first. if call_initialize { if let Ok(initialize) = instance.exports.get_function("_initialize") { - if let Err(err) = crate::run_wasi_func_start(initialize, &mut store, Vec::new()) { + if let Err(err) = crate::run_wasi_func_start(initialize, &mut store) { func_env .data(&store) .blocking_on_exit(Some(Errno::Noexec.into())); From cc95b8787eb1d86f168eef06299b53a1d1b1ea72 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Fri, 24 Nov 2023 01:06:05 +1100 Subject: [PATCH 079/129] Fixed the process exit record replay --- lib/wasix/src/journal/effector/process_exit.rs | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/lib/wasix/src/journal/effector/process_exit.rs b/lib/wasix/src/journal/effector/process_exit.rs index e181a6640c0..fb3facac6b8 100644 --- a/lib/wasix/src/journal/effector/process_exit.rs +++ b/lib/wasix/src/journal/effector/process_exit.rs @@ -1,3 +1,5 @@ +use virtual_mio::InlineWaker; + use super::*; impl JournalEffector { @@ -9,7 +11,15 @@ impl JournalEffector { } pub fn apply_process_exit(env: &WasiEnv, exit_code: Option) -> anyhow::Result<()> { - env.blocking_on_exit(exit_code); + // If we are in the phase of replaying journals then we + // close all the file descriptors but we don't actually send + // any signals + if env.replaying_journal { + let state = env.state.clone(); + InlineWaker::block_on(state.fs.close_all()); + } else { + env.blocking_on_exit(exit_code); + } Ok(()) } } From 10e9c02492c407518979ae61660e6f300f16f8bd Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Fri, 24 Nov 2023 01:07:48 +1100 Subject: [PATCH 080/129] Fixed a bug where the journals were not being returned by the runtime --- lib/wasix/src/runtime/mod.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/wasix/src/runtime/mod.rs b/lib/wasix/src/runtime/mod.rs index 76cc56db94b..088c82ab356 100644 --- a/lib/wasix/src/runtime/mod.rs +++ b/lib/wasix/src/runtime/mod.rs @@ -111,7 +111,7 @@ where /// The list of journals which will be used to restore the state of the /// runtime at a particular point in time #[cfg(feature = "journal")] - fn journals(&self) -> &'static Vec> { + fn journals(&self) -> &'_ Vec> { &EMPTY_JOURNAL_LIST } @@ -339,6 +339,11 @@ impl Runtime for PluggableRuntime { self.module_cache.clone() } + #[cfg(feature = "journal")] + fn journals(&self) -> &'_ Vec> { + &self.journals + } + #[cfg(feature = "journal")] fn active_journal(&self) -> Option<&DynJournal> { self.journals.iter().last().map(|a| a.as_ref()) From cc97e1bdf6e051f7a0f0b12d6b5813f21084afaf Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Fri, 24 Nov 2023 01:16:39 +1100 Subject: [PATCH 081/129] Fixed the bootstrapping process for one of the execution paths --- lib/wasix/src/state/builder.rs | 43 +++---------------------- lib/wasix/src/state/func_env.rs | 57 +++++++++++++++++++++++++++++++-- 2 files changed, 59 insertions(+), 41 deletions(-) diff --git a/lib/wasix/src/state/builder.rs b/lib/wasix/src/state/builder.rs index 6fab4a7a681..f72701119b4 100644 --- a/lib/wasix/src/state/builder.rs +++ b/lib/wasix/src/state/builder.rs @@ -12,6 +12,8 @@ use virtual_fs::{ArcFile, FileSystem, FsError, TmpFileSystem, VirtualFile}; use wasmer::{AsStoreMut, Instance, Module, RuntimeError, Store}; use wasmer_wasix_types::wasi::{Errno, ExitCode}; +#[cfg(feature = "journal")] +use crate::journal::{DynJournal, SnapshotTrigger}; #[cfg(feature = "sys")] use crate::PluggableRuntime; use crate::{ @@ -24,11 +26,6 @@ use crate::{ syscalls::types::{__WASI_STDERR_FILENO, __WASI_STDIN_FILENO, __WASI_STDOUT_FILENO}, RewindStateOption, Runtime, WasiEnv, WasiError, WasiFunctionEnv, WasiRuntimeError, }; -#[cfg(feature = "journal")] -use crate::{ - journal::{DynJournal, SnapshotTrigger}, - syscalls::restore_snapshot, -}; use super::env::WasiEnvInit; @@ -990,44 +987,12 @@ impl WasiEnvBuilder { #[cfg(feature = "sys-thread")] let _guard = _guard.as_ref().map(|r| r.enter()); - #[cfg(feature = "journal")] - let journals = self.journals.clone(); - let (_, env) = self.instantiate_ext(module, module_hash, &mut store)?; env.data(&store).thread.set_status_running(); - #[allow(unused_mut)] - let mut rewind_state = None; - - #[cfg(feature = "journal")] - { - env.data_mut(&mut store).replaying_journal = true; - - for journal in journals { - let ctx = env.env.clone().into_mut(&mut store); - let rewind = restore_snapshot(ctx, journal)?; - rewind_state = rewind.map(|rewind| (rewind, None)); - } - - env.data_mut(&mut store).replaying_journal = false; - - // The first event we save is an event that records the module hash. - // Note: This is used to detect if an incorrect journal is used on the wrong - // process or if a process has been recompiled - let wasm_hash = env.data(&store).process.module_hash.as_bytes(); - let mut ctx = env.env.clone().into_mut(&mut store); - crate::journal::JournalEffector::save_event( - &mut ctx, - crate::journal::JournalEntry::InitModule { wasm_hash }, - ) - .map_err(|err| { - WasiRuntimeError::Runtime(RuntimeError::new(format!( - "journal failied to save the module initialization event - {}", - err - ))) - })?; - } + // Bootstrap the process + let rewind_state = env.bootstrap(&mut store)?; let tasks = env.data(&store).tasks().clone(); let pid = env.data(&store).pid(); diff --git a/lib/wasix/src/state/func_env.rs b/lib/wasix/src/state/func_env.rs index 35ed4f1b7ef..844fae3b864 100644 --- a/lib/wasix/src/state/func_env.rs +++ b/lib/wasix/src/state/func_env.rs @@ -2,7 +2,8 @@ use std::sync::Arc; use tracing::trace; use wasmer::{ - AsStoreMut, AsStoreRef, ExportError, FunctionEnv, Imports, Instance, Memory, Module, Store, + AsStoreMut, AsStoreRef, ExportError, FunctionEnv, Imports, Instance, Memory, Module, + RuntimeError, Store, }; use wasmer_wasix_types::wasi::ExitCode; @@ -10,8 +11,9 @@ use crate::{ import_object_for_all_wasi_versions, runtime::SpawnMemoryType, state::WasiInstanceHandles, + syscalls::restore_snapshot, utils::{get_wasi_version, get_wasi_versions, store::restore_instance_snapshot}, - InstanceSnapshot, WasiEnv, WasiError, WasiThreadError, + InstanceSnapshot, RewindStateOption, WasiEnv, WasiError, WasiRuntimeError, WasiThreadError, }; /// The default stack size for WASIX - the number itself is the default that compilers @@ -232,4 +234,55 @@ impl WasiFunctionEnv { // Cleans up all the open files (if this is the main thread) self.data(store).blocking_on_exit(exit_code); } + + /// Bootstraps this main thread and context with any journals that + /// may be present + pub fn bootstrap( + &self, + mut store: &'_ mut impl AsStoreMut, + ) -> Result { + let mut rewind_state = None; + + #[cfg(feature = "journal")] + { + // If there are journals we need to restore then do so (this will + // prevent the initialization function from running + let restore_journals = self.data(&store).runtime.journals().clone(); + if !restore_journals.is_empty() { + self.data_mut(&mut store).replaying_journal = true; + + for journal in restore_journals { + let ctx = self.env.clone().into_mut(&mut store); + let rewind = match restore_snapshot(ctx, journal) { + Ok(r) => r, + Err(err) => { + self.data_mut(&mut store).replaying_journal = false; + return Err(err); + } + }; + rewind_state = rewind.map(|rewind| (rewind, None)); + } + + self.data_mut(&mut store).replaying_journal = false; + } + + // The first event we save is an event that records the module hash. + // Note: This is used to detect if an incorrect journal is used on the wrong + // process or if a process has been recompiled + let wasm_hash = self.data(&store).process.module_hash.as_bytes(); + let mut ctx = self.env.clone().into_mut(&mut store); + crate::journal::JournalEffector::save_event( + &mut ctx, + crate::journal::JournalEntry::InitModule { wasm_hash }, + ) + .map_err(|err| { + WasiRuntimeError::Runtime(RuntimeError::new(format!( + "journal failied to save the module initialization event - {}", + err + ))) + })?; + } + + Ok(rewind_state) + } } From 0320f045328a07d2930e8ae1326b832952e0b994 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Fri, 24 Nov 2023 01:32:34 +1100 Subject: [PATCH 082/129] Added another bootstrap and cleaned up the an interface --- lib/wasix/src/bin_factory/exec.rs | 8 ++++---- lib/wasix/src/state/builder.rs | 13 ++++++++++++- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/lib/wasix/src/bin_factory/exec.rs b/lib/wasix/src/bin_factory/exec.rs index fda008bc2eb..b28310bd115 100644 --- a/lib/wasix/src/bin_factory/exec.rs +++ b/lib/wasix/src/bin_factory/exec.rs @@ -153,7 +153,7 @@ fn call_module( ctx: WasiFunctionEnv, mut store: Store, handle: WasiThreadRunGuard, - rewind_state: Option<(RewindState, Bytes)>, + rewind_state: Option<(RewindState, Option)>, ) { let env = ctx.data(&store); let pid = env.pid(); @@ -169,7 +169,7 @@ fn call_module( rewind_state.memory_stack, rewind_state.rewind_stack, rewind_state.store_data, - Some(rewind_result), + rewind_result, ); if res != Errno::Success { ctx.data().blocking_on_exit(Some(res.into())); @@ -181,7 +181,7 @@ fn call_module( rewind_state.memory_stack, rewind_state.rewind_stack, rewind_state.store_data, - Some(rewind_result), + rewind_result, ); if res != Errno::Success { ctx.data().blocking_on_exit(Some(res.into())); @@ -212,7 +212,7 @@ fn call_module( let respawn = { move |ctx, store, rewind_result| { // Call the thread - call_module(ctx, store, handle, Some((rewind, rewind_result))); + call_module(ctx, store, handle, Some((rewind, Some(rewind_result)))); } }; diff --git a/lib/wasix/src/state/builder.rs b/lib/wasix/src/state/builder.rs index f72701119b4..71a40e82776 100644 --- a/lib/wasix/src/state/builder.rs +++ b/lib/wasix/src/state/builder.rs @@ -23,7 +23,10 @@ use crate::{ os::task::control_plane::{ControlPlaneConfig, ControlPlaneError, WasiControlPlane}, runtime::{module_cache::ModuleHash, task_manager::InlineWaker}, state::WasiState, - syscalls::types::{__WASI_STDERR_FILENO, __WASI_STDIN_FILENO, __WASI_STDOUT_FILENO}, + syscalls::{ + rewind_ext2, + types::{__WASI_STDERR_FILENO, __WASI_STDIN_FILENO, __WASI_STDOUT_FILENO}, + }, RewindStateOption, Runtime, WasiEnv, WasiError, WasiFunctionEnv, WasiRuntimeError, }; @@ -944,6 +947,14 @@ impl WasiEnvBuilder { let (instance, env) = self.instantiate_ext(module, module_hash, store)?; + // Bootstrap the process + let rewind_state = env.bootstrap(store)?; + if rewind_state.is_some() { + let mut ctx = env.env.clone().into_mut(store); + rewind_ext2(&mut ctx, rewind_state) + .map_err(|exit| WasiRuntimeError::Wasi(WasiError::Exit(exit)))?; + } + let start = instance.exports.get_function("_start")?; env.data(&store).thread.set_status_running(); From 36cb0530b99e6ded065d1cd8699251914798bf9b Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Fri, 24 Nov 2023 01:56:47 +1100 Subject: [PATCH 083/129] The journals are now properly being restored --- lib/wasix/src/bin_factory/exec.rs | 13 +++++++++- lib/wasix/src/journal/effector/thread_exit.rs | 17 +++++++++++++ lib/wasix/src/runtime/mod.rs | 25 ++++++++++++++++++- lib/wasix/src/state/func_env.rs | 2 +- lib/wasix/src/syscalls/mod.rs | 18 +++++++------ 5 files changed, 65 insertions(+), 10 deletions(-) diff --git a/lib/wasix/src/bin_factory/exec.rs b/lib/wasix/src/bin_factory/exec.rs index b28310bd115..6301b5fee43 100644 --- a/lib/wasix/src/bin_factory/exec.rs +++ b/lib/wasix/src/bin_factory/exec.rs @@ -119,12 +119,23 @@ pub fn spawn_exec_module( WasiFunctionEnv { env: ctx.env } }; + // Bootstrap the process + let rewind_state = match ctx.bootstrap(&mut store) { + Ok(r) => r, + Err(err) => { + thread.thread.set_status_finished(Err(err.into())); + ctx.data(&store) + .blocking_on_exit(Some(Errno::Noexec.into())); + return; + } + }; + // If there is a start function debug!("wasi[{}]::called main()", pid); // TODO: rewrite to use crate::run_wasi_func // Call the module - call_module(ctx, store, thread, None); + call_module(ctx, store, thread, rewind_state); } }; diff --git a/lib/wasix/src/journal/effector/thread_exit.rs b/lib/wasix/src/journal/effector/thread_exit.rs index 359cbfe13f1..4d7975a8159 100644 --- a/lib/wasix/src/journal/effector/thread_exit.rs +++ b/lib/wasix/src/journal/effector/thread_exit.rs @@ -1,3 +1,5 @@ +use wasmer_wasix_types::wasi::Signal; + use super::*; impl JournalEffector { @@ -11,4 +13,19 @@ impl JournalEffector { .map_err(map_snapshot_err)?; Ok(()) } + + pub fn apply_thread_exit( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + tid: WasiThreadId, + exit_code: Option, + ) -> anyhow::Result<()> { + let env = ctx.data(); + if let Some(thread) = env.process.get_thread(&tid) { + if let Some(code) = exit_code { + thread.set_status_finished(Ok(code)); + } + thread.signal(Signal::Sigkill); + } + Ok(()) + } } diff --git a/lib/wasix/src/runtime/mod.rs b/lib/wasix/src/runtime/mod.rs index 088c82ab356..c84f122c862 100644 --- a/lib/wasix/src/runtime/mod.rs +++ b/lib/wasix/src/runtime/mod.rs @@ -11,7 +11,7 @@ use self::{ use std::{ fmt, - sync::{Arc, Mutex}, + sync::{atomic::AtomicBool, Arc, Mutex}, }; use derivative::Derivative; @@ -115,6 +115,13 @@ where &EMPTY_JOURNAL_LIST } + /// Pops the restore list of journals that will be replayed before the + /// instance gets going + #[cfg(feature = "journal")] + fn pop_restore_journals(&self) -> &'_ Vec> { + &EMPTY_JOURNAL_LIST + } + /// The snapshot capturer takes and restores snapshots of the WASM process at specific /// points in time by reading and writing log entries #[cfg(feature = "journal")] @@ -202,6 +209,7 @@ pub struct PluggableRuntime { #[cfg(feature = "journal")] #[derivative(Debug = "ignore")] pub journals: Vec>, + pub restored_journals: Arc, } impl PluggableRuntime { @@ -238,6 +246,7 @@ impl PluggableRuntime { module_cache: Arc::new(module_cache::in_memory()), #[cfg(feature = "journal")] journals: Vec::new(), + restored_journals: Arc::new(AtomicBool::new(false)), } } @@ -344,6 +353,20 @@ impl Runtime for PluggableRuntime { &self.journals } + #[cfg(feature = "journal")] + fn pop_restore_journals(&self) -> &'_ Vec> { + use std::sync::atomic::Ordering; + if self + .restored_journals + .compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst) + .is_ok() + { + &self.journals + } else { + &EMPTY_JOURNAL_LIST + } + } + #[cfg(feature = "journal")] fn active_journal(&self) -> Option<&DynJournal> { self.journals.iter().last().map(|a| a.as_ref()) diff --git a/lib/wasix/src/state/func_env.rs b/lib/wasix/src/state/func_env.rs index 844fae3b864..7c8fe22defd 100644 --- a/lib/wasix/src/state/func_env.rs +++ b/lib/wasix/src/state/func_env.rs @@ -247,7 +247,7 @@ impl WasiFunctionEnv { { // If there are journals we need to restore then do so (this will // prevent the initialization function from running - let restore_journals = self.data(&store).runtime.journals().clone(); + let restore_journals = self.data(&store).runtime.pop_restore_journals().clone(); if !restore_journals.is_empty() { self.data_mut(&mut store).replaying_journal = true; diff --git a/lib/wasix/src/syscalls/mod.rs b/lib/wasix/src/syscalls/mod.rs index b86b998a9a7..a21f8af8362 100644 --- a/lib/wasix/src/syscalls/mod.rs +++ b/lib/wasix/src/syscalls/mod.rs @@ -1332,7 +1332,7 @@ pub fn restore_snapshot( ) -> Result, WasiRuntimeError> { use crate::journal::Journal; - let mut is_same_module = false; + let mut is_same_module = true; let mut rewind = None; while let Some(next) = journal.read().map_err(anyhow_err_to_runtime_err)? { tracing::trace!("Restoring snapshot event - {next:?}"); @@ -1341,6 +1341,7 @@ pub fn restore_snapshot( is_same_module = ctx.data().process.module_hash.as_bytes()[0..16] == wasm_hash; } crate::journal::JournalEntry::ProcessExit { exit_code } => { + rewind = None; JournalEffector::apply_process_exit(ctx.data(), exit_code) .map_err(anyhow_err_to_runtime_err)?; } @@ -1370,12 +1371,11 @@ pub fn restore_snapshot( } crate::journal::JournalEntry::CloseThread { id, exit_code } => { if id == ctx.data().tid() { - return Err(WasiRuntimeError::Runtime(RuntimeError::user( - anyhow::format_err!( - "Snapshot restoration has already closed the main thread." - ) - .into(), - ))); + JournalEffector::apply_process_exit(ctx.data(), exit_code) + .map_err(anyhow_err_to_runtime_err)?; + } else { + JournalEffector::apply_thread_exit(&mut ctx, id, exit_code) + .map_err(anyhow_err_to_runtime_err)?; } } crate::journal::JournalEntry::SetThread { @@ -1751,7 +1751,11 @@ pub fn restore_snapshot( // that simulates closing the process (hence keeps everything // in a clean state) if !is_same_module { + eprintln!( + "The WASM module hash does not match the journal module hash - forcing a restart" + ); JournalEffector::apply_process_exit(ctx.data(), None).map_err(anyhow_err_to_runtime_err)?; + rewind = None; } else { tracing::debug!( "journal used on a different module - the process will simulate a restart." From 953a0af77ee10cdea9bf8ffe7694902f15af03d2 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Fri, 24 Nov 2023 07:11:43 +1100 Subject: [PATCH 084/129] Fixed the linting and compile error --- lib/wasix/src/bin_factory/exec.rs | 2 +- lib/wasix/src/runners/wasi.rs | 2 +- lib/wasix/src/state/func_env.rs | 4 +++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/wasix/src/bin_factory/exec.rs b/lib/wasix/src/bin_factory/exec.rs index 6301b5fee43..044adc528eb 100644 --- a/lib/wasix/src/bin_factory/exec.rs +++ b/lib/wasix/src/bin_factory/exec.rs @@ -123,7 +123,7 @@ pub fn spawn_exec_module( let rewind_state = match ctx.bootstrap(&mut store) { Ok(r) => r, Err(err) => { - thread.thread.set_status_finished(Err(err.into())); + thread.thread.set_status_finished(Err(err)); ctx.data(&store) .blocking_on_exit(Some(Errno::Noexec.into())); return; diff --git a/lib/wasix/src/runners/wasi.rs b/lib/wasix/src/runners/wasi.rs index c4b429a731a..ab2f16a7f46 100644 --- a/lib/wasix/src/runners/wasi.rs +++ b/lib/wasix/src/runners/wasi.rs @@ -376,7 +376,7 @@ impl crate::runners::Runner for WasiRunner { WasiRuntimeError::Instantiation(a.clone()) } WasiRuntimeError::Wasi(WasiError::Exit(a)) => { - WasiRuntimeError::Wasi(WasiError::Exit(a.clone())) + WasiRuntimeError::Wasi(WasiError::Exit(*a)) } WasiRuntimeError::Wasi(WasiError::UnknownWasiVersion) => { WasiRuntimeError::Wasi(WasiError::UnknownWasiVersion) diff --git a/lib/wasix/src/state/func_env.rs b/lib/wasix/src/state/func_env.rs index 7c8fe22defd..d5b805fded8 100644 --- a/lib/wasix/src/state/func_env.rs +++ b/lib/wasix/src/state/func_env.rs @@ -7,11 +7,12 @@ use wasmer::{ }; use wasmer_wasix_types::wasi::ExitCode; +#[cfg(feature = "journal")] +use crate::syscalls::restore_snapshot; use crate::{ import_object_for_all_wasi_versions, runtime::SpawnMemoryType, state::WasiInstanceHandles, - syscalls::restore_snapshot, utils::{get_wasi_version, get_wasi_versions, store::restore_instance_snapshot}, InstanceSnapshot, RewindStateOption, WasiEnv, WasiError, WasiRuntimeError, WasiThreadError, }; @@ -237,6 +238,7 @@ impl WasiFunctionEnv { /// Bootstraps this main thread and context with any journals that /// may be present + #[allow(clippy::result_large_err)] pub fn bootstrap( &self, mut store: &'_ mut impl AsStoreMut, From 4baa50a393be535846e60313c7e284107d613579 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Fri, 24 Nov 2023 10:37:20 +1100 Subject: [PATCH 085/129] Added safety to the journal restore code --- lib/api/src/externals/memory.rs | 6 ++++ lib/api/src/sys/externals/memory.rs | 5 ++++ lib/api/src/sys/tunables.rs | 6 ++++ .../src/memory/fd_memory/memories.rs | 23 +++++++++++++++ lib/vm/src/memory.rs | 28 +++++++++++++++++++ lib/wasix/src/bin_factory/exec.rs | 4 ++- .../journal/effector/memory_and_snapshot.rs | 5 +++- .../src/journal/effector/process_exit.rs | 13 ++++++++- lib/wasix/src/state/builder.rs | 19 ++++++++++--- lib/wasix/src/state/func_env.rs | 6 +++- lib/wasix/src/syscalls/mod.rs | 28 +++++++++++-------- 11 files changed, 124 insertions(+), 19 deletions(-) diff --git a/lib/api/src/externals/memory.rs b/lib/api/src/externals/memory.rs index 8f2e18fa21d..c852f0ae79e 100644 --- a/lib/api/src/externals/memory.rs +++ b/lib/api/src/externals/memory.rs @@ -121,6 +121,12 @@ impl Memory { self.0.grow(store, delta) } + /// Resets the memory back to zero length + pub fn reset(&self, store: &mut impl AsStoreMut) -> Result<(), MemoryError> { + self.0.reset(store)?; + Ok(()) + } + /// Attempts to duplicate this memory (if its clonable) in a new store /// (copied memory) pub fn copy_to_store( diff --git a/lib/api/src/sys/externals/memory.rs b/lib/api/src/sys/externals/memory.rs index 90c02ac3051..19c7e591019 100644 --- a/lib/api/src/sys/externals/memory.rs +++ b/lib/api/src/sys/externals/memory.rs @@ -51,6 +51,11 @@ impl Memory { self.handle.get_mut(store.objects_mut()).grow(delta.into()) } + pub fn reset(&self, store: &mut impl AsStoreMut) -> Result<(), MemoryError> { + self.handle.get_mut(store.objects_mut()).reset()?; + Ok(()) + } + pub(crate) fn from_vm_extern(store: &impl AsStoreRef, vm_extern: VMExternMemory) -> Self { Self { handle: unsafe { diff --git a/lib/api/src/sys/tunables.rs b/lib/api/src/sys/tunables.rs index c2853be44ff..fb3d7a31c56 100644 --- a/lib/api/src/sys/tunables.rs +++ b/lib/api/src/sys/tunables.rs @@ -106,6 +106,12 @@ mod tests { attempted_delta: delta, }) } + + fn reset(&mut self) -> Result<(), MemoryError> { + self.mem.fill(0); + Ok(()) + } + fn vmmemory(&self) -> NonNull { unsafe { NonNull::new( diff --git a/lib/sys-utils/src/memory/fd_memory/memories.rs b/lib/sys-utils/src/memory/fd_memory/memories.rs index 4b71a13d3aa..32844df6130 100644 --- a/lib/sys-utils/src/memory/fd_memory/memories.rs +++ b/lib/sys-utils/src/memory/fd_memory/memories.rs @@ -131,6 +131,11 @@ impl WasmMmap { Ok(prev_pages) } + fn reset(&mut self) -> Result<(), MemoryError> { + self.size.0 = 0; + Ok(()) + } + /// Copies the memory /// (in this case it performs a copy-on-write to save memory) pub fn copy(&mut self) -> Result { @@ -337,6 +342,11 @@ impl LinearMemory for VMOwnedMemory { self.mmap.grow(delta, self.config.clone()) } + fn reset(&mut self) -> Result<(), MemoryError> { + self.mmap.reset()?; + Ok(()) + } + /// Return a `VMMemoryDefinition` for exposing the memory to compiled wasm code. fn vmmemory(&self) -> NonNull { self.mmap.vm_memory_definition.as_ptr() @@ -418,6 +428,13 @@ impl LinearMemory for VMSharedMemory { guard.size() } + /// Resets the memory back down to zero size + fn reset(&mut self) -> Result<(), MemoryError> { + let mut guard = self.mmap.write().unwrap(); + guard.reset(); + Ok(()) + } + /// Returns the memory style for this memory. fn style(&self) -> MemoryStyle { self.config.style() @@ -503,6 +520,12 @@ impl LinearMemory for VMMemory { self.0.grow(delta) } + /// Resets the memory down to a zero size + fn reset(&mut self) -> Result<(), MemoryError> { + self.0.reset(); + OK(()) + } + /// Returns the memory style for this memory. fn style(&self) -> MemoryStyle { self.0.style() diff --git a/lib/vm/src/memory.rs b/lib/vm/src/memory.rs index 37a6551b886..001e8a3c9f8 100644 --- a/lib/vm/src/memory.rs +++ b/lib/vm/src/memory.rs @@ -120,6 +120,12 @@ impl WasmMmap { Ok(prev_pages) } + /// Resets the memory down to a zero size + fn reset(&mut self) -> Result<(), MemoryError> { + self.size.0 = 0; + Ok(()) + } + /// Copies the memory /// (in this case it performs a copy-on-write to save memory) pub fn copy(&mut self) -> Result { @@ -326,6 +332,12 @@ impl LinearMemory for VMOwnedMemory { self.mmap.grow(delta, self.config.clone()) } + /// Resets the memory down to a zero size + fn reset(&mut self) -> Result<(), MemoryError> { + self.mmap.reset()?; + Ok(()) + } + /// Return a `VMMemoryDefinition` for exposing the memory to compiled wasm code. fn vmmemory(&self) -> NonNull { self.mmap.vm_memory_definition.as_ptr() @@ -422,6 +434,13 @@ impl LinearMemory for VMSharedMemory { guard.grow(delta, self.config.clone()) } + /// Resets the memory down to a zero size + fn reset(&mut self) -> Result<(), MemoryError> { + let mut guard = self.mmap.write().unwrap(); + guard.reset()?; + Ok(()) + } + /// Return a `VMMemoryDefinition` for exposing the memory to compiled wasm code. fn vmmemory(&self) -> NonNull { let guard = self.mmap.read().unwrap(); @@ -495,6 +514,12 @@ impl LinearMemory for VMMemory { self.0.grow(delta) } + /// Resets the memory down to a zero size + fn reset(&mut self) -> Result<(), MemoryError> { + self.0.reset()?; + Ok(()) + } + /// Returns the memory style for this memory. fn style(&self) -> MemoryStyle { self.0.style() @@ -633,6 +658,9 @@ where /// of wasm pages. fn grow(&mut self, delta: Pages) -> Result; + /// Resets the memory back to zero length + fn reset(&mut self) -> Result<(), MemoryError>; + /// Return a `VMMemoryDefinition` for exposing the memory to compiled wasm code. fn vmmemory(&self) -> NonNull; diff --git a/lib/wasix/src/bin_factory/exec.rs b/lib/wasix/src/bin_factory/exec.rs index 044adc528eb..eda19f8d24f 100644 --- a/lib/wasix/src/bin_factory/exec.rs +++ b/lib/wasix/src/bin_factory/exec.rs @@ -120,7 +120,9 @@ pub fn spawn_exec_module( }; // Bootstrap the process - let rewind_state = match ctx.bootstrap(&mut store) { + // Unsafe: The bootstrap must be executed in the same thread that runs the + // actual WASM code + let rewind_state = match unsafe { ctx.bootstrap(&mut store) } { Ok(r) => r, Err(err) => { thread.thread.set_status_finished(Err(err)); diff --git a/lib/wasix/src/journal/effector/memory_and_snapshot.rs b/lib/wasix/src/journal/effector/memory_and_snapshot.rs index 7659a8b8814..1160232ce73 100644 --- a/lib/wasix/src/journal/effector/memory_and_snapshot.rs +++ b/lib/wasix/src/journal/effector/memory_and_snapshot.rs @@ -70,7 +70,10 @@ impl JournalEffector { Ok(()) } - pub fn apply_memory( + /// Safety: This function manipulates the memory of the process and thus must + /// be executed by the WASM process thread itself. + /// + pub unsafe fn apply_memory( ctx: &mut FunctionEnvMut<'_, WasiEnv>, region: Range, data: &[u8], diff --git a/lib/wasix/src/journal/effector/process_exit.rs b/lib/wasix/src/journal/effector/process_exit.rs index fb3facac6b8..eabb2b45287 100644 --- a/lib/wasix/src/journal/effector/process_exit.rs +++ b/lib/wasix/src/journal/effector/process_exit.rs @@ -10,7 +10,14 @@ impl JournalEffector { Ok(()) } - pub fn apply_process_exit(env: &WasiEnv, exit_code: Option) -> anyhow::Result<()> { + /// Safety: This function manipulates the memory of the process and thus must + /// be executed by the WASM process thread itself. + /// + pub unsafe fn apply_process_exit( + ctx: &mut FunctionEnvMut<'_, WasiEnv>, + exit_code: Option, + ) -> anyhow::Result<()> { + let env = ctx.data(); // If we are in the phase of replaying journals then we // close all the file descriptors but we don't actually send // any signals @@ -20,6 +27,10 @@ impl JournalEffector { } else { env.blocking_on_exit(exit_code); } + + // Reset the memory back to a zero size + let memory = ctx.data_mut().inner().memory().clone(); + memory.reset(ctx)?; Ok(()) } } diff --git a/lib/wasix/src/state/builder.rs b/lib/wasix/src/state/builder.rs index 71a40e82776..eb4e68daac2 100644 --- a/lib/wasix/src/state/builder.rs +++ b/lib/wasix/src/state/builder.rs @@ -948,7 +948,9 @@ impl WasiEnvBuilder { let (instance, env) = self.instantiate_ext(module, module_hash, store)?; // Bootstrap the process - let rewind_state = env.bootstrap(store)?; + // Unsafe: The bootstrap must be executed in the same thread that runs the + // actual WASM code + let rewind_state = unsafe { env.bootstrap(store)? }; if rewind_state.is_some() { let mut ctx = env.env.clone().into_mut(store); rewind_ext2(&mut ctx, rewind_state) @@ -1002,9 +1004,6 @@ impl WasiEnvBuilder { env.data(&store).thread.set_status_running(); - // Bootstrap the process - let rewind_state = env.bootstrap(&mut store)?; - let tasks = env.data(&store).tasks().clone(); let pid = env.data(&store).pid(); let tid = env.data(&store).tid(); @@ -1015,6 +1014,18 @@ impl WasiEnvBuilder { let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel(); tasks.task_dedicated(Box::new(move || { + // Unsafe: The bootstrap must be executed in the same thread that runs the + // actual WASM code + let rewind_state = unsafe { + match env.bootstrap(&mut store) { + Ok(a) => a, + Err(err) => { + env.on_exit(&mut store, None); + tx.send(Err(err)).ok(); + return; + } + } + }; run_with_deep_sleep(store, rewind_state, env, tx); }))?; diff --git a/lib/wasix/src/state/func_env.rs b/lib/wasix/src/state/func_env.rs index d5b805fded8..1e3c5f97711 100644 --- a/lib/wasix/src/state/func_env.rs +++ b/lib/wasix/src/state/func_env.rs @@ -238,8 +238,12 @@ impl WasiFunctionEnv { /// Bootstraps this main thread and context with any journals that /// may be present + /// + /// Safety: This function manipulates the memory of the process and thus must + /// be executed by the WASM process thread itself. + /// #[allow(clippy::result_large_err)] - pub fn bootstrap( + pub unsafe fn bootstrap( &self, mut store: &'_ mut impl AsStoreMut, ) -> Result { diff --git a/lib/wasix/src/syscalls/mod.rs b/lib/wasix/src/syscalls/mod.rs index a21f8af8362..58672ec51e1 100644 --- a/lib/wasix/src/syscalls/mod.rs +++ b/lib/wasix/src/syscalls/mod.rs @@ -1324,25 +1324,29 @@ pub fn anyhow_err_to_runtime_err(err: anyhow::Error) -> WasiRuntimeError { WasiRuntimeError::Runtime(RuntimeError::user(err.into())) } +/// Safety: This function manipulates the memory of the process and thus must +/// be executed by the WASM process thread itself. +/// #[allow(clippy::result_large_err)] #[cfg(feature = "journal")] -pub fn restore_snapshot( +pub unsafe fn restore_snapshot( mut ctx: FunctionEnvMut<'_, WasiEnv>, journal: Arc, ) -> Result, WasiRuntimeError> { use crate::journal::Journal; - let mut is_same_module = true; + let cur_module_hash = Some(ctx.data().process.module_hash.as_bytes()); + let mut journal_module_hash = None; let mut rewind = None; while let Some(next) = journal.read().map_err(anyhow_err_to_runtime_err)? { tracing::trace!("Restoring snapshot event - {next:?}"); match next { crate::journal::JournalEntry::InitModule { wasm_hash } => { - is_same_module = ctx.data().process.module_hash.as_bytes()[0..16] == wasm_hash; + journal_module_hash.replace(wasm_hash); } crate::journal::JournalEntry::ProcessExit { exit_code } => { rewind = None; - JournalEffector::apply_process_exit(ctx.data(), exit_code) + JournalEffector::apply_process_exit(&mut ctx, exit_code) .map_err(anyhow_err_to_runtime_err)?; } crate::journal::JournalEntry::FileDescriptorWrite { @@ -1363,7 +1367,7 @@ pub fn restore_snapshot( .map_err(anyhow_err_to_runtime_err)?; } crate::journal::JournalEntry::UpdateMemoryRegion { region, data } => { - if !is_same_module { + if cur_module_hash != journal_module_hash { continue; } JournalEffector::apply_memory(&mut ctx, region, &data) @@ -1371,7 +1375,7 @@ pub fn restore_snapshot( } crate::journal::JournalEntry::CloseThread { id, exit_code } => { if id == ctx.data().tid() { - JournalEffector::apply_process_exit(ctx.data(), exit_code) + JournalEffector::apply_process_exit(&mut ctx, exit_code) .map_err(anyhow_err_to_runtime_err)?; } else { JournalEffector::apply_thread_exit(&mut ctx, id, exit_code) @@ -1385,7 +1389,7 @@ pub fn restore_snapshot( store_data, is_64bit, } => { - if !is_same_module { + if cur_module_hash != journal_module_hash { continue; } if id == ctx.data().tid() { @@ -1451,7 +1455,7 @@ pub fn restore_snapshot( when: _, trigger: _, } => { - if !is_same_module { + if cur_module_hash != journal_module_hash { continue; } } @@ -1750,11 +1754,13 @@ pub fn restore_snapshot( // If we are not in the same module then we fire off an exit // that simulates closing the process (hence keeps everything // in a clean state) - if !is_same_module { + if journal_module_hash.is_some() && cur_module_hash != journal_module_hash { eprintln!( - "The WASM module hash does not match the journal module hash - forcing a restart" + "The WASM module hash does not match the journal module hash (journal_hash={:x?} vs module_hash{:x?}) - forcing a restart", + journal_module_hash.unwrap(), + cur_module_hash.unwrap() ); - JournalEffector::apply_process_exit(ctx.data(), None).map_err(anyhow_err_to_runtime_err)?; + JournalEffector::apply_process_exit(&mut ctx, None).map_err(anyhow_err_to_runtime_err)?; rewind = None; } else { tracing::debug!( From fc19f5f38b460489ea4abadab0271ff8b9677f91 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Fri, 24 Nov 2023 10:39:34 +1100 Subject: [PATCH 086/129] Linting fixes --- lib/cli/src/commands/journal/filter.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/cli/src/commands/journal/filter.rs b/lib/cli/src/commands/journal/filter.rs index 9a94854150f..d383b932dcc 100644 --- a/lib/cli/src/commands/journal/filter.rs +++ b/lib/cli/src/commands/journal/filter.rs @@ -74,7 +74,7 @@ impl CmdJournalFilter { .to_string_lossy() .to_string(); temp_filename.insert_str(0, ".staging."); - let temp_path = self.target_path.clone().with_file_name(&temp_filename); + let temp_path = self.target_path.with_file_name(&temp_filename); std::fs::remove_file(&temp_path).ok(); // Load the source journal and the target journal (in the temp location) @@ -100,7 +100,7 @@ impl CmdJournalFilter { std::fs::rename(temp_path, self.target_path.clone())?; // Now print the outcome - let journal = LogFileJournal::new(self.target_path.clone())?; + let journal = LogFileJournal::new(&self.target_path)?; let printer = PrintingJournal::default(); copy_journal(&journal, &printer)?; Ok(()) From 4abc21556df3b86225b59daa0bb622e59f42cd16 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Fri, 24 Nov 2023 11:59:05 +1100 Subject: [PATCH 087/129] More compile fixes --- lib/api/src/js/externals/memory.rs | 4 ++++ lib/api/src/jsc/externals/memory.rs | 4 ++++ lib/sys-utils/src/memory/fd_memory/memories.rs | 2 +- 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/lib/api/src/js/externals/memory.rs b/lib/api/src/js/externals/memory.rs index a6aed3deb54..183faa8bf4b 100644 --- a/lib/api/src/js/externals/memory.rs +++ b/lib/api/src/js/externals/memory.rs @@ -144,6 +144,10 @@ impl Memory { Ok(Pages(new_pages)) } + pub fn reset(&self, _store: &mut impl AsStoreMut) -> Result<(), MemoryError> { + Ok(()) + } + pub(crate) fn from_vm_extern(_store: &mut impl AsStoreMut, internal: VMMemory) -> Self { Self { handle: internal } } diff --git a/lib/api/src/jsc/externals/memory.rs b/lib/api/src/jsc/externals/memory.rs index 38fcc952e3d..0e153b341f6 100644 --- a/lib/api/src/jsc/externals/memory.rs +++ b/lib/api/src/jsc/externals/memory.rs @@ -140,6 +140,10 @@ impl Memory { // Ok(Pages(new_pages)) } + pub fn reset(&self, _store: &mut impl AsStoreMut) -> Result<(), MemoryError> { + Ok(()) + } + pub fn copy_to_store( &self, store: &impl AsStoreRef, diff --git a/lib/sys-utils/src/memory/fd_memory/memories.rs b/lib/sys-utils/src/memory/fd_memory/memories.rs index 32844df6130..8d8ac926078 100644 --- a/lib/sys-utils/src/memory/fd_memory/memories.rs +++ b/lib/sys-utils/src/memory/fd_memory/memories.rs @@ -523,7 +523,7 @@ impl LinearMemory for VMMemory { /// Resets the memory down to a zero size fn reset(&mut self) -> Result<(), MemoryError> { self.0.reset(); - OK(()) + Ok(()) } /// Returns the memory style for this memory. From 52ee3ea168525675ecf673393f7713f6f5f931f4 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Fri, 24 Nov 2023 12:02:57 +1100 Subject: [PATCH 088/129] Module mismatch is now a warning --- lib/wasix/src/syscalls/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/wasix/src/syscalls/mod.rs b/lib/wasix/src/syscalls/mod.rs index 58672ec51e1..54a74cd7e8f 100644 --- a/lib/wasix/src/syscalls/mod.rs +++ b/lib/wasix/src/syscalls/mod.rs @@ -1755,7 +1755,7 @@ pub unsafe fn restore_snapshot( // that simulates closing the process (hence keeps everything // in a clean state) if journal_module_hash.is_some() && cur_module_hash != journal_module_hash { - eprintln!( + tracing::error!( "The WASM module hash does not match the journal module hash (journal_hash={:x?} vs module_hash{:x?}) - forcing a restart", journal_module_hash.unwrap(), cur_module_hash.unwrap() From ec617dbb4a7e920c4615eeb0f4e554e533161c1d Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Mon, 27 Nov 2023 08:38:19 +1100 Subject: [PATCH 089/129] We now restore the TTY state after the program executes --- lib/cli/src/commands/run/mod.rs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/lib/cli/src/commands/run/mod.rs b/lib/cli/src/commands/run/mod.rs index 7b3a97f8176..267844f1f6d 100644 --- a/lib/cli/src/commands/run/mod.rs +++ b/lib/cli/src/commands/run/mod.rs @@ -121,17 +121,25 @@ impl Run { pb.finish_and_clear(); + // push the TTY state so we can restore it after the program finishes + let tty = runtime.tty().map(|tty| tty.tty_get()); + let result = { match target { ExecutableTarget::WebAssembly { module, module_hash, path, - } => self.execute_wasm(&path, &module, module_hash, store, runtime), - ExecutableTarget::Package(pkg) => self.execute_webc(&pkg, runtime), + } => self.execute_wasm(&path, &module, module_hash, store, runtime.clone()), + ExecutableTarget::Package(pkg) => self.execute_webc(&pkg, runtime.clone()), } }; + // restore the TTY state as teh execution may have changed it + if let Some(state) = tty { + runtime.tty().map(|tty| tty.tty_set(state)); + } + if let Err(e) = &result { self.maybe_save_coredump(e); } From 1fea8d7792f02b616f5c927673ba153ee19bf7ca Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Mon, 27 Nov 2023 09:37:45 +1100 Subject: [PATCH 090/129] Fixed an issue where the restoration of the journal was not properly handling the boostrap process --- lib/wasix/src/runtime/mod.rs | 1 + lib/wasix/src/state/env.rs | 2 +- lib/wasix/src/state/func_env.rs | 2 +- lib/wasix/src/syscalls/mod.rs | 111 +++++++++++++++++++++++++------- 4 files changed, 91 insertions(+), 25 deletions(-) diff --git a/lib/wasix/src/runtime/mod.rs b/lib/wasix/src/runtime/mod.rs index c84f122c862..1c60b342f54 100644 --- a/lib/wasix/src/runtime/mod.rs +++ b/lib/wasix/src/runtime/mod.rs @@ -355,6 +355,7 @@ impl Runtime for PluggableRuntime { #[cfg(feature = "journal")] fn pop_restore_journals(&self) -> &'_ Vec> { + tracing::error!("TODO: remove pop_restore_journals"); use std::sync::atomic::Ordering; if self .restored_journals diff --git a/lib/wasix/src/state/env.rs b/lib/wasix/src/state/env.rs index d3498e801d7..0630a9adb6e 100644 --- a/lib/wasix/src/state/env.rs +++ b/lib/wasix/src/state/env.rs @@ -874,7 +874,7 @@ impl WasiEnv { /// Returns the active journal or fails with an error #[cfg(feature = "journal")] pub fn active_journal(&self) -> Result<&DynJournal, Errno> { - self.runtime().active_journal().ok_or({ + self.runtime().active_journal().ok_or_else(|| { tracing::warn!("failed to save thread exit as there is not active journal"); Errno::Fault }) diff --git a/lib/wasix/src/state/func_env.rs b/lib/wasix/src/state/func_env.rs index 1e3c5f97711..3db7171efa1 100644 --- a/lib/wasix/src/state/func_env.rs +++ b/lib/wasix/src/state/func_env.rs @@ -259,7 +259,7 @@ impl WasiFunctionEnv { for journal in restore_journals { let ctx = self.env.clone().into_mut(&mut store); - let rewind = match restore_snapshot(ctx, journal) { + let rewind = match restore_snapshot(ctx, journal, true) { Ok(r) => r, Err(err) => { self.data_mut(&mut store).replaying_journal = false; diff --git a/lib/wasix/src/syscalls/mod.rs b/lib/wasix/src/syscalls/mod.rs index 54a74cd7e8f..7b7a43ad634 100644 --- a/lib/wasix/src/syscalls/mod.rs +++ b/lib/wasix/src/syscalls/mod.rs @@ -1332,9 +1332,23 @@ pub fn anyhow_err_to_runtime_err(err: anyhow::Error) -> WasiRuntimeError { pub unsafe fn restore_snapshot( mut ctx: FunctionEnvMut<'_, WasiEnv>, journal: Arc, + bootstrapping: bool, ) -> Result, WasiRuntimeError> { + use std::ops::Range; + use crate::journal::Journal; + // We delay the spawning of threads until the end as its + // possible that the threads will be cancelled before all the + // events finished the streaming process + let mut spawn_threads: HashMap = Default::default(); + + // We delay the memory updates until the end as its possible the + // memory will be cleared before all the events finished the + // streaming process + let mut update_memory: HashMap, Cow<'_, [u8]>> = Default::default(); + + // Loop through all the events and process them let cur_module_hash = Some(ctx.data().process.module_hash.as_bytes()); let mut journal_module_hash = None; let mut rewind = None; @@ -1345,9 +1359,14 @@ pub unsafe fn restore_snapshot( journal_module_hash.replace(wasm_hash); } crate::journal::JournalEntry::ProcessExit { exit_code } => { - rewind = None; - JournalEffector::apply_process_exit(&mut ctx, exit_code) - .map_err(anyhow_err_to_runtime_err)?; + if bootstrapping { + rewind = None; + spawn_threads.clear(); + update_memory.clear(); + } else { + JournalEffector::apply_process_exit(&mut ctx, exit_code) + .map_err(anyhow_err_to_runtime_err)?; + } } crate::journal::JournalEntry::FileDescriptorWrite { fd, @@ -1370,16 +1389,31 @@ pub unsafe fn restore_snapshot( if cur_module_hash != journal_module_hash { continue; } - JournalEffector::apply_memory(&mut ctx, region, &data) - .map_err(anyhow_err_to_runtime_err)?; + + if bootstrapping { + update_memory.insert(region, data.clone()); + } else { + JournalEffector::apply_memory(&mut ctx, region, &data) + .map_err(anyhow_err_to_runtime_err)?; + } } crate::journal::JournalEntry::CloseThread { id, exit_code } => { if id == ctx.data().tid() { - JournalEffector::apply_process_exit(&mut ctx, exit_code) - .map_err(anyhow_err_to_runtime_err)?; + if bootstrapping { + rewind = None; + spawn_threads.clear(); + update_memory.clear(); + } else { + JournalEffector::apply_process_exit(&mut ctx, exit_code) + .map_err(anyhow_err_to_runtime_err)?; + } } else { - JournalEffector::apply_thread_exit(&mut ctx, id, exit_code) - .map_err(anyhow_err_to_runtime_err)?; + if bootstrapping { + spawn_threads.remove(&id); + } else { + JournalEffector::apply_thread_exit(&mut ctx, id, exit_code) + .map_err(anyhow_err_to_runtime_err)?; + } } } crate::journal::JournalEntry::SetThread { @@ -1392,20 +1426,27 @@ pub unsafe fn restore_snapshot( if cur_module_hash != journal_module_hash { continue; } + + let state = RewindState { + memory_stack: memory_stack.to_vec().into(), + rewind_stack: call_stack.to_vec().into(), + store_data: store_data.to_vec().into(), + is_64bit, + }; + if id == ctx.data().tid() { - rewind.replace(RewindState { - memory_stack: memory_stack.to_vec().into(), - rewind_stack: call_stack.to_vec().into(), - store_data: store_data.to_vec().into(), - is_64bit, - }); + rewind.replace(state); } else { - return Err(WasiRuntimeError::Runtime(RuntimeError::user( - anyhow::format_err!( - "Snapshot restoration does not currently support multiple threads." - ) - .into(), - ))); + if bootstrapping { + spawn_threads.insert(id, state); + } else { + return Err(WasiRuntimeError::Runtime(RuntimeError::user( + anyhow::format_err!( + "Snapshot restoration does not currently support live updates of running threads." + ) + .into(), + ))); + } } } crate::journal::JournalEntry::CloseFileDescriptor { fd } => { @@ -1751,6 +1792,7 @@ pub unsafe fn restore_snapshot( .map_err(anyhow_err_to_runtime_err)?, } } + // If we are not in the same module then we fire off an exit // that simulates closing the process (hence keeps everything // in a clean state) @@ -1760,14 +1802,37 @@ pub unsafe fn restore_snapshot( journal_module_hash.unwrap(), cur_module_hash.unwrap() ); - JournalEffector::apply_process_exit(&mut ctx, None).map_err(anyhow_err_to_runtime_err)?; - rewind = None; + if bootstrapping { + rewind = None; + spawn_threads.clear(); + update_memory.clear(); + } else { + JournalEffector::apply_process_exit(&mut ctx, None) + .map_err(anyhow_err_to_runtime_err)?; + } } else { tracing::debug!( "journal used on a different module - the process will simulate a restart." ); } + // We do not yet support multi threading + if !spawn_threads.is_empty() { + return Err(WasiRuntimeError::Runtime(RuntimeError::user( + anyhow::format_err!( + "Snapshot restoration does not currently support multiple threads." + ) + .into(), + ))); + } + + // Next we apply all the memory updates that were delayed while the logs + // were processed to completion. + for (region, data) in update_memory { + JournalEffector::apply_memory(&mut ctx, region, &data) + .map_err(anyhow_err_to_runtime_err)?; + } + Ok(rewind) } From 489f82c1726dee49afbba0256ad34981691b33c1 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Mon, 27 Nov 2023 10:08:53 +1100 Subject: [PATCH 091/129] Fixed a linting issue --- lib/wasix/src/journal/effector/memory_and_snapshot.rs | 6 ++++-- lib/wasix/src/journal/effector/process_exit.rs | 6 ++++-- lib/wasix/src/state/func_env.rs | 6 ++++-- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/lib/wasix/src/journal/effector/memory_and_snapshot.rs b/lib/wasix/src/journal/effector/memory_and_snapshot.rs index 1160232ce73..fc4c69656eb 100644 --- a/lib/wasix/src/journal/effector/memory_and_snapshot.rs +++ b/lib/wasix/src/journal/effector/memory_and_snapshot.rs @@ -70,8 +70,10 @@ impl JournalEffector { Ok(()) } - /// Safety: This function manipulates the memory of the process and thus must - /// be executed by the WASM process thread itself. + /// # Safety + /// + /// This function manipulates the memory of the process and thus must be executed + /// by the WASM process thread itself. /// pub unsafe fn apply_memory( ctx: &mut FunctionEnvMut<'_, WasiEnv>, diff --git a/lib/wasix/src/journal/effector/process_exit.rs b/lib/wasix/src/journal/effector/process_exit.rs index eabb2b45287..837d7130294 100644 --- a/lib/wasix/src/journal/effector/process_exit.rs +++ b/lib/wasix/src/journal/effector/process_exit.rs @@ -10,8 +10,10 @@ impl JournalEffector { Ok(()) } - /// Safety: This function manipulates the memory of the process and thus must - /// be executed by the WASM process thread itself. + /// # Safety + /// + /// This function manipulates the memory of the process and thus must be executed + /// by the WASM process thread itself. /// pub unsafe fn apply_process_exit( ctx: &mut FunctionEnvMut<'_, WasiEnv>, diff --git a/lib/wasix/src/state/func_env.rs b/lib/wasix/src/state/func_env.rs index 3db7171efa1..e75ff789c7b 100644 --- a/lib/wasix/src/state/func_env.rs +++ b/lib/wasix/src/state/func_env.rs @@ -239,8 +239,10 @@ impl WasiFunctionEnv { /// Bootstraps this main thread and context with any journals that /// may be present /// - /// Safety: This function manipulates the memory of the process and thus must - /// be executed by the WASM process thread itself. + /// # Safety + /// + /// This function manipulates the memory of the process and thus must be executed + /// by the WASM process thread itself. /// #[allow(clippy::result_large_err)] pub unsafe fn bootstrap( From 3eaabd96790ed09b8a6a72baf9b5ea43167702ea Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Mon, 27 Nov 2023 10:15:44 +1100 Subject: [PATCH 092/129] The TTY is now delayed during the journal bootstrap process --- lib/wasix/src/syscalls/mod.rs | 42 ++++++++++++++++++++++------------- 1 file changed, 26 insertions(+), 16 deletions(-) diff --git a/lib/wasix/src/syscalls/mod.rs b/lib/wasix/src/syscalls/mod.rs index 7b7a43ad634..b694a32b590 100644 --- a/lib/wasix/src/syscalls/mod.rs +++ b/lib/wasix/src/syscalls/mod.rs @@ -1347,6 +1347,7 @@ pub unsafe fn restore_snapshot( // memory will be cleared before all the events finished the // streaming process let mut update_memory: HashMap, Cow<'_, [u8]>> = Default::default(); + let mut update_tty = None; // Loop through all the events and process them let cur_module_hash = Some(ctx.data().process.module_hash.as_bytes()); @@ -1363,6 +1364,7 @@ pub unsafe fn restore_snapshot( rewind = None; spawn_threads.clear(); update_memory.clear(); + update_tty.take(); } else { JournalEffector::apply_process_exit(&mut ctx, exit_code) .map_err(anyhow_err_to_runtime_err)?; @@ -1403,6 +1405,7 @@ pub unsafe fn restore_snapshot( rewind = None; spawn_threads.clear(); update_memory.clear(); + update_tty.take(); } else { JournalEffector::apply_process_exit(&mut ctx, exit_code) .map_err(anyhow_err_to_runtime_err)?; @@ -1616,22 +1619,25 @@ pub unsafe fn restore_snapshot( .map_err(anyhow_err_to_runtime_err)?; } crate::journal::JournalEntry::TtySet { tty, line_feeds } => { - JournalEffector::apply_tty_set( - &mut ctx, - crate::WasiTtyState { - cols: tty.cols, - rows: tty.rows, - width: tty.width, - height: tty.height, - stdin_tty: tty.stdin_tty, - stdout_tty: tty.stdout_tty, - stderr_tty: tty.stderr_tty, - echo: tty.echo, - line_buffered: tty.line_buffered, - line_feeds, - }, - ) - .map_err(anyhow_err_to_runtime_err)?; + let state = crate::WasiTtyState { + cols: tty.cols, + rows: tty.rows, + width: tty.width, + height: tty.height, + stdin_tty: tty.stdin_tty, + stdout_tty: tty.stdout_tty, + stderr_tty: tty.stderr_tty, + echo: tty.echo, + line_buffered: tty.line_buffered, + line_feeds, + }; + + if bootstrapping { + update_tty.replace(state); + } else { + JournalEffector::apply_tty_set(&mut ctx, state) + .map_err(anyhow_err_to_runtime_err)?; + } } crate::journal::JournalEntry::PortAddAddr { cidr } => { JournalEffector::apply_port_addr_add(&mut ctx, cidr) @@ -1806,6 +1812,7 @@ pub unsafe fn restore_snapshot( rewind = None; spawn_threads.clear(); update_memory.clear(); + update_tty.take(); } else { JournalEffector::apply_process_exit(&mut ctx, None) .map_err(anyhow_err_to_runtime_err)?; @@ -1832,6 +1839,9 @@ pub unsafe fn restore_snapshot( JournalEffector::apply_memory(&mut ctx, region, &data) .map_err(anyhow_err_to_runtime_err)?; } + if let Some(state) = update_tty { + JournalEffector::apply_tty_set(&mut ctx, state).map_err(anyhow_err_to_runtime_err)?; + } Ok(rewind) } From e1c2570ceb2f0ab36afcb49cecb0660ef6ad2419 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Mon, 27 Nov 2023 10:20:18 +1100 Subject: [PATCH 093/129] Fixed another linting issue --- lib/wasix/src/syscalls/mod.rs | 28 ++++++++++++---------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/lib/wasix/src/syscalls/mod.rs b/lib/wasix/src/syscalls/mod.rs index b694a32b590..d64243d8dcd 100644 --- a/lib/wasix/src/syscalls/mod.rs +++ b/lib/wasix/src/syscalls/mod.rs @@ -1410,13 +1410,11 @@ pub unsafe fn restore_snapshot( JournalEffector::apply_process_exit(&mut ctx, exit_code) .map_err(anyhow_err_to_runtime_err)?; } + } else if bootstrapping { + spawn_threads.remove(&id); } else { - if bootstrapping { - spawn_threads.remove(&id); - } else { - JournalEffector::apply_thread_exit(&mut ctx, id, exit_code) - .map_err(anyhow_err_to_runtime_err)?; - } + JournalEffector::apply_thread_exit(&mut ctx, id, exit_code) + .map_err(anyhow_err_to_runtime_err)?; } } crate::journal::JournalEntry::SetThread { @@ -1439,17 +1437,15 @@ pub unsafe fn restore_snapshot( if id == ctx.data().tid() { rewind.replace(state); + } else if bootstrapping { + spawn_threads.insert(id, state); } else { - if bootstrapping { - spawn_threads.insert(id, state); - } else { - return Err(WasiRuntimeError::Runtime(RuntimeError::user( - anyhow::format_err!( - "Snapshot restoration does not currently support live updates of running threads." - ) - .into(), - ))); - } + return Err(WasiRuntimeError::Runtime(RuntimeError::user( + anyhow::format_err!( + "Snapshot restoration does not currently support live updates of running threads." + ) + .into(), + ))); } } crate::journal::JournalEntry::CloseFileDescriptor { fd } => { From 88c5d8cf27590fb020f16c8359f7e72c1c24c066 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Mon, 27 Nov 2023 10:24:54 +1100 Subject: [PATCH 094/129] Fixed the padding on the change directory event --- lib/wasix/src/journal/concrete/archived.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/lib/wasix/src/journal/concrete/archived.rs b/lib/wasix/src/journal/concrete/archived.rs index d63207796e1..be10c400ef4 100644 --- a/lib/wasix/src/journal/concrete/archived.rs +++ b/lib/wasix/src/journal/concrete/archived.rs @@ -641,7 +641,6 @@ impl<'a> JournalEntry<'a> { }), JournalEntry::ChangeDirectory { path } => { serializer.serialize_value(&JournalEntryChangeDirectoryV1 { - _padding: padding(path.as_bytes().len()), path: path.into_owned(), }) } @@ -1210,7 +1209,6 @@ pub struct JournalEntryPathRenameV1 { #[archive_attr(derive(CheckBytes))] pub struct JournalEntryChangeDirectoryV1 { pub path: String, - pub _padding: Vec, } #[repr(C)] @@ -2565,7 +2563,6 @@ impl<'a> From> for JournalEntry<'a> { }, ArchivedJournalEntry::ChangeDirectoryV1(ArchivedJournalEntryChangeDirectoryV1 { path, - _padding: _, }) => Self::ChangeDirectory { path: path.as_ref().into(), }, From 46f1503b0d7d7300011b22c3568482e093c4e7ce Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Mon, 27 Nov 2023 10:27:43 +1100 Subject: [PATCH 095/129] Removed the pop journals call which is not needed --- lib/wasix/src/runtime/mod.rs | 26 +------------------------- lib/wasix/src/state/func_env.rs | 2 +- 2 files changed, 2 insertions(+), 26 deletions(-) diff --git a/lib/wasix/src/runtime/mod.rs b/lib/wasix/src/runtime/mod.rs index 1c60b342f54..088c82ab356 100644 --- a/lib/wasix/src/runtime/mod.rs +++ b/lib/wasix/src/runtime/mod.rs @@ -11,7 +11,7 @@ use self::{ use std::{ fmt, - sync::{atomic::AtomicBool, Arc, Mutex}, + sync::{Arc, Mutex}, }; use derivative::Derivative; @@ -115,13 +115,6 @@ where &EMPTY_JOURNAL_LIST } - /// Pops the restore list of journals that will be replayed before the - /// instance gets going - #[cfg(feature = "journal")] - fn pop_restore_journals(&self) -> &'_ Vec> { - &EMPTY_JOURNAL_LIST - } - /// The snapshot capturer takes and restores snapshots of the WASM process at specific /// points in time by reading and writing log entries #[cfg(feature = "journal")] @@ -209,7 +202,6 @@ pub struct PluggableRuntime { #[cfg(feature = "journal")] #[derivative(Debug = "ignore")] pub journals: Vec>, - pub restored_journals: Arc, } impl PluggableRuntime { @@ -246,7 +238,6 @@ impl PluggableRuntime { module_cache: Arc::new(module_cache::in_memory()), #[cfg(feature = "journal")] journals: Vec::new(), - restored_journals: Arc::new(AtomicBool::new(false)), } } @@ -353,21 +344,6 @@ impl Runtime for PluggableRuntime { &self.journals } - #[cfg(feature = "journal")] - fn pop_restore_journals(&self) -> &'_ Vec> { - tracing::error!("TODO: remove pop_restore_journals"); - use std::sync::atomic::Ordering; - if self - .restored_journals - .compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst) - .is_ok() - { - &self.journals - } else { - &EMPTY_JOURNAL_LIST - } - } - #[cfg(feature = "journal")] fn active_journal(&self) -> Option<&DynJournal> { self.journals.iter().last().map(|a| a.as_ref()) diff --git a/lib/wasix/src/state/func_env.rs b/lib/wasix/src/state/func_env.rs index e75ff789c7b..d246e47e5c0 100644 --- a/lib/wasix/src/state/func_env.rs +++ b/lib/wasix/src/state/func_env.rs @@ -255,7 +255,7 @@ impl WasiFunctionEnv { { // If there are journals we need to restore then do so (this will // prevent the initialization function from running - let restore_journals = self.data(&store).runtime.pop_restore_journals().clone(); + let restore_journals = self.data(&store).runtime.journals().clone(); if !restore_journals.is_empty() { self.data_mut(&mut store).replaying_journal = true; From f62f56868dd009a9c3a17bf0e247d584584001d4 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Mon, 27 Nov 2023 12:24:10 +1100 Subject: [PATCH 096/129] Fixed a bug where the offsets were not updating properly on journal writes --- lib/wasix/src/journal/effector/syscalls/fd_write.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/wasix/src/journal/effector/syscalls/fd_write.rs b/lib/wasix/src/journal/effector/syscalls/fd_write.rs index a9f449457fc..9301a16357d 100644 --- a/lib/wasix/src/journal/effector/syscalls/fd_write.rs +++ b/lib/wasix/src/journal/effector/syscalls/fd_write.rs @@ -4,7 +4,7 @@ impl JournalEffector { pub fn save_fd_write( ctx: &FunctionEnvMut<'_, WasiEnv>, fd: Fd, - offset: u64, + mut offset: u64, written: usize, iovs: WasmPtr<__wasi_ciovec_t, M>, iovs_len: M::Offset, @@ -27,15 +27,20 @@ impl JournalEffector { .map_err(mem_error_to_wasi)? .access() .map_err(mem_error_to_wasi)?; + let data = Cow::Borrowed(buf.as_ref()); + let data_len = data.len(); + ctx.data() .active_journal()? .write(JournalEntry::FileDescriptorWrite { fd, offset, - data: Cow::Borrowed(buf.as_ref()), + data, is_64bit: M::is_64bit(), }) .map_err(map_snapshot_err)?; + + offset += data_len as u64; } Ok(()) } From 6ab87f9fd423c2d8bef581620ffd19eb24b1e4d0 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Mon, 27 Nov 2023 12:35:56 +1100 Subject: [PATCH 097/129] Fixed a bug where the module hash plumbing was not correct --- lib/wasix/src/runners/wasi.rs | 1 + lib/wasix/src/state/builder.rs | 22 ++++++++++++++++++++-- lib/wasix/src/state/env.rs | 11 ++++++----- 3 files changed, 27 insertions(+), 7 deletions(-) diff --git a/lib/wasix/src/runners/wasi.rs b/lib/wasix/src/runners/wasi.rs index ab2f16a7f46..93911957bb7 100644 --- a/lib/wasix/src/runners/wasi.rs +++ b/lib/wasix/src/runners/wasi.rs @@ -263,6 +263,7 @@ impl WasiRunner { let container_fs = if let Some(pkg) = pkg { builder.add_webc(pkg.clone()); + builder.set_module_hash(pkg.hash()); Some(Arc::clone(&pkg.webc_fs)) } else { None diff --git a/lib/wasix/src/state/builder.rs b/lib/wasix/src/state/builder.rs index eb4e68daac2..3f1a1885a6b 100644 --- a/lib/wasix/src/state/builder.rs +++ b/lib/wasix/src/state/builder.rs @@ -71,6 +71,8 @@ pub struct WasiEnvBuilder { /// List of webc dependencies to be injected. pub(super) uses: Vec, + pub(super) module_hash: Option, + /// List of host commands to map into the WASI instance. pub(super) map_commands: HashMap, @@ -294,6 +296,14 @@ impl WasiEnvBuilder { self } + /// Sets the module hash for the running process. This ensures that the journal + /// can restore the records for the right module. If no module hash is supplied + /// then the process will start with a random module hash. + pub fn set_module_hash(&mut self, hash: ModuleHash) -> &mut Self { + self.module_hash.replace(hash); + self + } + /// Adds a container this module inherits from. /// /// This will make all of the container's files and commands available to the @@ -857,8 +867,12 @@ impl WasiEnvBuilder { #[allow(clippy::result_large_err)] pub fn build(self) -> Result { + let module_hash = self + .module_hash + .clone() + .unwrap_or_else(|| ModuleHash::random()); let init = self.build_init()?; - WasiEnv::from_init(init) + WasiEnv::from_init(init, module_hash) } /// Construct a [`WasiFunctionEnv`]. @@ -871,8 +885,12 @@ impl WasiEnvBuilder { self, store: &mut impl AsStoreMut, ) -> Result { + let module_hash = self + .module_hash + .clone() + .unwrap_or_else(|| ModuleHash::random()); let init = self.build_init()?; - let env = WasiEnv::from_init(init)?; + let env = WasiEnv::from_init(init, module_hash)?; let func_env = WasiFunctionEnv::new(store, env); Ok(func_env) } diff --git a/lib/wasix/src/state/env.rs b/lib/wasix/src/state/env.rs index 0630a9adb6e..f765544822c 100644 --- a/lib/wasix/src/state/env.rs +++ b/lib/wasix/src/state/env.rs @@ -430,11 +430,14 @@ impl WasiEnv { } #[allow(clippy::result_large_err)] - pub(crate) fn from_init(init: WasiEnvInit) -> Result { + pub(crate) fn from_init( + init: WasiEnvInit, + module_hash: ModuleHash, + ) -> Result { let process = if let Some(p) = init.process { p } else { - init.control_plane.new_process(ModuleHash::random())? + init.control_plane.new_process(module_hash)? }; let layout = WasiMemoryLayout::default(); @@ -496,9 +499,7 @@ impl WasiEnv { } } - let mut env = Self::from_init(init)?; - env.process.module_hash = module_hash; - + let env = Self::from_init(init, module_hash)?; let pid = env.process.pid(); let mut store = store.as_store_mut(); From fd4f55eddb874e80f9fbc165dad0a3e659062de4 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Mon, 27 Nov 2023 17:04:08 +1100 Subject: [PATCH 098/129] Implemented fixes so the journal restoration works properly --- lib/api/src/externals/memory.rs | 10 + lib/api/src/sys/externals/memory.rs | 10 + lib/api/src/sys/tunables.rs | 12 ++ .../src/memory/fd_memory/memories.rs | 38 +++- lib/vm/src/memory.rs | 38 +++- lib/wasix/src/journal/concrete/compacting.rs | 175 +++++++++++------- .../journal/effector/memory_and_snapshot.rs | 6 +- lib/wasix/src/journal/snapshot.rs | 11 ++ lib/wasix/src/state/env.rs | 16 +- lib/wasix/src/syscalls/mod.rs | 7 +- 10 files changed, 250 insertions(+), 73 deletions(-) diff --git a/lib/api/src/externals/memory.rs b/lib/api/src/externals/memory.rs index c852f0ae79e..1c49d866b1d 100644 --- a/lib/api/src/externals/memory.rs +++ b/lib/api/src/externals/memory.rs @@ -121,6 +121,16 @@ impl Memory { self.0.grow(store, delta) } + /// Grows the memory to at least a minimum size. If the memory is already big enough + /// for the min size then this function does nothing + pub fn grow_at_least( + &self, + store: &mut impl AsStoreMut, + min_size: u64, + ) -> Result<(), MemoryError> { + self.0.grow_at_least(store, min_size) + } + /// Resets the memory back to zero length pub fn reset(&self, store: &mut impl AsStoreMut) -> Result<(), MemoryError> { self.0.reset(store)?; diff --git a/lib/api/src/sys/externals/memory.rs b/lib/api/src/sys/externals/memory.rs index 19c7e591019..148c7fe4e67 100644 --- a/lib/api/src/sys/externals/memory.rs +++ b/lib/api/src/sys/externals/memory.rs @@ -51,6 +51,16 @@ impl Memory { self.handle.get_mut(store.objects_mut()).grow(delta.into()) } + pub fn grow_at_least( + &self, + store: &mut impl AsStoreMut, + min_size: u64, + ) -> Result<(), MemoryError> { + self.handle + .get_mut(store.objects_mut()) + .grow_at_least(min_size) + } + pub fn reset(&self, store: &mut impl AsStoreMut) -> Result<(), MemoryError> { self.handle.get_mut(store.objects_mut()).reset()?; Ok(()) diff --git a/lib/api/src/sys/tunables.rs b/lib/api/src/sys/tunables.rs index fb3d7a31c56..df63e3b1d80 100644 --- a/lib/api/src/sys/tunables.rs +++ b/lib/api/src/sys/tunables.rs @@ -107,6 +107,18 @@ mod tests { }) } + fn grow_at_least(&mut self, min_size: u64) -> Result<(), MemoryError> { + let cur_size = self.size().0 as u64 * WASM_PAGE_SIZE as u64; + if min_size > cur_size { + let delta = min_size - cur_size; + return Err(MemoryError::CouldNotGrow { + current: Pages::from(100u32), + attempted_delta: Pages(delta as u32), + }); + } + Ok(()) + } + fn reset(&mut self) -> Result<(), MemoryError> { self.mem.fill(0); Ok(()) diff --git a/lib/sys-utils/src/memory/fd_memory/memories.rs b/lib/sys-utils/src/memory/fd_memory/memories.rs index 8d8ac926078..a499652c265 100644 --- a/lib/sys-utils/src/memory/fd_memory/memories.rs +++ b/lib/sys-utils/src/memory/fd_memory/memories.rs @@ -12,7 +12,7 @@ use std::{ }; use wasmer::{Bytes, MemoryError, MemoryType, Pages}; -use wasmer_types::MemoryStyle; +use wasmer_types::{MemoryStyle, WASM_PAGE_SIZE}; use wasmer_vm::{ LinearMemory, MaybeInstanceOwned, ThreadConditions, Trap, VMMemoryDefinition, WaiterError, }; @@ -131,6 +131,19 @@ impl WasmMmap { Ok(prev_pages) } + /// Grows the memory to at least a minimum size. If the memory is already big enough + /// for the min size then this function does nothing + fn grow_at_least(&mut self, min_size: u64, conf: VMMemoryConfig) -> Result<(), MemoryError> { + let cur_size = self.size.bytes().0 as u64; + if cur_size < min_size { + let growth = min_size - cur_size; + let growth_pages = ((growth - 1) / WASM_PAGE_SIZE as u64) + 1; + self.grow(Pages(growth_pages as u32), conf)?; + } + + Ok(()) + } + fn reset(&mut self) -> Result<(), MemoryError> { self.size.0 = 0; Ok(()) @@ -342,6 +355,12 @@ impl LinearMemory for VMOwnedMemory { self.mmap.grow(delta, self.config.clone()) } + /// Grows the memory to at least a minimum size. If the memory is already big enough + /// for the min size then this function does nothing + fn grow_at_least(&mut self, min_size: u64) -> Result<(), MemoryError> { + self.mmap.grow_at_least(min_size, self.config.clone()) + } + fn reset(&mut self) -> Result<(), MemoryError> { self.mmap.reset()?; Ok(()) @@ -431,7 +450,7 @@ impl LinearMemory for VMSharedMemory { /// Resets the memory back down to zero size fn reset(&mut self) -> Result<(), MemoryError> { let mut guard = self.mmap.write().unwrap(); - guard.reset(); + guard.reset()?; Ok(()) } @@ -449,6 +468,13 @@ impl LinearMemory for VMSharedMemory { guard.grow(delta, self.config.clone()) } + /// Grows the memory to at least a minimum size. If the memory is already big enough + /// for the min size then this function does nothing + fn grow_at_least(&mut self, min_size: u64) -> Result<(), MemoryError> { + let mut guard = self.mmap.write().unwrap(); + guard.grow_at_least(min_size, self.config.clone()) + } + /// Return a `VMMemoryDefinition` for exposing the memory to compiled wasm code. fn vmmemory(&self) -> NonNull { let guard = self.mmap.read().unwrap(); @@ -520,9 +546,15 @@ impl LinearMemory for VMMemory { self.0.grow(delta) } + /// Grows the memory to at least a minimum size. If the memory is already big enough + /// for the min size then this function does nothing + fn grow_at_least(&mut self, min_size: u64) -> Result<(), MemoryError> { + self.0.grow_at_least(min_size) + } + /// Resets the memory down to a zero size fn reset(&mut self) -> Result<(), MemoryError> { - self.0.reset(); + self.0.reset()?; Ok(()) } diff --git a/lib/vm/src/memory.rs b/lib/vm/src/memory.rs index 001e8a3c9f8..ca600d67512 100644 --- a/lib/vm/src/memory.rs +++ b/lib/vm/src/memory.rs @@ -16,7 +16,7 @@ use std::ptr::NonNull; use std::slice; use std::sync::{Arc, RwLock}; use std::time::Duration; -use wasmer_types::{Bytes, MemoryError, MemoryStyle, MemoryType, Pages}; +use wasmer_types::{Bytes, MemoryError, MemoryStyle, MemoryType, Pages, WASM_PAGE_SIZE}; // The memory mapped area #[derive(Debug)] @@ -120,6 +120,19 @@ impl WasmMmap { Ok(prev_pages) } + /// Grows the memory to at least a minimum size. If the memory is already big enough + /// for the min size then this function does nothing + fn grow_at_least(&mut self, min_size: u64, conf: VMMemoryConfig) -> Result<(), MemoryError> { + let cur_size = self.size.bytes().0 as u64; + if cur_size < min_size { + let growth = min_size - cur_size; + let growth_pages = ((growth - 1) / WASM_PAGE_SIZE as u64) + 1; + self.grow(Pages(growth_pages as u32), conf)?; + } + + Ok(()) + } + /// Resets the memory down to a zero size fn reset(&mut self) -> Result<(), MemoryError> { self.size.0 = 0; @@ -332,6 +345,12 @@ impl LinearMemory for VMOwnedMemory { self.mmap.grow(delta, self.config.clone()) } + /// Grows the memory to at least a minimum size. If the memory is already big enough + /// for the min size then this function does nothing + fn grow_at_least(&mut self, min_size: u64) -> Result<(), MemoryError> { + self.mmap.grow_at_least(min_size, self.config.clone()) + } + /// Resets the memory down to a zero size fn reset(&mut self) -> Result<(), MemoryError> { self.mmap.reset()?; @@ -434,6 +453,13 @@ impl LinearMemory for VMSharedMemory { guard.grow(delta, self.config.clone()) } + /// Grows the memory to at least a minimum size. If the memory is already big enough + /// for the min size then this function does nothing + fn grow_at_least(&mut self, min_size: u64) -> Result<(), MemoryError> { + let mut guard = self.mmap.write().unwrap(); + guard.grow_at_least(min_size, self.config.clone()) + } + /// Resets the memory down to a zero size fn reset(&mut self) -> Result<(), MemoryError> { let mut guard = self.mmap.write().unwrap(); @@ -514,6 +540,12 @@ impl LinearMemory for VMMemory { self.0.grow(delta) } + /// Grows the memory to at least a minimum size. If the memory is already big enough + /// for the min size then this function does nothing + fn grow_at_least(&mut self, min_size: u64) -> Result<(), MemoryError> { + self.0.grow_at_least(min_size) + } + /// Resets the memory down to a zero size fn reset(&mut self) -> Result<(), MemoryError> { self.0.reset()?; @@ -658,6 +690,10 @@ where /// of wasm pages. fn grow(&mut self, delta: Pages) -> Result; + /// Grows the memory to at least a minimum size. If the memory is already big enough + /// for the min size then this function does nothing + fn grow_at_least(&mut self, min_size: u64) -> Result<(), MemoryError>; + /// Resets the memory back to zero length fn reset(&mut self) -> Result<(), MemoryError>; diff --git a/lib/wasix/src/journal/concrete/compacting.rs b/lib/wasix/src/journal/concrete/compacting.rs index 9c1f0a7d639..b29a617d65e 100644 --- a/lib/wasix/src/journal/concrete/compacting.rs +++ b/lib/wasix/src/journal/concrete/compacting.rs @@ -8,7 +8,7 @@ use virtual_fs::Fd; use super::*; -#[derive(Debug)] +#[derive(Debug, Default)] struct StateDescriptor { events: Vec, write_map: HashMap, @@ -28,9 +28,14 @@ impl From> for MemoryRange { } } +#[derive(Debug, Clone, Copy, PartialOrd, Ord, PartialEq, Eq, Hash)] +struct DescriptorLookup(u64); + #[derive(Derivative)] #[derivative(Debug)] struct State { + /// The descriptor seed is used generate descriptor lookups + descriptor_seed: u64, // We maintain a memory map of the events that are significant memory_map: HashMap, // List of all the snapshots @@ -42,10 +47,17 @@ struct State { thread_map: HashMap, // Any descriptors are assumed to be read only operations until // they actually do something that changes the system - suspect_descriptors: HashMap>, + suspect_descriptors: HashMap, // Any descriptors are assumed to be read only operations until // they actually do something that changes the system - keep_descriptors: HashMap, + keep_descriptors: HashMap, + // We put the IO related to stdio into a special list + // which can be purged when the program exits as its no longer + // important. + stdio_descriptors: HashMap, + // We abstract the descriptor state so that multiple file descriptors + // can refer to the same file descriptors + descriptors: HashMap, // Everything that will be retained during the next compact whitelist: HashSet, // We use an event index to track what to keep @@ -80,17 +92,19 @@ impl State { for t in self.thread_map.iter() { filter.add_event_to_whitelist(*t.1); } - for (_, d) in self.suspect_descriptors.iter() { - for e in d.iter() { - filter.add_event_to_whitelist(*e); - } - } - for (_, d) in self.keep_descriptors.iter() { - for e in d.events.iter() { - filter.add_event_to_whitelist(*e); - } - for e in d.write_map.values() { - filter.add_event_to_whitelist(*e); + for (_, l) in self + .suspect_descriptors + .iter() + .chain(self.keep_descriptors.iter()) + .chain(self.stdio_descriptors.iter()) + { + if let Some(d) = self.descriptors.get(&l) { + for e in d.events.iter() { + filter.add_event_to_whitelist(*e); + } + for e in d.write_map.values() { + filter.add_event_to_whitelist(*e); + } } } filter @@ -125,21 +139,25 @@ impl CompactingJournal { J: Journal, { let (tx, rx) = inner.split(); + let state = State { + inner_tx: tx, + inner_rx: rx.as_restarted()?, + tty: None, + snapshots: Default::default(), + memory_map: Default::default(), + thread_map: Default::default(), + suspect_descriptors: Default::default(), + keep_descriptors: Default::default(), + stdio_descriptors: Default::default(), + descriptor_seed: 0, + descriptors: Default::default(), + whitelist: Default::default(), + delta_list: None, + event_index: 0, + }; Ok(Self { tx: CompactingJournalTx { - state: Arc::new(Mutex::new(State { - inner_tx: tx, - inner_rx: rx.as_restarted()?, - tty: None, - snapshots: Default::default(), - memory_map: Default::default(), - thread_map: Default::default(), - suspect_descriptors: Default::default(), - keep_descriptors: Default::default(), - whitelist: Default::default(), - delta_list: None, - event_index: 0, - })), + state: Arc::new(Mutex::new(state)), compacting: Arc::new(Mutex::new(())), }, rx: CompactingJournalRx { inner: rx }, @@ -242,21 +260,17 @@ impl WritableJournal for CompactingJournalTx { JournalEntry::ProcessExit { .. } => { state.thread_map.clear(); state.memory_map.clear(); + for (_, lookup) in state.suspect_descriptors.clone() { + state.descriptors.remove(&lookup); + } state.suspect_descriptors.clear(); + for (_, lookup) in state.stdio_descriptors.clone() { + state.descriptors.remove(&lookup); + } + state.stdio_descriptors.clear(); state.whitelist.insert(event_index); state.snapshots.clear(); } - JournalEntry::CloseFileDescriptor { fd } => { - // If its not suspect we need to record this event - if state.suspect_descriptors.remove(fd).is_some() { - // suspect descriptors that are closed are dropped - // as they made no material difference to the state - } else if let Some(e) = state.keep_descriptors.get_mut(fd) { - e.events.push(event_index); - } else { - state.whitelist.insert(event_index); - } - } JournalEntry::TtySet { .. } => { state.tty.replace(event_index); } @@ -264,14 +278,29 @@ impl WritableJournal for CompactingJournalTx { // All file descriptors are opened in a suspect state which // means if they are closed without modifying the file system // then the events will be ignored. - state.suspect_descriptors.insert(*fd, vec![event_index]); + let lookup = state.descriptor_seed; + state.descriptor_seed += 1; + state + .suspect_descriptors + .insert(*fd, DescriptorLookup(lookup)); } // We keep non-mutable events for file descriptors that are suspect - JournalEntry::FileDescriptorSeek { fd, .. } => { - if let Some(events) = state.suspect_descriptors.get_mut(fd) { - events.push(event_index); - } else if let Some(s) = state.keep_descriptors.get_mut(fd) { - s.events.push(event_index); + JournalEntry::FileDescriptorSeek { fd, .. } + | JournalEntry::CloseFileDescriptor { fd } => { + // Get the lookup + let lookup = state + .suspect_descriptors + .get(fd) + .cloned() + .or_else(|| state.keep_descriptors.get(fd).cloned()) + .or_else(|| state.stdio_descriptors.get(fd).cloned()); + + if let Some(lookup) = lookup { + let state = state + .descriptors + .entry(lookup) + .or_insert_with(|| Default::default()); + state.events.push(event_index); } else { state.whitelist.insert(event_index); } @@ -282,21 +311,26 @@ impl WritableJournal for CompactingJournalTx { | JournalEntry::FileDescriptorAllocate { fd, .. } | JournalEntry::FileDescriptorSetFlags { fd, .. } | JournalEntry::FileDescriptorSetTimes { fd, .. } - | JournalEntry::FileDescriptorWrite { fd, .. } - | JournalEntry::DuplicateFileDescriptor { - original_fd: fd, .. - } => { + | JournalEntry::FileDescriptorWrite { fd, .. } => { // Its no longer suspect - if let Some(events) = state.suspect_descriptors.remove(fd) { - state.keep_descriptors.insert( - *fd, - StateDescriptor { - events, - write_map: Default::default(), - }, - ); + if let Some(lookup) = state.suspect_descriptors.remove(fd) { + state.keep_descriptors.insert(*fd, lookup); } - if let Some(state) = state.keep_descriptors.get_mut(fd) { + + // Get the lookup + let lookup = state + .suspect_descriptors + .get(fd) + .cloned() + .or_else(|| state.keep_descriptors.get(fd).cloned()) + .or_else(|| state.stdio_descriptors.get(fd).cloned()); + + // Update the state + if let Some(lookup) = lookup { + let state = state + .descriptors + .entry(lookup) + .or_insert_with(|| Default::default()); if let JournalEntry::FileDescriptorWrite { offset, data, .. } = &entry { state.write_map.insert( MemoryRange { @@ -312,14 +346,29 @@ impl WritableJournal for CompactingJournalTx { state.whitelist.insert(event_index); } } + // Duplicating the file descriptor + JournalEntry::DuplicateFileDescriptor { + original_fd, + copied_fd, + } => { + if let Some(lookup) = state.suspect_descriptors.remove(original_fd) { + state.suspect_descriptors.insert(*copied_fd, lookup); + } else if let Some(lookup) = state.keep_descriptors.remove(original_fd) { + state.keep_descriptors.insert(*copied_fd, lookup); + } else if let Some(lookup) = state.stdio_descriptors.remove(original_fd) { + state.stdio_descriptors.insert(*copied_fd, lookup); + } else { + state.whitelist.insert(event_index); + } + } // Renumbered file descriptors will retain their suspect status JournalEntry::RenumberFileDescriptor { old_fd, new_fd } => { - if let Some(mut events) = state.suspect_descriptors.remove(old_fd) { - events.push(event_index); - state.suspect_descriptors.insert(*new_fd, events); - } else if let Some(mut s) = state.keep_descriptors.remove(old_fd) { - s.events.push(event_index); - state.keep_descriptors.insert(*new_fd, s); + if let Some(lookup) = state.suspect_descriptors.remove(old_fd) { + state.suspect_descriptors.insert(*new_fd, lookup); + } else if let Some(lookup) = state.keep_descriptors.remove(old_fd) { + state.keep_descriptors.insert(*new_fd, lookup); + } else if let Some(lookup) = state.stdio_descriptors.remove(old_fd) { + state.stdio_descriptors.insert(*new_fd, lookup); } else { state.whitelist.insert(event_index); } diff --git a/lib/wasix/src/journal/effector/memory_and_snapshot.rs b/lib/wasix/src/journal/effector/memory_and_snapshot.rs index fc4c69656eb..16400b61f07 100644 --- a/lib/wasix/src/journal/effector/memory_and_snapshot.rs +++ b/lib/wasix/src/journal/effector/memory_and_snapshot.rs @@ -80,7 +80,11 @@ impl JournalEffector { region: Range, data: &[u8], ) -> anyhow::Result<()> { - let (env, store) = ctx.data_and_store_mut(); + let (env, mut store) = ctx.data_and_store_mut(); + + let memory = unsafe { env.memory() }; + memory.grow_at_least(&mut store, region.end + data.len() as u64)?; + let memory = unsafe { env.memory_view(&store) }; memory .write(region.start, data.as_ref()) diff --git a/lib/wasix/src/journal/snapshot.rs b/lib/wasix/src/journal/snapshot.rs index 2d7d9d09ab6..178596577eb 100644 --- a/lib/wasix/src/journal/snapshot.rs +++ b/lib/wasix/src/journal/snapshot.rs @@ -26,6 +26,17 @@ pub enum SnapshotTrigger { NonDeterministicCall, } +impl SnapshotTrigger { + pub fn only_once(&self) -> bool { + match self { + Self::FirstListen => true, + Self::FirstEnviron => true, + Self::FirstStdin => true, + _ => false, + } + } +} + pub const DEFAULT_SNAPSHOT_TRIGGERS: [SnapshotTrigger; 4] = [ SnapshotTrigger::Idle, SnapshotTrigger::FirstEnviron, diff --git a/lib/wasix/src/state/env.rs b/lib/wasix/src/state/env.rs index f765544822c..df4249efa8e 100644 --- a/lib/wasix/src/state/env.rs +++ b/lib/wasix/src/state/env.rs @@ -818,6 +818,16 @@ impl WasiEnv { self.try_inner().map(|i| i.memory()) } + /// Providers safe access to the memory + /// (it must be initialized before it can be used) + /// This has been marked as unsafe as it will panic if its executed + /// on the wrong thread or before the inner is set + pub(crate) unsafe fn memory<'a>(&self) -> WasiInstanceGuardMemory<'_> { + self.try_memory().expect( + "You must initialize the WasiEnv before using it and can not pass it between threads", + ) + } + /// Providers safe access to the memory /// (it must be initialized before it can be used) pub(crate) fn try_memory_view<'a>( @@ -890,7 +900,11 @@ impl WasiEnv { /// Returns true if a particular snapshot trigger is enabled #[cfg(feature = "journal")] pub fn pop_snapshot_trigger(&mut self, trigger: SnapshotTrigger) -> bool { - self.snapshot_on.remove(&trigger) + if trigger.only_once() { + self.snapshot_on.remove(&trigger) + } else { + self.snapshot_on.contains(&trigger) + } } /// Internal helper function to get a standard device handle. diff --git a/lib/wasix/src/syscalls/mod.rs b/lib/wasix/src/syscalls/mod.rs index d64243d8dcd..48605fae817 100644 --- a/lib/wasix/src/syscalls/mod.rs +++ b/lib/wasix/src/syscalls/mod.rs @@ -1491,13 +1491,11 @@ pub unsafe fn restore_snapshot( JournalEffector::apply_path_rename(&mut ctx, old_fd, &old_path, new_fd, &new_path) .map_err(anyhow_err_to_runtime_err)?; } - crate::journal::JournalEntry::Snapshot { - when: _, - trigger: _, - } => { + crate::journal::JournalEntry::Snapshot { when: _, trigger } => { if cur_module_hash != journal_module_hash { continue; } + ctx.data_mut().pop_snapshot_trigger(trigger); } crate::journal::JournalEntry::SetClockTime { clock_id, time } => { JournalEffector::apply_clock_time_set(&mut ctx, clock_id, time) @@ -1836,6 +1834,7 @@ pub unsafe fn restore_snapshot( .map_err(anyhow_err_to_runtime_err)?; } if let Some(state) = update_tty { + tracing::error!("BLAH1 {:?}", state); JournalEffector::apply_tty_set(&mut ctx, state).map_err(anyhow_err_to_runtime_err)?; } From 69bcc612f7502e48693c7624104f81cb09c59932 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Mon, 27 Nov 2023 17:21:07 +1100 Subject: [PATCH 099/129] The stdout and stderr are no longer restored if the process exits --- lib/wasix/src/syscalls/mod.rs | 68 ++++++++++++++++++++++++++++++++++- 1 file changed, 67 insertions(+), 1 deletion(-) diff --git a/lib/wasix/src/syscalls/mod.rs b/lib/wasix/src/syscalls/mod.rs index 48605fae817..38097b7f36d 100644 --- a/lib/wasix/src/syscalls/mod.rs +++ b/lib/wasix/src/syscalls/mod.rs @@ -1349,6 +1349,14 @@ pub unsafe fn restore_snapshot( let mut update_memory: HashMap, Cow<'_, [u8]>> = Default::default(); let mut update_tty = None; + // We capture the stdout and stderr while we replay + let mut stdout = Vec::new(); + let mut stderr = Vec::new(); + let mut stdout_fds = HashSet::new(); + let mut stderr_fds = HashSet::new(); + stdout_fds.insert(1 as WasiFd); + stderr_fds.insert(2 as WasiFd); + // Loop through all the events and process them let cur_module_hash = Some(ctx.data().process.module_hash.as_bytes()); let mut journal_module_hash = None; @@ -1365,6 +1373,12 @@ pub unsafe fn restore_snapshot( spawn_threads.clear(); update_memory.clear(); update_tty.take(); + stdout.clear(); + stderr.clear(); + stdout_fds.clear(); + stderr_fds.clear(); + stdout_fds.insert(1 as WasiFd); + stderr_fds.insert(2 as WasiFd); } else { JournalEffector::apply_process_exit(&mut ctx, exit_code) .map_err(anyhow_err_to_runtime_err)?; @@ -1376,6 +1390,15 @@ pub unsafe fn restore_snapshot( data, is_64bit, } => { + if stdout_fds.contains(&fd) { + stdout.push((offset, data, is_64bit)); + continue; + } + if stderr_fds.contains(&fd) { + stderr.push((offset, data, is_64bit)); + continue; + } + if is_64bit { JournalEffector::apply_fd_write::(&ctx, fd, offset, data) } else { @@ -1406,6 +1429,12 @@ pub unsafe fn restore_snapshot( spawn_threads.clear(); update_memory.clear(); update_tty.take(); + stdout.clear(); + stderr.clear(); + stdout_fds.clear(); + stderr_fds.clear(); + stdout_fds.insert(1 as WasiFd); + stderr_fds.insert(2 as WasiFd); } else { JournalEffector::apply_process_exit(&mut ctx, exit_code) .map_err(anyhow_err_to_runtime_err)?; @@ -1449,6 +1478,8 @@ pub unsafe fn restore_snapshot( } } crate::journal::JournalEntry::CloseFileDescriptor { fd } => { + stdout_fds.remove(&fd); + stderr_fds.remove(&fd); JournalEffector::apply_fd_close(&mut ctx, fd).map_err(anyhow_err_to_runtime_err)?; } crate::journal::JournalEntry::OpenFileDescriptor { @@ -1502,6 +1533,12 @@ pub unsafe fn restore_snapshot( .map_err(anyhow_err_to_runtime_err)?; } crate::journal::JournalEntry::RenumberFileDescriptor { old_fd, new_fd } => { + if stdout_fds.remove(&old_fd) { + stdout_fds.insert(new_fd); + } + if stderr_fds.remove(&old_fd) { + stderr_fds.insert(new_fd); + } JournalEffector::apply_fd_renumber(&mut ctx, old_fd, new_fd) .map_err(anyhow_err_to_runtime_err)?; } @@ -1509,6 +1546,12 @@ pub unsafe fn restore_snapshot( original_fd, copied_fd, } => { + if stdout_fds.contains(&original_fd) { + stdout_fds.insert(copied_fd); + } + if stderr_fds.contains(&original_fd) { + stderr_fds.insert(copied_fd); + } JournalEffector::apply_fd_duplicate(&mut ctx, original_fd, copied_fd) .map_err(anyhow_err_to_runtime_err)?; } @@ -1807,6 +1850,12 @@ pub unsafe fn restore_snapshot( spawn_threads.clear(); update_memory.clear(); update_tty.take(); + stdout.clear(); + stderr.clear(); + stdout_fds.clear(); + stderr_fds.clear(); + stdout_fds.insert(1 as WasiFd); + stderr_fds.insert(2 as WasiFd); } else { JournalEffector::apply_process_exit(&mut ctx, None) .map_err(anyhow_err_to_runtime_err)?; @@ -1827,6 +1876,24 @@ pub unsafe fn restore_snapshot( ))); } + // Now output the stdout and stderr + for (offset, data, is_64bit) in stdout { + if is_64bit { + JournalEffector::apply_fd_write::(&ctx, 1, offset, data) + } else { + JournalEffector::apply_fd_write::(&ctx, 1, offset, data) + } + .map_err(anyhow_err_to_runtime_err)?; + } + + for (offset, data, is_64bit) in stderr { + if is_64bit { + JournalEffector::apply_fd_write::(&ctx, 2, offset, data) + } else { + JournalEffector::apply_fd_write::(&ctx, 2, offset, data) + } + .map_err(anyhow_err_to_runtime_err)?; + } // Next we apply all the memory updates that were delayed while the logs // were processed to completion. for (region, data) in update_memory { @@ -1834,7 +1901,6 @@ pub unsafe fn restore_snapshot( .map_err(anyhow_err_to_runtime_err)?; } if let Some(state) = update_tty { - tracing::error!("BLAH1 {:?}", state); JournalEffector::apply_tty_set(&mut ctx, state).map_err(anyhow_err_to_runtime_err)?; } From d494348155425a96ed0a58cb69245ef1fb794a60 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Tue, 28 Nov 2023 09:54:44 +1100 Subject: [PATCH 100/129] Changes made after review with Christoph plus added compression of journals --- Cargo.lock | 20 + lib/cli/src/commands/journal/filter.rs | 8 +- lib/wasix/Cargo.toml | 1 + lib/wasix/src/journal/base64.rs | 13 +- lib/wasix/src/journal/concrete/archived.rs | 654 ++++++++--------- lib/wasix/src/journal/concrete/compacting.rs | 148 ++-- lib/wasix/src/journal/concrete/filter.rs | 114 +-- lib/wasix/src/journal/concrete/log_file.rs | 36 +- lib/wasix/src/journal/concrete/printing.rs | 122 ++-- .../journal/effector/memory_and_snapshot.rs | 4 +- .../src/journal/effector/process_exit.rs | 2 +- .../src/journal/effector/syscalls/chdir.rs | 2 +- .../journal/effector/syscalls/clock_time.rs | 2 +- .../journal/effector/syscalls/epoll_create.rs | 2 +- .../journal/effector/syscalls/epoll_ctl.rs | 2 +- .../journal/effector/syscalls/fd_advise.rs | 2 +- .../journal/effector/syscalls/fd_allocate.rs | 2 +- .../src/journal/effector/syscalls/fd_close.rs | 2 +- .../journal/effector/syscalls/fd_duplicate.rs | 2 +- .../src/journal/effector/syscalls/fd_event.rs | 2 +- .../src/journal/effector/syscalls/fd_pipe.rs | 2 +- .../journal/effector/syscalls/fd_renumber.rs | 2 +- .../src/journal/effector/syscalls/fd_seek.rs | 5 +- .../journal/effector/syscalls/fd_set_flags.rs | 2 +- .../effector/syscalls/fd_set_rights.rs | 2 +- .../journal/effector/syscalls/fd_set_size.rs | 2 +- .../journal/effector/syscalls/fd_set_times.rs | 2 +- .../src/journal/effector/syscalls/fd_write.rs | 2 +- .../syscalls/path_create_directory.rs | 2 +- .../journal/effector/syscalls/path_link.rs | 2 +- .../journal/effector/syscalls/path_open.rs | 2 +- .../syscalls/path_remove_directory.rs | 2 +- .../journal/effector/syscalls/path_rename.rs | 2 +- .../effector/syscalls/path_set_times.rs | 2 +- .../journal/effector/syscalls/path_symlink.rs | 2 +- .../journal/effector/syscalls/path_unlink.rs | 2 +- .../effector/syscalls/port_addr_add.rs | 2 +- .../effector/syscalls/port_addr_clear.rs | 2 +- .../effector/syscalls/port_addr_remove.rs | 2 +- .../journal/effector/syscalls/port_bridge.rs | 2 +- .../effector/syscalls/port_dhcp_acquire.rs | 2 +- .../effector/syscalls/port_gateway_set.rs | 2 +- .../effector/syscalls/port_route_add.rs | 2 +- .../effector/syscalls/port_route_clear.rs | 2 +- .../effector/syscalls/port_route_remove.rs | 2 +- .../effector/syscalls/port_unbridge.rs | 2 +- .../journal/effector/syscalls/sock_accept.rs | 4 +- .../journal/effector/syscalls/sock_bind.rs | 2 +- .../journal/effector/syscalls/sock_connect.rs | 2 +- .../syscalls/sock_join_ipv4_multicast.rs | 2 +- .../syscalls/sock_join_ipv6_multicast.rs | 4 +- .../syscalls/sock_leave_ipv4_multicast.rs | 4 +- .../syscalls/sock_leave_ipv6_multicast.rs | 4 +- .../journal/effector/syscalls/sock_listen.rs | 2 +- .../journal/effector/syscalls/sock_open.rs | 2 +- .../journal/effector/syscalls/sock_send.rs | 2 +- .../effector/syscalls/sock_send_file.rs | 2 +- .../journal/effector/syscalls/sock_send_to.rs | 2 +- .../effector/syscalls/sock_set_opt_flag.rs | 2 +- .../effector/syscalls/sock_set_opt_size.rs | 2 +- .../effector/syscalls/sock_set_opt_time.rs | 2 +- .../effector/syscalls/sock_shutdown.rs | 2 +- .../src/journal/effector/syscalls/tty_set.rs | 2 +- lib/wasix/src/journal/effector/thread_exit.rs | 2 +- .../src/journal/effector/thread_state.rs | 2 +- lib/wasix/src/journal/entry.rs | 440 ++++++------ lib/wasix/src/state/func_env.rs | 2 +- lib/wasix/src/syscalls/journal.rs | 660 ++++++++++++++++++ lib/wasix/src/syscalls/mod.rs | 653 +---------------- 69 files changed, 1530 insertions(+), 1466 deletions(-) create mode 100644 lib/wasix/src/syscalls/journal.rs diff --git a/Cargo.lock b/Cargo.lock index 3962e83b5e6..ce1965180f8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2452,6 +2452,15 @@ version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" +[[package]] +name = "lz4_flex" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ea9b256699eda7b0387ffbc776dd625e28bde3918446381781245b7a50349d8" +dependencies = [ + "twox-hash", +] + [[package]] name = "lzma-rs" version = "0.2.0" @@ -4897,6 +4906,16 @@ dependencies = [ "utf-8", ] +[[package]] +name = "twox-hash" +version = "1.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675" +dependencies = [ + "cfg-if", + "static_assertions", +] + [[package]] name = "typenum" version = "1.17.0" @@ -6238,6 +6257,7 @@ dependencies = [ "lazy_static", "libc", "linked_hash_set", + "lz4_flex", "num_enum", "once_cell", "petgraph", diff --git a/lib/cli/src/commands/journal/filter.rs b/lib/cli/src/commands/journal/filter.rs index d383b932dcc..57dc9c44c54 100644 --- a/lib/cli/src/commands/journal/filter.rs +++ b/lib/cli/src/commands/journal/filter.rs @@ -47,7 +47,13 @@ pub struct CmdJournalFilter { /// Path to the journal that will be the output of the filter #[clap(index = 2)] target_path: PathBuf, - /// Filters to be applied to the output journal + /// Filters to be applied to the output journal, filter options are + /// - 'mem' | 'memory' -> removes all WASM memory related events + /// - 'thread' | 'threads' -> removes all events related to the state of the threads + /// - 'fs' | 'file' -> removes file system mutation events + /// - 'core' -> removes core operating system operations such as TTY + /// - 'snap' | 'snapshot' -> removes the snapshots from the journal + /// - 'net' | 'network' -> removes network socket and interface events #[clap(short, long = "filter")] filters: Vec, } diff --git a/lib/wasix/Cargo.toml b/lib/wasix/Cargo.toml index ff0110ad0a6..5bbb2a845d1 100644 --- a/lib/wasix/Cargo.toml +++ b/lib/wasix/Cargo.toml @@ -81,6 +81,7 @@ bytecheck = "0.6.8" shared-buffer = "0.1" petgraph = "0.6.3" base64 = "0.21" +lz4_flex = { version = "0.11" } rayon = { version = "1.7.0", optional = true } wasm-bindgen = { version = "0.2.87", optional = true } js-sys = { version = "0.3.64", optional = true } diff --git a/lib/wasix/src/journal/base64.rs b/lib/wasix/src/journal/base64.rs index cc4cfb702c7..cd4124af9ae 100644 --- a/lib/wasix/src/journal/base64.rs +++ b/lib/wasix/src/journal/base64.rs @@ -1,18 +1,23 @@ use std::borrow::Cow; +use lz4_flex::block::{compress_prepend_size, decompress_size_prepended}; + use serde::{Deserialize, Serialize}; use serde::{Deserializer, Serializer}; pub fn serialize(v: &[u8], s: S) -> Result { #[allow(deprecated)] - let base64 = base64::encode(v); + let base64 = base64::encode(compress_prepend_size(v)); String::serialize(&base64, s) } pub fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result, D::Error> { let base64 = String::deserialize(d)?; #[allow(deprecated)] - base64::decode(base64.as_bytes()) - .map_err(serde::de::Error::custom) - .map(|d| d.into()) + base64::decode( + decompress_size_prepended(base64.as_bytes()) + .map_err(|err| serde::de::Error::custom(err))?, + ) + .map_err(serde::de::Error::custom) + .map(|d| d.into()) } diff --git a/lib/wasix/src/journal/concrete/archived.rs b/lib/wasix/src/journal/concrete/archived.rs index be10c400ef4..0bf1bc8cb11 100644 --- a/lib/wasix/src/journal/concrete/archived.rs +++ b/lib/wasix/src/journal/concrete/archived.rs @@ -1,3 +1,4 @@ +use lz4_flex::block::{compress_prepend_size, decompress_size_prepended}; use num_enum::{IntoPrimitive, TryFromPrimitive}; use rkyv::ser::{ScratchSpace, Serializer}; use rkyv::{Archive, CheckBytes, Deserialize as RkyvDeserialize, Serialize as RkyvSerialize}; @@ -26,64 +27,64 @@ pub const JOURNAL_MAGIC_NUMBER_BYTES: [u8; 8] = JOURNAL_MAGIC_NUMBER.to_be_bytes )] #[archive_attr(derive(CheckBytes))] pub enum JournalEntryRecordType { - InitModuleV1, - ProcessExitV1, - SetThreadV1, - CloseThreadV1, - FileDescriptorSeekV1, - FileDescriptorWriteV1, - UpdateMemoryRegionV1, - SetClockTimeV1, - OpenFileDescriptorV1, - CloseFileDescriptorV1, - RenumberFileDescriptorV1, - DuplicateFileDescriptorV1, - CreateDirectoryV1, - RemoveDirectoryV1, - PathSetTimesV1, - FileDescriptorSetTimesV1, - FileDescriptorSetSizeV1, - FileDescriptorSetFlagsV1, - FileDescriptorSetRightsV1, - FileDescriptorAdviseV1, - FileDescriptorAllocateV1, - CreateHardLinkV1, - CreateSymbolicLinkV1, - UnlinkFileV1, - PathRenameV1, - ChangeDirectoryV1, - EpollCreateV1, - EpollCtlV1, - TtySetV1, - CreatePipeV1, - CreateEventV1, - PortAddAddrV1, - PortDelAddrV1, - PortAddrClearV1, - PortBridgeV1, - PortUnbridgeV1, - PortDhcpAcquireV1, - PortGatewaySetV1, - PortRouteAddV1, - PortRouteClearV1, - PortRouteDelV1, - SocketOpenV1, - SocketListenV1, - SocketBindV1, - SocketConnectedV1, - SocketAcceptedV1, - SocketJoinIpv4MulticastV1, - SocketJoinIpv6MulticastV1, - SocketLeaveIpv4MulticastV1, - SocketLeaveIpv6MulticastV1, - SocketSendFileV1, - SocketSendToV1, - SocketSendV1, - SocketSetOptFlagV1, - SocketSetOptSizeV1, - SocketSetOptTimeV1, - SocketShutdownV1, - SnapshotV1, + InitModuleV1 = 1, + ProcessExitV1 = 2, + SetThreadV1 = 3, + CloseThreadV1 = 4, + FileDescriptorSeekV1 = 5, + FileDescriptorWriteV1 = 6, + UpdateMemoryRegionV1 = 7, + SetClockTimeV1 = 9, + OpenFileDescriptorV1 = 10, + CloseFileDescriptorV1 = 11, + RenumberFileDescriptorV1 = 12, + DuplicateFileDescriptorV1 = 13, + CreateDirectoryV1 = 14, + RemoveDirectoryV1 = 15, + PathSetTimesV1 = 16, + FileDescriptorSetTimesV1 = 17, + FileDescriptorSetSizeV1 = 18, + FileDescriptorSetFlagsV1 = 19, + FileDescriptorSetRightsV1 = 20, + FileDescriptorAdviseV1 = 21, + FileDescriptorAllocateV1 = 22, + CreateHardLinkV1 = 23, + CreateSymbolicLinkV1 = 24, + UnlinkFileV1 = 25, + PathRenameV1 = 26, + ChangeDirectoryV1 = 27, + EpollCreateV1 = 28, + EpollCtlV1 = 29, + TtySetV1 = 30, + CreatePipeV1 = 31, + CreateEventV1 = 32, + PortAddAddrV1 = 33, + PortDelAddrV1 = 34, + PortAddrClearV1 = 35, + PortBridgeV1 = 36, + PortUnbridgeV1 = 37, + PortDhcpAcquireV1 = 38, + PortGatewaySetV1 = 39, + PortRouteAddV1 = 40, + PortRouteClearV1 = 41, + PortRouteDelV1 = 42, + SocketOpenV1 = 43, + SocketListenV1 = 44, + SocketBindV1 = 45, + SocketConnectedV1 = 46, + SocketAcceptedV1 = 47, + SocketJoinIpv4MulticastV1 = 48, + SocketJoinIpv6MulticastV1 = 49, + SocketLeaveIpv4MulticastV1 = 50, + SocketLeaveIpv6MulticastV1 = 51, + SocketSendFileV1 = 52, + SocketSendToV1 = 53, + SocketSendV1 = 54, + SocketSetOptFlagV1 = 55, + SocketSetOptSizeV1 = 56, + SocketSetOptTimeV1 = 57, + SocketShutdownV1 = 58, + SnapshotV1 = 59, } impl JournalEntryRecordType { @@ -92,8 +93,8 @@ impl JournalEntryRecordType { /// `rykv` makes direct memory references to achieve high performance /// however this does mean care must be taken that the data itself /// can not be manipulated or corrupted. - pub unsafe fn deserialize_archive(self, data: &[u8]) -> JournalEntry<'_> { - match self { + pub unsafe fn deserialize_archive(self, data: &[u8]) -> anyhow::Result> { + Ok(match self { JournalEntryRecordType::InitModuleV1 => ArchivedJournalEntry::InitModuleV1( rkyv::archived_root::(data), ), @@ -233,12 +234,14 @@ impl JournalEntryRecordType { JournalEntryRecordType::PortDelAddrV1 => ArchivedJournalEntry::PortDelAddrV1( rkyv::archived_root::(data), ), - JournalEntryRecordType::PortAddrClearV1 => return JournalEntry::PortAddrClear, + JournalEntryRecordType::PortAddrClearV1 => return Ok(JournalEntry::PortAddrClearV1), JournalEntryRecordType::PortBridgeV1 => ArchivedJournalEntry::PortBridgeV1( rkyv::archived_root::(data), ), - JournalEntryRecordType::PortUnbridgeV1 => return JournalEntry::PortUnbridge, - JournalEntryRecordType::PortDhcpAcquireV1 => return JournalEntry::PortDhcpAcquire, + JournalEntryRecordType::PortUnbridgeV1 => return Ok(JournalEntry::PortUnbridgeV1), + JournalEntryRecordType::PortDhcpAcquireV1 => { + return Ok(JournalEntry::PortDhcpAcquireV1) + } JournalEntryRecordType::PortGatewaySetV1 => { ArchivedJournalEntry::PortGatewaySetV1(rkyv::archived_root::< JournalEntryPortGatewaySetV1, @@ -249,7 +252,7 @@ impl JournalEntryRecordType { JournalEntryPortRouteAddV1, >(data)) } - JournalEntryRecordType::PortRouteClearV1 => return JournalEntry::PortRouteClear, + JournalEntryRecordType::PortRouteClearV1 => return Ok(JournalEntry::PortRouteClearV1), JournalEntryRecordType::PortRouteDelV1 => { ArchivedJournalEntry::PortRouteDelV1(rkyv::archived_root::< JournalEntryPortRouteDelV1, @@ -333,83 +336,91 @@ impl JournalEntryRecordType { rkyv::archived_root::(data), ), } - .into() + .try_into()?) } } impl<'a> JournalEntry<'a> { pub fn archive_record_type(&self) -> JournalEntryRecordType { match self { - Self::InitModule { .. } => JournalEntryRecordType::InitModuleV1, - Self::UpdateMemoryRegion { .. } => JournalEntryRecordType::UpdateMemoryRegionV1, - Self::ProcessExit { .. } => JournalEntryRecordType::ProcessExitV1, - Self::SetThread { .. } => JournalEntryRecordType::SetThreadV1, - Self::CloseThread { .. } => JournalEntryRecordType::CloseThreadV1, - Self::FileDescriptorSeek { .. } => JournalEntryRecordType::FileDescriptorSeekV1, - Self::FileDescriptorWrite { .. } => JournalEntryRecordType::FileDescriptorWriteV1, - Self::SetClockTime { .. } => JournalEntryRecordType::SetClockTimeV1, - Self::CloseFileDescriptor { .. } => JournalEntryRecordType::CloseFileDescriptorV1, - Self::OpenFileDescriptor { .. } => JournalEntryRecordType::OpenFileDescriptorV1, - Self::RenumberFileDescriptor { .. } => JournalEntryRecordType::RenumberFileDescriptorV1, - Self::DuplicateFileDescriptor { .. } => { + Self::InitModuleV1 { .. } => JournalEntryRecordType::InitModuleV1, + Self::UpdateMemoryRegionV1 { .. } => JournalEntryRecordType::UpdateMemoryRegionV1, + Self::ProcessExitV1 { .. } => JournalEntryRecordType::ProcessExitV1, + Self::SetThreadV1 { .. } => JournalEntryRecordType::SetThreadV1, + Self::CloseThreadV1 { .. } => JournalEntryRecordType::CloseThreadV1, + Self::FileDescriptorSeekV1 { .. } => JournalEntryRecordType::FileDescriptorSeekV1, + Self::FileDescriptorWriteV1 { .. } => JournalEntryRecordType::FileDescriptorWriteV1, + Self::SetClockTimeV1 { .. } => JournalEntryRecordType::SetClockTimeV1, + Self::CloseFileDescriptorV1 { .. } => JournalEntryRecordType::CloseFileDescriptorV1, + Self::OpenFileDescriptorV1 { .. } => JournalEntryRecordType::OpenFileDescriptorV1, + Self::RenumberFileDescriptorV1 { .. } => { + JournalEntryRecordType::RenumberFileDescriptorV1 + } + Self::DuplicateFileDescriptorV1 { .. } => { JournalEntryRecordType::DuplicateFileDescriptorV1 } - Self::CreateDirectory { .. } => JournalEntryRecordType::CreateDirectoryV1, - Self::RemoveDirectory { .. } => JournalEntryRecordType::RemoveDirectoryV1, - Self::PathSetTimes { .. } => JournalEntryRecordType::PathSetTimesV1, - Self::FileDescriptorSetTimes { .. } => JournalEntryRecordType::FileDescriptorSetTimesV1, - Self::FileDescriptorSetFlags { .. } => JournalEntryRecordType::FileDescriptorSetFlagsV1, - Self::FileDescriptorSetRights { .. } => { + Self::CreateDirectoryV1 { .. } => JournalEntryRecordType::CreateDirectoryV1, + Self::RemoveDirectoryV1 { .. } => JournalEntryRecordType::RemoveDirectoryV1, + Self::PathSetTimesV1 { .. } => JournalEntryRecordType::PathSetTimesV1, + Self::FileDescriptorSetTimesV1 { .. } => { + JournalEntryRecordType::FileDescriptorSetTimesV1 + } + Self::FileDescriptorSetFlagsV1 { .. } => { + JournalEntryRecordType::FileDescriptorSetFlagsV1 + } + Self::FileDescriptorSetRightsV1 { .. } => { JournalEntryRecordType::FileDescriptorSetRightsV1 } - Self::FileDescriptorSetSize { .. } => JournalEntryRecordType::FileDescriptorSetSizeV1, - Self::FileDescriptorAdvise { .. } => JournalEntryRecordType::FileDescriptorAdviseV1, - Self::FileDescriptorAllocate { .. } => JournalEntryRecordType::FileDescriptorAllocateV1, - Self::CreateHardLink { .. } => JournalEntryRecordType::CreateHardLinkV1, - Self::CreateSymbolicLink { .. } => JournalEntryRecordType::CreateSymbolicLinkV1, - Self::UnlinkFile { .. } => JournalEntryRecordType::UnlinkFileV1, - Self::PathRename { .. } => JournalEntryRecordType::PathRenameV1, - Self::ChangeDirectory { .. } => JournalEntryRecordType::ChangeDirectoryV1, - Self::EpollCreate { .. } => JournalEntryRecordType::EpollCreateV1, - Self::EpollCtl { .. } => JournalEntryRecordType::EpollCtlV1, - Self::TtySet { .. } => JournalEntryRecordType::TtySetV1, - Self::CreatePipe { .. } => JournalEntryRecordType::CreatePipeV1, - Self::CreateEvent { .. } => JournalEntryRecordType::CreateEventV1, - Self::PortAddAddr { .. } => JournalEntryRecordType::PortAddAddrV1, - Self::PortDelAddr { .. } => JournalEntryRecordType::PortDelAddrV1, - Self::PortAddrClear => JournalEntryRecordType::PortAddrClearV1, - Self::PortBridge { .. } => JournalEntryRecordType::PortBridgeV1, - Self::PortUnbridge => JournalEntryRecordType::PortUnbridgeV1, - Self::PortDhcpAcquire => JournalEntryRecordType::PortDhcpAcquireV1, - Self::PortGatewaySet { .. } => JournalEntryRecordType::PortGatewaySetV1, - Self::PortRouteAdd { .. } => JournalEntryRecordType::PortRouteAddV1, - Self::PortRouteClear => JournalEntryRecordType::PortRouteClearV1, - Self::PortRouteDel { .. } => JournalEntryRecordType::PortRouteDelV1, - Self::SocketOpen { .. } => JournalEntryRecordType::SocketOpenV1, - Self::SocketListen { .. } => JournalEntryRecordType::SocketListenV1, - Self::SocketBind { .. } => JournalEntryRecordType::SocketBindV1, - Self::SocketConnected { .. } => JournalEntryRecordType::SocketConnectedV1, - Self::SocketAccepted { .. } => JournalEntryRecordType::SocketAcceptedV1, - Self::SocketJoinIpv4Multicast { .. } => { + Self::FileDescriptorSetSizeV1 { .. } => JournalEntryRecordType::FileDescriptorSetSizeV1, + Self::FileDescriptorAdviseV1 { .. } => JournalEntryRecordType::FileDescriptorAdviseV1, + Self::FileDescriptorAllocateV1 { .. } => { + JournalEntryRecordType::FileDescriptorAllocateV1 + } + Self::CreateHardLinkV1 { .. } => JournalEntryRecordType::CreateHardLinkV1, + Self::CreateSymbolicLinkV1 { .. } => JournalEntryRecordType::CreateSymbolicLinkV1, + Self::UnlinkFileV1 { .. } => JournalEntryRecordType::UnlinkFileV1, + Self::PathRenameV1 { .. } => JournalEntryRecordType::PathRenameV1, + Self::ChangeDirectoryV1 { .. } => JournalEntryRecordType::ChangeDirectoryV1, + Self::EpollCreateV1 { .. } => JournalEntryRecordType::EpollCreateV1, + Self::EpollCtlV1 { .. } => JournalEntryRecordType::EpollCtlV1, + Self::TtySetV1 { .. } => JournalEntryRecordType::TtySetV1, + Self::CreatePipeV1 { .. } => JournalEntryRecordType::CreatePipeV1, + Self::CreateEventV1 { .. } => JournalEntryRecordType::CreateEventV1, + Self::PortAddAddrV1 { .. } => JournalEntryRecordType::PortAddAddrV1, + Self::PortDelAddrV1 { .. } => JournalEntryRecordType::PortDelAddrV1, + Self::PortAddrClearV1 => JournalEntryRecordType::PortAddrClearV1, + Self::PortBridgeV1 { .. } => JournalEntryRecordType::PortBridgeV1, + Self::PortUnbridgeV1 => JournalEntryRecordType::PortUnbridgeV1, + Self::PortDhcpAcquireV1 => JournalEntryRecordType::PortDhcpAcquireV1, + Self::PortGatewaySetV1 { .. } => JournalEntryRecordType::PortGatewaySetV1, + Self::PortRouteAddV1 { .. } => JournalEntryRecordType::PortRouteAddV1, + Self::PortRouteClearV1 => JournalEntryRecordType::PortRouteClearV1, + Self::PortRouteDelV1 { .. } => JournalEntryRecordType::PortRouteDelV1, + Self::SocketOpenV1 { .. } => JournalEntryRecordType::SocketOpenV1, + Self::SocketListenV1 { .. } => JournalEntryRecordType::SocketListenV1, + Self::SocketBindV1 { .. } => JournalEntryRecordType::SocketBindV1, + Self::SocketConnectedV1 { .. } => JournalEntryRecordType::SocketConnectedV1, + Self::SocketAcceptedV1 { .. } => JournalEntryRecordType::SocketAcceptedV1, + Self::SocketJoinIpv4MulticastV1 { .. } => { JournalEntryRecordType::SocketJoinIpv4MulticastV1 } - Self::SocketJoinIpv6Multicast { .. } => { + Self::SocketJoinIpv6MulticastV1 { .. } => { JournalEntryRecordType::SocketJoinIpv6MulticastV1 } - Self::SocketLeaveIpv4Multicast { .. } => { + Self::SocketLeaveIpv4MulticastV1 { .. } => { JournalEntryRecordType::SocketLeaveIpv4MulticastV1 } - Self::SocketLeaveIpv6Multicast { .. } => { + Self::SocketLeaveIpv6MulticastV1 { .. } => { JournalEntryRecordType::SocketLeaveIpv6MulticastV1 } - Self::SocketSendFile { .. } => JournalEntryRecordType::SocketSendFileV1, - Self::SocketSendTo { .. } => JournalEntryRecordType::SocketSendToV1, - Self::SocketSend { .. } => JournalEntryRecordType::SocketSendV1, - Self::SocketSetOptFlag { .. } => JournalEntryRecordType::SocketSetOptFlagV1, - Self::SocketSetOptSize { .. } => JournalEntryRecordType::SocketSetOptSizeV1, - Self::SocketSetOptTime { .. } => JournalEntryRecordType::SocketSetOptTimeV1, - Self::SocketShutdown { .. } => JournalEntryRecordType::SocketShutdownV1, - Self::Snapshot { .. } => JournalEntryRecordType::SnapshotV1, + Self::SocketSendFileV1 { .. } => JournalEntryRecordType::SocketSendFileV1, + Self::SocketSendToV1 { .. } => JournalEntryRecordType::SocketSendToV1, + Self::SocketSendV1 { .. } => JournalEntryRecordType::SocketSendV1, + Self::SocketSetOptFlagV1 { .. } => JournalEntryRecordType::SocketSetOptFlagV1, + Self::SocketSetOptSizeV1 { .. } => JournalEntryRecordType::SocketSetOptSizeV1, + Self::SocketSetOptTimeV1 { .. } => JournalEntryRecordType::SocketSetOptTimeV1, + Self::SocketShutdownV1 { .. } => JournalEntryRecordType::SocketShutdownV1, + Self::SnapshotV1 { .. } => JournalEntryRecordType::SnapshotV1, } } @@ -429,24 +440,24 @@ impl<'a> JournalEntry<'a> { vec![0u8; padding] }; match self { - JournalEntry::InitModule { wasm_hash } => { + JournalEntry::InitModuleV1 { wasm_hash } => { serializer.serialize_value(&JournalEntryInitModuleV1 { wasm_hash }) } - JournalEntry::UpdateMemoryRegion { region, data } => { + JournalEntry::UpdateMemoryRegionV1 { region, data } => { serializer.serialize_value(&JournalEntryUpdateMemoryRegionV1 { start: region.start, end: region.end, _padding: padding(data.len()), - data: data.into_owned(), + compressed_data: compress_prepend_size(data.as_ref()), }) } - JournalEntry::ProcessExit { exit_code } => { + JournalEntry::ProcessExitV1 { exit_code } => { serializer.serialize_value(&JournalEntryProcessExitV1 { exit_code: exit_code.map(|e| e.into()), _padding: 0, }) } - JournalEntry::SetThread { + JournalEntry::SetThreadV1 { id, call_stack, memory_stack, @@ -460,20 +471,19 @@ impl<'a> JournalEntry<'a> { store_data: store_data.into_owned(), is_64bit, }), - JournalEntry::CloseThread { id, exit_code } => { + JournalEntry::CloseThreadV1 { id, exit_code } => { serializer.serialize_value(&JournalEntryCloseThreadV1 { id: id.into(), exit_code: exit_code.map(|e| e.into()), }) } - JournalEntry::FileDescriptorSeek { fd, offset, whence } => { - serializer.serialize_value(&JournalEntryFileDescriptorSeekV1 { + JournalEntry::FileDescriptorSeekV1 { fd, offset, whence } => serializer + .serialize_value(&JournalEntryFileDescriptorSeekV1 { fd, offset, whence: whence.into(), - }) - } - JournalEntry::FileDescriptorWrite { + }), + JournalEntry::FileDescriptorWriteV1 { fd, offset, data, @@ -485,16 +495,16 @@ impl<'a> JournalEntry<'a> { data: data.into_owned(), is_64bit, }), - JournalEntry::SetClockTime { clock_id, time } => { + JournalEntry::SetClockTimeV1 { clock_id, time } => { serializer.serialize_value(&JournalEntrySetClockTimeV1 { clock_id: clock_id.into(), time, }) } - JournalEntry::CloseFileDescriptor { fd } => { + JournalEntry::CloseFileDescriptorV1 { fd } => { serializer.serialize_value(&JournalEntryCloseFileDescriptorV1 { fd, _padding: 0 }) } - JournalEntry::OpenFileDescriptor { + JournalEntry::OpenFileDescriptorV1 { fd, dirfd, dirflags, @@ -514,31 +524,31 @@ impl<'a> JournalEntry<'a> { fs_rights_inheriting: fs_rights_inheriting.bits(), fs_flags: fs_flags.bits(), }), - JournalEntry::RenumberFileDescriptor { old_fd, new_fd } => { + JournalEntry::RenumberFileDescriptorV1 { old_fd, new_fd } => { serializer.serialize_value(&JournalEntryRenumberFileDescriptorV1 { old_fd, new_fd }) } - JournalEntry::DuplicateFileDescriptor { + JournalEntry::DuplicateFileDescriptorV1 { original_fd, copied_fd, } => serializer.serialize_value(&JournalEntryDuplicateFileDescriptorV1 { original_fd, copied_fd, }), - JournalEntry::CreateDirectory { fd, path } => { + JournalEntry::CreateDirectoryV1 { fd, path } => { serializer.serialize_value(&JournalEntryCreateDirectoryV1 { fd, _padding: padding(path.as_bytes().len()), path: path.into_owned(), }) } - JournalEntry::RemoveDirectory { fd, path } => { + JournalEntry::RemoveDirectoryV1 { fd, path } => { serializer.serialize_value(&JournalEntryRemoveDirectoryV1 { fd, _padding: padding(path.as_bytes().len()), path: path.into_owned(), }) } - JournalEntry::PathSetTimes { + JournalEntry::PathSetTimesV1 { fd, flags, path, @@ -554,7 +564,7 @@ impl<'a> JournalEntry<'a> { st_mtim, fst_flags: fst_flags.bits(), }), - JournalEntry::FileDescriptorSetTimes { + JournalEntry::FileDescriptorSetTimesV1 { fd, st_atim, st_mtim, @@ -565,13 +575,13 @@ impl<'a> JournalEntry<'a> { st_mtim, fst_flags: fst_flags.bits(), }), - JournalEntry::FileDescriptorSetFlags { fd, flags } => { + JournalEntry::FileDescriptorSetFlagsV1 { fd, flags } => { serializer.serialize_value(&JournalEntryFileDescriptorSetFlagsV1 { fd, flags: flags.bits(), }) } - JournalEntry::FileDescriptorSetRights { + JournalEntry::FileDescriptorSetRightsV1 { fd, fs_rights_base, fs_rights_inheriting, @@ -580,10 +590,10 @@ impl<'a> JournalEntry<'a> { fs_rights_base: fs_rights_base.bits(), fs_rights_inheriting: fs_rights_inheriting.bits(), }), - JournalEntry::FileDescriptorSetSize { fd, st_size } => { + JournalEntry::FileDescriptorSetSizeV1 { fd, st_size } => { serializer.serialize_value(&JournalEntryFileDescriptorSetSizeV1 { fd, st_size }) } - JournalEntry::FileDescriptorAdvise { + JournalEntry::FileDescriptorAdviseV1 { fd, offset, len, @@ -594,9 +604,9 @@ impl<'a> JournalEntry<'a> { len, advice: advice.into(), }), - JournalEntry::FileDescriptorAllocate { fd, offset, len } => serializer + JournalEntry::FileDescriptorAllocateV1 { fd, offset, len } => serializer .serialize_value(&JournalEntryFileDescriptorAllocateV1 { fd, offset, len }), - JournalEntry::CreateHardLink { + JournalEntry::CreateHardLinkV1 { old_fd, old_path, old_flags, @@ -610,7 +620,7 @@ impl<'a> JournalEntry<'a> { new_fd, new_path: new_path.into_owned(), }), - JournalEntry::CreateSymbolicLink { + JournalEntry::CreateSymbolicLinkV1 { old_path, fd, new_path, @@ -620,14 +630,14 @@ impl<'a> JournalEntry<'a> { fd, new_path: new_path.into_owned(), }), - JournalEntry::UnlinkFile { fd, path } => { + JournalEntry::UnlinkFileV1 { fd, path } => { serializer.serialize_value(&JournalEntryUnlinkFileV1 { fd, _padding: padding(path.as_bytes().len()), path: path.into_owned(), }) } - JournalEntry::PathRename { + JournalEntry::PathRenameV1 { old_fd, old_path, new_fd, @@ -639,15 +649,15 @@ impl<'a> JournalEntry<'a> { new_fd, new_path: new_path.into_owned(), }), - JournalEntry::ChangeDirectory { path } => { + JournalEntry::ChangeDirectoryV1 { path } => { serializer.serialize_value(&JournalEntryChangeDirectoryV1 { path: path.into_owned(), }) } - JournalEntry::EpollCreate { fd } => { + JournalEntry::EpollCreateV1 { fd } => { serializer.serialize_value(&JournalEntryEpollCreateV1 { fd, _padding: 0 }) } - JournalEntry::EpollCtl { + JournalEntry::EpollCtlV1 { epfd, op, fd, @@ -658,7 +668,7 @@ impl<'a> JournalEntry<'a> { fd, event: event.map(|e| e.into()), }), - JournalEntry::TtySet { tty, line_feeds } => { + JournalEntry::TtySetV1 { tty, line_feeds } => { serializer.serialize_value(&JournalEntryTtySetV1 { cols: tty.cols, rows: tty.rows, @@ -672,10 +682,10 @@ impl<'a> JournalEntry<'a> { line_feeds, }) } - JournalEntry::CreatePipe { fd1, fd2 } => { + JournalEntry::CreatePipeV1 { fd1, fd2 } => { serializer.serialize_value(&JournalEntryCreatePipeV1 { fd1, fd2 }) } - JournalEntry::CreateEvent { + JournalEntry::CreateEventV1 { initial_val, flags, fd, @@ -684,14 +694,14 @@ impl<'a> JournalEntry<'a> { flags, fd, }), - JournalEntry::PortAddAddr { cidr } => { + JournalEntry::PortAddAddrV1 { cidr } => { serializer.serialize_value(&JournalEntryPortAddAddrV1 { cidr }) } - JournalEntry::PortDelAddr { addr } => { + JournalEntry::PortDelAddrV1 { addr } => { serializer.serialize_value(&JournalEntryPortDelAddrV1 { addr }) } - JournalEntry::PortAddrClear => return Ok(()), - JournalEntry::PortBridge { + JournalEntry::PortAddrClearV1 => return Ok(()), + JournalEntry::PortBridgeV1 { network, token, security, @@ -701,12 +711,12 @@ impl<'a> JournalEntry<'a> { token: token.into_owned(), security: security.into(), }), - JournalEntry::PortUnbridge => return Ok(()), - JournalEntry::PortDhcpAcquire => return Ok(()), - JournalEntry::PortGatewaySet { ip } => { + JournalEntry::PortUnbridgeV1 => return Ok(()), + JournalEntry::PortDhcpAcquireV1 => return Ok(()), + JournalEntry::PortGatewaySetV1 { ip } => { serializer.serialize_value(&JournalEntryPortGatewaySetV1 { ip }) } - JournalEntry::PortRouteAdd { + JournalEntry::PortRouteAddV1 { cidr, via_router, preferred_until, @@ -717,11 +727,11 @@ impl<'a> JournalEntry<'a> { preferred_until, expires_at, }), - JournalEntry::PortRouteClear => return Ok(()), - JournalEntry::PortRouteDel { ip } => { + JournalEntry::PortRouteClearV1 => return Ok(()), + JournalEntry::PortRouteDelV1 { ip } => { serializer.serialize_value(&JournalEntryPortRouteDelV1 { ip }) } - JournalEntry::SocketOpen { af, ty, pt, fd } => { + JournalEntry::SocketOpenV1 { af, ty, pt, fd } => { serializer.serialize_value(&JournalEntrySocketOpenV1 { af: af.into(), ty: ty.into(), @@ -729,21 +739,21 @@ impl<'a> JournalEntry<'a> { fd, }) } - JournalEntry::SocketListen { fd, backlog } => { + JournalEntry::SocketListenV1 { fd, backlog } => { serializer.serialize_value(&JournalEntrySocketListenV1 { fd, backlog }) } - JournalEntry::SocketBind { fd, addr } => { + JournalEntry::SocketBindV1 { fd, addr } => { serializer.serialize_value(&JournalEntrySocketBindV1 { fd, addr }) } - JournalEntry::SocketConnected { fd, addr } => { + JournalEntry::SocketConnectedV1 { fd, addr } => { serializer.serialize_value(&JournalEntrySocketConnectedV1 { fd, addr }) } - JournalEntry::SocketAccepted { + JournalEntry::SocketAcceptedV1 { listen_fd, fd, peer_addr, fd_flags, - nonblocking, + non_blocking: nonblocking, } => serializer.serialize_value(&JournalEntrySocketAcceptedV1 { listen_fd, fd, @@ -751,7 +761,7 @@ impl<'a> JournalEntry<'a> { fd_flags: fd_flags.bits(), nonblocking, }), - JournalEntry::SocketJoinIpv4Multicast { + JournalEntry::SocketJoinIpv4MulticastV1 { fd, multiaddr, iface, @@ -760,34 +770,34 @@ impl<'a> JournalEntry<'a> { multiaddr, iface, }), - JournalEntry::SocketJoinIpv6Multicast { + JournalEntry::SocketJoinIpv6MulticastV1 { fd, - multiaddr, + multi_addr: multiaddr, iface, } => serializer.serialize_value(&JournalEntrySocketJoinIpv6MulticastV1 { fd, multiaddr, iface, }), - JournalEntry::SocketLeaveIpv4Multicast { + JournalEntry::SocketLeaveIpv4MulticastV1 { fd, - multiaddr, + multi_addr: multiaddr, iface, } => serializer.serialize_value(&JournalEntrySocketLeaveIpv4MulticastV1 { fd, multiaddr, iface, }), - JournalEntry::SocketLeaveIpv6Multicast { + JournalEntry::SocketLeaveIpv6MulticastV1 { fd, - multiaddr, + multi_addr: multiaddr, iface, } => serializer.serialize_value(&JournalEntrySocketLeaveIpv6MulticastV1 { fd, multiaddr, iface, }), - JournalEntry::SocketSendFile { + JournalEntry::SocketSendFileV1 { socket_fd, file_fd, offset, @@ -798,7 +808,7 @@ impl<'a> JournalEntry<'a> { offset, count, }), - JournalEntry::SocketSendTo { + JournalEntry::SocketSendToV1 { fd, data, flags, @@ -812,7 +822,7 @@ impl<'a> JournalEntry<'a> { addr, is_64bit, }), - JournalEntry::SocketSend { + JournalEntry::SocketSendV1 { fd, data, flags, @@ -824,34 +834,34 @@ impl<'a> JournalEntry<'a> { flags, is_64bit, }), - JournalEntry::SocketSetOptFlag { fd, opt, flag } => { + JournalEntry::SocketSetOptFlagV1 { fd, opt, flag } => { serializer.serialize_value(&JournalEntrySocketSetOptFlagV1 { fd, opt: opt.into(), flag, }) } - JournalEntry::SocketSetOptSize { fd, opt, size } => { + JournalEntry::SocketSetOptSizeV1 { fd, opt, size } => { serializer.serialize_value(&JournalEntrySocketSetOptSizeV1 { fd, opt: opt.into(), size, }) } - JournalEntry::SocketSetOptTime { fd, ty, time } => { + JournalEntry::SocketSetOptTimeV1 { fd, ty, time } => { serializer.serialize_value(&JournalEntrySocketSetOptTimeV1 { fd, ty: ty.into(), time, }) } - JournalEntry::SocketShutdown { fd, how } => { + JournalEntry::SocketShutdownV1 { fd, how } => { serializer.serialize_value(&JournalEntrySocketShutdownV1 { fd, how: how.into(), }) } - JournalEntry::Snapshot { when, trigger } => { + JournalEntry::SnapshotV1 { when, trigger } => { serializer.serialize_value(&JournalEntrySnapshotV1 { since_epoch: when .duration_since(SystemTime::UNIX_EPOCH) @@ -1007,7 +1017,7 @@ pub struct JournalEntryFileDescriptorWriteV1 { pub struct JournalEntryUpdateMemoryRegionV1 { pub start: u64, pub end: u64, - pub data: Vec, + pub compressed_data: Vec, pub _padding: Vec, } @@ -2300,11 +2310,13 @@ impl From<&'_ ArchivedJournalSocketShutdownV1> for SocketShutdownHow { } } -impl<'a> From> for JournalEntry<'a> { - fn from(value: ArchivedJournalEntry<'a>) -> Self { - match value { +impl<'a> TryFrom> for JournalEntry<'a> { + type Error = anyhow::Error; + + fn try_from(value: ArchivedJournalEntry<'a>) -> anyhow::Result { + Ok(match value { ArchivedJournalEntry::InitModuleV1(ArchivedJournalEntryInitModuleV1 { wasm_hash }) => { - Self::InitModule { + Self::InitModuleV1 { wasm_hash: *wasm_hash, } } @@ -2312,17 +2324,17 @@ impl<'a> From> for JournalEntry<'a> { ArchivedJournalEntryUpdateMemoryRegionV1 { start, end, - data, + compressed_data, _padding: _, }, - ) => Self::UpdateMemoryRegion { + ) => Self::UpdateMemoryRegionV1 { region: (*start)..(*end), - data: Cow::Borrowed(data.as_ref()), + data: Cow::Owned(decompress_size_prepended(compressed_data.as_ref())?), }, ArchivedJournalEntry::ProcessExitV1(ArchivedJournalEntryProcessExitV1 { exit_code, _padding: _, - }) => Self::ProcessExit { + }) => Self::ProcessExitV1 { exit_code: exit_code.as_ref().map(|code| code.into()), }, ArchivedJournalEntry::SetThreadV1(ArchivedJournalEntrySetThreadV1 { @@ -2332,7 +2344,7 @@ impl<'a> From> for JournalEntry<'a> { store_data, _padding: _, is_64bit, - }) => Self::SetThread { + }) => Self::SetThreadV1 { id: (*id).into(), call_stack: call_stack.as_ref().into(), memory_stack: memory_stack.as_ref().into(), @@ -2342,7 +2354,7 @@ impl<'a> From> for JournalEntry<'a> { ArchivedJournalEntry::CloseThreadV1(ArchivedJournalEntryCloseThreadV1 { id, exit_code, - }) => Self::CloseThread { + }) => Self::CloseThreadV1 { id: (*id).into(), exit_code: exit_code.as_ref().map(|code| code.into()), }, @@ -2354,7 +2366,7 @@ impl<'a> From> for JournalEntry<'a> { is_64bit, _padding: _, }, - ) => Self::FileDescriptorWrite { + ) => Self::FileDescriptorWriteV1 { data: data.as_ref().into(), fd: *fd, offset: *offset, @@ -2366,7 +2378,7 @@ impl<'a> From> for JournalEntry<'a> { offset, ref whence, }, - ) => Self::FileDescriptorSeek { + ) => Self::FileDescriptorSeekV1 { fd: *fd, offset: *offset, whence: whence.into(), @@ -2383,7 +2395,7 @@ impl<'a> From> for JournalEntry<'a> { fs_flags, _padding: _, }, - ) => Self::OpenFileDescriptor { + ) => Self::OpenFileDescriptorV1 { fd: *fd, dirfd: *dirfd, dirflags: *dirflags, @@ -2395,12 +2407,12 @@ impl<'a> From> for JournalEntry<'a> { }, ArchivedJournalEntry::CloseFileDescriptorV1( ArchivedJournalEntryCloseFileDescriptorV1 { fd, _padding: _ }, - ) => Self::CloseFileDescriptor { fd: *fd }, + ) => Self::CloseFileDescriptorV1 { fd: *fd }, ArchivedJournalEntry::RemoveDirectoryV1(ArchivedJournalEntryRemoveDirectoryV1 { fd, path, _padding: _, - }) => Self::RemoveDirectory { + }) => Self::RemoveDirectoryV1 { fd: *fd, path: path.as_ref().into(), }, @@ -2408,7 +2420,7 @@ impl<'a> From> for JournalEntry<'a> { fd, path, _padding: _, - }) => Self::UnlinkFile { + }) => Self::UnlinkFileV1 { fd: *fd, path: path.as_ref().into(), }, @@ -2418,7 +2430,7 @@ impl<'a> From> for JournalEntry<'a> { new_fd, new_path, _padding: _, - }) => Self::PathRename { + }) => Self::PathRenameV1 { old_fd: *old_fd, old_path: old_path.as_ref().into(), new_fd: *new_fd, @@ -2427,7 +2439,7 @@ impl<'a> From> for JournalEntry<'a> { ArchivedJournalEntry::SnapshotV1(ArchivedJournalEntrySnapshotV1 { since_epoch, ref trigger, - }) => Self::Snapshot { + }) => Self::SnapshotV1 { when: SystemTime::UNIX_EPOCH .checked_add((*since_epoch).try_into().unwrap()) .unwrap_or(SystemTime::UNIX_EPOCH), @@ -2436,13 +2448,13 @@ impl<'a> From> for JournalEntry<'a> { ArchivedJournalEntry::SetClockTimeV1(ArchivedJournalEntrySetClockTimeV1 { ref clock_id, time, - }) => Self::SetClockTime { + }) => Self::SetClockTimeV1 { clock_id: clock_id.into(), time: *time, }, ArchivedJournalEntry::RenumberFileDescriptorV1( ArchivedJournalEntryRenumberFileDescriptorV1 { old_fd, new_fd }, - ) => Self::RenumberFileDescriptor { + ) => Self::RenumberFileDescriptorV1 { old_fd: *old_fd, new_fd: *new_fd, }, @@ -2451,7 +2463,7 @@ impl<'a> From> for JournalEntry<'a> { original_fd: old_fd, copied_fd: new_fd, }, - ) => Self::DuplicateFileDescriptor { + ) => Self::DuplicateFileDescriptorV1 { original_fd: *old_fd, copied_fd: *new_fd, }, @@ -2459,7 +2471,7 @@ impl<'a> From> for JournalEntry<'a> { fd, path, _padding: _, - }) => Self::CreateDirectory { + }) => Self::CreateDirectoryV1 { fd: *fd, path: path.as_ref().into(), }, @@ -2471,7 +2483,7 @@ impl<'a> From> for JournalEntry<'a> { st_mtim, fst_flags, _padding: _, - }) => Self::PathSetTimes { + }) => Self::PathSetTimesV1 { fd: *fd, path: path.as_ref().into(), flags: *flags, @@ -2486,7 +2498,7 @@ impl<'a> From> for JournalEntry<'a> { st_mtim, fst_flags, }, - ) => Self::FileDescriptorSetTimes { + ) => Self::FileDescriptorSetTimesV1 { fd: *fd, st_atim: *st_atim, st_mtim: *st_mtim, @@ -2494,13 +2506,13 @@ impl<'a> From> for JournalEntry<'a> { }, ArchivedJournalEntry::FileDescriptorSetSizeV1( ArchivedJournalEntryFileDescriptorSetSizeV1 { fd, st_size }, - ) => Self::FileDescriptorSetSize { + ) => Self::FileDescriptorSetSizeV1 { fd: *fd, st_size: *st_size, }, ArchivedJournalEntry::FileDescriptorSetFlagsV1( ArchivedJournalEntryFileDescriptorSetFlagsV1 { fd, flags }, - ) => Self::FileDescriptorSetFlags { + ) => Self::FileDescriptorSetFlagsV1 { fd: *fd, flags: Fdflags::from_bits_truncate(*flags), }, @@ -2510,7 +2522,7 @@ impl<'a> From> for JournalEntry<'a> { fs_rights_base, fs_rights_inheriting, }, - ) => Self::FileDescriptorSetRights { + ) => Self::FileDescriptorSetRightsV1 { fd: *fd, fs_rights_base: Rights::from_bits_truncate(*fs_rights_base), fs_rights_inheriting: Rights::from_bits_truncate(*fs_rights_inheriting), @@ -2522,7 +2534,7 @@ impl<'a> From> for JournalEntry<'a> { len, ref advice, }, - ) => Self::FileDescriptorAdvise { + ) => Self::FileDescriptorAdviseV1 { fd: *fd, offset: *offset, len: *len, @@ -2530,7 +2542,7 @@ impl<'a> From> for JournalEntry<'a> { }, ArchivedJournalEntry::FileDescriptorAllocateV1( ArchivedJournalEntryFileDescriptorAllocateV1 { fd, offset, len }, - ) => Self::FileDescriptorAllocate { + ) => Self::FileDescriptorAllocateV1 { fd: *fd, offset: *offset, len: *len, @@ -2542,7 +2554,7 @@ impl<'a> From> for JournalEntry<'a> { new_fd, new_path, _padding: _, - }) => Self::CreateHardLink { + }) => Self::CreateHardLinkV1 { old_fd: *old_fd, old_path: old_path.as_ref().into(), old_flags: *old_flags, @@ -2556,26 +2568,26 @@ impl<'a> From> for JournalEntry<'a> { new_path, _padding: _, }, - ) => Self::CreateSymbolicLink { + ) => Self::CreateSymbolicLinkV1 { old_path: old_path.as_ref().into(), fd: *fd, new_path: new_path.as_ref().into(), }, ArchivedJournalEntry::ChangeDirectoryV1(ArchivedJournalEntryChangeDirectoryV1 { path, - }) => Self::ChangeDirectory { + }) => Self::ChangeDirectoryV1 { path: path.as_ref().into(), }, ArchivedJournalEntry::EpollCreateV1(ArchivedJournalEntryEpollCreateV1 { fd, _padding: _, - }) => Self::EpollCreate { fd: *fd }, + }) => Self::EpollCreateV1 { fd: *fd }, ArchivedJournalEntry::EpollCtlV1(ArchivedJournalEntryEpollCtlV1 { epfd, ref op, fd, ref event, - }) => Self::EpollCtl { + }) => Self::EpollCtlV1 { epfd: *epfd, op: op.into(), fd: *fd, @@ -2592,7 +2604,7 @@ impl<'a> From> for JournalEntry<'a> { echo, line_buffered, line_feeds, - }) => Self::TtySet { + }) => Self::TtySetV1 { tty: wasi::Tty { cols: *cols, rows: *rows, @@ -2607,13 +2619,13 @@ impl<'a> From> for JournalEntry<'a> { line_feeds: *line_feeds, }, ArchivedJournalEntry::CreatePipeV1(ArchivedJournalEntryCreatePipeV1 { fd1, fd2 }) => { - Self::CreatePipe { + Self::CreatePipeV1 { fd1: *fd1, fd2: *fd2, } } ArchivedJournalEntry::PortAddAddrV1(ArchivedJournalEntryPortAddAddrV1 { cidr }) => { - Self::PortAddAddr { + Self::PortAddAddrV1 { cidr: IpCidr { ip: cidr.ip.as_ipaddr(), prefix: cidr.prefix, @@ -2621,32 +2633,32 @@ impl<'a> From> for JournalEntry<'a> { } } ArchivedJournalEntry::PortDelAddrV1(ArchivedJournalEntryPortDelAddrV1 { addr }) => { - Self::PortDelAddr { + Self::PortDelAddrV1 { addr: addr.as_ipaddr(), } } - ArchivedJournalEntry::PortAddrClearV1 => Self::PortAddrClear, + ArchivedJournalEntry::PortAddrClearV1 => Self::PortAddrClearV1, ArchivedJournalEntry::PortBridgeV1(ArchivedJournalEntryPortBridgeV1 { network, token, ref security, _padding: _, - }) => Self::PortBridge { + }) => Self::PortBridgeV1 { network: network.as_ref().into(), token: token.as_ref().into(), security: security.into(), }, - ArchivedJournalEntry::PortUnbridgeV1 => Self::PortUnbridge, - ArchivedJournalEntry::PortDhcpAcquireV1 => Self::PortDhcpAcquire, + ArchivedJournalEntry::PortUnbridgeV1 => Self::PortUnbridgeV1, + ArchivedJournalEntry::PortDhcpAcquireV1 => Self::PortDhcpAcquireV1, ArchivedJournalEntry::PortGatewaySetV1(ArchivedJournalEntryPortGatewaySetV1 { ip }) => { - Self::PortGatewaySet { ip: ip.as_ipaddr() } + Self::PortGatewaySetV1 { ip: ip.as_ipaddr() } } ArchivedJournalEntry::PortRouteAddV1(ArchivedJournalEntryPortRouteAddV1 { cidr, via_router, preferred_until, expires_at, - }) => Self::PortRouteAdd { + }) => Self::PortRouteAddV1 { cidr: IpCidr { ip: cidr.ip.as_ipaddr(), prefix: cidr.prefix, @@ -2657,16 +2669,16 @@ impl<'a> From> for JournalEntry<'a> { .map(|time| (*time).try_into().unwrap()), expires_at: expires_at.as_ref().map(|time| (*time).try_into().unwrap()), }, - ArchivedJournalEntry::PortRouteClearV1 => Self::PortRouteClear, + ArchivedJournalEntry::PortRouteClearV1 => Self::PortRouteClearV1, ArchivedJournalEntry::PortRouteDelV1(ArchivedJournalEntryPortRouteDelV1 { ip }) => { - Self::PortRouteDel { ip: ip.as_ipaddr() } + Self::PortRouteDelV1 { ip: ip.as_ipaddr() } } ArchivedJournalEntry::SocketOpenV1(ArchivedJournalEntrySocketOpenV1 { ref af, ref ty, pt, fd, - }) => Self::SocketOpen { + }) => Self::SocketOpenV1 { af: af.into(), ty: ty.into(), pt: (*pt).try_into().unwrap_or(wasi::SockProto::Max), @@ -2675,12 +2687,12 @@ impl<'a> From> for JournalEntry<'a> { ArchivedJournalEntry::SocketListenV1(ArchivedJournalEntrySocketListenV1 { fd, backlog, - }) => Self::SocketListen { + }) => Self::SocketListenV1 { fd: *fd, backlog: *backlog, }, ArchivedJournalEntry::SocketBindV1(ArchivedJournalEntrySocketBindV1 { fd, addr }) => { - Self::SocketBind { + Self::SocketBindV1 { fd: *fd, addr: addr.as_socket_addr(), } @@ -2688,7 +2700,7 @@ impl<'a> From> for JournalEntry<'a> { ArchivedJournalEntry::SocketConnectedV1(ArchivedJournalEntrySocketConnectedV1 { fd, addr, - }) => Self::SocketConnected { + }) => Self::SocketConnectedV1 { fd: *fd, addr: addr.as_socket_addr(), }, @@ -2698,12 +2710,12 @@ impl<'a> From> for JournalEntry<'a> { peer_addr, fd_flags, nonblocking, - }) => Self::SocketAccepted { + }) => Self::SocketAcceptedV1 { listen_fd: *listen_fd, fd: *fd, peer_addr: peer_addr.as_socket_addr(), fd_flags: Fdflags::from_bits_truncate(*fd_flags), - nonblocking: *nonblocking, + non_blocking: *nonblocking, }, ArchivedJournalEntry::SocketJoinIpv4MulticastV1( ArchivedJournalEntrySocketJoinIpv4MulticastV1 { @@ -2711,7 +2723,7 @@ impl<'a> From> for JournalEntry<'a> { multiaddr, iface, }, - ) => Self::SocketJoinIpv4Multicast { + ) => Self::SocketJoinIpv4MulticastV1 { fd: *fd, multiaddr: multiaddr.as_ipv4(), iface: iface.as_ipv4(), @@ -2722,9 +2734,9 @@ impl<'a> From> for JournalEntry<'a> { multiaddr, iface, }, - ) => Self::SocketJoinIpv6Multicast { + ) => Self::SocketJoinIpv6MulticastV1 { fd: *fd, - multiaddr: multiaddr.as_ipv6(), + multi_addr: multiaddr.as_ipv6(), iface: *iface, }, ArchivedJournalEntry::SocketLeaveIpv4MulticastV1( @@ -2733,9 +2745,9 @@ impl<'a> From> for JournalEntry<'a> { multiaddr, iface, }, - ) => Self::SocketLeaveIpv4Multicast { + ) => Self::SocketLeaveIpv4MulticastV1 { fd: *fd, - multiaddr: multiaddr.as_ipv4(), + multi_addr: multiaddr.as_ipv4(), iface: iface.as_ipv4(), }, ArchivedJournalEntry::SocketLeaveIpv6MulticastV1( @@ -2744,9 +2756,9 @@ impl<'a> From> for JournalEntry<'a> { multiaddr, iface, }, - ) => Self::SocketLeaveIpv6Multicast { + ) => Self::SocketLeaveIpv6MulticastV1 { fd: *fd, - multiaddr: multiaddr.as_ipv6(), + multi_addr: multiaddr.as_ipv6(), iface: *iface, }, ArchivedJournalEntry::SocketSendFileV1(ArchivedJournalEntrySocketSendFileV1 { @@ -2754,7 +2766,7 @@ impl<'a> From> for JournalEntry<'a> { file_fd, offset, count, - }) => Self::SocketSendFile { + }) => Self::SocketSendFileV1 { socket_fd: *socket_fd, file_fd: *file_fd, offset: *offset, @@ -2767,7 +2779,7 @@ impl<'a> From> for JournalEntry<'a> { addr, is_64bit, _padding: _, - }) => Self::SocketSendTo { + }) => Self::SocketSendToV1 { fd: *fd, data: data.as_ref().into(), flags: *flags, @@ -2780,7 +2792,7 @@ impl<'a> From> for JournalEntry<'a> { flags, is_64bit, _padding: _, - }) => Self::SocketSend { + }) => Self::SocketSendV1 { fd: *fd, data: data.as_ref().into(), flags: *flags, @@ -2790,7 +2802,7 @@ impl<'a> From> for JournalEntry<'a> { fd, ref opt, flag, - }) => Self::SocketSetOptFlag { + }) => Self::SocketSetOptFlagV1 { fd: *fd, opt: opt.into(), flag: *flag, @@ -2799,7 +2811,7 @@ impl<'a> From> for JournalEntry<'a> { fd, ref opt, size, - }) => Self::SocketSetOptSize { + }) => Self::SocketSetOptSizeV1 { fd: *fd, opt: opt.into(), size: *size, @@ -2808,7 +2820,7 @@ impl<'a> From> for JournalEntry<'a> { fd, ref ty, time, - }) => Self::SocketSetOptTime { + }) => Self::SocketSetOptTimeV1 { fd: *fd, ty: ty.into(), time: time.as_ref().map(|time| (*time).try_into().unwrap()), @@ -2816,7 +2828,7 @@ impl<'a> From> for JournalEntry<'a> { ArchivedJournalEntry::SocketShutdownV1(ArchivedJournalEntrySocketShutdownV1 { fd, ref how, - }) => Self::SocketShutdown { + }) => Self::SocketShutdownV1 { fd: *fd, how: how.into(), }, @@ -2824,12 +2836,12 @@ impl<'a> From> for JournalEntry<'a> { initial_val, flags, fd, - }) => Self::CreateEvent { + }) => Self::CreateEventV1 { initial_val: *initial_val, flags: *flags, fd: *fd, }, - } + }) } } @@ -2865,7 +2877,7 @@ mod tests { } // Deserialize it - let record2 = unsafe { record_type.deserialize_archive(buffer) }; + let record2 = unsafe { record_type.deserialize_archive(buffer).unwrap() }; tracing::info!("record2: {:?}", record2); // Check it @@ -2880,7 +2892,7 @@ mod tests { #[tracing_test::traced_test] #[test] pub fn test_record_init_module() { - run_test(JournalEntry::InitModule { + run_test(JournalEntry::InitModuleV1 { wasm_hash: [13u8; 32], }); } @@ -2888,7 +2900,7 @@ mod tests { #[tracing_test::traced_test] #[test] pub fn test_record_process_exit() { - run_test(JournalEntry::ProcessExit { + run_test(JournalEntry::ProcessExitV1 { exit_code: Some(wasi::ExitCode::Errno(wasi::Errno::Fault)), }); } @@ -2896,7 +2908,7 @@ mod tests { #[tracing_test::traced_test] #[test] pub fn test_record_set_thread() { - run_test(JournalEntry::SetThread { + run_test(JournalEntry::SetThreadV1 { id: 1234u32.into(), call_stack: vec![1, 2, 3].into(), memory_stack: vec![4, 5, 6, 7].into(), @@ -2908,7 +2920,7 @@ mod tests { #[tracing_test::traced_test] #[test] pub fn test_record_close_thread() { - run_test(JournalEntry::CloseThread { + run_test(JournalEntry::CloseThreadV1 { id: 987u32.into(), exit_code: Some(wasi::ExitCode::Errno(wasi::Errno::Fault)), }); @@ -2917,7 +2929,7 @@ mod tests { #[tracing_test::traced_test] #[test] pub fn test_record_descriptor_seek() { - run_test(JournalEntry::FileDescriptorSeek { + run_test(JournalEntry::FileDescriptorSeekV1 { fd: 765u32, offset: 9183722450971234i64, whence: wasi::Whence::End, @@ -2927,7 +2939,7 @@ mod tests { #[tracing_test::traced_test] #[test] pub fn test_record_descriptor_write() { - run_test(JournalEntry::FileDescriptorWrite { + run_test(JournalEntry::FileDescriptorWriteV1 { fd: 54321u32, offset: 13897412934u64, data: vec![74u8, 98u8, 36u8].into(), @@ -2938,7 +2950,7 @@ mod tests { #[tracing_test::traced_test] #[test] pub fn test_record_update_memory() { - run_test(JournalEntry::UpdateMemoryRegion { + run_test(JournalEntry::UpdateMemoryRegionV1 { region: 76u64..8237453u64, data: [74u8; 40960].to_vec().into(), }); @@ -2947,7 +2959,7 @@ mod tests { #[tracing_test::traced_test] #[test] pub fn test_record_set_clock_time() { - run_test(JournalEntry::SetClockTime { + run_test(JournalEntry::SetClockTimeV1 { clock_id: wasi::Snapshot0Clockid::Realtime, time: 7912837412934u64, }); @@ -2956,7 +2968,7 @@ mod tests { #[tracing_test::traced_test] #[test] pub fn test_record_open_file_descriptor() { - run_test(JournalEntry::OpenFileDescriptor { + run_test(JournalEntry::OpenFileDescriptorV1 { fd: 298745u32, dirfd: 23458922u32, dirflags: 134512345, @@ -2971,13 +2983,13 @@ mod tests { #[tracing_test::traced_test] #[test] pub fn test_record_close_descriptor() { - run_test(JournalEntry::CloseFileDescriptor { fd: 23845732u32 }); + run_test(JournalEntry::CloseFileDescriptorV1 { fd: 23845732u32 }); } #[tracing_test::traced_test] #[test] pub fn test_record_renumber_file_descriptor() { - run_test(JournalEntry::RenumberFileDescriptor { + run_test(JournalEntry::RenumberFileDescriptorV1 { old_fd: 27834u32, new_fd: 398452345u32, }); @@ -2986,7 +2998,7 @@ mod tests { #[tracing_test::traced_test] #[test] pub fn test_record_duplicate_file_descriptor() { - run_test(JournalEntry::DuplicateFileDescriptor { + run_test(JournalEntry::DuplicateFileDescriptorV1 { original_fd: 23482934u32, copied_fd: 9384529u32, }); @@ -2995,7 +3007,7 @@ mod tests { #[tracing_test::traced_test] #[test] pub fn test_record_create_directory() { - run_test(JournalEntry::CreateDirectory { + run_test(JournalEntry::CreateDirectoryV1 { fd: 238472u32, path: "/joasjdf/asdfn".into(), }); @@ -3004,7 +3016,7 @@ mod tests { #[tracing_test::traced_test] #[test] pub fn test_record_remove_directory() { - run_test(JournalEntry::RemoveDirectory { + run_test(JournalEntry::RemoveDirectoryV1 { fd: 23894952u32, path: "/blahblah".into(), }); @@ -3013,7 +3025,7 @@ mod tests { #[tracing_test::traced_test] #[test] pub fn test_record_path_set_times() { - run_test(JournalEntry::PathSetTimes { + run_test(JournalEntry::PathSetTimesV1 { fd: 1238934u32, flags: 234523, path: "/".into(), @@ -3026,7 +3038,7 @@ mod tests { #[tracing_test::traced_test] #[test] pub fn test_record_file_descriptor_set_times() { - run_test(JournalEntry::FileDescriptorSetTimes { + run_test(JournalEntry::FileDescriptorSetTimesV1 { fd: 898785u32, st_atim: 29834952345, st_mtim: 239845892345, @@ -3037,7 +3049,7 @@ mod tests { #[tracing_test::traced_test] #[test] pub fn test_record_file_descriptor_set_size() { - run_test(JournalEntry::FileDescriptorSetSize { + run_test(JournalEntry::FileDescriptorSetSizeV1 { fd: 34958234u32, st_size: 234958293845u64, }); @@ -3046,7 +3058,7 @@ mod tests { #[tracing_test::traced_test] #[test] pub fn test_record_file_descriptor_set_flags() { - run_test(JournalEntry::FileDescriptorSetFlags { + run_test(JournalEntry::FileDescriptorSetFlagsV1 { fd: 982348752u32, flags: wasi::Fdflags::all(), }); @@ -3055,7 +3067,7 @@ mod tests { #[tracing_test::traced_test] #[test] pub fn test_record_file_descriptor_set_rights() { - run_test(JournalEntry::FileDescriptorSetRights { + run_test(JournalEntry::FileDescriptorSetRightsV1 { fd: 872345u32, fs_rights_base: wasi::Rights::all(), fs_rights_inheriting: wasi::Rights::all(), @@ -3065,7 +3077,7 @@ mod tests { #[tracing_test::traced_test] #[test] pub fn test_record_file_descriptor_advise() { - run_test(JournalEntry::FileDescriptorAdvise { + run_test(JournalEntry::FileDescriptorAdviseV1 { fd: 298434u32, offset: 92834529092345, len: 23485928345, @@ -3076,7 +3088,7 @@ mod tests { #[tracing_test::traced_test] #[test] pub fn test_record_file_descriptor_allocate() { - run_test(JournalEntry::FileDescriptorAllocate { + run_test(JournalEntry::FileDescriptorAllocateV1 { fd: 2934852, offset: 23489582934523, len: 9845982345, @@ -3086,7 +3098,7 @@ mod tests { #[tracing_test::traced_test] #[test] pub fn test_record_create_hard_link() { - run_test(JournalEntry::CreateHardLink { + run_test(JournalEntry::CreateHardLinkV1 { old_fd: 324983845, old_path: "/asjdfiasidfasdf".into(), old_flags: 234857, @@ -3098,7 +3110,7 @@ mod tests { #[tracing_test::traced_test] #[test] pub fn test_record_create_symbolic_link() { - run_test(JournalEntry::CreateSymbolicLink { + run_test(JournalEntry::CreateSymbolicLinkV1 { old_path: "/asjbndfjasdf/asdafasdf".into(), fd: 235422345, new_path: "/asdf".into(), @@ -3108,7 +3120,7 @@ mod tests { #[tracing_test::traced_test] #[test] pub fn test_record_unlink_file() { - run_test(JournalEntry::UnlinkFile { + run_test(JournalEntry::UnlinkFileV1 { fd: 32452345, path: "/asdfasd".into(), }); @@ -3117,7 +3129,7 @@ mod tests { #[tracing_test::traced_test] #[test] pub fn test_record_path_rename() { - run_test(JournalEntry::PathRename { + run_test(JournalEntry::PathRenameV1 { old_fd: 32451345, old_path: "/asdfasdfas/asdfasdf".into(), new_fd: 23452345, @@ -3128,7 +3140,7 @@ mod tests { #[tracing_test::traced_test] #[test] pub fn test_record_change_directory() { - run_test(JournalEntry::ChangeDirectory { + run_test(JournalEntry::ChangeDirectoryV1 { path: "/etc".to_string().into(), }); } @@ -3136,13 +3148,13 @@ mod tests { #[tracing_test::traced_test] #[test] pub fn test_record_epoll_create() { - run_test(JournalEntry::EpollCreate { fd: 45384752 }); + run_test(JournalEntry::EpollCreateV1 { fd: 45384752 }); } #[tracing_test::traced_test] #[test] pub fn test_record_epoll_ctl() { - run_test(JournalEntry::EpollCtl { + run_test(JournalEntry::EpollCtlV1 { epfd: 34523455, op: wasi::EpollCtl::Unknown, fd: 23452345, @@ -3159,7 +3171,7 @@ mod tests { #[tracing_test::traced_test] #[test] pub fn test_record_tty_set() { - run_test(JournalEntry::TtySet { + run_test(JournalEntry::TtySetV1 { tty: wasi::Tty { cols: 1234, rows: 6754, @@ -3178,7 +3190,7 @@ mod tests { #[tracing_test::traced_test] #[test] pub fn test_record_create_pipe() { - run_test(JournalEntry::CreatePipe { + run_test(JournalEntry::CreatePipeV1 { fd1: 3452345, fd2: 2345163, }); @@ -3187,7 +3199,7 @@ mod tests { #[tracing_test::traced_test] #[test] pub fn test_record_create_event() { - run_test(JournalEntry::CreateEvent { + run_test(JournalEntry::CreateEventV1 { initial_val: 13451345, flags: 2343, fd: 5836544, @@ -3197,7 +3209,7 @@ mod tests { #[tracing_test::traced_test] #[test] pub fn test_record_port_add_addr() { - run_test(JournalEntry::PortAddAddr { + run_test(JournalEntry::PortAddAddrV1 { cidr: IpCidr { ip: Ipv4Addr::LOCALHOST.into(), prefix: 24, @@ -3208,7 +3220,7 @@ mod tests { #[tracing_test::traced_test] #[test] pub fn test_record_del_addr() { - run_test(JournalEntry::PortDelAddr { + run_test(JournalEntry::PortDelAddrV1 { addr: Ipv6Addr::LOCALHOST.into(), }); } @@ -3216,13 +3228,13 @@ mod tests { #[tracing_test::traced_test] #[test] pub fn test_record_addr_clear() { - run_test(JournalEntry::PortAddrClear); + run_test(JournalEntry::PortAddrClearV1); } #[tracing_test::traced_test] #[test] pub fn test_record_port_bridge() { - run_test(JournalEntry::PortBridge { + run_test(JournalEntry::PortBridgeV1 { network: "mynetwork".into(), token: format!("blh blah").into(), security: StreamSecurity::ClassicEncryption, @@ -3232,19 +3244,19 @@ mod tests { #[tracing_test::traced_test] #[test] pub fn test_record_unbridge() { - run_test(JournalEntry::PortUnbridge); + run_test(JournalEntry::PortUnbridgeV1); } #[tracing_test::traced_test] #[test] pub fn test_record_dhcp_acquire() { - run_test(JournalEntry::PortDhcpAcquire); + run_test(JournalEntry::PortDhcpAcquireV1); } #[tracing_test::traced_test] #[test] pub fn test_record_gateway_set() { - run_test(JournalEntry::PortGatewaySet { + run_test(JournalEntry::PortGatewaySetV1 { ip: Ipv4Addr::new(12, 34, 136, 220).into(), }); } @@ -3252,7 +3264,7 @@ mod tests { #[tracing_test::traced_test] #[test] pub fn test_record_route_add() { - run_test(JournalEntry::PortRouteAdd { + run_test(JournalEntry::PortRouteAddV1 { cidr: IpCidr { ip: Ipv4Addr::LOCALHOST.into(), prefix: 24, @@ -3266,13 +3278,13 @@ mod tests { #[tracing_test::traced_test] #[test] pub fn test_record_route_clear() { - run_test(JournalEntry::PortRouteClear); + run_test(JournalEntry::PortRouteClearV1); } #[tracing_test::traced_test] #[test] pub fn test_record_route_del() { - run_test(JournalEntry::PortRouteDel { + run_test(JournalEntry::PortRouteDelV1 { ip: Ipv4Addr::BROADCAST.into(), }); } @@ -3280,7 +3292,7 @@ mod tests { #[tracing_test::traced_test] #[test] pub fn test_record_socket_open() { - run_test(JournalEntry::SocketOpen { + run_test(JournalEntry::SocketOpenV1 { af: wasi::Addressfamily::Inet6, ty: wasi::Socktype::Stream, pt: wasi::SockProto::Tcp, @@ -3291,7 +3303,7 @@ mod tests { #[tracing_test::traced_test] #[test] pub fn test_record_socket_listen() { - run_test(JournalEntry::SocketListen { + run_test(JournalEntry::SocketListenV1 { fd: 12341234, backlog: 123, }); @@ -3300,7 +3312,7 @@ mod tests { #[tracing_test::traced_test] #[test] pub fn test_record_socket_bind() { - run_test(JournalEntry::SocketBind { + run_test(JournalEntry::SocketBindV1 { fd: 2341234, addr: SocketAddr::new(Ipv6Addr::UNSPECIFIED.into(), 1234), }); @@ -3309,7 +3321,7 @@ mod tests { #[tracing_test::traced_test] #[test] pub fn test_record_socket_connected() { - run_test(JournalEntry::SocketConnected { + run_test(JournalEntry::SocketConnectedV1 { fd: 12341, addr: SocketAddr::new(Ipv6Addr::UNSPECIFIED.into(), 1234), }); @@ -3318,19 +3330,19 @@ mod tests { #[tracing_test::traced_test] #[test] pub fn test_record_socket_accepted() { - run_test(JournalEntry::SocketAccepted { + run_test(JournalEntry::SocketAcceptedV1 { listen_fd: 21234, fd: 1, peer_addr: SocketAddr::new(Ipv6Addr::UNSPECIFIED.into(), 3452), fd_flags: wasi::Fdflags::all(), - nonblocking: true, + non_blocking: true, }); } #[tracing_test::traced_test] #[test] pub fn test_record_socket_join_ipv4_multicast() { - run_test(JournalEntry::SocketJoinIpv4Multicast { + run_test(JournalEntry::SocketJoinIpv4MulticastV1 { fd: 12, multiaddr: Ipv4Addr::new(123, 123, 123, 123).into(), iface: Ipv4Addr::new(128, 0, 0, 1).into(), @@ -3340,9 +3352,9 @@ mod tests { #[tracing_test::traced_test] #[test] pub fn test_record_socket_join_ipv6_multicast() { - run_test(JournalEntry::SocketJoinIpv6Multicast { + run_test(JournalEntry::SocketJoinIpv6MulticastV1 { fd: 12, - multiaddr: Ipv6Addr::new(123, 123, 123, 123, 1234, 12663, 31, 1324).into(), + multi_addr: Ipv6Addr::new(123, 123, 123, 123, 1234, 12663, 31, 1324).into(), iface: 23541, }); } @@ -3350,9 +3362,9 @@ mod tests { #[tracing_test::traced_test] #[test] pub fn test_record_socket_leave_ipv4_multicast() { - run_test(JournalEntry::SocketLeaveIpv4Multicast { + run_test(JournalEntry::SocketLeaveIpv4MulticastV1 { fd: 12, - multiaddr: Ipv4Addr::new(123, 123, 123, 123).into(), + multi_addr: Ipv4Addr::new(123, 123, 123, 123).into(), iface: Ipv4Addr::new(128, 0, 0, 1).into(), }); } @@ -3360,9 +3372,9 @@ mod tests { #[tracing_test::traced_test] #[test] pub fn test_record_socket_leave_ipv6_multicast() { - run_test(JournalEntry::SocketLeaveIpv6Multicast { + run_test(JournalEntry::SocketLeaveIpv6MulticastV1 { fd: 12, - multiaddr: Ipv6Addr::new(123, 123, 123, 123, 1234, 12663, 31, 1324).into(), + multi_addr: Ipv6Addr::new(123, 123, 123, 123, 1234, 12663, 31, 1324).into(), iface: 23541, }); } @@ -3370,7 +3382,7 @@ mod tests { #[tracing_test::traced_test] #[test] pub fn test_record_socket_send_file() { - run_test(JournalEntry::SocketSendFile { + run_test(JournalEntry::SocketSendFileV1 { socket_fd: 22234, file_fd: 989, offset: 124, @@ -3381,7 +3393,7 @@ mod tests { #[tracing_test::traced_test] #[test] pub fn test_record_socket_send_to() { - run_test(JournalEntry::SocketSendTo { + run_test(JournalEntry::SocketSendToV1 { fd: 123, data: [98u8; 102400].to_vec().into(), flags: 1234, @@ -3393,7 +3405,7 @@ mod tests { #[tracing_test::traced_test] #[test] pub fn test_record_socket_send() { - run_test(JournalEntry::SocketSend { + run_test(JournalEntry::SocketSendV1 { fd: 123, data: [98u8; 102400].to_vec().into(), flags: 1234, @@ -3404,7 +3416,7 @@ mod tests { #[tracing_test::traced_test] #[test] pub fn test_record_socket_set_opt_flag() { - run_test(JournalEntry::SocketSetOptFlag { + run_test(JournalEntry::SocketSetOptFlagV1 { fd: 0, opt: wasi::Sockoption::Linger, flag: true, @@ -3414,7 +3426,7 @@ mod tests { #[tracing_test::traced_test] #[test] pub fn test_record_socket_set_opt_size() { - run_test(JournalEntry::SocketSetOptSize { + run_test(JournalEntry::SocketSetOptSizeV1 { fd: 15, opt: wasi::Sockoption::Linger, size: 234234, @@ -3424,7 +3436,7 @@ mod tests { #[tracing_test::traced_test] #[test] pub fn test_record_socket_set_opt_time() { - run_test(JournalEntry::SocketSetOptTime { + run_test(JournalEntry::SocketSetOptTimeV1 { fd: 0, ty: SocketOptTimeType::AcceptTimeout, time: Some(Duration::ZERO), @@ -3434,7 +3446,7 @@ mod tests { #[tracing_test::traced_test] #[test] pub fn test_record_socket_shutdown() { - run_test(JournalEntry::SocketShutdown { + run_test(JournalEntry::SocketShutdownV1 { fd: 123, how: SocketShutdownHow::Both, }); @@ -3443,7 +3455,7 @@ mod tests { #[tracing_test::traced_test] #[test] pub fn test_record_snapshot() { - run_test(JournalEntry::Snapshot { + run_test(JournalEntry::SnapshotV1 { when: SystemTime::now(), trigger: SnapshotTrigger::Idle, }); diff --git a/lib/wasix/src/journal/concrete/compacting.rs b/lib/wasix/src/journal/concrete/compacting.rs index b29a617d65e..2c663f6af15 100644 --- a/lib/wasix/src/journal/concrete/compacting.rs +++ b/lib/wasix/src/journal/concrete/compacting.rs @@ -245,19 +245,19 @@ impl WritableJournal for CompactingJournalTx { } match &entry { - JournalEntry::UpdateMemoryRegion { region, .. } => { + JournalEntry::UpdateMemoryRegionV1 { region, .. } => { state.memory_map.insert(region.clone().into(), event_index); } - JournalEntry::SetThread { id, .. } => { + JournalEntry::SetThreadV1 { id, .. } => { state.thread_map.insert(*id, event_index); } - JournalEntry::CloseThread { id, .. } => { + JournalEntry::CloseThreadV1 { id, .. } => { state.thread_map.remove(id); } - JournalEntry::Snapshot { .. } => { + JournalEntry::SnapshotV1 { .. } => { state.snapshots.push(event_index); } - JournalEntry::ProcessExit { .. } => { + JournalEntry::ProcessExitV1 { .. } => { state.thread_map.clear(); state.memory_map.clear(); for (_, lookup) in state.suspect_descriptors.clone() { @@ -271,10 +271,10 @@ impl WritableJournal for CompactingJournalTx { state.whitelist.insert(event_index); state.snapshots.clear(); } - JournalEntry::TtySet { .. } => { + JournalEntry::TtySetV1 { .. } => { state.tty.replace(event_index); } - JournalEntry::OpenFileDescriptor { fd, .. } => { + JournalEntry::OpenFileDescriptorV1 { fd, .. } => { // All file descriptors are opened in a suspect state which // means if they are closed without modifying the file system // then the events will be ignored. @@ -285,8 +285,8 @@ impl WritableJournal for CompactingJournalTx { .insert(*fd, DescriptorLookup(lookup)); } // We keep non-mutable events for file descriptors that are suspect - JournalEntry::FileDescriptorSeek { fd, .. } - | JournalEntry::CloseFileDescriptor { fd } => { + JournalEntry::FileDescriptorSeekV1 { fd, .. } + | JournalEntry::CloseFileDescriptorV1 { fd } => { // Get the lookup let lookup = state .suspect_descriptors @@ -307,11 +307,11 @@ impl WritableJournal for CompactingJournalTx { } // Things that modify a file descriptor mean that it is // no longer suspect and thus it needs to be kept - JournalEntry::FileDescriptorAdvise { fd, .. } - | JournalEntry::FileDescriptorAllocate { fd, .. } - | JournalEntry::FileDescriptorSetFlags { fd, .. } - | JournalEntry::FileDescriptorSetTimes { fd, .. } - | JournalEntry::FileDescriptorWrite { fd, .. } => { + JournalEntry::FileDescriptorAdviseV1 { fd, .. } + | JournalEntry::FileDescriptorAllocateV1 { fd, .. } + | JournalEntry::FileDescriptorSetFlagsV1 { fd, .. } + | JournalEntry::FileDescriptorSetTimesV1 { fd, .. } + | JournalEntry::FileDescriptorWriteV1 { fd, .. } => { // Its no longer suspect if let Some(lookup) = state.suspect_descriptors.remove(fd) { state.keep_descriptors.insert(*fd, lookup); @@ -331,7 +331,7 @@ impl WritableJournal for CompactingJournalTx { .descriptors .entry(lookup) .or_insert_with(|| Default::default()); - if let JournalEntry::FileDescriptorWrite { offset, data, .. } = &entry { + if let JournalEntry::FileDescriptorWriteV1 { offset, data, .. } = &entry { state.write_map.insert( MemoryRange { start: *offset, @@ -347,7 +347,7 @@ impl WritableJournal for CompactingJournalTx { } } // Duplicating the file descriptor - JournalEntry::DuplicateFileDescriptor { + JournalEntry::DuplicateFileDescriptorV1 { original_fd, copied_fd, } => { @@ -362,7 +362,7 @@ impl WritableJournal for CompactingJournalTx { } } // Renumbered file descriptors will retain their suspect status - JournalEntry::RenumberFileDescriptor { old_fd, new_fd } => { + JournalEntry::RenumberFileDescriptorV1 { old_fd, new_fd } => { if let Some(lookup) = state.suspect_descriptors.remove(old_fd) { state.suspect_descriptors.insert(*new_fd, lookup); } else if let Some(lookup) = state.keep_descriptors.remove(old_fd) { @@ -471,16 +471,16 @@ mod tests { pub fn test_compact_purge_duplicate_memory_writes() { run_test( vec![ - JournalEntry::UpdateMemoryRegion { + JournalEntry::UpdateMemoryRegionV1 { region: 0..16, data: [11u8; 16].to_vec().into(), }, - JournalEntry::UpdateMemoryRegion { + JournalEntry::UpdateMemoryRegionV1 { region: 0..16, data: [22u8; 16].to_vec().into(), }, ], - vec![JournalEntry::UpdateMemoryRegion { + vec![JournalEntry::UpdateMemoryRegionV1 { region: 0..16, data: [22u8; 16].to_vec().into(), }], @@ -493,21 +493,21 @@ mod tests { pub fn test_compact_keep_overlapping_memory() { run_test( vec![ - JournalEntry::UpdateMemoryRegion { + JournalEntry::UpdateMemoryRegionV1 { region: 0..16, data: [11u8; 16].to_vec().into(), }, - JournalEntry::UpdateMemoryRegion { + JournalEntry::UpdateMemoryRegionV1 { region: 20..36, data: [22u8; 16].to_vec().into(), }, ], vec![ - JournalEntry::UpdateMemoryRegion { + JournalEntry::UpdateMemoryRegionV1 { region: 0..16, data: [11u8; 16].to_vec().into(), }, - JournalEntry::UpdateMemoryRegion { + JournalEntry::UpdateMemoryRegionV1 { region: 20..36, data: [22u8; 16].to_vec().into(), }, @@ -521,21 +521,21 @@ mod tests { pub fn test_compact_keep_adjacent_memory_writes() { run_test( vec![ - JournalEntry::UpdateMemoryRegion { + JournalEntry::UpdateMemoryRegionV1 { region: 0..16, data: [11u8; 16].to_vec().into(), }, - JournalEntry::UpdateMemoryRegion { + JournalEntry::UpdateMemoryRegionV1 { region: 16..32, data: [22u8; 16].to_vec().into(), }, ], vec![ - JournalEntry::UpdateMemoryRegion { + JournalEntry::UpdateMemoryRegionV1 { region: 0..16, data: [11u8; 16].to_vec().into(), }, - JournalEntry::UpdateMemoryRegion { + JournalEntry::UpdateMemoryRegionV1 { region: 16..32, data: [22u8; 16].to_vec().into(), }, @@ -549,16 +549,16 @@ mod tests { pub fn test_compact_purge_identical_memory_writes() { run_test( vec![ - JournalEntry::UpdateMemoryRegion { + JournalEntry::UpdateMemoryRegionV1 { region: 0..16, data: [11u8; 16].to_vec().into(), }, - JournalEntry::UpdateMemoryRegion { + JournalEntry::UpdateMemoryRegionV1 { region: 0..16, data: [11u8; 16].to_vec().into(), }, ], - vec![JournalEntry::UpdateMemoryRegion { + vec![JournalEntry::UpdateMemoryRegionV1 { region: 0..16, data: [11u8; 16].to_vec().into(), }], @@ -571,41 +571,41 @@ mod tests { pub fn test_compact_thread_stacks() { run_test( vec![ - JournalEntry::SetThread { + JournalEntry::SetThreadV1 { id: 4321.into(), call_stack: [44u8; 87].to_vec().into(), memory_stack: [55u8; 34].to_vec().into(), store_data: [66u8; 70].to_vec().into(), is_64bit: true, }, - JournalEntry::SetThread { + JournalEntry::SetThreadV1 { id: 1234.into(), call_stack: [11u8; 124].to_vec().into(), memory_stack: [22u8; 51].to_vec().into(), store_data: [33u8; 87].to_vec().into(), is_64bit: true, }, - JournalEntry::SetThread { + JournalEntry::SetThreadV1 { id: 65.into(), call_stack: [77u8; 34].to_vec().into(), memory_stack: [88u8; 51].to_vec().into(), store_data: [99u8; 12].to_vec().into(), is_64bit: true, }, - JournalEntry::CloseThread { + JournalEntry::CloseThreadV1 { id: 1234.into(), exit_code: None, }, ], vec![ - JournalEntry::SetThread { + JournalEntry::SetThreadV1 { id: 4321.into(), call_stack: [44u8; 87].to_vec().into(), memory_stack: [55u8; 34].to_vec().into(), store_data: [66u8; 70].to_vec().into(), is_64bit: true, }, - JournalEntry::SetThread { + JournalEntry::SetThreadV1 { id: 65.into(), call_stack: [77u8; 34].to_vec().into(), memory_stack: [88u8; 51].to_vec().into(), @@ -622,22 +622,22 @@ mod tests { pub fn test_compact_processed_exited() { run_test( vec![ - JournalEntry::UpdateMemoryRegion { + JournalEntry::UpdateMemoryRegionV1 { region: 0..16, data: [11u8; 16].to_vec().into(), }, - JournalEntry::SetThread { + JournalEntry::SetThreadV1 { id: 4321.into(), call_stack: [44u8; 87].to_vec().into(), memory_stack: [55u8; 34].to_vec().into(), store_data: [66u8; 70].to_vec().into(), is_64bit: true, }, - JournalEntry::Snapshot { + JournalEntry::SnapshotV1 { when: SystemTime::now(), trigger: SnapshotTrigger::FirstListen, }, - JournalEntry::OpenFileDescriptor { + JournalEntry::OpenFileDescriptorV1 { fd: 1234, dirfd: 3452345, dirflags: 0, @@ -647,9 +647,9 @@ mod tests { fs_rights_inheriting: wasi::Rights::all(), fs_flags: wasi::Fdflags::all(), }, - JournalEntry::ProcessExit { exit_code: None }, + JournalEntry::ProcessExitV1 { exit_code: None }, ], - vec![JournalEntry::ProcessExit { exit_code: None }], + vec![JournalEntry::ProcessExitV1 { exit_code: None }], ) .unwrap() } @@ -659,7 +659,7 @@ mod tests { pub fn test_compact_file_system_partial_write_survives() { run_test( vec![ - JournalEntry::OpenFileDescriptor { + JournalEntry::OpenFileDescriptorV1 { fd: 1234, dirfd: 3452345, dirflags: 0, @@ -669,7 +669,7 @@ mod tests { fs_rights_inheriting: wasi::Rights::all(), fs_flags: wasi::Fdflags::all(), }, - JournalEntry::FileDescriptorWrite { + JournalEntry::FileDescriptorWriteV1 { fd: 1234, offset: 1234, data: [1u8; 16].to_vec().into(), @@ -677,7 +677,7 @@ mod tests { }, ], vec![ - JournalEntry::OpenFileDescriptor { + JournalEntry::OpenFileDescriptorV1 { fd: 1234, dirfd: 3452345, dirflags: 0, @@ -687,7 +687,7 @@ mod tests { fs_rights_inheriting: wasi::Rights::all(), fs_flags: wasi::Fdflags::all(), }, - JournalEntry::FileDescriptorWrite { + JournalEntry::FileDescriptorWriteV1 { fd: 1234, offset: 1234, data: [1u8; 16].to_vec().into(), @@ -703,7 +703,7 @@ mod tests { pub fn test_compact_file_system_write_survives_close() { run_test( vec![ - JournalEntry::OpenFileDescriptor { + JournalEntry::OpenFileDescriptorV1 { fd: 1234, dirfd: 3452345, dirflags: 0, @@ -713,16 +713,16 @@ mod tests { fs_rights_inheriting: wasi::Rights::all(), fs_flags: wasi::Fdflags::all(), }, - JournalEntry::FileDescriptorWrite { + JournalEntry::FileDescriptorWriteV1 { fd: 1234, offset: 1234, data: [1u8; 16].to_vec().into(), is_64bit: true, }, - JournalEntry::CloseFileDescriptor { fd: 1234 }, + JournalEntry::CloseFileDescriptorV1 { fd: 1234 }, ], vec![ - JournalEntry::OpenFileDescriptor { + JournalEntry::OpenFileDescriptorV1 { fd: 1234, dirfd: 3452345, dirflags: 0, @@ -732,13 +732,13 @@ mod tests { fs_rights_inheriting: wasi::Rights::all(), fs_flags: wasi::Fdflags::all(), }, - JournalEntry::FileDescriptorWrite { + JournalEntry::FileDescriptorWriteV1 { fd: 1234, offset: 1234, data: [1u8; 16].to_vec().into(), is_64bit: true, }, - JournalEntry::CloseFileDescriptor { fd: 1234 }, + JournalEntry::CloseFileDescriptorV1 { fd: 1234 }, ], ) .unwrap() @@ -749,7 +749,7 @@ mod tests { pub fn test_compact_file_system_write_survives_exit() { run_test( vec![ - JournalEntry::OpenFileDescriptor { + JournalEntry::OpenFileDescriptorV1 { fd: 1234, dirfd: 3452345, dirflags: 0, @@ -759,16 +759,16 @@ mod tests { fs_rights_inheriting: wasi::Rights::all(), fs_flags: wasi::Fdflags::all(), }, - JournalEntry::FileDescriptorWrite { + JournalEntry::FileDescriptorWriteV1 { fd: 1234, offset: 1234, data: [1u8; 16].to_vec().into(), is_64bit: true, }, - JournalEntry::ProcessExit { exit_code: None }, + JournalEntry::ProcessExitV1 { exit_code: None }, ], vec![ - JournalEntry::OpenFileDescriptor { + JournalEntry::OpenFileDescriptorV1 { fd: 1234, dirfd: 3452345, dirflags: 0, @@ -778,13 +778,13 @@ mod tests { fs_rights_inheriting: wasi::Rights::all(), fs_flags: wasi::Fdflags::all(), }, - JournalEntry::FileDescriptorWrite { + JournalEntry::FileDescriptorWriteV1 { fd: 1234, offset: 1234, data: [1u8; 16].to_vec().into(), is_64bit: true, }, - JournalEntry::ProcessExit { exit_code: None }, + JournalEntry::ProcessExitV1 { exit_code: None }, ], ) .unwrap() @@ -795,7 +795,7 @@ mod tests { pub fn test_compact_file_system_read_is_ignored() { run_test( vec![ - JournalEntry::OpenFileDescriptor { + JournalEntry::OpenFileDescriptorV1 { fd: 1234, dirfd: 3452345, dirflags: 0, @@ -805,12 +805,12 @@ mod tests { fs_rights_inheriting: wasi::Rights::all(), fs_flags: wasi::Fdflags::all(), }, - JournalEntry::FileDescriptorSeek { + JournalEntry::FileDescriptorSeekV1 { fd: 1234, offset: 1234, whence: wasi::Whence::End, }, - JournalEntry::CloseFileDescriptor { fd: 1234 }, + JournalEntry::CloseFileDescriptorV1 { fd: 1234 }, ], Vec::new(), ) @@ -822,7 +822,7 @@ mod tests { pub fn test_compact_file_system_ignore_double_writes() { run_test( vec![ - JournalEntry::OpenFileDescriptor { + JournalEntry::OpenFileDescriptorV1 { fd: 1234, dirfd: 3452345, dirflags: 0, @@ -832,22 +832,22 @@ mod tests { fs_rights_inheriting: wasi::Rights::all(), fs_flags: wasi::Fdflags::all(), }, - JournalEntry::FileDescriptorWrite { + JournalEntry::FileDescriptorWriteV1 { fd: 1234, offset: 1234, data: [1u8; 16].to_vec().into(), is_64bit: true, }, - JournalEntry::FileDescriptorWrite { + JournalEntry::FileDescriptorWriteV1 { fd: 1234, offset: 1234, data: [5u8; 16].to_vec().into(), is_64bit: true, }, - JournalEntry::CloseFileDescriptor { fd: 1234 }, + JournalEntry::CloseFileDescriptorV1 { fd: 1234 }, ], vec![ - JournalEntry::OpenFileDescriptor { + JournalEntry::OpenFileDescriptorV1 { fd: 1234, dirfd: 3452345, dirflags: 0, @@ -857,13 +857,13 @@ mod tests { fs_rights_inheriting: wasi::Rights::all(), fs_flags: wasi::Fdflags::all(), }, - JournalEntry::FileDescriptorWrite { + JournalEntry::FileDescriptorWriteV1 { fd: 1234, offset: 1234, data: [5u8; 16].to_vec().into(), is_64bit: true, }, - JournalEntry::CloseFileDescriptor { fd: 1234 }, + JournalEntry::CloseFileDescriptorV1 { fd: 1234 }, ], ) .unwrap() @@ -873,11 +873,11 @@ mod tests { #[test] pub fn test_compact_file_system_create_directory() { run_test( - vec![JournalEntry::CreateDirectory { + vec![JournalEntry::CreateDirectoryV1 { fd: 1234, path: "/blah".into(), }], - vec![JournalEntry::CreateDirectory { + vec![JournalEntry::CreateDirectoryV1 { fd: 1234, path: "/blah".into(), }], @@ -890,7 +890,7 @@ mod tests { pub fn test_compact_duplicate_tty() { run_test( vec![ - JournalEntry::TtySet { + JournalEntry::TtySetV1 { tty: Tty { cols: 123, rows: 65, @@ -904,7 +904,7 @@ mod tests { }, line_feeds: true, }, - JournalEntry::TtySet { + JournalEntry::TtySetV1 { tty: Tty { cols: 12, rows: 65, @@ -919,7 +919,7 @@ mod tests { line_feeds: true, }, ], - vec![JournalEntry::TtySet { + vec![JournalEntry::TtySetV1 { tty: Tty { cols: 12, rows: 65, diff --git a/lib/wasix/src/journal/concrete/filter.rs b/lib/wasix/src/journal/concrete/filter.rs index 59decac6fa4..7b936afeb49 100644 --- a/lib/wasix/src/journal/concrete/filter.rs +++ b/lib/wasix/src/journal/concrete/filter.rs @@ -116,88 +116,88 @@ impl WritableJournal for FilteredJournalTx { } let evt = match entry { - JournalEntry::SetClockTime { .. } - | JournalEntry::InitModule { .. } - | JournalEntry::ProcessExit { .. } - | JournalEntry::EpollCreate { .. } - | JournalEntry::EpollCtl { .. } - | JournalEntry::TtySet { .. } => { + JournalEntry::SetClockTimeV1 { .. } + | JournalEntry::InitModuleV1 { .. } + | JournalEntry::ProcessExitV1 { .. } + | JournalEntry::EpollCreateV1 { .. } + | JournalEntry::EpollCtlV1 { .. } + | JournalEntry::TtySetV1 { .. } => { if self.filter_core { return Ok(0); } entry } - JournalEntry::SetThread { .. } | JournalEntry::CloseThread { .. } => { + JournalEntry::SetThreadV1 { .. } | JournalEntry::CloseThreadV1 { .. } => { if self.filter_threads { return Ok(0); } entry } - JournalEntry::UpdateMemoryRegion { .. } => { + JournalEntry::UpdateMemoryRegionV1 { .. } => { if self.filter_memory { return Ok(0); } entry } - JournalEntry::FileDescriptorSeek { .. } - | JournalEntry::FileDescriptorWrite { .. } - | JournalEntry::OpenFileDescriptor { .. } - | JournalEntry::CloseFileDescriptor { .. } - | JournalEntry::RemoveDirectory { .. } - | JournalEntry::UnlinkFile { .. } - | JournalEntry::PathRename { .. } - | JournalEntry::RenumberFileDescriptor { .. } - | JournalEntry::DuplicateFileDescriptor { .. } - | JournalEntry::CreateDirectory { .. } - | JournalEntry::PathSetTimes { .. } - | JournalEntry::FileDescriptorSetFlags { .. } - | JournalEntry::FileDescriptorAdvise { .. } - | JournalEntry::FileDescriptorAllocate { .. } - | JournalEntry::FileDescriptorSetRights { .. } - | JournalEntry::FileDescriptorSetTimes { .. } - | JournalEntry::FileDescriptorSetSize { .. } - | JournalEntry::CreateHardLink { .. } - | JournalEntry::CreateSymbolicLink { .. } - | JournalEntry::ChangeDirectory { .. } - | JournalEntry::CreatePipe { .. } - | JournalEntry::CreateEvent { .. } => { + JournalEntry::FileDescriptorSeekV1 { .. } + | JournalEntry::FileDescriptorWriteV1 { .. } + | JournalEntry::OpenFileDescriptorV1 { .. } + | JournalEntry::CloseFileDescriptorV1 { .. } + | JournalEntry::RemoveDirectoryV1 { .. } + | JournalEntry::UnlinkFileV1 { .. } + | JournalEntry::PathRenameV1 { .. } + | JournalEntry::RenumberFileDescriptorV1 { .. } + | JournalEntry::DuplicateFileDescriptorV1 { .. } + | JournalEntry::CreateDirectoryV1 { .. } + | JournalEntry::PathSetTimesV1 { .. } + | JournalEntry::FileDescriptorSetFlagsV1 { .. } + | JournalEntry::FileDescriptorAdviseV1 { .. } + | JournalEntry::FileDescriptorAllocateV1 { .. } + | JournalEntry::FileDescriptorSetRightsV1 { .. } + | JournalEntry::FileDescriptorSetTimesV1 { .. } + | JournalEntry::FileDescriptorSetSizeV1 { .. } + | JournalEntry::CreateHardLinkV1 { .. } + | JournalEntry::CreateSymbolicLinkV1 { .. } + | JournalEntry::ChangeDirectoryV1 { .. } + | JournalEntry::CreatePipeV1 { .. } + | JournalEntry::CreateEventV1 { .. } => { if self.filter_fs { return Ok(0); } entry } - JournalEntry::Snapshot { .. } => { + JournalEntry::SnapshotV1 { .. } => { if self.filter_snapshots { return Ok(0); } entry } - JournalEntry::PortAddAddr { .. } - | JournalEntry::PortDelAddr { .. } - | JournalEntry::PortAddrClear - | JournalEntry::PortBridge { .. } - | JournalEntry::PortUnbridge - | JournalEntry::PortDhcpAcquire - | JournalEntry::PortGatewaySet { .. } - | JournalEntry::PortRouteAdd { .. } - | JournalEntry::PortRouteClear - | JournalEntry::PortRouteDel { .. } - | JournalEntry::SocketOpen { .. } - | JournalEntry::SocketListen { .. } - | JournalEntry::SocketBind { .. } - | JournalEntry::SocketConnected { .. } - | JournalEntry::SocketAccepted { .. } - | JournalEntry::SocketJoinIpv4Multicast { .. } - | JournalEntry::SocketJoinIpv6Multicast { .. } - | JournalEntry::SocketLeaveIpv4Multicast { .. } - | JournalEntry::SocketLeaveIpv6Multicast { .. } - | JournalEntry::SocketSendFile { .. } - | JournalEntry::SocketSendTo { .. } - | JournalEntry::SocketSend { .. } - | JournalEntry::SocketSetOptFlag { .. } - | JournalEntry::SocketSetOptSize { .. } - | JournalEntry::SocketSetOptTime { .. } - | JournalEntry::SocketShutdown { .. } => { + JournalEntry::PortAddAddrV1 { .. } + | JournalEntry::PortDelAddrV1 { .. } + | JournalEntry::PortAddrClearV1 + | JournalEntry::PortBridgeV1 { .. } + | JournalEntry::PortUnbridgeV1 + | JournalEntry::PortDhcpAcquireV1 + | JournalEntry::PortGatewaySetV1 { .. } + | JournalEntry::PortRouteAddV1 { .. } + | JournalEntry::PortRouteClearV1 + | JournalEntry::PortRouteDelV1 { .. } + | JournalEntry::SocketOpenV1 { .. } + | JournalEntry::SocketListenV1 { .. } + | JournalEntry::SocketBindV1 { .. } + | JournalEntry::SocketConnectedV1 { .. } + | JournalEntry::SocketAcceptedV1 { .. } + | JournalEntry::SocketJoinIpv4MulticastV1 { .. } + | JournalEntry::SocketJoinIpv6MulticastV1 { .. } + | JournalEntry::SocketLeaveIpv4MulticastV1 { .. } + | JournalEntry::SocketLeaveIpv6MulticastV1 { .. } + | JournalEntry::SocketSendFileV1 { .. } + | JournalEntry::SocketSendToV1 { .. } + | JournalEntry::SocketSendV1 { .. } + | JournalEntry::SocketSetOptFlagV1 { .. } + | JournalEntry::SocketSetOptSizeV1 { .. } + | JournalEntry::SocketSetOptTimeV1 { .. } + | JournalEntry::SocketShutdownV1 { .. } => { if self.filter_net { return Ok(0); } diff --git a/lib/wasix/src/journal/concrete/log_file.rs b/lib/wasix/src/journal/concrete/log_file.rs index dbbeaca25e9..fdffdb304a8 100644 --- a/lib/wasix/src/journal/concrete/log_file.rs +++ b/lib/wasix/src/journal/concrete/log_file.rs @@ -221,7 +221,7 @@ impl ReadableJournal for LogFileJournalRx { } }; - let record = unsafe { record_type.deserialize_archive(entry) }; + let record = unsafe { record_type.deserialize_archive(entry)? }; return Ok(Some(record)); } } @@ -267,10 +267,10 @@ mod tests { // Write some events to it let journal = LogFileJournal::from_file(file.as_file().try_clone().unwrap()).unwrap(); journal - .write(JournalEntry::CreatePipe { fd1: 1, fd2: 2 }) + .write(JournalEntry::CreatePipeV1 { fd1: 1, fd2: 2 }) .unwrap(); journal - .write(JournalEntry::SetThread { + .write(JournalEntry::SetThreadV1 { id: 1.into(), call_stack: vec![11; 116].into(), memory_stack: vec![22; 16].into(), @@ -278,7 +278,7 @@ mod tests { is_64bit: false, }) .unwrap(); - journal.write(JournalEntry::PortAddrClear).unwrap(); + journal.write(JournalEntry::PortAddrClearV1).unwrap(); drop(journal); // Read the events and validate @@ -289,10 +289,10 @@ mod tests { let event4 = journal.read().unwrap(); // Check the events - assert_eq!(event1, Some(JournalEntry::CreatePipe { fd1: 1, fd2: 2 })); + assert_eq!(event1, Some(JournalEntry::CreatePipeV1 { fd1: 1, fd2: 2 })); assert_eq!( event2, - Some(JournalEntry::SetThread { + Some(JournalEntry::SetThreadV1 { id: 1.into(), call_stack: vec![11; 116].into(), memory_stack: vec![22; 16].into(), @@ -300,12 +300,12 @@ mod tests { is_64bit: false, }) ); - assert_eq!(event3, Some(JournalEntry::PortAddrClear)); + assert_eq!(event3, Some(JournalEntry::PortAddrClearV1)); assert_eq!(event4, None); // Now write another event journal - .write(JournalEntry::SocketSend { + .write(JournalEntry::SocketSendV1 { fd: 1234, data: [12; 1024].to_vec().into(), flags: 123, @@ -321,7 +321,7 @@ mod tests { // Before we read it, we will throw in another event journal - .write(JournalEntry::CreatePipe { + .write(JournalEntry::CreatePipeV1 { fd1: 1234, fd2: 5432, }) @@ -332,10 +332,10 @@ mod tests { let event3 = journal.read().unwrap(); let event4 = journal.read().unwrap(); let event5 = journal.read().unwrap(); - assert_eq!(event1, Some(JournalEntry::CreatePipe { fd1: 1, fd2: 2 })); + assert_eq!(event1, Some(JournalEntry::CreatePipeV1 { fd1: 1, fd2: 2 })); assert_eq!( event2, - Some(JournalEntry::SetThread { + Some(JournalEntry::SetThreadV1 { id: 1.into(), call_stack: vec![11; 116].into(), memory_stack: vec![22; 16].into(), @@ -343,10 +343,10 @@ mod tests { is_64bit: false, }) ); - assert_eq!(event3, Some(JournalEntry::PortAddrClear)); + assert_eq!(event3, Some(JournalEntry::PortAddrClearV1)); assert_eq!( event4, - Some(JournalEntry::SocketSend { + Some(JournalEntry::SocketSendV1 { fd: 1234, data: [12; 1024].to_vec().into(), flags: 123, @@ -372,10 +372,10 @@ mod tests { tracing::info!("event5 {:?}", event5); tracing::info!("event6 {:?}", event6); - assert_eq!(event1, Some(JournalEntry::CreatePipe { fd1: 1, fd2: 2 })); + assert_eq!(event1, Some(JournalEntry::CreatePipeV1 { fd1: 1, fd2: 2 })); assert_eq!( event2, - Some(JournalEntry::SetThread { + Some(JournalEntry::SetThreadV1 { id: 1.into(), call_stack: vec![11; 116].into(), memory_stack: vec![22; 16].into(), @@ -383,10 +383,10 @@ mod tests { is_64bit: false, }) ); - assert_eq!(event3, Some(JournalEntry::PortAddrClear)); + assert_eq!(event3, Some(JournalEntry::PortAddrClearV1)); assert_eq!( event4, - Some(JournalEntry::SocketSend { + Some(JournalEntry::SocketSendV1 { fd: 1234, data: [12; 1024].to_vec().into(), flags: 123, @@ -395,7 +395,7 @@ mod tests { ); assert_eq!( event5, - Some(JournalEntry::CreatePipe { + Some(JournalEntry::CreatePipeV1 { fd1: 1234, fd2: 5432, }) diff --git a/lib/wasix/src/journal/concrete/printing.rs b/lib/wasix/src/journal/concrete/printing.rs index 5a59a1f485d..93c363eafb5 100644 --- a/lib/wasix/src/journal/concrete/printing.rs +++ b/lib/wasix/src/journal/concrete/printing.rs @@ -61,20 +61,20 @@ impl Journal for PrintingJournal { impl<'a> fmt::Display for JournalEntry<'a> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { - JournalEntry::InitModule { wasm_hash } => { + JournalEntry::InitModuleV1 { wasm_hash } => { write!(f, "init-module (hash={:x?})", wasm_hash) } - JournalEntry::UpdateMemoryRegion { region, data } => write!( + JournalEntry::UpdateMemoryRegionV1 { region, data } => write!( f, "memory-update (start={}, end={}, data.len={})", region.start, region.end, data.len() ), - JournalEntry::ProcessExit { exit_code } => { + JournalEntry::ProcessExitV1 { exit_code } => { write!(f, "process-exit (code={:?})", exit_code) } - JournalEntry::SetThread { + JournalEntry::SetThreadV1 { id, call_stack, memory_stack, @@ -88,15 +88,15 @@ impl<'a> fmt::Display for JournalEntry<'a> { memory_stack.len(), store_data.len(), ), - JournalEntry::CloseThread { id, exit_code } => { + JournalEntry::CloseThreadV1 { id, exit_code } => { write!(f, "thread-close (id={}, code={:?})", id, exit_code) } - JournalEntry::FileDescriptorSeek { fd, offset, whence } => write!( + JournalEntry::FileDescriptorSeekV1 { fd, offset, whence } => write!( f, "fd-seek (fd={}, offset={}, whence={:?})", fd, offset, whence ), - JournalEntry::FileDescriptorWrite { + JournalEntry::FileDescriptorWriteV1 { fd, offset, data, .. } => write!( f, @@ -105,17 +105,17 @@ impl<'a> fmt::Display for JournalEntry<'a> { offset, data.len() ), - JournalEntry::SetClockTime { clock_id, time } => { + JournalEntry::SetClockTimeV1 { clock_id, time } => { write!(f, "set-clock-time (id={:?}, time={})", clock_id, time) } - JournalEntry::CloseFileDescriptor { fd } => write!(f, "fd-close (fd={})", fd), - JournalEntry::OpenFileDescriptor { fd, path, .. } => { + JournalEntry::CloseFileDescriptorV1 { fd } => write!(f, "fd-close (fd={})", fd), + JournalEntry::OpenFileDescriptorV1 { fd, path, .. } => { write!(f, "fd-open (path={}, fd={})", fd, path) } - JournalEntry::RenumberFileDescriptor { old_fd, new_fd } => { + JournalEntry::RenumberFileDescriptorV1 { old_fd, new_fd } => { write!(f, "fd-renumber (old={}, new={})", old_fd, new_fd) } - JournalEntry::DuplicateFileDescriptor { + JournalEntry::DuplicateFileDescriptorV1 { original_fd, copied_fd, } => write!( @@ -123,13 +123,13 @@ impl<'a> fmt::Display for JournalEntry<'a> { "fd-duplicate (original={}, copied={})", original_fd, copied_fd ), - JournalEntry::CreateDirectory { path, .. } => { + JournalEntry::CreateDirectoryV1 { path, .. } => { write!(f, "path-create-dir (path={})", path) } - JournalEntry::RemoveDirectory { path, .. } => { + JournalEntry::RemoveDirectoryV1 { path, .. } => { write!(f, "path-remove-dir (path={})", path) } - JournalEntry::PathSetTimes { + JournalEntry::PathSetTimesV1 { path, st_atim, st_mtim, @@ -139,7 +139,7 @@ impl<'a> fmt::Display for JournalEntry<'a> { "path-set-times (path={}, atime={}, mtime={}))", path, st_atim, st_mtim ), - JournalEntry::FileDescriptorSetTimes { + JournalEntry::FileDescriptorSetTimesV1 { fd, st_atim, st_mtim, @@ -149,10 +149,10 @@ impl<'a> fmt::Display for JournalEntry<'a> { "fd-set-times (fd={}, atime={}, mtime={})", fd, st_atim, st_mtim ), - JournalEntry::FileDescriptorSetFlags { fd, flags } => { + JournalEntry::FileDescriptorSetFlagsV1 { fd, flags } => { write!(f, "fd-set-flags (fd={}, flags={:?})", fd, flags) } - JournalEntry::FileDescriptorSetRights { + JournalEntry::FileDescriptorSetRightsV1 { fd, fs_rights_base, fs_rights_inheriting, @@ -161,82 +161,82 @@ impl<'a> fmt::Display for JournalEntry<'a> { "fd-set-rights (fd={}, base={:?}, inherited={:?})", fd, fs_rights_base, fs_rights_inheriting ), - JournalEntry::FileDescriptorSetSize { fd, st_size } => { + JournalEntry::FileDescriptorSetSizeV1 { fd, st_size } => { write!(f, "fd-set-size (fd={}, size={})", fd, st_size) } - JournalEntry::FileDescriptorAdvise { + JournalEntry::FileDescriptorAdviseV1 { fd, offset, len, .. } => write!(f, "fd-advise (fd={}, offset={}, len={})", fd, offset, len), - JournalEntry::FileDescriptorAllocate { fd, offset, len } => { + JournalEntry::FileDescriptorAllocateV1 { fd, offset, len } => { write!(f, "fd-allocate (fd={}, offset={}, len={})", fd, offset, len) } - JournalEntry::CreateHardLink { + JournalEntry::CreateHardLinkV1 { old_path, new_path, .. } => write!(f, "path-link (from={}, to={})", old_path, new_path), - JournalEntry::CreateSymbolicLink { + JournalEntry::CreateSymbolicLinkV1 { old_path, new_path, .. } => write!(f, "path-symlink (from={}, to={})", old_path, new_path), - JournalEntry::UnlinkFile { path, .. } => write!(f, "path-unlink (path={})", path), - JournalEntry::PathRename { + JournalEntry::UnlinkFileV1 { path, .. } => write!(f, "path-unlink (path={})", path), + JournalEntry::PathRenameV1 { old_path, new_path, .. } => write!( f, "path-rename (old-path={}, new-path={})", old_path, new_path ), - JournalEntry::ChangeDirectory { path } => write!(f, "chdir (path={})", path), - JournalEntry::EpollCreate { fd } => write!(f, "epoll-create (fd={})", fd), - JournalEntry::EpollCtl { epfd, op, fd, .. } => { + JournalEntry::ChangeDirectoryV1 { path } => write!(f, "chdir (path={})", path), + JournalEntry::EpollCreateV1 { fd } => write!(f, "epoll-create (fd={})", fd), + JournalEntry::EpollCtlV1 { epfd, op, fd, .. } => { write!(f, "epoll-ctl (epfd={}, op={:?}, fd={})", epfd, op, fd) } - JournalEntry::TtySet { tty, line_feeds } => write!( + JournalEntry::TtySetV1 { tty, line_feeds } => write!( f, "tty-set (echo={}, buffering={}, feeds={})", tty.echo, tty.line_buffered, line_feeds ), - JournalEntry::CreatePipe { fd1, fd2 } => { + JournalEntry::CreatePipeV1 { fd1, fd2 } => { write!(f, "fd-pipe (fd1={}, fd2={})", fd1, fd2) } - JournalEntry::CreateEvent { + JournalEntry::CreateEventV1 { initial_val, fd, .. } => write!(f, "fd-event (fd={}, initial={})", fd, initial_val), - JournalEntry::PortAddAddr { cidr } => { + JournalEntry::PortAddAddrV1 { cidr } => { write!(f, "port-addr-add (ip={}, prefix={})", cidr.ip, cidr.prefix) } - JournalEntry::PortDelAddr { addr } => write!(f, "port-addr-del (addr={})", addr), - JournalEntry::PortAddrClear => write!(f, "port-addr-clear"), - JournalEntry::PortBridge { network, .. } => { + JournalEntry::PortDelAddrV1 { addr } => write!(f, "port-addr-del (addr={})", addr), + JournalEntry::PortAddrClearV1 => write!(f, "port-addr-clear"), + JournalEntry::PortBridgeV1 { network, .. } => { write!(f, "port-bridge (network={})", network) } - JournalEntry::PortUnbridge => write!(f, "port-unbridge"), - JournalEntry::PortDhcpAcquire => write!(f, "port-dhcp-acquire"), - JournalEntry::PortGatewaySet { ip } => write!(f, "port-gateway-set (ip={})", ip), - JournalEntry::PortRouteAdd { + JournalEntry::PortUnbridgeV1 => write!(f, "port-unbridge"), + JournalEntry::PortDhcpAcquireV1 => write!(f, "port-dhcp-acquire"), + JournalEntry::PortGatewaySetV1 { ip } => write!(f, "port-gateway-set (ip={})", ip), + JournalEntry::PortRouteAddV1 { cidr, via_router, .. } => write!( f, "port-route-add (ip={}, prefix={}, via_router={})", cidr.ip, cidr.prefix, via_router ), - JournalEntry::PortRouteClear => write!(f, "port-route-clear"), - JournalEntry::PortRouteDel { ip } => write!(f, "port-route-del (ip={})", ip), - JournalEntry::SocketOpen { af, ty, pt, fd } => { + JournalEntry::PortRouteClearV1 => write!(f, "port-route-clear"), + JournalEntry::PortRouteDelV1 { ip } => write!(f, "port-route-del (ip={})", ip), + JournalEntry::SocketOpenV1 { af, ty, pt, fd } => { write!( f, "sock-open (fd={}, af={:?}, ty={:?}, pt={:?})", fd, af, ty, pt ) } - JournalEntry::SocketListen { fd, backlog } => { + JournalEntry::SocketListenV1 { fd, backlog } => { write!(f, "sock-listen (fd={}, backlog={})", fd, backlog) } - JournalEntry::SocketBind { fd, addr } => { + JournalEntry::SocketBindV1 { fd, addr } => { write!(f, "sock-bind (fd={}, addr={})", fd, addr) } - JournalEntry::SocketConnected { fd, addr } => { + JournalEntry::SocketConnectedV1 { fd, addr } => { write!(f, "sock-connect (fd={}, addr={})", fd, addr) } - JournalEntry::SocketAccepted { + JournalEntry::SocketAcceptedV1 { listen_fd, fd, peer_addr, @@ -246,7 +246,7 @@ impl<'a> fmt::Display for JournalEntry<'a> { "sock-accept (listen-fd={}, sock_fd={}, peer={})", listen_fd, fd, peer_addr ), - JournalEntry::SocketJoinIpv4Multicast { + JournalEntry::SocketJoinIpv4MulticastV1 { fd, multiaddr, iface, @@ -255,34 +255,34 @@ impl<'a> fmt::Display for JournalEntry<'a> { "sock-join-mcast-ipv4 (fd={}, addr={}, iface={})", fd, multiaddr, iface ), - JournalEntry::SocketJoinIpv6Multicast { + JournalEntry::SocketJoinIpv6MulticastV1 { fd, - multiaddr, + multi_addr: multiaddr, iface, } => write!( f, "sock-join-mcast-ipv6 (fd={}, addr={}, iface={})", fd, multiaddr, iface ), - JournalEntry::SocketLeaveIpv4Multicast { + JournalEntry::SocketLeaveIpv4MulticastV1 { fd, - multiaddr, + multi_addr: multiaddr, iface, } => write!( f, "sock-leave-mcast-ipv4 (fd={}, addr={}, iface={})", fd, multiaddr, iface ), - JournalEntry::SocketLeaveIpv6Multicast { + JournalEntry::SocketLeaveIpv6MulticastV1 { fd, - multiaddr, + multi_addr: multiaddr, iface, } => write!( f, "sock-leave-mcast-ipv6 (fd={}, addr={}, iface={})", fd, multiaddr, iface ), - JournalEntry::SocketSendFile { + JournalEntry::SocketSendFileV1 { socket_fd, file_fd, offset, @@ -292,29 +292,29 @@ impl<'a> fmt::Display for JournalEntry<'a> { "sock-send-file (sock-fd={}, file-fd={}, offset={}, count={})", socket_fd, file_fd, offset, count ), - JournalEntry::SocketSendTo { fd, data, addr, .. } => write!( + JournalEntry::SocketSendToV1 { fd, data, addr, .. } => write!( f, "sock-send-to (fd={}, data.len={}, addr={})", fd, data.len(), addr ), - JournalEntry::SocketSend { fd, data, .. } => { + JournalEntry::SocketSendV1 { fd, data, .. } => { write!(f, "sock-send (fd={}, data.len={}", fd, data.len()) } - JournalEntry::SocketSetOptFlag { fd, opt, flag } => { + JournalEntry::SocketSetOptFlagV1 { fd, opt, flag } => { write!(f, "sock-set-opt (fd={}, opt={:?}, flag={})", fd, opt, flag) } - JournalEntry::SocketSetOptSize { fd, opt, size } => { + JournalEntry::SocketSetOptSizeV1 { fd, opt, size } => { write!(f, "sock-set-opt (fd={}, opt={:?}, size={})", fd, opt, size) } - JournalEntry::SocketSetOptTime { fd, ty, time } => { + JournalEntry::SocketSetOptTimeV1 { fd, ty, time } => { write!(f, "sock-set-opt (fd={}, opt={:?}, time={:?})", fd, ty, time) } - JournalEntry::SocketShutdown { fd, how } => { + JournalEntry::SocketShutdownV1 { fd, how } => { write!(f, "sock-shutdown (fd={}, how={:?})", fd, how) } - JournalEntry::Snapshot { when, trigger } => { + JournalEntry::SnapshotV1 { when, trigger } => { write!(f, "snapshot (when={:?}, trigger={:?})", when, trigger) } } diff --git a/lib/wasix/src/journal/effector/memory_and_snapshot.rs b/lib/wasix/src/journal/effector/memory_and_snapshot.rs index 16400b61f07..a53ec4465d2 100644 --- a/lib/wasix/src/journal/effector/memory_and_snapshot.rs +++ b/lib/wasix/src/journal/effector/memory_and_snapshot.rs @@ -54,7 +54,7 @@ impl JournalEffector { // Now we write it to the snap snapshot capturer journal - .write(JournalEntry::UpdateMemoryRegion { + .write(JournalEntry::UpdateMemoryRegionV1 { region, data: data.into(), }) @@ -65,7 +65,7 @@ impl JournalEffector { // it can act as a restoration point let when = SystemTime::now(); journal - .write(JournalEntry::Snapshot { when, trigger }) + .write(JournalEntry::SnapshotV1 { when, trigger }) .map_err(map_snapshot_err)?; Ok(()) } diff --git a/lib/wasix/src/journal/effector/process_exit.rs b/lib/wasix/src/journal/effector/process_exit.rs index 837d7130294..d57264e50fa 100644 --- a/lib/wasix/src/journal/effector/process_exit.rs +++ b/lib/wasix/src/journal/effector/process_exit.rs @@ -5,7 +5,7 @@ use super::*; impl JournalEffector { pub fn save_process_exit(env: &WasiEnv, exit_code: Option) -> anyhow::Result<()> { env.active_journal()? - .write(JournalEntry::ProcessExit { exit_code }) + .write(JournalEntry::ProcessExitV1 { exit_code }) .map_err(map_snapshot_err)?; Ok(()) } diff --git a/lib/wasix/src/journal/effector/syscalls/chdir.rs b/lib/wasix/src/journal/effector/syscalls/chdir.rs index ee0d49f7d55..aa3041a1562 100644 --- a/lib/wasix/src/journal/effector/syscalls/chdir.rs +++ b/lib/wasix/src/journal/effector/syscalls/chdir.rs @@ -2,7 +2,7 @@ use super::*; impl JournalEffector { pub fn save_chdir(ctx: &mut FunctionEnvMut<'_, WasiEnv>, path: String) -> anyhow::Result<()> { - Self::save_event(ctx, JournalEntry::ChangeDirectory { path: path.into() }) + Self::save_event(ctx, JournalEntry::ChangeDirectoryV1 { path: path.into() }) } pub fn apply_chdir(ctx: &mut FunctionEnvMut<'_, WasiEnv>, path: &str) -> anyhow::Result<()> { diff --git a/lib/wasix/src/journal/effector/syscalls/clock_time.rs b/lib/wasix/src/journal/effector/syscalls/clock_time.rs index 4100c754797..c196732dfe2 100644 --- a/lib/wasix/src/journal/effector/syscalls/clock_time.rs +++ b/lib/wasix/src/journal/effector/syscalls/clock_time.rs @@ -6,7 +6,7 @@ impl JournalEffector { clock_id: Snapshot0Clockid, time: Timestamp, ) -> anyhow::Result<()> { - Self::save_event(ctx, JournalEntry::SetClockTime { clock_id, time }) + Self::save_event(ctx, JournalEntry::SetClockTimeV1 { clock_id, time }) } pub fn apply_clock_time_set( diff --git a/lib/wasix/src/journal/effector/syscalls/epoll_create.rs b/lib/wasix/src/journal/effector/syscalls/epoll_create.rs index de75591a5e7..7fbe2ae227a 100644 --- a/lib/wasix/src/journal/effector/syscalls/epoll_create.rs +++ b/lib/wasix/src/journal/effector/syscalls/epoll_create.rs @@ -2,7 +2,7 @@ use super::*; impl JournalEffector { pub fn save_epoll_create(ctx: &mut FunctionEnvMut<'_, WasiEnv>, fd: Fd) -> anyhow::Result<()> { - Self::save_event(ctx, JournalEntry::EpollCreate { fd }) + Self::save_event(ctx, JournalEntry::EpollCreateV1 { fd }) } pub fn apply_epoll_create(ctx: &mut FunctionEnvMut<'_, WasiEnv>, fd: Fd) -> anyhow::Result<()> { diff --git a/lib/wasix/src/journal/effector/syscalls/epoll_ctl.rs b/lib/wasix/src/journal/effector/syscalls/epoll_ctl.rs index 6aaf463b56f..e4719999db4 100644 --- a/lib/wasix/src/journal/effector/syscalls/epoll_ctl.rs +++ b/lib/wasix/src/journal/effector/syscalls/epoll_ctl.rs @@ -10,7 +10,7 @@ impl JournalEffector { ) -> anyhow::Result<()> { Self::save_event( ctx, - JournalEntry::EpollCtl { + JournalEntry::EpollCtlV1 { epfd, op, fd, diff --git a/lib/wasix/src/journal/effector/syscalls/fd_advise.rs b/lib/wasix/src/journal/effector/syscalls/fd_advise.rs index 900db9aca79..12cd5e17802 100644 --- a/lib/wasix/src/journal/effector/syscalls/fd_advise.rs +++ b/lib/wasix/src/journal/effector/syscalls/fd_advise.rs @@ -10,7 +10,7 @@ impl JournalEffector { ) -> anyhow::Result<()> { Self::save_event( ctx, - JournalEntry::FileDescriptorAdvise { + JournalEntry::FileDescriptorAdviseV1 { fd, offset, len, diff --git a/lib/wasix/src/journal/effector/syscalls/fd_allocate.rs b/lib/wasix/src/journal/effector/syscalls/fd_allocate.rs index 3ad49bb02fd..6ecacb827eb 100644 --- a/lib/wasix/src/journal/effector/syscalls/fd_allocate.rs +++ b/lib/wasix/src/journal/effector/syscalls/fd_allocate.rs @@ -9,7 +9,7 @@ impl JournalEffector { ) -> anyhow::Result<()> { Self::save_event( ctx, - JournalEntry::FileDescriptorAllocate { fd, offset, len }, + JournalEntry::FileDescriptorAllocateV1 { fd, offset, len }, ) } diff --git a/lib/wasix/src/journal/effector/syscalls/fd_close.rs b/lib/wasix/src/journal/effector/syscalls/fd_close.rs index d1d0b25aa2c..4ec66918699 100644 --- a/lib/wasix/src/journal/effector/syscalls/fd_close.rs +++ b/lib/wasix/src/journal/effector/syscalls/fd_close.rs @@ -2,7 +2,7 @@ use super::*; impl JournalEffector { pub fn save_fd_close(ctx: &mut FunctionEnvMut<'_, WasiEnv>, fd: Fd) -> anyhow::Result<()> { - Self::save_event(ctx, JournalEntry::CloseFileDescriptor { fd }) + Self::save_event(ctx, JournalEntry::CloseFileDescriptorV1 { fd }) } pub fn apply_fd_close(ctx: &mut FunctionEnvMut<'_, WasiEnv>, fd: Fd) -> anyhow::Result<()> { diff --git a/lib/wasix/src/journal/effector/syscalls/fd_duplicate.rs b/lib/wasix/src/journal/effector/syscalls/fd_duplicate.rs index 1d489ec5420..a53be32d4c4 100644 --- a/lib/wasix/src/journal/effector/syscalls/fd_duplicate.rs +++ b/lib/wasix/src/journal/effector/syscalls/fd_duplicate.rs @@ -8,7 +8,7 @@ impl JournalEffector { ) -> anyhow::Result<()> { Self::save_event( ctx, - JournalEntry::DuplicateFileDescriptor { + JournalEntry::DuplicateFileDescriptorV1 { original_fd, copied_fd, }, diff --git a/lib/wasix/src/journal/effector/syscalls/fd_event.rs b/lib/wasix/src/journal/effector/syscalls/fd_event.rs index 7efd599a582..a30738947a2 100644 --- a/lib/wasix/src/journal/effector/syscalls/fd_event.rs +++ b/lib/wasix/src/journal/effector/syscalls/fd_event.rs @@ -11,7 +11,7 @@ impl JournalEffector { ) -> anyhow::Result<()> { Self::save_event( ctx, - JournalEntry::CreateEvent { + JournalEntry::CreateEventV1 { initial_val, flags, fd, diff --git a/lib/wasix/src/journal/effector/syscalls/fd_pipe.rs b/lib/wasix/src/journal/effector/syscalls/fd_pipe.rs index 5238dd5ac14..21b32d8a402 100644 --- a/lib/wasix/src/journal/effector/syscalls/fd_pipe.rs +++ b/lib/wasix/src/journal/effector/syscalls/fd_pipe.rs @@ -6,7 +6,7 @@ impl JournalEffector { fd1: Fd, fd2: Fd, ) -> anyhow::Result<()> { - Self::save_event(ctx, JournalEntry::CreatePipe { fd1, fd2 }) + Self::save_event(ctx, JournalEntry::CreatePipeV1 { fd1, fd2 }) } pub fn apply_fd_pipe( diff --git a/lib/wasix/src/journal/effector/syscalls/fd_renumber.rs b/lib/wasix/src/journal/effector/syscalls/fd_renumber.rs index 5d72902449d..c439d753f23 100644 --- a/lib/wasix/src/journal/effector/syscalls/fd_renumber.rs +++ b/lib/wasix/src/journal/effector/syscalls/fd_renumber.rs @@ -8,7 +8,7 @@ impl JournalEffector { ) -> anyhow::Result<()> { Self::save_event( ctx, - JournalEntry::RenumberFileDescriptor { + JournalEntry::RenumberFileDescriptorV1 { old_fd: from, new_fd: to, }, diff --git a/lib/wasix/src/journal/effector/syscalls/fd_seek.rs b/lib/wasix/src/journal/effector/syscalls/fd_seek.rs index 16baa705eea..7cd7787ba69 100644 --- a/lib/wasix/src/journal/effector/syscalls/fd_seek.rs +++ b/lib/wasix/src/journal/effector/syscalls/fd_seek.rs @@ -7,7 +7,10 @@ impl JournalEffector { offset: i64, whence: Whence, ) -> anyhow::Result<()> { - Self::save_event(ctx, JournalEntry::FileDescriptorSeek { fd, offset, whence }) + Self::save_event( + ctx, + JournalEntry::FileDescriptorSeekV1 { fd, offset, whence }, + ) } pub fn apply_fd_seek( diff --git a/lib/wasix/src/journal/effector/syscalls/fd_set_flags.rs b/lib/wasix/src/journal/effector/syscalls/fd_set_flags.rs index 293902275d3..669041a88da 100644 --- a/lib/wasix/src/journal/effector/syscalls/fd_set_flags.rs +++ b/lib/wasix/src/journal/effector/syscalls/fd_set_flags.rs @@ -6,7 +6,7 @@ impl JournalEffector { fd: Fd, flags: Fdflags, ) -> anyhow::Result<()> { - Self::save_event(ctx, JournalEntry::FileDescriptorSetFlags { fd, flags }) + Self::save_event(ctx, JournalEntry::FileDescriptorSetFlagsV1 { fd, flags }) } pub fn apply_fd_set_flags( diff --git a/lib/wasix/src/journal/effector/syscalls/fd_set_rights.rs b/lib/wasix/src/journal/effector/syscalls/fd_set_rights.rs index a86a3f0a5e8..6d77752b17f 100644 --- a/lib/wasix/src/journal/effector/syscalls/fd_set_rights.rs +++ b/lib/wasix/src/journal/effector/syscalls/fd_set_rights.rs @@ -9,7 +9,7 @@ impl JournalEffector { ) -> anyhow::Result<()> { Self::save_event( ctx, - JournalEntry::FileDescriptorSetRights { + JournalEntry::FileDescriptorSetRightsV1 { fd, fs_rights_base, fs_rights_inheriting, diff --git a/lib/wasix/src/journal/effector/syscalls/fd_set_size.rs b/lib/wasix/src/journal/effector/syscalls/fd_set_size.rs index 2ad7531df88..58980c12207 100644 --- a/lib/wasix/src/journal/effector/syscalls/fd_set_size.rs +++ b/lib/wasix/src/journal/effector/syscalls/fd_set_size.rs @@ -6,7 +6,7 @@ impl JournalEffector { fd: Fd, st_size: Filesize, ) -> anyhow::Result<()> { - Self::save_event(ctx, JournalEntry::FileDescriptorSetSize { fd, st_size }) + Self::save_event(ctx, JournalEntry::FileDescriptorSetSizeV1 { fd, st_size }) } pub fn apply_fd_set_size( diff --git a/lib/wasix/src/journal/effector/syscalls/fd_set_times.rs b/lib/wasix/src/journal/effector/syscalls/fd_set_times.rs index 26f17565b8a..633ba079c03 100644 --- a/lib/wasix/src/journal/effector/syscalls/fd_set_times.rs +++ b/lib/wasix/src/journal/effector/syscalls/fd_set_times.rs @@ -10,7 +10,7 @@ impl JournalEffector { ) -> anyhow::Result<()> { Self::save_event( ctx, - JournalEntry::FileDescriptorSetTimes { + JournalEntry::FileDescriptorSetTimesV1 { fd, st_atim, st_mtim, diff --git a/lib/wasix/src/journal/effector/syscalls/fd_write.rs b/lib/wasix/src/journal/effector/syscalls/fd_write.rs index 9301a16357d..689adc97a87 100644 --- a/lib/wasix/src/journal/effector/syscalls/fd_write.rs +++ b/lib/wasix/src/journal/effector/syscalls/fd_write.rs @@ -32,7 +32,7 @@ impl JournalEffector { ctx.data() .active_journal()? - .write(JournalEntry::FileDescriptorWrite { + .write(JournalEntry::FileDescriptorWriteV1 { fd, offset, data, diff --git a/lib/wasix/src/journal/effector/syscalls/path_create_directory.rs b/lib/wasix/src/journal/effector/syscalls/path_create_directory.rs index 249a9df128c..447c60393af 100644 --- a/lib/wasix/src/journal/effector/syscalls/path_create_directory.rs +++ b/lib/wasix/src/journal/effector/syscalls/path_create_directory.rs @@ -8,7 +8,7 @@ impl JournalEffector { ) -> anyhow::Result<()> { Self::save_event( ctx, - JournalEntry::CreateDirectory { + JournalEntry::CreateDirectoryV1 { fd, path: path.into(), }, diff --git a/lib/wasix/src/journal/effector/syscalls/path_link.rs b/lib/wasix/src/journal/effector/syscalls/path_link.rs index fd85faf35fa..822f671d09c 100644 --- a/lib/wasix/src/journal/effector/syscalls/path_link.rs +++ b/lib/wasix/src/journal/effector/syscalls/path_link.rs @@ -11,7 +11,7 @@ impl JournalEffector { ) -> anyhow::Result<()> { Self::save_event( ctx, - JournalEntry::CreateHardLink { + JournalEntry::CreateHardLinkV1 { old_fd, old_flags, old_path: old_path.into(), diff --git a/lib/wasix/src/journal/effector/syscalls/path_open.rs b/lib/wasix/src/journal/effector/syscalls/path_open.rs index 5119f3570a5..f63e303bceb 100644 --- a/lib/wasix/src/journal/effector/syscalls/path_open.rs +++ b/lib/wasix/src/journal/effector/syscalls/path_open.rs @@ -15,7 +15,7 @@ impl JournalEffector { ) -> anyhow::Result<()> { Self::save_event( ctx, - JournalEntry::OpenFileDescriptor { + JournalEntry::OpenFileDescriptorV1 { fd, dirfd, dirflags, diff --git a/lib/wasix/src/journal/effector/syscalls/path_remove_directory.rs b/lib/wasix/src/journal/effector/syscalls/path_remove_directory.rs index d607cca4520..b45ec07d1a4 100644 --- a/lib/wasix/src/journal/effector/syscalls/path_remove_directory.rs +++ b/lib/wasix/src/journal/effector/syscalls/path_remove_directory.rs @@ -8,7 +8,7 @@ impl JournalEffector { ) -> anyhow::Result<()> { Self::save_event( ctx, - JournalEntry::RemoveDirectory { + JournalEntry::RemoveDirectoryV1 { fd, path: Cow::Owned(path), }, diff --git a/lib/wasix/src/journal/effector/syscalls/path_rename.rs b/lib/wasix/src/journal/effector/syscalls/path_rename.rs index 9eaafaff42c..41ca232f8b8 100644 --- a/lib/wasix/src/journal/effector/syscalls/path_rename.rs +++ b/lib/wasix/src/journal/effector/syscalls/path_rename.rs @@ -10,7 +10,7 @@ impl JournalEffector { ) -> anyhow::Result<()> { Self::save_event( ctx, - JournalEntry::PathRename { + JournalEntry::PathRenameV1 { old_fd, old_path: Cow::Owned(old_path), new_fd, diff --git a/lib/wasix/src/journal/effector/syscalls/path_set_times.rs b/lib/wasix/src/journal/effector/syscalls/path_set_times.rs index b9e72a0c5b4..e40b3bae698 100644 --- a/lib/wasix/src/journal/effector/syscalls/path_set_times.rs +++ b/lib/wasix/src/journal/effector/syscalls/path_set_times.rs @@ -12,7 +12,7 @@ impl JournalEffector { ) -> anyhow::Result<()> { Self::save_event( ctx, - JournalEntry::PathSetTimes { + JournalEntry::PathSetTimesV1 { fd, flags, path: path.into(), diff --git a/lib/wasix/src/journal/effector/syscalls/path_symlink.rs b/lib/wasix/src/journal/effector/syscalls/path_symlink.rs index 3f54de76412..60591cae7c4 100644 --- a/lib/wasix/src/journal/effector/syscalls/path_symlink.rs +++ b/lib/wasix/src/journal/effector/syscalls/path_symlink.rs @@ -9,7 +9,7 @@ impl JournalEffector { ) -> anyhow::Result<()> { Self::save_event( ctx, - JournalEntry::CreateSymbolicLink { + JournalEntry::CreateSymbolicLinkV1 { old_path: old_path.into(), fd, new_path: new_path.into(), diff --git a/lib/wasix/src/journal/effector/syscalls/path_unlink.rs b/lib/wasix/src/journal/effector/syscalls/path_unlink.rs index a21825b9258..9141a7fe8cf 100644 --- a/lib/wasix/src/journal/effector/syscalls/path_unlink.rs +++ b/lib/wasix/src/journal/effector/syscalls/path_unlink.rs @@ -8,7 +8,7 @@ impl JournalEffector { ) -> anyhow::Result<()> { Self::save_event( ctx, - JournalEntry::UnlinkFile { + JournalEntry::UnlinkFileV1 { fd, path: Cow::Owned(path), }, diff --git a/lib/wasix/src/journal/effector/syscalls/port_addr_add.rs b/lib/wasix/src/journal/effector/syscalls/port_addr_add.rs index d3d3d51edfc..ae508e774ea 100644 --- a/lib/wasix/src/journal/effector/syscalls/port_addr_add.rs +++ b/lib/wasix/src/journal/effector/syscalls/port_addr_add.rs @@ -7,7 +7,7 @@ impl JournalEffector { ctx: &mut FunctionEnvMut<'_, WasiEnv>, cidr: IpCidr, ) -> anyhow::Result<()> { - Self::save_event(ctx, JournalEntry::PortAddAddr { cidr }) + Self::save_event(ctx, JournalEntry::PortAddAddrV1 { cidr }) } pub fn apply_port_addr_add( diff --git a/lib/wasix/src/journal/effector/syscalls/port_addr_clear.rs b/lib/wasix/src/journal/effector/syscalls/port_addr_clear.rs index 078fab9157d..d68a2a8e4c6 100644 --- a/lib/wasix/src/journal/effector/syscalls/port_addr_clear.rs +++ b/lib/wasix/src/journal/effector/syscalls/port_addr_clear.rs @@ -2,7 +2,7 @@ use super::*; impl JournalEffector { pub fn save_port_addr_clear(ctx: &mut FunctionEnvMut<'_, WasiEnv>) -> anyhow::Result<()> { - Self::save_event(ctx, JournalEntry::PortAddrClear) + Self::save_event(ctx, JournalEntry::PortAddrClearV1) } pub fn apply_port_addr_clear(ctx: &mut FunctionEnvMut<'_, WasiEnv>) -> anyhow::Result<()> { diff --git a/lib/wasix/src/journal/effector/syscalls/port_addr_remove.rs b/lib/wasix/src/journal/effector/syscalls/port_addr_remove.rs index 88d31ec2f60..1463d2595cb 100644 --- a/lib/wasix/src/journal/effector/syscalls/port_addr_remove.rs +++ b/lib/wasix/src/journal/effector/syscalls/port_addr_remove.rs @@ -7,7 +7,7 @@ impl JournalEffector { ctx: &mut FunctionEnvMut<'_, WasiEnv>, addr: IpAddr, ) -> anyhow::Result<()> { - Self::save_event(ctx, JournalEntry::PortDelAddr { addr }) + Self::save_event(ctx, JournalEntry::PortDelAddrV1 { addr }) } pub fn apply_port_addr_remove( diff --git a/lib/wasix/src/journal/effector/syscalls/port_bridge.rs b/lib/wasix/src/journal/effector/syscalls/port_bridge.rs index ee69c557663..bdfd5ce54dc 100644 --- a/lib/wasix/src/journal/effector/syscalls/port_bridge.rs +++ b/lib/wasix/src/journal/effector/syscalls/port_bridge.rs @@ -11,7 +11,7 @@ impl JournalEffector { ) -> anyhow::Result<()> { Self::save_event( ctx, - JournalEntry::PortBridge { + JournalEntry::PortBridgeV1 { network: network.into(), token: token.into(), security, diff --git a/lib/wasix/src/journal/effector/syscalls/port_dhcp_acquire.rs b/lib/wasix/src/journal/effector/syscalls/port_dhcp_acquire.rs index cfa6ea3e930..98ef069b181 100644 --- a/lib/wasix/src/journal/effector/syscalls/port_dhcp_acquire.rs +++ b/lib/wasix/src/journal/effector/syscalls/port_dhcp_acquire.rs @@ -2,7 +2,7 @@ use super::*; impl JournalEffector { pub fn save_port_dhcp_acquire(ctx: &mut FunctionEnvMut<'_, WasiEnv>) -> anyhow::Result<()> { - Self::save_event(ctx, JournalEntry::PortDhcpAcquire) + Self::save_event(ctx, JournalEntry::PortDhcpAcquireV1) } pub fn apply_port_dhcp_acquire(ctx: &mut FunctionEnvMut<'_, WasiEnv>) -> anyhow::Result<()> { diff --git a/lib/wasix/src/journal/effector/syscalls/port_gateway_set.rs b/lib/wasix/src/journal/effector/syscalls/port_gateway_set.rs index 22247418c7f..230eba8d467 100644 --- a/lib/wasix/src/journal/effector/syscalls/port_gateway_set.rs +++ b/lib/wasix/src/journal/effector/syscalls/port_gateway_set.rs @@ -7,7 +7,7 @@ impl JournalEffector { ctx: &mut FunctionEnvMut<'_, WasiEnv>, ip: IpAddr, ) -> anyhow::Result<()> { - Self::save_event(ctx, JournalEntry::PortGatewaySet { ip }) + Self::save_event(ctx, JournalEntry::PortGatewaySetV1 { ip }) } pub fn apply_port_gateway_set( diff --git a/lib/wasix/src/journal/effector/syscalls/port_route_add.rs b/lib/wasix/src/journal/effector/syscalls/port_route_add.rs index 489d440dc60..0565e8b8f09 100644 --- a/lib/wasix/src/journal/effector/syscalls/port_route_add.rs +++ b/lib/wasix/src/journal/effector/syscalls/port_route_add.rs @@ -12,7 +12,7 @@ impl JournalEffector { ) -> anyhow::Result<()> { Self::save_event( ctx, - JournalEntry::PortRouteAdd { + JournalEntry::PortRouteAddV1 { cidr, via_router, preferred_until, diff --git a/lib/wasix/src/journal/effector/syscalls/port_route_clear.rs b/lib/wasix/src/journal/effector/syscalls/port_route_clear.rs index f02a3adad9e..dfb1774d10c 100644 --- a/lib/wasix/src/journal/effector/syscalls/port_route_clear.rs +++ b/lib/wasix/src/journal/effector/syscalls/port_route_clear.rs @@ -2,7 +2,7 @@ use super::*; impl JournalEffector { pub fn save_port_route_clear(ctx: &mut FunctionEnvMut<'_, WasiEnv>) -> anyhow::Result<()> { - Self::save_event(ctx, JournalEntry::PortRouteClear) + Self::save_event(ctx, JournalEntry::PortRouteClearV1) } pub fn apply_port_route_clear(ctx: &mut FunctionEnvMut<'_, WasiEnv>) -> anyhow::Result<()> { diff --git a/lib/wasix/src/journal/effector/syscalls/port_route_remove.rs b/lib/wasix/src/journal/effector/syscalls/port_route_remove.rs index 00af31c21da..61d371b8da1 100644 --- a/lib/wasix/src/journal/effector/syscalls/port_route_remove.rs +++ b/lib/wasix/src/journal/effector/syscalls/port_route_remove.rs @@ -7,7 +7,7 @@ impl JournalEffector { ctx: &mut FunctionEnvMut<'_, WasiEnv>, ip: IpAddr, ) -> anyhow::Result<()> { - Self::save_event(ctx, JournalEntry::PortRouteDel { ip }) + Self::save_event(ctx, JournalEntry::PortRouteDelV1 { ip }) } pub fn apply_port_route_remove( diff --git a/lib/wasix/src/journal/effector/syscalls/port_unbridge.rs b/lib/wasix/src/journal/effector/syscalls/port_unbridge.rs index d5a4e25ec78..0fc1ae98d4e 100644 --- a/lib/wasix/src/journal/effector/syscalls/port_unbridge.rs +++ b/lib/wasix/src/journal/effector/syscalls/port_unbridge.rs @@ -2,7 +2,7 @@ use super::*; impl JournalEffector { pub fn save_port_unbridge(ctx: &mut FunctionEnvMut<'_, WasiEnv>) -> anyhow::Result<()> { - Self::save_event(ctx, JournalEntry::PortUnbridge) + Self::save_event(ctx, JournalEntry::PortUnbridgeV1) } pub fn apply_port_unbridge(ctx: &mut FunctionEnvMut<'_, WasiEnv>) -> anyhow::Result<()> { diff --git a/lib/wasix/src/journal/effector/syscalls/sock_accept.rs b/lib/wasix/src/journal/effector/syscalls/sock_accept.rs index 0985509d05d..2bbbf3b382d 100644 --- a/lib/wasix/src/journal/effector/syscalls/sock_accept.rs +++ b/lib/wasix/src/journal/effector/syscalls/sock_accept.rs @@ -18,12 +18,12 @@ impl JournalEffector { ) -> anyhow::Result<()> { Self::save_event( ctx, - JournalEntry::SocketAccepted { + JournalEntry::SocketAcceptedV1 { listen_fd, fd, peer_addr, fd_flags, - nonblocking, + non_blocking: nonblocking, }, ) } diff --git a/lib/wasix/src/journal/effector/syscalls/sock_bind.rs b/lib/wasix/src/journal/effector/syscalls/sock_bind.rs index 25e63641542..fc0727e788b 100644 --- a/lib/wasix/src/journal/effector/syscalls/sock_bind.rs +++ b/lib/wasix/src/journal/effector/syscalls/sock_bind.rs @@ -8,7 +8,7 @@ impl JournalEffector { fd: Fd, addr: SocketAddr, ) -> anyhow::Result<()> { - Self::save_event(ctx, JournalEntry::SocketBind { fd, addr }) + Self::save_event(ctx, JournalEntry::SocketBindV1 { fd, addr }) } pub fn apply_sock_bind( diff --git a/lib/wasix/src/journal/effector/syscalls/sock_connect.rs b/lib/wasix/src/journal/effector/syscalls/sock_connect.rs index 9558b82af1e..5e4b2f89b6b 100644 --- a/lib/wasix/src/journal/effector/syscalls/sock_connect.rs +++ b/lib/wasix/src/journal/effector/syscalls/sock_connect.rs @@ -13,7 +13,7 @@ impl JournalEffector { fd: Fd, addr: SocketAddr, ) -> anyhow::Result<()> { - Self::save_event(ctx, JournalEntry::SocketConnected { fd, addr }) + Self::save_event(ctx, JournalEntry::SocketConnectedV1 { fd, addr }) } pub fn apply_sock_connect( diff --git a/lib/wasix/src/journal/effector/syscalls/sock_join_ipv4_multicast.rs b/lib/wasix/src/journal/effector/syscalls/sock_join_ipv4_multicast.rs index 6768175d189..a1c4d40005d 100644 --- a/lib/wasix/src/journal/effector/syscalls/sock_join_ipv4_multicast.rs +++ b/lib/wasix/src/journal/effector/syscalls/sock_join_ipv4_multicast.rs @@ -11,7 +11,7 @@ impl JournalEffector { ) -> anyhow::Result<()> { Self::save_event( ctx, - JournalEntry::SocketJoinIpv4Multicast { + JournalEntry::SocketJoinIpv4MulticastV1 { fd, multiaddr, iface, diff --git a/lib/wasix/src/journal/effector/syscalls/sock_join_ipv6_multicast.rs b/lib/wasix/src/journal/effector/syscalls/sock_join_ipv6_multicast.rs index aa42495e8f4..d130e0391f1 100644 --- a/lib/wasix/src/journal/effector/syscalls/sock_join_ipv6_multicast.rs +++ b/lib/wasix/src/journal/effector/syscalls/sock_join_ipv6_multicast.rs @@ -11,9 +11,9 @@ impl JournalEffector { ) -> anyhow::Result<()> { Self::save_event( ctx, - JournalEntry::SocketJoinIpv6Multicast { + JournalEntry::SocketJoinIpv6MulticastV1 { fd, - multiaddr, + multi_addr: multiaddr, iface, }, ) diff --git a/lib/wasix/src/journal/effector/syscalls/sock_leave_ipv4_multicast.rs b/lib/wasix/src/journal/effector/syscalls/sock_leave_ipv4_multicast.rs index df6d6cdf589..50abd7d6255 100644 --- a/lib/wasix/src/journal/effector/syscalls/sock_leave_ipv4_multicast.rs +++ b/lib/wasix/src/journal/effector/syscalls/sock_leave_ipv4_multicast.rs @@ -11,9 +11,9 @@ impl JournalEffector { ) -> anyhow::Result<()> { Self::save_event( ctx, - JournalEntry::SocketLeaveIpv4Multicast { + JournalEntry::SocketLeaveIpv4MulticastV1 { fd, - multiaddr, + multi_addr: multiaddr, iface, }, ) diff --git a/lib/wasix/src/journal/effector/syscalls/sock_leave_ipv6_multicast.rs b/lib/wasix/src/journal/effector/syscalls/sock_leave_ipv6_multicast.rs index faf48b08a6e..6d7fad5fea1 100644 --- a/lib/wasix/src/journal/effector/syscalls/sock_leave_ipv6_multicast.rs +++ b/lib/wasix/src/journal/effector/syscalls/sock_leave_ipv6_multicast.rs @@ -11,9 +11,9 @@ impl JournalEffector { ) -> anyhow::Result<()> { Self::save_event( ctx, - JournalEntry::SocketLeaveIpv6Multicast { + JournalEntry::SocketLeaveIpv6MulticastV1 { fd, - multiaddr, + multi_addr: multiaddr, iface, }, ) diff --git a/lib/wasix/src/journal/effector/syscalls/sock_listen.rs b/lib/wasix/src/journal/effector/syscalls/sock_listen.rs index b404cf0e597..40a420a2013 100644 --- a/lib/wasix/src/journal/effector/syscalls/sock_listen.rs +++ b/lib/wasix/src/journal/effector/syscalls/sock_listen.rs @@ -8,7 +8,7 @@ impl JournalEffector { ) -> anyhow::Result<()> { Self::save_event( ctx, - JournalEntry::SocketListen { + JournalEntry::SocketListenV1 { fd, backlog: backlog as u32, }, diff --git a/lib/wasix/src/journal/effector/syscalls/sock_open.rs b/lib/wasix/src/journal/effector/syscalls/sock_open.rs index 3dd3527cc08..eaa07b40919 100644 --- a/lib/wasix/src/journal/effector/syscalls/sock_open.rs +++ b/lib/wasix/src/journal/effector/syscalls/sock_open.rs @@ -10,7 +10,7 @@ impl JournalEffector { pt: SockProto, fd: Fd, ) -> anyhow::Result<()> { - Self::save_event(ctx, JournalEntry::SocketOpen { af, ty, pt, fd }) + Self::save_event(ctx, JournalEntry::SocketOpenV1 { af, ty, pt, fd }) } pub fn apply_sock_open( diff --git a/lib/wasix/src/journal/effector/syscalls/sock_send.rs b/lib/wasix/src/journal/effector/syscalls/sock_send.rs index 8b5ed27e6b3..710b9cc9224 100644 --- a/lib/wasix/src/journal/effector/syscalls/sock_send.rs +++ b/lib/wasix/src/journal/effector/syscalls/sock_send.rs @@ -33,7 +33,7 @@ impl JournalEffector { .map_err(mem_error_to_wasi)?; ctx.data() .active_journal()? - .write(JournalEntry::SocketSend { + .write(JournalEntry::SocketSendV1 { fd, data: Cow::Borrowed(buf.as_ref()), is_64bit: M::is_64bit(), diff --git a/lib/wasix/src/journal/effector/syscalls/sock_send_file.rs b/lib/wasix/src/journal/effector/syscalls/sock_send_file.rs index e3abe981dbc..abf21efb9ee 100644 --- a/lib/wasix/src/journal/effector/syscalls/sock_send_file.rs +++ b/lib/wasix/src/journal/effector/syscalls/sock_send_file.rs @@ -12,7 +12,7 @@ impl JournalEffector { ) -> anyhow::Result<()> { Self::save_event( ctx, - JournalEntry::SocketSendFile { + JournalEntry::SocketSendFileV1 { socket_fd, file_fd, offset, diff --git a/lib/wasix/src/journal/effector/syscalls/sock_send_to.rs b/lib/wasix/src/journal/effector/syscalls/sock_send_to.rs index 7bf0fd671e2..7df19d13eb0 100644 --- a/lib/wasix/src/journal/effector/syscalls/sock_send_to.rs +++ b/lib/wasix/src/journal/effector/syscalls/sock_send_to.rs @@ -35,7 +35,7 @@ impl JournalEffector { .map_err(mem_error_to_wasi)?; ctx.data() .active_journal()? - .write(JournalEntry::SocketSendTo { + .write(JournalEntry::SocketSendToV1 { fd, data: Cow::Borrowed(buf.as_ref()), addr, diff --git a/lib/wasix/src/journal/effector/syscalls/sock_set_opt_flag.rs b/lib/wasix/src/journal/effector/syscalls/sock_set_opt_flag.rs index 8cfc5582cee..ead9f509332 100644 --- a/lib/wasix/src/journal/effector/syscalls/sock_set_opt_flag.rs +++ b/lib/wasix/src/journal/effector/syscalls/sock_set_opt_flag.rs @@ -9,7 +9,7 @@ impl JournalEffector { opt: Sockoption, flag: bool, ) -> anyhow::Result<()> { - Self::save_event(ctx, JournalEntry::SocketSetOptFlag { fd, opt, flag }) + Self::save_event(ctx, JournalEntry::SocketSetOptFlagV1 { fd, opt, flag }) } pub fn apply_sock_set_opt_flag( diff --git a/lib/wasix/src/journal/effector/syscalls/sock_set_opt_size.rs b/lib/wasix/src/journal/effector/syscalls/sock_set_opt_size.rs index 4db8c1d8937..867f3512250 100644 --- a/lib/wasix/src/journal/effector/syscalls/sock_set_opt_size.rs +++ b/lib/wasix/src/journal/effector/syscalls/sock_set_opt_size.rs @@ -9,7 +9,7 @@ impl JournalEffector { opt: Sockoption, size: Filesize, ) -> anyhow::Result<()> { - Self::save_event(ctx, JournalEntry::SocketSetOptSize { fd, opt, size }) + Self::save_event(ctx, JournalEntry::SocketSetOptSizeV1 { fd, opt, size }) } pub fn apply_sock_set_opt_size( diff --git a/lib/wasix/src/journal/effector/syscalls/sock_set_opt_time.rs b/lib/wasix/src/journal/effector/syscalls/sock_set_opt_time.rs index b88f74d73a2..75574fb5cde 100644 --- a/lib/wasix/src/journal/effector/syscalls/sock_set_opt_time.rs +++ b/lib/wasix/src/journal/effector/syscalls/sock_set_opt_time.rs @@ -13,7 +13,7 @@ impl JournalEffector { ) -> anyhow::Result<()> { Self::save_event( ctx, - JournalEntry::SocketSetOptTime { + JournalEntry::SocketSetOptTimeV1 { fd, ty: ty.into(), time, diff --git a/lib/wasix/src/journal/effector/syscalls/sock_shutdown.rs b/lib/wasix/src/journal/effector/syscalls/sock_shutdown.rs index 1d22351ad43..61497214b70 100644 --- a/lib/wasix/src/journal/effector/syscalls/sock_shutdown.rs +++ b/lib/wasix/src/journal/effector/syscalls/sock_shutdown.rs @@ -10,7 +10,7 @@ impl JournalEffector { ) -> anyhow::Result<()> { Self::save_event( ctx, - JournalEntry::SocketShutdown { + JournalEntry::SocketShutdownV1 { fd, how: shutdown.into(), }, diff --git a/lib/wasix/src/journal/effector/syscalls/tty_set.rs b/lib/wasix/src/journal/effector/syscalls/tty_set.rs index 76d7ef44ac0..536c5a3a2fa 100644 --- a/lib/wasix/src/journal/effector/syscalls/tty_set.rs +++ b/lib/wasix/src/journal/effector/syscalls/tty_set.rs @@ -9,7 +9,7 @@ impl JournalEffector { ) -> anyhow::Result<()> { Self::save_event( ctx, - JournalEntry::TtySet { + JournalEntry::TtySetV1 { tty: wasmer_wasix_types::wasi::Tty { cols: state.cols, rows: state.rows, diff --git a/lib/wasix/src/journal/effector/thread_exit.rs b/lib/wasix/src/journal/effector/thread_exit.rs index 4d7975a8159..4c2d70b31c6 100644 --- a/lib/wasix/src/journal/effector/thread_exit.rs +++ b/lib/wasix/src/journal/effector/thread_exit.rs @@ -9,7 +9,7 @@ impl JournalEffector { exit_code: Option, ) -> anyhow::Result<()> { env.active_journal()? - .write(JournalEntry::CloseThread { id, exit_code }) + .write(JournalEntry::CloseThreadV1 { id, exit_code }) .map_err(map_snapshot_err)?; Ok(()) } diff --git a/lib/wasix/src/journal/effector/thread_state.rs b/lib/wasix/src/journal/effector/thread_state.rs index fe2dec80a20..510a5a28582 100644 --- a/lib/wasix/src/journal/effector/thread_state.rs +++ b/lib/wasix/src/journal/effector/thread_state.rs @@ -10,7 +10,7 @@ impl JournalEffector { ) -> anyhow::Result<()> { Self::save_event( ctx, - JournalEntry::SetThread { + JournalEntry::SetThreadV1 { id, call_stack: Cow::Owned(rewind_stack.into()), memory_stack: Cow::Owned(memory_stack.into()), diff --git a/lib/wasix/src/journal/entry.rs b/lib/wasix/src/journal/entry.rs index a096ef2a920..bfc48b10d4f 100644 --- a/lib/wasix/src/journal/entry.rs +++ b/lib/wasix/src/journal/entry.rs @@ -18,6 +18,7 @@ use crate::WasiThreadId; use super::SnapshotTrigger; #[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] pub enum SocketJournalEvent { TcpListen { listen_addr: SocketAddr, @@ -40,6 +41,7 @@ pub enum SocketJournalEvent { } #[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialOrd, Ord, PartialEq, Eq, Hash)] +#[serde(rename_all = "snake_case")] pub enum SocketShutdownHow { Read, Write, @@ -65,6 +67,7 @@ impl From for Shutdown { } #[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialOrd, Ord, PartialEq, Eq, Hash)] +#[serde(rename_all = "snake_case")] pub enum SocketOptTimeType { ReadTimeout, WriteTimeout, @@ -102,19 +105,20 @@ impl From for TimeType { /// state of a WASM process at a point in time. #[allow(clippy::large_enum_variant)] #[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "snake_case")] pub enum JournalEntry<'a> { - InitModule { + InitModuleV1 { wasm_hash: [u8; 32], }, - UpdateMemoryRegion { + UpdateMemoryRegionV1 { region: Range, #[serde(with = "base64")] data: Cow<'a, [u8]>, }, - ProcessExit { + ProcessExitV1 { exit_code: Option, }, - SetThread { + SetThreadV1 { id: WasiThreadId, #[serde(with = "base64")] call_stack: Cow<'a, [u8]>, @@ -124,30 +128,30 @@ pub enum JournalEntry<'a> { store_data: Cow<'a, [u8]>, is_64bit: bool, }, - CloseThread { + CloseThreadV1 { id: WasiThreadId, exit_code: Option, }, - FileDescriptorSeek { + FileDescriptorSeekV1 { fd: Fd, offset: FileDelta, whence: Whence, }, - FileDescriptorWrite { + FileDescriptorWriteV1 { fd: Fd, offset: u64, #[serde(with = "base64")] data: Cow<'a, [u8]>, is_64bit: bool, }, - SetClockTime { + SetClockTimeV1 { clock_id: Snapshot0Clockid, time: Timestamp, }, - CloseFileDescriptor { + CloseFileDescriptorV1 { fd: Fd, }, - OpenFileDescriptor { + OpenFileDescriptorV1 { fd: Fd, dirfd: Fd, dirflags: LookupFlags, @@ -157,23 +161,23 @@ pub enum JournalEntry<'a> { fs_rights_inheriting: Rights, fs_flags: Fdflags, }, - RenumberFileDescriptor { + RenumberFileDescriptorV1 { old_fd: Fd, new_fd: Fd, }, - DuplicateFileDescriptor { + DuplicateFileDescriptorV1 { original_fd: Fd, copied_fd: Fd, }, - CreateDirectory { + CreateDirectoryV1 { fd: Fd, path: Cow<'a, str>, }, - RemoveDirectory { + RemoveDirectoryV1 { fd: Fd, path: Cow<'a, str>, }, - PathSetTimes { + PathSetTimesV1 { fd: Fd, flags: LookupFlags, path: Cow<'a, str>, @@ -181,162 +185,162 @@ pub enum JournalEntry<'a> { st_mtim: Timestamp, fst_flags: Fstflags, }, - FileDescriptorSetTimes { + FileDescriptorSetTimesV1 { fd: Fd, st_atim: Timestamp, st_mtim: Timestamp, fst_flags: Fstflags, }, - FileDescriptorSetFlags { + FileDescriptorSetFlagsV1 { fd: Fd, flags: Fdflags, }, - FileDescriptorSetRights { + FileDescriptorSetRightsV1 { fd: Fd, fs_rights_base: Rights, fs_rights_inheriting: Rights, }, - FileDescriptorSetSize { + FileDescriptorSetSizeV1 { fd: Fd, st_size: Filesize, }, - FileDescriptorAdvise { + FileDescriptorAdviseV1 { fd: Fd, offset: Filesize, len: Filesize, advice: Advice, }, - FileDescriptorAllocate { + FileDescriptorAllocateV1 { fd: Fd, offset: Filesize, len: Filesize, }, - CreateHardLink { + CreateHardLinkV1 { old_fd: Fd, old_path: Cow<'a, str>, old_flags: LookupFlags, new_fd: Fd, new_path: Cow<'a, str>, }, - CreateSymbolicLink { + CreateSymbolicLinkV1 { old_path: Cow<'a, str>, fd: Fd, new_path: Cow<'a, str>, }, - UnlinkFile { + UnlinkFileV1 { fd: Fd, path: Cow<'a, str>, }, - PathRename { + PathRenameV1 { old_fd: Fd, old_path: Cow<'a, str>, new_fd: Fd, new_path: Cow<'a, str>, }, - ChangeDirectory { + ChangeDirectoryV1 { path: Cow<'a, str>, }, - EpollCreate { + EpollCreateV1 { fd: Fd, }, - EpollCtl { + EpollCtlV1 { epfd: Fd, op: EpollCtl, fd: Fd, event: Option, }, - TtySet { + TtySetV1 { tty: Tty, line_feeds: bool, }, - CreatePipe { + CreatePipeV1 { fd1: Fd, fd2: Fd, }, - CreateEvent { + CreateEventV1 { initial_val: u64, flags: EventFdFlags, fd: Fd, }, - PortAddAddr { + PortAddAddrV1 { cidr: IpCidr, }, - PortDelAddr { + PortDelAddrV1 { addr: IpAddr, }, - PortAddrClear, - PortBridge { + PortAddrClearV1, + PortBridgeV1 { network: Cow<'a, str>, token: Cow<'a, str>, security: StreamSecurity, }, - PortUnbridge, - PortDhcpAcquire, - PortGatewaySet { + PortUnbridgeV1, + PortDhcpAcquireV1, + PortGatewaySetV1 { ip: IpAddr, }, - PortRouteAdd { + PortRouteAddV1 { cidr: IpCidr, via_router: IpAddr, preferred_until: Option, expires_at: Option, }, - PortRouteClear, - PortRouteDel { + PortRouteClearV1, + PortRouteDelV1 { ip: IpAddr, }, - SocketOpen { + SocketOpenV1 { af: Addressfamily, ty: Socktype, pt: SockProto, fd: Fd, }, - SocketListen { + SocketListenV1 { fd: Fd, backlog: u32, }, - SocketBind { + SocketBindV1 { fd: Fd, addr: SocketAddr, }, - SocketConnected { + SocketConnectedV1 { fd: Fd, addr: SocketAddr, }, - SocketAccepted { + SocketAcceptedV1 { listen_fd: Fd, fd: Fd, peer_addr: SocketAddr, fd_flags: Fdflags, - nonblocking: bool, + non_blocking: bool, }, - SocketJoinIpv4Multicast { + SocketJoinIpv4MulticastV1 { fd: Fd, multiaddr: Ipv4Addr, iface: Ipv4Addr, }, - SocketJoinIpv6Multicast { + SocketJoinIpv6MulticastV1 { fd: Fd, - multiaddr: Ipv6Addr, + multi_addr: Ipv6Addr, iface: u32, }, - SocketLeaveIpv4Multicast { + SocketLeaveIpv4MulticastV1 { fd: Fd, - multiaddr: Ipv4Addr, + multi_addr: Ipv4Addr, iface: Ipv4Addr, }, - SocketLeaveIpv6Multicast { + SocketLeaveIpv6MulticastV1 { fd: Fd, - multiaddr: Ipv6Addr, + multi_addr: Ipv6Addr, iface: u32, }, - SocketSendFile { + SocketSendFileV1 { socket_fd: Fd, file_fd: Fd, offset: Filesize, count: Filesize, }, - SocketSendTo { + SocketSendToV1 { fd: Fd, #[serde(with = "base64")] data: Cow<'a, [u8]>, @@ -344,34 +348,34 @@ pub enum JournalEntry<'a> { addr: SocketAddr, is_64bit: bool, }, - SocketSend { + SocketSendV1 { fd: Fd, #[serde(with = "base64")] data: Cow<'a, [u8]>, flags: SiFlags, is_64bit: bool, }, - SocketSetOptFlag { + SocketSetOptFlagV1 { fd: Fd, opt: Sockoption, flag: bool, }, - SocketSetOptSize { + SocketSetOptSizeV1 { fd: Fd, opt: Sockoption, size: u64, }, - SocketSetOptTime { + SocketSetOptTimeV1 { fd: Fd, ty: SocketOptTimeType, time: Option, }, - SocketShutdown { + SocketShutdownV1 { fd: Fd, how: SocketShutdownHow, }, /// Represents the marker for the end of a snapshot - Snapshot { + SnapshotV1 { when: SystemTime, trigger: SnapshotTrigger, }, @@ -380,43 +384,45 @@ pub enum JournalEntry<'a> { impl<'a> JournalEntry<'a> { pub fn into_owned(self) -> JournalEntry<'static> { match self { - Self::InitModule { wasm_hash } => JournalEntry::InitModule { wasm_hash }, - Self::UpdateMemoryRegion { region, data } => JournalEntry::UpdateMemoryRegion { + Self::InitModuleV1 { wasm_hash } => JournalEntry::InitModuleV1 { wasm_hash }, + Self::UpdateMemoryRegionV1 { region, data } => JournalEntry::UpdateMemoryRegionV1 { region, data: data.into_owned().into(), }, - Self::ProcessExit { exit_code } => JournalEntry::ProcessExit { exit_code }, - Self::SetThread { + Self::ProcessExitV1 { exit_code } => JournalEntry::ProcessExitV1 { exit_code }, + Self::SetThreadV1 { id, call_stack, memory_stack, store_data, is_64bit, - } => JournalEntry::SetThread { + } => JournalEntry::SetThreadV1 { id, call_stack: call_stack.into_owned().into(), memory_stack: memory_stack.into_owned().into(), store_data: store_data.into_owned().into(), is_64bit, }, - Self::CloseThread { id, exit_code } => JournalEntry::CloseThread { id, exit_code }, - Self::FileDescriptorSeek { fd, offset, whence } => { - JournalEntry::FileDescriptorSeek { fd, offset, whence } + Self::CloseThreadV1 { id, exit_code } => JournalEntry::CloseThreadV1 { id, exit_code }, + Self::FileDescriptorSeekV1 { fd, offset, whence } => { + JournalEntry::FileDescriptorSeekV1 { fd, offset, whence } } - Self::FileDescriptorWrite { + Self::FileDescriptorWriteV1 { fd, offset, data, is_64bit, - } => JournalEntry::FileDescriptorWrite { + } => JournalEntry::FileDescriptorWriteV1 { fd, offset, data: data.into_owned().into(), is_64bit, }, - Self::SetClockTime { clock_id, time } => JournalEntry::SetClockTime { clock_id, time }, - Self::CloseFileDescriptor { fd } => JournalEntry::CloseFileDescriptor { fd }, - Self::OpenFileDescriptor { + Self::SetClockTimeV1 { clock_id, time } => { + JournalEntry::SetClockTimeV1 { clock_id, time } + } + Self::CloseFileDescriptorV1 { fd } => JournalEntry::CloseFileDescriptorV1 { fd }, + Self::OpenFileDescriptorV1 { fd, dirfd, dirflags, @@ -425,7 +431,7 @@ impl<'a> JournalEntry<'a> { fs_rights_base, fs_rights_inheriting, fs_flags, - } => JournalEntry::OpenFileDescriptor { + } => JournalEntry::OpenFileDescriptorV1 { fd, dirfd, dirflags, @@ -435,32 +441,32 @@ impl<'a> JournalEntry<'a> { fs_rights_inheriting, fs_flags, }, - Self::RenumberFileDescriptor { old_fd, new_fd } => { - JournalEntry::RenumberFileDescriptor { old_fd, new_fd } + Self::RenumberFileDescriptorV1 { old_fd, new_fd } => { + JournalEntry::RenumberFileDescriptorV1 { old_fd, new_fd } } - Self::DuplicateFileDescriptor { + Self::DuplicateFileDescriptorV1 { original_fd, copied_fd, - } => JournalEntry::DuplicateFileDescriptor { + } => JournalEntry::DuplicateFileDescriptorV1 { original_fd, copied_fd, }, - Self::CreateDirectory { fd, path } => JournalEntry::CreateDirectory { + Self::CreateDirectoryV1 { fd, path } => JournalEntry::CreateDirectoryV1 { fd, path: path.into_owned().into(), }, - Self::RemoveDirectory { fd, path } => JournalEntry::RemoveDirectory { + Self::RemoveDirectoryV1 { fd, path } => JournalEntry::RemoveDirectoryV1 { fd, path: path.into_owned().into(), }, - Self::PathSetTimes { + Self::PathSetTimesV1 { fd, flags, path, st_atim, st_mtim, fst_flags, - } => JournalEntry::PathSetTimes { + } => JournalEntry::PathSetTimesV1 { fd, flags, path: path.into_owned().into(), @@ -468,313 +474,313 @@ impl<'a> JournalEntry<'a> { st_mtim, fst_flags, }, - Self::FileDescriptorSetTimes { + Self::FileDescriptorSetTimesV1 { fd, st_atim, st_mtim, fst_flags, - } => JournalEntry::FileDescriptorSetTimes { + } => JournalEntry::FileDescriptorSetTimesV1 { fd, st_atim, st_mtim, fst_flags, }, - Self::FileDescriptorSetFlags { fd, flags } => { - JournalEntry::FileDescriptorSetFlags { fd, flags } + Self::FileDescriptorSetFlagsV1 { fd, flags } => { + JournalEntry::FileDescriptorSetFlagsV1 { fd, flags } } - Self::FileDescriptorSetRights { + Self::FileDescriptorSetRightsV1 { fd, fs_rights_base, fs_rights_inheriting, - } => JournalEntry::FileDescriptorSetRights { + } => JournalEntry::FileDescriptorSetRightsV1 { fd, fs_rights_base, fs_rights_inheriting, }, - Self::FileDescriptorSetSize { fd, st_size } => { - JournalEntry::FileDescriptorSetSize { fd, st_size } + Self::FileDescriptorSetSizeV1 { fd, st_size } => { + JournalEntry::FileDescriptorSetSizeV1 { fd, st_size } } - Self::FileDescriptorAdvise { + Self::FileDescriptorAdviseV1 { fd, offset, len, advice, - } => JournalEntry::FileDescriptorAdvise { + } => JournalEntry::FileDescriptorAdviseV1 { fd, offset, len, advice, }, - Self::FileDescriptorAllocate { fd, offset, len } => { - JournalEntry::FileDescriptorAllocate { fd, offset, len } + Self::FileDescriptorAllocateV1 { fd, offset, len } => { + JournalEntry::FileDescriptorAllocateV1 { fd, offset, len } } - Self::CreateHardLink { + Self::CreateHardLinkV1 { old_fd, old_path, old_flags, new_fd, new_path, - } => JournalEntry::CreateHardLink { + } => JournalEntry::CreateHardLinkV1 { old_fd, old_path: old_path.into_owned().into(), old_flags, new_fd, new_path: new_path.into_owned().into(), }, - Self::CreateSymbolicLink { + Self::CreateSymbolicLinkV1 { old_path, fd, new_path, - } => JournalEntry::CreateSymbolicLink { + } => JournalEntry::CreateSymbolicLinkV1 { old_path: old_path.into_owned().into(), fd, new_path: new_path.into_owned().into(), }, - Self::UnlinkFile { fd, path } => JournalEntry::UnlinkFile { + Self::UnlinkFileV1 { fd, path } => JournalEntry::UnlinkFileV1 { fd, path: path.into_owned().into(), }, - Self::PathRename { + Self::PathRenameV1 { old_fd, old_path, new_fd, new_path, - } => JournalEntry::PathRename { + } => JournalEntry::PathRenameV1 { old_fd, old_path: old_path.into_owned().into(), new_fd, new_path: new_path.into_owned().into(), }, - Self::ChangeDirectory { path } => JournalEntry::ChangeDirectory { + Self::ChangeDirectoryV1 { path } => JournalEntry::ChangeDirectoryV1 { path: path.into_owned().into(), }, - Self::EpollCreate { fd } => JournalEntry::EpollCreate { fd }, - Self::EpollCtl { + Self::EpollCreateV1 { fd } => JournalEntry::EpollCreateV1 { fd }, + Self::EpollCtlV1 { epfd, op, fd, event, - } => JournalEntry::EpollCtl { + } => JournalEntry::EpollCtlV1 { epfd, op, fd, event, }, - Self::TtySet { tty, line_feeds } => JournalEntry::TtySet { tty, line_feeds }, - Self::CreatePipe { fd1, fd2 } => JournalEntry::CreatePipe { fd1, fd2 }, - Self::CreateEvent { + Self::TtySetV1 { tty, line_feeds } => JournalEntry::TtySetV1 { tty, line_feeds }, + Self::CreatePipeV1 { fd1, fd2 } => JournalEntry::CreatePipeV1 { fd1, fd2 }, + Self::CreateEventV1 { initial_val, flags, fd, - } => JournalEntry::CreateEvent { + } => JournalEntry::CreateEventV1 { initial_val, flags, fd, }, - Self::PortAddAddr { cidr } => JournalEntry::PortAddAddr { cidr }, - Self::PortDelAddr { addr } => JournalEntry::PortDelAddr { addr }, - Self::PortAddrClear => JournalEntry::PortAddrClear, - Self::PortBridge { + Self::PortAddAddrV1 { cidr } => JournalEntry::PortAddAddrV1 { cidr }, + Self::PortDelAddrV1 { addr } => JournalEntry::PortDelAddrV1 { addr }, + Self::PortAddrClearV1 => JournalEntry::PortAddrClearV1, + Self::PortBridgeV1 { network, token, security, - } => JournalEntry::PortBridge { + } => JournalEntry::PortBridgeV1 { network: network.into_owned().into(), token: token.into_owned().into(), security, }, - Self::PortUnbridge => JournalEntry::PortUnbridge, - Self::PortDhcpAcquire => JournalEntry::PortDhcpAcquire, - Self::PortGatewaySet { ip } => JournalEntry::PortGatewaySet { ip }, - Self::PortRouteAdd { + Self::PortUnbridgeV1 => JournalEntry::PortUnbridgeV1, + Self::PortDhcpAcquireV1 => JournalEntry::PortDhcpAcquireV1, + Self::PortGatewaySetV1 { ip } => JournalEntry::PortGatewaySetV1 { ip }, + Self::PortRouteAddV1 { cidr, via_router, preferred_until, expires_at, - } => JournalEntry::PortRouteAdd { + } => JournalEntry::PortRouteAddV1 { cidr, via_router, preferred_until, expires_at, }, - Self::PortRouteClear => JournalEntry::PortRouteClear, - Self::PortRouteDel { ip } => JournalEntry::PortRouteDel { ip }, - Self::SocketOpen { af, ty, pt, fd } => JournalEntry::SocketOpen { af, ty, pt, fd }, - Self::SocketListen { fd, backlog } => JournalEntry::SocketListen { fd, backlog }, - Self::SocketBind { fd, addr } => JournalEntry::SocketBind { fd, addr }, - Self::SocketConnected { fd, addr } => JournalEntry::SocketConnected { fd, addr }, - Self::SocketAccepted { + Self::PortRouteClearV1 => JournalEntry::PortRouteClearV1, + Self::PortRouteDelV1 { ip } => JournalEntry::PortRouteDelV1 { ip }, + Self::SocketOpenV1 { af, ty, pt, fd } => JournalEntry::SocketOpenV1 { af, ty, pt, fd }, + Self::SocketListenV1 { fd, backlog } => JournalEntry::SocketListenV1 { fd, backlog }, + Self::SocketBindV1 { fd, addr } => JournalEntry::SocketBindV1 { fd, addr }, + Self::SocketConnectedV1 { fd, addr } => JournalEntry::SocketConnectedV1 { fd, addr }, + Self::SocketAcceptedV1 { listen_fd, fd, peer_addr, fd_flags, - nonblocking, - } => JournalEntry::SocketAccepted { + non_blocking: nonblocking, + } => JournalEntry::SocketAcceptedV1 { listen_fd, fd, peer_addr, fd_flags, - nonblocking, + non_blocking: nonblocking, }, - Self::SocketJoinIpv4Multicast { + Self::SocketJoinIpv4MulticastV1 { fd, multiaddr, iface, - } => JournalEntry::SocketJoinIpv4Multicast { + } => JournalEntry::SocketJoinIpv4MulticastV1 { fd, multiaddr, iface, }, - Self::SocketJoinIpv6Multicast { + Self::SocketJoinIpv6MulticastV1 { fd, - multiaddr, + multi_addr: multiaddr, iface, - } => JournalEntry::SocketJoinIpv6Multicast { + } => JournalEntry::SocketJoinIpv6MulticastV1 { fd, - multiaddr, + multi_addr: multiaddr, iface, }, - Self::SocketLeaveIpv4Multicast { + Self::SocketLeaveIpv4MulticastV1 { fd, - multiaddr, + multi_addr: multiaddr, iface, - } => JournalEntry::SocketLeaveIpv4Multicast { + } => JournalEntry::SocketLeaveIpv4MulticastV1 { fd, - multiaddr, + multi_addr: multiaddr, iface, }, - Self::SocketLeaveIpv6Multicast { + Self::SocketLeaveIpv6MulticastV1 { fd, - multiaddr, + multi_addr: multiaddr, iface, - } => JournalEntry::SocketLeaveIpv6Multicast { + } => JournalEntry::SocketLeaveIpv6MulticastV1 { fd, - multiaddr, + multi_addr: multiaddr, iface, }, - Self::SocketSendFile { + Self::SocketSendFileV1 { socket_fd, file_fd, offset, count, - } => JournalEntry::SocketSendFile { + } => JournalEntry::SocketSendFileV1 { socket_fd, file_fd, offset, count, }, - Self::SocketSendTo { + Self::SocketSendToV1 { fd, data, flags, addr, is_64bit, - } => JournalEntry::SocketSendTo { + } => JournalEntry::SocketSendToV1 { fd, data: data.into_owned().into(), flags, addr, is_64bit, }, - Self::SocketSend { + Self::SocketSendV1 { fd, data, flags, is_64bit, - } => JournalEntry::SocketSend { + } => JournalEntry::SocketSendV1 { fd, data: data.into_owned().into(), flags, is_64bit, }, - Self::SocketSetOptFlag { fd, opt, flag } => { - JournalEntry::SocketSetOptFlag { fd, opt, flag } + Self::SocketSetOptFlagV1 { fd, opt, flag } => { + JournalEntry::SocketSetOptFlagV1 { fd, opt, flag } } - Self::SocketSetOptSize { fd, opt, size } => { - JournalEntry::SocketSetOptSize { fd, opt, size } + Self::SocketSetOptSizeV1 { fd, opt, size } => { + JournalEntry::SocketSetOptSizeV1 { fd, opt, size } } - Self::SocketSetOptTime { fd, ty, time } => { - JournalEntry::SocketSetOptTime { fd, ty, time } + Self::SocketSetOptTimeV1 { fd, ty, time } => { + JournalEntry::SocketSetOptTimeV1 { fd, ty, time } } - Self::SocketShutdown { fd, how } => JournalEntry::SocketShutdown { fd, how }, - Self::Snapshot { when, trigger } => JournalEntry::Snapshot { when, trigger }, + Self::SocketShutdownV1 { fd, how } => JournalEntry::SocketShutdownV1 { fd, how }, + Self::SnapshotV1 { when, trigger } => JournalEntry::SnapshotV1 { when, trigger }, } } pub fn estimate_size(&self) -> usize { let base_size = std::mem::size_of_val(self); match self { - JournalEntry::InitModule { .. } => base_size, - JournalEntry::UpdateMemoryRegion { data, .. } => base_size + data.len(), - JournalEntry::ProcessExit { .. } => base_size, - JournalEntry::SetThread { + JournalEntry::InitModuleV1 { .. } => base_size, + JournalEntry::UpdateMemoryRegionV1 { data, .. } => base_size + data.len(), + JournalEntry::ProcessExitV1 { .. } => base_size, + JournalEntry::SetThreadV1 { call_stack, memory_stack, store_data, .. } => base_size + call_stack.len() + memory_stack.len() + store_data.len(), - JournalEntry::CloseThread { .. } => base_size, - JournalEntry::FileDescriptorSeek { .. } => base_size, - JournalEntry::FileDescriptorWrite { data, .. } => base_size + data.len(), - JournalEntry::SetClockTime { .. } => base_size, - JournalEntry::CloseFileDescriptor { .. } => base_size, - JournalEntry::OpenFileDescriptor { path, .. } => base_size + path.as_bytes().len(), - JournalEntry::RenumberFileDescriptor { .. } => base_size, - JournalEntry::DuplicateFileDescriptor { .. } => base_size, - JournalEntry::CreateDirectory { path, .. } => base_size + path.as_bytes().len(), - JournalEntry::RemoveDirectory { path, .. } => base_size + path.as_bytes().len(), - JournalEntry::PathSetTimes { path, .. } => base_size + path.as_bytes().len(), - JournalEntry::FileDescriptorSetTimes { .. } => base_size, - JournalEntry::FileDescriptorSetFlags { .. } => base_size, - JournalEntry::FileDescriptorSetRights { .. } => base_size, - JournalEntry::FileDescriptorSetSize { .. } => base_size, - JournalEntry::FileDescriptorAdvise { .. } => base_size, - JournalEntry::FileDescriptorAllocate { .. } => base_size, - JournalEntry::CreateHardLink { + JournalEntry::CloseThreadV1 { .. } => base_size, + JournalEntry::FileDescriptorSeekV1 { .. } => base_size, + JournalEntry::FileDescriptorWriteV1 { data, .. } => base_size + data.len(), + JournalEntry::SetClockTimeV1 { .. } => base_size, + JournalEntry::CloseFileDescriptorV1 { .. } => base_size, + JournalEntry::OpenFileDescriptorV1 { path, .. } => base_size + path.as_bytes().len(), + JournalEntry::RenumberFileDescriptorV1 { .. } => base_size, + JournalEntry::DuplicateFileDescriptorV1 { .. } => base_size, + JournalEntry::CreateDirectoryV1 { path, .. } => base_size + path.as_bytes().len(), + JournalEntry::RemoveDirectoryV1 { path, .. } => base_size + path.as_bytes().len(), + JournalEntry::PathSetTimesV1 { path, .. } => base_size + path.as_bytes().len(), + JournalEntry::FileDescriptorSetTimesV1 { .. } => base_size, + JournalEntry::FileDescriptorSetFlagsV1 { .. } => base_size, + JournalEntry::FileDescriptorSetRightsV1 { .. } => base_size, + JournalEntry::FileDescriptorSetSizeV1 { .. } => base_size, + JournalEntry::FileDescriptorAdviseV1 { .. } => base_size, + JournalEntry::FileDescriptorAllocateV1 { .. } => base_size, + JournalEntry::CreateHardLinkV1 { old_path, new_path, .. } => base_size + old_path.as_bytes().len() + new_path.as_bytes().len(), - JournalEntry::CreateSymbolicLink { + JournalEntry::CreateSymbolicLinkV1 { old_path, new_path, .. } => base_size + old_path.as_bytes().len() + new_path.as_bytes().len(), - JournalEntry::UnlinkFile { path, .. } => base_size + path.as_bytes().len(), - JournalEntry::PathRename { + JournalEntry::UnlinkFileV1 { path, .. } => base_size + path.as_bytes().len(), + JournalEntry::PathRenameV1 { old_path, new_path, .. } => base_size + old_path.as_bytes().len() + new_path.as_bytes().len(), - JournalEntry::ChangeDirectory { path } => base_size + path.as_bytes().len(), - JournalEntry::EpollCreate { .. } => base_size, - JournalEntry::EpollCtl { .. } => base_size, - JournalEntry::TtySet { .. } => base_size, - JournalEntry::CreatePipe { .. } => base_size, - JournalEntry::CreateEvent { .. } => base_size, - JournalEntry::PortAddAddr { .. } => base_size, - JournalEntry::PortDelAddr { .. } => base_size, - JournalEntry::PortAddrClear => base_size, - JournalEntry::PortBridge { network, token, .. } => { + JournalEntry::ChangeDirectoryV1 { path } => base_size + path.as_bytes().len(), + JournalEntry::EpollCreateV1 { .. } => base_size, + JournalEntry::EpollCtlV1 { .. } => base_size, + JournalEntry::TtySetV1 { .. } => base_size, + JournalEntry::CreatePipeV1 { .. } => base_size, + JournalEntry::CreateEventV1 { .. } => base_size, + JournalEntry::PortAddAddrV1 { .. } => base_size, + JournalEntry::PortDelAddrV1 { .. } => base_size, + JournalEntry::PortAddrClearV1 => base_size, + JournalEntry::PortBridgeV1 { network, token, .. } => { base_size + network.as_bytes().len() + token.as_bytes().len() } - JournalEntry::PortUnbridge => base_size, - JournalEntry::PortDhcpAcquire => base_size, - JournalEntry::PortGatewaySet { .. } => base_size, - JournalEntry::PortRouteAdd { .. } => base_size, - JournalEntry::PortRouteClear => base_size, - JournalEntry::PortRouteDel { .. } => base_size, - JournalEntry::SocketOpen { .. } => base_size, - JournalEntry::SocketListen { .. } => base_size, - JournalEntry::SocketBind { .. } => base_size, - JournalEntry::SocketConnected { .. } => base_size, - JournalEntry::SocketAccepted { .. } => base_size, - JournalEntry::SocketJoinIpv4Multicast { .. } => base_size, - JournalEntry::SocketJoinIpv6Multicast { .. } => base_size, - JournalEntry::SocketLeaveIpv4Multicast { .. } => base_size, - JournalEntry::SocketLeaveIpv6Multicast { .. } => base_size, - JournalEntry::SocketSendFile { .. } => base_size, - JournalEntry::SocketSendTo { data, .. } => base_size + data.len(), - JournalEntry::SocketSend { data, .. } => base_size + data.len(), - JournalEntry::SocketSetOptFlag { .. } => base_size, - JournalEntry::SocketSetOptSize { .. } => base_size, - JournalEntry::SocketSetOptTime { .. } => base_size, - JournalEntry::SocketShutdown { .. } => base_size, - JournalEntry::Snapshot { .. } => base_size, + JournalEntry::PortUnbridgeV1 => base_size, + JournalEntry::PortDhcpAcquireV1 => base_size, + JournalEntry::PortGatewaySetV1 { .. } => base_size, + JournalEntry::PortRouteAddV1 { .. } => base_size, + JournalEntry::PortRouteClearV1 => base_size, + JournalEntry::PortRouteDelV1 { .. } => base_size, + JournalEntry::SocketOpenV1 { .. } => base_size, + JournalEntry::SocketListenV1 { .. } => base_size, + JournalEntry::SocketBindV1 { .. } => base_size, + JournalEntry::SocketConnectedV1 { .. } => base_size, + JournalEntry::SocketAcceptedV1 { .. } => base_size, + JournalEntry::SocketJoinIpv4MulticastV1 { .. } => base_size, + JournalEntry::SocketJoinIpv6MulticastV1 { .. } => base_size, + JournalEntry::SocketLeaveIpv4MulticastV1 { .. } => base_size, + JournalEntry::SocketLeaveIpv6MulticastV1 { .. } => base_size, + JournalEntry::SocketSendFileV1 { .. } => base_size, + JournalEntry::SocketSendToV1 { data, .. } => base_size + data.len(), + JournalEntry::SocketSendV1 { data, .. } => base_size + data.len(), + JournalEntry::SocketSetOptFlagV1 { .. } => base_size, + JournalEntry::SocketSetOptSizeV1 { .. } => base_size, + JournalEntry::SocketSetOptTimeV1 { .. } => base_size, + JournalEntry::SocketShutdownV1 { .. } => base_size, + JournalEntry::SnapshotV1 { .. } => base_size, } } } diff --git a/lib/wasix/src/state/func_env.rs b/lib/wasix/src/state/func_env.rs index d246e47e5c0..83febd3a8b8 100644 --- a/lib/wasix/src/state/func_env.rs +++ b/lib/wasix/src/state/func_env.rs @@ -281,7 +281,7 @@ impl WasiFunctionEnv { let mut ctx = self.env.clone().into_mut(&mut store); crate::journal::JournalEffector::save_event( &mut ctx, - crate::journal::JournalEntry::InitModule { wasm_hash }, + crate::journal::JournalEntry::InitModuleV1 { wasm_hash }, ) .map_err(|err| { WasiRuntimeError::Runtime(RuntimeError::new(format!( diff --git a/lib/wasix/src/syscalls/journal.rs b/lib/wasix/src/syscalls/journal.rs new file mode 100644 index 00000000000..a09c8b8aee2 --- /dev/null +++ b/lib/wasix/src/syscalls/journal.rs @@ -0,0 +1,660 @@ +use super::*; + +#[allow(clippy::extra_unused_type_parameters)] +#[cfg(not(feature = "journal"))] +pub fn maybe_snapshot_once( + ctx: FunctionEnvMut<'_, WasiEnv>, + _trigger: crate::journal::SnapshotTrigger, +) -> WasiResult> { + Ok(Ok(ctx)) +} + +#[cfg(feature = "journal")] +pub fn maybe_snapshot_once( + mut ctx: FunctionEnvMut<'_, WasiEnv>, + trigger: crate::journal::SnapshotTrigger, +) -> WasiResult> { + use crate::os::task::process::{WasiProcessCheckpoint, WasiProcessInner}; + + unsafe { handle_rewind_ext::(&mut ctx, HandleRewindType::Resultless) }; + + if !ctx.data().enable_journal { + return Ok(Ok(ctx)); + } + + if ctx.data_mut().pop_snapshot_trigger(trigger) { + let inner = ctx.data().process.inner.clone(); + let res = wasi_try_ok_ok!(WasiProcessInner::checkpoint::( + inner, + ctx, + WasiProcessCheckpoint::Snapshot { trigger }, + )?); + match res { + MaybeCheckpointResult::Unwinding => return Ok(Err(Errno::Success)), + MaybeCheckpointResult::NotThisTime(c) => { + ctx = c; + } + } + } + Ok(Ok(ctx)) +} + +#[allow(clippy::extra_unused_type_parameters)] +#[cfg(not(feature = "journal"))] +pub fn maybe_snapshot( + ctx: FunctionEnvMut<'_, WasiEnv>, +) -> WasiResult> { + Ok(Ok(ctx)) +} + +#[cfg(feature = "journal")] +pub fn maybe_snapshot( + mut ctx: FunctionEnvMut<'_, WasiEnv>, +) -> WasiResult> { + use crate::os::task::process::{WasiProcessCheckpoint, WasiProcessInner}; + + if !ctx.data().enable_journal { + return Ok(Ok(ctx)); + } + + let inner = ctx.data().process.inner.clone(); + let res = wasi_try_ok_ok!(WasiProcessInner::maybe_checkpoint::(inner, ctx)?); + match res { + MaybeCheckpointResult::Unwinding => return Ok(Err(Errno::Success)), + MaybeCheckpointResult::NotThisTime(c) => { + ctx = c; + } + } + Ok(Ok(ctx)) +} + +/// Safety: This function manipulates the memory of the process and thus must +/// be executed by the WASM process thread itself. +/// +#[allow(clippy::result_large_err)] +#[cfg(feature = "journal")] +pub unsafe fn restore_snapshot( + mut ctx: FunctionEnvMut<'_, WasiEnv>, + journal: Arc, + bootstrapping: bool, +) -> Result, WasiRuntimeError> { + use std::ops::Range; + + use crate::journal::Journal; + + // We delay the spawning of threads until the end as its + // possible that the threads will be cancelled before all the + // events finished the streaming process + let mut spawn_threads: HashMap = Default::default(); + + // We delay the memory updates until the end as its possible the + // memory will be cleared before all the events finished the + // streaming process + let mut update_memory: HashMap, Cow<'_, [u8]>> = Default::default(); + let mut update_tty = None; + + // We capture the stdout and stderr while we replay + let mut stdout = Vec::new(); + let mut stderr = Vec::new(); + let mut stdout_fds = HashSet::new(); + let mut stderr_fds = HashSet::new(); + stdout_fds.insert(1 as WasiFd); + stderr_fds.insert(2 as WasiFd); + + // Loop through all the events and process them + let cur_module_hash = Some(ctx.data().process.module_hash.as_bytes()); + let mut journal_module_hash = None; + let mut rewind = None; + while let Some(next) = journal.read().map_err(anyhow_err_to_runtime_err)? { + tracing::trace!("Restoring snapshot event - {next:?}"); + match next { + crate::journal::JournalEntry::InitModuleV1 { wasm_hash } => { + journal_module_hash.replace(wasm_hash); + } + crate::journal::JournalEntry::ProcessExitV1 { exit_code } => { + if bootstrapping { + rewind = None; + spawn_threads.clear(); + update_memory.clear(); + update_tty.take(); + stdout.clear(); + stderr.clear(); + stdout_fds.clear(); + stderr_fds.clear(); + stdout_fds.insert(1 as WasiFd); + stderr_fds.insert(2 as WasiFd); + } else { + JournalEffector::apply_process_exit(&mut ctx, exit_code) + .map_err(anyhow_err_to_runtime_err)?; + } + } + crate::journal::JournalEntry::FileDescriptorWriteV1 { + fd, + offset, + data, + is_64bit, + } => { + if stdout_fds.contains(&fd) { + stdout.push((offset, data, is_64bit)); + continue; + } + if stderr_fds.contains(&fd) { + stderr.push((offset, data, is_64bit)); + continue; + } + + if is_64bit { + JournalEffector::apply_fd_write::(&ctx, fd, offset, data) + } else { + JournalEffector::apply_fd_write::(&ctx, fd, offset, data) + } + .map_err(anyhow_err_to_runtime_err)?; + } + crate::journal::JournalEntry::FileDescriptorSeekV1 { fd, offset, whence } => { + JournalEffector::apply_fd_seek(&mut ctx, fd, offset, whence) + .map_err(anyhow_err_to_runtime_err)?; + } + crate::journal::JournalEntry::UpdateMemoryRegionV1 { region, data } => { + if cur_module_hash != journal_module_hash { + continue; + } + + if bootstrapping { + update_memory.insert(region, data.clone()); + } else { + JournalEffector::apply_memory(&mut ctx, region, &data) + .map_err(anyhow_err_to_runtime_err)?; + } + } + crate::journal::JournalEntry::CloseThreadV1 { id, exit_code } => { + if id == ctx.data().tid() { + if bootstrapping { + rewind = None; + spawn_threads.clear(); + update_memory.clear(); + update_tty.take(); + stdout.clear(); + stderr.clear(); + stdout_fds.clear(); + stderr_fds.clear(); + stdout_fds.insert(1 as WasiFd); + stderr_fds.insert(2 as WasiFd); + } else { + JournalEffector::apply_process_exit(&mut ctx, exit_code) + .map_err(anyhow_err_to_runtime_err)?; + } + } else if bootstrapping { + spawn_threads.remove(&id); + } else { + JournalEffector::apply_thread_exit(&mut ctx, id, exit_code) + .map_err(anyhow_err_to_runtime_err)?; + } + } + crate::journal::JournalEntry::SetThreadV1 { + id, + call_stack, + memory_stack, + store_data, + is_64bit, + } => { + if cur_module_hash != journal_module_hash { + continue; + } + + let state = RewindState { + memory_stack: memory_stack.to_vec().into(), + rewind_stack: call_stack.to_vec().into(), + store_data: store_data.to_vec().into(), + is_64bit, + }; + + if id == ctx.data().tid() { + rewind.replace(state); + } else if bootstrapping { + spawn_threads.insert(id, state); + } else { + return Err(WasiRuntimeError::Runtime(RuntimeError::user( + anyhow::format_err!( + "Snapshot restoration does not currently support live updates of running threads." + ) + .into(), + ))); + } + } + crate::journal::JournalEntry::CloseFileDescriptorV1 { fd } => { + stdout_fds.remove(&fd); + stderr_fds.remove(&fd); + JournalEffector::apply_fd_close(&mut ctx, fd).map_err(anyhow_err_to_runtime_err)?; + } + crate::journal::JournalEntry::OpenFileDescriptorV1 { + fd, + dirfd, + dirflags, + path, + o_flags, + fs_rights_base, + fs_rights_inheriting, + fs_flags, + } => { + JournalEffector::apply_path_open( + &mut ctx, + fd, + dirfd, + dirflags, + &path, + o_flags, + fs_rights_base, + fs_rights_inheriting, + fs_flags, + ) + .map_err(anyhow_err_to_runtime_err)?; + } + crate::journal::JournalEntry::RemoveDirectoryV1 { fd, path } => { + JournalEffector::apply_path_remove_directory(&mut ctx, fd, &path) + .map_err(anyhow_err_to_runtime_err)?; + } + crate::journal::JournalEntry::UnlinkFileV1 { fd, path } => { + JournalEffector::apply_path_unlink(&mut ctx, fd, &path) + .map_err(anyhow_err_to_runtime_err)?; + } + crate::journal::JournalEntry::PathRenameV1 { + old_fd, + old_path, + new_fd, + new_path, + } => { + JournalEffector::apply_path_rename(&mut ctx, old_fd, &old_path, new_fd, &new_path) + .map_err(anyhow_err_to_runtime_err)?; + } + crate::journal::JournalEntry::SnapshotV1 { when: _, trigger } => { + if cur_module_hash != journal_module_hash { + continue; + } + ctx.data_mut().pop_snapshot_trigger(trigger); + } + crate::journal::JournalEntry::SetClockTimeV1 { clock_id, time } => { + JournalEffector::apply_clock_time_set(&mut ctx, clock_id, time) + .map_err(anyhow_err_to_runtime_err)?; + } + crate::journal::JournalEntry::RenumberFileDescriptorV1 { old_fd, new_fd } => { + if old_fd != new_fd { + stdout_fds.remove(&new_fd); + stderr_fds.remove(&new_fd); + } + if stdout_fds.remove(&old_fd) { + stdout_fds.insert(new_fd); + } + if stderr_fds.remove(&old_fd) { + stderr_fds.insert(new_fd); + } + JournalEffector::apply_fd_renumber(&mut ctx, old_fd, new_fd) + .map_err(anyhow_err_to_runtime_err)?; + } + crate::journal::JournalEntry::DuplicateFileDescriptorV1 { + original_fd, + copied_fd, + } => { + if original_fd != copied_fd { + stdout_fds.remove(&copied_fd); + stderr_fds.remove(&copied_fd); + } + if stdout_fds.contains(&original_fd) { + stdout_fds.insert(copied_fd); + } + if stderr_fds.contains(&original_fd) { + stderr_fds.insert(copied_fd); + } + JournalEffector::apply_fd_duplicate(&mut ctx, original_fd, copied_fd) + .map_err(anyhow_err_to_runtime_err)?; + } + crate::journal::JournalEntry::CreateDirectoryV1 { fd, path } => { + JournalEffector::apply_path_create_directory(&mut ctx, fd, &path) + .map_err(anyhow_err_to_runtime_err)?; + } + crate::journal::JournalEntry::PathSetTimesV1 { + fd, + flags, + path, + st_atim, + st_mtim, + fst_flags, + } => { + JournalEffector::apply_path_set_times( + &mut ctx, fd, flags, &path, st_atim, st_mtim, fst_flags, + ) + .map_err(anyhow_err_to_runtime_err)?; + } + crate::journal::JournalEntry::FileDescriptorSetTimesV1 { + fd, + st_atim, + st_mtim, + fst_flags, + } => { + JournalEffector::apply_fd_set_times(&mut ctx, fd, st_atim, st_mtim, fst_flags) + .map_err(anyhow_err_to_runtime_err)?; + } + crate::journal::JournalEntry::FileDescriptorSetSizeV1 { fd, st_size } => { + JournalEffector::apply_fd_set_size(&mut ctx, fd, st_size) + .map_err(anyhow_err_to_runtime_err)?; + } + crate::journal::JournalEntry::FileDescriptorSetFlagsV1 { fd, flags } => { + JournalEffector::apply_fd_set_flags(&mut ctx, fd, flags) + .map_err(anyhow_err_to_runtime_err)?; + } + crate::journal::JournalEntry::FileDescriptorSetRightsV1 { + fd, + fs_rights_base, + fs_rights_inheriting, + } => { + JournalEffector::apply_fd_set_rights( + &mut ctx, + fd, + fs_rights_base, + fs_rights_inheriting, + ) + .map_err(anyhow_err_to_runtime_err)?; + } + crate::journal::JournalEntry::FileDescriptorAdviseV1 { + fd, + offset, + len, + advice, + } => { + JournalEffector::apply_fd_advise(&mut ctx, fd, offset, len, advice) + .map_err(anyhow_err_to_runtime_err)?; + } + crate::journal::JournalEntry::FileDescriptorAllocateV1 { fd, offset, len } => { + JournalEffector::apply_fd_allocate(&mut ctx, fd, offset, len) + .map_err(anyhow_err_to_runtime_err)?; + } + crate::journal::JournalEntry::CreateHardLinkV1 { + old_fd, + old_path, + old_flags, + new_fd, + new_path, + } => { + JournalEffector::apply_path_link( + &mut ctx, old_fd, old_flags, &old_path, new_fd, &new_path, + ) + .map_err(anyhow_err_to_runtime_err)?; + } + crate::journal::JournalEntry::CreateSymbolicLinkV1 { + old_path, + fd, + new_path, + } => { + JournalEffector::apply_path_symlink(&mut ctx, &old_path, fd, &new_path) + .map_err(anyhow_err_to_runtime_err)?; + } + crate::journal::JournalEntry::ChangeDirectoryV1 { path } => { + JournalEffector::apply_chdir(&mut ctx, &path).map_err(anyhow_err_to_runtime_err)?; + } + crate::journal::JournalEntry::CreatePipeV1 { fd1, fd2 } => { + JournalEffector::apply_fd_pipe(&mut ctx, fd1, fd2) + .map_err(anyhow_err_to_runtime_err)?; + } + crate::journal::JournalEntry::EpollCreateV1 { fd } => { + JournalEffector::apply_epoll_create(&mut ctx, fd) + .map_err(anyhow_err_to_runtime_err)?; + } + crate::journal::JournalEntry::EpollCtlV1 { + epfd, + op, + fd, + event, + } => { + JournalEffector::apply_epoll_ctl(&mut ctx, epfd, op, fd, event) + .map_err(anyhow_err_to_runtime_err)?; + } + crate::journal::JournalEntry::TtySetV1 { tty, line_feeds } => { + let state = crate::WasiTtyState { + cols: tty.cols, + rows: tty.rows, + width: tty.width, + height: tty.height, + stdin_tty: tty.stdin_tty, + stdout_tty: tty.stdout_tty, + stderr_tty: tty.stderr_tty, + echo: tty.echo, + line_buffered: tty.line_buffered, + line_feeds, + }; + + if bootstrapping { + update_tty.replace(state); + } else { + JournalEffector::apply_tty_set(&mut ctx, state) + .map_err(anyhow_err_to_runtime_err)?; + } + } + crate::journal::JournalEntry::PortAddAddrV1 { cidr } => { + JournalEffector::apply_port_addr_add(&mut ctx, cidr) + .map_err(anyhow_err_to_runtime_err)? + } + crate::journal::JournalEntry::PortDelAddrV1 { addr } => { + JournalEffector::apply_port_addr_remove(&mut ctx, addr) + .map_err(anyhow_err_to_runtime_err)? + } + crate::journal::JournalEntry::PortAddrClearV1 => { + JournalEffector::apply_port_addr_clear(&mut ctx) + .map_err(anyhow_err_to_runtime_err)? + } + crate::journal::JournalEntry::PortBridgeV1 { + network, + token, + security, + } => JournalEffector::apply_port_bridge(&mut ctx, &network, &token, security) + .map_err(anyhow_err_to_runtime_err)?, + crate::journal::JournalEntry::PortUnbridgeV1 => { + JournalEffector::apply_port_unbridge(&mut ctx).map_err(anyhow_err_to_runtime_err)? + } + crate::journal::JournalEntry::PortDhcpAcquireV1 => { + JournalEffector::apply_port_dhcp_acquire(&mut ctx) + .map_err(anyhow_err_to_runtime_err)? + } + crate::journal::JournalEntry::PortGatewaySetV1 { ip } => { + JournalEffector::apply_port_gateway_set(&mut ctx, ip) + .map_err(anyhow_err_to_runtime_err)? + } + crate::journal::JournalEntry::PortRouteAddV1 { + cidr, + via_router, + preferred_until, + expires_at, + } => JournalEffector::apply_port_route_add( + &mut ctx, + cidr, + via_router, + preferred_until, + expires_at, + ) + .map_err(anyhow_err_to_runtime_err)?, + crate::journal::JournalEntry::PortRouteClearV1 => { + JournalEffector::apply_port_route_clear(&mut ctx) + .map_err(anyhow_err_to_runtime_err)? + } + crate::journal::JournalEntry::PortRouteDelV1 { ip } => { + JournalEffector::apply_port_route_remove(&mut ctx, ip) + .map_err(anyhow_err_to_runtime_err)? + } + crate::journal::JournalEntry::SocketOpenV1 { af, ty, pt, fd } => { + JournalEffector::apply_sock_open(&mut ctx, af, ty, pt, fd) + .map_err(anyhow_err_to_runtime_err)? + } + crate::journal::JournalEntry::SocketListenV1 { fd, backlog } => { + JournalEffector::apply_sock_listen(&mut ctx, fd, backlog as usize) + .map_err(anyhow_err_to_runtime_err)? + } + crate::journal::JournalEntry::SocketBindV1 { fd, addr } => { + JournalEffector::apply_sock_bind(&mut ctx, fd, addr) + .map_err(anyhow_err_to_runtime_err)? + } + crate::journal::JournalEntry::SocketConnectedV1 { fd, addr } => { + JournalEffector::apply_sock_connect(&mut ctx, fd, addr) + .map_err(anyhow_err_to_runtime_err)? + } + crate::journal::JournalEntry::SocketAcceptedV1 { + listen_fd, + fd, + peer_addr, + fd_flags, + non_blocking: nonblocking, + } => JournalEffector::apply_sock_accepted( + &mut ctx, + listen_fd, + fd, + peer_addr, + fd_flags, + nonblocking, + ) + .map_err(anyhow_err_to_runtime_err)?, + crate::journal::JournalEntry::SocketJoinIpv4MulticastV1 { + fd, + multiaddr, + iface, + } => JournalEffector::apply_sock_join_ipv4_multicast(&mut ctx, fd, multiaddr, iface) + .map_err(anyhow_err_to_runtime_err)?, + crate::journal::JournalEntry::SocketJoinIpv6MulticastV1 { + fd, + multi_addr: multiaddr, + iface, + } => JournalEffector::apply_sock_join_ipv6_multicast(&mut ctx, fd, multiaddr, iface) + .map_err(anyhow_err_to_runtime_err)?, + crate::journal::JournalEntry::SocketLeaveIpv4MulticastV1 { + fd, + multi_addr: multiaddr, + iface, + } => JournalEffector::apply_sock_leave_ipv4_multicast(&mut ctx, fd, multiaddr, iface) + .map_err(anyhow_err_to_runtime_err)?, + crate::journal::JournalEntry::SocketLeaveIpv6MulticastV1 { + fd, + multi_addr: multiaddr, + iface, + } => JournalEffector::apply_sock_leave_ipv6_multicast(&mut ctx, fd, multiaddr, iface) + .map_err(anyhow_err_to_runtime_err)?, + crate::journal::JournalEntry::SocketSendFileV1 { + socket_fd, + file_fd, + offset, + count, + } => JournalEffector::apply_sock_send_file(&mut ctx, socket_fd, file_fd, offset, count) + .map_err(anyhow_err_to_runtime_err)?, + crate::journal::JournalEntry::SocketSendToV1 { + fd, + data, + flags, + addr, + is_64bit, + } => if is_64bit { + JournalEffector::apply_sock_send_to::(&ctx, fd, data, flags, addr) + } else { + JournalEffector::apply_sock_send_to::(&ctx, fd, data, flags, addr) + } + .map_err(anyhow_err_to_runtime_err)?, + crate::journal::JournalEntry::SocketSendV1 { + fd, + data, + flags, + is_64bit, + } => if is_64bit { + JournalEffector::apply_sock_send::(&ctx, fd, data, flags) + } else { + JournalEffector::apply_sock_send::(&ctx, fd, data, flags) + } + .map_err(anyhow_err_to_runtime_err)?, + crate::journal::JournalEntry::SocketSetOptFlagV1 { fd, opt, flag } => { + JournalEffector::apply_sock_set_opt_flag(&mut ctx, fd, opt, flag) + .map_err(anyhow_err_to_runtime_err)? + } + crate::journal::JournalEntry::SocketSetOptSizeV1 { fd, opt, size } => { + JournalEffector::apply_sock_set_opt_size(&mut ctx, fd, opt, size) + .map_err(anyhow_err_to_runtime_err)? + } + crate::journal::JournalEntry::SocketSetOptTimeV1 { fd, ty, time } => { + JournalEffector::apply_sock_set_opt_time(&mut ctx, fd, ty.into(), time) + .map_err(anyhow_err_to_runtime_err)? + } + crate::journal::JournalEntry::SocketShutdownV1 { fd, how } => { + JournalEffector::apply_sock_shutdown(&mut ctx, fd, how.into()) + .map_err(anyhow_err_to_runtime_err)? + } + crate::journal::JournalEntry::CreateEventV1 { + initial_val, + flags, + fd, + } => JournalEffector::apply_fd_event(&mut ctx, initial_val, flags, fd) + .map_err(anyhow_err_to_runtime_err)?, + } + } + + // If we are not in the same module then we fire off an exit + // that simulates closing the process (hence keeps everything + // in a clean state) + if journal_module_hash.is_some() && cur_module_hash != journal_module_hash { + tracing::error!( + "The WASM module hash does not match the journal module hash (journal_hash={:x?} vs module_hash{:x?}) - forcing a restart", + journal_module_hash.unwrap(), + cur_module_hash.unwrap() + ); + if bootstrapping { + rewind = None; + spawn_threads.clear(); + update_memory.clear(); + update_tty.take(); + stdout.clear(); + stderr.clear(); + stdout_fds.clear(); + stderr_fds.clear(); + stdout_fds.insert(1 as WasiFd); + stderr_fds.insert(2 as WasiFd); + } else { + JournalEffector::apply_process_exit(&mut ctx, None) + .map_err(anyhow_err_to_runtime_err)?; + } + } else { + tracing::debug!( + "journal used on a different module - the process will simulate a restart." + ); + } + + // We do not yet support multi threading + if !spawn_threads.is_empty() { + return Err(WasiRuntimeError::Runtime(RuntimeError::user( + anyhow::format_err!( + "Snapshot restoration does not currently support multiple threads." + ) + .into(), + ))); + } + + // Now output the stdout and stderr + for (offset, data, is_64bit) in stdout { + if is_64bit { + JournalEffector::apply_fd_write::(&ctx, 1, offset, data) + } else { + JournalEffector::apply_fd_write::(&ctx, 1, offset, data) + } + .map_err(anyhow_err_to_runtime_err)?; + } + + for (offset, data, is_64bit) in stderr { + if is_64bit { + JournalEffector::apply_fd_write::(&ctx, 2, offset, data) + } else { + JournalEffector::apply_fd_write::(&ctx, 2, offset, data) + } + .map_err(anyhow_err_to_runtime_err)?; + } + // Next we apply all the memory updates that were delayed while the logs + // were processed to completion. + for (region, data) in update_memory { + JournalEffector::apply_memory(&mut ctx, region, &data) + .map_err(anyhow_err_to_runtime_err)?; + } + if let Some(state) = update_tty { + JournalEffector::apply_tty_set(&mut ctx, state).map_err(anyhow_err_to_runtime_err)?; + } + + Ok(rewind) +} diff --git a/lib/wasix/src/syscalls/mod.rs b/lib/wasix/src/syscalls/mod.rs index 38097b7f36d..610f4075419 100644 --- a/lib/wasix/src/syscalls/mod.rs +++ b/lib/wasix/src/syscalls/mod.rs @@ -16,6 +16,7 @@ pub mod wasm; #[cfg(any(target_os = "windows"))] pub mod windows; +pub mod journal; pub mod wasi; pub mod wasix; @@ -54,6 +55,7 @@ use std::{io::IoSlice, marker::PhantomData, mem::MaybeUninit, task::Waker, time: pub(crate) use bytes::{Bytes, BytesMut}; pub(crate) use cooked_waker::IntoWaker; +pub use journal::*; pub(crate) use sha2::Sha256; pub(crate) use tracing::{debug, error, trace, warn}; #[cfg(any( @@ -1252,661 +1254,10 @@ pub fn rewind_ext2( Ok(()) } -#[allow(clippy::extra_unused_type_parameters)] -#[cfg(not(feature = "journal"))] -pub fn maybe_snapshot_once( - ctx: FunctionEnvMut<'_, WasiEnv>, - _trigger: crate::journal::SnapshotTrigger, -) -> WasiResult> { - Ok(Ok(ctx)) -} - -#[cfg(feature = "journal")] -pub fn maybe_snapshot_once( - mut ctx: FunctionEnvMut<'_, WasiEnv>, - trigger: crate::journal::SnapshotTrigger, -) -> WasiResult> { - use crate::os::task::process::{WasiProcessCheckpoint, WasiProcessInner}; - - unsafe { handle_rewind_ext::(&mut ctx, HandleRewindType::Resultless) }; - - if !ctx.data().enable_journal { - return Ok(Ok(ctx)); - } - - if ctx.data_mut().pop_snapshot_trigger(trigger) { - let inner = ctx.data().process.inner.clone(); - let res = wasi_try_ok_ok!(WasiProcessInner::checkpoint::( - inner, - ctx, - WasiProcessCheckpoint::Snapshot { trigger }, - )?); - match res { - MaybeCheckpointResult::Unwinding => return Ok(Err(Errno::Success)), - MaybeCheckpointResult::NotThisTime(c) => { - ctx = c; - } - } - } - Ok(Ok(ctx)) -} - -#[allow(clippy::extra_unused_type_parameters)] -#[cfg(not(feature = "journal"))] -pub fn maybe_snapshot( - ctx: FunctionEnvMut<'_, WasiEnv>, -) -> WasiResult> { - Ok(Ok(ctx)) -} - -#[cfg(feature = "journal")] -pub fn maybe_snapshot( - mut ctx: FunctionEnvMut<'_, WasiEnv>, -) -> WasiResult> { - use crate::os::task::process::{WasiProcessCheckpoint, WasiProcessInner}; - - if !ctx.data().enable_journal { - return Ok(Ok(ctx)); - } - - let inner = ctx.data().process.inner.clone(); - let res = wasi_try_ok_ok!(WasiProcessInner::maybe_checkpoint::(inner, ctx)?); - match res { - MaybeCheckpointResult::Unwinding => return Ok(Err(Errno::Success)), - MaybeCheckpointResult::NotThisTime(c) => { - ctx = c; - } - } - Ok(Ok(ctx)) -} - pub fn anyhow_err_to_runtime_err(err: anyhow::Error) -> WasiRuntimeError { WasiRuntimeError::Runtime(RuntimeError::user(err.into())) } -/// Safety: This function manipulates the memory of the process and thus must -/// be executed by the WASM process thread itself. -/// -#[allow(clippy::result_large_err)] -#[cfg(feature = "journal")] -pub unsafe fn restore_snapshot( - mut ctx: FunctionEnvMut<'_, WasiEnv>, - journal: Arc, - bootstrapping: bool, -) -> Result, WasiRuntimeError> { - use std::ops::Range; - - use crate::journal::Journal; - - // We delay the spawning of threads until the end as its - // possible that the threads will be cancelled before all the - // events finished the streaming process - let mut spawn_threads: HashMap = Default::default(); - - // We delay the memory updates until the end as its possible the - // memory will be cleared before all the events finished the - // streaming process - let mut update_memory: HashMap, Cow<'_, [u8]>> = Default::default(); - let mut update_tty = None; - - // We capture the stdout and stderr while we replay - let mut stdout = Vec::new(); - let mut stderr = Vec::new(); - let mut stdout_fds = HashSet::new(); - let mut stderr_fds = HashSet::new(); - stdout_fds.insert(1 as WasiFd); - stderr_fds.insert(2 as WasiFd); - - // Loop through all the events and process them - let cur_module_hash = Some(ctx.data().process.module_hash.as_bytes()); - let mut journal_module_hash = None; - let mut rewind = None; - while let Some(next) = journal.read().map_err(anyhow_err_to_runtime_err)? { - tracing::trace!("Restoring snapshot event - {next:?}"); - match next { - crate::journal::JournalEntry::InitModule { wasm_hash } => { - journal_module_hash.replace(wasm_hash); - } - crate::journal::JournalEntry::ProcessExit { exit_code } => { - if bootstrapping { - rewind = None; - spawn_threads.clear(); - update_memory.clear(); - update_tty.take(); - stdout.clear(); - stderr.clear(); - stdout_fds.clear(); - stderr_fds.clear(); - stdout_fds.insert(1 as WasiFd); - stderr_fds.insert(2 as WasiFd); - } else { - JournalEffector::apply_process_exit(&mut ctx, exit_code) - .map_err(anyhow_err_to_runtime_err)?; - } - } - crate::journal::JournalEntry::FileDescriptorWrite { - fd, - offset, - data, - is_64bit, - } => { - if stdout_fds.contains(&fd) { - stdout.push((offset, data, is_64bit)); - continue; - } - if stderr_fds.contains(&fd) { - stderr.push((offset, data, is_64bit)); - continue; - } - - if is_64bit { - JournalEffector::apply_fd_write::(&ctx, fd, offset, data) - } else { - JournalEffector::apply_fd_write::(&ctx, fd, offset, data) - } - .map_err(anyhow_err_to_runtime_err)?; - } - crate::journal::JournalEntry::FileDescriptorSeek { fd, offset, whence } => { - JournalEffector::apply_fd_seek(&mut ctx, fd, offset, whence) - .map_err(anyhow_err_to_runtime_err)?; - } - crate::journal::JournalEntry::UpdateMemoryRegion { region, data } => { - if cur_module_hash != journal_module_hash { - continue; - } - - if bootstrapping { - update_memory.insert(region, data.clone()); - } else { - JournalEffector::apply_memory(&mut ctx, region, &data) - .map_err(anyhow_err_to_runtime_err)?; - } - } - crate::journal::JournalEntry::CloseThread { id, exit_code } => { - if id == ctx.data().tid() { - if bootstrapping { - rewind = None; - spawn_threads.clear(); - update_memory.clear(); - update_tty.take(); - stdout.clear(); - stderr.clear(); - stdout_fds.clear(); - stderr_fds.clear(); - stdout_fds.insert(1 as WasiFd); - stderr_fds.insert(2 as WasiFd); - } else { - JournalEffector::apply_process_exit(&mut ctx, exit_code) - .map_err(anyhow_err_to_runtime_err)?; - } - } else if bootstrapping { - spawn_threads.remove(&id); - } else { - JournalEffector::apply_thread_exit(&mut ctx, id, exit_code) - .map_err(anyhow_err_to_runtime_err)?; - } - } - crate::journal::JournalEntry::SetThread { - id, - call_stack, - memory_stack, - store_data, - is_64bit, - } => { - if cur_module_hash != journal_module_hash { - continue; - } - - let state = RewindState { - memory_stack: memory_stack.to_vec().into(), - rewind_stack: call_stack.to_vec().into(), - store_data: store_data.to_vec().into(), - is_64bit, - }; - - if id == ctx.data().tid() { - rewind.replace(state); - } else if bootstrapping { - spawn_threads.insert(id, state); - } else { - return Err(WasiRuntimeError::Runtime(RuntimeError::user( - anyhow::format_err!( - "Snapshot restoration does not currently support live updates of running threads." - ) - .into(), - ))); - } - } - crate::journal::JournalEntry::CloseFileDescriptor { fd } => { - stdout_fds.remove(&fd); - stderr_fds.remove(&fd); - JournalEffector::apply_fd_close(&mut ctx, fd).map_err(anyhow_err_to_runtime_err)?; - } - crate::journal::JournalEntry::OpenFileDescriptor { - fd, - dirfd, - dirflags, - path, - o_flags, - fs_rights_base, - fs_rights_inheriting, - fs_flags, - } => { - JournalEffector::apply_path_open( - &mut ctx, - fd, - dirfd, - dirflags, - &path, - o_flags, - fs_rights_base, - fs_rights_inheriting, - fs_flags, - ) - .map_err(anyhow_err_to_runtime_err)?; - } - crate::journal::JournalEntry::RemoveDirectory { fd, path } => { - JournalEffector::apply_path_remove_directory(&mut ctx, fd, &path) - .map_err(anyhow_err_to_runtime_err)?; - } - crate::journal::JournalEntry::UnlinkFile { fd, path } => { - JournalEffector::apply_path_unlink(&mut ctx, fd, &path) - .map_err(anyhow_err_to_runtime_err)?; - } - crate::journal::JournalEntry::PathRename { - old_fd, - old_path, - new_fd, - new_path, - } => { - JournalEffector::apply_path_rename(&mut ctx, old_fd, &old_path, new_fd, &new_path) - .map_err(anyhow_err_to_runtime_err)?; - } - crate::journal::JournalEntry::Snapshot { when: _, trigger } => { - if cur_module_hash != journal_module_hash { - continue; - } - ctx.data_mut().pop_snapshot_trigger(trigger); - } - crate::journal::JournalEntry::SetClockTime { clock_id, time } => { - JournalEffector::apply_clock_time_set(&mut ctx, clock_id, time) - .map_err(anyhow_err_to_runtime_err)?; - } - crate::journal::JournalEntry::RenumberFileDescriptor { old_fd, new_fd } => { - if stdout_fds.remove(&old_fd) { - stdout_fds.insert(new_fd); - } - if stderr_fds.remove(&old_fd) { - stderr_fds.insert(new_fd); - } - JournalEffector::apply_fd_renumber(&mut ctx, old_fd, new_fd) - .map_err(anyhow_err_to_runtime_err)?; - } - crate::journal::JournalEntry::DuplicateFileDescriptor { - original_fd, - copied_fd, - } => { - if stdout_fds.contains(&original_fd) { - stdout_fds.insert(copied_fd); - } - if stderr_fds.contains(&original_fd) { - stderr_fds.insert(copied_fd); - } - JournalEffector::apply_fd_duplicate(&mut ctx, original_fd, copied_fd) - .map_err(anyhow_err_to_runtime_err)?; - } - crate::journal::JournalEntry::CreateDirectory { fd, path } => { - JournalEffector::apply_path_create_directory(&mut ctx, fd, &path) - .map_err(anyhow_err_to_runtime_err)?; - } - crate::journal::JournalEntry::PathSetTimes { - fd, - flags, - path, - st_atim, - st_mtim, - fst_flags, - } => { - JournalEffector::apply_path_set_times( - &mut ctx, fd, flags, &path, st_atim, st_mtim, fst_flags, - ) - .map_err(anyhow_err_to_runtime_err)?; - } - crate::journal::JournalEntry::FileDescriptorSetTimes { - fd, - st_atim, - st_mtim, - fst_flags, - } => { - JournalEffector::apply_fd_set_times(&mut ctx, fd, st_atim, st_mtim, fst_flags) - .map_err(anyhow_err_to_runtime_err)?; - } - crate::journal::JournalEntry::FileDescriptorSetSize { fd, st_size } => { - JournalEffector::apply_fd_set_size(&mut ctx, fd, st_size) - .map_err(anyhow_err_to_runtime_err)?; - } - crate::journal::JournalEntry::FileDescriptorSetFlags { fd, flags } => { - JournalEffector::apply_fd_set_flags(&mut ctx, fd, flags) - .map_err(anyhow_err_to_runtime_err)?; - } - crate::journal::JournalEntry::FileDescriptorSetRights { - fd, - fs_rights_base, - fs_rights_inheriting, - } => { - JournalEffector::apply_fd_set_rights( - &mut ctx, - fd, - fs_rights_base, - fs_rights_inheriting, - ) - .map_err(anyhow_err_to_runtime_err)?; - } - crate::journal::JournalEntry::FileDescriptorAdvise { - fd, - offset, - len, - advice, - } => { - JournalEffector::apply_fd_advise(&mut ctx, fd, offset, len, advice) - .map_err(anyhow_err_to_runtime_err)?; - } - crate::journal::JournalEntry::FileDescriptorAllocate { fd, offset, len } => { - JournalEffector::apply_fd_allocate(&mut ctx, fd, offset, len) - .map_err(anyhow_err_to_runtime_err)?; - } - crate::journal::JournalEntry::CreateHardLink { - old_fd, - old_path, - old_flags, - new_fd, - new_path, - } => { - JournalEffector::apply_path_link( - &mut ctx, old_fd, old_flags, &old_path, new_fd, &new_path, - ) - .map_err(anyhow_err_to_runtime_err)?; - } - crate::journal::JournalEntry::CreateSymbolicLink { - old_path, - fd, - new_path, - } => { - JournalEffector::apply_path_symlink(&mut ctx, &old_path, fd, &new_path) - .map_err(anyhow_err_to_runtime_err)?; - } - crate::journal::JournalEntry::ChangeDirectory { path } => { - JournalEffector::apply_chdir(&mut ctx, &path).map_err(anyhow_err_to_runtime_err)?; - } - crate::journal::JournalEntry::CreatePipe { fd1, fd2 } => { - JournalEffector::apply_fd_pipe(&mut ctx, fd1, fd2) - .map_err(anyhow_err_to_runtime_err)?; - } - crate::journal::JournalEntry::EpollCreate { fd } => { - JournalEffector::apply_epoll_create(&mut ctx, fd) - .map_err(anyhow_err_to_runtime_err)?; - } - crate::journal::JournalEntry::EpollCtl { - epfd, - op, - fd, - event, - } => { - JournalEffector::apply_epoll_ctl(&mut ctx, epfd, op, fd, event) - .map_err(anyhow_err_to_runtime_err)?; - } - crate::journal::JournalEntry::TtySet { tty, line_feeds } => { - let state = crate::WasiTtyState { - cols: tty.cols, - rows: tty.rows, - width: tty.width, - height: tty.height, - stdin_tty: tty.stdin_tty, - stdout_tty: tty.stdout_tty, - stderr_tty: tty.stderr_tty, - echo: tty.echo, - line_buffered: tty.line_buffered, - line_feeds, - }; - - if bootstrapping { - update_tty.replace(state); - } else { - JournalEffector::apply_tty_set(&mut ctx, state) - .map_err(anyhow_err_to_runtime_err)?; - } - } - crate::journal::JournalEntry::PortAddAddr { cidr } => { - JournalEffector::apply_port_addr_add(&mut ctx, cidr) - .map_err(anyhow_err_to_runtime_err)? - } - crate::journal::JournalEntry::PortDelAddr { addr } => { - JournalEffector::apply_port_addr_remove(&mut ctx, addr) - .map_err(anyhow_err_to_runtime_err)? - } - crate::journal::JournalEntry::PortAddrClear => { - JournalEffector::apply_port_addr_clear(&mut ctx) - .map_err(anyhow_err_to_runtime_err)? - } - crate::journal::JournalEntry::PortBridge { - network, - token, - security, - } => JournalEffector::apply_port_bridge(&mut ctx, &network, &token, security) - .map_err(anyhow_err_to_runtime_err)?, - crate::journal::JournalEntry::PortUnbridge => { - JournalEffector::apply_port_unbridge(&mut ctx).map_err(anyhow_err_to_runtime_err)? - } - crate::journal::JournalEntry::PortDhcpAcquire => { - JournalEffector::apply_port_dhcp_acquire(&mut ctx) - .map_err(anyhow_err_to_runtime_err)? - } - crate::journal::JournalEntry::PortGatewaySet { ip } => { - JournalEffector::apply_port_gateway_set(&mut ctx, ip) - .map_err(anyhow_err_to_runtime_err)? - } - crate::journal::JournalEntry::PortRouteAdd { - cidr, - via_router, - preferred_until, - expires_at, - } => JournalEffector::apply_port_route_add( - &mut ctx, - cidr, - via_router, - preferred_until, - expires_at, - ) - .map_err(anyhow_err_to_runtime_err)?, - crate::journal::JournalEntry::PortRouteClear => { - JournalEffector::apply_port_route_clear(&mut ctx) - .map_err(anyhow_err_to_runtime_err)? - } - crate::journal::JournalEntry::PortRouteDel { ip } => { - JournalEffector::apply_port_route_remove(&mut ctx, ip) - .map_err(anyhow_err_to_runtime_err)? - } - crate::journal::JournalEntry::SocketOpen { af, ty, pt, fd } => { - JournalEffector::apply_sock_open(&mut ctx, af, ty, pt, fd) - .map_err(anyhow_err_to_runtime_err)? - } - crate::journal::JournalEntry::SocketListen { fd, backlog } => { - JournalEffector::apply_sock_listen(&mut ctx, fd, backlog as usize) - .map_err(anyhow_err_to_runtime_err)? - } - crate::journal::JournalEntry::SocketBind { fd, addr } => { - JournalEffector::apply_sock_bind(&mut ctx, fd, addr) - .map_err(anyhow_err_to_runtime_err)? - } - crate::journal::JournalEntry::SocketConnected { fd, addr } => { - JournalEffector::apply_sock_connect(&mut ctx, fd, addr) - .map_err(anyhow_err_to_runtime_err)? - } - crate::journal::JournalEntry::SocketAccepted { - listen_fd, - fd, - peer_addr, - fd_flags, - nonblocking, - } => JournalEffector::apply_sock_accepted( - &mut ctx, - listen_fd, - fd, - peer_addr, - fd_flags, - nonblocking, - ) - .map_err(anyhow_err_to_runtime_err)?, - crate::journal::JournalEntry::SocketJoinIpv4Multicast { - fd, - multiaddr, - iface, - } => JournalEffector::apply_sock_join_ipv4_multicast(&mut ctx, fd, multiaddr, iface) - .map_err(anyhow_err_to_runtime_err)?, - crate::journal::JournalEntry::SocketJoinIpv6Multicast { - fd, - multiaddr, - iface, - } => JournalEffector::apply_sock_join_ipv6_multicast(&mut ctx, fd, multiaddr, iface) - .map_err(anyhow_err_to_runtime_err)?, - crate::journal::JournalEntry::SocketLeaveIpv4Multicast { - fd, - multiaddr, - iface, - } => JournalEffector::apply_sock_leave_ipv4_multicast(&mut ctx, fd, multiaddr, iface) - .map_err(anyhow_err_to_runtime_err)?, - crate::journal::JournalEntry::SocketLeaveIpv6Multicast { - fd, - multiaddr, - iface, - } => JournalEffector::apply_sock_leave_ipv6_multicast(&mut ctx, fd, multiaddr, iface) - .map_err(anyhow_err_to_runtime_err)?, - crate::journal::JournalEntry::SocketSendFile { - socket_fd, - file_fd, - offset, - count, - } => JournalEffector::apply_sock_send_file(&mut ctx, socket_fd, file_fd, offset, count) - .map_err(anyhow_err_to_runtime_err)?, - crate::journal::JournalEntry::SocketSendTo { - fd, - data, - flags, - addr, - is_64bit, - } => if is_64bit { - JournalEffector::apply_sock_send_to::(&ctx, fd, data, flags, addr) - } else { - JournalEffector::apply_sock_send_to::(&ctx, fd, data, flags, addr) - } - .map_err(anyhow_err_to_runtime_err)?, - crate::journal::JournalEntry::SocketSend { - fd, - data, - flags, - is_64bit, - } => if is_64bit { - JournalEffector::apply_sock_send::(&ctx, fd, data, flags) - } else { - JournalEffector::apply_sock_send::(&ctx, fd, data, flags) - } - .map_err(anyhow_err_to_runtime_err)?, - crate::journal::JournalEntry::SocketSetOptFlag { fd, opt, flag } => { - JournalEffector::apply_sock_set_opt_flag(&mut ctx, fd, opt, flag) - .map_err(anyhow_err_to_runtime_err)? - } - crate::journal::JournalEntry::SocketSetOptSize { fd, opt, size } => { - JournalEffector::apply_sock_set_opt_size(&mut ctx, fd, opt, size) - .map_err(anyhow_err_to_runtime_err)? - } - crate::journal::JournalEntry::SocketSetOptTime { fd, ty, time } => { - JournalEffector::apply_sock_set_opt_time(&mut ctx, fd, ty.into(), time) - .map_err(anyhow_err_to_runtime_err)? - } - crate::journal::JournalEntry::SocketShutdown { fd, how } => { - JournalEffector::apply_sock_shutdown(&mut ctx, fd, how.into()) - .map_err(anyhow_err_to_runtime_err)? - } - crate::journal::JournalEntry::CreateEvent { - initial_val, - flags, - fd, - } => JournalEffector::apply_fd_event(&mut ctx, initial_val, flags, fd) - .map_err(anyhow_err_to_runtime_err)?, - } - } - - // If we are not in the same module then we fire off an exit - // that simulates closing the process (hence keeps everything - // in a clean state) - if journal_module_hash.is_some() && cur_module_hash != journal_module_hash { - tracing::error!( - "The WASM module hash does not match the journal module hash (journal_hash={:x?} vs module_hash{:x?}) - forcing a restart", - journal_module_hash.unwrap(), - cur_module_hash.unwrap() - ); - if bootstrapping { - rewind = None; - spawn_threads.clear(); - update_memory.clear(); - update_tty.take(); - stdout.clear(); - stderr.clear(); - stdout_fds.clear(); - stderr_fds.clear(); - stdout_fds.insert(1 as WasiFd); - stderr_fds.insert(2 as WasiFd); - } else { - JournalEffector::apply_process_exit(&mut ctx, None) - .map_err(anyhow_err_to_runtime_err)?; - } - } else { - tracing::debug!( - "journal used on a different module - the process will simulate a restart." - ); - } - - // We do not yet support multi threading - if !spawn_threads.is_empty() { - return Err(WasiRuntimeError::Runtime(RuntimeError::user( - anyhow::format_err!( - "Snapshot restoration does not currently support multiple threads." - ) - .into(), - ))); - } - - // Now output the stdout and stderr - for (offset, data, is_64bit) in stdout { - if is_64bit { - JournalEffector::apply_fd_write::(&ctx, 1, offset, data) - } else { - JournalEffector::apply_fd_write::(&ctx, 1, offset, data) - } - .map_err(anyhow_err_to_runtime_err)?; - } - - for (offset, data, is_64bit) in stderr { - if is_64bit { - JournalEffector::apply_fd_write::(&ctx, 2, offset, data) - } else { - JournalEffector::apply_fd_write::(&ctx, 2, offset, data) - } - .map_err(anyhow_err_to_runtime_err)?; - } - // Next we apply all the memory updates that were delayed while the logs - // were processed to completion. - for (region, data) in update_memory { - JournalEffector::apply_memory(&mut ctx, region, &data) - .map_err(anyhow_err_to_runtime_err)?; - } - if let Some(state) = update_tty { - JournalEffector::apply_tty_set(&mut ctx, state).map_err(anyhow_err_to_runtime_err)?; - } - - Ok(rewind) -} - pub(crate) unsafe fn handle_rewind( ctx: &mut FunctionEnvMut<'_, WasiEnv>, ) -> Option From a41c9966b9532527902524defcfbe9c70afd96c0 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Tue, 28 Nov 2023 10:15:49 +1100 Subject: [PATCH 101/129] Added some linting fixes and fix for compile erors --- lib/api/src/js/externals/memory.rs | 15 +++++++++++++++ lib/wasix/src/journal/concrete/compacting.rs | 6 +++--- lib/wasix/src/journal/snapshot.rs | 10 ++++------ lib/wasix/src/state/builder.rs | 10 ++-------- lib/wasix/src/state/env.rs | 2 +- 5 files changed, 25 insertions(+), 18 deletions(-) diff --git a/lib/api/src/js/externals/memory.rs b/lib/api/src/js/externals/memory.rs index 183faa8bf4b..7944a0e3e7b 100644 --- a/lib/api/src/js/externals/memory.rs +++ b/lib/api/src/js/externals/memory.rs @@ -144,6 +144,21 @@ impl Memory { Ok(Pages(new_pages)) } + pub fn grow_at_least( + &self, + store: &mut impl AsStoreMut, + min_size: u64, + ) -> Result<(), MemoryError> { + let cur_size = self.view(store).data_size(); + if min_size > cur_size { + let delta = min_size - cur_size; + let pages = ((delta - 1) / wasmer_types::WASM_PAGE_SIZE) + 1; + + self.grow(store, Pages(pages))?; + } + Ok(()) + } + pub fn reset(&self, _store: &mut impl AsStoreMut) -> Result<(), MemoryError> { Ok(()) } diff --git a/lib/wasix/src/journal/concrete/compacting.rs b/lib/wasix/src/journal/concrete/compacting.rs index 2c663f6af15..0b1d7937dcf 100644 --- a/lib/wasix/src/journal/concrete/compacting.rs +++ b/lib/wasix/src/journal/concrete/compacting.rs @@ -98,7 +98,7 @@ impl State { .chain(self.keep_descriptors.iter()) .chain(self.stdio_descriptors.iter()) { - if let Some(d) = self.descriptors.get(&l) { + if let Some(d) = self.descriptors.get(l) { for e in d.events.iter() { filter.add_event_to_whitelist(*e); } @@ -299,7 +299,7 @@ impl WritableJournal for CompactingJournalTx { let state = state .descriptors .entry(lookup) - .or_insert_with(|| Default::default()); + .or_insert_with(Default::default); state.events.push(event_index); } else { state.whitelist.insert(event_index); @@ -330,7 +330,7 @@ impl WritableJournal for CompactingJournalTx { let state = state .descriptors .entry(lookup) - .or_insert_with(|| Default::default()); + .or_insert_with(Default::default); if let JournalEntry::FileDescriptorWriteV1 { offset, data, .. } = &entry { state.write_map.insert( MemoryRange { diff --git a/lib/wasix/src/journal/snapshot.rs b/lib/wasix/src/journal/snapshot.rs index 178596577eb..15afcb6ae92 100644 --- a/lib/wasix/src/journal/snapshot.rs +++ b/lib/wasix/src/journal/snapshot.rs @@ -28,12 +28,10 @@ pub enum SnapshotTrigger { impl SnapshotTrigger { pub fn only_once(&self) -> bool { - match self { - Self::FirstListen => true, - Self::FirstEnviron => true, - Self::FirstStdin => true, - _ => false, - } + matches!( + self, + Self::FirstListen | Self::FirstEnviron | Self::FirstStdin + ) } } diff --git a/lib/wasix/src/state/builder.rs b/lib/wasix/src/state/builder.rs index 3f1a1885a6b..082fed4880d 100644 --- a/lib/wasix/src/state/builder.rs +++ b/lib/wasix/src/state/builder.rs @@ -867,10 +867,7 @@ impl WasiEnvBuilder { #[allow(clippy::result_large_err)] pub fn build(self) -> Result { - let module_hash = self - .module_hash - .clone() - .unwrap_or_else(|| ModuleHash::random()); + let module_hash = self.module_hash.unwrap_or_else(ModuleHash::random); let init = self.build_init()?; WasiEnv::from_init(init, module_hash) } @@ -885,10 +882,7 @@ impl WasiEnvBuilder { self, store: &mut impl AsStoreMut, ) -> Result { - let module_hash = self - .module_hash - .clone() - .unwrap_or_else(|| ModuleHash::random()); + let module_hash = self.module_hash.unwrap_or_else(ModuleHash::random); let init = self.build_init()?; let env = WasiEnv::from_init(init, module_hash)?; let func_env = WasiFunctionEnv::new(store, env); diff --git a/lib/wasix/src/state/env.rs b/lib/wasix/src/state/env.rs index df4249efa8e..d934bd39ad8 100644 --- a/lib/wasix/src/state/env.rs +++ b/lib/wasix/src/state/env.rs @@ -822,7 +822,7 @@ impl WasiEnv { /// (it must be initialized before it can be used) /// This has been marked as unsafe as it will panic if its executed /// on the wrong thread or before the inner is set - pub(crate) unsafe fn memory<'a>(&self) -> WasiInstanceGuardMemory<'_> { + pub(crate) unsafe fn memory(&self) -> WasiInstanceGuardMemory<'_> { self.try_memory().expect( "You must initialize the WasiEnv before using it and can not pass it between threads", ) From dd74d0836670d539d9e72b9149b20f267efa6a7f Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Thu, 21 Dec 2023 11:05:07 +1100 Subject: [PATCH 102/129] Added the basics of a DCGI runner --- lib/cli/Cargo.toml | 1 + lib/cli/src/commands/run/mod.rs | 46 ++++-- lib/wasix/Cargo.toml | 2 + lib/wasix/src/runners/dcgi/handler.rs | 67 +++++++++ lib/wasix/src/runners/dcgi/mod.rs | 5 + lib/wasix/src/runners/dcgi/runner.rs | 202 ++++++++++++++++++++++++++ lib/wasix/src/runners/mod.rs | 2 + lib/wasix/src/runners/wcgi/handler.rs | 85 +++++++++-- lib/wasix/src/runners/wcgi/mod.rs | 1 + lib/wasix/src/runners/wcgi/runner.rs | 71 ++++++--- 10 files changed, 434 insertions(+), 48 deletions(-) create mode 100644 lib/wasix/src/runners/dcgi/handler.rs create mode 100644 lib/wasix/src/runners/dcgi/mod.rs create mode 100644 lib/wasix/src/runners/dcgi/runner.rs diff --git a/lib/cli/Cargo.toml b/lib/cli/Cargo.toml index fc6ec57003f..3388685904e 100644 --- a/lib/cli/Cargo.toml +++ b/lib/cli/Cargo.toml @@ -39,6 +39,7 @@ wasmer-vm = { version = "=4.2.4", path = "../vm", optional = true } wasmer-wasix = { version = "0.17.0", path = "../wasix", features = [ "logging", "webc_runner_rt_wcgi", + "webc_runner_rt_dcgi", "webc_runner_rt_emscripten", "host-fs", ] } diff --git a/lib/cli/src/commands/run/mod.rs b/lib/cli/src/commands/run/mod.rs index 267844f1f6d..3db59f3ce71 100644 --- a/lib/cli/src/commands/run/mod.rs +++ b/lib/cli/src/commands/run/mod.rs @@ -32,7 +32,7 @@ use wasmer_registry::{wasmer_env::WasmerEnv, Package}; use wasmer_wasix::journal::{LogFileJournal, SnapshotTrigger}; use wasmer_wasix::{ bin_factory::BinaryPackage, - runners::{MappedCommand, MappedDirectory, Runner}, + runners::{wcgi, MappedCommand, MappedDirectory, Runner}, runtime::{ module_cache::{CacheError, ModuleHash}, package_loader::PackageLoader, @@ -43,6 +43,7 @@ use wasmer_wasix::{ }; use wasmer_wasix::{ runners::{ + dcgi::DcgiRunner, emscripten::EmscriptenRunner, wasi::WasiRunner, wcgi::{AbortHandle, WcgiRunner}, @@ -181,7 +182,9 @@ impl Run { let uses = self.load_injected_packages(&runtime)?; - if WcgiRunner::can_run_command(cmd.metadata())? { + if DcgiRunner::can_run_command(cmd.metadata())? { + self.run_dcgi(id, pkg, uses, runtime) + } else if WcgiRunner::can_run_command(cmd.metadata())? { self.run_wcgi(id, pkg, uses, runtime) } else if WasiRunner::can_run_command(cmd.metadata())? { self.run_wasi(id, pkg, uses, runtime) @@ -240,27 +243,34 @@ impl Run { runtime: Arc, ) -> Result<(), Error> { let mut runner = wasmer_wasix::runners::wcgi::WcgiRunner::new(); + self.config_wcgi(runner.config(), uses)?; + runner.run_command(command_name, pkg, runtime) + } - runner - .config() + fn config_wcgi( + &self, + config: &mut wcgi::Config, + uses: Vec, + ) -> Result<(), Error> { + config .args(self.args.clone()) .addr(self.wcgi.addr) .envs(self.wasi.env_vars.clone()) .map_directories(self.wasi.mapped_dirs.clone()) .callbacks(Callbacks::new(self.wcgi.addr)) .inject_packages(uses); - *runner.config().capabilities() = self.wasi.capabilities(); + *config.capabilities() = self.wasi.capabilities(); if self.wasi.forward_host_env { - runner.config().forward_host_env(); + config.forward_host_env(); } #[cfg(feature = "journal")] { for trigger in self.wasi.snapshot_on.iter().cloned() { - runner.config().add_snapshot_trigger(trigger); + config.add_snapshot_trigger(trigger); } if self.wasi.snapshot_on.is_empty() && !self.wasi.journals.is_empty() { - runner.config().add_default_snapshot_triggers(); + config.add_default_snapshot_triggers(); } if let Some(period) = self.wasi.snapshot_interval { if self.wasi.journals.is_empty() { @@ -268,17 +278,25 @@ impl Run { "If you specify a snapshot interval then you must also specify a journal file" )); } - runner - .config() - .with_snapshot_interval(Duration::from_millis(period)); + config.with_snapshot_interval(Duration::from_millis(period)); } for journal in self.wasi.journals.clone() { - runner - .config() - .add_journal(Arc::new(LogFileJournal::new(journal)?)); + config.add_journal(Arc::new(LogFileJournal::new(journal)?)); } } + Ok(()) + } + + fn run_dcgi( + &self, + command_name: &str, + pkg: &BinaryPackage, + uses: Vec, + runtime: Arc, + ) -> Result<(), Error> { + let mut runner = wasmer_wasix::runners::dcgi::DcgiRunner::new(); + self.config_wcgi(runner.config().inner(), uses); runner.run_command(command_name, pkg, runtime) } diff --git a/lib/wasix/Cargo.toml b/lib/wasix/Cargo.toml index d02ff5b4620..fcfb3039982 100644 --- a/lib/wasix/Cargo.toml +++ b/lib/wasix/Cargo.toml @@ -139,6 +139,7 @@ default = ["sys-default"] time = ["tokio/time"] webc_runner_rt_wcgi = ["hyper", "wcgi", "wcgi-host", "tower", "tower-http"] +webc_runner_rt_dcgi = ["webc_runner_rt_wcgi", "journal"] webc_runner_rt_emscripten = ["wasmer-emscripten"] sys = ["webc/mmap", "time", "virtual-mio/sys"] @@ -190,6 +191,7 @@ enable-serde = [ features = [ "wasmer/sys", "webc_runner_rt_wcgi", + "webc_runner_rt_dcgi", "webc_runner_rt_emscripten", "sys-default", ] diff --git a/lib/wasix/src/runners/dcgi/handler.rs b/lib/wasix/src/runners/dcgi/handler.rs new file mode 100644 index 00000000000..94b7223c498 --- /dev/null +++ b/lib/wasix/src/runners/dcgi/handler.rs @@ -0,0 +1,67 @@ +use std::{ops::Deref, pin::Pin, sync::Arc, task::Poll}; + +use anyhow::Error; +use futures::Future; +use http::{Request, Response}; +use hyper::{service::Service, Body}; + +use crate::runners::wcgi; + +/// The shared object that manages the instantiaion of WASI executables and +/// communicating with them via the CGI protocol. +#[derive(Clone, Debug)] +pub(crate) struct Handler { + state: Arc, + inner: wcgi::Handler, +} + +impl Handler { + pub(crate) fn new(state: SharedState) -> Self { + Handler { + inner: wcgi::Handler::new(state.inner.clone()), + state: Arc::new(state), + } + } + + pub(crate) fn from_wcgi_handler(handler: wcgi::Handler) -> Self { + Handler { + state: Arc::new(SharedState { + inner: handler.deref().clone(), + }), + inner: handler, + } + } + + #[tracing::instrument(level = "debug", skip_all, err)] + pub(crate) async fn handle(&self, req: Request) -> Result, Error> { + self.inner.handle(req).await + } +} + +impl Deref for Handler { + type Target = Arc; + + fn deref(&self) -> &Self::Target { + &self.state + } +} + +#[derive(derivative::Derivative, Clone)] +#[derivative(Debug)] +pub(crate) struct SharedState { + pub(crate) inner: Arc, +} + +impl Service> for Handler { + type Response = Response; + type Error = Error; + type Future = Pin, Error>> + Send>>; + + fn poll_ready(&mut self, cx: &mut std::task::Context<'_>) -> Poll> { + self.inner.poll_ready(cx) + } + + fn call(&mut self, request: Request) -> Self::Future { + self.inner.call(request) + } +} diff --git a/lib/wasix/src/runners/dcgi/mod.rs b/lib/wasix/src/runners/dcgi/mod.rs new file mode 100644 index 00000000000..6e0b572cb84 --- /dev/null +++ b/lib/wasix/src/runners/dcgi/mod.rs @@ -0,0 +1,5 @@ +mod handler; +mod runner; + +pub use self::runner::{Config, DcgiRunner}; +pub use futures::future::AbortHandle; diff --git a/lib/wasix/src/runners/dcgi/runner.rs b/lib/wasix/src/runners/dcgi/runner.rs new file mode 100644 index 00000000000..e1034aa37db --- /dev/null +++ b/lib/wasix/src/runners/dcgi/runner.rs @@ -0,0 +1,202 @@ +use std::{net::SocketAddr, sync::Arc}; + +use anyhow::Error; +use wcgi_host::CgiDialect; +use webc::metadata::Command; + +use crate::{ + bin_factory::BinaryPackage, + capabilities::Capabilities, + runners::{dcgi::handler::Handler, wcgi, MappedDirectory}, + Runtime, +}; + +#[derive(Debug, Default)] +pub struct DcgiRunner { + config: Config, + inner: wcgi::WcgiRunner, +} + +impl DcgiRunner { + pub fn new() -> Self { + DcgiRunner::default() + } + + pub fn config(&mut self) -> &mut Config { + &mut self.config + } + + #[tracing::instrument(skip_all)] + fn prepare_handler( + &mut self, + command_name: &str, + pkg: &BinaryPackage, + runtime: Arc, + ) -> Result { + let inner: wcgi::Handler = + self.inner + .prepare_handler(command_name, pkg, true, CgiDialect::Rfc3875, runtime)?; + Ok(Handler::from_wcgi_handler(inner)) + } +} + +/// The base URI used by a [`Dcgi`] runner. +pub const DCGI_RUNNER_URI: &str = "https://webc.org/runner/dcgi"; + +impl crate::runners::Runner for DcgiRunner { + fn can_run_command(command: &Command) -> Result { + Ok(command.runner.starts_with(DCGI_RUNNER_URI)) + } + + fn run_command( + &mut self, + command_name: &str, + pkg: &BinaryPackage, + runtime: Arc, + ) -> Result<(), Error> { + let handler = self.prepare_handler(command_name, pkg, Arc::clone(&runtime))?; + self.inner.run_command_with_handler(handler, runtime) + } +} + +#[derive(Debug)] +pub struct Config { + inner: wcgi::Config, +} + +impl Config { + pub fn inner(&mut self) -> &mut wcgi::Config { + &mut self.inner + } + + pub fn addr(&mut self, addr: SocketAddr) -> &mut Self { + self.inner.addr(addr); + self + } + + /// Add an argument to the WASI executable's command-line arguments. + pub fn arg(&mut self, arg: impl Into) -> &mut Self { + self.inner.arg(arg); + self + } + + /// Add multiple arguments to the WASI executable's command-line arguments. + pub fn args(&mut self, args: A) -> &mut Self + where + A: IntoIterator, + S: Into, + { + self.inner.args(args); + self + } + + /// Expose an environment variable to the guest. + pub fn env(&mut self, name: impl Into, value: impl Into) -> &mut Self { + self.inner.env(name, value); + self + } + + /// Expose multiple environment variables to the guest. + pub fn envs(&mut self, variables: I) -> &mut Self + where + I: IntoIterator, + K: Into, + V: Into, + { + self.inner.envs(variables); + self + } + + /// Forward all of the host's environment variables to the guest. + pub fn forward_host_env(&mut self) -> &mut Self { + self.inner.forward_host_env(); + self + } + + pub fn map_directory(&mut self, dir: MappedDirectory) -> &mut Self { + self.inner.map_directory(dir); + self + } + + pub fn map_directories( + &mut self, + mappings: impl IntoIterator, + ) -> &mut Self { + self.inner.map_directories(mappings); + self + } + + /// Set callbacks that will be triggered at various points in the runner's + /// lifecycle. + pub fn callbacks( + &mut self, + callbacks: impl wcgi::Callbacks + Send + Sync + 'static, + ) -> &mut Self { + self.inner.callbacks(callbacks); + self + } + + /// Add a package that should be available to the instance at runtime. + pub fn inject_package(&mut self, pkg: BinaryPackage) -> &mut Self { + self.inner.inject_package(pkg); + self + } + + /// Add packages that should be available to the instance at runtime. + pub fn inject_packages( + &mut self, + packages: impl IntoIterator, + ) -> &mut Self { + self.inner.inject_packages(packages); + self + } + + pub fn capabilities(&mut self) -> &mut Capabilities { + self.inner.capabilities() + } + + pub fn add_snapshot_trigger(&mut self, on: crate::journal::SnapshotTrigger) { + self.inner.add_snapshot_trigger(on); + } + + pub fn add_default_snapshot_triggers(&mut self) -> &mut Self { + self.inner.add_default_snapshot_triggers(); + self + } + + pub fn has_snapshot_trigger(&self, on: crate::journal::SnapshotTrigger) -> bool { + self.inner.has_snapshot_trigger(on) + } + + pub fn with_snapshot_interval(&mut self, period: std::time::Duration) -> &mut Self { + self.inner.with_snapshot_interval(period); + self + } + + pub fn add_journal(&mut self, journal: Arc) -> &mut Self { + self.inner.add_journal(journal); + self + } +} + +impl Default for Config { + fn default() -> Self { + Self { + inner: Default::default(), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn send_and_sync() { + fn assert_send() {} + fn assert_sync() {} + + assert_send::(); + assert_sync::(); + } +} diff --git a/lib/wasix/src/runners/mod.rs b/lib/wasix/src/runners/mod.rs index ed9fa74f006..e32221049ec 100644 --- a/lib/wasix/src/runners/mod.rs +++ b/lib/wasix/src/runners/mod.rs @@ -1,5 +1,7 @@ mod runner; +#[cfg(feature = "webc_runner_rt_dcgi")] +pub mod dcgi; #[cfg(feature = "webc_runner_rt_emscripten")] pub mod emscripten; pub mod wasi; diff --git a/lib/wasix/src/runners/wcgi/handler.rs b/lib/wasix/src/runners/wcgi/handler.rs index f3aee79e3eb..dcc89101620 100644 --- a/lib/wasix/src/runners/wcgi/handler.rs +++ b/lib/wasix/src/runners/wcgi/handler.rs @@ -2,7 +2,7 @@ use std::{collections::HashMap, ops::Deref, pin::Pin, sync::Arc, task::Poll}; use anyhow::Error; use futures::{Future, FutureExt, StreamExt}; -use http::{Request, Response}; +use http::{Request, Response, StatusCode}; use hyper::{service::Service, Body}; use tokio::io::{AsyncBufReadExt, AsyncRead, AsyncWrite, AsyncWriteExt}; use tracing::Instrument; @@ -20,8 +20,8 @@ use crate::{ pub(crate) struct Handler(Arc); impl Handler { - pub(crate) fn new(state: SharedState) -> Self { - Handler(Arc::new(state)) + pub(crate) fn new(state: Arc) -> Self { + Handler(state) } #[tracing::instrument(level = "debug", skip_all, err)] @@ -44,8 +44,14 @@ impl Handler { // anything specified by WASI annotations so users get a chance to // override things like $DOCUMENT_ROOT and $SCRIPT_FILENAME. let mut request_specific_env = HashMap::new(); + request_specific_env.insert("REQUEST_METHOD".to_string(), parts.method.to_string()); + request_specific_env.insert("SCRIPT_NAME".to_string(), parts.uri.path().to_string()); + if let Some(query) = parts.uri.query() { + request_specific_env.insert("QUERY_STRING".to_string(), query.to_string()); + } self.dialect .prepare_environment_variables(parts, &mut request_specific_env); + builder.add_envs(request_specific_env); let builder = builder @@ -80,13 +86,15 @@ impl Handler { let mut res_body_receiver = tokio::io::BufReader::new(res_body_receiver); let callbacks = Arc::clone(&self.callbacks); - let work_consume_stderr = async move { - consume_stderr(stderr_receiver, callbacks).await; - } - .in_current_span(); - task_manager - .task_shared(Box::new(move || Box::pin(work_consume_stderr))) - .ok(); + let propagate_stderr = self.propagate_stderr; + let work_consume_stderr = + async move { consume_stderr(stderr_receiver, callbacks, propagate_stderr).await } + .in_current_span(); + + tracing::trace!( + dialect=%self.dialect, + "spawning request forwarder", + ); let work_drive_io = async move { if let Err(e) = drive_request_to_completion(done, body, req_body_sender).await { @@ -101,10 +109,44 @@ impl Handler { .task_shared(Box::new(move || Box::pin(work_drive_io))) .ok(); + tracing::trace!( + dialect=%self.dialect, + "extracting response parts", + ); + let parts = self .dialect .extract_response_header(&mut res_body_receiver) - .await?; + .await; + + // When set this will cause any stderr responses to + // take precedence over nominal responses but it + // will cause the stderr pipe to be read to the end + // before transmitting the body + if propagate_stderr { + if let Some(stderr) = work_consume_stderr.await { + if !stderr.is_empty() { + return Ok(Response::builder() + .status(StatusCode::INTERNAL_SERVER_ERROR) + .body(Body::from(stderr))?); + } + } + } else { + task_manager + .task_shared(Box::new(move || { + Box::pin(async move { + work_consume_stderr.await; + }) + })) + .ok(); + } + let parts = parts?; + + tracing::trace!( + dialect=%self.dialect, + status=%parts.status, + "received response parts", + ); let chunks = futures::stream::try_unfold(res_body_receiver, |mut r| async move { match r.fill_buf().await { @@ -119,6 +161,11 @@ impl Handler { }); let body = hyper::Body::wrap_stream(chunks); + tracing::trace!( + dialect=%self.dialect, + "returning response with body stream", + ); + let response = hyper::Response::from_parts(parts, body); Ok(response) } @@ -173,9 +220,15 @@ async fn drive_request_to_completion( async fn consume_stderr( stderr: impl AsyncRead + Send + Unpin + 'static, callbacks: Arc, -) { + propagate_stderr: bool, +) -> Option> { let mut stderr = tokio::io::BufReader::new(stderr); + let mut propagate = match propagate_stderr { + true => Some(Vec::new()), + false => None, + }; + // Note: we don't want to just read_to_end() because a reading error // would cause us to lose all of stderr. At least this way we'll be // able to show users the partial result. @@ -186,16 +239,23 @@ async fn consume_stderr( break; } Ok(chunk) => { + tracing::trace!("received stderr (len={})", chunk.len()); + if let Some(propogate) = propagate.as_mut() { + propogate.write_all(chunk).await.ok(); + } callbacks.on_stderr(chunk); let bytes_read = chunk.len(); stderr.consume(bytes_read); } Err(e) => { + tracing::trace!("received stderr (err={})", e); callbacks.on_stderr_error(e); break; } } } + + propagate } type SetupBuilder = Box Result<(), anyhow::Error> + Send + Sync>; @@ -207,6 +267,7 @@ pub(crate) struct SharedState { pub(crate) module_hash: ModuleHash, pub(crate) dialect: CgiDialect, pub(crate) program_name: String, + pub(crate) propagate_stderr: bool, #[derivative(Debug = "ignore")] pub(crate) setup_builder: SetupBuilder, #[derivative(Debug = "ignore")] diff --git a/lib/wasix/src/runners/wcgi/mod.rs b/lib/wasix/src/runners/wcgi/mod.rs index e39e197f6af..4fb1fccd9cb 100644 --- a/lib/wasix/src/runners/wcgi/mod.rs +++ b/lib/wasix/src/runners/wcgi/mod.rs @@ -3,3 +3,4 @@ mod runner; pub use self::runner::{Callbacks, Config, WcgiRunner}; pub use futures::future::AbortHandle; +pub(crate) use handler::{Handler, SharedState}; diff --git a/lib/wasix/src/runners/wcgi/runner.rs b/lib/wasix/src/runners/wcgi/runner.rs index 918fc4157e2..3bdd6fcdb23 100644 --- a/lib/wasix/src/runners/wcgi/runner.rs +++ b/lib/wasix/src/runners/wcgi/runner.rs @@ -4,7 +4,7 @@ use anyhow::{Context, Error}; use futures::future::AbortHandle; use http::{Request, Response}; use hyper::Body; -use tower::{make::Shared, ServiceBuilder}; +use tower::{make::Shared, Service, ServiceBuilder}; use tower_http::{catch_panic::CatchPanicLayer, cors::CorsLayer, trace::TraceLayer}; use tracing::Span; use wcgi_host::CgiDialect; @@ -40,10 +40,12 @@ impl WcgiRunner { } #[tracing::instrument(skip_all)] - fn prepare_handler( + pub(crate) fn prepare_handler( &mut self, command_name: &str, pkg: &BinaryPackage, + propagate_stderr: bool, + default_dialect: CgiDialect, runtime: Arc, ) -> Result { let cmd = pkg @@ -59,7 +61,7 @@ impl WcgiRunner { let Wcgi { dialect, .. } = metadata.annotation("wcgi")?.unwrap_or_default(); let dialect = match dialect { Some(d) => d.parse().context("Unable to parse the CGI dialect")?, - None => CgiDialect::Wcgi, + None => default_dialect, }; let container_fs = Arc::clone(&pkg.webc_fs); @@ -77,32 +79,32 @@ impl WcgiRunner { module, module_hash: pkg.hash(), dialect, + propagate_stderr, program_name: command_name.to_string(), setup_builder: Box::new(setup_builder), callbacks: Arc::clone(&self.config.callbacks), runtime, }; - Ok(Handler::new(shared)) + Ok(Handler::new(Arc::new(shared))) } -} -impl crate::runners::Runner for WcgiRunner { - fn can_run_command(command: &Command) -> Result { - Ok(command - .runner - .starts_with(webc::metadata::annotations::WCGI_RUNNER_URI)) - } - - fn run_command( + pub(crate) fn run_command_with_handler( &mut self, - command_name: &str, - pkg: &BinaryPackage, + handler: S, runtime: Arc, - ) -> Result<(), Error> { - let handler = self.prepare_handler(command_name, pkg, Arc::clone(&runtime))?; - let callbacks = Arc::clone(&self.config.callbacks); - + ) -> Result<(), Error> + where + S: Service< + Request, + Response = http::Response, + Error = anyhow::Error, + Future = std::pin::Pin< + Box, Error>> + Send>, + >, + >, + S: Clone + Send + Sync + 'static, + { let service = ServiceBuilder::new() .layer( TraceLayer::new_for_http() @@ -126,6 +128,7 @@ impl crate::runners::Runner for WcgiRunner { let address = self.config.addr; tracing::info!(%address, "Starting the server"); + let callbacks = Arc::clone(&self.config.callbacks); runtime .task_manager() .spawn_and_block_on(async move { @@ -148,13 +151,37 @@ impl crate::runners::Runner for WcgiRunner { } } +impl crate::runners::Runner for WcgiRunner { + fn can_run_command(command: &Command) -> Result { + Ok(command + .runner + .starts_with(webc::metadata::annotations::WCGI_RUNNER_URI)) + } + + fn run_command( + &mut self, + command_name: &str, + pkg: &BinaryPackage, + runtime: Arc, + ) -> Result<(), Error> { + let handler = self.prepare_handler( + command_name, + pkg, + false, + CgiDialect::Wcgi, + Arc::clone(&runtime), + )?; + self.run_command_with_handler(handler, runtime) + } +} + #[derive(derivative::Derivative)] #[derivative(Debug)] pub struct Config { - wasi: CommonWasiOptions, - addr: SocketAddr, + pub(crate) wasi: CommonWasiOptions, + pub(crate) addr: SocketAddr, #[derivative(Debug = "ignore")] - callbacks: Arc, + pub(crate) callbacks: Arc, } impl Config { From aba3c65ef86a2407dde0538f24bdec0a4914beb8 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Thu, 21 Dec 2023 21:32:23 +1100 Subject: [PATCH 103/129] Finished off the basics of DCGI --- lib/cli/src/commands/run/mod.rs | 19 +- lib/wasix/src/fs/mod.rs | 427 +++++++++--------- lib/wasix/src/os/task/control_plane.rs | 2 +- lib/wasix/src/runners/dcgi/callbacks.rs | 55 +++ lib/wasix/src/runners/dcgi/factory.rs | 162 +++++++ lib/wasix/src/runners/dcgi/handler.rs | 51 ++- lib/wasix/src/runners/dcgi/instance.rs | 9 + lib/wasix/src/runners/dcgi/meta.rs | 21 + lib/wasix/src/runners/dcgi/mod.rs | 8 + lib/wasix/src/runners/dcgi/runner.rs | 38 +- lib/wasix/src/runners/wasi_common.rs | 1 + lib/wasix/src/runners/wcgi/callbacks.rs | 69 +++ lib/wasix/src/runners/wcgi/create_env.rs | 54 +++ lib/wasix/src/runners/wcgi/handler.rs | 147 +++--- lib/wasix/src/runners/wcgi/mod.rs | 7 +- lib/wasix/src/runners/wcgi/runner.rs | 73 +-- lib/wasix/src/state/builder.rs | 215 +-------- lib/wasix/src/state/env.rs | 45 +- lib/wasix/src/state/mod.rs | 6 +- lib/wasix/src/state/run.rs | 204 +++++++++ lib/wasix/src/syscalls/wasi/environ_get.rs | 8 +- .../src/syscalls/wasi/environ_sizes_get.rs | 11 +- lib/wasix/tests/runners.rs | 2 +- 23 files changed, 1081 insertions(+), 553 deletions(-) create mode 100644 lib/wasix/src/runners/dcgi/callbacks.rs create mode 100644 lib/wasix/src/runners/dcgi/factory.rs create mode 100644 lib/wasix/src/runners/dcgi/instance.rs create mode 100644 lib/wasix/src/runners/dcgi/meta.rs create mode 100644 lib/wasix/src/runners/wcgi/callbacks.rs create mode 100644 lib/wasix/src/runners/wcgi/create_env.rs create mode 100644 lib/wasix/src/state/run.rs diff --git a/lib/cli/src/commands/run/mod.rs b/lib/cli/src/commands/run/mod.rs index 3db59f3ce71..496af2c9cc0 100644 --- a/lib/cli/src/commands/run/mod.rs +++ b/lib/cli/src/commands/run/mod.rs @@ -32,7 +32,7 @@ use wasmer_registry::{wasmer_env::WasmerEnv, Package}; use wasmer_wasix::journal::{LogFileJournal, SnapshotTrigger}; use wasmer_wasix::{ bin_factory::BinaryPackage, - runners::{wcgi, MappedCommand, MappedDirectory, Runner}, + runners::{dcgi::DcgiInstanceFactory, wcgi, MappedCommand, MappedDirectory, Runner}, runtime::{ module_cache::{CacheError, ModuleHash}, package_loader::PackageLoader, @@ -247,11 +247,14 @@ impl Run { runner.run_command(command_name, pkg, runtime) } - fn config_wcgi( + fn config_wcgi( &self, - config: &mut wcgi::Config, + config: &mut wcgi::Config, uses: Vec, - ) -> Result<(), Error> { + ) -> Result<(), Error> + where + M: Send + Sync + 'static, + { config .args(self.args.clone()) .addr(self.wcgi.addr) @@ -295,7 +298,8 @@ impl Run { uses: Vec, runtime: Arc, ) -> Result<(), Error> { - let mut runner = wasmer_wasix::runners::dcgi::DcgiRunner::new(); + let factory = DcgiInstanceFactory::new(); + let mut runner = wasmer_wasix::runners::dcgi::DcgiRunner::new(factory); self.config_wcgi(runner.config().inner(), uses); runner.run_command(command_name, pkg, runtime) } @@ -790,7 +794,10 @@ impl Callbacks { } } -impl wasmer_wasix::runners::wcgi::Callbacks for Callbacks { +impl wasmer_wasix::runners::wcgi::Callbacks for Callbacks +where + M: Send + Sync + 'static, +{ fn started(&self, _abort: AbortHandle) { println!("WCGI Server running at http://{}/", self.addr); } diff --git a/lib/wasix/src/fs/mod.rs b/lib/wasix/src/fs/mod.rs index a14e298d05f..ceada11883e 100644 --- a/lib/wasix/src/fs/mod.rs +++ b/lib/wasix/src/fs/mod.rs @@ -515,175 +515,8 @@ impl WasiFs { vfs_preopens: &[String], fs_backing: WasiFsRoot, ) -> Result { - let (wasi_fs, root_inode) = Self::new_init(fs_backing, inodes)?; - - for preopen_name in vfs_preopens { - let kind = Kind::Dir { - parent: root_inode.downgrade(), - path: PathBuf::from(preopen_name), - entries: Default::default(), - }; - let rights = Rights::FD_ADVISE - | Rights::FD_TELL - | Rights::FD_SEEK - | Rights::FD_READ - | Rights::PATH_OPEN - | Rights::FD_READDIR - | Rights::PATH_READLINK - | Rights::PATH_FILESTAT_GET - | Rights::FD_FILESTAT_GET - | Rights::PATH_LINK_SOURCE - | Rights::PATH_RENAME_SOURCE - | Rights::POLL_FD_READWRITE - | Rights::SOCK_SHUTDOWN; - let inode = wasi_fs - .create_inode(inodes, kind, true, preopen_name.clone()) - .map_err(|e| { - format!( - "Failed to create inode for preopened dir (name `{}`): WASI error code: {}", - preopen_name, e - ) - })?; - let fd_flags = Fd::READ; - let fd = wasi_fs - .create_fd(rights, rights, Fdflags::empty(), fd_flags, inode.clone()) - .map_err(|e| format!("Could not open fd for file {:?}: {}", preopen_name, e))?; - { - let mut guard = root_inode.write(); - if let Kind::Root { entries } = guard.deref_mut() { - let existing_entry = entries.insert(preopen_name.clone(), inode); - if existing_entry.is_some() { - return Err(format!( - "Found duplicate entry for alias `{}`", - preopen_name - )); - } - assert!(existing_entry.is_none()) - } - } - wasi_fs.preopen_fds.write().unwrap().push(fd); - } - - for PreopenedDir { - path, - alias, - read, - write, - create, - } in preopens - { - debug!( - "Attempting to preopen {} with alias {:?}", - &path.to_string_lossy(), - &alias - ); - let cur_dir_metadata = wasi_fs - .root_fs - .metadata(path) - .map_err(|e| format!("Could not get metadata for file {:?}: {}", path, e))?; - - let kind = if cur_dir_metadata.is_dir() { - Kind::Dir { - parent: root_inode.downgrade(), - path: path.clone(), - entries: Default::default(), - } - } else { - return Err(format!( - "WASI only supports pre-opened directories right now; found \"{}\"", - &path.to_string_lossy() - )); - }; - - let rights = { - // TODO: review tell' and fd_readwrite - let mut rights = Rights::FD_ADVISE | Rights::FD_TELL | Rights::FD_SEEK; - if *read { - rights |= Rights::FD_READ - | Rights::PATH_OPEN - | Rights::FD_READDIR - | Rights::PATH_READLINK - | Rights::PATH_FILESTAT_GET - | Rights::FD_FILESTAT_GET - | Rights::PATH_LINK_SOURCE - | Rights::PATH_RENAME_SOURCE - | Rights::POLL_FD_READWRITE - | Rights::SOCK_SHUTDOWN; - } - if *write { - rights |= Rights::FD_DATASYNC - | Rights::FD_FDSTAT_SET_FLAGS - | Rights::FD_WRITE - | Rights::FD_SYNC - | Rights::FD_ALLOCATE - | Rights::PATH_OPEN - | Rights::PATH_RENAME_TARGET - | Rights::PATH_FILESTAT_SET_SIZE - | Rights::PATH_FILESTAT_SET_TIMES - | Rights::FD_FILESTAT_SET_SIZE - | Rights::FD_FILESTAT_SET_TIMES - | Rights::PATH_REMOVE_DIRECTORY - | Rights::PATH_UNLINK_FILE - | Rights::POLL_FD_READWRITE - | Rights::SOCK_SHUTDOWN; - } - if *create { - rights |= Rights::PATH_CREATE_DIRECTORY - | Rights::PATH_CREATE_FILE - | Rights::PATH_LINK_TARGET - | Rights::PATH_OPEN - | Rights::PATH_RENAME_TARGET - | Rights::PATH_SYMLINK; - } - - rights - }; - let inode = if let Some(alias) = &alias { - wasi_fs.create_inode(inodes, kind, true, alias.clone()) - } else { - wasi_fs.create_inode(inodes, kind, true, path.to_string_lossy().into_owned()) - } - .map_err(|e| { - format!( - "Failed to create inode for preopened dir: WASI error code: {}", - e - ) - })?; - let fd_flags = { - let mut fd_flags = 0; - if *read { - fd_flags |= Fd::READ; - } - if *write { - // TODO: introduce API for finer grained control - fd_flags |= Fd::WRITE | Fd::APPEND | Fd::TRUNCATE; - } - if *create { - fd_flags |= Fd::CREATE; - } - fd_flags - }; - let fd = wasi_fs - .create_fd(rights, rights, Fdflags::empty(), fd_flags, inode.clone()) - .map_err(|e| format!("Could not open fd for file {:?}: {}", path, e))?; - { - let mut guard = root_inode.write(); - if let Kind::Root { entries } = guard.deref_mut() { - let key = if let Some(alias) = &alias { - alias.clone() - } else { - path.to_string_lossy().into_owned() - }; - let existing_entry = entries.insert(key.clone(), inode); - if existing_entry.is_some() { - return Err(format!("Found duplicate entry for alias `{}`", key)); - } - assert!(existing_entry.is_none()) - } - } - wasi_fs.preopen_fds.write().unwrap().push(fd); - } - + let wasi_fs = Self::new_init(fs_backing, inodes)?; + wasi_fs.create_preopens(inodes, preopens, vfs_preopens)?; Ok(wasi_fs) } @@ -701,7 +534,7 @@ impl WasiFs { /// Private helper function to init the filesystem, called in `new` and /// `new_with_preopen` - fn new_init(fs_backing: WasiFsRoot, inodes: &WasiInodes) -> Result<(Self, InodeGuard), String> { + fn new_init(fs_backing: WasiFsRoot, inodes: &WasiInodes) -> Result { debug!("Initializing WASI filesystem"); let stat = Filestat { @@ -731,40 +564,9 @@ impl WasiFs { wasi_fs.create_stdin(inodes); wasi_fs.create_stdout(inodes); wasi_fs.create_stderr(inodes); + wasi_fs.create_rootfd()?; - // create virtual root - let all_rights = ALL_RIGHTS; - // TODO: make this a list of positive rigths instead of negative ones - // root gets all right for now - let root_rights = all_rights - /* - & (!Rights::FD_WRITE) - & (!Rights::FD_ALLOCATE) - & (!Rights::PATH_CREATE_DIRECTORY) - & (!Rights::PATH_CREATE_FILE) - & (!Rights::PATH_LINK_SOURCE) - & (!Rights::PATH_RENAME_SOURCE) - & (!Rights::PATH_RENAME_TARGET) - & (!Rights::PATH_FILESTAT_SET_SIZE) - & (!Rights::PATH_FILESTAT_SET_TIMES) - & (!Rights::FD_FILESTAT_SET_SIZE) - & (!Rights::FD_FILESTAT_SET_TIMES) - & (!Rights::PATH_SYMLINK) - & (!Rights::PATH_UNLINK_FILE) - & (!Rights::PATH_REMOVE_DIRECTORY) - */; - let fd = wasi_fs - .create_fd( - root_rights, - root_rights, - Fdflags::empty(), - Fd::READ, - root_inode.clone(), - ) - .map_err(|e| format!("Could not create root fd: {}", e))?; - wasi_fs.preopen_fds.write().unwrap().push(fd); - - Ok((wasi_fs, root_inode)) + Ok(wasi_fs) } /// This function is like create dir all, but it also opens it. @@ -1738,7 +1540,7 @@ impl WasiFs { guard.lookup.remove(&ino).and_then(|a| Weak::upgrade(&a)) } - fn create_stdout(&self, inodes: &WasiInodes) { + pub(crate) fn create_stdout(&self, inodes: &WasiInodes) { self.create_std_dev_inner( inodes, Box::::default(), @@ -1749,7 +1551,7 @@ impl WasiFs { ); } - fn create_stdin(&self, inodes: &WasiInodes) { + pub(crate) fn create_stdin(&self, inodes: &WasiInodes) { self.create_std_dev_inner( inodes, Box::::default(), @@ -1760,7 +1562,7 @@ impl WasiFs { ); } - fn create_stderr(&self, inodes: &WasiInodes) { + pub(crate) fn create_stderr(&self, inodes: &WasiInodes) { self.create_std_dev_inner( inodes, Box::::default(), @@ -1771,7 +1573,218 @@ impl WasiFs { ); } - fn create_std_dev_inner( + pub(crate) fn create_rootfd(&self) -> Result<(), String> { + // create virtual root + let all_rights = ALL_RIGHTS; + // TODO: make this a list of positive rigths instead of negative ones + // root gets all right for now + let root_rights = all_rights + /* + & (!Rights::FD_WRITE) + & (!Rights::FD_ALLOCATE) + & (!Rights::PATH_CREATE_DIRECTORY) + & (!Rights::PATH_CREATE_FILE) + & (!Rights::PATH_LINK_SOURCE) + & (!Rights::PATH_RENAME_SOURCE) + & (!Rights::PATH_RENAME_TARGET) + & (!Rights::PATH_FILESTAT_SET_SIZE) + & (!Rights::PATH_FILESTAT_SET_TIMES) + & (!Rights::FD_FILESTAT_SET_SIZE) + & (!Rights::FD_FILESTAT_SET_TIMES) + & (!Rights::PATH_SYMLINK) + & (!Rights::PATH_UNLINK_FILE) + & (!Rights::PATH_REMOVE_DIRECTORY) + */; + let fd = self + .create_fd( + root_rights, + root_rights, + Fdflags::empty(), + Fd::READ, + self.root_inode.clone(), + ) + .map_err(|e| format!("Could not create root fd: {}", e))?; + self.preopen_fds.write().unwrap().push(fd); + Ok(()) + } + + pub(crate) fn create_preopens( + &self, + inodes: &WasiInodes, + preopens: &[PreopenedDir], + vfs_preopens: &[String], + ) -> Result<(), String> { + for preopen_name in vfs_preopens { + let kind = Kind::Dir { + parent: self.root_inode.downgrade(), + path: PathBuf::from(preopen_name), + entries: Default::default(), + }; + let rights = Rights::FD_ADVISE + | Rights::FD_TELL + | Rights::FD_SEEK + | Rights::FD_READ + | Rights::PATH_OPEN + | Rights::FD_READDIR + | Rights::PATH_READLINK + | Rights::PATH_FILESTAT_GET + | Rights::FD_FILESTAT_GET + | Rights::PATH_LINK_SOURCE + | Rights::PATH_RENAME_SOURCE + | Rights::POLL_FD_READWRITE + | Rights::SOCK_SHUTDOWN; + let inode = self + .create_inode(inodes, kind, true, preopen_name.clone()) + .map_err(|e| { + format!( + "Failed to create inode for preopened dir (name `{}`): WASI error code: {}", + preopen_name, e + ) + })?; + let fd_flags = Fd::READ; + let fd = self + .create_fd(rights, rights, Fdflags::empty(), fd_flags, inode.clone()) + .map_err(|e| format!("Could not open fd for file {:?}: {}", preopen_name, e))?; + { + let mut guard = self.root_inode.write(); + if let Kind::Root { entries } = guard.deref_mut() { + let existing_entry = entries.insert(preopen_name.clone(), inode); + if existing_entry.is_some() { + return Err(format!( + "Found duplicate entry for alias `{}`", + preopen_name + )); + } + assert!(existing_entry.is_none()) + } + } + self.preopen_fds.write().unwrap().push(fd); + } + + for PreopenedDir { + path, + alias, + read, + write, + create, + } in preopens + { + debug!( + "Attempting to preopen {} with alias {:?}", + &path.to_string_lossy(), + &alias + ); + let cur_dir_metadata = self + .root_fs + .metadata(path) + .map_err(|e| format!("Could not get metadata for file {:?}: {}", path, e))?; + + let kind = if cur_dir_metadata.is_dir() { + Kind::Dir { + parent: self.root_inode.downgrade(), + path: path.clone(), + entries: Default::default(), + } + } else { + return Err(format!( + "WASI only supports pre-opened directories right now; found \"{}\"", + &path.to_string_lossy() + )); + }; + + let rights = { + // TODO: review tell' and fd_readwrite + let mut rights = Rights::FD_ADVISE | Rights::FD_TELL | Rights::FD_SEEK; + if *read { + rights |= Rights::FD_READ + | Rights::PATH_OPEN + | Rights::FD_READDIR + | Rights::PATH_READLINK + | Rights::PATH_FILESTAT_GET + | Rights::FD_FILESTAT_GET + | Rights::PATH_LINK_SOURCE + | Rights::PATH_RENAME_SOURCE + | Rights::POLL_FD_READWRITE + | Rights::SOCK_SHUTDOWN; + } + if *write { + rights |= Rights::FD_DATASYNC + | Rights::FD_FDSTAT_SET_FLAGS + | Rights::FD_WRITE + | Rights::FD_SYNC + | Rights::FD_ALLOCATE + | Rights::PATH_OPEN + | Rights::PATH_RENAME_TARGET + | Rights::PATH_FILESTAT_SET_SIZE + | Rights::PATH_FILESTAT_SET_TIMES + | Rights::FD_FILESTAT_SET_SIZE + | Rights::FD_FILESTAT_SET_TIMES + | Rights::PATH_REMOVE_DIRECTORY + | Rights::PATH_UNLINK_FILE + | Rights::POLL_FD_READWRITE + | Rights::SOCK_SHUTDOWN; + } + if *create { + rights |= Rights::PATH_CREATE_DIRECTORY + | Rights::PATH_CREATE_FILE + | Rights::PATH_LINK_TARGET + | Rights::PATH_OPEN + | Rights::PATH_RENAME_TARGET + | Rights::PATH_SYMLINK; + } + + rights + }; + let inode = if let Some(alias) = &alias { + self.create_inode(inodes, kind, true, alias.clone()) + } else { + self.create_inode(inodes, kind, true, path.to_string_lossy().into_owned()) + } + .map_err(|e| { + format!( + "Failed to create inode for preopened dir: WASI error code: {}", + e + ) + })?; + let fd_flags = { + let mut fd_flags = 0; + if *read { + fd_flags |= Fd::READ; + } + if *write { + // TODO: introduce API for finer grained control + fd_flags |= Fd::WRITE | Fd::APPEND | Fd::TRUNCATE; + } + if *create { + fd_flags |= Fd::CREATE; + } + fd_flags + }; + let fd = self + .create_fd(rights, rights, Fdflags::empty(), fd_flags, inode.clone()) + .map_err(|e| format!("Could not open fd for file {:?}: {}", path, e))?; + { + let mut guard = self.root_inode.write(); + if let Kind::Root { entries } = guard.deref_mut() { + let key = if let Some(alias) = &alias { + alias.clone() + } else { + path.to_string_lossy().into_owned() + }; + let existing_entry = entries.insert(key.clone(), inode); + if existing_entry.is_some() { + return Err(format!("Found duplicate entry for alias `{}`", key)); + } + assert!(existing_entry.is_none()) + } + } + self.preopen_fds.write().unwrap().push(fd); + } + + Ok(()) + } + + pub(crate) fn create_std_dev_inner( &self, inodes: &WasiInodes, handle: Box, diff --git a/lib/wasix/src/os/task/control_plane.rs b/lib/wasix/src/os/task/control_plane.rs index 7bbc7894887..96d5575e62e 100644 --- a/lib/wasix/src/os/task/control_plane.rs +++ b/lib/wasix/src/os/task/control_plane.rs @@ -109,7 +109,7 @@ impl WasiControlPlane { /// Register a new task. /// // Currently just increments the task counter. - pub(super) fn register_task(&self) -> Result { + pub(crate) fn register_task(&self) -> Result { let count = self.state.task_count.fetch_add(1, Ordering::SeqCst); if let Some(max) = self.state.config.max_task_count { if count > max { diff --git a/lib/wasix/src/runners/dcgi/callbacks.rs b/lib/wasix/src/runners/dcgi/callbacks.rs new file mode 100644 index 00000000000..048db03b364 --- /dev/null +++ b/lib/wasix/src/runners/dcgi/callbacks.rs @@ -0,0 +1,55 @@ +use std::sync::Arc; + +use derivative::Derivative; + +use super::*; +use crate::runners::wcgi::{self, CreateEnvConfig, CreateEnvResult, RecycleEnvConfig}; + +#[derive(Derivative, Clone)] +#[derivative(Debug)] +pub struct DcgiCallbacks { + #[derivative(Debug = "ignore")] + inner: Arc>, + factory: DcgiInstanceFactory, +} + +impl DcgiCallbacks { + pub fn new(factory: DcgiInstanceFactory, inner: C) -> Self + where + C: wcgi::Callbacks, + { + Self { + inner: Arc::new(inner), + factory, + } + } +} + +#[async_trait::async_trait] +impl wcgi::Callbacks for DcgiCallbacks { + fn started(&self, abort: AbortHandle) { + self.inner.started(abort) + } + + fn on_stderr(&self, stderr: &[u8]) { + self.inner.on_stderr(stderr) + } + + fn on_stderr_error(&self, error: std::io::Error) { + self.inner.on_stderr_error(error) + } + + async fn recycle_env(&self, conf: RecycleEnvConfig) { + self.factory.release(conf).await; + } + + async fn create_env( + &self, + mut conf: CreateEnvConfig, + ) -> anyhow::Result { + if let Some(res) = self.factory.acquire(&mut conf).await { + return Ok(res); + } + self.inner.create_env(conf).await + } +} diff --git a/lib/wasix/src/runners/dcgi/factory.rs b/lib/wasix/src/runners/dcgi/factory.rs new file mode 100644 index 00000000000..53c67b3bde6 --- /dev/null +++ b/lib/wasix/src/runners/dcgi/factory.rs @@ -0,0 +1,162 @@ +use std::{ + collections::{HashMap, VecDeque}, + sync::{Arc, Mutex}, + time::Instant, +}; + +use virtual_fs::Pipe; +use wasmer_wasix_types::types::{__WASI_STDERR_FILENO, __WASI_STDIN_FILENO, __WASI_STDOUT_FILENO}; + +use crate::{ + runners::wcgi::{CreateEnvConfig, CreateEnvResult, RecycleEnvConfig}, + state::conv_env_vars, + WasiStateCreationError, +}; + +use super::*; + +#[derive(Debug)] +struct StateShard { + instances: VecDeque, + last_acquire: Instant, + master_lock: Arc>, +} + +impl Default for StateShard { + fn default() -> Self { + Self { + instances: Default::default(), + last_acquire: Instant::now(), + master_lock: Arc::new(Default::default()), + } + } +} + +#[derive(Debug, Default)] +struct State { + // List of the shards and a queue of DCGI instances that + // are running for these shards + shards: HashMap, +} + +/// This factory will store and reuse instances between invocations thus +/// allowing for the instances to be stateful. +#[derive(Debug, Clone, Default)] +pub struct DcgiInstanceFactory { + state: Arc>, +} + +impl DcgiInstanceFactory { + pub fn new() -> Self { + Self { + state: Default::default(), + } + } + + pub async fn release(&self, mut conf: RecycleEnvConfig) { + let shard = conf.meta.shard; + + let mut state = self.state.lock().unwrap(); + state + .shards + .entry(shard) + .or_default() + .instances + .push_front(DcgiInstance { + env: conf.env, + store: conf.store, + }); + + drop(state); + conf.meta.master_lock.take(); + } + + pub async fn acquire( + &self, + conf: &mut CreateEnvConfig, + ) -> Option { + let shard = conf.meta.shard.clone(); + + // We take a short lock that looks for existing instances + // that have been recycled, otherwise we will use the + // master lock to prevent concurrent creations + let master_lock = { + let mut state = self.state.lock().unwrap(); + let shard = state.shards.entry(shard.clone()).or_default(); + shard.last_acquire = Instant::now(); + shard.master_lock.clone() + }; + + // We acquire a master lock whenever creating a new instance and hold it + // until the instance dies or the instance is returned to the factory. This + // is done using the `DcgiMetadata` + let master_lock = master_lock.clone().lock_owned().await; + conf.meta.master_lock.replace(Arc::new(master_lock)); + + // We check the shard again under a short lock as maybe one was returned + { + let mut state = self.state.lock().unwrap(); + let shard = state.shards.entry(shard).or_default(); + shard.last_acquire = Instant::now(); + if let Some(inst) = shard.instances.pop_front() { + if let Ok(converted) = convert_instance(inst, conf) { + return Some(converted); + } + } + } + + None + } +} + +fn convert_instance( + inst: DcgiInstance, + conf: &mut CreateEnvConfig, +) -> anyhow::Result { + let env = inst.env; + let mut store = inst.store; + + let (req_body_sender, req_body_receiver) = Pipe::channel(); + let (res_body_sender, res_body_receiver) = Pipe::channel(); + let (stderr_sender, stderr_receiver) = Pipe::channel(); + + { + // Reinitialize the environment + let env = env.data_mut(&mut store); + env.reinit()?; + + // Replace the environment variables as these will change + // depending on the WCGI call + *env.state.envs.lock().unwrap() = conv_env_vars( + conf.env + .iter() + .map(|(k, v)| (k.clone(), v.as_bytes().to_vec())) + .collect(), + ); + + // The stdio have to be reattached on each call as they are + // read to completion (EOF) during nominal flows + env.state + .fs + .swap_file(__WASI_STDIN_FILENO, Box::new(req_body_receiver)) + .map_err(WasiStateCreationError::FileSystemError)?; + + env.state + .fs + .swap_file(__WASI_STDOUT_FILENO, Box::new(res_body_sender)) + .map_err(WasiStateCreationError::FileSystemError)?; + + env.state + .fs + .swap_file(__WASI_STDERR_FILENO, Box::new(stderr_sender)) + .map_err(WasiStateCreationError::FileSystemError)?; + } + + Ok(CreateEnvResult { + env, + store, + body_sender: req_body_sender, + body_receiver: res_body_receiver, + stderr_receiver, + }) +} diff --git a/lib/wasix/src/runners/dcgi/handler.rs b/lib/wasix/src/runners/dcgi/handler.rs index 94b7223c498..6b90bdae2af 100644 --- a/lib/wasix/src/runners/dcgi/handler.rs +++ b/lib/wasix/src/runners/dcgi/handler.rs @@ -1,40 +1,40 @@ use std::{ops::Deref, pin::Pin, sync::Arc, task::Poll}; use anyhow::Error; -use futures::Future; +use futures::{Future, FutureExt}; use http::{Request, Response}; use hyper::{service::Service, Body}; use crate::runners::wcgi; +use super::{DcgiInstanceFactory, DcgiMetadata}; + /// The shared object that manages the instantiaion of WASI executables and /// communicating with them via the CGI protocol. #[derive(Clone, Debug)] pub(crate) struct Handler { state: Arc, - inner: wcgi::Handler, + inner: wcgi::Handler, } impl Handler { - pub(crate) fn new(state: SharedState) -> Self { - Handler { - inner: wcgi::Handler::new(state.inner.clone()), - state: Arc::new(state), - } - } - - pub(crate) fn from_wcgi_handler(handler: wcgi::Handler) -> Self { + pub(crate) fn from_wcgi_handler(handler: wcgi::Handler) -> Self { Handler { state: Arc::new(SharedState { inner: handler.deref().clone(), + factory: DcgiInstanceFactory::default(), }), inner: handler, } } #[tracing::instrument(level = "debug", skip_all, err)] - pub(crate) async fn handle(&self, req: Request) -> Result, Error> { - self.inner.handle(req).await + pub(crate) async fn handle( + &self, + req: Request, + meta: DcgiMetadata, + ) -> Result, Error> { + self.inner.handle(req, meta).await } } @@ -49,7 +49,8 @@ impl Deref for Handler { #[derive(derivative::Derivative, Clone)] #[derivative(Debug)] pub(crate) struct SharedState { - pub(crate) inner: Arc, + pub(crate) inner: Arc>, + factory: DcgiInstanceFactory, } impl Service> for Handler { @@ -57,11 +58,29 @@ impl Service> for Handler { type Error = Error; type Future = Pin, Error>> + Send>>; - fn poll_ready(&mut self, cx: &mut std::task::Context<'_>) -> Poll> { - self.inner.poll_ready(cx) + fn poll_ready(&mut self, _cx: &mut std::task::Context<'_>) -> Poll> { + // TODO: We probably should implement some sort of backpressure here... + Poll::Ready(Ok(())) } fn call(&mut self, request: Request) -> Self::Future { - self.inner.call(request) + // We determine the shard that this DCGI request will run against + // (multiple shards can be served by the same endpoint) + let shard = request + .headers() + .get("X-Shard") + .map(|s| s.to_str().unwrap_or_else(|_| "").to_string()) + .unwrap_or_else(|| "".to_string()); + + // Grab the metadata from the request + let meta = DcgiMetadata { + shard, + master_lock: None, + }; + + // Note: all fields are reference-counted so cloning is pretty cheap + let handler = self.clone(); + let fut = async move { handler.handle(request, meta).await }; + fut.boxed() } } diff --git a/lib/wasix/src/runners/dcgi/instance.rs b/lib/wasix/src/runners/dcgi/instance.rs new file mode 100644 index 00000000000..dec077445e8 --- /dev/null +++ b/lib/wasix/src/runners/dcgi/instance.rs @@ -0,0 +1,9 @@ +use wasmer::Store; + +use crate::WasiFunctionEnv; + +#[derive(Debug)] +pub(crate) struct DcgiInstance { + pub env: WasiFunctionEnv, + pub store: Store, +} diff --git a/lib/wasix/src/runners/dcgi/meta.rs b/lib/wasix/src/runners/dcgi/meta.rs new file mode 100644 index 00000000000..5db2e1d258d --- /dev/null +++ b/lib/wasix/src/runners/dcgi/meta.rs @@ -0,0 +1,21 @@ +use std::sync::Arc; + +#[derive(Debug, Clone)] +pub struct DcgiMetadata { + /// Shard associated with this WCGI + pub shard: String, + /// This master lock prevents multiple writable instances + /// from running at the same time. It is held for the duration + /// of the instance running until it returns to the factory + /// or its dropped, for example if an error occurs + pub master_lock: Option>>, +} + +impl Default for DcgiMetadata { + fn default() -> Self { + DcgiMetadata { + shard: Default::default(), + master_lock: None, + } + } +} diff --git a/lib/wasix/src/runners/dcgi/mod.rs b/lib/wasix/src/runners/dcgi/mod.rs index 6e0b572cb84..101b880c7c8 100644 --- a/lib/wasix/src/runners/dcgi/mod.rs +++ b/lib/wasix/src/runners/dcgi/mod.rs @@ -1,5 +1,13 @@ +mod callbacks; +mod factory; mod handler; +mod instance; +mod meta; mod runner; pub use self::runner::{Config, DcgiRunner}; +pub use callbacks::DcgiCallbacks; +pub use factory::DcgiInstanceFactory; pub use futures::future::AbortHandle; +pub(crate) use instance::DcgiInstance; +pub use meta::DcgiMetadata; diff --git a/lib/wasix/src/runners/dcgi/runner.rs b/lib/wasix/src/runners/dcgi/runner.rs index e1034aa37db..67963f7b9ce 100644 --- a/lib/wasix/src/runners/dcgi/runner.rs +++ b/lib/wasix/src/runners/dcgi/runner.rs @@ -7,19 +7,31 @@ use webc::metadata::Command; use crate::{ bin_factory::BinaryPackage, capabilities::Capabilities, - runners::{dcgi::handler::Handler, wcgi, MappedDirectory}, + runners::{ + dcgi::handler::Handler, + wcgi::{self, NoopCallbacks}, + MappedDirectory, + }, Runtime, }; -#[derive(Debug, Default)] +use super::{DcgiCallbacks, DcgiInstanceFactory, DcgiMetadata}; + +#[derive(Debug)] pub struct DcgiRunner { config: Config, - inner: wcgi::WcgiRunner, + inner: wcgi::WcgiRunner, } impl DcgiRunner { - pub fn new() -> Self { - DcgiRunner::default() + pub fn new(factory: DcgiInstanceFactory) -> Self { + let callbacks = DcgiCallbacks::new(factory, NoopCallbacks); + DcgiRunner { + config: Config { + inner: wcgi::Config::new(callbacks), + }, + inner: Default::default(), + } } pub fn config(&mut self) -> &mut Config { @@ -33,7 +45,7 @@ impl DcgiRunner { pkg: &BinaryPackage, runtime: Arc, ) -> Result { - let inner: wcgi::Handler = + let inner: wcgi::Handler = self.inner .prepare_handler(command_name, pkg, true, CgiDialect::Rfc3875, runtime)?; Ok(Handler::from_wcgi_handler(inner)) @@ -61,11 +73,11 @@ impl crate::runners::Runner for DcgiRunner { #[derive(Debug)] pub struct Config { - inner: wcgi::Config, + inner: wcgi::Config, } impl Config { - pub fn inner(&mut self) -> &mut wcgi::Config { + pub fn inner(&mut self) -> &mut wcgi::Config { &mut self.inner } @@ -130,7 +142,7 @@ impl Config { /// lifecycle. pub fn callbacks( &mut self, - callbacks: impl wcgi::Callbacks + Send + Sync + 'static, + callbacks: impl wcgi::Callbacks + Send + Sync + 'static, ) -> &mut Self { self.inner.callbacks(callbacks); self @@ -179,14 +191,6 @@ impl Config { } } -impl Default for Config { - fn default() -> Self { - Self { - inner: Default::default(), - } - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/lib/wasix/src/runners/wasi_common.rs b/lib/wasix/src/runners/wasi_common.rs index f037c3d7f1e..453fa3c077c 100644 --- a/lib/wasix/src/runners/wasi_common.rs +++ b/lib/wasix/src/runners/wasi_common.rs @@ -52,6 +52,7 @@ impl CommonWasiOptions { root_fs: Option, ) -> Result<(), anyhow::Error> { let root_fs = root_fs.unwrap_or_else(|| RootFileSystemBuilder::default().build()); + let fs = prepare_filesystem(root_fs, &self.mapped_dirs, container_fs, builder)?; builder.add_preopen_dir("/")?; diff --git a/lib/wasix/src/runners/wcgi/callbacks.rs b/lib/wasix/src/runners/wcgi/callbacks.rs new file mode 100644 index 00000000000..a8b1cfbe7b8 --- /dev/null +++ b/lib/wasix/src/runners/wcgi/callbacks.rs @@ -0,0 +1,69 @@ +use std::{collections::HashMap, sync::Arc}; + +use virtual_fs::Pipe; +use wasmer::{Module, Store}; + +use crate::{runtime::module_cache::ModuleHash, WasiFunctionEnv}; + +use super::{create_env::default_recycle_env, handler::SetupBuilder, *}; + +/// Configuration used for creating a new environment +pub struct CreateEnvConfig +where + M: Send + Sync + 'static, +{ + pub meta: M, + pub env: HashMap, + pub program_name: String, + pub module: Module, + pub module_hash: ModuleHash, + pub runtime: Arc, + pub setup_builder: SetupBuilder, +} + +/// Result of a create operation on a new environment +pub struct CreateEnvResult { + pub env: WasiFunctionEnv, + pub store: Store, + pub body_sender: Pipe, + pub body_receiver: Pipe, + pub stderr_receiver: Pipe, +} + +/// Configuration used for reusing an new environment +pub struct RecycleEnvConfig { + pub meta: M, + pub env: WasiFunctionEnv, + pub store: Store, +} + +/// Callbacks that are triggered at various points in the lifecycle of a runner +/// and any WebAssembly instances it may start. +#[async_trait::async_trait] +pub trait Callbacks: Send + Sync + 'static +where + M: Send + Sync + 'static, +{ + /// A callback that is called whenever the server starts. + fn started(&self, _abort: AbortHandle) {} + + /// Data was written to stderr by an instance. + fn on_stderr(&self, _stderr: &[u8]) {} + + /// Reading from stderr failed. + fn on_stderr_error(&self, _error: std::io::Error) {} + + /// Recycle the WASI environment + async fn recycle_env(&self, conf: RecycleEnvConfig) { + default_recycle_env(conf).await + } + + /// Create the WASI environment + async fn create_env(&self, conf: CreateEnvConfig) -> anyhow::Result { + default_create_env(conf).await + } +} + +pub(crate) struct NoopCallbacks; + +impl Callbacks for NoopCallbacks where M: Send + Sync + 'static {} diff --git a/lib/wasix/src/runners/wcgi/create_env.rs b/lib/wasix/src/runners/wcgi/create_env.rs new file mode 100644 index 00000000000..2c57e80e738 --- /dev/null +++ b/lib/wasix/src/runners/wcgi/create_env.rs @@ -0,0 +1,54 @@ +use virtual_fs::Pipe; + +use crate::{ + capabilities::Capabilities, http::HttpClientCapabilityV1, + runners::wcgi::callbacks::CreateEnvResult, WasiEnvBuilder, +}; + +use super::{callbacks::CreateEnvConfig, RecycleEnvConfig}; + +pub(crate) async fn default_recycle_env(_conf: RecycleEnvConfig) +where + M: Send + Sync + 'static, +{ + tracing::debug!("Destroying the WebAssembly instance"); +} + +pub(crate) async fn default_create_env( + conf: CreateEnvConfig, +) -> anyhow::Result +where + M: Send + Sync + 'static, +{ + tracing::debug!("Creating the WebAssembly instance"); + + let (req_body_sender, req_body_receiver) = Pipe::channel(); + let (res_body_sender, res_body_receiver) = Pipe::channel(); + let (stderr_sender, stderr_receiver) = Pipe::channel(); + + let mut builder = WasiEnvBuilder::new(&conf.program_name); + + (conf.setup_builder)(&mut builder)?; + + builder.add_envs(conf.env); + + let builder = builder + .stdin(Box::new(req_body_receiver)) + .stdout(Box::new(res_body_sender)) + .stderr(Box::new(stderr_sender)) + .capabilities(Capabilities { + insecure_allow_all: true, + http_client: HttpClientCapabilityV1::new_allow_all(), + threading: Default::default(), + }); + + let mut store = conf.runtime.new_store(); + let (_, env) = builder.instantiate_ext(conf.module, conf.module_hash, &mut store)?; + Ok(CreateEnvResult { + env, + store, + body_sender: req_body_sender, + body_receiver: res_body_receiver, + stderr_receiver, + }) +} diff --git a/lib/wasix/src/runners/wcgi/handler.rs b/lib/wasix/src/runners/wcgi/handler.rs index dcc89101620..9ba00ddfd49 100644 --- a/lib/wasix/src/runners/wcgi/handler.rs +++ b/lib/wasix/src/runners/wcgi/handler.rs @@ -10,36 +10,38 @@ use wasmer::Module; use wcgi_host::CgiDialect; use crate::{ - capabilities::Capabilities, http::HttpClientCapabilityV1, runners::wcgi::Callbacks, - runtime::module_cache::ModuleHash, Pipe, Runtime, VirtualTaskManager, WasiEnvBuilder, + runners::wcgi::{ + callbacks::{CreateEnvConfig, RecycleEnvConfig}, + Callbacks, + }, + runtime::module_cache::ModuleHash, + Runtime, VirtualTaskManager, WasiEnvBuilder, }; /// The shared object that manages the instantiaion of WASI executables and /// communicating with them via the CGI protocol. #[derive(Clone, Debug)] -pub(crate) struct Handler(Arc); - -impl Handler { - pub(crate) fn new(state: Arc) -> Self { +pub(crate) struct Handler(Arc>) +where + M: Send + Sync + 'static; + +impl Handler +where + M: Send + Sync + 'static, +{ + pub(crate) fn new(state: Arc>) -> Self { Handler(state) } #[tracing::instrument(level = "debug", skip_all, err)] - pub(crate) async fn handle(&self, req: Request) -> Result, Error> { + pub(crate) async fn handle(&self, req: Request, meta: M) -> Result, Error> + where + M: Clone, + { tracing::debug!(headers=?req.headers()); let (parts, body) = req.into_parts(); - let (req_body_sender, req_body_receiver) = Pipe::channel(); - let (res_body_sender, res_body_receiver) = Pipe::channel(); - let (stderr_sender, stderr_receiver) = Pipe::channel(); - - tracing::debug!("Creating the WebAssembly instance"); - - let mut builder = WasiEnvBuilder::new(&self.program_name); - - (self.setup_builder)(&mut builder)?; - // Note: We want to apply the CGI environment variables *after* // anything specified by WASI annotations so users get a chance to // override things like $DOCUMENT_ROOT and $SCRIPT_FILENAME. @@ -52,20 +54,18 @@ impl Handler { self.dialect .prepare_environment_variables(parts, &mut request_specific_env); - builder.add_envs(request_specific_env); - - let builder = builder - .stdin(Box::new(req_body_receiver)) - .stdout(Box::new(res_body_sender)) - .stderr(Box::new(stderr_sender)) - .capabilities(Capabilities { - insecure_allow_all: true, - http_client: HttpClientCapabilityV1::new_allow_all(), - threading: Default::default(), - }); - - let module = self.module.clone(); - let module_hash = self.module_hash; + let create = self + .callbacks + .create_env(CreateEnvConfig { + meta: meta.clone(), + env: request_specific_env, + program_name: self.program_name.clone(), + module: self.module.clone(), + module_hash: self.module_hash, + runtime: self.runtime.clone(), + setup_builder: self.setup_builder.clone(), + }) + .await?; tracing::debug!( dialect=%self.dialect, @@ -73,38 +73,51 @@ impl Handler { ); let task_manager = self.runtime.task_manager(); - let store = self.runtime.new_store(); - + let env = create.env; + let store = create.store; let (run_tx, mut run_rx) = tokio::sync::mpsc::unbounded_channel(); task_manager.task_dedicated(Box::new(move || { - run_tx - .send(builder.run_with_store_async(module, module_hash, store)) - .ok(); + run_tx.send(env.run_async(store)).ok(); }))?; let done = async move { run_rx.recv().await.unwrap().map_err(Error::from) }; - let mut res_body_receiver = tokio::io::BufReader::new(res_body_receiver); + let mut res_body_receiver = tokio::io::BufReader::new(create.body_receiver); + let stderr_receiver = create.stderr_receiver; let callbacks = Arc::clone(&self.callbacks); let propagate_stderr = self.propagate_stderr; - let work_consume_stderr = + let work_consume_stderr = { + let callbacks = callbacks.clone(); async move { consume_stderr(stderr_receiver, callbacks, propagate_stderr).await } - .in_current_span(); + .in_current_span() + }; tracing::trace!( dialect=%self.dialect, "spawning request forwarder", ); - let work_drive_io = async move { - if let Err(e) = drive_request_to_completion(done, body, req_body_sender).await { - tracing::error!( - error = &*e as &dyn std::error::Error, - "Unable to drive the request to completion" - ); + let req_body_sender = create.body_sender; + let callbacks = Arc::clone(&self.callbacks); + let work_drive_io = { + async move { + let ret = drive_request_to_completion(done, body, req_body_sender).await; + match ret { + Ok((env, store)) => { + callbacks + .recycle_env(RecycleEnvConfig { meta, env, store }) + .await; + } + Err(e) => { + tracing::error!( + error = &*e as &dyn std::error::Error, + "Unable to drive the request to completion" + ); + } + } } - } - .in_current_span(); + .in_current_span() + }; task_manager .task_shared(Box::new(move || Box::pin(work_drive_io))) .ok(); @@ -171,8 +184,11 @@ impl Handler { } } -impl Deref for Handler { - type Target = Arc; +impl Deref for Handler +where + M: Send + Sync + 'static, +{ + type Target = Arc>; fn deref(&self) -> &Self::Target { &self.0 @@ -181,11 +197,11 @@ impl Deref for Handler { /// Drive the request to completion by streaming the request body to the /// instance and waiting for it to exit. -async fn drive_request_to_completion( - done: impl Future>, +async fn drive_request_to_completion( + done: impl Future>, mut request_body: hyper::Body, mut instance_stdin: impl AsyncWrite + Send + Unpin + 'static, -) -> Result<(), Error> { +) -> Result { let request_body_send = async move { // Copy the request into our instance, chunk-by-chunk. If the instance // dies before we finish writing the body, the instance's side of the @@ -209,19 +225,21 @@ async fn drive_request_to_completion( } .in_current_span(); - futures::try_join!(done, request_body_send)?; - - Ok(()) + let (ret, _) = futures::try_join!(done, request_body_send)?; + Ok(ret) } /// Read the instance's stderr, taking care to preserve output even when WASI /// pipe errors occur so users still have *something* they use for /// troubleshooting. -async fn consume_stderr( +async fn consume_stderr( stderr: impl AsyncRead + Send + Unpin + 'static, - callbacks: Arc, + callbacks: Arc>, propagate_stderr: bool, -) -> Option> { +) -> Option> +where + M: Send + Sync + 'static, +{ let mut stderr = tokio::io::BufReader::new(stderr); let mut propagate = match propagate_stderr { @@ -258,11 +276,14 @@ async fn consume_stderr( propagate } -type SetupBuilder = Box Result<(), anyhow::Error> + Send + Sync>; +pub type SetupBuilder = Arc Result<(), anyhow::Error> + Send + Sync>; #[derive(derivative::Derivative)] #[derivative(Debug)] -pub(crate) struct SharedState { +pub(crate) struct SharedState +where + M: Send + Sync + 'static, +{ pub(crate) module: Module, pub(crate) module_hash: ModuleHash, pub(crate) dialect: CgiDialect, @@ -271,12 +292,12 @@ pub(crate) struct SharedState { #[derivative(Debug = "ignore")] pub(crate) setup_builder: SetupBuilder, #[derivative(Debug = "ignore")] - pub(crate) callbacks: Arc, + pub(crate) callbacks: Arc>, #[derivative(Debug = "ignore")] pub(crate) runtime: Arc, } -impl Service> for Handler { +impl Service> for Handler<()> { type Response = Response; type Error = Error; type Future = Pin, Error>> + Send>>; @@ -289,7 +310,7 @@ impl Service> for Handler { fn call(&mut self, request: Request) -> Self::Future { // Note: all fields are reference-counted so cloning is pretty cheap let handler = self.clone(); - let fut = async move { handler.handle(request).await }; + let fut = async move { handler.handle(request, ()).await }; fut.boxed() } } diff --git a/lib/wasix/src/runners/wcgi/mod.rs b/lib/wasix/src/runners/wcgi/mod.rs index 4fb1fccd9cb..9990c0af7cf 100644 --- a/lib/wasix/src/runners/wcgi/mod.rs +++ b/lib/wasix/src/runners/wcgi/mod.rs @@ -1,6 +1,11 @@ +mod callbacks; +mod create_env; mod handler; mod runner; -pub use self::runner::{Callbacks, Config, WcgiRunner}; +pub use self::runner::{Config, WcgiRunner}; +pub(crate) use callbacks::NoopCallbacks; +pub use callbacks::{Callbacks, CreateEnvConfig, CreateEnvResult, RecycleEnvConfig}; +pub(crate) use create_env::default_create_env; pub use futures::future::AbortHandle; pub(crate) use handler::{Handler, SharedState}; diff --git a/lib/wasix/src/runners/wcgi/runner.rs b/lib/wasix/src/runners/wcgi/runner.rs index 3bdd6fcdb23..f0982c07260 100644 --- a/lib/wasix/src/runners/wcgi/runner.rs +++ b/lib/wasix/src/runners/wcgi/runner.rs @@ -1,7 +1,6 @@ use std::{net::SocketAddr, sync::Arc, time::Duration}; use anyhow::{Context, Error}; -use futures::future::AbortHandle; use http::{Request, Response}; use hyper::Body; use tower::{make::Shared, Service, ServiceBuilder}; @@ -25,17 +24,28 @@ use crate::{ Runtime, WasiEnvBuilder, }; +use super::{Callbacks, NoopCallbacks}; + #[derive(Debug, Default)] -pub struct WcgiRunner { - config: Config, +pub struct WcgiRunner +where + M: Send + Sync + 'static, +{ + config: Config, } -impl WcgiRunner { - pub fn new() -> Self { +impl WcgiRunner +where + M: Send + Sync + 'static, +{ + pub fn new() -> Self + where + M: Default, + { WcgiRunner::default() } - pub fn config(&mut self) -> &mut Config { + pub fn config(&mut self) -> &mut Config { &mut self.config } @@ -47,7 +57,7 @@ impl WcgiRunner { propagate_stderr: bool, default_dialect: CgiDialect, runtime: Arc, - ) -> Result { + ) -> Result, Error> { let cmd = pkg .get_command(command_name) .with_context(|| format!("The package doesn't contain a \"{command_name}\" command"))?; @@ -81,7 +91,7 @@ impl WcgiRunner { dialect, propagate_stderr, program_name: command_name.to_string(), - setup_builder: Box::new(setup_builder), + setup_builder: Arc::new(setup_builder), callbacks: Arc::clone(&self.config.callbacks), runtime, }; @@ -177,14 +187,17 @@ impl crate::runners::Runner for WcgiRunner { #[derive(derivative::Derivative)] #[derivative(Debug)] -pub struct Config { +pub struct Config { pub(crate) wasi: CommonWasiOptions, pub(crate) addr: SocketAddr, #[derivative(Debug = "ignore")] - pub(crate) callbacks: Arc, + pub(crate) callbacks: Arc>, } -impl Config { +impl Config +where + M: Send + Sync + 'static, +{ pub fn addr(&mut self, addr: SocketAddr) -> &mut Self { self.addr = addr; self @@ -246,7 +259,7 @@ impl Config { /// Set callbacks that will be triggered at various points in the runner's /// lifecycle. - pub fn callbacks(&mut self, callbacks: impl Callbacks + Send + Sync + 'static) -> &mut Self { + pub fn callbacks(&mut self, callbacks: impl Callbacks + Send + Sync + 'static) -> &mut Self { self.callbacks = Arc::new(callbacks); self } @@ -306,33 +319,31 @@ impl Config { } } -impl Default for Config { +impl Default for Config +where + M: Send + Sync + 'static, +{ fn default() -> Self { + Self::new(NoopCallbacks) + } +} + +impl Config +where + M: Send + Sync + 'static, +{ + pub fn new(callbacks: C) -> Self + where + C: Callbacks, + { Self { addr: ([127, 0, 0, 1], 8000).into(), wasi: CommonWasiOptions::default(), - callbacks: Arc::new(NoopCallbacks), + callbacks: Arc::new(callbacks), } } } -/// Callbacks that are triggered at various points in the lifecycle of a runner -/// and any WebAssembly instances it may start. -pub trait Callbacks: Send + Sync + 'static { - /// A callback that is called whenever the server starts. - fn started(&self, _abort: AbortHandle) {} - - /// Data was written to stderr by an instance. - fn on_stderr(&self, _stderr: &[u8]) {} - - /// Reading from stderr failed. - fn on_stderr_error(&self, _error: std::io::Error) {} -} - -struct NoopCallbacks; - -impl Callbacks for NoopCallbacks {} - #[cfg(test)] mod tests { use super::*; diff --git a/lib/wasix/src/state/builder.rs b/lib/wasix/src/state/builder.rs index 082fed4880d..7a23c5e3712 100644 --- a/lib/wasix/src/state/builder.rs +++ b/lib/wasix/src/state/builder.rs @@ -9,8 +9,7 @@ use std::{ use rand::Rng; use thiserror::Error; use virtual_fs::{ArcFile, FileSystem, FsError, TmpFileSystem, VirtualFile}; -use wasmer::{AsStoreMut, Instance, Module, RuntimeError, Store}; -use wasmer_wasix_types::wasi::{Errno, ExitCode}; +use wasmer::{AsStoreMut, Instance, Module, Store}; #[cfg(feature = "journal")] use crate::journal::{DynJournal, SnapshotTrigger}; @@ -21,13 +20,13 @@ use crate::{ capabilities::Capabilities, fs::{WasiFs, WasiFsRoot, WasiInodes}, os::task::control_plane::{ControlPlaneConfig, ControlPlaneError, WasiControlPlane}, - runtime::{module_cache::ModuleHash, task_manager::InlineWaker}, + runtime::module_cache::ModuleHash, state::WasiState, syscalls::{ rewind_ext2, types::{__WASI_STDERR_FILENO, __WASI_STDIN_FILENO, __WASI_STDOUT_FILENO}, }, - RewindStateOption, Runtime, WasiEnv, WasiError, WasiFunctionEnv, WasiRuntimeError, + Runtime, WasiEnv, WasiError, WasiFunctionEnv, WasiRuntimeError, }; use super::env::WasiEnvInit; @@ -786,19 +785,6 @@ impl WasiEnvBuilder { wasi_fs.set_current_dir(s); } - let envs = self - .envs - .into_iter() - .map(|(key, value)| { - let mut env = Vec::with_capacity(key.len() + value.len() + 1); - env.extend_from_slice(key.as_bytes()); - env.push(b'='); - env.extend_from_slice(&value); - - env - }) - .collect(); - let state = WasiState { fs: wasi_fs, secret: rand::thread_rng().gen::<[u8; 32]>(), @@ -807,7 +793,7 @@ impl WasiEnvBuilder { preopen: self.vfs_preopens.clone(), futexs: Default::default(), clock_offset: Default::default(), - envs, + envs: std::sync::Mutex::new(conv_env_vars(self.envs)), }; let runtime = self.runtime.unwrap_or_else(|| { @@ -973,7 +959,7 @@ impl WasiEnvBuilder { env.data(&store).thread.set_status_running(); let result = crate::run_wasi_func_start(start, store); - let (result, exit_code) = wasi_exit_code(result); + let (result, exit_code) = super::wasi_exit_code(result); let pid = env.data(&store).pid(); let tid = env.data(&store).tid(); @@ -998,190 +984,23 @@ impl WasiEnvBuilder { module_hash: ModuleHash, mut store: Store, ) -> Result<(), WasiRuntimeError> { - // If no handle or runtime exists then create one - #[cfg(feature = "sys-thread")] - let _guard = if tokio::runtime::Handle::try_current().is_err() { - let runtime = tokio::runtime::Builder::new_multi_thread() - .enable_all() - .build() - .unwrap(); - Some(runtime) - } else { - None - }; - #[cfg(feature = "sys-thread")] - let _guard = _guard.as_ref().map(|r| r.enter()); - let (_, env) = self.instantiate_ext(module, module_hash, &mut store)?; - - env.data(&store).thread.set_status_running(); - - let tasks = env.data(&store).tasks().clone(); - let pid = env.data(&store).pid(); - let tid = env.data(&store).tid(); - - // The return value is passed synchronously and will block until the result - // is returned this is because the main thread can go into a deep sleep and - // exit the dedicated thread - let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel(); - - tasks.task_dedicated(Box::new(move || { - // Unsafe: The bootstrap must be executed in the same thread that runs the - // actual WASM code - let rewind_state = unsafe { - match env.bootstrap(&mut store) { - Ok(a) => a, - Err(err) => { - env.on_exit(&mut store, None); - tx.send(Err(err)).ok(); - return; - } - } - }; - run_with_deep_sleep(store, rewind_state, env, tx); - }))?; - - let result = InlineWaker::block_on(rx.recv()); - let result = result.unwrap_or_else(|| { - Err(WasiRuntimeError::Runtime(RuntimeError::new( - "main thread terminated without a result, this normally means a panic occurred", - ))) - }); - let (result, exit_code) = wasi_exit_code(result); - - tracing::trace!( - %pid, - %tid, - %exit_code, - error=result.as_ref().err().map(|e| e as &dyn std::error::Error), - "main exit", - ); - - result - } -} - -/// Extract the exit code from a `Result<(), WasiRuntimeError>`. -/// -/// We need this because calling `exit(0)` inside a WASI program technically -/// triggers [`WasiError`] with an exit code of `0`, but the end user won't want -/// that treated as an error. -fn wasi_exit_code( - mut result: Result<(), WasiRuntimeError>, -) -> (Result<(), WasiRuntimeError>, ExitCode) { - let exit_code = match &result { - Ok(_) => Errno::Success.into(), - Err(err) => match err.as_exit_code() { - Some(code) if code.is_success() => { - // This is actually not an error, so we need to fix up the - // result - result = Ok(()); - Errno::Success.into() - } - Some(other) => other, - None => Errno::Noexec.into(), - }, - }; - - (result, exit_code) -} - -fn run_with_deep_sleep( - mut store: Store, - rewind_state: RewindStateOption, - env: WasiFunctionEnv, - sender: tokio::sync::mpsc::UnboundedSender>, -) { - if let Some((rewind_state, rewind_result)) = rewind_state { - tracing::trace!("Rewinding"); - let mut ctx = env.env.clone().into_mut(&mut store); - let errno = if rewind_state.is_64bit { - crate::rewind_ext::( - &mut ctx, - rewind_state.memory_stack, - rewind_state.rewind_stack, - rewind_state.store_data, - rewind_result, - ) - } else { - crate::rewind_ext::( - &mut ctx, - rewind_state.memory_stack, - rewind_state.rewind_stack, - rewind_state.store_data, - rewind_result, - ) - }; - - if errno != Errno::Success { - let exit_code = ExitCode::from(errno); - env.on_exit(&mut store, Some(exit_code)); - let _ = sender.send(Err(WasiRuntimeError::Wasi(WasiError::Exit(exit_code)))); - return; - } + env.run_async(store)?; + Ok(()) } - - let instance = match env.data(&store).try_clone_instance() { - Some(instance) => instance, - None => { - tracing::debug!("Unable to clone the instance"); - env.on_exit(&mut store, None); - let _ = sender.send(Err(WasiRuntimeError::Wasi(WasiError::Exit( - Errno::Noexec.into(), - )))); - return; - } - }; - - let start = match instance.exports.get_function("_start") { - Ok(start) => start, - Err(e) => { - tracing::debug!("Unable to get the _start function"); - env.on_exit(&mut store, None); - let _ = sender.send(Err(e.into())); - return; - } - }; - - let result = start.call(&mut store, &[]); - handle_result(store, env, result, sender); } -fn handle_result( - mut store: Store, - env: WasiFunctionEnv, - result: Result, RuntimeError>, - sender: tokio::sync::mpsc::UnboundedSender>, -) { - let result = match result.map_err(|e| e.downcast::()) { - Err(Ok(WasiError::DeepSleep(work))) => { - let pid = env.data(&store).pid(); - let tid = env.data(&store).tid(); - tracing::trace!(%pid, %tid, "entered a deep sleep"); - - let tasks = env.data(&store).tasks().clone(); - let rewind = work.rewind; - let respawn = move |ctx, store, res| { - run_with_deep_sleep(store, Some((rewind, Some(res))), ctx, sender) - }; - - // Spawns the WASM process after a trigger - unsafe { - tasks - .resume_wasm_after_poller(Box::new(respawn), env, store, work.trigger) - .unwrap(); - } +pub(crate) fn conv_env_vars(envs: Vec<(String, Vec)>) -> Vec> { + envs.into_iter() + .map(|(key, value)| { + let mut env = Vec::with_capacity(key.len() + value.len() + 1); + env.extend_from_slice(key.as_bytes()); + env.push(b'='); + env.extend_from_slice(&value); - return; - } - Ok(_) => Ok(()), - Err(Ok(other)) => Err(other.into()), - Err(Err(e)) => Err(e.into()), - }; - - let (result, exit_code) = wasi_exit_code(result); - env.on_exit(&mut store, Some(exit_code)); - sender.send(result).ok(); + env + }) + .collect() } /// Builder for preopened directories. diff --git a/lib/wasix/src/state/env.rs b/lib/wasix/src/state/env.rs index d934bd39ad8..f1755f7d2bd 100644 --- a/lib/wasix/src/state/env.rs +++ b/lib/wasix/src/state/env.rs @@ -261,7 +261,7 @@ impl WasiEnvInit { self.state.clock_offset.lock().unwrap().clone(), ), args: self.state.args.clone(), - envs: self.state.envs.clone(), + envs: std::sync::Mutex::new(self.state.envs.lock().unwrap().deref().clone()), preopen: self.state.preopen.clone(), }, runtime: self.runtime.clone(), @@ -407,6 +407,49 @@ impl WasiEnv { self.thread.tid() } + /// Re-initializes this environment so that it can be executed again + pub fn reinit(&mut self) -> Result<(), WasiStateCreationError> { + // First we clear any open files as the descriptors would + // otherwise clash + if let Ok(mut map) = self.state.fs.fd_map.write() { + map.clear(); + } + self.state.fs.preopen_fds.write().unwrap().clear(); + self.state + .fs + .next_fd + .store(3, std::sync::atomic::Ordering::SeqCst); + *self.state.fs.current_dir.lock().unwrap() = "/".to_string(); + + // We need to rebuild the basic file descriptors + // (note: this does not include pre-opened files which are not + // supported yet with reinit calls) + self.state.fs.create_stdin(&self.state.inodes); + self.state.fs.create_stdout(&self.state.inodes); + self.state.fs.create_stderr(&self.state.inodes); + self.state + .fs + .create_rootfd() + .map_err(WasiStateCreationError::WasiFsSetupError)?; + + // The process and thread state need to be reset + self.process = WasiProcess::new( + self.process.pid, + self.process.module_hash, + self.process.compute.clone(), + ); + self.thread = WasiThread::new( + self.thread.pid(), + self.thread.tid(), + self.thread.is_main(), + self.process.finished.clone(), + self.process.compute.must_upgrade().register_task()?, + self.thread.memory_layout().clone(), + ); + + Ok(()) + } + /// Returns true if this module is capable of deep sleep /// (needs asyncify to unwind and rewin) /// diff --git a/lib/wasix/src/state/mod.rs b/lib/wasix/src/state/mod.rs index e18dac9fbbd..aa8f6dc91a9 100644 --- a/lib/wasix/src/state/mod.rs +++ b/lib/wasix/src/state/mod.rs @@ -19,6 +19,7 @@ mod builder; mod env; mod func_env; mod handles; +mod run; mod types; use std::{ @@ -29,6 +30,7 @@ use std::{ time::Duration, }; +use run::*; #[cfg(feature = "enable-serde")] use serde::{Deserialize, Serialize}; use virtual_fs::{FileOpener, FileSystem, FsError, OpenOptions, VirtualFile}; @@ -131,7 +133,7 @@ pub(crate) struct WasiState { pub futexs: Mutex, pub clock_offset: Mutex>, pub args: Vec, - pub envs: Vec>, + pub envs: Mutex>>, // TODO: should not be here, since this requires active work to resolve. // State should only hold active runtime state that can be reproducibly re-created. @@ -253,7 +255,7 @@ impl WasiState { futexs: Default::default(), clock_offset: Mutex::new(self.clock_offset.lock().unwrap().clone()), args: self.args.clone(), - envs: self.envs.clone(), + envs: Mutex::new(self.envs.lock().unwrap().clone()), preopen: self.preopen.clone(), } } diff --git a/lib/wasix/src/state/run.rs b/lib/wasix/src/state/run.rs new file mode 100644 index 00000000000..48ba2b15816 --- /dev/null +++ b/lib/wasix/src/state/run.rs @@ -0,0 +1,204 @@ +use virtual_mio::InlineWaker; +use wasmer::{RuntimeError, Store}; +use wasmer_wasix_types::wasi::ExitCode; + +use crate::{RewindStateOption, WasiError, WasiRuntimeError}; + +use super::*; + +impl WasiFunctionEnv { + pub fn run_async(self, mut store: Store) -> Result<(Self, Store), WasiRuntimeError> { + // If no handle or runtime exists then create one + #[cfg(feature = "sys-thread")] + let _guard = if tokio::runtime::Handle::try_current().is_err() { + let runtime = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .unwrap(); + Some(runtime) + } else { + None + }; + #[cfg(feature = "sys-thread")] + let _guard = _guard.as_ref().map(|r| r.enter()); + + self.data(&store).thread.set_status_running(); + + let tasks = self.data(&store).tasks().clone(); + let pid = self.data(&store).pid(); + let tid = self.data(&store).tid(); + + // The return value is passed synchronously and will block until the result + // is returned this is because the main thread can go into a deep sleep and + // exit the dedicated thread + let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel(); + + let this = self.clone(); + tasks.task_dedicated(Box::new(move || { + // Unsafe: The bootstrap must be executed in the same thread that runs the + // actual WASM code + let rewind_state = unsafe { + match this.bootstrap(&mut store) { + Ok(a) => a, + Err(err) => { + this.on_exit(&mut store, None); + tx.send(Err(err)).ok(); + return; + } + } + }; + run_with_deep_sleep(store, rewind_state, this, tx); + }))?; + + let result = InlineWaker::block_on(rx.recv()); + let store = match result { + Some(result) => { + tracing::trace!( + %pid, + %tid, + error=result.as_ref().err().map(|e| e as &dyn std::error::Error), + "main exit", + ); + result? + } + None => { + tracing::trace!( + %pid, + %tid, + "main premature termination", + ); + return Err(WasiRuntimeError::Runtime(RuntimeError::new( + "main thread terminated without a result, this normally means a panic occurred", + ))); + } + }; + Ok((self, store)) + } +} + +fn run_with_deep_sleep( + mut store: Store, + rewind_state: RewindStateOption, + env: WasiFunctionEnv, + sender: tokio::sync::mpsc::UnboundedSender>, +) { + if let Some((rewind_state, rewind_result)) = rewind_state { + tracing::trace!("Rewinding"); + let mut ctx = env.env.clone().into_mut(&mut store); + let errno = if rewind_state.is_64bit { + crate::rewind_ext::( + &mut ctx, + rewind_state.memory_stack, + rewind_state.rewind_stack, + rewind_state.store_data, + rewind_result, + ) + } else { + crate::rewind_ext::( + &mut ctx, + rewind_state.memory_stack, + rewind_state.rewind_stack, + rewind_state.store_data, + rewind_result, + ) + }; + + if errno != Errno::Success { + let exit_code = ExitCode::from(errno); + env.on_exit(&mut store, Some(exit_code)); + if exit_code.is_success() { + let _ = sender.send(Ok(store)); + } else { + let _ = sender.send(Err(WasiRuntimeError::Wasi(WasiError::Exit(exit_code)))); + } + return; + } + } + + let instance = match env.data(&store).try_clone_instance() { + Some(instance) => instance, + None => { + tracing::debug!("Unable to clone the instance"); + env.on_exit(&mut store, None); + let _ = sender.send(Err(WasiRuntimeError::Wasi(WasiError::Exit( + Errno::Noexec.into(), + )))); + return; + } + }; + + let start = match instance.exports.get_function("_start") { + Ok(start) => start, + Err(e) => { + tracing::debug!("Unable to get the _start function"); + env.on_exit(&mut store, None); + let _ = sender.send(Err(e.into())); + return; + } + }; + + let result = start.call(&mut store, &[]); + handle_result(store, env, result, sender); +} + +fn handle_result( + mut store: Store, + env: WasiFunctionEnv, + result: Result, RuntimeError>, + sender: tokio::sync::mpsc::UnboundedSender>, +) { + let result: Result<_, WasiRuntimeError> = match result.map_err(|e| e.downcast::()) { + Err(Ok(WasiError::DeepSleep(work))) => { + let pid = env.data(&store).pid(); + let tid = env.data(&store).tid(); + tracing::trace!(%pid, %tid, "entered a deep sleep"); + + let tasks = env.data(&store).tasks().clone(); + let rewind = work.rewind; + let respawn = move |ctx, store, res| { + run_with_deep_sleep(store, Some((rewind, Some(res))), ctx, sender) + }; + + // Spawns the WASM process after a trigger + unsafe { + tasks + .resume_wasm_after_poller(Box::new(respawn), env, store, work.trigger) + .unwrap(); + } + + return; + } + Ok(_) => Ok(()), + Err(Ok(other)) => Err(other.into()), + Err(Err(e)) => Err(e.into()), + }; + + let (result, exit_code) = wasi_exit_code(result); + env.on_exit(&mut store, Some(exit_code)); + sender.send(result.map(|_| store)).ok(); +} + +/// Extract the exit code from a `Result<(), WasiRuntimeError>`. +/// +/// We need this because calling `exit(0)` inside a WASI program technically +/// triggers [`WasiError`] with an exit code of `0`, but the end user won't want +/// that treated as an error. +pub(super) fn wasi_exit_code( + mut result: Result<(), WasiRuntimeError>, +) -> (Result<(), WasiRuntimeError>, ExitCode) { + let exit_code = match &result { + Ok(_) => Errno::Success.into(), + Err(err) => match err.as_exit_code() { + Some(code) if code.is_success() => { + // This is actually not an error, so we need to fix up the + // result + result = Ok(()); + Errno::Success.into() + } + Some(other) => other, + None => Errno::Noexec.into(), + }, + }; + + (result, exit_code) +} diff --git a/lib/wasix/src/syscalls/wasi/environ_get.rs b/lib/wasix/src/syscalls/wasi/environ_get.rs index a39a5faa5f6..0c3b0c60735 100644 --- a/lib/wasix/src/syscalls/wasi/environ_get.rs +++ b/lib/wasix/src/syscalls/wasi/environ_get.rs @@ -23,10 +23,6 @@ pub fn environ_get( let env = ctx.data(); let (memory, mut state) = unsafe { env.get_memory_and_wasi_state(&ctx, 0) }; - Ok(write_buffer_array( - &memory, - &state.envs, - environ, - environ_buf, - )) + let envs = state.envs.lock().unwrap(); + Ok(write_buffer_array(&memory, &envs, environ, environ_buf)) } diff --git a/lib/wasix/src/syscalls/wasi/environ_sizes_get.rs b/lib/wasix/src/syscalls/wasi/environ_sizes_get.rs index 846b8e92839..73f4dddd9a6 100644 --- a/lib/wasix/src/syscalls/wasi/environ_sizes_get.rs +++ b/lib/wasix/src/syscalls/wasi/environ_sizes_get.rs @@ -25,9 +25,14 @@ pub fn environ_sizes_get( let environ_count = environ_count.deref(&memory); let environ_buf_size = environ_buf_size.deref(&memory); - let env_var_count: M::Offset = - wasi_try_ok!(state.envs.len().try_into().map_err(|_| Errno::Overflow)); - let env_buf_size: usize = state.envs.iter().map(|v| v.len() + 1).sum(); + let env_var_count: M::Offset = wasi_try_ok!(state + .envs + .lock() + .unwrap() + .len() + .try_into() + .map_err(|_| Errno::Overflow)); + let env_buf_size: usize = state.envs.lock().unwrap().iter().map(|v| v.len() + 1).sum(); let env_buf_size: M::Offset = wasi_try_ok!(env_buf_size.try_into().map_err(|_| Errno::Overflow)); wasi_try_mem_ok!(environ_count.write(env_var_count)); diff --git a/lib/wasix/tests/runners.rs b/lib/wasix/tests/runners.rs index 95852f3458a..dbcf99cabd9 100644 --- a/lib/wasix/tests/runners.rs +++ b/lib/wasix/tests/runners.rs @@ -175,7 +175,7 @@ mod wcgi { handle: Handle, } - impl wasmer_wasix::runners::wcgi::Callbacks for Callbacks { + impl wasmer_wasix::runners::wcgi::Callbacks<()> for Callbacks { fn started(&self, abort: futures::stream::AbortHandle) { let mut sender = self.sender.clone(); self.handle.spawn(async move { From 355fcfa5f183798e7581592c29eb0f5b7e627514 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Thu, 21 Dec 2023 21:40:37 +1100 Subject: [PATCH 104/129] Now opening the preopens when the WASM environment is reinitialized --- lib/wasix/src/fs/mod.rs | 26 ++++++++++++++++---------- lib/wasix/src/state/env.rs | 6 ++++-- 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/lib/wasix/src/fs/mod.rs b/lib/wasix/src/fs/mod.rs index ceada11883e..849a53f2cb1 100644 --- a/lib/wasix/src/fs/mod.rs +++ b/lib/wasix/src/fs/mod.rs @@ -429,6 +429,11 @@ pub struct WasiFs { // but it shouldn't be necessary // It should not be necessary at all. is_wasix: AtomicBool, + + // The preopens when this was initialized + pub(crate) init_preopens: Vec, + // The virtual file system preopens when this was initialized + pub(crate) init_vfs_preopens: Vec, } impl WasiFs { @@ -454,6 +459,8 @@ impl WasiFs { root_fs: self.root_fs.clone(), root_inode: self.root_inode.clone(), has_unioned: Arc::new(Mutex::new(HashSet::new())), + init_preopens: self.init_preopens.clone(), + init_vfs_preopens: self.init_vfs_preopens.clone(), } } @@ -515,8 +522,10 @@ impl WasiFs { vfs_preopens: &[String], fs_backing: WasiFsRoot, ) -> Result { - let wasi_fs = Self::new_init(fs_backing, inodes)?; - wasi_fs.create_preopens(inodes, preopens, vfs_preopens)?; + let mut wasi_fs = Self::new_init(fs_backing, inodes)?; + wasi_fs.init_preopens = preopens.iter().cloned().collect(); + wasi_fs.init_vfs_preopens = vfs_preopens.iter().cloned().collect(); + wasi_fs.create_preopens(inodes)?; Ok(wasi_fs) } @@ -560,6 +569,8 @@ impl WasiFs { root_fs: fs_backing, root_inode: root_inode.clone(), has_unioned: Arc::new(Mutex::new(HashSet::new())), + init_preopens: Default::default(), + init_vfs_preopens: Default::default(), }; wasi_fs.create_stdin(inodes); wasi_fs.create_stdout(inodes); @@ -1608,13 +1619,8 @@ impl WasiFs { Ok(()) } - pub(crate) fn create_preopens( - &self, - inodes: &WasiInodes, - preopens: &[PreopenedDir], - vfs_preopens: &[String], - ) -> Result<(), String> { - for preopen_name in vfs_preopens { + pub(crate) fn create_preopens(&self, inodes: &WasiInodes) -> Result<(), String> { + for preopen_name in self.init_vfs_preopens.iter() { let kind = Kind::Dir { parent: self.root_inode.downgrade(), path: PathBuf::from(preopen_name), @@ -1667,7 +1673,7 @@ impl WasiFs { read, write, create, - } in preopens + } in self.init_preopens.iter() { debug!( "Attempting to preopen {} with alias {:?}", diff --git a/lib/wasix/src/state/env.rs b/lib/wasix/src/state/env.rs index f1755f7d2bd..9deaef32d63 100644 --- a/lib/wasix/src/state/env.rs +++ b/lib/wasix/src/state/env.rs @@ -422,8 +422,6 @@ impl WasiEnv { *self.state.fs.current_dir.lock().unwrap() = "/".to_string(); // We need to rebuild the basic file descriptors - // (note: this does not include pre-opened files which are not - // supported yet with reinit calls) self.state.fs.create_stdin(&self.state.inodes); self.state.fs.create_stdout(&self.state.inodes); self.state.fs.create_stderr(&self.state.inodes); @@ -431,6 +429,10 @@ impl WasiEnv { .fs .create_rootfd() .map_err(WasiStateCreationError::WasiFsSetupError)?; + self.state + .fs + .create_preopens(&self.state.inodes) + .map_err(WasiStateCreationError::WasiFsSetupError)?; // The process and thread state need to be reset self.process = WasiProcess::new( From a06789d1a7dae0d46c163e05605bb8e8b75cc6d6 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Thu, 21 Dec 2023 21:48:30 +1100 Subject: [PATCH 105/129] Fixed an issue where the callbacks for DCGI were not be wired properly --- lib/cli/src/commands/run/mod.rs | 8 ++++++-- lib/wasix/src/runners/dcgi/runner.rs | 8 ++++---- lib/wasix/src/runners/wcgi/callbacks.rs | 4 ++-- lib/wasix/src/runners/wcgi/mod.rs | 2 +- lib/wasix/src/runners/wcgi/runner.rs | 21 +++++++-------------- 5 files changed, 20 insertions(+), 23 deletions(-) diff --git a/lib/cli/src/commands/run/mod.rs b/lib/cli/src/commands/run/mod.rs index 496af2c9cc0..629d098522d 100644 --- a/lib/cli/src/commands/run/mod.rs +++ b/lib/cli/src/commands/run/mod.rs @@ -32,7 +32,11 @@ use wasmer_registry::{wasmer_env::WasmerEnv, Package}; use wasmer_wasix::journal::{LogFileJournal, SnapshotTrigger}; use wasmer_wasix::{ bin_factory::BinaryPackage, - runners::{dcgi::DcgiInstanceFactory, wcgi, MappedCommand, MappedDirectory, Runner}, + runners::{ + dcgi::DcgiInstanceFactory, + wcgi::{self, NoOpWcgiCallbacks}, + MappedCommand, MappedDirectory, Runner, + }, runtime::{ module_cache::{CacheError, ModuleHash}, package_loader::PackageLoader, @@ -242,7 +246,7 @@ impl Run { uses: Vec, runtime: Arc, ) -> Result<(), Error> { - let mut runner = wasmer_wasix::runners::wcgi::WcgiRunner::new(); + let mut runner = wasmer_wasix::runners::wcgi::WcgiRunner::new(NoOpWcgiCallbacks); self.config_wcgi(runner.config(), uses)?; runner.run_command(command_name, pkg, runtime) } diff --git a/lib/wasix/src/runners/dcgi/runner.rs b/lib/wasix/src/runners/dcgi/runner.rs index 67963f7b9ce..b95a230024f 100644 --- a/lib/wasix/src/runners/dcgi/runner.rs +++ b/lib/wasix/src/runners/dcgi/runner.rs @@ -9,7 +9,7 @@ use crate::{ capabilities::Capabilities, runners::{ dcgi::handler::Handler, - wcgi::{self, NoopCallbacks}, + wcgi::{self, NoOpWcgiCallbacks, WcgiRunner}, MappedDirectory, }, Runtime, @@ -25,12 +25,12 @@ pub struct DcgiRunner { impl DcgiRunner { pub fn new(factory: DcgiInstanceFactory) -> Self { - let callbacks = DcgiCallbacks::new(factory, NoopCallbacks); + let callbacks = DcgiCallbacks::new(factory, NoOpWcgiCallbacks); DcgiRunner { config: Config { - inner: wcgi::Config::new(callbacks), + inner: wcgi::Config::new(callbacks.clone()), }, - inner: Default::default(), + inner: WcgiRunner::new(callbacks), } } diff --git a/lib/wasix/src/runners/wcgi/callbacks.rs b/lib/wasix/src/runners/wcgi/callbacks.rs index a8b1cfbe7b8..2300aefec5e 100644 --- a/lib/wasix/src/runners/wcgi/callbacks.rs +++ b/lib/wasix/src/runners/wcgi/callbacks.rs @@ -64,6 +64,6 @@ where } } -pub(crate) struct NoopCallbacks; +pub struct NoOpWcgiCallbacks; -impl Callbacks for NoopCallbacks where M: Send + Sync + 'static {} +impl Callbacks for NoOpWcgiCallbacks where M: Send + Sync + 'static {} diff --git a/lib/wasix/src/runners/wcgi/mod.rs b/lib/wasix/src/runners/wcgi/mod.rs index 9990c0af7cf..29dad55f56b 100644 --- a/lib/wasix/src/runners/wcgi/mod.rs +++ b/lib/wasix/src/runners/wcgi/mod.rs @@ -4,7 +4,7 @@ mod handler; mod runner; pub use self::runner::{Config, WcgiRunner}; -pub(crate) use callbacks::NoopCallbacks; +pub use callbacks::NoOpWcgiCallbacks; pub use callbacks::{Callbacks, CreateEnvConfig, CreateEnvResult, RecycleEnvConfig}; pub(crate) use create_env::default_create_env; pub use futures::future::AbortHandle; diff --git a/lib/wasix/src/runners/wcgi/runner.rs b/lib/wasix/src/runners/wcgi/runner.rs index f0982c07260..f8e9431eac6 100644 --- a/lib/wasix/src/runners/wcgi/runner.rs +++ b/lib/wasix/src/runners/wcgi/runner.rs @@ -24,9 +24,9 @@ use crate::{ Runtime, WasiEnvBuilder, }; -use super::{Callbacks, NoopCallbacks}; +use super::Callbacks; -#[derive(Debug, Default)] +#[derive(Debug)] pub struct WcgiRunner where M: Send + Sync + 'static, @@ -38,11 +38,13 @@ impl WcgiRunner where M: Send + Sync + 'static, { - pub fn new() -> Self + pub fn new(callbacks: C) -> Self where - M: Default, + C: Callbacks, { - WcgiRunner::default() + Self { + config: Config::new(callbacks), + } } pub fn config(&mut self) -> &mut Config { @@ -319,15 +321,6 @@ where } } -impl Default for Config -where - M: Send + Sync + 'static, -{ - fn default() -> Self { - Self::new(NoopCallbacks) - } -} - impl Config where M: Send + Sync + 'static, From 8b9b8f6f4c3b83dfed5526dcd6c49dcce34b1713 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Fri, 22 Dec 2023 06:27:06 +1100 Subject: [PATCH 106/129] Now properly propogating the errors to the caller --- lib/wasix/src/fs/mod.rs | 14 ++++---- lib/wasix/src/runners/dcgi/callbacks.rs | 44 ++++++++++++++++++++++-- lib/wasix/src/runners/dcgi/factory.rs | 14 +++++--- lib/wasix/src/runners/wcgi/create_env.rs | 11 +++++- lib/wasix/src/runners/wcgi/handler.rs | 35 ++++++++----------- lib/wasix/src/state/env.rs | 11 ++++-- lib/wasix/tests/runners.rs | 7 ++-- 7 files changed, 98 insertions(+), 38 deletions(-) diff --git a/lib/wasix/src/fs/mod.rs b/lib/wasix/src/fs/mod.rs index 849a53f2cb1..ee826690d47 100644 --- a/lib/wasix/src/fs/mod.rs +++ b/lib/wasix/src/fs/mod.rs @@ -525,7 +525,7 @@ impl WasiFs { let mut wasi_fs = Self::new_init(fs_backing, inodes)?; wasi_fs.init_preopens = preopens.iter().cloned().collect(); wasi_fs.init_vfs_preopens = vfs_preopens.iter().cloned().collect(); - wasi_fs.create_preopens(inodes)?; + wasi_fs.create_preopens(inodes, false)?; Ok(wasi_fs) } @@ -1619,7 +1619,11 @@ impl WasiFs { Ok(()) } - pub(crate) fn create_preopens(&self, inodes: &WasiInodes) -> Result<(), String> { + pub(crate) fn create_preopens( + &self, + inodes: &WasiInodes, + ignore_duplicates: bool, + ) -> Result<(), String> { for preopen_name in self.init_vfs_preopens.iter() { let kind = Kind::Dir { parent: self.root_inode.downgrade(), @@ -1655,13 +1659,12 @@ impl WasiFs { let mut guard = self.root_inode.write(); if let Kind::Root { entries } = guard.deref_mut() { let existing_entry = entries.insert(preopen_name.clone(), inode); - if existing_entry.is_some() { + if existing_entry.is_some() && !ignore_duplicates { return Err(format!( "Found duplicate entry for alias `{}`", preopen_name )); } - assert!(existing_entry.is_none()) } } self.preopen_fds.write().unwrap().push(fd); @@ -1778,10 +1781,9 @@ impl WasiFs { path.to_string_lossy().into_owned() }; let existing_entry = entries.insert(key.clone(), inode); - if existing_entry.is_some() { + if existing_entry.is_some() && !ignore_duplicates { return Err(format!("Found duplicate entry for alias `{}`", key)); } - assert!(existing_entry.is_none()) } } self.preopen_fds.write().unwrap().push(fd); diff --git a/lib/wasix/src/runners/dcgi/callbacks.rs b/lib/wasix/src/runners/dcgi/callbacks.rs index 048db03b364..b6c49ab6890 100644 --- a/lib/wasix/src/runners/dcgi/callbacks.rs +++ b/lib/wasix/src/runners/dcgi/callbacks.rs @@ -4,6 +4,8 @@ use derivative::Derivative; use super::*; use crate::runners::wcgi::{self, CreateEnvConfig, CreateEnvResult, RecycleEnvConfig}; +use virtual_fs::NullFile; +use wasmer_wasix_types::types::{__WASI_STDERR_FILENO, __WASI_STDIN_FILENO, __WASI_STDOUT_FILENO}; #[derive(Derivative, Clone)] #[derivative(Debug)] @@ -39,7 +41,30 @@ impl wcgi::Callbacks for DcgiCallbacks { self.inner.on_stderr_error(error) } - async fn recycle_env(&self, conf: RecycleEnvConfig) { + async fn recycle_env(&self, mut conf: RecycleEnvConfig) { + tracing::debug!(shard = conf.meta.shard, "recycling DCGI instance"); + + // We cycle out the stdio so that the pipes close properly + { + let env = conf.env.data_mut(&mut conf.store); + + // The stdio have to be reattached on each call as they are + // read to completion (EOF) during nominal flows + env.state + .fs + .swap_file(__WASI_STDIN_FILENO, Box::new(NullFile::default())) + .ok(); + env.state + .fs + .swap_file(__WASI_STDOUT_FILENO, Box::new(NullFile::default())) + .ok(); + env.state + .fs + .swap_file(__WASI_STDERR_FILENO, Box::new(NullFile::default())) + .ok(); + } + + // Now we make the instance available for reuse self.factory.release(conf).await; } @@ -47,9 +72,24 @@ impl wcgi::Callbacks for DcgiCallbacks { &self, mut conf: CreateEnvConfig, ) -> anyhow::Result { + tracing::debug!( + shard = conf.meta.shard, + "attempting to acquire existing DCGI instance" + ); + if let Some(res) = self.factory.acquire(&mut conf).await { + tracing::debug!(shard = conf.meta.shard, "found existing DCGI instance"); return Ok(res); } - self.inner.create_env(conf).await + + let mut ret = self.inner.create_env(conf).await; + + if let Ok(ret) = ret.as_mut() { + // We disable the cleanup to prevent the instance so that the + // resources can be reused + ret.env.clone().data_mut(&mut ret.store).disable_cleanup = true; + } + + ret } } diff --git a/lib/wasix/src/runners/dcgi/factory.rs b/lib/wasix/src/runners/dcgi/factory.rs index 53c67b3bde6..5995f4b6f33 100644 --- a/lib/wasix/src/runners/dcgi/factory.rs +++ b/lib/wasix/src/runners/dcgi/factory.rs @@ -98,9 +98,17 @@ impl DcgiInstanceFactory { let mut state = self.state.lock().unwrap(); let shard = state.shards.entry(shard).or_default(); shard.last_acquire = Instant::now(); + if let Some(inst) = shard.instances.pop_front() { - if let Ok(converted) = convert_instance(inst, conf) { - return Some(converted); + tracing::debug!( + shard = conf.meta.shard, + "attempting to reinitialize DCGI instance" + ); + match convert_instance(inst, conf) { + Ok(converted) => return Some(converted), + Err(err) => { + tracing::warn!("failed to reinitialize DCGI instance - {}", err); + } } } } @@ -121,9 +129,7 @@ fn convert_instance( let (stderr_sender, stderr_receiver) = Pipe::channel(); { - // Reinitialize the environment let env = env.data_mut(&mut store); - env.reinit()?; // Replace the environment variables as these will change // depending on the WCGI call diff --git a/lib/wasix/src/runners/wcgi/create_env.rs b/lib/wasix/src/runners/wcgi/create_env.rs index 2c57e80e738..1e6a806a26a 100644 --- a/lib/wasix/src/runners/wcgi/create_env.rs +++ b/lib/wasix/src/runners/wcgi/create_env.rs @@ -7,11 +7,20 @@ use crate::{ use super::{callbacks::CreateEnvConfig, RecycleEnvConfig}; -pub(crate) async fn default_recycle_env(_conf: RecycleEnvConfig) +pub(crate) async fn default_recycle_env(conf: RecycleEnvConfig) where M: Send + Sync + 'static, { tracing::debug!("Destroying the WebAssembly instance"); + + let env = conf.env; + let mut store = conf.store; + { + // We enable the cleanup again and trigger the exit function + let env = env.data_mut(&mut store); + env.disable_cleanup = false; + env.on_exit(None).await; + } } pub(crate) async fn default_create_env( diff --git a/lib/wasix/src/runners/wcgi/handler.rs b/lib/wasix/src/runners/wcgi/handler.rs index 9ba00ddfd49..8069bc35cd6 100644 --- a/lib/wasix/src/runners/wcgi/handler.rs +++ b/lib/wasix/src/runners/wcgi/handler.rs @@ -99,28 +99,21 @@ where let req_body_sender = create.body_sender; let callbacks = Arc::clone(&self.callbacks); - let work_drive_io = { - async move { - let ret = drive_request_to_completion(done, body, req_body_sender).await; - match ret { - Ok((env, store)) => { - callbacks - .recycle_env(RecycleEnvConfig { meta, env, store }) - .await; - } - Err(e) => { - tracing::error!( - error = &*e as &dyn std::error::Error, - "Unable to drive the request to completion" - ); - } - } + let ret = drive_request_to_completion(done, body, req_body_sender).await; + match ret { + Ok((env, store)) => { + callbacks + .recycle_env(RecycleEnvConfig { meta, env, store }) + .await; } - .in_current_span() - }; - task_manager - .task_shared(Box::new(move || Box::pin(work_drive_io))) - .ok(); + Err(e) => { + let e = e.to_string(); + tracing::error!(error = e, "Unable to drive the request to completion"); + return Ok(Response::builder() + .status(StatusCode::INTERNAL_SERVER_ERROR) + .body(Body::from(e.as_bytes().to_vec()))?); + } + } tracing::trace!( dialect=%self.dialect, diff --git a/lib/wasix/src/state/env.rs b/lib/wasix/src/state/env.rs index 9deaef32d63..a5058bdd377 100644 --- a/lib/wasix/src/state/env.rs +++ b/lib/wasix/src/state/env.rs @@ -318,6 +318,10 @@ pub struct WasiEnv { /// (and hence it should not record new events) pub replaying_journal: bool, + /// Flag that indicates the cleanup of the environment is to be disabled + /// (this is normally used so that the instance can be reused later on) + pub(crate) disable_cleanup: bool, + /// List of situations that the process will checkpoint on #[cfg(feature = "journal")] snapshot_on: HashSet, @@ -355,6 +359,7 @@ impl Clone for WasiEnv { replaying_journal: self.replaying_journal, #[cfg(feature = "journal")] snapshot_on: self.snapshot_on.clone(), + disable_cleanup: self.disable_cleanup, } } } @@ -395,6 +400,7 @@ impl WasiEnv { replaying_journal: false, #[cfg(feature = "journal")] snapshot_on: self.snapshot_on.clone(), + disable_cleanup: self.disable_cleanup, }; Ok((new_env, handle)) } @@ -431,7 +437,7 @@ impl WasiEnv { .map_err(WasiStateCreationError::WasiFsSetupError)?; self.state .fs - .create_preopens(&self.state.inodes) + .create_preopens(&self.state.inodes, true) .map_err(WasiStateCreationError::WasiFsSetupError)?; // The process and thread state need to be reset @@ -513,6 +519,7 @@ impl WasiEnv { capabilities: init.capabilities, #[cfg(feature = "journal")] snapshot_on: init.snapshot_on.into_iter().collect(), + disable_cleanup: false, }; env.owned_handles.push(thread); @@ -1183,7 +1190,7 @@ impl WasiEnv { } // If this is the main thread then also close all the files - if self.thread.is_main() { + if self.thread.is_main() && !self.disable_cleanup { tracing::trace!(pid=%self.pid(), "cleaning up open file handles"); let process = self.process.clone(); diff --git a/lib/wasix/tests/runners.rs b/lib/wasix/tests/runners.rs index dbcf99cabd9..10924a5c3fc 100644 --- a/lib/wasix/tests/runners.rs +++ b/lib/wasix/tests/runners.rs @@ -103,7 +103,10 @@ mod wcgi { use futures::{channel::mpsc::Sender, future::AbortHandle, SinkExt, StreamExt}; use rand::Rng; use tokio::runtime::Handle; - use wasmer_wasix::{bin_factory::BinaryPackage, runners::wcgi::WcgiRunner}; + use wasmer_wasix::{ + bin_factory::BinaryPackage, + runners::wcgi::{NoOpWcgiCallbacks, WcgiRunner}, + }; use super::*; @@ -121,7 +124,7 @@ mod wcgi { let webc = download_cached("https://wasmer.io/Michael-F-Bryan/staticserver@1.0.3").await; let (rt, tasks) = runtime(); let container = Container::from_bytes(webc).unwrap(); - let mut runner = WcgiRunner::new(); + let mut runner = WcgiRunner::new(NoOpWcgiCallbacks); let port = rand::thread_rng().gen_range(10000_u16..65535_u16); let (cb, started) = callbacks(Handle::current()); runner From 9e3f3c03160030cd94e25ff464f7d217a032b95d Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Fri, 22 Dec 2023 15:18:43 +1100 Subject: [PATCH 107/129] Now reusing DCGI instances but not the memory associated with them --- lib/wasix/src/bin_factory/exec.rs | 143 ++++++++++++-------- lib/wasix/src/bin_factory/mod.rs | 2 +- lib/wasix/src/os/task/task_join_handle.rs | 4 + lib/wasix/src/runners/dcgi/callbacks.rs | 38 +++--- lib/wasix/src/runners/dcgi/factory.rs | 61 +++++---- lib/wasix/src/runners/dcgi/instance.rs | 7 +- lib/wasix/src/runners/wcgi/callbacks.rs | 10 +- lib/wasix/src/runners/wcgi/create_env.rs | 18 +-- lib/wasix/src/runners/wcgi/handler.rs | 81 ++++++++--- lib/wasix/src/runtime/task_manager/mod.rs | 31 ++++- lib/wasix/src/runtime/task_manager/tokio.rs | 3 + lib/wasix/src/state/env.rs | 91 +++++++------ 12 files changed, 303 insertions(+), 186 deletions(-) diff --git a/lib/wasix/src/bin_factory/exec.rs b/lib/wasix/src/bin_factory/exec.rs index eda19f8d24f..558ca3fbe40 100644 --- a/lib/wasix/src/bin_factory/exec.rs +++ b/lib/wasix/src/bin_factory/exec.rs @@ -2,7 +2,9 @@ use std::{pin::Pin, sync::Arc}; use crate::{ os::task::{thread::WasiThreadRunGuard, TaskJoinHandle}, - runtime::task_manager::{TaskWasm, TaskWasmRunProperties}, + runtime::task_manager::{ + TaskWasm, TaskWasmRecycle, TaskWasmRecycleProperties, TaskWasmRunProperties, + }, syscalls::rewind_ext, RewindState, SpawnError, WasiError, WasiRuntimeError, }; @@ -91,58 +93,10 @@ pub fn spawn_exec_module( // Create a thread that will run this process let tasks_outer = tasks.clone(); - let run = { - move |props: TaskWasmRunProperties| { - let ctx = props.ctx; - let mut store = props.store; - - // Create the WasiFunctionEnv - let thread = WasiThreadRunGuard::new(ctx.data(&store).thread.clone()); - - // Perform the initialization - let ctx = { - // If this module exports an _initialize function, run that first. - if let Ok(initialize) = unsafe { ctx.data(&store).inner() } - .instance - .exports - .get_function("_initialize") - { - let initialize = initialize.clone(); - if let Err(err) = initialize.call(&mut store, &[]) { - thread.thread.set_status_finished(Err(err.into())); - ctx.data(&store) - .blocking_on_exit(Some(Errno::Noexec.into())); - return; - } - } - - WasiFunctionEnv { env: ctx.env } - }; - - // Bootstrap the process - // Unsafe: The bootstrap must be executed in the same thread that runs the - // actual WASM code - let rewind_state = match unsafe { ctx.bootstrap(&mut store) } { - Ok(r) => r, - Err(err) => { - thread.thread.set_status_finished(Err(err)); - ctx.data(&store) - .blocking_on_exit(Some(Errno::Noexec.into())); - return; - } - }; - - // If there is a start function - debug!("wasi[{}]::called main()", pid); - // TODO: rewrite to use crate::run_wasi_func - - // Call the module - call_module(ctx, store, thread, rewind_state); - } - }; - tasks_outer - .task_wasm(TaskWasm::new(Box::new(run), env, module, true).with_memory(memory_spawn)) + .task_wasm( + TaskWasm::new(Box::new(run_exec), env, module, true).with_memory(memory_spawn), + ) .map_err(|err| { error!("wasi[{}]::failed to launch module - {}", pid, err); SpawnError::UnknownError @@ -152,6 +106,78 @@ pub fn spawn_exec_module( Ok(join_handle) } +/// # SAFETY +/// This must be executed from the same thread that owns the instance as +/// otherwise it will cause a panic +unsafe fn run_recycle( + callback: Option>, + ctx: WasiFunctionEnv, + mut store: Store, +) { + if let Some(callback) = callback { + let env = ctx.data_mut(&mut store); + let memory = env.memory().clone(); + + let props = TaskWasmRecycleProperties { + env: env.clone(), + memory, + store, + }; + callback(props); + } +} + +pub fn run_exec(props: TaskWasmRunProperties) { + let ctx = props.ctx; + let mut store = props.store; + + // Create the WasiFunctionEnv + let thread = WasiThreadRunGuard::new(ctx.data(&store).thread.clone()); + let recycle = props.recycle; + + // Perform the initialization + let ctx = { + // If this module exports an _initialize function, run that first. + if let Ok(initialize) = unsafe { ctx.data(&store).inner() } + .instance + .exports + .get_function("_initialize") + { + let initialize = initialize.clone(); + if let Err(err) = initialize.call(&mut store, &[]) { + thread.thread.set_status_finished(Err(err.into())); + ctx.data(&store) + .blocking_on_exit(Some(Errno::Noexec.into())); + unsafe { run_recycle(recycle, ctx, store) }; + return; + } + } + + WasiFunctionEnv { env: ctx.env } + }; + + // Bootstrap the process + // Unsafe: The bootstrap must be executed in the same thread that runs the + // actual WASM code + let rewind_state = match unsafe { ctx.bootstrap(&mut store) } { + Ok(r) => r, + Err(err) => { + thread.thread.set_status_finished(Err(err)); + ctx.data(&store) + .blocking_on_exit(Some(Errno::Noexec.into())); + unsafe { run_recycle(recycle, ctx, store) }; + return; + } + }; + + // If there is a start function + debug!("wasi[{}]::called main()", ctx.data(&store).pid()); + // TODO: rewrite to use crate::run_wasi_func + + // Call the module + call_module(ctx, store, thread, rewind_state, recycle); +} + fn get_start(ctx: &WasiFunctionEnv, store: &Store) -> Option { unsafe { ctx.data(store).inner() } .instance @@ -167,6 +193,7 @@ fn call_module( mut store: Store, handle: WasiThreadRunGuard, rewind_state: Option<(RewindState, Option)>, + recycle: Option>, ) { let env = ctx.data(&store); let pid = env.pid(); @@ -186,6 +213,7 @@ fn call_module( ); if res != Errno::Success { ctx.data().blocking_on_exit(Some(res.into())); + unsafe { run_recycle(recycle, WasiFunctionEnv { env: ctx.as_ref() }, store) }; return; } } else { @@ -198,6 +226,7 @@ fn call_module( ); if res != Errno::Success { ctx.data().blocking_on_exit(Some(res.into())); + unsafe { run_recycle(recycle, WasiFunctionEnv { env: ctx.as_ref() }, store) }; return; } }; @@ -212,6 +241,7 @@ fn call_module( debug!("wasi[{}]::exec-failed: missing _start function", pid); ctx.data(&store) .blocking_on_exit(Some(Errno::Noexec.into())); + unsafe { run_recycle(recycle, ctx, store) }; return; }; @@ -225,7 +255,13 @@ fn call_module( let respawn = { move |ctx, store, rewind_result| { // Call the thread - call_module(ctx, store, handle, Some((rewind, Some(rewind_result)))); + call_module( + ctx, + store, + handle, + Some((rewind, Some(rewind_result))), + recycle, + ); } }; @@ -256,6 +292,7 @@ fn call_module( // Cleanup the environment ctx.data(&store).blocking_on_exit(Some(code)); + unsafe { run_recycle(recycle, ctx, store) }; debug!("wasi[{pid}]::main() has exited with {code}"); handle.thread.set_status_finished(ret.map(|a| a.into())); diff --git a/lib/wasix/src/bin_factory/mod.rs b/lib/wasix/src/bin_factory/mod.rs index 4fba6d838d6..9a08d7d4011 100644 --- a/lib/wasix/src/bin_factory/mod.rs +++ b/lib/wasix/src/bin_factory/mod.rs @@ -14,7 +14,7 @@ mod exec; pub use self::{ binary_package::*, - exec::{spawn_exec, spawn_exec_module}, + exec::{run_exec, spawn_exec, spawn_exec_module}, }; use crate::{os::command::Commands, Runtime}; diff --git a/lib/wasix/src/os/task/task_join_handle.rs b/lib/wasix/src/os/task/task_join_handle.rs index c374305fd96..792b8d271c2 100644 --- a/lib/wasix/src/os/task/task_join_handle.rs +++ b/lib/wasix/src/os/task/task_join_handle.rs @@ -138,6 +138,10 @@ impl OwnedTaskStatus { } } + pub(crate) async fn await_termination_anyhow(&self) -> anyhow::Result { + Ok(self.await_termination().await?) + } + pub fn handle(&self) -> TaskJoinHandle { TaskJoinHandle { watch: self.watch_tx.subscribe(), diff --git a/lib/wasix/src/runners/dcgi/callbacks.rs b/lib/wasix/src/runners/dcgi/callbacks.rs index b6c49ab6890..2bbfe6c3b80 100644 --- a/lib/wasix/src/runners/dcgi/callbacks.rs +++ b/lib/wasix/src/runners/dcgi/callbacks.rs @@ -44,25 +44,23 @@ impl wcgi::Callbacks for DcgiCallbacks { async fn recycle_env(&self, mut conf: RecycleEnvConfig) { tracing::debug!(shard = conf.meta.shard, "recycling DCGI instance"); - // We cycle out the stdio so that the pipes close properly - { - let env = conf.env.data_mut(&mut conf.store); - - // The stdio have to be reattached on each call as they are - // read to completion (EOF) during nominal flows - env.state - .fs - .swap_file(__WASI_STDIN_FILENO, Box::new(NullFile::default())) - .ok(); - env.state - .fs - .swap_file(__WASI_STDOUT_FILENO, Box::new(NullFile::default())) - .ok(); - env.state - .fs - .swap_file(__WASI_STDERR_FILENO, Box::new(NullFile::default())) - .ok(); - } + // The stdio have to be reattached on each call as they are + // read to completion (EOF) during nominal flows + conf.env + .state + .fs + .swap_file(__WASI_STDIN_FILENO, Box::new(NullFile::default())) + .ok(); + conf.env + .state + .fs + .swap_file(__WASI_STDOUT_FILENO, Box::new(NullFile::default())) + .ok(); + conf.env + .state + .fs + .swap_file(__WASI_STDERR_FILENO, Box::new(NullFile::default())) + .ok(); // Now we make the instance available for reuse self.factory.release(conf).await; @@ -87,7 +85,7 @@ impl wcgi::Callbacks for DcgiCallbacks { if let Ok(ret) = ret.as_mut() { // We disable the cleanup to prevent the instance so that the // resources can be reused - ret.env.clone().data_mut(&mut ret.store).disable_cleanup = true; + ret.env.disable_fs_cleanup = true; } ret diff --git a/lib/wasix/src/runners/dcgi/factory.rs b/lib/wasix/src/runners/dcgi/factory.rs index 5995f4b6f33..0c9c419821e 100644 --- a/lib/wasix/src/runners/dcgi/factory.rs +++ b/lib/wasix/src/runners/dcgi/factory.rs @@ -64,6 +64,7 @@ impl DcgiInstanceFactory { .instances .push_front(DcgiInstance { env: conf.env, + memory: conf.memory, store: conf.store, }); @@ -121,46 +122,46 @@ fn convert_instance( inst: DcgiInstance, conf: &mut CreateEnvConfig, ) -> anyhow::Result { - let env = inst.env; + let mut env = inst.env; let mut store = inst.store; + let memory = inst.memory; let (req_body_sender, req_body_receiver) = Pipe::channel(); let (res_body_sender, res_body_receiver) = Pipe::channel(); let (stderr_sender, stderr_receiver) = Pipe::channel(); - { - let env = env.data_mut(&mut store); - - // Replace the environment variables as these will change - // depending on the WCGI call - *env.state.envs.lock().unwrap() = conv_env_vars( - conf.env - .iter() - .map(|(k, v)| (k.clone(), v.as_bytes().to_vec())) - .collect(), - ); - - // The stdio have to be reattached on each call as they are - // read to completion (EOF) during nominal flows - env.state - .fs - .swap_file(__WASI_STDIN_FILENO, Box::new(req_body_receiver)) - .map_err(WasiStateCreationError::FileSystemError)?; - - env.state - .fs - .swap_file(__WASI_STDOUT_FILENO, Box::new(res_body_sender)) - .map_err(WasiStateCreationError::FileSystemError)?; - - env.state - .fs - .swap_file(__WASI_STDERR_FILENO, Box::new(stderr_sender)) - .map_err(WasiStateCreationError::FileSystemError)?; - } + env.reinit()?; + + // Replace the environment variables as these will change + // depending on the WCGI call + *env.state.envs.lock().unwrap() = conv_env_vars( + conf.env + .iter() + .map(|(k, v)| (k.clone(), v.as_bytes().to_vec())) + .collect(), + ); + + // The stdio have to be reattached on each call as they are + // read to completion (EOF) during nominal flows + env.state + .fs + .swap_file(__WASI_STDIN_FILENO, Box::new(req_body_receiver)) + .map_err(WasiStateCreationError::FileSystemError)?; + + env.state + .fs + .swap_file(__WASI_STDOUT_FILENO, Box::new(res_body_sender)) + .map_err(WasiStateCreationError::FileSystemError)?; + + env.state + .fs + .swap_file(__WASI_STDERR_FILENO, Box::new(stderr_sender)) + .map_err(WasiStateCreationError::FileSystemError)?; Ok(CreateEnvResult { env, store, + memory: Some(memory), body_sender: req_body_sender, body_receiver: res_body_receiver, stderr_receiver, diff --git a/lib/wasix/src/runners/dcgi/instance.rs b/lib/wasix/src/runners/dcgi/instance.rs index dec077445e8..83d6787152d 100644 --- a/lib/wasix/src/runners/dcgi/instance.rs +++ b/lib/wasix/src/runners/dcgi/instance.rs @@ -1,9 +1,10 @@ -use wasmer::Store; +use wasmer::{Memory, Store}; -use crate::WasiFunctionEnv; +use crate::WasiEnv; #[derive(Debug)] pub(crate) struct DcgiInstance { - pub env: WasiFunctionEnv, + pub env: WasiEnv, + pub memory: Memory, pub store: Store, } diff --git a/lib/wasix/src/runners/wcgi/callbacks.rs b/lib/wasix/src/runners/wcgi/callbacks.rs index 2300aefec5e..5b1450a54ac 100644 --- a/lib/wasix/src/runners/wcgi/callbacks.rs +++ b/lib/wasix/src/runners/wcgi/callbacks.rs @@ -1,9 +1,9 @@ use std::{collections::HashMap, sync::Arc}; use virtual_fs::Pipe; -use wasmer::{Module, Store}; +use wasmer::{Memory, Module, Store}; -use crate::{runtime::module_cache::ModuleHash, WasiFunctionEnv}; +use crate::{runtime::module_cache::ModuleHash, WasiEnv}; use super::{create_env::default_recycle_env, handler::SetupBuilder, *}; @@ -23,7 +23,8 @@ where /// Result of a create operation on a new environment pub struct CreateEnvResult { - pub env: WasiFunctionEnv, + pub env: WasiEnv, + pub memory: Option, pub store: Store, pub body_sender: Pipe, pub body_receiver: Pipe, @@ -33,7 +34,8 @@ pub struct CreateEnvResult { /// Configuration used for reusing an new environment pub struct RecycleEnvConfig { pub meta: M, - pub env: WasiFunctionEnv, + pub env: WasiEnv, + pub memory: Memory, pub store: Store, } diff --git a/lib/wasix/src/runners/wcgi/create_env.rs b/lib/wasix/src/runners/wcgi/create_env.rs index 1e6a806a26a..0204e33cf69 100644 --- a/lib/wasix/src/runners/wcgi/create_env.rs +++ b/lib/wasix/src/runners/wcgi/create_env.rs @@ -7,20 +7,14 @@ use crate::{ use super::{callbacks::CreateEnvConfig, RecycleEnvConfig}; -pub(crate) async fn default_recycle_env(conf: RecycleEnvConfig) +pub(crate) async fn default_recycle_env(mut conf: RecycleEnvConfig) where M: Send + Sync + 'static, { tracing::debug!("Destroying the WebAssembly instance"); - let env = conf.env; - let mut store = conf.store; - { - // We enable the cleanup again and trigger the exit function - let env = env.data_mut(&mut store); - env.disable_cleanup = false; - env.on_exit(None).await; - } + conf.env.disable_fs_cleanup = false; + conf.env.on_exit(None).await; } pub(crate) async fn default_create_env( @@ -50,12 +44,12 @@ where http_client: HttpClientCapabilityV1::new_allow_all(), threading: Default::default(), }); + let env = builder.build()?; - let mut store = conf.runtime.new_store(); - let (_, env) = builder.instantiate_ext(conf.module, conf.module_hash, &mut store)?; Ok(CreateEnvResult { env, - store, + memory: None, + store: conf.runtime.new_store(), body_sender: req_body_sender, body_receiver: res_body_receiver, stderr_receiver, diff --git a/lib/wasix/src/runners/wcgi/handler.rs b/lib/wasix/src/runners/wcgi/handler.rs index 8069bc35cd6..29b6f8f4b4e 100644 --- a/lib/wasix/src/runners/wcgi/handler.rs +++ b/lib/wasix/src/runners/wcgi/handler.rs @@ -6,15 +6,23 @@ use http::{Request, Response, StatusCode}; use hyper::{service::Service, Body}; use tokio::io::{AsyncBufReadExt, AsyncRead, AsyncWrite, AsyncWriteExt}; use tracing::Instrument; -use wasmer::Module; +use virtual_mio::InlineWaker; +use wasmer::{AsStoreRef, Module}; +use wasmer_wasix_types::wasi::ExitCode; use wcgi_host::CgiDialect; use crate::{ + bin_factory::run_exec, + os::task::OwnedTaskStatus, runners::wcgi::{ callbacks::{CreateEnvConfig, RecycleEnvConfig}, Callbacks, }, - runtime::module_cache::ModuleHash, + runtime::{ + module_cache::ModuleHash, + task_manager::{TaskWasm, TaskWasmRecycleProperties}, + SpawnMemoryType, + }, Runtime, VirtualTaskManager, WasiEnvBuilder, }; @@ -75,16 +83,54 @@ where let task_manager = self.runtime.task_manager(); let env = create.env; let store = create.store; - let (run_tx, mut run_rx) = tokio::sync::mpsc::unbounded_channel(); - task_manager.task_dedicated(Box::new(move || { - run_tx.send(env.run_async(store)).ok(); - }))?; - let done = async move { run_rx.recv().await.unwrap().map_err(Error::from) }; + let module = self.module.clone(); + + // The recycle function will attempt to reuse the instance + let callbacks = Arc::clone(&self.callbacks); + let recycle = { + let callbacks = callbacks.clone(); + move |props: TaskWasmRecycleProperties| { + InlineWaker::block_on(callbacks.recycle_env(RecycleEnvConfig { + meta, + env: props.env, + store: props.store, + memory: props.memory, + })); + } + }; + let finished = env.process.finished.clone(); + + /* + // Determine if we are going to create memory and import it or just rely on self creation of memory + let spawn_type = match create.memory { + Some(memory) => SpawnMemoryType::ShareMemory(memory, store.as_store_ref()), + None => { + let shared_memory = module.imports().memories().next().map(|a| *a.ty()); + match shared_memory { + Some(ty) => SpawnMemoryType::CreateMemoryOfType(ty), + None => SpawnMemoryType::CreateMemory, + } + } + }; + */ + + // We run the WCGI thread on the dedicated WASM + // thread pool that has support for asynchronous + // threading, etc... + task_manager + .task_wasm( + TaskWasm::new(Box::new(run_exec), env, module, false) + //.with_memory(spawn_type) + .with_recycle(Box::new(recycle)), + ) + .map_err(|err| { + tracing::warn!("failed to execute WCGI thread - {}", err); + err + })?; let mut res_body_receiver = tokio::io::BufReader::new(create.body_receiver); let stderr_receiver = create.stderr_receiver; - let callbacks = Arc::clone(&self.callbacks); let propagate_stderr = self.propagate_stderr; let work_consume_stderr = { let callbacks = callbacks.clone(); @@ -98,14 +144,9 @@ where ); let req_body_sender = create.body_sender; - let callbacks = Arc::clone(&self.callbacks); - let ret = drive_request_to_completion(done, body, req_body_sender).await; + let ret = drive_request_to_completion(finished, body, req_body_sender).await; match ret { - Ok((env, store)) => { - callbacks - .recycle_env(RecycleEnvConfig { meta, env, store }) - .await; - } + Ok(_) => {} Err(e) => { let e = e.to_string(); tracing::error!(error = e, "Unable to drive the request to completion"); @@ -190,11 +231,11 @@ where /// Drive the request to completion by streaming the request body to the /// instance and waiting for it to exit. -async fn drive_request_to_completion( - done: impl Future>, +async fn drive_request_to_completion( + finished: Arc, mut request_body: hyper::Body, - mut instance_stdin: impl AsyncWrite + Send + Unpin + 'static, -) -> Result { + mut instance_stdin: impl AsyncWrite + Send + Sync + Unpin + 'static, +) -> Result { let request_body_send = async move { // Copy the request into our instance, chunk-by-chunk. If the instance // dies before we finish writing the body, the instance's side of the @@ -218,7 +259,7 @@ async fn drive_request_to_completion( } .in_current_span(); - let (ret, _) = futures::try_join!(done, request_body_send)?; + let (ret, _) = futures::try_join!(finished.await_termination_anyhow(), request_body_send)?; Ok(ret) } diff --git a/lib/wasix/src/runtime/task_manager/mod.rs b/lib/wasix/src/runtime/task_manager/mod.rs index 9ace594cd9a..6dbb8769471 100644 --- a/lib/wasix/src/runtime/task_manager/mod.rs +++ b/lib/wasix/src/runtime/task_manager/mod.rs @@ -7,6 +7,7 @@ use std::task::{Context, Poll}; use std::{pin::Pin, time::Duration}; use bytes::Bytes; +use derivative::Derivative; use futures::future::BoxFuture; use futures::Future; use wasmer::{AsStoreMut, AsStoreRef, Memory, MemoryType, Module, Store, StoreMut, StoreRef}; @@ -38,7 +39,8 @@ pub type WasmResumeTrigger = dyn FnOnce() -> Pin>, + /// The instance will be recycled back to this function when the WASM run has finished + #[derivative(Debug = "ignore")] + pub recycle: Option>, } /// Callback that will be invoked @@ -54,9 +59,21 @@ pub type TaskWasmRun = dyn FnOnce(TaskWasmRunProperties) + Send + 'static; /// Callback that will be invoked pub type TaskExecModule = dyn FnOnce(Module) + Send + 'static; +/// The properties passed to the task +#[derive(Debug)] +pub struct TaskWasmRecycleProperties { + pub env: WasiEnv, + pub memory: Memory, + pub store: Store, +} + +/// Callback that will be invoked +pub type TaskWasmRecycle = dyn FnOnce(TaskWasmRecycleProperties) + Send + 'static; + /// Represents a WASM task that will be executed on a dedicated thread pub struct TaskWasm<'a, 'b> { pub run: Box, + pub recycle: Option>, pub env: WasiEnv, pub module: Module, pub snapshot: Option<&'b InstanceSnapshot>, @@ -66,14 +83,19 @@ pub struct TaskWasm<'a, 'b> { } impl<'a, 'b> TaskWasm<'a, 'b> { pub fn new(run: Box, env: WasiEnv, module: Module, update_layout: bool) -> Self { + let shared_memory = module.imports().memories().next().map(|a| *a.ty()); Self { run, env, module, snapshot: None, - spawn_type: SpawnMemoryType::CreateMemory, + spawn_type: match shared_memory { + Some(ty) => SpawnMemoryType::CreateMemoryOfType(ty), + None => SpawnMemoryType::CreateMemory, + }, trigger: None, update_layout, + recycle: None, } } @@ -91,6 +113,11 @@ impl<'a, 'b> TaskWasm<'a, 'b> { self.trigger.replace(trigger); self } + + pub fn with_recycle(mut self, trigger: Box) -> Self { + self.recycle.replace(trigger); + self + } } /// A task executor backed by a thread pool. diff --git a/lib/wasix/src/runtime/task_manager/tokio.rs b/lib/wasix/src/runtime/task_manager/tokio.rs index b9f2a4c098c..7ee53897873 100644 --- a/lib/wasix/src/runtime/task_manager/tokio.rs +++ b/lib/wasix/src/runtime/task_manager/tokio.rs @@ -147,6 +147,7 @@ impl VirtualTaskManager for TokioTaskManager { fn task_wasm(&self, task: TaskWasm) -> Result<(), WasiThreadError> { // Create the context on a new store let run = task.run; + let recycle = task.recycle; let (ctx, store) = WasiFunctionEnv::new_with_store( task.module, task.env, @@ -171,6 +172,7 @@ impl VirtualTaskManager for TokioTaskManager { ctx, store, trigger_result: Some(result), + recycle, }); }); }); @@ -186,6 +188,7 @@ impl VirtualTaskManager for TokioTaskManager { ctx, store, trigger_result: None, + recycle, }); }); } diff --git a/lib/wasix/src/state/env.rs b/lib/wasix/src/state/env.rs index a5058bdd377..4f76db6c90f 100644 --- a/lib/wasix/src/state/env.rs +++ b/lib/wasix/src/state/env.rs @@ -320,7 +320,7 @@ pub struct WasiEnv { /// Flag that indicates the cleanup of the environment is to be disabled /// (this is normally used so that the instance can be reused later on) - pub(crate) disable_cleanup: bool, + pub(crate) disable_fs_cleanup: bool, /// List of situations that the process will checkpoint on #[cfg(feature = "journal")] @@ -359,7 +359,7 @@ impl Clone for WasiEnv { replaying_journal: self.replaying_journal, #[cfg(feature = "journal")] snapshot_on: self.snapshot_on.clone(), - disable_cleanup: self.disable_cleanup, + disable_fs_cleanup: self.disable_fs_cleanup, } } } @@ -400,7 +400,7 @@ impl WasiEnv { replaying_journal: false, #[cfg(feature = "journal")] snapshot_on: self.snapshot_on.clone(), - disable_cleanup: self.disable_cleanup, + disable_fs_cleanup: self.disable_fs_cleanup, }; Ok((new_env, handle)) } @@ -415,30 +415,35 @@ impl WasiEnv { /// Re-initializes this environment so that it can be executed again pub fn reinit(&mut self) -> Result<(), WasiStateCreationError> { - // First we clear any open files as the descriptors would - // otherwise clash - if let Ok(mut map) = self.state.fs.fd_map.write() { - map.clear(); + // If the cleanup logic is enabled then we need to rebuild the + // file descriptors which would have been destroyed when the + // main thread exited + if !self.disable_fs_cleanup { + // First we clear any open files as the descriptors would + // otherwise clash + if let Ok(mut map) = self.state.fs.fd_map.write() { + map.clear(); + } + self.state.fs.preopen_fds.write().unwrap().clear(); + self.state + .fs + .next_fd + .store(3, std::sync::atomic::Ordering::SeqCst); + *self.state.fs.current_dir.lock().unwrap() = "/".to_string(); + + // We need to rebuild the basic file descriptors + self.state.fs.create_stdin(&self.state.inodes); + self.state.fs.create_stdout(&self.state.inodes); + self.state.fs.create_stderr(&self.state.inodes); + self.state + .fs + .create_rootfd() + .map_err(WasiStateCreationError::WasiFsSetupError)?; + self.state + .fs + .create_preopens(&self.state.inodes, true) + .map_err(WasiStateCreationError::WasiFsSetupError)?; } - self.state.fs.preopen_fds.write().unwrap().clear(); - self.state - .fs - .next_fd - .store(3, std::sync::atomic::Ordering::SeqCst); - *self.state.fs.current_dir.lock().unwrap() = "/".to_string(); - - // We need to rebuild the basic file descriptors - self.state.fs.create_stdin(&self.state.inodes); - self.state.fs.create_stdout(&self.state.inodes); - self.state.fs.create_stderr(&self.state.inodes); - self.state - .fs - .create_rootfd() - .map_err(WasiStateCreationError::WasiFsSetupError)?; - self.state - .fs - .create_preopens(&self.state.inodes, true) - .map_err(WasiStateCreationError::WasiFsSetupError)?; // The process and thread state need to be reset self.process = WasiProcess::new( @@ -519,7 +524,7 @@ impl WasiEnv { capabilities: init.capabilities, #[cfg(feature = "journal")] snapshot_on: init.snapshot_on.into_iter().collect(), - disable_cleanup: false, + disable_fs_cleanup: false, }; env.owned_handles.push(thread); @@ -1190,26 +1195,30 @@ impl WasiEnv { } // If this is the main thread then also close all the files - if self.thread.is_main() && !self.disable_cleanup { - tracing::trace!(pid=%self.pid(), "cleaning up open file handles"); - + if self.thread.is_main() { let process = self.process.clone(); + let disable_fs_cleanup = self.disable_fs_cleanup; + let pid = self.pid(); let timeout = self.tasks().sleep_now(CLEANUP_TIMEOUT); let state = self.state.clone(); Box::pin(async move { - // Perform the clean operation using the asynchronous runtime - tokio::select! { - _ = timeout => { - tracing::debug!( - "WasiEnv::cleanup has timed out after {CLEANUP_TIMEOUT:?}" - ); - }, - _ = state.fs.close_all() => { } - } + if !disable_fs_cleanup { + tracing::trace!(pid = %pid, "cleaning up open file handles"); + + // Perform the clean operation using the asynchronous runtime + tokio::select! { + _ = timeout => { + tracing::debug!( + "WasiEnv::cleanup has timed out after {CLEANUP_TIMEOUT:?}" + ); + }, + _ = state.fs.close_all() => { } + } - // Now send a signal that the thread is terminated - process.signal_process(Signal::Sigquit); + // Now send a signal that the thread is terminated + process.signal_process(Signal::Sigquit); + } // Terminate the process let exit_code = exit_code.unwrap_or_else(|| Errno::Canceled.into()); From ffb45eac06c348813f75fed3a52fdff19ea91c09 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Fri, 22 Dec 2023 15:39:15 +1100 Subject: [PATCH 108/129] DCGI can not reuse memory for now due to several restrictions --- lib/wasix/src/bin_factory/exec.rs | 15 ++---------- lib/wasix/src/runners/dcgi/callbacks.rs | 2 +- lib/wasix/src/runners/dcgi/factory.rs | 10 ++++---- lib/wasix/src/runners/dcgi/instance.rs | 6 ++--- lib/wasix/src/runners/wcgi/callbacks.rs | 3 +-- lib/wasix/src/runners/wcgi/create_env.rs | 1 - lib/wasix/src/runners/wcgi/handler.rs | 28 +++++++++++------------ lib/wasix/src/runtime/task_manager/mod.rs | 7 ++++++ 8 files changed, 32 insertions(+), 40 deletions(-) diff --git a/lib/wasix/src/bin_factory/exec.rs b/lib/wasix/src/bin_factory/exec.rs index 558ca3fbe40..1f82d365c1b 100644 --- a/lib/wasix/src/bin_factory/exec.rs +++ b/lib/wasix/src/bin_factory/exec.rs @@ -15,7 +15,7 @@ use wasmer::{Function, FunctionEnvMut, Memory32, Memory64, Module, Store}; use wasmer_wasix_types::wasi::Errno; use super::{BinFactory, BinaryPackage}; -use crate::{runtime::SpawnMemoryType, Runtime, WasiEnv, WasiFunctionEnv}; +use crate::{Runtime, WasiEnv, WasiFunctionEnv}; #[tracing::instrument(level = "trace", skip_all, fields(%name, %binary.package_name))] pub async fn spawn_exec( @@ -81,22 +81,11 @@ pub fn spawn_exec_module( let join_handle = env.thread.join_handle(); { - // Determine if shared memory needs to be created and imported - let shared_memory = module.imports().memories().next().map(|a| *a.ty()); - - // Determine if we are going to create memory and import it or just rely on self creation of memory - let memory_spawn = match shared_memory { - Some(ty) => SpawnMemoryType::CreateMemoryOfType(ty), - None => SpawnMemoryType::CreateMemory, - }; - // Create a thread that will run this process let tasks_outer = tasks.clone(); tasks_outer - .task_wasm( - TaskWasm::new(Box::new(run_exec), env, module, true).with_memory(memory_spawn), - ) + .task_wasm(TaskWasm::new(Box::new(run_exec), env, module, true)) .map_err(|err| { error!("wasi[{}]::failed to launch module - {}", pid, err); SpawnError::UnknownError diff --git a/lib/wasix/src/runners/dcgi/callbacks.rs b/lib/wasix/src/runners/dcgi/callbacks.rs index 2bbfe6c3b80..53d0a2b186a 100644 --- a/lib/wasix/src/runners/dcgi/callbacks.rs +++ b/lib/wasix/src/runners/dcgi/callbacks.rs @@ -41,7 +41,7 @@ impl wcgi::Callbacks for DcgiCallbacks { self.inner.on_stderr_error(error) } - async fn recycle_env(&self, mut conf: RecycleEnvConfig) { + async fn recycle_env(&self, conf: RecycleEnvConfig) { tracing::debug!(shard = conf.meta.shard, "recycling DCGI instance"); // The stdio have to be reattached on each call as they are diff --git a/lib/wasix/src/runners/dcgi/factory.rs b/lib/wasix/src/runners/dcgi/factory.rs index 0c9c419821e..12d4caf432e 100644 --- a/lib/wasix/src/runners/dcgi/factory.rs +++ b/lib/wasix/src/runners/dcgi/factory.rs @@ -64,8 +64,8 @@ impl DcgiInstanceFactory { .instances .push_front(DcgiInstance { env: conf.env, - memory: conf.memory, - store: conf.store, + //memory: conf.memory, + //store: conf.store, }); drop(state); @@ -123,8 +123,6 @@ fn convert_instance( conf: &mut CreateEnvConfig, ) -> anyhow::Result { let mut env = inst.env; - let mut store = inst.store; - let memory = inst.memory; let (req_body_sender, req_body_receiver) = Pipe::channel(); let (res_body_sender, res_body_receiver) = Pipe::channel(); @@ -160,8 +158,8 @@ fn convert_instance( Ok(CreateEnvResult { env, - store, - memory: Some(memory), + //memory: Some((inst.memory, inst.store)), + memory: None, body_sender: req_body_sender, body_receiver: res_body_receiver, stderr_receiver, diff --git a/lib/wasix/src/runners/dcgi/instance.rs b/lib/wasix/src/runners/dcgi/instance.rs index 83d6787152d..df04de1a593 100644 --- a/lib/wasix/src/runners/dcgi/instance.rs +++ b/lib/wasix/src/runners/dcgi/instance.rs @@ -1,10 +1,10 @@ -use wasmer::{Memory, Store}; +//use wasmer::{Memory, Store}; use crate::WasiEnv; #[derive(Debug)] pub(crate) struct DcgiInstance { pub env: WasiEnv, - pub memory: Memory, - pub store: Store, + //pub memory: Memory, + //pub store: Store, } diff --git a/lib/wasix/src/runners/wcgi/callbacks.rs b/lib/wasix/src/runners/wcgi/callbacks.rs index 5b1450a54ac..101a24f68dc 100644 --- a/lib/wasix/src/runners/wcgi/callbacks.rs +++ b/lib/wasix/src/runners/wcgi/callbacks.rs @@ -24,8 +24,7 @@ where /// Result of a create operation on a new environment pub struct CreateEnvResult { pub env: WasiEnv, - pub memory: Option, - pub store: Store, + pub memory: Option<(Memory, Store)>, pub body_sender: Pipe, pub body_receiver: Pipe, pub stderr_receiver: Pipe, diff --git a/lib/wasix/src/runners/wcgi/create_env.rs b/lib/wasix/src/runners/wcgi/create_env.rs index 0204e33cf69..53f5537ef7a 100644 --- a/lib/wasix/src/runners/wcgi/create_env.rs +++ b/lib/wasix/src/runners/wcgi/create_env.rs @@ -49,7 +49,6 @@ where Ok(CreateEnvResult { env, memory: None, - store: conf.runtime.new_store(), body_sender: req_body_sender, body_receiver: res_body_receiver, stderr_receiver, diff --git a/lib/wasix/src/runners/wcgi/handler.rs b/lib/wasix/src/runners/wcgi/handler.rs index 29b6f8f4b4e..c1a9ab20d59 100644 --- a/lib/wasix/src/runners/wcgi/handler.rs +++ b/lib/wasix/src/runners/wcgi/handler.rs @@ -7,7 +7,7 @@ use hyper::{service::Service, Body}; use tokio::io::{AsyncBufReadExt, AsyncRead, AsyncWrite, AsyncWriteExt}; use tracing::Instrument; use virtual_mio::InlineWaker; -use wasmer::{AsStoreRef, Module}; +use wasmer::Module; use wasmer_wasix_types::wasi::ExitCode; use wcgi_host::CgiDialect; @@ -21,7 +21,6 @@ use crate::{ runtime::{ module_cache::ModuleHash, task_manager::{TaskWasm, TaskWasmRecycleProperties}, - SpawnMemoryType, }, Runtime, VirtualTaskManager, WasiEnvBuilder, }; @@ -82,7 +81,6 @@ where let task_manager = self.runtime.task_manager(); let env = create.env; - let store = create.store; let module = self.module.clone(); // The recycle function will attempt to reuse the instance @@ -101,17 +99,19 @@ where let finished = env.process.finished.clone(); /* + * TODO: Reusing memory for DCGI calls and not just the file system + * + * DCGI does not support reusing the memory for the following reasons + * 1. The environment variables can not be overridden after libc does its lazy loading + * 2. The HTTP request variables are passed as environment variables and hence can not be changed + * after the first call is made on the memory + * 3. The `SpawnMemoryType` is not send however this handler is running as a Send async. In order + * to fix this the entire handler would need to run in its own non-Send thread. + // Determine if we are going to create memory and import it or just rely on self creation of memory - let spawn_type = match create.memory { - Some(memory) => SpawnMemoryType::ShareMemory(memory, store.as_store_ref()), - None => { - let shared_memory = module.imports().memories().next().map(|a| *a.ty()); - match shared_memory { - Some(ty) => SpawnMemoryType::CreateMemoryOfType(ty), - None => SpawnMemoryType::CreateMemory, - } - } - }; + let spawn_type = create + .memory + .map(|memory| SpawnMemoryType::ShareMemory(memory, store.as_store_ref())); */ // We run the WCGI thread on the dedicated WASM @@ -120,7 +120,7 @@ where task_manager .task_wasm( TaskWasm::new(Box::new(run_exec), env, module, false) - //.with_memory(spawn_type) + //.with_optional_memory(spawn_type) .with_recycle(Box::new(recycle)), ) .map_err(|err| { diff --git a/lib/wasix/src/runtime/task_manager/mod.rs b/lib/wasix/src/runtime/task_manager/mod.rs index 6dbb8769471..814f73cb889 100644 --- a/lib/wasix/src/runtime/task_manager/mod.rs +++ b/lib/wasix/src/runtime/task_manager/mod.rs @@ -104,6 +104,13 @@ impl<'a, 'b> TaskWasm<'a, 'b> { self } + pub fn with_optional_memory(mut self, spawn_type: Option>) -> Self { + if let Some(spawn_type) = spawn_type { + self.spawn_type = spawn_type; + } + self + } + pub fn with_snapshot(mut self, snapshot: &'b InstanceSnapshot) -> Self { self.snapshot.replace(snapshot); self From 25a4824147e6f016b9c4172ec7938c710ffd6668 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Fri, 22 Dec 2023 15:51:31 +1100 Subject: [PATCH 109/129] Added a fix so that stderr is properly returned on DCGI calls --- lib/wasix/src/runners/wcgi/handler.rs | 41 ++++++++++++++------------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/lib/wasix/src/runners/wcgi/handler.rs b/lib/wasix/src/runners/wcgi/handler.rs index c1a9ab20d59..d8ea21f40ab 100644 --- a/lib/wasix/src/runners/wcgi/handler.rs +++ b/lib/wasix/src/runners/wcgi/handler.rs @@ -145,26 +145,6 @@ where let req_body_sender = create.body_sender; let ret = drive_request_to_completion(finished, body, req_body_sender).await; - match ret { - Ok(_) => {} - Err(e) => { - let e = e.to_string(); - tracing::error!(error = e, "Unable to drive the request to completion"); - return Ok(Response::builder() - .status(StatusCode::INTERNAL_SERVER_ERROR) - .body(Body::from(e.as_bytes().to_vec()))?); - } - } - - tracing::trace!( - dialect=%self.dialect, - "extracting response parts", - ); - - let parts = self - .dialect - .extract_response_header(&mut res_body_receiver) - .await; // When set this will cause any stderr responses to // take precedence over nominal responses but it @@ -187,6 +167,27 @@ where })) .ok(); } + + match ret { + Ok(_) => {} + Err(e) => { + let e = e.to_string(); + tracing::error!(error = e, "Unable to drive the request to completion"); + return Ok(Response::builder() + .status(StatusCode::INTERNAL_SERVER_ERROR) + .body(Body::from(e.as_bytes().to_vec()))?); + } + } + + tracing::trace!( + dialect=%self.dialect, + "extracting response parts", + ); + + let parts = self + .dialect + .extract_response_header(&mut res_body_receiver) + .await; let parts = parts?; tracing::trace!( From e732ff85862f7f0b037339c1ec6336313448c81c Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Sat, 23 Dec 2023 11:16:46 +1100 Subject: [PATCH 110/129] Fixes after the latest merge --- lib/cli/src/commands/run/mod.rs | 4 ++-- lib/wasix/src/journal/concrete/archived.rs | 2 +- lib/wasix/src/journal/entry.rs | 2 +- lib/wasix/src/runtime/module_cache/types.rs | 1 + tests/lib/wast/src/wasi_wast.rs | 2 +- 5 files changed, 6 insertions(+), 5 deletions(-) diff --git a/lib/cli/src/commands/run/mod.rs b/lib/cli/src/commands/run/mod.rs index 629d098522d..75739e54ab6 100644 --- a/lib/cli/src/commands/run/mod.rs +++ b/lib/cli/src/commands/run/mod.rs @@ -695,7 +695,7 @@ impl ExecutableTarget { Ok(ExecutableTarget::WebAssembly { module, - module_hash: ModuleHash::sha256(&wasm), + module_hash: ModuleHash::hash(&wasm), path: path.to_path_buf(), }) } @@ -705,7 +705,7 @@ impl ExecutableTarget { let module = unsafe { Module::deserialize_from_file(&engine, path)? }; let module_hash = { let wasm = std::fs::read(path)?; - ModuleHash::sha256(wasm) + ModuleHash::hash(wasm) }; Ok(ExecutableTarget::WebAssembly { diff --git a/lib/wasix/src/journal/concrete/archived.rs b/lib/wasix/src/journal/concrete/archived.rs index 0bf1bc8cb11..c6db6e66c97 100644 --- a/lib/wasix/src/journal/concrete/archived.rs +++ b/lib/wasix/src/journal/concrete/archived.rs @@ -954,7 +954,7 @@ pub enum ArchivedJournalEntry<'a> { #[derive(Debug, Clone, RkyvSerialize, RkyvDeserialize, Archive)] #[archive_attr(derive(CheckBytes))] pub struct JournalEntryInitModuleV1 { - pub wasm_hash: [u8; 32], + pub wasm_hash: [u8; 8], } #[repr(C)] diff --git a/lib/wasix/src/journal/entry.rs b/lib/wasix/src/journal/entry.rs index bfc48b10d4f..d29a80fbcde 100644 --- a/lib/wasix/src/journal/entry.rs +++ b/lib/wasix/src/journal/entry.rs @@ -108,7 +108,7 @@ impl From for TimeType { #[serde(rename_all = "snake_case")] pub enum JournalEntry<'a> { InitModuleV1 { - wasm_hash: [u8; 32], + wasm_hash: [u8; 8], }, UpdateMemoryRegionV1 { region: Range, diff --git a/lib/wasix/src/runtime/module_cache/types.rs b/lib/wasix/src/runtime/module_cache/types.rs index 0f59c1eda98..68b7667a2ad 100644 --- a/lib/wasix/src/runtime/module_cache/types.rs +++ b/lib/wasix/src/runtime/module_cache/types.rs @@ -5,6 +5,7 @@ use std::{ path::PathBuf, }; +use rand::RngCore; use wasmer::{Engine, Module}; use crate::runtime::module_cache::FallbackCache; diff --git a/tests/lib/wast/src/wasi_wast.rs b/tests/lib/wast/src/wasi_wast.rs index e3bf84d8e10..806209dbb68 100644 --- a/tests/lib/wast/src/wasi_wast.rs +++ b/tests/lib/wast/src/wasi_wast.rs @@ -122,7 +122,7 @@ impl<'a> WasiTest<'a> { wasm_module.read_to_end(&mut out)?; out }; - let module_hash = ModuleHash::sha256(&wasm_bytes); + let module_hash = ModuleHash::hash(&wasm_bytes); let module = Module::new(store, wasm_bytes)?; let (builder, _tempdirs, mut stdin_tx, stdout_rx, stderr_rx) = From 9caf4bfd783935c795096477f307ae0dfddbb2c5 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Sat, 23 Dec 2023 11:28:45 +1100 Subject: [PATCH 111/129] Lint fixes and compile fixes --- lib/api/src/js/externals/memory.rs | 4 ++-- lib/wasix/src/fs/mod.rs | 6 +++--- lib/wasix/src/journal/base64.rs | 9 +++------ lib/wasix/src/journal/concrete/archived.rs | 4 ++-- lib/wasix/src/os/task/task_join_handle.rs | 2 +- lib/wasix/src/state/run.rs | 1 + 6 files changed, 12 insertions(+), 14 deletions(-) diff --git a/lib/api/src/js/externals/memory.rs b/lib/api/src/js/externals/memory.rs index 7944a0e3e7b..0fb067771e5 100644 --- a/lib/api/src/js/externals/memory.rs +++ b/lib/api/src/js/externals/memory.rs @@ -152,9 +152,9 @@ impl Memory { let cur_size = self.view(store).data_size(); if min_size > cur_size { let delta = min_size - cur_size; - let pages = ((delta - 1) / wasmer_types::WASM_PAGE_SIZE) + 1; + let pages = ((delta - 1) / wasmer_types::WASM_PAGE_SIZE as u64) + 1; - self.grow(store, Pages(pages))?; + self.grow(store, Pages(pages as u32))?; } Ok(()) } diff --git a/lib/wasix/src/fs/mod.rs b/lib/wasix/src/fs/mod.rs index 7b0be99f3eb..965377b3cff 100644 --- a/lib/wasix/src/fs/mod.rs +++ b/lib/wasix/src/fs/mod.rs @@ -523,8 +523,8 @@ impl WasiFs { fs_backing: WasiFsRoot, ) -> Result { let mut wasi_fs = Self::new_init(fs_backing, inodes)?; - wasi_fs.init_preopens = preopens.iter().cloned().collect(); - wasi_fs.init_vfs_preopens = vfs_preopens.iter().cloned().collect(); + wasi_fs.init_preopens = preopens.to_vec(); + wasi_fs.init_vfs_preopens = vfs_preopens.to_vec(); wasi_fs.create_preopens(inodes, false)?; Ok(wasi_fs) } @@ -567,7 +567,7 @@ impl WasiFs { current_dir: Mutex::new("/".to_string()), is_wasix: AtomicBool::new(false), root_fs: fs_backing, - root_inode: root_inode.clone(), + root_inode, has_unioned: Arc::new(Mutex::new(HashSet::new())), init_preopens: Default::default(), init_vfs_preopens: Default::default(), diff --git a/lib/wasix/src/journal/base64.rs b/lib/wasix/src/journal/base64.rs index cd4124af9ae..50d251f8329 100644 --- a/lib/wasix/src/journal/base64.rs +++ b/lib/wasix/src/journal/base64.rs @@ -14,10 +14,7 @@ pub fn serialize(v: &[u8], s: S) -> Result { pub fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result, D::Error> { let base64 = String::deserialize(d)?; #[allow(deprecated)] - base64::decode( - decompress_size_prepended(base64.as_bytes()) - .map_err(|err| serde::de::Error::custom(err))?, - ) - .map_err(serde::de::Error::custom) - .map(|d| d.into()) + base64::decode(decompress_size_prepended(base64.as_bytes()).map_err(serde::de::Error::custom)?) + .map_err(serde::de::Error::custom) + .map(|d| d.into()) } diff --git a/lib/wasix/src/journal/concrete/archived.rs b/lib/wasix/src/journal/concrete/archived.rs index c6db6e66c97..3ffe6af01d8 100644 --- a/lib/wasix/src/journal/concrete/archived.rs +++ b/lib/wasix/src/journal/concrete/archived.rs @@ -94,7 +94,7 @@ impl JournalEntryRecordType { /// however this does mean care must be taken that the data itself /// can not be manipulated or corrupted. pub unsafe fn deserialize_archive(self, data: &[u8]) -> anyhow::Result> { - Ok(match self { + match self { JournalEntryRecordType::InitModuleV1 => ArchivedJournalEntry::InitModuleV1( rkyv::archived_root::(data), ), @@ -336,7 +336,7 @@ impl JournalEntryRecordType { rkyv::archived_root::(data), ), } - .try_into()?) + .try_into() } } diff --git a/lib/wasix/src/os/task/task_join_handle.rs b/lib/wasix/src/os/task/task_join_handle.rs index 792b8d271c2..18d7271bac6 100644 --- a/lib/wasix/src/os/task/task_join_handle.rs +++ b/lib/wasix/src/os/task/task_join_handle.rs @@ -138,7 +138,7 @@ impl OwnedTaskStatus { } } - pub(crate) async fn await_termination_anyhow(&self) -> anyhow::Result { + pub async fn await_termination_anyhow(&self) -> anyhow::Result { Ok(self.await_termination().await?) } diff --git a/lib/wasix/src/state/run.rs b/lib/wasix/src/state/run.rs index 48ba2b15816..45f450d6d5e 100644 --- a/lib/wasix/src/state/run.rs +++ b/lib/wasix/src/state/run.rs @@ -7,6 +7,7 @@ use crate::{RewindStateOption, WasiError, WasiRuntimeError}; use super::*; impl WasiFunctionEnv { + #[allow(clippy::result_large_err)] pub fn run_async(self, mut store: Store) -> Result<(Self, Store), WasiRuntimeError> { // If no handle or runtime exists then create one #[cfg(feature = "sys-thread")] From 472d9180adb8fd7024aec1dbc2dd0cf3106b780d Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Sat, 23 Dec 2023 13:20:27 +1100 Subject: [PATCH 112/129] More linting and compile fixes --- lib/api/src/jsc/externals/memory.rs | 15 +++++++++++++++ lib/wasix/src/journal/concrete/archived.rs | 2 +- lib/wasix/src/runners/dcgi/callbacks.rs | 6 +++--- lib/wasix/src/runners/dcgi/handler.rs | 2 +- lib/wasix/src/runners/dcgi/meta.rs | 11 +---------- lib/wasix/src/state/func_env.rs | 7 ++++--- 6 files changed, 25 insertions(+), 18 deletions(-) diff --git a/lib/api/src/jsc/externals/memory.rs b/lib/api/src/jsc/externals/memory.rs index 0e153b341f6..e628e4d49ee 100644 --- a/lib/api/src/jsc/externals/memory.rs +++ b/lib/api/src/jsc/externals/memory.rs @@ -140,6 +140,21 @@ impl Memory { // Ok(Pages(new_pages)) } + pub fn grow_at_least( + &self, + store: &mut impl AsStoreMut, + min_size: u64, + ) -> Result<(), MemoryError> { + let cur_size = self.view(store).data_size(); + if min_size > cur_size { + let delta = min_size - cur_size; + let pages = ((delta - 1) / wasmer_types::WASM_PAGE_SIZE as u64) + 1; + + self.grow(store, Pages(pages as u32))?; + } + Ok(()) + } + pub fn reset(&self, _store: &mut impl AsStoreMut) -> Result<(), MemoryError> { Ok(()) } diff --git a/lib/wasix/src/journal/concrete/archived.rs b/lib/wasix/src/journal/concrete/archived.rs index 3ffe6af01d8..b5eb9852d5e 100644 --- a/lib/wasix/src/journal/concrete/archived.rs +++ b/lib/wasix/src/journal/concrete/archived.rs @@ -2893,7 +2893,7 @@ mod tests { #[test] pub fn test_record_init_module() { run_test(JournalEntry::InitModuleV1 { - wasm_hash: [13u8; 32], + wasm_hash: [13u8; 8], }); } diff --git a/lib/wasix/src/runners/dcgi/callbacks.rs b/lib/wasix/src/runners/dcgi/callbacks.rs index 53d0a2b186a..68122c892ce 100644 --- a/lib/wasix/src/runners/dcgi/callbacks.rs +++ b/lib/wasix/src/runners/dcgi/callbacks.rs @@ -49,17 +49,17 @@ impl wcgi::Callbacks for DcgiCallbacks { conf.env .state .fs - .swap_file(__WASI_STDIN_FILENO, Box::new(NullFile::default())) + .swap_file(__WASI_STDIN_FILENO, Box::::default()) .ok(); conf.env .state .fs - .swap_file(__WASI_STDOUT_FILENO, Box::new(NullFile::default())) + .swap_file(__WASI_STDOUT_FILENO, Box::::default()) .ok(); conf.env .state .fs - .swap_file(__WASI_STDERR_FILENO, Box::new(NullFile::default())) + .swap_file(__WASI_STDERR_FILENO, Box::::default()) .ok(); // Now we make the instance available for reuse diff --git a/lib/wasix/src/runners/dcgi/handler.rs b/lib/wasix/src/runners/dcgi/handler.rs index 6b90bdae2af..a0546a8f27e 100644 --- a/lib/wasix/src/runners/dcgi/handler.rs +++ b/lib/wasix/src/runners/dcgi/handler.rs @@ -69,7 +69,7 @@ impl Service> for Handler { let shard = request .headers() .get("X-Shard") - .map(|s| s.to_str().unwrap_or_else(|_| "").to_string()) + .map(|s| s.to_str().unwrap_or("").to_string()) .unwrap_or_else(|| "".to_string()); // Grab the metadata from the request diff --git a/lib/wasix/src/runners/dcgi/meta.rs b/lib/wasix/src/runners/dcgi/meta.rs index 5db2e1d258d..9308804bbf3 100644 --- a/lib/wasix/src/runners/dcgi/meta.rs +++ b/lib/wasix/src/runners/dcgi/meta.rs @@ -1,6 +1,6 @@ use std::sync::Arc; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Default)] pub struct DcgiMetadata { /// Shard associated with this WCGI pub shard: String, @@ -10,12 +10,3 @@ pub struct DcgiMetadata { /// or its dropped, for example if an error occurs pub master_lock: Option>>, } - -impl Default for DcgiMetadata { - fn default() -> Self { - DcgiMetadata { - shard: Default::default(), - master_lock: None, - } - } -} diff --git a/lib/wasix/src/state/func_env.rs b/lib/wasix/src/state/func_env.rs index 83febd3a8b8..cff6ed1918a 100644 --- a/lib/wasix/src/state/func_env.rs +++ b/lib/wasix/src/state/func_env.rs @@ -2,8 +2,7 @@ use std::sync::Arc; use tracing::trace; use wasmer::{ - AsStoreMut, AsStoreRef, ExportError, FunctionEnv, Imports, Instance, Memory, Module, - RuntimeError, Store, + AsStoreMut, AsStoreRef, ExportError, FunctionEnv, Imports, Instance, Memory, Module, Store, }; use wasmer_wasix_types::wasi::ExitCode; @@ -245,10 +244,12 @@ impl WasiFunctionEnv { /// by the WASM process thread itself. /// #[allow(clippy::result_large_err)] + #[allow(unused_variables, unused_mut)] pub unsafe fn bootstrap( &self, mut store: &'_ mut impl AsStoreMut, ) -> Result { + #[allow(unused_mut)] let mut rewind_state = None; #[cfg(feature = "journal")] @@ -284,7 +285,7 @@ impl WasiFunctionEnv { crate::journal::JournalEntry::InitModuleV1 { wasm_hash }, ) .map_err(|err| { - WasiRuntimeError::Runtime(RuntimeError::new(format!( + WasiRuntimeError::Runtime(wasmer::RuntimeError::new(format!( "journal failied to save the module initialization event - {}", err ))) From 79f4d26bed3e1bc3489bc001c210bd772d6624a4 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Sat, 23 Dec 2023 15:09:44 +1100 Subject: [PATCH 113/129] Added support for journals to DCGI and removed sharding --- lib/cli/src/commands/run/mod.rs | 20 ++-- lib/wasix/src/journal/factories/mod.rs | 3 + .../src/journal/factories/passthru_factory.rs | 23 ++++ lib/wasix/src/journal/factory.rs | 34 ++++++ lib/wasix/src/journal/mod.rs | 4 + lib/wasix/src/runners/dcgi/callbacks.rs | 22 ++-- lib/wasix/src/runners/dcgi/factory.rs | 102 ++++-------------- lib/wasix/src/runners/dcgi/handler.rs | 42 +++----- lib/wasix/src/runners/dcgi/meta.rs | 12 --- lib/wasix/src/runners/dcgi/mod.rs | 2 - lib/wasix/src/runners/dcgi/runner.rs | 26 +++-- lib/wasix/src/runners/wcgi/callbacks.rs | 20 ++-- lib/wasix/src/runners/wcgi/create_env.rs | 12 +-- lib/wasix/src/runners/wcgi/handler.rs | 53 ++++----- lib/wasix/src/runners/wcgi/runner.rs | 52 ++++----- 15 files changed, 193 insertions(+), 234 deletions(-) create mode 100644 lib/wasix/src/journal/factories/mod.rs create mode 100644 lib/wasix/src/journal/factories/passthru_factory.rs create mode 100644 lib/wasix/src/journal/factory.rs delete mode 100644 lib/wasix/src/runners/dcgi/meta.rs diff --git a/lib/cli/src/commands/run/mod.rs b/lib/cli/src/commands/run/mod.rs index 75739e54ab6..2ed0137c3bf 100644 --- a/lib/cli/src/commands/run/mod.rs +++ b/lib/cli/src/commands/run/mod.rs @@ -140,9 +140,11 @@ impl Run { } }; - // restore the TTY state as teh execution may have changed it + // restore the TTY state as the execution may have changed it if let Some(state) = tty { - runtime.tty().map(|tty| tty.tty_set(state)); + if let Some(tty) = runtime.tty() { + tty.tty_set(state); + } } if let Err(e) = &result { @@ -251,14 +253,11 @@ impl Run { runner.run_command(command_name, pkg, runtime) } - fn config_wcgi( + fn config_wcgi( &self, - config: &mut wcgi::Config, + config: &mut wcgi::Config, uses: Vec, - ) -> Result<(), Error> - where - M: Send + Sync + 'static, - { + ) -> Result<(), Error> { config .args(self.args.clone()) .addr(self.wcgi.addr) @@ -798,10 +797,7 @@ impl Callbacks { } } -impl wasmer_wasix::runners::wcgi::Callbacks for Callbacks -where - M: Send + Sync + 'static, -{ +impl wasmer_wasix::runners::wcgi::Callbacks for Callbacks { fn started(&self, _abort: AbortHandle) { println!("WCGI Server running at http://{}/", self.addr); } diff --git a/lib/wasix/src/journal/factories/mod.rs b/lib/wasix/src/journal/factories/mod.rs new file mode 100644 index 00000000000..df8fb436105 --- /dev/null +++ b/lib/wasix/src/journal/factories/mod.rs @@ -0,0 +1,3 @@ +mod passthru_factory; + +pub use passthru_factory::*; diff --git a/lib/wasix/src/journal/factories/passthru_factory.rs b/lib/wasix/src/journal/factories/passthru_factory.rs new file mode 100644 index 00000000000..e0cb5412ece --- /dev/null +++ b/lib/wasix/src/journal/factories/passthru_factory.rs @@ -0,0 +1,23 @@ +use std::sync::Arc; + +use crate::journal::{DynJournal, JournalFactory}; + +#[derive(Clone)] +pub struct PassthruJournalFactory { + journals: Vec>, +} + +impl PassthruJournalFactory { + pub fn new(journals: Vec>) -> Self { + Self { journals } + } +} + +impl JournalFactory for PassthruJournalFactory { + fn load_or_create( + &self, + _desc: crate::journal::JournalDescriptor, + ) -> anyhow::Result>> { + Ok(self.journals.clone()) + } +} diff --git a/lib/wasix/src/journal/factory.rs b/lib/wasix/src/journal/factory.rs new file mode 100644 index 00000000000..44cc0d5221c --- /dev/null +++ b/lib/wasix/src/journal/factory.rs @@ -0,0 +1,34 @@ +use std::sync::Arc; + +use semver::Version; + +use crate::runtime::module_cache::ModuleHash; + +use super::DynJournal; + +/// Describes the journal to be loaded or created +#[derive(Debug, Clone)] +pub struct JournalDescriptor { + /// Name of the package that holds the module (i.e. `wasmer/wapm2pirita`) + pub package_name: String, + /// Version of the package that contains the module (i.e. `1.0``) + pub version: Version, + /// Hash of the module that will use this journal + pub module_hash: ModuleHash, +} + +/// The journal factory creates or loads journals based of a set of +/// properties that describe the journal. +pub trait JournalFactory { + /// Creates or loads a journal based on a descriptor that makes it unique + /// + /// It is the responsibility of the implementor of this function + /// to reuse and reload existing journals based on the descriptor data + /// that is supplied. + /// + /// The factory can return more than one journal where only the last + /// journal is the active one while the former journals are base journals + fn load_or_create(&self, desc: JournalDescriptor) -> anyhow::Result>>; +} + +pub type DynJournalFactory = dyn JournalFactory + Send + Sync; diff --git a/lib/wasix/src/journal/mod.rs b/lib/wasix/src/journal/mod.rs index 14b5cb5984b..2802cc1b29e 100644 --- a/lib/wasix/src/journal/mod.rs +++ b/lib/wasix/src/journal/mod.rs @@ -6,12 +6,16 @@ mod effector; #[path = "effector/unimplemented.rs"] mod effector; mod entry; +mod factories; +mod factory; mod snapshot; mod util; pub use concrete::*; pub use effector::*; pub use entry::*; +pub use factories::*; +pub use factory::*; pub use snapshot::*; pub use util::*; diff --git a/lib/wasix/src/runners/dcgi/callbacks.rs b/lib/wasix/src/runners/dcgi/callbacks.rs index 68122c892ce..87b430b5c3a 100644 --- a/lib/wasix/src/runners/dcgi/callbacks.rs +++ b/lib/wasix/src/runners/dcgi/callbacks.rs @@ -11,14 +11,14 @@ use wasmer_wasix_types::types::{__WASI_STDERR_FILENO, __WASI_STDIN_FILENO, __WAS #[derivative(Debug)] pub struct DcgiCallbacks { #[derivative(Debug = "ignore")] - inner: Arc>, + inner: Arc, factory: DcgiInstanceFactory, } impl DcgiCallbacks { pub fn new(factory: DcgiInstanceFactory, inner: C) -> Self where - C: wcgi::Callbacks, + C: wcgi::Callbacks, { Self { inner: Arc::new(inner), @@ -28,7 +28,7 @@ impl DcgiCallbacks { } #[async_trait::async_trait] -impl wcgi::Callbacks for DcgiCallbacks { +impl wcgi::Callbacks for DcgiCallbacks { fn started(&self, abort: AbortHandle) { self.inner.started(abort) } @@ -41,8 +41,8 @@ impl wcgi::Callbacks for DcgiCallbacks { self.inner.on_stderr_error(error) } - async fn recycle_env(&self, conf: RecycleEnvConfig) { - tracing::debug!(shard = conf.meta.shard, "recycling DCGI instance"); + async fn recycle_env(&self, conf: RecycleEnvConfig) { + tracing::debug!("recycling DCGI instance"); // The stdio have to be reattached on each call as they are // read to completion (EOF) during nominal flows @@ -66,17 +66,11 @@ impl wcgi::Callbacks for DcgiCallbacks { self.factory.release(conf).await; } - async fn create_env( - &self, - mut conf: CreateEnvConfig, - ) -> anyhow::Result { - tracing::debug!( - shard = conf.meta.shard, - "attempting to acquire existing DCGI instance" - ); + async fn create_env(&self, mut conf: CreateEnvConfig) -> anyhow::Result { + tracing::debug!("attempting to acquire existing DCGI instance"); if let Some(res) = self.factory.acquire(&mut conf).await { - tracing::debug!(shard = conf.meta.shard, "found existing DCGI instance"); + tracing::debug!("found existing DCGI instance"); return Ok(res); } diff --git a/lib/wasix/src/runners/dcgi/factory.rs b/lib/wasix/src/runners/dcgi/factory.rs index 12d4caf432e..056e490164b 100644 --- a/lib/wasix/src/runners/dcgi/factory.rs +++ b/lib/wasix/src/runners/dcgi/factory.rs @@ -1,9 +1,6 @@ -use std::{ - collections::{HashMap, VecDeque}, - sync::{Arc, Mutex}, - time::Instant, -}; +use std::sync::{Arc, Mutex}; +use derivative::Derivative; use virtual_fs::Pipe; use wasmer_wasix_types::types::{__WASI_STDERR_FILENO, __WASI_STDIN_FILENO, __WASI_STDOUT_FILENO}; @@ -15,33 +12,16 @@ use crate::{ use super::*; -#[derive(Debug)] -struct StateShard { - instances: VecDeque, - last_acquire: Instant, - master_lock: Arc>, -} - -impl Default for StateShard { - fn default() -> Self { - Self { - instances: Default::default(), - last_acquire: Instant::now(), - master_lock: Arc::new(Default::default()), - } - } -} - #[derive(Debug, Default)] struct State { - // List of the shards and a queue of DCGI instances that - // are running for these shards - shards: HashMap, + /// Once the instance is running it will + instance: Option, } /// This factory will store and reuse instances between invocations thus /// allowing for the instances to be stateful. -#[derive(Debug, Clone, Default)] +#[derive(Derivative, Clone)] +#[derivative(Debug)] pub struct DcgiInstanceFactory { state: Arc>, } @@ -53,63 +33,23 @@ impl DcgiInstanceFactory { } } - pub async fn release(&self, mut conf: RecycleEnvConfig) { - let shard = conf.meta.shard; - + pub async fn release(&self, conf: RecycleEnvConfig) { let mut state = self.state.lock().unwrap(); - state - .shards - .entry(shard) - .or_default() - .instances - .push_front(DcgiInstance { - env: conf.env, - //memory: conf.memory, - //store: conf.store, - }); - - drop(state); - conf.meta.master_lock.take(); + state.instance.replace(DcgiInstance { + env: conf.env, + //memory: conf.memory, + //store: conf.store, + }); } - pub async fn acquire( - &self, - conf: &mut CreateEnvConfig, - ) -> Option { - let shard = conf.meta.shard.clone(); - - // We take a short lock that looks for existing instances - // that have been recycled, otherwise we will use the - // master lock to prevent concurrent creations - let master_lock = { - let mut state = self.state.lock().unwrap(); - let shard = state.shards.entry(shard.clone()).or_default(); - shard.last_acquire = Instant::now(); - shard.master_lock.clone() - }; - - // We acquire a master lock whenever creating a new instance and hold it - // until the instance dies or the instance is returned to the factory. This - // is done using the `DcgiMetadata` - let master_lock = master_lock.clone().lock_owned().await; - conf.meta.master_lock.replace(Arc::new(master_lock)); - - // We check the shard again under a short lock as maybe one was returned - { - let mut state = self.state.lock().unwrap(); - let shard = state.shards.entry(shard).or_default(); - shard.last_acquire = Instant::now(); - - if let Some(inst) = shard.instances.pop_front() { - tracing::debug!( - shard = conf.meta.shard, - "attempting to reinitialize DCGI instance" - ); - match convert_instance(inst, conf) { - Ok(converted) => return Some(converted), - Err(err) => { - tracing::warn!("failed to reinitialize DCGI instance - {}", err); - } + pub async fn acquire(&self, conf: &mut CreateEnvConfig) -> Option { + let mut state = self.state.lock().unwrap(); + if let Some(inst) = state.instance.take() { + tracing::debug!("attempting to reinitialize DCGI instance"); + match convert_instance(inst, conf) { + Ok(converted) => return Some(converted), + Err(err) => { + tracing::warn!("failed to reinitialize DCGI instance - {}", err); } } } @@ -120,7 +60,7 @@ impl DcgiInstanceFactory { fn convert_instance( inst: DcgiInstance, - conf: &mut CreateEnvConfig, + conf: &mut CreateEnvConfig, ) -> anyhow::Result { let mut env = inst.env; diff --git a/lib/wasix/src/runners/dcgi/handler.rs b/lib/wasix/src/runners/dcgi/handler.rs index a0546a8f27e..627751fd4ec 100644 --- a/lib/wasix/src/runners/dcgi/handler.rs +++ b/lib/wasix/src/runners/dcgi/handler.rs @@ -7,34 +7,37 @@ use hyper::{service::Service, Body}; use crate::runners::wcgi; -use super::{DcgiInstanceFactory, DcgiMetadata}; +use super::DcgiInstanceFactory; /// The shared object that manages the instantiaion of WASI executables and /// communicating with them via the CGI protocol. #[derive(Clone, Debug)] pub(crate) struct Handler { state: Arc, - inner: wcgi::Handler, + inner: wcgi::Handler, } impl Handler { - pub(crate) fn from_wcgi_handler(handler: wcgi::Handler) -> Self { + pub(crate) fn new(handler: wcgi::Handler) -> Self { Handler { state: Arc::new(SharedState { inner: handler.deref().clone(), - factory: DcgiInstanceFactory::default(), + factory: DcgiInstanceFactory::new(), + master_lock: Default::default(), }), inner: handler, } } #[tracing::instrument(level = "debug", skip_all, err)] - pub(crate) async fn handle( - &self, - req: Request, - meta: DcgiMetadata, - ) -> Result, Error> { - self.inner.handle(req, meta).await + pub(crate) async fn handle(&self, req: Request) -> Result, Error> { + // we acquire a guard token so that only one request at a time can be processed + // which effectively means that DCGI is single-threaded. This is a limitation + // of the MVP which should be rectified in future releases. + let guard_token = self.state.master_lock.clone().lock_owned().await; + + // Process the request as a normal WCGI request + self.inner.handle(req, guard_token).await } } @@ -49,8 +52,9 @@ impl Deref for Handler { #[derive(derivative::Derivative, Clone)] #[derivative(Debug)] pub(crate) struct SharedState { - pub(crate) inner: Arc>, + pub(crate) inner: Arc, factory: DcgiInstanceFactory, + master_lock: Arc>, } impl Service> for Handler { @@ -64,23 +68,9 @@ impl Service> for Handler { } fn call(&mut self, request: Request) -> Self::Future { - // We determine the shard that this DCGI request will run against - // (multiple shards can be served by the same endpoint) - let shard = request - .headers() - .get("X-Shard") - .map(|s| s.to_str().unwrap_or("").to_string()) - .unwrap_or_else(|| "".to_string()); - - // Grab the metadata from the request - let meta = DcgiMetadata { - shard, - master_lock: None, - }; - // Note: all fields are reference-counted so cloning is pretty cheap let handler = self.clone(); - let fut = async move { handler.handle(request, meta).await }; + let fut = async move { handler.handle(request).await }; fut.boxed() } } diff --git a/lib/wasix/src/runners/dcgi/meta.rs b/lib/wasix/src/runners/dcgi/meta.rs deleted file mode 100644 index 9308804bbf3..00000000000 --- a/lib/wasix/src/runners/dcgi/meta.rs +++ /dev/null @@ -1,12 +0,0 @@ -use std::sync::Arc; - -#[derive(Debug, Clone, Default)] -pub struct DcgiMetadata { - /// Shard associated with this WCGI - pub shard: String, - /// This master lock prevents multiple writable instances - /// from running at the same time. It is held for the duration - /// of the instance running until it returns to the factory - /// or its dropped, for example if an error occurs - pub master_lock: Option>>, -} diff --git a/lib/wasix/src/runners/dcgi/mod.rs b/lib/wasix/src/runners/dcgi/mod.rs index 101b880c7c8..f9b46080d8c 100644 --- a/lib/wasix/src/runners/dcgi/mod.rs +++ b/lib/wasix/src/runners/dcgi/mod.rs @@ -2,7 +2,6 @@ mod callbacks; mod factory; mod handler; mod instance; -mod meta; mod runner; pub use self::runner::{Config, DcgiRunner}; @@ -10,4 +9,3 @@ pub use callbacks::DcgiCallbacks; pub use factory::DcgiInstanceFactory; pub use futures::future::AbortHandle; pub(crate) use instance::DcgiInstance; -pub use meta::DcgiMetadata; diff --git a/lib/wasix/src/runners/dcgi/runner.rs b/lib/wasix/src/runners/dcgi/runner.rs index b95a230024f..c19da0f036c 100644 --- a/lib/wasix/src/runners/dcgi/runner.rs +++ b/lib/wasix/src/runners/dcgi/runner.rs @@ -7,6 +7,7 @@ use webc::metadata::Command; use crate::{ bin_factory::BinaryPackage, capabilities::Capabilities, + journal::PassthruJournalFactory, runners::{ dcgi::handler::Handler, wcgi::{self, NoOpWcgiCallbacks, WcgiRunner}, @@ -15,12 +16,12 @@ use crate::{ Runtime, }; -use super::{DcgiCallbacks, DcgiInstanceFactory, DcgiMetadata}; +use super::{DcgiCallbacks, DcgiInstanceFactory}; #[derive(Debug)] pub struct DcgiRunner { config: Config, - inner: wcgi::WcgiRunner, + inner: wcgi::WcgiRunner, } impl DcgiRunner { @@ -45,10 +46,17 @@ impl DcgiRunner { pkg: &BinaryPackage, runtime: Arc, ) -> Result { - let inner: wcgi::Handler = - self.inner - .prepare_handler(command_name, pkg, true, CgiDialect::Rfc3875, runtime)?; - Ok(Handler::from_wcgi_handler(inner)) + let inner: wcgi::Handler = self.inner.prepare_handler( + command_name, + pkg, + true, + CgiDialect::Rfc3875, + runtime, + Arc::new(PassthruJournalFactory::new( + self.config.inner.wasi.journals.clone(), + )), + )?; + Ok(Handler::new(inner)) } } @@ -73,11 +81,11 @@ impl crate::runners::Runner for DcgiRunner { #[derive(Debug)] pub struct Config { - inner: wcgi::Config, + inner: wcgi::Config, } impl Config { - pub fn inner(&mut self) -> &mut wcgi::Config { + pub fn inner(&mut self) -> &mut wcgi::Config { &mut self.inner } @@ -142,7 +150,7 @@ impl Config { /// lifecycle. pub fn callbacks( &mut self, - callbacks: impl wcgi::Callbacks + Send + Sync + 'static, + callbacks: impl wcgi::Callbacks + Send + Sync + 'static, ) -> &mut Self { self.inner.callbacks(callbacks); self diff --git a/lib/wasix/src/runners/wcgi/callbacks.rs b/lib/wasix/src/runners/wcgi/callbacks.rs index 101a24f68dc..1716691b314 100644 --- a/lib/wasix/src/runners/wcgi/callbacks.rs +++ b/lib/wasix/src/runners/wcgi/callbacks.rs @@ -8,11 +8,7 @@ use crate::{runtime::module_cache::ModuleHash, WasiEnv}; use super::{create_env::default_recycle_env, handler::SetupBuilder, *}; /// Configuration used for creating a new environment -pub struct CreateEnvConfig -where - M: Send + Sync + 'static, -{ - pub meta: M, +pub struct CreateEnvConfig { pub env: HashMap, pub program_name: String, pub module: Module, @@ -31,8 +27,7 @@ pub struct CreateEnvResult { } /// Configuration used for reusing an new environment -pub struct RecycleEnvConfig { - pub meta: M, +pub struct RecycleEnvConfig { pub env: WasiEnv, pub memory: Memory, pub store: Store, @@ -41,10 +36,7 @@ pub struct RecycleEnvConfig { /// Callbacks that are triggered at various points in the lifecycle of a runner /// and any WebAssembly instances it may start. #[async_trait::async_trait] -pub trait Callbacks: Send + Sync + 'static -where - M: Send + Sync + 'static, -{ +pub trait Callbacks: Send + Sync + 'static { /// A callback that is called whenever the server starts. fn started(&self, _abort: AbortHandle) {} @@ -55,16 +47,16 @@ where fn on_stderr_error(&self, _error: std::io::Error) {} /// Recycle the WASI environment - async fn recycle_env(&self, conf: RecycleEnvConfig) { + async fn recycle_env(&self, conf: RecycleEnvConfig) { default_recycle_env(conf).await } /// Create the WASI environment - async fn create_env(&self, conf: CreateEnvConfig) -> anyhow::Result { + async fn create_env(&self, conf: CreateEnvConfig) -> anyhow::Result { default_create_env(conf).await } } pub struct NoOpWcgiCallbacks; -impl Callbacks for NoOpWcgiCallbacks where M: Send + Sync + 'static {} +impl Callbacks for NoOpWcgiCallbacks {} diff --git a/lib/wasix/src/runners/wcgi/create_env.rs b/lib/wasix/src/runners/wcgi/create_env.rs index 53f5537ef7a..c8ab9fe9ac8 100644 --- a/lib/wasix/src/runners/wcgi/create_env.rs +++ b/lib/wasix/src/runners/wcgi/create_env.rs @@ -7,22 +7,14 @@ use crate::{ use super::{callbacks::CreateEnvConfig, RecycleEnvConfig}; -pub(crate) async fn default_recycle_env(mut conf: RecycleEnvConfig) -where - M: Send + Sync + 'static, -{ +pub(crate) async fn default_recycle_env(mut conf: RecycleEnvConfig) { tracing::debug!("Destroying the WebAssembly instance"); conf.env.disable_fs_cleanup = false; conf.env.on_exit(None).await; } -pub(crate) async fn default_create_env( - conf: CreateEnvConfig, -) -> anyhow::Result -where - M: Send + Sync + 'static, -{ +pub(crate) async fn default_create_env(conf: CreateEnvConfig) -> anyhow::Result { tracing::debug!("Creating the WebAssembly instance"); let (req_body_sender, req_body_receiver) = Pipe::channel(); diff --git a/lib/wasix/src/runners/wcgi/handler.rs b/lib/wasix/src/runners/wcgi/handler.rs index d8ea21f40ab..6f0eb012bc8 100644 --- a/lib/wasix/src/runners/wcgi/handler.rs +++ b/lib/wasix/src/runners/wcgi/handler.rs @@ -28,22 +28,21 @@ use crate::{ /// The shared object that manages the instantiaion of WASI executables and /// communicating with them via the CGI protocol. #[derive(Clone, Debug)] -pub(crate) struct Handler(Arc>) -where - M: Send + Sync + 'static; - -impl Handler -where - M: Send + Sync + 'static, -{ - pub(crate) fn new(state: Arc>) -> Self { +pub(crate) struct Handler(Arc); + +impl Handler { + pub(crate) fn new(state: Arc) -> Self { Handler(state) } #[tracing::instrument(level = "debug", skip_all, err)] - pub(crate) async fn handle(&self, req: Request, meta: M) -> Result, Error> + pub(crate) async fn handle( + &self, + req: Request, + token: T, + ) -> Result, Error> where - M: Clone, + T: Send + 'static, { tracing::debug!(headers=?req.headers()); @@ -64,7 +63,6 @@ where let create = self .callbacks .create_env(CreateEnvConfig { - meta: meta.clone(), env: request_specific_env, program_name: self.program_name.clone(), module: self.module.clone(), @@ -89,11 +87,15 @@ where let callbacks = callbacks.clone(); move |props: TaskWasmRecycleProperties| { InlineWaker::block_on(callbacks.recycle_env(RecycleEnvConfig { - meta, env: props.env, store: props.store, memory: props.memory, })); + + // We release the token after we recycle the environment + // so that race conditions (such as reusing instances) are + // avoided + drop(token); } }; let finished = env.process.finished.clone(); @@ -219,11 +221,8 @@ where } } -impl Deref for Handler -where - M: Send + Sync + 'static, -{ - type Target = Arc>; +impl Deref for Handler { + type Target = Arc; fn deref(&self) -> &Self::Target { &self.0 @@ -267,14 +266,11 @@ async fn drive_request_to_completion( /// Read the instance's stderr, taking care to preserve output even when WASI /// pipe errors occur so users still have *something* they use for /// troubleshooting. -async fn consume_stderr( +async fn consume_stderr( stderr: impl AsyncRead + Send + Unpin + 'static, - callbacks: Arc>, + callbacks: Arc, propagate_stderr: bool, -) -> Option> -where - M: Send + Sync + 'static, -{ +) -> Option> { let mut stderr = tokio::io::BufReader::new(stderr); let mut propagate = match propagate_stderr { @@ -315,10 +311,7 @@ pub type SetupBuilder = Arc Result<(), anyhow::Er #[derive(derivative::Derivative)] #[derivative(Debug)] -pub(crate) struct SharedState -where - M: Send + Sync + 'static, -{ +pub(crate) struct SharedState { pub(crate) module: Module, pub(crate) module_hash: ModuleHash, pub(crate) dialect: CgiDialect, @@ -327,12 +320,12 @@ where #[derivative(Debug = "ignore")] pub(crate) setup_builder: SetupBuilder, #[derivative(Debug = "ignore")] - pub(crate) callbacks: Arc>, + pub(crate) callbacks: Arc, #[derivative(Debug = "ignore")] pub(crate) runtime: Arc, } -impl Service> for Handler<()> { +impl Service> for Handler { type Response = Response; type Error = Error; type Future = Pin, Error>> + Send>>; diff --git a/lib/wasix/src/runners/wcgi/runner.rs b/lib/wasix/src/runners/wcgi/runner.rs index f8e9431eac6..e7040ae25af 100644 --- a/lib/wasix/src/runners/wcgi/runner.rs +++ b/lib/wasix/src/runners/wcgi/runner.rs @@ -15,6 +15,7 @@ use webc::metadata::{ use crate::{ bin_factory::BinaryPackage, capabilities::Capabilities, + journal::{DynJournalFactory, JournalDescriptor, PassthruJournalFactory}, runners::{ wasi_common::CommonWasiOptions, wcgi::handler::{Handler, SharedState}, @@ -27,27 +28,21 @@ use crate::{ use super::Callbacks; #[derive(Debug)] -pub struct WcgiRunner -where - M: Send + Sync + 'static, -{ - config: Config, +pub struct WcgiRunner { + config: Config, } -impl WcgiRunner -where - M: Send + Sync + 'static, -{ +impl WcgiRunner { pub fn new(callbacks: C) -> Self where - C: Callbacks, + C: Callbacks, { Self { config: Config::new(callbacks), } } - pub fn config(&mut self) -> &mut Config { + pub fn config(&mut self) -> &mut Config { &mut self.config } @@ -59,7 +54,8 @@ where propagate_stderr: bool, default_dialect: CgiDialect, runtime: Arc, - ) -> Result, Error> { + journal_factory: Arc, + ) -> Result { let cmd = pkg .get_command(command_name) .with_context(|| format!("The package doesn't contain a \"{command_name}\" command"))?; @@ -78,12 +74,23 @@ where let container_fs = Arc::clone(&pkg.webc_fs); + let journal_desc = JournalDescriptor { + package_name: pkg.package_name.clone(), + version: pkg.version.clone(), + module_hash: pkg.hash(), + }; + let wasi_common = self.config.wasi.clone(); let rt = Arc::clone(&runtime); let setup_builder = move |builder: &mut WasiEnvBuilder| { wasi_common.prepare_webc_env(builder, Some(Arc::clone(&container_fs)), &wasi, None)?; builder.set_runtime(Arc::clone(&rt)); + let journals = journal_factory.load_or_create(journal_desc.clone())?; + for journal in journals { + builder.add_journal(journal); + } + Ok(()) }; @@ -182,6 +189,9 @@ impl crate::runners::Runner for WcgiRunner { false, CgiDialect::Wcgi, Arc::clone(&runtime), + Arc::new(PassthruJournalFactory::new( + self.config.wasi.journals.clone(), + )), )?; self.run_command_with_handler(handler, runtime) } @@ -189,17 +199,14 @@ impl crate::runners::Runner for WcgiRunner { #[derive(derivative::Derivative)] #[derivative(Debug)] -pub struct Config { +pub struct Config { pub(crate) wasi: CommonWasiOptions, pub(crate) addr: SocketAddr, #[derivative(Debug = "ignore")] - pub(crate) callbacks: Arc>, + pub(crate) callbacks: Arc, } -impl Config -where - M: Send + Sync + 'static, -{ +impl Config { pub fn addr(&mut self, addr: SocketAddr) -> &mut Self { self.addr = addr; self @@ -261,7 +268,7 @@ where /// Set callbacks that will be triggered at various points in the runner's /// lifecycle. - pub fn callbacks(&mut self, callbacks: impl Callbacks + Send + Sync + 'static) -> &mut Self { + pub fn callbacks(&mut self, callbacks: impl Callbacks + Send + Sync + 'static) -> &mut Self { self.callbacks = Arc::new(callbacks); self } @@ -321,13 +328,10 @@ where } } -impl Config -where - M: Send + Sync + 'static, -{ +impl Config { pub fn new(callbacks: C) -> Self where - C: Callbacks, + C: Callbacks, { Self { addr: ([127, 0, 0, 1], 8000).into(), From b4131b0dbbacedaf4fac378a78ab08d5d845b056 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Sat, 23 Dec 2023 17:34:21 +1100 Subject: [PATCH 114/129] Fixed the journal wiring to DCGI --- lib/cli/src/commands/run/mod.rs | 9 +- lib/cli/src/commands/run/wasi.rs | 23 +- lib/wasix/src/journal/concrete/arc.rs | 41 ++++ lib/wasix/src/journal/concrete/compacting.rs | 9 +- .../journal/concrete/compacting_log_file.rs | 8 +- lib/wasix/src/journal/concrete/filter.rs | 100 +++++++-- lib/wasix/src/journal/concrete/mod.rs | 2 + lib/wasix/src/journal/concrete/null.rs | 10 +- lib/wasix/src/journal/concrete/printing.rs | 10 +- lib/wasix/src/journal/entry.rs | 13 +- lib/wasix/src/journal/factories/mod.rs | 3 - .../src/journal/factories/passthru_factory.rs | 23 -- lib/wasix/src/journal/factory.rs | 34 --- lib/wasix/src/journal/mod.rs | 4 - lib/wasix/src/runners/dcgi/runner.rs | 41 ++-- lib/wasix/src/runners/wcgi/runner.rs | 17 -- lib/wasix/src/runtime/mod.rs | 201 ++++++++++++++++++ lib/wasix/tests/runners.rs | 2 +- 18 files changed, 431 insertions(+), 119 deletions(-) create mode 100644 lib/wasix/src/journal/concrete/arc.rs delete mode 100644 lib/wasix/src/journal/factories/mod.rs delete mode 100644 lib/wasix/src/journal/factories/passthru_factory.rs delete mode 100644 lib/wasix/src/journal/factory.rs diff --git a/lib/cli/src/commands/run/mod.rs b/lib/cli/src/commands/run/mod.rs index 2ed0137c3bf..cde97f181bd 100644 --- a/lib/cli/src/commands/run/mod.rs +++ b/lib/cli/src/commands/run/mod.rs @@ -32,6 +32,7 @@ use wasmer_registry::{wasmer_env::WasmerEnv, Package}; use wasmer_wasix::journal::{LogFileJournal, SnapshotTrigger}; use wasmer_wasix::{ bin_factory::BinaryPackage, + journal::CompactingLogFileJournal, runners::{ dcgi::DcgiInstanceFactory, wcgi::{self, NoOpWcgiCallbacks}, @@ -287,7 +288,13 @@ impl Run { config.with_snapshot_interval(Duration::from_millis(period)); } for journal in self.wasi.journals.clone() { - config.add_journal(Arc::new(LogFileJournal::new(journal)?)); + if self.wasi.enable_compaction { + config.add_journal(Arc::new( + CompactingLogFileJournal::new(journal)?.with_compact_on_drop(), + )); + } else { + config.add_journal(Arc::new(LogFileJournal::new(journal)?)); + } } } diff --git a/lib/cli/src/commands/run/wasi.rs b/lib/cli/src/commands/run/wasi.rs index 22fd08af861..0db1a3fa928 100644 --- a/lib/cli/src/commands/run/wasi.rs +++ b/lib/cli/src/commands/run/wasi.rs @@ -21,6 +21,7 @@ use wasmer_wasix::{ capabilities::Capabilities, default_fs_backing, get_wasi_versions, http::HttpClient, + journal::CompactingLogFileJournal, os::{tty_sys::SysTty, TtyBridge}, rewind_ext, runners::{MappedCommand, MappedDirectory}, @@ -121,6 +122,12 @@ pub struct Wasi { #[clap(long = "journal")] pub journals: Vec, + /// Flag that indicates if the journal will be automatically compacted + /// as it fills up and when the process exits + #[cfg(feature = "journal")] + #[clap(long = "enable-compaction")] + pub enable_compaction: bool, + /// Indicates what events will cause a snapshot to be taken /// and written to the journal file. /// @@ -351,7 +358,13 @@ impl Wasi { builder.with_snapshot_interval(std::time::Duration::from_millis(interval)); } for journal in self.journals.iter() { - builder.add_journal(Arc::new(LogFileJournal::new(journal)?)); + if self.enable_compaction { + builder.add_journal(Arc::new( + CompactingLogFileJournal::new(journal)?.with_compact_on_drop(), + )); + } else { + builder.add_journal(Arc::new(LogFileJournal::new(journal)?)); + } } } @@ -510,7 +523,13 @@ impl Wasi { #[cfg(feature = "journal")] for journal in self.journals.clone() { - rt.add_journal(Arc::new(LogFileJournal::new(journal)?)); + if self.enable_compaction { + rt.add_journal(Arc::new( + CompactingLogFileJournal::new(journal)?.with_compact_on_drop(), + )); + } else { + rt.add_journal(Arc::new(LogFileJournal::new(journal)?)); + } } if !self.no_tty { diff --git a/lib/wasix/src/journal/concrete/arc.rs b/lib/wasix/src/journal/concrete/arc.rs new file mode 100644 index 00000000000..fa7a84146d8 --- /dev/null +++ b/lib/wasix/src/journal/concrete/arc.rs @@ -0,0 +1,41 @@ +use super::*; +use std::ops::Deref; +use std::sync::Arc; + +impl ReadableJournal for Arc { + fn read(&self) -> anyhow::Result>> { + self.deref().read() + } + + fn as_restarted(&self) -> anyhow::Result> { + self.deref().as_restarted() + } +} + +impl WritableJournal for Arc { + fn write<'a>(&'a self, entry: JournalEntry<'a>) -> anyhow::Result { + self.deref().write(entry) + } +} + +impl ReadableJournal for Arc { + fn read(&self) -> anyhow::Result>> { + self.deref().read() + } + + fn as_restarted(&self) -> anyhow::Result> { + self.deref().as_restarted() + } +} + +impl WritableJournal for Arc { + fn write<'a>(&'a self, entry: JournalEntry<'a>) -> anyhow::Result { + self.deref().write(entry) + } +} + +impl Journal for Arc { + fn split(self) -> (Box, Box) { + (Box::new(self.clone()), Box::new(self.clone())) + } +} diff --git a/lib/wasix/src/journal/concrete/compacting.rs b/lib/wasix/src/journal/concrete/compacting.rs index 0b1d7937dcf..75d5e9f054b 100644 --- a/lib/wasix/src/journal/concrete/compacting.rs +++ b/lib/wasix/src/journal/concrete/compacting.rs @@ -396,8 +396,13 @@ impl CompactingJournal { (self.tx, self.rx) } - pub fn replace_inner(&self, inner: J) { - self.tx.replace_inner(inner) + pub fn replace_inner(&mut self, inner: J) { + let (inner_tx, inner_rx) = inner.split(); + let inner_rx_restarted = inner_rx.as_restarted().unwrap(); + + self.tx + .replace_inner(RecombinedJournal::new(inner_tx, inner_rx)); + self.rx.inner = inner_rx_restarted; } } diff --git a/lib/wasix/src/journal/concrete/compacting_log_file.rs b/lib/wasix/src/journal/concrete/compacting_log_file.rs index 3ee8078f694..4085d947b66 100644 --- a/lib/wasix/src/journal/concrete/compacting_log_file.rs +++ b/lib/wasix/src/journal/concrete/compacting_log_file.rs @@ -39,7 +39,7 @@ impl CompactingLogFileJournal { pub fn new(path: impl AsRef) -> anyhow::Result { // We prepare a compacting journal which does nothing // with the events other than learn from them - let compacting = CompactingJournal::new(NullJournal::default())?; + let mut compacting = CompactingJournal::new(NullJournal::default())?; // We first feed all the entries into the compactor so that // it learns all the records @@ -199,3 +199,9 @@ impl WritableJournal for CompactingLogFileJournal { self.tx.write(entry) } } + +impl Journal for CompactingLogFileJournal { + fn split(self) -> (Box, Box) { + (Box::new(self.tx), Box::new(self.rx)) + } +} diff --git a/lib/wasix/src/journal/concrete/filter.rs b/lib/wasix/src/journal/concrete/filter.rs index 7b936afeb49..cfeb22f4852 100644 --- a/lib/wasix/src/journal/concrete/filter.rs +++ b/lib/wasix/src/journal/concrete/filter.rs @@ -24,6 +24,7 @@ pub struct FilteredJournalTx { filter_memory: bool, filter_threads: bool, filter_fs: bool, + filter_stdio: bool, filter_core: bool, filter_snapshots: bool, filter_net: bool, @@ -50,6 +51,7 @@ impl FilteredJournal { filter_memory: false, filter_threads: false, filter_fs: false, + filter_stdio: false, filter_core: false, filter_snapshots: false, filter_net: false, @@ -60,6 +62,28 @@ impl FilteredJournal { } } + pub fn clone_with_inner(&self, inner: J) -> Self + where + J: Journal, + { + let (tx, rx) = inner.split(); + Self { + tx: FilteredJournalTx { + inner: tx, + filter_memory: self.tx.filter_memory, + filter_threads: self.tx.filter_threads, + filter_fs: self.tx.filter_fs, + filter_stdio: self.tx.filter_stdio, + filter_core: self.tx.filter_core, + filter_snapshots: self.tx.filter_snapshots, + filter_net: self.tx.filter_net, + filter_events: self.tx.filter_events.clone(), + event_index: AtomicUsize::new(self.tx.event_index.load(Ordering::SeqCst)), + }, + rx: FilteredJournalRx { inner: rx }, + } + } + pub fn with_ignore_memory(mut self, val: bool) -> Self { self.tx.filter_memory = val; self @@ -75,6 +99,11 @@ impl FilteredJournal { self } + pub fn with_ignore_stdio(mut self, val: bool) -> Self { + self.tx.filter_stdio = val; + self + } + pub fn with_ignore_core(mut self, val: bool) -> Self { self.tx.filter_core = val; self @@ -101,6 +130,41 @@ impl FilteredJournal { } } + pub fn set_ignore_memory(&mut self, val: bool) -> &mut Self { + self.tx.filter_memory = val; + self + } + + pub fn set_ignore_threads(&mut self, val: bool) -> &mut Self { + self.tx.filter_threads = val; + self + } + + pub fn set_ignore_fs(&mut self, val: bool) -> &mut Self { + self.tx.filter_fs = val; + self + } + + pub fn set_ignore_stdio(&mut self, val: bool) -> &mut Self { + self.tx.filter_stdio = val; + self + } + + pub fn set_ignore_core(&mut self, val: bool) -> &mut Self { + self.tx.filter_core = val; + self + } + + pub fn set_ignore_snapshots(&mut self, val: bool) -> &mut Self { + self.tx.filter_snapshots = val; + self + } + + pub fn set_ignore_networking(&mut self, val: bool) -> &mut Self { + self.tx.filter_net = val; + self + } + pub fn into_inner(self) -> RecombinedJournal { RecombinedJournal::new(self.tx.inner, self.rx.inner) } @@ -139,23 +203,33 @@ impl WritableJournal for FilteredJournalTx { } entry } - JournalEntry::FileDescriptorSeekV1 { .. } - | JournalEntry::FileDescriptorWriteV1 { .. } - | JournalEntry::OpenFileDescriptorV1 { .. } - | JournalEntry::CloseFileDescriptorV1 { .. } - | JournalEntry::RemoveDirectoryV1 { .. } + JournalEntry::FileDescriptorSeekV1 { fd, .. } + | JournalEntry::FileDescriptorWriteV1 { fd, .. } + | JournalEntry::OpenFileDescriptorV1 { fd, .. } + | JournalEntry::CloseFileDescriptorV1 { fd, .. } + | JournalEntry::RenumberFileDescriptorV1 { old_fd: fd, .. } + | JournalEntry::DuplicateFileDescriptorV1 { + original_fd: fd, .. + } + | JournalEntry::FileDescriptorSetFlagsV1 { fd, .. } + | JournalEntry::FileDescriptorAdviseV1 { fd, .. } + | JournalEntry::FileDescriptorAllocateV1 { fd, .. } + | JournalEntry::FileDescriptorSetRightsV1 { fd, .. } + | JournalEntry::FileDescriptorSetTimesV1 { fd, .. } + | JournalEntry::FileDescriptorSetSizeV1 { fd, .. } => { + if self.filter_stdio && fd <= 2 { + return Ok(0); + } + if self.filter_fs { + return Ok(0); + } + entry + } + JournalEntry::RemoveDirectoryV1 { .. } | JournalEntry::UnlinkFileV1 { .. } | JournalEntry::PathRenameV1 { .. } - | JournalEntry::RenumberFileDescriptorV1 { .. } - | JournalEntry::DuplicateFileDescriptorV1 { .. } | JournalEntry::CreateDirectoryV1 { .. } | JournalEntry::PathSetTimesV1 { .. } - | JournalEntry::FileDescriptorSetFlagsV1 { .. } - | JournalEntry::FileDescriptorAdviseV1 { .. } - | JournalEntry::FileDescriptorAllocateV1 { .. } - | JournalEntry::FileDescriptorSetRightsV1 { .. } - | JournalEntry::FileDescriptorSetTimesV1 { .. } - | JournalEntry::FileDescriptorSetSizeV1 { .. } | JournalEntry::CreateHardLinkV1 { .. } | JournalEntry::CreateSymbolicLinkV1 { .. } | JournalEntry::ChangeDirectoryV1 { .. } diff --git a/lib/wasix/src/journal/concrete/mod.rs b/lib/wasix/src/journal/concrete/mod.rs index 1e142ab02eb..644a7eb5df3 100644 --- a/lib/wasix/src/journal/concrete/mod.rs +++ b/lib/wasix/src/journal/concrete/mod.rs @@ -1,3 +1,4 @@ +mod arc; mod archived; mod boxed; mod buffered; @@ -15,6 +16,7 @@ mod unsupported; pub(super) use super::*; +pub use arc::*; pub use archived::*; pub use boxed::*; pub use buffered::*; diff --git a/lib/wasix/src/journal/concrete/null.rs b/lib/wasix/src/journal/concrete/null.rs index 72b54f005c4..779e9d8ce28 100644 --- a/lib/wasix/src/journal/concrete/null.rs +++ b/lib/wasix/src/journal/concrete/null.rs @@ -1,10 +1,12 @@ use super::*; -pub static NULL_JOURNAL: NullJournal = NullJournal {}; +pub static NULL_JOURNAL: NullJournal = NullJournal { debug_print: false }; /// The null journal sends all the records into the abyss #[derive(Debug, Default)] -pub struct NullJournal {} +pub struct NullJournal { + debug_print: bool, +} impl ReadableJournal for NullJournal { fn read(&self) -> anyhow::Result>> { @@ -18,7 +20,9 @@ impl ReadableJournal for NullJournal { impl WritableJournal for NullJournal { fn write<'a>(&'a self, entry: JournalEntry<'a>) -> anyhow::Result { - tracing::debug!("journal event: {:?}", entry); + if self.debug_print { + tracing::debug!("journal event: {:?}", entry); + } Ok(entry.estimate_size() as u64) } } diff --git a/lib/wasix/src/journal/concrete/printing.rs b/lib/wasix/src/journal/concrete/printing.rs index 93c363eafb5..df017592806 100644 --- a/lib/wasix/src/journal/concrete/printing.rs +++ b/lib/wasix/src/journal/concrete/printing.rs @@ -109,8 +109,14 @@ impl<'a> fmt::Display for JournalEntry<'a> { write!(f, "set-clock-time (id={:?}, time={})", clock_id, time) } JournalEntry::CloseFileDescriptorV1 { fd } => write!(f, "fd-close (fd={})", fd), - JournalEntry::OpenFileDescriptorV1 { fd, path, .. } => { - write!(f, "fd-open (path={}, fd={})", fd, path) + JournalEntry::OpenFileDescriptorV1 { + fd, path, o_flags, .. + } => { + if o_flags.contains(Oflags::TRUNC) { + write!(f, "fd-open-new (path={}, fd={})", fd, path) + } else { + write!(f, "fd-open (path={}, fd={})", fd, path) + } } JournalEntry::RenumberFileDescriptorV1 { old_fd, new_fd } => { write!(f, "fd-renumber (old={}, new={})", old_fd, new_fd) diff --git a/lib/wasix/src/journal/entry.rs b/lib/wasix/src/journal/entry.rs index d29a80fbcde..37bb494163f 100644 --- a/lib/wasix/src/journal/entry.rs +++ b/lib/wasix/src/journal/entry.rs @@ -1,4 +1,5 @@ use super::base64; +use derivative::Derivative; use serde::{Deserialize, Serialize}; use std::net::{Shutdown, SocketAddr}; use std::time::SystemTime; @@ -104,7 +105,8 @@ impl From for TimeType { /// Represents a log entry in a snapshot log stream that represents the total /// state of a WASM process at a point in time. #[allow(clippy::large_enum_variant)] -#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] +#[derive(Derivative, Clone, PartialEq, serde::Serialize, serde::Deserialize)] +#[derivative(Debug)] #[serde(rename_all = "snake_case")] pub enum JournalEntry<'a> { InitModuleV1 { @@ -112,6 +114,7 @@ pub enum JournalEntry<'a> { }, UpdateMemoryRegionV1 { region: Range, + #[derivative(Debug = "ignore")] #[serde(with = "base64")] data: Cow<'a, [u8]>, }, @@ -120,10 +123,13 @@ pub enum JournalEntry<'a> { }, SetThreadV1 { id: WasiThreadId, + #[derivative(Debug = "ignore")] #[serde(with = "base64")] call_stack: Cow<'a, [u8]>, + #[derivative(Debug = "ignore")] #[serde(with = "base64")] memory_stack: Cow<'a, [u8]>, + #[derivative(Debug = "ignore")] #[serde(with = "base64")] store_data: Cow<'a, [u8]>, is_64bit: bool, @@ -140,6 +146,7 @@ pub enum JournalEntry<'a> { FileDescriptorWriteV1 { fd: Fd, offset: u64, + #[derivative(Debug = "ignore")] #[serde(with = "base64")] data: Cow<'a, [u8]>, is_64bit: bool, @@ -157,7 +164,9 @@ pub enum JournalEntry<'a> { dirflags: LookupFlags, path: Cow<'a, str>, o_flags: Oflags, + #[derivative(Debug = "ignore")] fs_rights_base: Rights, + #[derivative(Debug = "ignore")] fs_rights_inheriting: Rights, fs_flags: Fdflags, }, @@ -342,6 +351,7 @@ pub enum JournalEntry<'a> { }, SocketSendToV1 { fd: Fd, + #[derivative(Debug = "ignore")] #[serde(with = "base64")] data: Cow<'a, [u8]>, flags: SiFlags, @@ -350,6 +360,7 @@ pub enum JournalEntry<'a> { }, SocketSendV1 { fd: Fd, + #[derivative(Debug = "ignore")] #[serde(with = "base64")] data: Cow<'a, [u8]>, flags: SiFlags, diff --git a/lib/wasix/src/journal/factories/mod.rs b/lib/wasix/src/journal/factories/mod.rs deleted file mode 100644 index df8fb436105..00000000000 --- a/lib/wasix/src/journal/factories/mod.rs +++ /dev/null @@ -1,3 +0,0 @@ -mod passthru_factory; - -pub use passthru_factory::*; diff --git a/lib/wasix/src/journal/factories/passthru_factory.rs b/lib/wasix/src/journal/factories/passthru_factory.rs deleted file mode 100644 index e0cb5412ece..00000000000 --- a/lib/wasix/src/journal/factories/passthru_factory.rs +++ /dev/null @@ -1,23 +0,0 @@ -use std::sync::Arc; - -use crate::journal::{DynJournal, JournalFactory}; - -#[derive(Clone)] -pub struct PassthruJournalFactory { - journals: Vec>, -} - -impl PassthruJournalFactory { - pub fn new(journals: Vec>) -> Self { - Self { journals } - } -} - -impl JournalFactory for PassthruJournalFactory { - fn load_or_create( - &self, - _desc: crate::journal::JournalDescriptor, - ) -> anyhow::Result>> { - Ok(self.journals.clone()) - } -} diff --git a/lib/wasix/src/journal/factory.rs b/lib/wasix/src/journal/factory.rs deleted file mode 100644 index 44cc0d5221c..00000000000 --- a/lib/wasix/src/journal/factory.rs +++ /dev/null @@ -1,34 +0,0 @@ -use std::sync::Arc; - -use semver::Version; - -use crate::runtime::module_cache::ModuleHash; - -use super::DynJournal; - -/// Describes the journal to be loaded or created -#[derive(Debug, Clone)] -pub struct JournalDescriptor { - /// Name of the package that holds the module (i.e. `wasmer/wapm2pirita`) - pub package_name: String, - /// Version of the package that contains the module (i.e. `1.0``) - pub version: Version, - /// Hash of the module that will use this journal - pub module_hash: ModuleHash, -} - -/// The journal factory creates or loads journals based of a set of -/// properties that describe the journal. -pub trait JournalFactory { - /// Creates or loads a journal based on a descriptor that makes it unique - /// - /// It is the responsibility of the implementor of this function - /// to reuse and reload existing journals based on the descriptor data - /// that is supplied. - /// - /// The factory can return more than one journal where only the last - /// journal is the active one while the former journals are base journals - fn load_or_create(&self, desc: JournalDescriptor) -> anyhow::Result>>; -} - -pub type DynJournalFactory = dyn JournalFactory + Send + Sync; diff --git a/lib/wasix/src/journal/mod.rs b/lib/wasix/src/journal/mod.rs index 2802cc1b29e..14b5cb5984b 100644 --- a/lib/wasix/src/journal/mod.rs +++ b/lib/wasix/src/journal/mod.rs @@ -6,16 +6,12 @@ mod effector; #[path = "effector/unimplemented.rs"] mod effector; mod entry; -mod factories; -mod factory; mod snapshot; mod util; pub use concrete::*; pub use effector::*; pub use entry::*; -pub use factories::*; -pub use factory::*; pub use snapshot::*; pub use util::*; diff --git a/lib/wasix/src/runners/dcgi/runner.rs b/lib/wasix/src/runners/dcgi/runner.rs index c19da0f036c..2681608913b 100644 --- a/lib/wasix/src/runners/dcgi/runner.rs +++ b/lib/wasix/src/runners/dcgi/runner.rs @@ -7,12 +7,13 @@ use webc::metadata::Command; use crate::{ bin_factory::BinaryPackage, capabilities::Capabilities, - journal::PassthruJournalFactory, + journal::{DynJournal, FilteredJournal}, runners::{ dcgi::handler::Handler, wcgi::{self, NoOpWcgiCallbacks, WcgiRunner}, MappedDirectory, }, + runtime::{DynRuntime, OverriddenRuntime}, Runtime, }; @@ -46,16 +47,9 @@ impl DcgiRunner { pkg: &BinaryPackage, runtime: Arc, ) -> Result { - let inner: wcgi::Handler = self.inner.prepare_handler( - command_name, - pkg, - true, - CgiDialect::Rfc3875, - runtime, - Arc::new(PassthruJournalFactory::new( - self.config.inner.wasi.journals.clone(), - )), - )?; + let inner: wcgi::Handler = + self.inner + .prepare_handler(command_name, pkg, true, CgiDialect::Rfc3875, runtime)?; Ok(Handler::new(inner)) } } @@ -72,8 +66,31 @@ impl crate::runners::Runner for DcgiRunner { &mut self, command_name: &str, pkg: &BinaryPackage, - runtime: Arc, + runtime: Arc, ) -> Result<(), Error> { + // We use a filter in front of the journals supplied to the runtime. + // The reason for this is that DCGI currently only supports persisting the + // file system changes as it is unable to run the main function more than + // once due to limitations in the runtime + let journals = runtime + .journals() + .clone() + .into_iter() + .map(|journal| { + let journal = FilteredJournal::new(journal) + .with_ignore_memory(true) + .with_ignore_threads(true) + .with_ignore_core(true) + .with_ignore_snapshots(true) + .with_ignore_networking(true) + .with_ignore_stdio(true); + Arc::new(journal) as Arc + }) + .collect::>(); + let runtime = OverriddenRuntime::new(runtime).with_journals(journals); + let runtime = Arc::new(runtime) as Arc; + + //We now pass the runtime to the the handlers let handler = self.prepare_handler(command_name, pkg, Arc::clone(&runtime))?; self.inner.run_command_with_handler(handler, runtime) } diff --git a/lib/wasix/src/runners/wcgi/runner.rs b/lib/wasix/src/runners/wcgi/runner.rs index e7040ae25af..882e8ade5bf 100644 --- a/lib/wasix/src/runners/wcgi/runner.rs +++ b/lib/wasix/src/runners/wcgi/runner.rs @@ -15,7 +15,6 @@ use webc::metadata::{ use crate::{ bin_factory::BinaryPackage, capabilities::Capabilities, - journal::{DynJournalFactory, JournalDescriptor, PassthruJournalFactory}, runners::{ wasi_common::CommonWasiOptions, wcgi::handler::{Handler, SharedState}, @@ -54,7 +53,6 @@ impl WcgiRunner { propagate_stderr: bool, default_dialect: CgiDialect, runtime: Arc, - journal_factory: Arc, ) -> Result { let cmd = pkg .get_command(command_name) @@ -74,23 +72,11 @@ impl WcgiRunner { let container_fs = Arc::clone(&pkg.webc_fs); - let journal_desc = JournalDescriptor { - package_name: pkg.package_name.clone(), - version: pkg.version.clone(), - module_hash: pkg.hash(), - }; - let wasi_common = self.config.wasi.clone(); let rt = Arc::clone(&runtime); let setup_builder = move |builder: &mut WasiEnvBuilder| { wasi_common.prepare_webc_env(builder, Some(Arc::clone(&container_fs)), &wasi, None)?; builder.set_runtime(Arc::clone(&rt)); - - let journals = journal_factory.load_or_create(journal_desc.clone())?; - for journal in journals { - builder.add_journal(journal); - } - Ok(()) }; @@ -189,9 +175,6 @@ impl crate::runners::Runner for WcgiRunner { false, CgiDialect::Wcgi, Arc::clone(&runtime), - Arc::new(PassthruJournalFactory::new( - self.config.wasi.journals.clone(), - )), )?; self.run_command_with_handler(handler, runtime) } diff --git a/lib/wasix/src/runtime/mod.rs b/lib/wasix/src/runtime/mod.rs index a6c100df7d3..51ce73d441b 100644 --- a/lib/wasix/src/runtime/mod.rs +++ b/lib/wasix/src/runtime/mod.rs @@ -11,6 +11,7 @@ use self::{ use std::{ fmt, + ops::Deref, sync::{Arc, Mutex}, }; @@ -123,6 +124,8 @@ where } } +pub type DynRuntime = dyn Runtime + Send + Sync; + #[cfg(feature = "journal")] static EMPTY_JOURNAL_LIST: Vec> = Vec::new(); @@ -349,3 +352,201 @@ impl Runtime for PluggableRuntime { self.journals.iter().last().map(|a| a.as_ref()) } } + +/// Runtime that allows for certain things to be overridden +/// such as the active journals +#[derive(Clone, Derivative)] +#[derivative(Debug)] +pub struct OverriddenRuntime { + inner: Arc, + task_manager: Option>, + networking: Option, + http_client: Option, + package_loader: Option>, + source: Option>, + engine: Option, + module_cache: Option>, + #[derivative(Debug = "ignore")] + tty: Option>, + #[cfg(feature = "journal")] + #[derivative(Debug = "ignore")] + journals: Option>>, +} + +impl OverriddenRuntime { + pub fn new(inner: Arc) -> Self { + Self { + inner, + task_manager: None, + networking: None, + http_client: None, + package_loader: None, + source: None, + engine: None, + module_cache: None, + tty: None, + journals: None, + } + } + + pub fn with_task_manager(mut self, task_manager: Arc) -> Self { + self.task_manager.replace(task_manager); + self + } + + pub fn with_networking(mut self, networking: DynVirtualNetworking) -> Self { + self.networking.replace(networking); + self + } + + pub fn with_http_client(mut self, http_client: DynHttpClient) -> Self { + self.http_client.replace(http_client); + self + } + + pub fn with_package_loader( + mut self, + package_loader: Arc, + ) -> Self { + self.package_loader.replace(package_loader); + self + } + + pub fn with_source(mut self, source: Arc) -> Self { + self.source.replace(source); + self + } + + pub fn with_engine(mut self, engine: wasmer::Engine) -> Self { + self.engine.replace(engine); + self + } + + pub fn with_module_cache(mut self, module_cache: Arc) -> Self { + self.module_cache.replace(module_cache); + self + } + + #[cfg(feature = "journal")] + pub fn with_tty(mut self, tty: Arc) -> Self { + self.tty.replace(tty); + self + } + + #[cfg(feature = "journal")] + pub fn with_journals(mut self, journals: Vec>) -> Self { + self.journals.replace(journals); + self + } +} + +impl Runtime for OverriddenRuntime { + fn networking(&self) -> &DynVirtualNetworking { + if let Some(net) = self.networking.as_ref() { + net + } else { + self.inner.networking() + } + } + + fn task_manager(&self) -> &Arc { + if let Some(rt) = self.task_manager.as_ref() { + rt + } else { + self.inner.task_manager() + } + } + + fn source(&self) -> Arc { + if let Some(source) = self.source.clone() { + source + } else { + self.inner.source() + } + } + + fn package_loader(&self) -> Arc { + if let Some(loader) = self.package_loader.clone() { + loader + } else { + self.inner.package_loader() + } + } + + fn module_cache(&self) -> Arc { + if let Some(cache) = self.module_cache.clone() { + cache + } else { + self.inner.module_cache() + } + } + + fn engine(&self) -> wasmer::Engine { + if let Some(engine) = self.engine.clone() { + engine + } else { + self.inner.engine() + } + } + + fn new_store(&self) -> wasmer::Store { + if let Some(engine) = self.engine.clone() { + wasmer::Store::new(engine) + } else { + self.inner.new_store() + } + } + + fn http_client(&self) -> Option<&DynHttpClient> { + if let Some(client) = self.http_client.as_ref() { + Some(client) + } else { + self.inner.http_client() + } + } + + fn tty(&self) -> Option<&(dyn TtyBridge + Send + Sync)> { + if let Some(tty) = self.tty.as_ref() { + Some(tty.deref()) + } else { + self.inner.tty() + } + } + + #[cfg(feature = "journal")] + fn journals(&self) -> &'_ Vec> { + if let Some(journals) = self.journals.as_ref() { + journals + } else { + self.inner.journals() + } + } + + #[cfg(feature = "journal")] + fn active_journal(&self) -> Option<&'_ DynJournal> { + if let Some(journals) = self.journals.as_ref() { + journals.iter().last().map(|a| a.as_ref()) + } else { + self.inner.active_journal() + } + } + + fn load_module<'a>(&'a self, wasm: &'a [u8]) -> BoxFuture<'a, anyhow::Result> { + if self.engine.is_some() || self.module_cache.is_some() { + let engine = self.engine(); + let module_cache = self.module_cache(); + let task = async move { load_module(&engine, &module_cache, wasm).await }; + Box::pin(task) + } else { + self.inner.load_module(wasm) + } + } + + fn load_module_sync(&self, wasm: &[u8]) -> Result { + if self.engine.is_some() || self.module_cache.is_some() { + InlineWaker::block_on(self.load_module(wasm)) + } else { + self.inner.load_module_sync(wasm) + } + } +} diff --git a/lib/wasix/tests/runners.rs b/lib/wasix/tests/runners.rs index 10924a5c3fc..e38afd9a294 100644 --- a/lib/wasix/tests/runners.rs +++ b/lib/wasix/tests/runners.rs @@ -178,7 +178,7 @@ mod wcgi { handle: Handle, } - impl wasmer_wasix::runners::wcgi::Callbacks<()> for Callbacks { + impl wasmer_wasix::runners::wcgi::Callbacks for Callbacks { fn started(&self, abort: futures::stream::AbortHandle) { let mut sender = self.sender.clone(); self.handle.spawn(async move { From ca401d875ff42ebbc6b3e083091db4a827a6cb90 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Sat, 23 Dec 2023 18:16:30 +1100 Subject: [PATCH 115/129] Compacting a journal will now remove more redundant file system events in the journal --- lib/wasix/src/journal/concrete/compacting.rs | 95 ++++++++++++++++++-- lib/wasix/src/journal/concrete/printing.rs | 14 ++- 2 files changed, 97 insertions(+), 12 deletions(-) diff --git a/lib/wasix/src/journal/concrete/compacting.rs b/lib/wasix/src/journal/concrete/compacting.rs index 75d5e9f054b..43729a4c134 100644 --- a/lib/wasix/src/journal/concrete/compacting.rs +++ b/lib/wasix/src/journal/concrete/compacting.rs @@ -42,6 +42,13 @@ struct State { snapshots: Vec, // Last tty event thats been set tty: Option, + // Events that create a particular directory + create_directory: HashMap, + // Events that remove a particular directory + remove_directory: HashMap, + // When creating and truncating a file we have a special + // lookup so that duplicates can be erased + create_trunc_file: HashMap, // Thread events are only maintained while the thread and the // process are still running thread_map: HashMap, @@ -78,6 +85,8 @@ impl State { where J: Journal, { + let has_threads = !self.thread_map.is_empty(); + let mut filter = FilteredJournal::new(inner) .with_filter_events(self.whitelist.clone().into_iter().collect()); if let Some(tty) = self.tty.as_ref() { @@ -92,11 +101,16 @@ impl State { for t in self.thread_map.iter() { filter.add_event_to_whitelist(*t.1); } + for (_, e) in self.create_directory.iter() { + filter.add_event_to_whitelist(*e); + } + for (_, e) in self.remove_directory.iter() { + filter.add_event_to_whitelist(*e); + } for (_, l) in self .suspect_descriptors .iter() .chain(self.keep_descriptors.iter()) - .chain(self.stdio_descriptors.iter()) { if let Some(d) = self.descriptors.get(l) { for e in d.events.iter() { @@ -107,6 +121,18 @@ impl State { } } } + if has_threads { + for (_, l) in self.stdio_descriptors.iter() { + if let Some(d) = self.descriptors.get(l) { + for e in d.events.iter() { + filter.add_event_to_whitelist(*e); + } + for e in d.write_map.values() { + filter.add_event_to_whitelist(*e); + } + } + } + } filter } } @@ -146,6 +172,9 @@ impl CompactingJournal { snapshots: Default::default(), memory_map: Default::default(), thread_map: Default::default(), + create_directory: Default::default(), + remove_directory: Default::default(), + create_trunc_file: Default::default(), suspect_descriptors: Default::default(), keep_descriptors: Default::default(), stdio_descriptors: Default::default(), @@ -274,24 +303,56 @@ impl WritableJournal for CompactingJournalTx { JournalEntry::TtySetV1 { .. } => { state.tty.replace(event_index); } - JournalEntry::OpenFileDescriptorV1 { fd, .. } => { + JournalEntry::OpenFileDescriptorV1 { + fd, o_flags, path, .. + } => { // All file descriptors are opened in a suspect state which // means if they are closed without modifying the file system // then the events will be ignored. - let lookup = state.descriptor_seed; + let lookup = DescriptorLookup(state.descriptor_seed); state.descriptor_seed += 1; + state.suspect_descriptors.insert(*fd, lookup); + + // There is an exception to the rule which is if the create + // flag is specified its always recorded as a mutating operation + // because it may create a file that does not exist on the file system + if o_flags.contains(Oflags::CREATE) { + if let Some(lookup) = state.suspect_descriptors.remove(fd) { + state.keep_descriptors.insert(*fd, lookup); + } + } + + // The event itself must be recorded in a staging area state - .suspect_descriptors - .insert(*fd, DescriptorLookup(lookup)); + .descriptors + .entry(lookup) + .or_insert_with(Default::default) + .events + .push(event_index); + + // Creating a file and erasing anything that was there before means + // the entire create branch that exists before this one can be ignored + if o_flags.contains(Oflags::CREATE) && o_flags.contains(Oflags::TRUNC) { + let path = path.to_string(); + if let Some(existing) = state.create_trunc_file.remove(&path) { + state.suspect_descriptors.remove(&existing); + state.keep_descriptors.remove(&existing); + } + state.create_trunc_file.insert(path, *fd); + } } // We keep non-mutable events for file descriptors that are suspect JournalEntry::FileDescriptorSeekV1 { fd, .. } | JournalEntry::CloseFileDescriptorV1 { fd } => { // Get the lookup - let lookup = state - .suspect_descriptors - .get(fd) - .cloned() + // (if its suspect then it will remove the entry and + // thus the entire branch of events it represents is discarded) + let lookup = if matches!(&entry, JournalEntry::CloseFileDescriptorV1 { .. }) { + state.suspect_descriptors.remove(fd) + } else { + state.suspect_descriptors.get(fd).cloned() + }; + let lookup = lookup .or_else(|| state.keep_descriptors.get(fd).cloned()) .or_else(|| state.stdio_descriptors.get(fd).cloned()); @@ -373,6 +434,22 @@ impl WritableJournal for CompactingJournalTx { state.whitelist.insert(event_index); } } + // Creating a new directory only needs to be done once + JournalEntry::CreateDirectoryV1 { path, .. } => { + let path = path.to_string(); + state.remove_directory.remove(&path); + if state.create_directory.contains_key(&path) == false { + state.create_directory.insert(path, event_index); + } + } + // Deleting a directory only needs to be done once + JournalEntry::RemoveDirectoryV1 { path, .. } => { + let path = path.to_string(); + state.create_directory.remove(&path); + if state.remove_directory.contains_key(&path) == false { + state.remove_directory.insert(path, event_index); + } + } _ => { // The fallthrough is to whitelist the event so that it will // be reflected in the next compaction event diff --git a/lib/wasix/src/journal/concrete/printing.rs b/lib/wasix/src/journal/concrete/printing.rs index df017592806..3d74b0723d4 100644 --- a/lib/wasix/src/journal/concrete/printing.rs +++ b/lib/wasix/src/journal/concrete/printing.rs @@ -112,10 +112,18 @@ impl<'a> fmt::Display for JournalEntry<'a> { JournalEntry::OpenFileDescriptorV1 { fd, path, o_flags, .. } => { - if o_flags.contains(Oflags::TRUNC) { - write!(f, "fd-open-new (path={}, fd={})", fd, path) + if o_flags.contains(Oflags::CREATE) { + if o_flags.contains(Oflags::TRUNC) { + write!(f, "fd-create-new (path={}, fd={})", fd, path) + } else { + write!(f, "fd-create (path={}, fd={})", fd, path) + } } else { - write!(f, "fd-open (path={}, fd={})", fd, path) + if o_flags.contains(Oflags::TRUNC) { + write!(f, "fd-open-new (path={}, fd={})", fd, path) + } else { + write!(f, "fd-open (path={}, fd={})", fd, path) + } } } JournalEntry::RenumberFileDescriptorV1 { old_fd, new_fd } => { From 515aa1664be24c9c35fef9fc7bd029599719bc26 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Sat, 23 Dec 2023 20:11:24 +1100 Subject: [PATCH 116/129] Fixed an issue where the compacting logs were not rotating and compacting properly --- lib/cli/src/commands/run/mod.rs | 14 ++-- lib/cli/src/commands/run/wasi.rs | 55 ++++++++++----- lib/wasix/src/journal/concrete/compacting.rs | 53 ++++++++++++--- .../journal/concrete/compacting_log_file.rs | 67 +++++++++++++++++-- lib/wasix/src/journal/concrete/counting.rs | 48 +++++++++++++ lib/wasix/src/journal/concrete/mod.rs | 2 + 6 files changed, 196 insertions(+), 43 deletions(-) create mode 100644 lib/wasix/src/journal/concrete/counting.rs diff --git a/lib/cli/src/commands/run/mod.rs b/lib/cli/src/commands/run/mod.rs index cde97f181bd..1f430493635 100644 --- a/lib/cli/src/commands/run/mod.rs +++ b/lib/cli/src/commands/run/mod.rs @@ -287,14 +287,8 @@ impl Run { } config.with_snapshot_interval(Duration::from_millis(period)); } - for journal in self.wasi.journals.clone() { - if self.wasi.enable_compaction { - config.add_journal(Arc::new( - CompactingLogFileJournal::new(journal)?.with_compact_on_drop(), - )); - } else { - config.add_journal(Arc::new(LogFileJournal::new(journal)?)); - } + for journal in self.wasi.build_journals()? { + config.add_journal(journal); } } @@ -389,8 +383,8 @@ impl Run { } runner.with_snapshot_interval(Duration::from_millis(period)); } - for journal in self.wasi.journals.clone() { - runner.add_journal(Arc::new(LogFileJournal::new(journal)?)); + for journal in self.wasi.build_journals()? { + runner.add_journal(journal); } } diff --git a/lib/cli/src/commands/run/wasi.rs b/lib/cli/src/commands/run/wasi.rs index 0db1a3fa928..905419a3327 100644 --- a/lib/cli/src/commands/run/wasi.rs +++ b/lib/cli/src/commands/run/wasi.rs @@ -21,7 +21,7 @@ use wasmer_wasix::{ capabilities::Capabilities, default_fs_backing, get_wasi_versions, http::HttpClient, - journal::CompactingLogFileJournal, + journal::{CompactingLogFileJournal, DynJournal}, os::{tty_sys::SysTty, TtyBridge}, rewind_ext, runners::{MappedCommand, MappedDirectory}, @@ -128,6 +128,20 @@ pub struct Wasi { #[clap(long = "enable-compaction")] pub enable_compaction: bool, + /// Tells the compactor not to compact when the journal log file is closed + #[cfg(feature = "journal")] + #[clap(long = "without-compact-on-drop")] + pub without_compact_on_drop: bool, + + /// Tells the compactor to compact when it grows by a certain factor of + /// its original size. (i.e. '0.2' would be it compacts after the journal + /// has grown by 20 percent) + /// + /// Default is to compact on growth that exceeds 15% + #[cfg(feature = "journal")] + #[clap(long = "with-compact-on-growth", default_value = "0.15")] + pub with_compact_on_growth: f32, + /// Indicates what events will cause a snapshot to be taken /// and written to the journal file. /// @@ -357,14 +371,8 @@ impl Wasi { if let Some(interval) = self.snapshot_interval { builder.with_snapshot_interval(std::time::Duration::from_millis(interval)); } - for journal in self.journals.iter() { - if self.enable_compaction { - builder.add_journal(Arc::new( - CompactingLogFileJournal::new(journal)?.with_compact_on_drop(), - )); - } else { - builder.add_journal(Arc::new(LogFileJournal::new(journal)?)); - } + for journal in self.build_journals()? { + builder.add_journal(journal); } } @@ -379,6 +387,25 @@ impl Wasi { Ok(builder) } + pub fn build_journals(&self) -> anyhow::Result>> { + let mut ret = Vec::new(); + for journal in self.journals.clone() { + if self.enable_compaction { + let mut journal = CompactingLogFileJournal::new(journal)?; + if !self.without_compact_on_drop { + journal = journal.with_compact_on_drop() + } + if self.with_compact_on_growth.is_normal() && self.with_compact_on_growth != 0f32 { + journal = journal.with_compact_on_factor_size(self.with_compact_on_growth); + } + ret.push(Arc::new(journal) as Arc); + } else { + ret.push(Arc::new(LogFileJournal::new(journal)?)); + } + } + Ok(ret) + } + pub fn build_mapped_directories(&self) -> Result, anyhow::Error> { let mut mapped_dirs = Vec::new(); @@ -522,14 +549,8 @@ impl Wasi { } #[cfg(feature = "journal")] - for journal in self.journals.clone() { - if self.enable_compaction { - rt.add_journal(Arc::new( - CompactingLogFileJournal::new(journal)?.with_compact_on_drop(), - )); - } else { - rt.add_journal(Arc::new(LogFileJournal::new(journal)?)); - } + for journal in self.build_journals()? { + rt.add_journal(journal); } if !self.no_tty { diff --git a/lib/wasix/src/journal/concrete/compacting.rs b/lib/wasix/src/journal/concrete/compacting.rs index 43729a4c134..cf395f260dd 100644 --- a/lib/wasix/src/journal/concrete/compacting.rs +++ b/lib/wasix/src/journal/concrete/compacting.rs @@ -1,7 +1,7 @@ use derivative::Derivative; use std::{ collections::{HashMap, HashSet}, - ops::Range, + ops::{DerefMut, Range}, sync::{Arc, Mutex}, }; use virtual_fs::Fd; @@ -153,6 +153,13 @@ pub struct CompactingJournalRx { inner: Box, } +impl CompactingJournalRx { + pub fn swap_inner(&mut self, mut with: Box) -> Box { + std::mem::swap(&mut self.inner, &mut with); + with + } +} + #[derive(Debug)] pub struct CompactingJournal { tx: CompactingJournalTx, @@ -194,6 +201,13 @@ impl CompactingJournal { } } +/// Represents the results of a compaction operation +#[derive(Debug, Default)] +pub struct CompactResult { + pub total_size: u64, + pub total_events: usize, +} + impl CompactingJournalTx { pub fn create_filter(&self, inner: J) -> FilteredJournal where @@ -203,8 +217,17 @@ impl CompactingJournalTx { state.create_filter(inner) } + pub fn swap(&self, other: Self) -> Self { + let mut state1 = self.state.lock().unwrap(); + let mut state2 = other.state.lock().unwrap(); + std::mem::swap(state1.deref_mut(), state2.deref_mut()); + drop(state1); + drop(state2); + other + } + /// Compacts the inner journal into a new journal - pub fn compact_to(&self, new_journal: J) -> anyhow::Result<()> + pub fn compact_to(&self, new_journal: J) -> anyhow::Result where J: Journal, { @@ -222,17 +245,25 @@ impl CompactingJournalTx { ) }; - // Read all the events and feed them into the filtered journal + let mut result = CompactResult::default(); + + // Read all the events and feed them into the filtered journal and then + // strip off the filter so that its a normal journal again while let Some(entry) = replay_rx.read()? { - new_journal.write(entry)?; + let amt = new_journal.write(entry)?; + if amt > 0 { + result.total_size += amt; + result.total_events += 1; + } } + let new_journal = new_journal.into_inner(); // We now go into a blocking situation which will freeze the journals let mut state = self.state.lock().unwrap(); // Now we build a filtered journal which will pick up any events that were - // added which we did the compacting - let new_journal = FilteredJournal::new(new_journal.into_inner()).with_filter_events( + // added which we did the compacting. + let new_journal = FilteredJournal::new(new_journal).with_filter_events( state .delta_list .take() @@ -241,18 +272,20 @@ impl CompactingJournalTx { .collect(), ); - // Now we feed all the events into the new journal using the delta filter + // Now we feed all the events into the new journal using the delta filter. After the + // extra events are added we strip off the filter again let replay_rx = state.inner_rx.as_restarted()?; while let Some(entry) = replay_rx.read()? { new_journal.write(entry)?; } + let new_journal = new_journal.into_inner(); // Now we install the new journal - let (mut tx, mut rx) = new_journal.into_inner().split(); + let (mut tx, mut rx) = new_journal.split(); std::mem::swap(&mut state.inner_tx, &mut tx); std::mem::swap(&mut state.inner_rx, &mut rx); - Ok(()) + Ok(result) } pub fn replace_inner(&self, inner: J) { @@ -462,7 +495,7 @@ impl WritableJournal for CompactingJournalTx { impl CompactingJournal { /// Compacts the inner journal into a new journal - pub fn compact_to(&mut self, new_journal: J) -> anyhow::Result<()> + pub fn compact_to(&mut self, new_journal: J) -> anyhow::Result where J: Journal, { diff --git a/lib/wasix/src/journal/concrete/compacting_log_file.rs b/lib/wasix/src/journal/concrete/compacting_log_file.rs index 4085d947b66..5a1f5fa0317 100644 --- a/lib/wasix/src/journal/concrete/compacting_log_file.rs +++ b/lib/wasix/src/journal/concrete/compacting_log_file.rs @@ -9,9 +9,11 @@ use super::*; struct State { on_n_records: Option, on_n_size: Option, + on_factor_size: Option, on_drop: bool, cnt_records: u64, cnt_size: u64, + ref_size: u64, } #[derive(Debug)] @@ -35,11 +37,18 @@ pub struct CompactingLogFileJournalRx { inner: CompactingJournalRx, } +impl CompactingLogFileJournalRx { + pub fn swap_inner(&mut self, with: Box) -> Box { + self.inner.swap_inner(with) + } +} + impl CompactingLogFileJournal { pub fn new(path: impl AsRef) -> anyhow::Result { // We prepare a compacting journal which does nothing // with the events other than learn from them - let mut compacting = CompactingJournal::new(NullJournal::default())?; + let counting = CountingJournal::default(); + let mut compacting = CompactingJournal::new(counting.clone())?; // We first feed all the entries into the compactor so that // it learns all the records @@ -69,8 +78,10 @@ impl CompactingLogFileJournal { on_drop: false, on_n_records: None, on_n_size: None, + on_factor_size: None, cnt_records: 0, cnt_size: 0, + ref_size: counting.size(), })); let tx = CompactingLogFileJournalTx { state: state.clone(), @@ -83,8 +94,10 @@ impl CompactingLogFileJournal { Ok(Self { tx, rx }) } - pub fn compact_now(&mut self) -> anyhow::Result<()> { - self.tx.compact_now() + pub fn compact_now(&mut self) -> anyhow::Result { + let (result, new_rx) = self.tx.compact_now()?; + self.rx.inner = new_rx; + Ok(result) } pub fn with_compact_on_drop(self) -> Self { @@ -106,10 +119,20 @@ impl CompactingLogFileJournal { self.tx.state.lock().unwrap().on_n_size.replace(n_size); self } + + pub fn with_compact_on_factor_size(self, factor_size: f32) -> Self { + self.tx + .state + .lock() + .unwrap() + .on_factor_size + .replace(factor_size); + self + } } impl CompactingLogFileJournalTx { - pub fn compact_now(&self) -> anyhow::Result<()> { + pub fn compact_now(&self) -> anyhow::Result<(CompactResult, CompactingJournalRx)> { // Reset the counters self.reset_counters(); @@ -118,9 +141,33 @@ impl CompactingLogFileJournalTx { let target = LogFileJournal::new(self.temp_path.clone())?; // Compact the data into the new target and rename it over the last one - self.inner.compact_to(target)?; + let result = self.inner.compact_to(target)?; std::fs::rename(&self.temp_path, &self.main_path)?; - Ok(()) + + // Renaming the file has quite a detrimental effect on the file as + // it means any new mmap operations will fail, hence we need to + // reopen the log file, seek to the end and reattach it + let target = LogFileJournal::new(self.main_path.clone())?; + + // We prepare a compacting journal which does nothing + // with the events other than learn from them + let counting = CountingJournal::default(); + let mut compacting = CompactingJournal::new(counting.clone())?; + copy_journal(&target, &compacting)?; + + // Now everything is learned its time to attach the log file to the compacting journal + // and replace the current one + compacting.replace_inner(target); + let (tx, rx) = compacting.into_split(); + self.inner.swap(tx); + + // We take a new reference point for the size of the journal + { + let mut state = self.state.lock().unwrap(); + state.ref_size = result.total_size; + } + + Ok((result, rx)) } pub fn reset_counters(&self) { @@ -173,6 +220,14 @@ impl WritableJournal for CompactingLogFileJournalTx { triggered = true; } } + + if let Some(factor) = state.on_factor_size.as_ref() { + let next_ref = (*factor * state.ref_size as f32) as u64; + if state.cnt_size > next_ref { + triggered = true; + } + } + triggered }; diff --git a/lib/wasix/src/journal/concrete/counting.rs b/lib/wasix/src/journal/concrete/counting.rs new file mode 100644 index 00000000000..ddeda96a5a7 --- /dev/null +++ b/lib/wasix/src/journal/concrete/counting.rs @@ -0,0 +1,48 @@ +use std::sync::{ + atomic::{AtomicU64, AtomicUsize, Ordering}, + Arc, +}; + +use super::*; + +/// Journal that counts the size of the entries that are written to it +#[derive(Debug, Clone, Default)] +pub struct CountingJournal { + n_cnt: Arc, + n_size: Arc, +} + +impl CountingJournal { + pub fn cnt(&self) -> usize { + self.n_cnt.load(Ordering::SeqCst) + } + + pub fn size(&self) -> u64 { + self.n_size.load(Ordering::SeqCst) + } +} + +impl ReadableJournal for CountingJournal { + fn read(&self) -> anyhow::Result>> { + Ok(None) + } + + fn as_restarted(&self) -> anyhow::Result> { + Ok(Box::::default()) + } +} + +impl WritableJournal for CountingJournal { + fn write<'a>(&'a self, entry: JournalEntry<'a>) -> anyhow::Result { + let size = entry.estimate_size() as u64; + self.n_cnt.fetch_add(1, Ordering::SeqCst); + self.n_size.fetch_add(size, Ordering::SeqCst); + Ok(size) + } +} + +impl Journal for CountingJournal { + fn split(self) -> (Box, Box) { + (Box::new(self.clone()), Box::new(self)) + } +} diff --git a/lib/wasix/src/journal/concrete/mod.rs b/lib/wasix/src/journal/concrete/mod.rs index 644a7eb5df3..f51d780000f 100644 --- a/lib/wasix/src/journal/concrete/mod.rs +++ b/lib/wasix/src/journal/concrete/mod.rs @@ -5,6 +5,7 @@ mod buffered; mod compacting; #[cfg(feature = "journal")] mod compacting_log_file; +mod counting; mod filter; #[cfg(feature = "journal")] mod log_file; @@ -23,6 +24,7 @@ pub use buffered::*; pub use compacting::*; #[cfg(feature = "journal")] pub use compacting_log_file::*; +pub use counting::*; pub use filter::*; #[cfg(feature = "journal")] pub use log_file::*; From 803529a630320e5602012c727cbce895d6ccb117 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Sat, 23 Dec 2023 20:21:44 +1100 Subject: [PATCH 117/129] Linting and compile fixes --- lib/wasix/src/journal/concrete/compacting.rs | 11 ++++++----- .../src/journal/concrete/compacting_log_file.rs | 2 +- lib/wasix/src/journal/concrete/printing.rs | 17 ++++++++--------- lib/wasix/src/runtime/mod.rs | 1 + 4 files changed, 16 insertions(+), 15 deletions(-) diff --git a/lib/wasix/src/journal/concrete/compacting.rs b/lib/wasix/src/journal/concrete/compacting.rs index cf395f260dd..079cae1bee3 100644 --- a/lib/wasix/src/journal/concrete/compacting.rs +++ b/lib/wasix/src/journal/concrete/compacting.rs @@ -5,6 +5,7 @@ use std::{ sync::{Arc, Mutex}, }; use virtual_fs::Fd; +use wasmer_wasix_types::wasi; use super::*; @@ -349,7 +350,7 @@ impl WritableJournal for CompactingJournalTx { // There is an exception to the rule which is if the create // flag is specified its always recorded as a mutating operation // because it may create a file that does not exist on the file system - if o_flags.contains(Oflags::CREATE) { + if o_flags.contains(wasi::Oflags::CREATE) { if let Some(lookup) = state.suspect_descriptors.remove(fd) { state.keep_descriptors.insert(*fd, lookup); } @@ -365,7 +366,7 @@ impl WritableJournal for CompactingJournalTx { // Creating a file and erasing anything that was there before means // the entire create branch that exists before this one can be ignored - if o_flags.contains(Oflags::CREATE) && o_flags.contains(Oflags::TRUNC) { + if o_flags.contains(wasi::Oflags::CREATE) && o_flags.contains(wasi::Oflags::TRUNC) { let path = path.to_string(); if let Some(existing) = state.create_trunc_file.remove(&path) { state.suspect_descriptors.remove(&existing); @@ -471,7 +472,7 @@ impl WritableJournal for CompactingJournalTx { JournalEntry::CreateDirectoryV1 { path, .. } => { let path = path.to_string(); state.remove_directory.remove(&path); - if state.create_directory.contains_key(&path) == false { + if !state.create_directory.contains_key(&path) { state.create_directory.insert(path, event_index); } } @@ -479,7 +480,7 @@ impl WritableJournal for CompactingJournalTx { JournalEntry::RemoveDirectoryV1 { path, .. } => { let path = path.to_string(); state.create_directory.remove(&path); - if state.remove_directory.contains_key(&path) == false { + if !state.remove_directory.contains_key(&path) { state.remove_directory.insert(path, event_index); } } @@ -554,7 +555,7 @@ impl Journal for CompactingJournal { mod tests { use super::*; - use wasmer_wasix_types::wasi::{self, Tty}; + use wasmer_wasix_types::wasi::Tty; pub fn run_test<'a>( in_records: Vec>, diff --git a/lib/wasix/src/journal/concrete/compacting_log_file.rs b/lib/wasix/src/journal/concrete/compacting_log_file.rs index 5a1f5fa0317..1face3b2ecd 100644 --- a/lib/wasix/src/journal/concrete/compacting_log_file.rs +++ b/lib/wasix/src/journal/concrete/compacting_log_file.rs @@ -152,7 +152,7 @@ impl CompactingLogFileJournalTx { // We prepare a compacting journal which does nothing // with the events other than learn from them let counting = CountingJournal::default(); - let mut compacting = CompactingJournal::new(counting.clone())?; + let mut compacting = CompactingJournal::new(counting)?; copy_journal(&target, &compacting)?; // Now everything is learned its time to attach the log file to the compacting journal diff --git a/lib/wasix/src/journal/concrete/printing.rs b/lib/wasix/src/journal/concrete/printing.rs index 3d74b0723d4..c12ea6e2fac 100644 --- a/lib/wasix/src/journal/concrete/printing.rs +++ b/lib/wasix/src/journal/concrete/printing.rs @@ -1,6 +1,7 @@ use std::fmt; use super::*; +use wasmer_wasix_types::wasi; /// Type of printing mode to use #[derive(Debug)] @@ -112,18 +113,16 @@ impl<'a> fmt::Display for JournalEntry<'a> { JournalEntry::OpenFileDescriptorV1 { fd, path, o_flags, .. } => { - if o_flags.contains(Oflags::CREATE) { - if o_flags.contains(Oflags::TRUNC) { - write!(f, "fd-create-new (path={}, fd={})", fd, path) + if o_flags.contains(wasi::Oflags::CREATE) { + if o_flags.contains(wasi::Oflags::TRUNC) { + write!(f, "fd-create-new (fd={}, path={})", fd, path) } else { - write!(f, "fd-create (path={}, fd={})", fd, path) + write!(f, "fd-create (fd={}, path={})", fd, path) } + } else if o_flags.contains(wasi::Oflags::TRUNC) { + write!(f, "fd-open-new (fd={}, path={})", fd, path) } else { - if o_flags.contains(Oflags::TRUNC) { - write!(f, "fd-open-new (path={}, fd={})", fd, path) - } else { - write!(f, "fd-open (path={}, fd={})", fd, path) - } + write!(f, "fd-open (fd={}, path={})", fd, path) } } JournalEntry::RenumberFileDescriptorV1 { old_fd, new_fd } => { diff --git a/lib/wasix/src/runtime/mod.rs b/lib/wasix/src/runtime/mod.rs index 51ce73d441b..56f40b9defe 100644 --- a/lib/wasix/src/runtime/mod.rs +++ b/lib/wasix/src/runtime/mod.rs @@ -385,6 +385,7 @@ impl OverriddenRuntime { engine: None, module_cache: None, tty: None, + #[cfg(feature = "journal")] journals: None, } } From b2b528eb5896538c0b90908af94e6b556b4af851 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Sat, 23 Dec 2023 20:41:28 +1100 Subject: [PATCH 118/129] Added more unit test fixes, linting and compile errors --- lib/cli/src/commands/run/wasi.rs | 6 + lib/wasix/src/journal/concrete/compacting.rs | 149 +++++++++++++++++-- 2 files changed, 139 insertions(+), 16 deletions(-) diff --git a/lib/cli/src/commands/run/wasi.rs b/lib/cli/src/commands/run/wasi.rs index 905419a3327..c57d0d595ff 100644 --- a/lib/cli/src/commands/run/wasi.rs +++ b/lib/cli/src/commands/run/wasi.rs @@ -387,6 +387,7 @@ impl Wasi { Ok(builder) } + #[cfg(feature = "journal")] pub fn build_journals(&self) -> anyhow::Result>> { let mut ret = Vec::new(); for journal in self.journals.clone() { @@ -406,6 +407,11 @@ impl Wasi { Ok(ret) } + #[cfg(not(feature = "journal"))] + pub fn build_journals(&self) -> anyhow::Result>> { + Ok(Vec::new()) + } + pub fn build_mapped_directories(&self) -> Result, anyhow::Error> { let mut mapped_dirs = Vec::new(); diff --git a/lib/wasix/src/journal/concrete/compacting.rs b/lib/wasix/src/journal/concrete/compacting.rs index 079cae1bee3..40bf148f1b9 100644 --- a/lib/wasix/src/journal/concrete/compacting.rs +++ b/lib/wasix/src/journal/concrete/compacting.rs @@ -472,17 +472,13 @@ impl WritableJournal for CompactingJournalTx { JournalEntry::CreateDirectoryV1 { path, .. } => { let path = path.to_string(); state.remove_directory.remove(&path); - if !state.create_directory.contains_key(&path) { - state.create_directory.insert(path, event_index); - } + state.create_directory.entry(path).or_insert(event_index); } // Deleting a directory only needs to be done once JournalEntry::RemoveDirectoryV1 { path, .. } => { let path = path.to_string(); state.create_directory.remove(&path); - if !state.remove_directory.contains_key(&path) { - state.remove_directory.insert(path, event_index); - } + state.remove_directory.entry(path).or_insert(event_index); } _ => { // The fallthrough is to whitelist the event so that it will @@ -758,7 +754,7 @@ mod tests { dirfd: 3452345, dirflags: 0, path: "/blah".into(), - o_flags: wasi::Oflags::all(), + o_flags: wasi::Oflags::empty(), fs_rights_base: wasi::Rights::all(), fs_rights_inheriting: wasi::Rights::all(), fs_flags: wasi::Fdflags::all(), @@ -780,7 +776,7 @@ mod tests { dirfd: 3452345, dirflags: 0, path: "/blah".into(), - o_flags: wasi::Oflags::all(), + o_flags: wasi::Oflags::empty(), fs_rights_base: wasi::Rights::all(), fs_rights_inheriting: wasi::Rights::all(), fs_flags: wasi::Fdflags::all(), @@ -798,7 +794,7 @@ mod tests { dirfd: 3452345, dirflags: 0, path: "/blah".into(), - o_flags: wasi::Oflags::all(), + o_flags: wasi::Oflags::empty(), fs_rights_base: wasi::Rights::all(), fs_rights_inheriting: wasi::Rights::all(), fs_flags: wasi::Fdflags::all(), @@ -824,7 +820,7 @@ mod tests { dirfd: 3452345, dirflags: 0, path: "/blah".into(), - o_flags: wasi::Oflags::all(), + o_flags: wasi::Oflags::empty(), fs_rights_base: wasi::Rights::all(), fs_rights_inheriting: wasi::Rights::all(), fs_flags: wasi::Fdflags::all(), @@ -843,7 +839,7 @@ mod tests { dirfd: 3452345, dirflags: 0, path: "/blah".into(), - o_flags: wasi::Oflags::all(), + o_flags: wasi::Oflags::empty(), fs_rights_base: wasi::Rights::all(), fs_rights_inheriting: wasi::Rights::all(), fs_flags: wasi::Fdflags::all(), @@ -870,7 +866,7 @@ mod tests { dirfd: 3452345, dirflags: 0, path: "/blah".into(), - o_flags: wasi::Oflags::all(), + o_flags: wasi::Oflags::empty(), fs_rights_base: wasi::Rights::all(), fs_rights_inheriting: wasi::Rights::all(), fs_flags: wasi::Fdflags::all(), @@ -889,7 +885,7 @@ mod tests { dirfd: 3452345, dirflags: 0, path: "/blah".into(), - o_flags: wasi::Oflags::all(), + o_flags: wasi::Oflags::empty(), fs_rights_base: wasi::Rights::all(), fs_rights_inheriting: wasi::Rights::all(), fs_flags: wasi::Fdflags::all(), @@ -916,7 +912,7 @@ mod tests { dirfd: 3452345, dirflags: 0, path: "/blah".into(), - o_flags: wasi::Oflags::all(), + o_flags: wasi::Oflags::empty(), fs_rights_base: wasi::Rights::all(), fs_rights_inheriting: wasi::Rights::all(), fs_flags: wasi::Fdflags::all(), @@ -933,6 +929,105 @@ mod tests { .unwrap() } + #[tracing_test::traced_test] + #[test] + pub fn test_compact_file_system_touch() { + run_test( + vec![ + JournalEntry::OpenFileDescriptorV1 { + fd: 1234, + dirfd: 3452345, + dirflags: 0, + path: "/blah".into(), + o_flags: wasi::Oflags::CREATE | wasi::Oflags::TRUNC, + fs_rights_base: wasi::Rights::all(), + fs_rights_inheriting: wasi::Rights::all(), + fs_flags: wasi::Fdflags::all(), + }, + JournalEntry::CloseFileDescriptorV1 { fd: 1234 }, + JournalEntry::ProcessExitV1 { exit_code: None }, + ], + vec![ + JournalEntry::OpenFileDescriptorV1 { + fd: 1234, + dirfd: 3452345, + dirflags: 0, + path: "/blah".into(), + o_flags: wasi::Oflags::CREATE | wasi::Oflags::TRUNC, + fs_rights_base: wasi::Rights::all(), + fs_rights_inheriting: wasi::Rights::all(), + fs_flags: wasi::Fdflags::all(), + }, + JournalEntry::CloseFileDescriptorV1 { fd: 1234 }, + JournalEntry::ProcessExitV1 { exit_code: None }, + ], + ) + .unwrap() + } + + #[tracing_test::traced_test] + #[test] + pub fn test_compact_file_system_redundant_file() { + run_test( + vec![ + JournalEntry::OpenFileDescriptorV1 { + fd: 1234, + dirfd: 3452345, + dirflags: 0, + path: "/blah".into(), + o_flags: wasi::Oflags::CREATE | wasi::Oflags::TRUNC, + fs_rights_base: wasi::Rights::all(), + fs_rights_inheriting: wasi::Rights::all(), + fs_flags: wasi::Fdflags::all(), + }, + JournalEntry::FileDescriptorWriteV1 { + fd: 1234, + offset: 1234, + data: [5u8; 16].to_vec().into(), + is_64bit: true, + }, + JournalEntry::CloseFileDescriptorV1 { fd: 1234 }, + JournalEntry::OpenFileDescriptorV1 { + fd: 1235, + dirfd: 3452345, + dirflags: 0, + path: "/blah".into(), + o_flags: wasi::Oflags::CREATE | wasi::Oflags::TRUNC, + fs_rights_base: wasi::Rights::all(), + fs_rights_inheriting: wasi::Rights::all(), + fs_flags: wasi::Fdflags::all(), + }, + JournalEntry::FileDescriptorWriteV1 { + fd: 1235, + offset: 1234, + data: [6u8; 16].to_vec().into(), + is_64bit: true, + }, + JournalEntry::CloseFileDescriptorV1 { fd: 1235 }, + ], + vec![ + JournalEntry::OpenFileDescriptorV1 { + fd: 1235, + dirfd: 3452345, + dirflags: 0, + path: "/blah".into(), + o_flags: wasi::Oflags::CREATE | wasi::Oflags::TRUNC, + fs_rights_base: wasi::Rights::all(), + fs_rights_inheriting: wasi::Rights::all(), + fs_flags: wasi::Fdflags::all(), + }, + JournalEntry::FileDescriptorWriteV1 { + fd: 1235, + offset: 1234, + data: [6u8; 16].to_vec().into(), + is_64bit: true, + }, + JournalEntry::CloseFileDescriptorV1 { fd: 1235 }, + ], + ) + .unwrap() + } + #[tracing_test::traced_test] #[test] pub fn test_compact_file_system_ignore_double_writes() { @@ -943,7 +1038,7 @@ mod tests { dirfd: 3452345, dirflags: 0, path: "/blah".into(), - o_flags: wasi::Oflags::all(), + o_flags: wasi::Oflags::empty(), fs_rights_base: wasi::Rights::all(), fs_rights_inheriting: wasi::Rights::all(), fs_flags: wasi::Fdflags::all(), @@ -968,7 +1063,7 @@ mod tests { dirfd: 3452345, dirflags: 0, path: "/blah".into(), - o_flags: wasi::Oflags::all(), + o_flags: wasi::Oflags::empty(), fs_rights_base: wasi::Rights::all(), fs_rights_inheriting: wasi::Rights::all(), fs_flags: wasi::Fdflags::all(), @@ -1001,6 +1096,28 @@ mod tests { .unwrap() } + #[tracing_test::traced_test] + #[test] + pub fn test_compact_file_system_redundant_create_directory() { + run_test( + vec![ + JournalEntry::CreateDirectoryV1 { + fd: 1234, + path: "/blah".into(), + }, + JournalEntry::CreateDirectoryV1 { + fd: 1235, + path: "/blah".into(), + }, + ], + vec![JournalEntry::CreateDirectoryV1 { + fd: 1234, + path: "/blah".into(), + }], + ) + .unwrap() + } + #[tracing_test::traced_test] #[test] pub fn test_compact_duplicate_tty() { From 4bd2b9d2e3645e943ae1bc2d48a3cb03274f1d82 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Sat, 23 Dec 2023 21:01:17 +1100 Subject: [PATCH 119/129] Linting fix --- lib/wasix/src/runners/dcgi/factory.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/lib/wasix/src/runners/dcgi/factory.rs b/lib/wasix/src/runners/dcgi/factory.rs index 056e490164b..9e10a74f09b 100644 --- a/lib/wasix/src/runners/dcgi/factory.rs +++ b/lib/wasix/src/runners/dcgi/factory.rs @@ -20,7 +20,7 @@ struct State { /// This factory will store and reuse instances between invocations thus /// allowing for the instances to be stateful. -#[derive(Derivative, Clone)] +#[derive(Derivative, Clone, Default)] #[derivative(Debug)] pub struct DcgiInstanceFactory { state: Arc>, @@ -28,9 +28,7 @@ pub struct DcgiInstanceFactory { impl DcgiInstanceFactory { pub fn new() -> Self { - Self { - state: Default::default(), - } + Default::default() } pub async fn release(&self, conf: RecycleEnvConfig) { From 9bb6f1887fdaffd3e14312e9bffbc5773e2ba09b Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Sun, 24 Dec 2023 09:30:58 +1100 Subject: [PATCH 120/129] Fixed a race condition in the virtual-net unit tests --- Cargo.lock | 30 ++++++++++++++++++++++++-- lib/virtual-net/Cargo.toml | 1 + lib/virtual-net/src/client.rs | 40 +++++++++++++++++++++++------------ lib/virtual-net/src/tests.rs | 13 +++++++++++- 4 files changed, 68 insertions(+), 16 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 95666e8d8c1..9069409501b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4162,7 +4162,21 @@ checksum = "e0bccbcf40c8938196944a3da0e133e031a33f4d6b72db3bda3cc556e361905d" dependencies = [ "lazy_static", "parking_lot 0.11.2", - "serial_test_derive", + "serial_test_derive 0.5.1", +] + +[[package]] +name = "serial_test" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e56dd856803e253c8f298af3f4d7eb0ae5e23a737252cd90bb4f3b435033b2d" +dependencies = [ + "dashmap", + "futures", + "lazy_static", + "log", + "parking_lot 0.12.1", + "serial_test_derive 2.0.0", ] [[package]] @@ -4176,6 +4190,17 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "serial_test_derive" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91d129178576168c589c9ec973feedf7d3126c01ac2bf08795109aa35b69fb8f" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.42", +] + [[package]] name = "sha1" version = "0.10.6" @@ -5293,6 +5318,7 @@ dependencies = [ "pin-project-lite", "rkyv", "serde", + "serial_test 2.0.0", "socket2 0.4.10", "thiserror", "tokio", @@ -6448,7 +6474,7 @@ dependencies = [ "glob", "lazy_static", "rustc_version 0.4.0", - "serial_test", + "serial_test 0.5.1", "tempfile", "test-generator", "test-log", diff --git a/lib/virtual-net/Cargo.toml b/lib/virtual-net/Cargo.toml index dc390b4006c..db23e664d3d 100644 --- a/lib/virtual-net/Cargo.toml +++ b/lib/virtual-net/Cargo.toml @@ -37,6 +37,7 @@ bytecheck = { version = "0.6.8", optional = true } [dev-dependencies] tokio = { version = "1", default_features = false, features = [ "macros", "rt-multi-thread" ] } tracing-test = { version = "0.2" } +serial_test = "2.0.0" [features] default = [ "host-net", "remote", "json", "messagepack", "cbor", "hyper", "tokio-tungstenite" ] diff --git a/lib/virtual-net/src/client.rs b/lib/virtual-net/src/client.rs index 5777c338810..879b99244ab 100644 --- a/lib/virtual-net/src/client.rs +++ b/lib/virtual-net/src/client.rs @@ -335,7 +335,9 @@ impl Future for RemoteNetworkingClientDriver { let guard = self.common.recv_tx.lock().unwrap(); match guard.get(&socket_id) { Some(tx) => tx.clone(), - None => continue, + None => { + continue; + } } }; let common = self.common.clone(); @@ -409,7 +411,6 @@ impl Future for RemoteNetworkingClientDriver { } } }, - MessageResponse::FinishAccept { socket_id, child_id, @@ -872,7 +873,7 @@ struct RemoteSocket { tx_waker: Waker, rx_accept: mpsc::Receiver, rx_sent: mpsc::Receiver, - pending_accept: Option, + pending_accept: Option<(SocketId, mpsc::Receiver>)>, buffer_recv_with_addr: VecDeque, buffer_accept: VecDeque, send_available: u64, @@ -930,7 +931,11 @@ impl RemoteSocket { .fetch_add(1, Ordering::SeqCst) .into(); self.io_socket_fire_and_forget(RequestType::BeginAccept(child_id))?; - self.pending_accept.replace(child_id); + + let (tx, rx_recv) = tokio::sync::mpsc::channel(100); + self.common.recv_tx.lock().unwrap().insert(child_id, tx); + + self.pending_accept.replace((child_id, rx_recv)); Ok(()) } } @@ -1059,17 +1064,26 @@ impl VirtualTcpListener for RemoteSocket { // This placed here will mean there is always an accept request pending at the // server as the constructor invokes this method and we invoke it here after // receiving a child connection. - self.pending_accept.take(); + let mut rx_recv = None; + if let Some((rx_socket, existing_rx_recv)) = self.pending_accept.take() { + if accepted.socket == rx_socket { + rx_recv.replace(existing_rx_recv); + } + } + let rx_recv = match rx_recv { + Some(rx_recv) => rx_recv, + None => { + let (tx, rx_recv) = tokio::sync::mpsc::channel(100); + self.common + .recv_tx + .lock() + .unwrap() + .insert(accepted.socket, tx); + rx_recv + } + }; self.touch_begin_accept().ok(); - // Now we construct the child - let (tx, rx_recv) = tokio::sync::mpsc::channel(100); - self.common - .recv_tx - .lock() - .unwrap() - .insert(accepted.socket, tx); - let (tx, rx_recv_with_addr) = tokio::sync::mpsc::channel(100); self.common .recv_with_addr_tx diff --git a/lib/virtual-net/src/tests.rs b/lib/virtual-net/src/tests.rs index 576c9cba029..1b0149a1b76 100644 --- a/lib/virtual-net/src/tests.rs +++ b/lib/virtual-net/src/tests.rs @@ -80,7 +80,7 @@ async fn test_tcp(client: RemoteNetworkingClient, _server: RemoteNetworkingServe ) .await .unwrap(); - let addr = listener.addr_local().unwrap(); + let addr: SocketAddr = listener.addr_local().unwrap(); tracing::info!("listening on {addr}"); const TEST1: &str = "the cat ran up the wall!"; @@ -126,6 +126,7 @@ async fn test_tcp(client: RemoteNetworkingClient, _server: RemoteNetworkingServe #[cfg_attr(windows, ignore)] #[traced_test] #[tokio::test(flavor = "multi_thread")] +#[serial_test::serial] async fn test_tcp_with_mpsc() { let (client, server) = setup_mpsc().await; test_tcp(client, server).await @@ -135,6 +136,7 @@ async fn test_tcp_with_mpsc() { #[cfg_attr(windows, ignore)] #[traced_test] #[tokio::test(flavor = "multi_thread")] +#[serial_test::serial] async fn test_tcp_with_small_pipe_using_bincode() { let (client, server) = setup_pipe(10, FrameSerializationFormat::Bincode).await; test_tcp(client, server).await @@ -144,6 +146,7 @@ async fn test_tcp_with_small_pipe_using_bincode() { #[cfg_attr(windows, ignore)] #[traced_test] #[tokio::test(flavor = "multi_thread")] +#[serial_test::serial] async fn test_tcp_with_large_pipe_using_bincode() { let (client, server) = setup_pipe(1024000, FrameSerializationFormat::Bincode).await; test_tcp(client, server).await @@ -154,6 +157,7 @@ async fn test_tcp_with_large_pipe_using_bincode() { #[cfg_attr(windows, ignore)] #[traced_test] #[tokio::test(flavor = "multi_thread")] +#[serial_test::serial] async fn test_tcp_with_small_pipe_using_json() { let (client, server) = setup_pipe(10, FrameSerializationFormat::Json).await; test_tcp(client, server).await @@ -164,6 +168,7 @@ async fn test_tcp_with_small_pipe_using_json() { #[cfg_attr(windows, ignore)] #[traced_test] #[tokio::test(flavor = "multi_thread")] +#[serial_test::serial] async fn test_tcp_with_large_pipe_json_using_json() { let (client, server) = setup_pipe(1024000, FrameSerializationFormat::Json).await; test_tcp(client, server).await @@ -174,6 +179,7 @@ async fn test_tcp_with_large_pipe_json_using_json() { #[cfg_attr(windows, ignore)] #[traced_test] #[tokio::test(flavor = "multi_thread")] +#[serial_test::serial] async fn test_tcp_with_small_pipe_using_messagepack() { let (client, server) = setup_pipe(10, FrameSerializationFormat::MessagePack).await; test_tcp(client, server).await @@ -184,6 +190,7 @@ async fn test_tcp_with_small_pipe_using_messagepack() { #[cfg_attr(windows, ignore)] #[traced_test] #[tokio::test(flavor = "multi_thread")] +#[serial_test::serial] async fn test_tcp_with_large_pipe_json_using_messagepack() { let (client, server) = setup_pipe(1024000, FrameSerializationFormat::MessagePack).await; test_tcp(client, server).await @@ -194,6 +201,7 @@ async fn test_tcp_with_large_pipe_json_using_messagepack() { #[cfg_attr(windows, ignore)] #[traced_test] #[tokio::test(flavor = "multi_thread")] +#[serial_test::serial] async fn test_tcp_with_small_pipe_using_cbor() { let (client, server) = setup_pipe(10, FrameSerializationFormat::Cbor).await; test_tcp(client, server).await @@ -204,6 +212,7 @@ async fn test_tcp_with_small_pipe_using_cbor() { #[cfg_attr(windows, ignore)] #[traced_test] #[tokio::test(flavor = "multi_thread")] +#[serial_test::serial] async fn test_tcp_with_large_pipe_json_using_cbor() { let (client, server) = setup_pipe(1024000, FrameSerializationFormat::Cbor).await; test_tcp(client, server).await @@ -212,6 +221,7 @@ async fn test_tcp_with_large_pipe_json_using_cbor() { #[cfg_attr(windows, ignore)] #[traced_test] #[tokio::test] +#[serial_test::serial] async fn test_google_poll() { use futures_util::Future; @@ -303,6 +313,7 @@ async fn test_google_poll() { #[cfg_attr(windows, ignore)] #[traced_test] #[tokio::test] +#[serial_test::serial] async fn test_google_epoll() { use futures_util::Future; use virtual_mio::SharedWakerInterestHandler; From 4420e617b2470fe4f533f07ff9f6b8f044d27ca9 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Sun, 24 Dec 2023 09:37:20 +1100 Subject: [PATCH 121/129] Excluding tokio from the docs build --- Makefile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 647cf2b9493..e6d51de1b93 100644 --- a/Makefile +++ b/Makefile @@ -439,10 +439,10 @@ test-build-docs-rs: fi; \ printf "*** Building doc for package with manifest $$manifest_path ***\n\n"; \ if [ -z "$$features" ]; then \ - RUSTDOCFLAGS="--cfg=docsrs" $(CARGO_BINARY) +nightly doc $(CARGO_TARGET_FLAG) --manifest-path "$$manifest_path" --locked || exit 1; \ + RUSTDOCFLAGS="--cfg=docsrs" $(CARGO_BINARY) +nightly doc $(CARGO_TARGET_FLAG) --manifest-path "$$manifest_path" --exclude tokio --locked || exit 1; \ else \ printf "Following features are inferred from Cargo.toml: $$features\n\n\n"; \ - RUSTDOCFLAGS="--cfg=docsrs" $(CARGO_BINARY) +nightly doc $(CARGO_TARGET_FLAG) --manifest-path "$$manifest_path" --features "$$features" --locked || exit 1; \ + RUSTDOCFLAGS="--cfg=docsrs" $(CARGO_BINARY) +nightly doc $(CARGO_TARGET_FLAG) --manifest-path "$$manifest_path" --exclude tokio --features "$$features" --locked || exit 1; \ fi; \ done @@ -457,10 +457,10 @@ test-build-docs-rs-ci: fi; \ printf "*** Building doc for package with manifest $$manifest_path ***\n\n"; \ if [ -z "$$features" ]; then \ - RUSTDOCFLAGS="--cfg=docsrs" $(CARGO_BINARY) +nightly-2023-05-25 doc $(CARGO_TARGET_FLAG) --manifest-path "$$manifest_path" --locked || exit 1; \ + RUSTDOCFLAGS="--cfg=docsrs" $(CARGO_BINARY) +nightly-2023-05-25 doc $(CARGO_TARGET_FLAG) --manifest-path "$$manifest_path" --no-deps --locked || exit 1; \ else \ printf "Following features are inferred from Cargo.toml: $$features\n\n\n"; \ - RUSTDOCFLAGS="--cfg=docsrs" $(CARGO_BINARY) +nightly-2023-05-25 doc $(CARGO_TARGET_FLAG) --manifest-path "$$manifest_path" --features "$$features" --locked || exit 1; \ + RUSTDOCFLAGS="--cfg=docsrs" $(CARGO_BINARY) +nightly-2023-05-25 doc $(CARGO_TARGET_FLAG) --manifest-path "$$manifest_path" --no-deps --features "$$features" --locked || exit 1; \ fi; \ done From eb96a9b95f883351dd45425d52e7e4b9ecd13870 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Thu, 4 Jan 2024 09:39:41 +1100 Subject: [PATCH 122/129] Added comments on the JournalEffector --- lib/wasix/src/journal/effector/mod.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/lib/wasix/src/journal/effector/mod.rs b/lib/wasix/src/journal/effector/mod.rs index f730c73d120..bf9e464da14 100644 --- a/lib/wasix/src/journal/effector/mod.rs +++ b/lib/wasix/src/journal/effector/mod.rs @@ -91,5 +91,15 @@ mod thread_exit; #[cfg(feature = "journal")] mod thread_state; +/// The journal effector is an adapter that will be removed in a future refactor. +/// Its purpose is to put the code that does mappings from WASM memory through its +/// abstractions into concrete journal objects that can be stored. Instead of this +/// what should be done is that the syscalls themselves can be represented as a +/// strongly typed object that can be passed directly to the journal but in order +/// to do this we require an extensive refactoring of the WASIX syscalls which +/// is not in scope at this time. +/// +/// Separating this out now makes it easier to eliminate later without hurting the +/// journal event abstraction through leaking abstraction layers. #[derive(Debug, Clone)] pub struct JournalEffector {} From fb6d4d639d775e1f8313ec4cfc1c1808f4884758 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Thu, 4 Jan 2024 09:57:00 +1100 Subject: [PATCH 123/129] Split out the config responsibility from the FilteredJournal --- lib/wasix/src/journal/concrete/compacting.rs | 22 +-- lib/wasix/src/journal/concrete/filter.rs | 174 +++++++++++-------- 2 files changed, 117 insertions(+), 79 deletions(-) diff --git a/lib/wasix/src/journal/concrete/compacting.rs b/lib/wasix/src/journal/concrete/compacting.rs index 40bf148f1b9..dd42e3632f8 100644 --- a/lib/wasix/src/journal/concrete/compacting.rs +++ b/lib/wasix/src/journal/concrete/compacting.rs @@ -88,7 +88,7 @@ impl State { { let has_threads = !self.thread_map.is_empty(); - let mut filter = FilteredJournal::new(inner) + let mut filter = FilteredJournalBuilder::new() .with_filter_events(self.whitelist.clone().into_iter().collect()); if let Some(tty) = self.tty.as_ref() { filter.add_event_to_whitelist(*tty); @@ -134,7 +134,7 @@ impl State { } } } - filter + filter.build(inner) } } @@ -264,14 +264,16 @@ impl CompactingJournalTx { // Now we build a filtered journal which will pick up any events that were // added which we did the compacting. - let new_journal = FilteredJournal::new(new_journal).with_filter_events( - state - .delta_list - .take() - .unwrap_or_default() - .into_iter() - .collect(), - ); + let new_journal = FilteredJournalBuilder::new() + .with_filter_events( + state + .delta_list + .take() + .unwrap_or_default() + .into_iter() + .collect(), + ) + .build(new_journal); // Now we feed all the events into the new journal using the delta filter. After the // extra events are added we strip off the filter again diff --git a/lib/wasix/src/journal/concrete/filter.rs b/lib/wasix/src/journal/concrete/filter.rs index cfeb22f4852..4ed3bd0f408 100644 --- a/lib/wasix/src/journal/concrete/filter.rs +++ b/lib/wasix/src/journal/concrete/filter.rs @@ -16,11 +16,9 @@ pub struct FilteredJournal { rx: FilteredJournalRx, } -#[derive(Derivative)] -#[derivative(Debug)] -pub struct FilteredJournalTx { - #[derivative(Debug = "ignore")] - inner: Box, +/// Represents what will be filtered by the filtering process +#[derive(Debug)] +struct FilteredJournalConfig { filter_memory: bool, filter_threads: bool, filter_fs: bool, @@ -32,6 +30,46 @@ pub struct FilteredJournalTx { event_index: AtomicUsize, } +impl Default for FilteredJournalConfig { + fn default() -> Self { + Self { + filter_memory: false, + filter_threads: false, + filter_fs: false, + filter_stdio: false, + filter_core: false, + filter_snapshots: false, + filter_net: false, + filter_events: None, + event_index: AtomicUsize::new(0), + } + } +} + +impl Clone for FilteredJournalConfig { + fn clone(&self) -> Self { + Self { + filter_memory: self.filter_memory, + filter_threads: self.filter_threads, + filter_fs: self.filter_fs, + filter_stdio: self.filter_stdio, + filter_core: self.filter_core, + filter_snapshots: self.filter_snapshots, + filter_net: self.filter_net, + filter_events: self.filter_events, + event_index: AtomicUsize::new(self.event_index.load(Ordering::SeqCst)), + } + } +} + +#[derive(Derivative)] +#[derivative(Debug)] +pub struct FilteredJournalTx { + #[derivative(Debug = "ignore")] + inner: Box, + config: FilteredJournalConfig, +} + #[derive(Derivative)] #[derivative(Debug)] pub struct FilteredJournalRx { @@ -39,131 +77,129 @@ pub struct FilteredJournalRx { inner: Box, } -impl FilteredJournal { - pub fn new(inner: J) -> Self - where - J: Journal, - { - let (tx, rx) = inner.split(); - Self { - tx: FilteredJournalTx { - inner: tx, - filter_memory: false, - filter_threads: false, - filter_fs: false, - filter_stdio: false, - filter_core: false, - filter_snapshots: false, - filter_net: false, - filter_events: None, - event_index: AtomicUsize::new(0), - }, - rx: FilteredJournalRx { inner: rx }, - } +/// Constructs a filter with a set of parameters that will be filtered on +#[derive(Debug, Default)] +pub struct FilteredJournalBuilder { + config: FilteredJournalConfig, +} + +impl FilteredJournalBuilder { + pub fn new() -> Self { + Self::default() } - pub fn clone_with_inner(&self, inner: J) -> Self + pub fn build(self, inner: J) -> FilteredJournal where J: Journal, { - let (tx, rx) = inner.split(); - Self { - tx: FilteredJournalTx { - inner: tx, - filter_memory: self.tx.filter_memory, - filter_threads: self.tx.filter_threads, - filter_fs: self.tx.filter_fs, - filter_stdio: self.tx.filter_stdio, - filter_core: self.tx.filter_core, - filter_snapshots: self.tx.filter_snapshots, - filter_net: self.tx.filter_net, - filter_events: self.tx.filter_events.clone(), - event_index: AtomicUsize::new(self.tx.event_index.load(Ordering::SeqCst)), - }, - rx: FilteredJournalRx { inner: rx }, - } + FilteredJournal::new(inner, self.config) } pub fn with_ignore_memory(mut self, val: bool) -> Self { - self.tx.filter_memory = val; + self.config.filter_memory = val; self } pub fn with_ignore_threads(mut self, val: bool) -> Self { - self.tx.filter_threads = val; + self.config.filter_threads = val; self } pub fn with_ignore_fs(mut self, val: bool) -> Self { - self.tx.filter_fs = val; + self.config.filter_fs = val; self } pub fn with_ignore_stdio(mut self, val: bool) -> Self { - self.tx.filter_stdio = val; + self.config.filter_stdio = val; self } pub fn with_ignore_core(mut self, val: bool) -> Self { - self.tx.filter_core = val; + self.config.filter_core = val; self } pub fn with_ignore_snapshots(mut self, val: bool) -> Self { - self.tx.filter_snapshots = val; + self.config.filter_snapshots = val; self } pub fn with_ignore_networking(mut self, val: bool) -> Self { - self.tx.filter_net = val; + self.config.filter_net = val; self } pub fn with_filter_events(mut self, events: HashSet) -> Self { - self.tx.filter_events = Some(events); + self.config.filter_events = Some(events); self } pub fn add_event_to_whitelist(&mut self, event_index: usize) { - if let Some(filter) = self.tx.filter_events.as_mut() { + if let Some(filter) = self.config.filter_events.as_mut() { filter.insert(event_index); } } pub fn set_ignore_memory(&mut self, val: bool) -> &mut Self { - self.tx.filter_memory = val; + self.config.filter_memory = val; self } pub fn set_ignore_threads(&mut self, val: bool) -> &mut Self { - self.tx.filter_threads = val; + self.config.filter_threads = val; self } pub fn set_ignore_fs(&mut self, val: bool) -> &mut Self { - self.tx.filter_fs = val; + self.config.filter_fs = val; self } pub fn set_ignore_stdio(&mut self, val: bool) -> &mut Self { - self.tx.filter_stdio = val; + self.config.filter_stdio = val; self } pub fn set_ignore_core(&mut self, val: bool) -> &mut Self { - self.tx.filter_core = val; + self.config.filter_core = val; self } pub fn set_ignore_snapshots(&mut self, val: bool) -> &mut Self { - self.tx.filter_snapshots = val; + self.config.filter_snapshots = val; self } pub fn set_ignore_networking(&mut self, val: bool) -> &mut Self { - self.tx.filter_net = val; + self.config.filter_net = val; self } +} + +impl FilteredJournal { + fn new(inner: J, config: FilteredJournalConfig) -> Self + where + J: Journal, + { + let (tx, rx) = inner.split(); + Self { + tx: FilteredJournalTx { inner: tx, config }, + rx: FilteredJournalRx { inner: rx }, + } + } + + pub fn clone_with_inner(&self, inner: J) -> Self + where + J: Journal, + { + let config = self.tx.config.clone(); + let (tx, rx) = inner.split(); + Self { + tx: FilteredJournalTx { config, inner: tx }, + rx: FilteredJournalRx { inner: rx }, + } + } pub fn into_inner(self) -> RecombinedJournal { RecombinedJournal::new(self.tx.inner, self.rx.inner) @@ -172,8 +208,8 @@ impl FilteredJournal { impl WritableJournal for FilteredJournalTx { fn write<'a>(&'a self, entry: JournalEntry<'a>) -> anyhow::Result { - let event_index = self.event_index.fetch_add(1, Ordering::SeqCst); - if let Some(events) = self.filter_events.as_ref() { + let event_index = self.config.event_index.fetch_add(1, Ordering::SeqCst); + if let Some(events) = self.config.filter_events.as_ref() { if !events.contains(&event_index) { return Ok(0); } @@ -186,19 +222,19 @@ impl WritableJournal for FilteredJournalTx { | JournalEntry::EpollCreateV1 { .. } | JournalEntry::EpollCtlV1 { .. } | JournalEntry::TtySetV1 { .. } => { - if self.filter_core { + if self.config.filter_core { return Ok(0); } entry } JournalEntry::SetThreadV1 { .. } | JournalEntry::CloseThreadV1 { .. } => { - if self.filter_threads { + if self.config.filter_threads { return Ok(0); } entry } JournalEntry::UpdateMemoryRegionV1 { .. } => { - if self.filter_memory { + if self.config.filter_memory { return Ok(0); } entry @@ -217,10 +253,10 @@ impl WritableJournal for FilteredJournalTx { | JournalEntry::FileDescriptorSetRightsV1 { fd, .. } | JournalEntry::FileDescriptorSetTimesV1 { fd, .. } | JournalEntry::FileDescriptorSetSizeV1 { fd, .. } => { - if self.filter_stdio && fd <= 2 { + if self.config.filter_stdio && fd <= 2 { return Ok(0); } - if self.filter_fs { + if self.config.filter_fs { return Ok(0); } entry @@ -235,13 +271,13 @@ impl WritableJournal for FilteredJournalTx { | JournalEntry::ChangeDirectoryV1 { .. } | JournalEntry::CreatePipeV1 { .. } | JournalEntry::CreateEventV1 { .. } => { - if self.filter_fs { + if self.config.filter_fs { return Ok(0); } entry } JournalEntry::SnapshotV1 { .. } => { - if self.filter_snapshots { + if self.config.filter_snapshots { return Ok(0); } entry @@ -272,7 +308,7 @@ impl WritableJournal for FilteredJournalTx { | JournalEntry::SocketSetOptSizeV1 { .. } | JournalEntry::SocketSetOptTimeV1 { .. } | JournalEntry::SocketShutdownV1 { .. } => { - if self.filter_net { + if self.config.filter_net { return Ok(0); } entry From 58a89419db1c2cb5a088cfc490143f7db7f25b13 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Thu, 4 Jan 2024 10:07:28 +1100 Subject: [PATCH 124/129] Added a comment on the excluding of tokio from the docs build --- Makefile | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/Makefile b/Makefile index e6d51de1b93..90e19f081b2 100644 --- a/Makefile +++ b/Makefile @@ -428,6 +428,16 @@ endif build-docs: $(CARGO_BINARY) doc $(CARGO_TARGET_FLAG) --release $(compiler_features) --document-private-items --no-deps --workspace --exclude wasmer-c-api --locked +# The tokio crate was excluded from the docs build because the code (which is not under our control) +# does not currently compile its docs successfully +# +# ``` +# error[E0432]: unresolved import `self::doc::os` +# --> /home/runner/.cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.35.1/src/lib.rs:636:16 +# | +# 636 | pub(crate) use self::doc::os; +# | ^^^^^^^^^^^^^ no `os` in `doc` +# ``` test-build-docs-rs: @manifest_docs_rs_features_path="package.metadata.docs.rs.features"; \ for manifest_path in lib/*/Cargo.toml; do \ @@ -438,6 +448,7 @@ test-build-docs-rs: features=$$(toml get "$$manifest_path" "$$manifest_docs_rs_features_path" | sed 's/\[//; s/\]//; s/"\([^"]*\)"/\1/g'); \ fi; \ printf "*** Building doc for package with manifest $$manifest_path ***\n\n"; \ + if [ -z "$$features" ]; then \ RUSTDOCFLAGS="--cfg=docsrs" $(CARGO_BINARY) +nightly doc $(CARGO_TARGET_FLAG) --manifest-path "$$manifest_path" --exclude tokio --locked || exit 1; \ else \ From 7c1b68df6b11c47c825ab2b01202a336c3e3370c Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Thu, 4 Jan 2024 10:43:26 +1100 Subject: [PATCH 125/129] Moved most of the journal functionality out into its own crate --- Cargo.lock | 24 +++++++++ Cargo.toml | 1 + lib/journal/Cargo.toml | 32 ++++++++++++ .../src/journal => journal/src}/base64.rs | 0 .../journal => journal/src}/concrete/arc.rs | 0 .../src}/concrete/archived.rs | 8 +-- .../journal => journal/src}/concrete/boxed.rs | 0 .../src}/concrete/buffered.rs | 0 .../src}/concrete/compacting.rs | 5 +- .../src}/concrete/compacting_log_file.rs | 0 .../src}/concrete/counting.rs | 0 .../src}/concrete/filter.rs | 2 +- .../src}/concrete/log_file.rs | 0 .../journal => journal/src}/concrete/mod.rs | 0 .../journal => journal/src}/concrete/null.rs | 0 .../journal => journal/src}/concrete/pipe.rs | 0 .../src}/concrete/printing.rs | 0 .../src}/concrete/recombined.rs | 0 .../src}/concrete/unsupported.rs | 0 .../src/journal => journal/src}/entry.rs | 36 ++----------- lib/journal/src/lib.rs | 50 +++++++++++++++++++ .../src/journal => journal/src}/snapshot.rs | 0 .../src/journal => journal/src}/util.rs | 0 lib/wasix/Cargo.toml | 1 + lib/wasix/src/journal/effector/thread_exit.rs | 5 +- .../src/journal/effector/thread_state.rs | 2 +- lib/wasix/src/journal/mod.rs | 50 +------------------ lib/wasix/src/net/socket.rs | 28 +++++++++++ lib/wasix/src/runners/dcgi/runner.rs | 8 +-- lib/wasix/src/syscalls/journal.rs | 13 +++-- 30 files changed, 168 insertions(+), 97 deletions(-) create mode 100644 lib/journal/Cargo.toml rename lib/{wasix/src/journal => journal/src}/base64.rs (100%) rename lib/{wasix/src/journal => journal/src}/concrete/arc.rs (100%) rename lib/{wasix/src/journal => journal/src}/concrete/archived.rs (99%) rename lib/{wasix/src/journal => journal/src}/concrete/boxed.rs (100%) rename lib/{wasix/src/journal => journal/src}/concrete/buffered.rs (100%) rename lib/{wasix/src/journal => journal/src}/concrete/compacting.rs (99%) rename lib/{wasix/src/journal => journal/src}/concrete/compacting_log_file.rs (100%) rename lib/{wasix/src/journal => journal/src}/concrete/counting.rs (100%) rename lib/{wasix/src/journal => journal/src}/concrete/filter.rs (99%) rename lib/{wasix/src/journal => journal/src}/concrete/log_file.rs (100%) rename lib/{wasix/src/journal => journal/src}/concrete/mod.rs (100%) rename lib/{wasix/src/journal => journal/src}/concrete/null.rs (100%) rename lib/{wasix/src/journal => journal/src}/concrete/pipe.rs (100%) rename lib/{wasix/src/journal => journal/src}/concrete/printing.rs (100%) rename lib/{wasix/src/journal => journal/src}/concrete/recombined.rs (100%) rename lib/{wasix/src/journal => journal/src}/concrete/unsupported.rs (100%) rename lib/{wasix/src/journal => journal/src}/entry.rs (95%) create mode 100644 lib/journal/src/lib.rs rename lib/{wasix/src/journal => journal/src}/snapshot.rs (100%) rename lib/{wasix/src/journal => journal/src}/util.rs (100%) diff --git a/Cargo.lock b/Cargo.lock index 9069409501b..a9442017666 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6123,6 +6123,29 @@ dependencies = [ name = "wasmer-integration-tests-ios" version = "4.2.5" +[[package]] +name = "wasmer-journal" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-trait", + "base64", + "bincode", + "bytecheck", + "bytes", + "derivative", + "lz4_flex", + "num_enum", + "rkyv", + "serde", + "serde_json", + "thiserror", + "tracing", + "tracing-test", + "virtual-net 0.6.2", + "wasmer-wasix-types", +] + [[package]] name = "wasmer-middlewares" version = "4.2.5" @@ -6372,6 +6395,7 @@ dependencies = [ "wasm-bindgen-test", "wasmer", "wasmer-emscripten", + "wasmer-journal", "wasmer-types", "wasmer-wasix-types", "wcgi", diff --git a/Cargo.toml b/Cargo.toml index b067e8260a1..3019bed58a7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -65,6 +65,7 @@ members = [ "lib/wasix", "lib/wasix-http-client", "lib/wasm-interface", + "lib/journal", "tests/integration/cli", "tests/integration/ios", "tests/lib/compiler-test-derive", diff --git a/lib/journal/Cargo.toml b/lib/journal/Cargo.toml new file mode 100644 index 00000000000..011a2922a21 --- /dev/null +++ b/lib/journal/Cargo.toml @@ -0,0 +1,32 @@ +[package] +name = "wasmer-journal" +version = "0.1.0" +description = "Journaling functionality used by Wasmer to save and restore WASM state" +authors.workspace = true +edition.workspace = true +homepage.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true + +[dependencies] +wasmer-wasix-types = { path = "../wasi-types", version = "0.18.0", features = [ "enable-serde" ] } +virtual-net = { path = "../virtual-net", version = "0.6.2", default-features = false, features = ["rkyv"] } + +thiserror = "1" +bytes = "1.1" +async-trait = { version = "^0.1" } +tracing = "0.1" +derivative = { version = "^2" } +base64 = "0.21" +bincode = { version = "1.3" } +serde = { version = "1.0", default-features = false, features = ["derive"] } +anyhow = "1.0" +rkyv = { version = "0.7.40", features = ["indexmap", "validation", "strict"] } +bytecheck = { version = "0.6.8" } +lz4_flex = { version = "0.11" } +num_enum = "0.5.7" +serde_json = { version = "^1" } + +[dev-dependencies] +tracing-test = "0.2.4" \ No newline at end of file diff --git a/lib/wasix/src/journal/base64.rs b/lib/journal/src/base64.rs similarity index 100% rename from lib/wasix/src/journal/base64.rs rename to lib/journal/src/base64.rs diff --git a/lib/wasix/src/journal/concrete/arc.rs b/lib/journal/src/concrete/arc.rs similarity index 100% rename from lib/wasix/src/journal/concrete/arc.rs rename to lib/journal/src/concrete/arc.rs diff --git a/lib/wasix/src/journal/concrete/archived.rs b/lib/journal/src/concrete/archived.rs similarity index 99% rename from lib/wasix/src/journal/concrete/archived.rs rename to lib/journal/src/concrete/archived.rs index b5eb9852d5e..1255431c969 100644 --- a/lib/wasix/src/journal/concrete/archived.rs +++ b/lib/journal/src/concrete/archived.rs @@ -2345,7 +2345,7 @@ impl<'a> TryFrom> for JournalEntry<'a> { _padding: _, is_64bit, }) => Self::SetThreadV1 { - id: (*id).into(), + id: *id, call_stack: call_stack.as_ref().into(), memory_stack: memory_stack.as_ref().into(), store_data: store_data.as_ref().into(), @@ -2355,7 +2355,7 @@ impl<'a> TryFrom> for JournalEntry<'a> { id, exit_code, }) => Self::CloseThreadV1 { - id: (*id).into(), + id: *id, exit_code: exit_code.as_ref().map(|code| code.into()), }, ArchivedJournalEntry::FileDescriptorWriteV1( @@ -2909,7 +2909,7 @@ mod tests { #[test] pub fn test_record_set_thread() { run_test(JournalEntry::SetThreadV1 { - id: 1234u32.into(), + id: 1234u32, call_stack: vec![1, 2, 3].into(), memory_stack: vec![4, 5, 6, 7].into(), store_data: vec![10, 11].into(), @@ -2921,7 +2921,7 @@ mod tests { #[test] pub fn test_record_close_thread() { run_test(JournalEntry::CloseThreadV1 { - id: 987u32.into(), + id: 987u32, exit_code: Some(wasi::ExitCode::Errno(wasi::Errno::Fault)), }); } diff --git a/lib/wasix/src/journal/concrete/boxed.rs b/lib/journal/src/concrete/boxed.rs similarity index 100% rename from lib/wasix/src/journal/concrete/boxed.rs rename to lib/journal/src/concrete/boxed.rs diff --git a/lib/wasix/src/journal/concrete/buffered.rs b/lib/journal/src/concrete/buffered.rs similarity index 100% rename from lib/wasix/src/journal/concrete/buffered.rs rename to lib/journal/src/concrete/buffered.rs diff --git a/lib/wasix/src/journal/concrete/compacting.rs b/lib/journal/src/concrete/compacting.rs similarity index 99% rename from lib/wasix/src/journal/concrete/compacting.rs rename to lib/journal/src/concrete/compacting.rs index dd42e3632f8..8098c2ce2b5 100644 --- a/lib/wasix/src/journal/concrete/compacting.rs +++ b/lib/journal/src/concrete/compacting.rs @@ -4,11 +4,12 @@ use std::{ ops::{DerefMut, Range}, sync::{Arc, Mutex}, }; -use virtual_fs::Fd; use wasmer_wasix_types::wasi; use super::*; +pub type Fd = u32; + #[derive(Debug, Default)] struct StateDescriptor { events: Vec, @@ -52,7 +53,7 @@ struct State { create_trunc_file: HashMap, // Thread events are only maintained while the thread and the // process are still running - thread_map: HashMap, + thread_map: HashMap, // Any descriptors are assumed to be read only operations until // they actually do something that changes the system suspect_descriptors: HashMap, diff --git a/lib/wasix/src/journal/concrete/compacting_log_file.rs b/lib/journal/src/concrete/compacting_log_file.rs similarity index 100% rename from lib/wasix/src/journal/concrete/compacting_log_file.rs rename to lib/journal/src/concrete/compacting_log_file.rs diff --git a/lib/wasix/src/journal/concrete/counting.rs b/lib/journal/src/concrete/counting.rs similarity index 100% rename from lib/wasix/src/journal/concrete/counting.rs rename to lib/journal/src/concrete/counting.rs diff --git a/lib/wasix/src/journal/concrete/filter.rs b/lib/journal/src/concrete/filter.rs similarity index 99% rename from lib/wasix/src/journal/concrete/filter.rs rename to lib/journal/src/concrete/filter.rs index 4ed3bd0f408..32a8ee3ce7c 100644 --- a/lib/wasix/src/journal/concrete/filter.rs +++ b/lib/journal/src/concrete/filter.rs @@ -56,7 +56,7 @@ impl Clone for FilteredJournalConfig { filter_core: self.filter_core, filter_snapshots: self.filter_snapshots, filter_net: self.filter_net, - filter_events: self.filter_events, + filter_events: self.filter_events.clone(), event_index: AtomicUsize::new(self.event_index.load(Ordering::SeqCst)), } } diff --git a/lib/wasix/src/journal/concrete/log_file.rs b/lib/journal/src/concrete/log_file.rs similarity index 100% rename from lib/wasix/src/journal/concrete/log_file.rs rename to lib/journal/src/concrete/log_file.rs diff --git a/lib/wasix/src/journal/concrete/mod.rs b/lib/journal/src/concrete/mod.rs similarity index 100% rename from lib/wasix/src/journal/concrete/mod.rs rename to lib/journal/src/concrete/mod.rs diff --git a/lib/wasix/src/journal/concrete/null.rs b/lib/journal/src/concrete/null.rs similarity index 100% rename from lib/wasix/src/journal/concrete/null.rs rename to lib/journal/src/concrete/null.rs diff --git a/lib/wasix/src/journal/concrete/pipe.rs b/lib/journal/src/concrete/pipe.rs similarity index 100% rename from lib/wasix/src/journal/concrete/pipe.rs rename to lib/journal/src/concrete/pipe.rs diff --git a/lib/wasix/src/journal/concrete/printing.rs b/lib/journal/src/concrete/printing.rs similarity index 100% rename from lib/wasix/src/journal/concrete/printing.rs rename to lib/journal/src/concrete/printing.rs diff --git a/lib/wasix/src/journal/concrete/recombined.rs b/lib/journal/src/concrete/recombined.rs similarity index 100% rename from lib/wasix/src/journal/concrete/recombined.rs rename to lib/journal/src/concrete/recombined.rs diff --git a/lib/wasix/src/journal/concrete/unsupported.rs b/lib/journal/src/concrete/unsupported.rs similarity index 100% rename from lib/wasix/src/journal/concrete/unsupported.rs rename to lib/journal/src/concrete/unsupported.rs diff --git a/lib/wasix/src/journal/entry.rs b/lib/journal/src/entry.rs similarity index 95% rename from lib/wasix/src/journal/entry.rs rename to lib/journal/src/entry.rs index 37bb494163f..336a7d26d62 100644 --- a/lib/wasix/src/journal/entry.rs +++ b/lib/journal/src/entry.rs @@ -1,4 +1,3 @@ -use super::base64; use derivative::Derivative; use serde::{Deserialize, Serialize}; use std::net::{Shutdown, SocketAddr}; @@ -11,12 +10,9 @@ use wasmer_wasix_types::wasi::{ Sockoption, Socktype, Timestamp, Tty, Whence, }; -use virtual_fs::Fd; +use crate::{base64, SnapshotTrigger}; -use crate::net::socket::TimeType; -use crate::WasiThreadId; - -use super::SnapshotTrigger; +type Fd = u32; #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] @@ -77,30 +73,6 @@ pub enum SocketOptTimeType { BindTimeout, Linger, } -impl From for SocketOptTimeType { - fn from(value: TimeType) -> Self { - match value { - TimeType::ReadTimeout => Self::ReadTimeout, - TimeType::WriteTimeout => Self::WriteTimeout, - TimeType::AcceptTimeout => Self::AcceptTimeout, - TimeType::ConnectTimeout => Self::ConnectTimeout, - TimeType::BindTimeout => Self::BindTimeout, - TimeType::Linger => Self::Linger, - } - } -} -impl From for TimeType { - fn from(value: SocketOptTimeType) -> Self { - match value { - SocketOptTimeType::ReadTimeout => Self::ReadTimeout, - SocketOptTimeType::WriteTimeout => Self::WriteTimeout, - SocketOptTimeType::AcceptTimeout => Self::AcceptTimeout, - SocketOptTimeType::ConnectTimeout => Self::ConnectTimeout, - SocketOptTimeType::BindTimeout => Self::BindTimeout, - SocketOptTimeType::Linger => Self::Linger, - } - } -} /// Represents a log entry in a snapshot log stream that represents the total /// state of a WASM process at a point in time. @@ -122,7 +94,7 @@ pub enum JournalEntry<'a> { exit_code: Option, }, SetThreadV1 { - id: WasiThreadId, + id: u32, #[derivative(Debug = "ignore")] #[serde(with = "base64")] call_stack: Cow<'a, [u8]>, @@ -135,7 +107,7 @@ pub enum JournalEntry<'a> { is_64bit: bool, }, CloseThreadV1 { - id: WasiThreadId, + id: u32, exit_code: Option, }, FileDescriptorSeekV1 { diff --git a/lib/journal/src/lib.rs b/lib/journal/src/lib.rs new file mode 100644 index 00000000000..d92fa4cf8cf --- /dev/null +++ b/lib/journal/src/lib.rs @@ -0,0 +1,50 @@ +mod base64; +mod concrete; +mod entry; +mod snapshot; +mod util; + +pub use concrete::*; +pub use entry::*; +pub use snapshot::*; +pub use util::*; + +use serde::{Deserialize, Serialize}; +use std::str::FromStr; + +/// The snapshot capturer will take a series of objects that represents the state of +/// a WASM process at a point in time and saves it so that it can be restored. +/// It also allows for the restoration of that state at a later moment +#[allow(unused_variables)] +pub trait WritableJournal { + /// Takes in a stream of snapshot log entries and saves them so that they + /// may be restored at a later moment + fn write<'a>(&'a self, entry: JournalEntry<'a>) -> anyhow::Result; +} + +/// The snapshot capturer will take a series of objects that represents the state of +/// a WASM process at a point in time and saves it so that it can be restored. +/// It also allows for the restoration of that state at a later moment +#[allow(unused_variables)] +pub trait ReadableJournal { + /// Returns a stream of snapshot objects that the runtime will use + /// to restore the state of a WASM process to a previous moment in time + fn read(&self) -> anyhow::Result>>; + + /// Resets the journal so that reads will start from the + /// beginning again + fn as_restarted(&self) -> anyhow::Result>; +} + +/// The snapshot capturer will take a series of objects that represents the state of +/// a WASM process at a point in time and saves it so that it can be restored. +/// It also allows for the restoration of that state at a later moment +#[allow(unused_variables)] +pub trait Journal: WritableJournal + ReadableJournal { + /// Splits the journal into a read and write side + fn split(self) -> (Box, Box); +} + +pub type DynJournal = dyn Journal + Send + Sync; +pub type DynWritableJournal = dyn WritableJournal + Send + Sync; +pub type DynReadableJournal = dyn ReadableJournal + Send + Sync; diff --git a/lib/wasix/src/journal/snapshot.rs b/lib/journal/src/snapshot.rs similarity index 100% rename from lib/wasix/src/journal/snapshot.rs rename to lib/journal/src/snapshot.rs diff --git a/lib/wasix/src/journal/util.rs b/lib/journal/src/util.rs similarity index 100% rename from lib/wasix/src/journal/util.rs rename to lib/journal/src/util.rs diff --git a/lib/wasix/Cargo.toml b/lib/wasix/Cargo.toml index e574df659e2..4c2ad56c50f 100644 --- a/lib/wasix/Cargo.toml +++ b/lib/wasix/Cargo.toml @@ -25,6 +25,7 @@ wasmer = { path = "../api", version = "=4.2.5", default-features = false, featur virtual-mio = { path = "../virtual-io", version = "0.3.0", default-features = false } virtual-fs = { path = "../virtual-fs", version = "0.11.0", default-features = false, features = ["webc-fs"] } virtual-net = { path = "../virtual-net", version = "0.6.2", default-features = false, features = ["rkyv"] } +wasmer-journal = { path = "../journal", version = "0.1.0" } wasmer-emscripten = { path = "../emscripten", version = "=4.2.5", optional = true } typetag = { version = "0.1", optional = true } serde = { version = "1.0", default-features = false, features = ["derive"] } diff --git a/lib/wasix/src/journal/effector/thread_exit.rs b/lib/wasix/src/journal/effector/thread_exit.rs index 4c2d70b31c6..eae7f8813c8 100644 --- a/lib/wasix/src/journal/effector/thread_exit.rs +++ b/lib/wasix/src/journal/effector/thread_exit.rs @@ -9,7 +9,10 @@ impl JournalEffector { exit_code: Option, ) -> anyhow::Result<()> { env.active_journal()? - .write(JournalEntry::CloseThreadV1 { id, exit_code }) + .write(JournalEntry::CloseThreadV1 { + id: id.raw(), + exit_code, + }) .map_err(map_snapshot_err)?; Ok(()) } diff --git a/lib/wasix/src/journal/effector/thread_state.rs b/lib/wasix/src/journal/effector/thread_state.rs index 510a5a28582..93b4d3cf9fe 100644 --- a/lib/wasix/src/journal/effector/thread_state.rs +++ b/lib/wasix/src/journal/effector/thread_state.rs @@ -11,7 +11,7 @@ impl JournalEffector { Self::save_event( ctx, JournalEntry::SetThreadV1 { - id, + id: id.raw(), call_stack: Cow::Owned(rewind_stack.into()), memory_stack: Cow::Owned(memory_stack.into()), store_data: Cow::Owned(store_data.into()), diff --git a/lib/wasix/src/journal/mod.rs b/lib/wasix/src/journal/mod.rs index 14b5cb5984b..34787b065f2 100644 --- a/lib/wasix/src/journal/mod.rs +++ b/lib/wasix/src/journal/mod.rs @@ -1,56 +1,8 @@ -mod base64; -mod concrete; #[cfg(feature = "journal")] mod effector; #[cfg(not(feature = "journal"))] #[path = "effector/unimplemented.rs"] mod effector; -mod entry; -mod snapshot; -mod util; -pub use concrete::*; pub use effector::*; -pub use entry::*; -pub use snapshot::*; -pub use util::*; - -use serde::{Deserialize, Serialize}; -use std::str::FromStr; - -/// The snapshot capturer will take a series of objects that represents the state of -/// a WASM process at a point in time and saves it so that it can be restored. -/// It also allows for the restoration of that state at a later moment -#[allow(unused_variables)] -pub trait WritableJournal { - /// Takes in a stream of snapshot log entries and saves them so that they - /// may be restored at a later moment - fn write<'a>(&'a self, entry: JournalEntry<'a>) -> anyhow::Result; -} - -/// The snapshot capturer will take a series of objects that represents the state of -/// a WASM process at a point in time and saves it so that it can be restored. -/// It also allows for the restoration of that state at a later moment -#[allow(unused_variables)] -pub trait ReadableJournal { - /// Returns a stream of snapshot objects that the runtime will use - /// to restore the state of a WASM process to a previous moment in time - fn read(&self) -> anyhow::Result>>; - - /// Resets the journal so that reads will start from the - /// beginning again - fn as_restarted(&self) -> anyhow::Result>; -} - -/// The snapshot capturer will take a series of objects that represents the state of -/// a WASM process at a point in time and saves it so that it can be restored. -/// It also allows for the restoration of that state at a later moment -#[allow(unused_variables)] -pub trait Journal: WritableJournal + ReadableJournal { - /// Splits the journal into a read and write side - fn split(self) -> (Box, Box); -} - -pub type DynJournal = dyn Journal + Send + Sync; -pub type DynWritableJournal = dyn WritableJournal + Send + Sync; -pub type DynReadableJournal = dyn ReadableJournal + Send + Sync; +pub use wasmer_journal::*; diff --git a/lib/wasix/src/net/socket.rs b/lib/wasix/src/net/socket.rs index 6b372bc1c3a..a338735a4d5 100644 --- a/lib/wasix/src/net/socket.rs +++ b/lib/wasix/src/net/socket.rs @@ -161,6 +161,34 @@ pub enum TimeType { Linger, } +impl Into for TimeType { + fn into(self) -> wasmer_journal::SocketOptTimeType { + use wasmer_journal::SocketOptTimeType; + match self { + TimeType::ReadTimeout => SocketOptTimeType::ReadTimeout, + TimeType::WriteTimeout => SocketOptTimeType::WriteTimeout, + TimeType::AcceptTimeout => SocketOptTimeType::AcceptTimeout, + TimeType::ConnectTimeout => SocketOptTimeType::ConnectTimeout, + TimeType::BindTimeout => SocketOptTimeType::BindTimeout, + TimeType::Linger => SocketOptTimeType::Linger, + } + } +} + +impl From for TimeType { + fn from(value: wasmer_journal::SocketOptTimeType) -> Self { + use wasmer_journal::SocketOptTimeType; + match value { + SocketOptTimeType::ReadTimeout => TimeType::ReadTimeout, + SocketOptTimeType::WriteTimeout => TimeType::WriteTimeout, + SocketOptTimeType::AcceptTimeout => TimeType::AcceptTimeout, + SocketOptTimeType::ConnectTimeout => TimeType::ConnectTimeout, + SocketOptTimeType::BindTimeout => TimeType::BindTimeout, + SocketOptTimeType::Linger => TimeType::Linger, + } + } +} + #[derive(Debug)] //#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] pub(crate) struct InodeSocketProtected { diff --git a/lib/wasix/src/runners/dcgi/runner.rs b/lib/wasix/src/runners/dcgi/runner.rs index 2681608913b..5731728d9fb 100644 --- a/lib/wasix/src/runners/dcgi/runner.rs +++ b/lib/wasix/src/runners/dcgi/runner.rs @@ -1,13 +1,14 @@ use std::{net::SocketAddr, sync::Arc}; use anyhow::Error; +use wasmer_journal::FilteredJournalBuilder; use wcgi_host::CgiDialect; use webc::metadata::Command; use crate::{ bin_factory::BinaryPackage, capabilities::Capabilities, - journal::{DynJournal, FilteredJournal}, + journal::DynJournal, runners::{ dcgi::handler::Handler, wcgi::{self, NoOpWcgiCallbacks, WcgiRunner}, @@ -77,13 +78,14 @@ impl crate::runners::Runner for DcgiRunner { .clone() .into_iter() .map(|journal| { - let journal = FilteredJournal::new(journal) + let journal = FilteredJournalBuilder::new() .with_ignore_memory(true) .with_ignore_threads(true) .with_ignore_core(true) .with_ignore_snapshots(true) .with_ignore_networking(true) - .with_ignore_stdio(true); + .with_ignore_stdio(true) + .build(journal); Arc::new(journal) as Arc }) .collect::>(); diff --git a/lib/wasix/src/syscalls/journal.rs b/lib/wasix/src/syscalls/journal.rs index a09c8b8aee2..c2040111291 100644 --- a/lib/wasix/src/syscalls/journal.rs +++ b/lib/wasix/src/syscalls/journal.rs @@ -167,7 +167,7 @@ pub unsafe fn restore_snapshot( } } crate::journal::JournalEntry::CloseThreadV1 { id, exit_code } => { - if id == ctx.data().tid() { + if id == ctx.data().tid().raw() { if bootstrapping { rewind = None; spawn_threads.clear(); @@ -184,10 +184,14 @@ pub unsafe fn restore_snapshot( .map_err(anyhow_err_to_runtime_err)?; } } else if bootstrapping { - spawn_threads.remove(&id); + spawn_threads.remove(&Into::::into(id)); } else { - JournalEffector::apply_thread_exit(&mut ctx, id, exit_code) - .map_err(anyhow_err_to_runtime_err)?; + JournalEffector::apply_thread_exit( + &mut ctx, + Into::::into(id), + exit_code, + ) + .map_err(anyhow_err_to_runtime_err)?; } } crate::journal::JournalEntry::SetThreadV1 { @@ -208,6 +212,7 @@ pub unsafe fn restore_snapshot( is_64bit, }; + let id = Into::::into(id); if id == ctx.data().tid() { rewind.replace(state); } else if bootstrapping { From 21a23ef1187ac33d6c4bda9263e16f283bd8e464 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Thu, 4 Jan 2024 11:04:19 +1100 Subject: [PATCH 126/129] Fixed some linting and compile issues from last refactor --- Cargo.lock | 2 ++ lib/journal/Cargo.toml | 8 +++++++- lib/journal/src/concrete/archived.rs | 4 ++-- lib/journal/src/concrete/log_file.rs | 8 ++++---- lib/journal/src/concrete/mod.rs | 8 ++++---- lib/wasix/Cargo.toml | 4 ++-- 6 files changed, 21 insertions(+), 13 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a9442017666..046b25962d6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6139,6 +6139,8 @@ dependencies = [ "rkyv", "serde", "serde_json", + "shared-buffer", + "tempfile", "thiserror", "tracing", "tracing-test", diff --git a/lib/journal/Cargo.toml b/lib/journal/Cargo.toml index 011a2922a21..66b22face8c 100644 --- a/lib/journal/Cargo.toml +++ b/lib/journal/Cargo.toml @@ -9,10 +9,15 @@ license.workspace = true repository.workspace = true rust-version.workspace = true +[features] +default = [ "log-file" ] +log-file = [ "shared-buffer" ] + [dependencies] wasmer-wasix-types = { path = "../wasi-types", version = "0.18.0", features = [ "enable-serde" ] } virtual-net = { path = "../virtual-net", version = "0.6.2", default-features = false, features = ["rkyv"] } +shared-buffer = { workspace = true, optional = true } thiserror = "1" bytes = "1.1" async-trait = { version = "^0.1" } @@ -29,4 +34,5 @@ num_enum = "0.5.7" serde_json = { version = "^1" } [dev-dependencies] -tracing-test = "0.2.4" \ No newline at end of file +tracing-test = "0.2.4" +tempfile = "3.6.0" \ No newline at end of file diff --git a/lib/journal/src/concrete/archived.rs b/lib/journal/src/concrete/archived.rs index 1255431c969..5912aefa277 100644 --- a/lib/journal/src/concrete/archived.rs +++ b/lib/journal/src/concrete/archived.rs @@ -464,7 +464,7 @@ impl<'a> JournalEntry<'a> { store_data, is_64bit, } => serializer.serialize_value(&JournalEntrySetThreadV1 { - id: id.into(), + id, _padding: padding(call_stack.len() + memory_stack.len() + store_data.len()), call_stack: call_stack.into_owned(), memory_stack: memory_stack.into_owned(), @@ -473,7 +473,7 @@ impl<'a> JournalEntry<'a> { }), JournalEntry::CloseThreadV1 { id, exit_code } => { serializer.serialize_value(&JournalEntryCloseThreadV1 { - id: id.into(), + id, exit_code: exit_code.map(|e| e.into()), }) } diff --git a/lib/journal/src/concrete/log_file.rs b/lib/journal/src/concrete/log_file.rs index fdffdb304a8..a254b7e7e11 100644 --- a/lib/journal/src/concrete/log_file.rs +++ b/lib/journal/src/concrete/log_file.rs @@ -271,7 +271,7 @@ mod tests { .unwrap(); journal .write(JournalEntry::SetThreadV1 { - id: 1.into(), + id: 1, call_stack: vec![11; 116].into(), memory_stack: vec![22; 16].into(), store_data: vec![33; 136].into(), @@ -293,7 +293,7 @@ mod tests { assert_eq!( event2, Some(JournalEntry::SetThreadV1 { - id: 1.into(), + id: 1, call_stack: vec![11; 116].into(), memory_stack: vec![22; 16].into(), store_data: vec![33; 136].into(), @@ -336,7 +336,7 @@ mod tests { assert_eq!( event2, Some(JournalEntry::SetThreadV1 { - id: 1.into(), + id: 1, call_stack: vec![11; 116].into(), memory_stack: vec![22; 16].into(), store_data: vec![33; 136].into(), @@ -376,7 +376,7 @@ mod tests { assert_eq!( event2, Some(JournalEntry::SetThreadV1 { - id: 1.into(), + id: 1, call_stack: vec![11; 116].into(), memory_stack: vec![22; 16].into(), store_data: vec![33; 136].into(), diff --git a/lib/journal/src/concrete/mod.rs b/lib/journal/src/concrete/mod.rs index f51d780000f..f6b666d8be1 100644 --- a/lib/journal/src/concrete/mod.rs +++ b/lib/journal/src/concrete/mod.rs @@ -3,11 +3,11 @@ mod archived; mod boxed; mod buffered; mod compacting; -#[cfg(feature = "journal")] +#[cfg(feature = "log-file")] mod compacting_log_file; mod counting; mod filter; -#[cfg(feature = "journal")] +#[cfg(feature = "log-file")] mod log_file; mod null; mod pipe; @@ -22,11 +22,11 @@ pub use archived::*; pub use boxed::*; pub use buffered::*; pub use compacting::*; -#[cfg(feature = "journal")] +#[cfg(feature = "log-file")] pub use compacting_log_file::*; pub use counting::*; pub use filter::*; -#[cfg(feature = "journal")] +#[cfg(feature = "log-file")] pub use log_file::*; pub use null::*; pub use pipe::*; diff --git a/lib/wasix/Cargo.toml b/lib/wasix/Cargo.toml index 4c2ad56c50f..f623e774bd4 100644 --- a/lib/wasix/Cargo.toml +++ b/lib/wasix/Cargo.toml @@ -25,7 +25,7 @@ wasmer = { path = "../api", version = "=4.2.5", default-features = false, featur virtual-mio = { path = "../virtual-io", version = "0.3.0", default-features = false } virtual-fs = { path = "../virtual-fs", version = "0.11.0", default-features = false, features = ["webc-fs"] } virtual-net = { path = "../virtual-net", version = "0.6.2", default-features = false, features = ["rkyv"] } -wasmer-journal = { path = "../journal", version = "0.1.0" } +wasmer-journal = { path = "../journal", version = "0.1.0", default-features = false } wasmer-emscripten = { path = "../emscripten", version = "=4.2.5", optional = true } typetag = { version = "0.1", optional = true } serde = { version = "1.0", default-features = false, features = ["derive"] } @@ -158,7 +158,7 @@ sys-default = [ ] sys-poll = [] sys-thread = ["tokio/rt", "tokio/time", "tokio/rt-multi-thread", "rusty_pool"] -journal = ["tokio/fs"] +journal = ["tokio/fs", "wasmer-journal/log-file"] # Deprecated. Kept it for compatibility compiler = [] From 77bb3cce25007226a58f0f73eddefb3f0e4bfa81 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Thu, 4 Jan 2024 11:22:11 +1100 Subject: [PATCH 127/129] More linting fixes --- lib/cli/src/commands/journal/filter.rs | 21 ++++++++++++--------- lib/wasix/src/net/socket.rs | 19 +++++++++---------- 2 files changed, 21 insertions(+), 19 deletions(-) diff --git a/lib/cli/src/commands/journal/filter.rs b/lib/cli/src/commands/journal/filter.rs index 4e0a4b5ea62..815078ec354 100644 --- a/lib/cli/src/commands/journal/filter.rs +++ b/lib/cli/src/commands/journal/filter.rs @@ -2,7 +2,9 @@ use std::{path::PathBuf, str::FromStr}; use clap::Parser; use wasmer_edge_cli::cmd::AsyncCliCommand; -use wasmer_wasix::journal::{copy_journal, FilteredJournal, LogFileJournal, PrintingJournal}; +use wasmer_wasix::journal::{ + copy_journal, FilteredJournalBuilder, LogFileJournal, PrintingJournal, +}; /// Flags that specify what should be filtered out #[derive(Debug, Clone, Copy, PartialOrd, Ord, PartialEq, Eq, Hash)] @@ -88,17 +90,18 @@ impl CmdJournalFilter { let target = LogFileJournal::new(temp_path.clone())?; // Put a filter on the farget - let mut target = FilteredJournal::new(target); + let mut builder = FilteredJournalBuilder::new(); for filter in self.filters { - target = match filter { - FilterOut::Memory => target.with_ignore_memory(true), - FilterOut::Threads => target.with_ignore_threads(true), - FilterOut::FileSystem => target.with_ignore_fs(true), - FilterOut::Core => target.with_ignore_core(true), - FilterOut::Snapshots => target.with_ignore_snapshots(true), - FilterOut::Networking => target.with_ignore_networking(true), + builder = match filter { + FilterOut::Memory => builder.with_ignore_memory(true), + FilterOut::Threads => builder.with_ignore_threads(true), + FilterOut::FileSystem => builder.with_ignore_fs(true), + FilterOut::Core => builder.with_ignore_core(true), + FilterOut::Snapshots => builder.with_ignore_snapshots(true), + FilterOut::Networking => builder.with_ignore_networking(true), } } + let target = builder.build(target); // Copy the journal over and rename the temp file to the target file copy_journal(&source, &target)?; diff --git a/lib/wasix/src/net/socket.rs b/lib/wasix/src/net/socket.rs index a338735a4d5..958a01578b1 100644 --- a/lib/wasix/src/net/socket.rs +++ b/lib/wasix/src/net/socket.rs @@ -161,16 +161,15 @@ pub enum TimeType { Linger, } -impl Into for TimeType { - fn into(self) -> wasmer_journal::SocketOptTimeType { - use wasmer_journal::SocketOptTimeType; - match self { - TimeType::ReadTimeout => SocketOptTimeType::ReadTimeout, - TimeType::WriteTimeout => SocketOptTimeType::WriteTimeout, - TimeType::AcceptTimeout => SocketOptTimeType::AcceptTimeout, - TimeType::ConnectTimeout => SocketOptTimeType::ConnectTimeout, - TimeType::BindTimeout => SocketOptTimeType::BindTimeout, - TimeType::Linger => SocketOptTimeType::Linger, +impl From for wasmer_journal::SocketOptTimeType { + fn from(value: TimeType) -> Self { + match value { + TimeType::ReadTimeout => Self::ReadTimeout, + TimeType::WriteTimeout => Self::WriteTimeout, + TimeType::AcceptTimeout => Self::AcceptTimeout, + TimeType::ConnectTimeout => Self::ConnectTimeout, + TimeType::BindTimeout => Self::BindTimeout, + TimeType::Linger => Self::Linger, } } } From d80d469739115c6bb4c3763b8438719ec70af429 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Thu, 4 Jan 2024 22:36:42 +1100 Subject: [PATCH 128/129] No longer re-exporting standard types in virtual-net --- lib/journal/src/concrete/archived.rs | 5 +++-- lib/journal/src/entry.rs | 5 +++-- lib/virtual-net/src/lib.rs | 10 +++++----- lib/virtual-net/src/meta.rs | 11 ++++++----- .../src/journal/effector/syscalls/port_addr_remove.rs | 2 +- .../src/journal/effector/syscalls/port_gateway_set.rs | 2 +- .../src/journal/effector/syscalls/port_route_add.rs | 3 ++- .../journal/effector/syscalls/port_route_remove.rs | 2 +- .../src/journal/effector/syscalls/sock_accept.rs | 2 +- lib/wasix/src/journal/effector/syscalls/sock_bind.rs | 2 +- .../src/journal/effector/syscalls/sock_connect.rs | 2 +- .../effector/syscalls/sock_join_ipv4_multicast.rs | 2 +- .../effector/syscalls/sock_join_ipv6_multicast.rs | 2 +- .../effector/syscalls/sock_leave_ipv4_multicast.rs | 2 +- .../effector/syscalls/sock_leave_ipv6_multicast.rs | 2 +- .../src/journal/effector/syscalls/sock_send_to.rs | 2 +- .../journal/effector/syscalls/sock_set_opt_time.rs | 2 +- 17 files changed, 31 insertions(+), 27 deletions(-) diff --git a/lib/journal/src/concrete/archived.rs b/lib/journal/src/concrete/archived.rs index 5912aefa277..f9462e1e67f 100644 --- a/lib/journal/src/concrete/archived.rs +++ b/lib/journal/src/concrete/archived.rs @@ -3,8 +3,9 @@ use num_enum::{IntoPrimitive, TryFromPrimitive}; use rkyv::ser::{ScratchSpace, Serializer}; use rkyv::{Archive, CheckBytes, Deserialize as RkyvDeserialize, Serialize as RkyvSerialize}; use std::borrow::Cow; -use std::time::SystemTime; -use virtual_net::{Duration, IpAddr, IpCidr, Ipv4Addr, Ipv6Addr, SocketAddr, StreamSecurity}; +use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr}; +use std::time::{Duration, SystemTime}; +use virtual_net::{IpCidr, StreamSecurity}; use wasmer_wasix_types::wasi::{self, EpollEventCtl, EpollType, Fdflags, Rights, Sockoption}; use super::*; diff --git a/lib/journal/src/entry.rs b/lib/journal/src/entry.rs index 336a7d26d62..aace436d22d 100644 --- a/lib/journal/src/entry.rs +++ b/lib/journal/src/entry.rs @@ -1,9 +1,10 @@ use derivative::Derivative; use serde::{Deserialize, Serialize}; +use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; use std::net::{Shutdown, SocketAddr}; -use std::time::SystemTime; +use std::time::{Duration, SystemTime}; use std::{borrow::Cow, ops::Range}; -use virtual_net::{Duration, IpAddr, IpCidr, Ipv4Addr, Ipv6Addr, StreamSecurity}; +use virtual_net::{IpCidr, StreamSecurity}; use wasmer_wasix_types::wasi::{ Addressfamily, Advice, EpollCtl, EpollEventCtl, EventFdFlags, ExitCode, Fdflags, FileDelta, Filesize, Fstflags, LookupFlags, Oflags, Rights, SiFlags, Snapshot0Clockid, SockProto, diff --git a/lib/virtual-net/src/lib.rs b/lib/virtual-net/src/lib.rs index c59f613f083..fd0ff3b0a00 100644 --- a/lib/virtual-net/src/lib.rs +++ b/lib/virtual-net/src/lib.rs @@ -22,16 +22,16 @@ use rkyv::{Archive, CheckBytes, Deserialize as RkyvDeserialize, Serialize as Rky pub use server::{RemoteNetworkingServer, RemoteNetworkingServerDriver}; use std::fmt; use std::mem::MaybeUninit; -pub use std::net::IpAddr; -pub use std::net::Ipv4Addr; -pub use std::net::Ipv6Addr; +use std::net::IpAddr; +use std::net::Ipv4Addr; +use std::net::Ipv6Addr; use std::net::Shutdown; -pub use std::net::SocketAddr; +use std::net::SocketAddr; use std::pin::Pin; use std::sync::Arc; use std::task::Context; use std::task::Poll; -pub use std::time::Duration; +use std::time::Duration; use thiserror::Error; #[cfg(feature = "tokio")] use tokio::io::AsyncRead; diff --git a/lib/virtual-net/src/meta.rs b/lib/virtual-net/src/meta.rs index 8744f0af9ca..c8875c9c81b 100644 --- a/lib/virtual-net/src/meta.rs +++ b/lib/virtual-net/src/meta.rs @@ -1,13 +1,14 @@ +use std::net::IpAddr; +use std::net::Ipv4Addr; +use std::net::Ipv6Addr; +use std::net::SocketAddr; +use std::time::Duration; + use serde::{Deserialize, Serialize}; -pub use super::Duration; -pub use super::IpAddr; pub use super::IpCidr; pub use super::IpRoute; -pub use super::Ipv4Addr; -pub use super::Ipv6Addr; pub use super::NetworkError; -pub use super::SocketAddr; pub use super::SocketStatus; pub use super::StreamSecurity; diff --git a/lib/wasix/src/journal/effector/syscalls/port_addr_remove.rs b/lib/wasix/src/journal/effector/syscalls/port_addr_remove.rs index 1463d2595cb..b2d46f926db 100644 --- a/lib/wasix/src/journal/effector/syscalls/port_addr_remove.rs +++ b/lib/wasix/src/journal/effector/syscalls/port_addr_remove.rs @@ -1,4 +1,4 @@ -use virtual_net::IpAddr; +use std::net::IpAddr; use super::*; diff --git a/lib/wasix/src/journal/effector/syscalls/port_gateway_set.rs b/lib/wasix/src/journal/effector/syscalls/port_gateway_set.rs index 230eba8d467..a0251a02e88 100644 --- a/lib/wasix/src/journal/effector/syscalls/port_gateway_set.rs +++ b/lib/wasix/src/journal/effector/syscalls/port_gateway_set.rs @@ -1,4 +1,4 @@ -use virtual_net::IpAddr; +use std::net::IpAddr; use super::*; diff --git a/lib/wasix/src/journal/effector/syscalls/port_route_add.rs b/lib/wasix/src/journal/effector/syscalls/port_route_add.rs index 0565e8b8f09..e2f44eb84e1 100644 --- a/lib/wasix/src/journal/effector/syscalls/port_route_add.rs +++ b/lib/wasix/src/journal/effector/syscalls/port_route_add.rs @@ -1,4 +1,5 @@ -use virtual_net::{Duration, IpAddr, IpCidr}; +use std::{net::IpAddr, time::Duration}; +use virtual_net::IpCidr; use super::*; diff --git a/lib/wasix/src/journal/effector/syscalls/port_route_remove.rs b/lib/wasix/src/journal/effector/syscalls/port_route_remove.rs index 61d371b8da1..3a980bfff65 100644 --- a/lib/wasix/src/journal/effector/syscalls/port_route_remove.rs +++ b/lib/wasix/src/journal/effector/syscalls/port_route_remove.rs @@ -1,4 +1,4 @@ -use virtual_net::IpAddr; +use std::net::IpAddr; use super::*; diff --git a/lib/wasix/src/journal/effector/syscalls/sock_accept.rs b/lib/wasix/src/journal/effector/syscalls/sock_accept.rs index 2bbbf3b382d..3482074dcfe 100644 --- a/lib/wasix/src/journal/effector/syscalls/sock_accept.rs +++ b/lib/wasix/src/journal/effector/syscalls/sock_accept.rs @@ -1,4 +1,4 @@ -use virtual_net::SocketAddr; +use std::net::SocketAddr; use crate::{ fs::Kind, diff --git a/lib/wasix/src/journal/effector/syscalls/sock_bind.rs b/lib/wasix/src/journal/effector/syscalls/sock_bind.rs index fc0727e788b..73c3e3a7b81 100644 --- a/lib/wasix/src/journal/effector/syscalls/sock_bind.rs +++ b/lib/wasix/src/journal/effector/syscalls/sock_bind.rs @@ -1,4 +1,4 @@ -use virtual_net::SocketAddr; +use std::net::SocketAddr; use super::*; diff --git a/lib/wasix/src/journal/effector/syscalls/sock_connect.rs b/lib/wasix/src/journal/effector/syscalls/sock_connect.rs index 5e4b2f89b6b..3c7af982efd 100644 --- a/lib/wasix/src/journal/effector/syscalls/sock_connect.rs +++ b/lib/wasix/src/journal/effector/syscalls/sock_connect.rs @@ -1,4 +1,4 @@ -use virtual_net::SocketAddr; +use std::net::SocketAddr; use crate::{ fs::Kind, diff --git a/lib/wasix/src/journal/effector/syscalls/sock_join_ipv4_multicast.rs b/lib/wasix/src/journal/effector/syscalls/sock_join_ipv4_multicast.rs index a1c4d40005d..589bb08cafd 100644 --- a/lib/wasix/src/journal/effector/syscalls/sock_join_ipv4_multicast.rs +++ b/lib/wasix/src/journal/effector/syscalls/sock_join_ipv4_multicast.rs @@ -1,4 +1,4 @@ -use virtual_net::Ipv4Addr; +use std::net::Ipv4Addr; use super::*; diff --git a/lib/wasix/src/journal/effector/syscalls/sock_join_ipv6_multicast.rs b/lib/wasix/src/journal/effector/syscalls/sock_join_ipv6_multicast.rs index d130e0391f1..17e7d875799 100644 --- a/lib/wasix/src/journal/effector/syscalls/sock_join_ipv6_multicast.rs +++ b/lib/wasix/src/journal/effector/syscalls/sock_join_ipv6_multicast.rs @@ -1,4 +1,4 @@ -use virtual_net::Ipv6Addr; +use std::net::Ipv6Addr; use super::*; diff --git a/lib/wasix/src/journal/effector/syscalls/sock_leave_ipv4_multicast.rs b/lib/wasix/src/journal/effector/syscalls/sock_leave_ipv4_multicast.rs index 50abd7d6255..9fd3b78eba5 100644 --- a/lib/wasix/src/journal/effector/syscalls/sock_leave_ipv4_multicast.rs +++ b/lib/wasix/src/journal/effector/syscalls/sock_leave_ipv4_multicast.rs @@ -1,4 +1,4 @@ -use virtual_net::Ipv4Addr; +use std::net::Ipv4Addr; use super::*; diff --git a/lib/wasix/src/journal/effector/syscalls/sock_leave_ipv6_multicast.rs b/lib/wasix/src/journal/effector/syscalls/sock_leave_ipv6_multicast.rs index 6d7fad5fea1..b91f71093bc 100644 --- a/lib/wasix/src/journal/effector/syscalls/sock_leave_ipv6_multicast.rs +++ b/lib/wasix/src/journal/effector/syscalls/sock_leave_ipv6_multicast.rs @@ -1,4 +1,4 @@ -use virtual_net::Ipv6Addr; +use std::net::Ipv6Addr; use super::*; diff --git a/lib/wasix/src/journal/effector/syscalls/sock_send_to.rs b/lib/wasix/src/journal/effector/syscalls/sock_send_to.rs index 7df19d13eb0..f1e7608e5d6 100644 --- a/lib/wasix/src/journal/effector/syscalls/sock_send_to.rs +++ b/lib/wasix/src/journal/effector/syscalls/sock_send_to.rs @@ -1,4 +1,4 @@ -use virtual_net::SocketAddr; +use std::net::SocketAddr; use wasmer_wasix_types::wasi::SiFlags; use crate::syscalls::sock_send_to_internal; diff --git a/lib/wasix/src/journal/effector/syscalls/sock_set_opt_time.rs b/lib/wasix/src/journal/effector/syscalls/sock_set_opt_time.rs index 75574fb5cde..0dbdf7ab339 100644 --- a/lib/wasix/src/journal/effector/syscalls/sock_set_opt_time.rs +++ b/lib/wasix/src/journal/effector/syscalls/sock_set_opt_time.rs @@ -1,4 +1,4 @@ -use virtual_net::Duration; +use std::time::Duration; use crate::net::socket::TimeType; From b6ef7dd65ab55ba9574768db8062a39d07984cf5 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Thu, 4 Jan 2024 23:43:20 +1100 Subject: [PATCH 129/129] Removed the wasix and virtual-net dependencies from the archived events --- Cargo.lock | 1 + lib/journal/Cargo.toml | 3 +- lib/journal/src/concrete/archived.rs | 1808 +-------------------- lib/journal/src/concrete/archived_from.rs | 1107 +++++++++++++ lib/journal/src/concrete/mod.rs | 3 + lib/journal/src/concrete/tests.rs | 715 ++++++++ 6 files changed, 1840 insertions(+), 1797 deletions(-) create mode 100644 lib/journal/src/concrete/archived_from.rs create mode 100644 lib/journal/src/concrete/tests.rs diff --git a/Cargo.lock b/Cargo.lock index 046b25962d6..5d8acc2bce7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6145,6 +6145,7 @@ dependencies = [ "tracing", "tracing-test", "virtual-net 0.6.2", + "wasmer", "wasmer-wasix-types", ] diff --git a/lib/journal/Cargo.toml b/lib/journal/Cargo.toml index 66b22face8c..452454aef47 100644 --- a/lib/journal/Cargo.toml +++ b/lib/journal/Cargo.toml @@ -10,10 +10,11 @@ repository.workspace = true rust-version.workspace = true [features] -default = [ "log-file" ] +default = [ "log-file", "wasmer/sys" ] log-file = [ "shared-buffer" ] [dependencies] +wasmer = { default-features = false, path = "../api", version = "=4.2.5" } wasmer-wasix-types = { path = "../wasi-types", version = "0.18.0", features = [ "enable-serde" ] } virtual-net = { path = "../virtual-net", version = "0.6.2", default-features = false, features = ["rkyv"] } diff --git a/lib/journal/src/concrete/archived.rs b/lib/journal/src/concrete/archived.rs index f9462e1e67f..ad0cbf98a88 100644 --- a/lib/journal/src/concrete/archived.rs +++ b/lib/journal/src/concrete/archived.rs @@ -1,12 +1,9 @@ -use lz4_flex::block::{compress_prepend_size, decompress_size_prepended}; +use lz4_flex::block::compress_prepend_size; use num_enum::{IntoPrimitive, TryFromPrimitive}; use rkyv::ser::{ScratchSpace, Serializer}; use rkyv::{Archive, CheckBytes, Deserialize as RkyvDeserialize, Serialize as RkyvSerialize}; -use std::borrow::Cow; use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr}; use std::time::{Duration, SystemTime}; -use virtual_net::{IpCidr, StreamSecurity}; -use wasmer_wasix_types::wasi::{self, EpollEventCtl, EpollType, Fdflags, Rights, Sockoption}; use super::*; @@ -696,7 +693,7 @@ impl<'a> JournalEntry<'a> { fd, }), JournalEntry::PortAddAddrV1 { cidr } => { - serializer.serialize_value(&JournalEntryPortAddAddrV1 { cidr }) + serializer.serialize_value(&JournalEntryPortAddAddrV1 { cidr: cidr.into() }) } JournalEntry::PortDelAddrV1 { addr } => { serializer.serialize_value(&JournalEntryPortDelAddrV1 { addr }) @@ -723,7 +720,7 @@ impl<'a> JournalEntry<'a> { preferred_until, expires_at, } => serializer.serialize_value(&JournalEntryPortRouteAddV1 { - cidr, + cidr: cidr.into(), via_router, preferred_until, expires_at, @@ -1283,7 +1280,7 @@ pub struct JournalEntryCreateEventV1 { #[derive(Debug, Clone, RkyvSerialize, RkyvDeserialize, Archive)] #[archive_attr(derive(CheckBytes))] pub struct JournalEntryPortAddAddrV1 { - pub cidr: IpCidr, + pub cidr: JournalIpCidrV1, } #[repr(C)] @@ -1318,7 +1315,7 @@ pub struct JournalEntryPortGatewaySetV1 { #[derive(Debug, Clone, RkyvSerialize, RkyvDeserialize, Archive)] #[archive_attr(derive(CheckBytes))] pub struct JournalEntryPortRouteAddV1 { - pub cidr: IpCidr, + pub cidr: JournalIpCidrV1, pub via_router: IpAddr, pub preferred_until: Option, pub expires_at: Option, @@ -1517,46 +1514,6 @@ pub enum JournalSnapshot0ClockidV1 { Unknown = 255, } -impl From for JournalSnapshot0ClockidV1 { - fn from(val: wasi::Snapshot0Clockid) -> Self { - match val { - wasi::Snapshot0Clockid::Realtime => JournalSnapshot0ClockidV1::Realtime, - wasi::Snapshot0Clockid::Monotonic => JournalSnapshot0ClockidV1::Monotonic, - wasi::Snapshot0Clockid::ProcessCputimeId => JournalSnapshot0ClockidV1::ProcessCputimeId, - wasi::Snapshot0Clockid::ThreadCputimeId => JournalSnapshot0ClockidV1::ThreadCputimeId, - wasi::Snapshot0Clockid::Unknown => JournalSnapshot0ClockidV1::Unknown, - } - } -} - -impl From for wasi::Snapshot0Clockid { - fn from(val: JournalSnapshot0ClockidV1) -> Self { - match val { - JournalSnapshot0ClockidV1::Realtime => wasi::Snapshot0Clockid::Realtime, - JournalSnapshot0ClockidV1::Monotonic => wasi::Snapshot0Clockid::Monotonic, - JournalSnapshot0ClockidV1::ProcessCputimeId => wasi::Snapshot0Clockid::ProcessCputimeId, - JournalSnapshot0ClockidV1::ThreadCputimeId => wasi::Snapshot0Clockid::ThreadCputimeId, - JournalSnapshot0ClockidV1::Unknown => wasi::Snapshot0Clockid::Unknown, - } - } -} - -impl From<&'_ ArchivedJournalSnapshot0ClockidV1> for wasi::Snapshot0Clockid { - fn from(val: &'_ ArchivedJournalSnapshot0ClockidV1) -> Self { - match val { - ArchivedJournalSnapshot0ClockidV1::Realtime => wasi::Snapshot0Clockid::Realtime, - ArchivedJournalSnapshot0ClockidV1::Monotonic => wasi::Snapshot0Clockid::Monotonic, - ArchivedJournalSnapshot0ClockidV1::ProcessCputimeId => { - wasi::Snapshot0Clockid::ProcessCputimeId - } - ArchivedJournalSnapshot0ClockidV1::ThreadCputimeId => { - wasi::Snapshot0Clockid::ThreadCputimeId - } - ArchivedJournalSnapshot0ClockidV1::Unknown => wasi::Snapshot0Clockid::Unknown, - } - } -} - #[repr(C)] #[derive(Debug, Clone, RkyvSerialize, RkyvDeserialize, Archive)] #[archive_attr(derive(CheckBytes))] @@ -1567,39 +1524,6 @@ pub enum JournalWhenceV1 { Unknown = 255, } -impl From for JournalWhenceV1 { - fn from(val: wasi::Whence) -> Self { - match val { - wasi::Whence::Set => JournalWhenceV1::Set, - wasi::Whence::Cur => JournalWhenceV1::Cur, - wasi::Whence::End => JournalWhenceV1::End, - wasi::Whence::Unknown => JournalWhenceV1::Unknown, - } - } -} - -impl From for wasi::Whence { - fn from(val: JournalWhenceV1) -> Self { - match val { - JournalWhenceV1::Set => wasi::Whence::Set, - JournalWhenceV1::Cur => wasi::Whence::Cur, - JournalWhenceV1::End => wasi::Whence::End, - JournalWhenceV1::Unknown => wasi::Whence::Unknown, - } - } -} - -impl From<&'_ ArchivedJournalWhenceV1> for wasi::Whence { - fn from(val: &'_ ArchivedJournalWhenceV1) -> Self { - match val { - ArchivedJournalWhenceV1::Set => wasi::Whence::Set, - ArchivedJournalWhenceV1::Cur => wasi::Whence::Cur, - ArchivedJournalWhenceV1::End => wasi::Whence::End, - ArchivedJournalWhenceV1::Unknown => wasi::Whence::Unknown, - } - } -} - #[repr(C)] #[derive(Debug, Clone, RkyvSerialize, RkyvDeserialize, Archive)] #[archive_attr(derive(CheckBytes))] @@ -1613,46 +1537,13 @@ pub enum JournalAdviceV1 { Unknown = 255, } -impl From for JournalAdviceV1 { - fn from(val: wasi::Advice) -> Self { - match val { - wasi::Advice::Normal => JournalAdviceV1::Normal, - wasi::Advice::Sequential => JournalAdviceV1::Sequential, - wasi::Advice::Random => JournalAdviceV1::Random, - wasi::Advice::Willneed => JournalAdviceV1::Willneed, - wasi::Advice::Dontneed => JournalAdviceV1::Dontneed, - wasi::Advice::Noreuse => JournalAdviceV1::Noreuse, - wasi::Advice::Unknown => JournalAdviceV1::Unknown, - } - } -} - -impl From for wasi::Advice { - fn from(val: JournalAdviceV1) -> Self { - match val { - JournalAdviceV1::Normal => wasi::Advice::Normal, - JournalAdviceV1::Sequential => wasi::Advice::Sequential, - JournalAdviceV1::Random => wasi::Advice::Random, - JournalAdviceV1::Willneed => wasi::Advice::Willneed, - JournalAdviceV1::Dontneed => wasi::Advice::Dontneed, - JournalAdviceV1::Noreuse => wasi::Advice::Noreuse, - JournalAdviceV1::Unknown => wasi::Advice::Unknown, - } - } -} - -impl From<&'_ ArchivedJournalAdviceV1> for wasi::Advice { - fn from(val: &'_ ArchivedJournalAdviceV1) -> Self { - match val { - ArchivedJournalAdviceV1::Normal => wasi::Advice::Normal, - ArchivedJournalAdviceV1::Sequential => wasi::Advice::Sequential, - ArchivedJournalAdviceV1::Random => wasi::Advice::Random, - ArchivedJournalAdviceV1::Willneed => wasi::Advice::Willneed, - ArchivedJournalAdviceV1::Dontneed => wasi::Advice::Dontneed, - ArchivedJournalAdviceV1::Noreuse => wasi::Advice::Noreuse, - ArchivedJournalAdviceV1::Unknown => wasi::Advice::Unknown, - } - } +#[repr(C)] +#[repr(align(8))] +#[derive(Debug, Clone, RkyvSerialize, RkyvDeserialize, Archive)] +#[archive_attr(derive(CheckBytes))] +pub struct JournalIpCidrV1 { + pub ip: IpAddr, + pub prefix: u8, } #[repr(C)] @@ -1663,37 +1554,6 @@ pub enum JournalExitCodeV1 { Other(i32), } -impl From for JournalExitCodeV1 { - fn from(val: wasi::ExitCode) -> Self { - match val { - wasi::ExitCode::Errno(errno) => JournalExitCodeV1::Errno(errno as u16), - wasi::ExitCode::Other(id) => JournalExitCodeV1::Other(id), - } - } -} - -impl From for wasi::ExitCode { - fn from(val: JournalExitCodeV1) -> Self { - match val { - JournalExitCodeV1::Errno(errno) => { - wasi::ExitCode::Errno(errno.try_into().unwrap_or(wasi::Errno::Unknown)) - } - JournalExitCodeV1::Other(id) => wasi::ExitCode::Other(id), - } - } -} - -impl From<&'_ ArchivedJournalExitCodeV1> for wasi::ExitCode { - fn from(val: &'_ ArchivedJournalExitCodeV1) -> Self { - match val { - ArchivedJournalExitCodeV1::Errno(errno) => { - wasi::ExitCode::Errno((*errno).try_into().unwrap_or(wasi::Errno::Unknown)) - } - ArchivedJournalExitCodeV1::Other(id) => wasi::ExitCode::Other(*id), - } - } -} - #[repr(C)] #[derive( Debug, @@ -1722,59 +1582,6 @@ pub enum JournalSnapshotTriggerV1 { NonDeterministicCall, } -impl From for JournalSnapshotTriggerV1 { - fn from(val: SnapshotTrigger) -> Self { - match val { - SnapshotTrigger::Idle => JournalSnapshotTriggerV1::Idle, - SnapshotTrigger::FirstListen => JournalSnapshotTriggerV1::Listen, - SnapshotTrigger::FirstEnviron => JournalSnapshotTriggerV1::Environ, - SnapshotTrigger::FirstStdin => JournalSnapshotTriggerV1::Stdin, - SnapshotTrigger::PeriodicInterval => JournalSnapshotTriggerV1::Timer, - SnapshotTrigger::Sigint => JournalSnapshotTriggerV1::Sigint, - SnapshotTrigger::Sigalrm => JournalSnapshotTriggerV1::Sigalrm, - SnapshotTrigger::Sigtstp => JournalSnapshotTriggerV1::Sigtstp, - SnapshotTrigger::Sigstop => JournalSnapshotTriggerV1::Sigstop, - SnapshotTrigger::NonDeterministicCall => JournalSnapshotTriggerV1::NonDeterministicCall, - } - } -} - -impl From for SnapshotTrigger { - fn from(val: JournalSnapshotTriggerV1) -> Self { - match val { - JournalSnapshotTriggerV1::Idle => SnapshotTrigger::Idle, - JournalSnapshotTriggerV1::Listen => SnapshotTrigger::FirstListen, - JournalSnapshotTriggerV1::Environ => SnapshotTrigger::FirstEnviron, - JournalSnapshotTriggerV1::Stdin => SnapshotTrigger::FirstStdin, - JournalSnapshotTriggerV1::Timer => SnapshotTrigger::PeriodicInterval, - JournalSnapshotTriggerV1::Sigint => SnapshotTrigger::Sigint, - JournalSnapshotTriggerV1::Sigalrm => SnapshotTrigger::Sigalrm, - JournalSnapshotTriggerV1::Sigtstp => SnapshotTrigger::Sigtstp, - JournalSnapshotTriggerV1::Sigstop => SnapshotTrigger::Sigstop, - JournalSnapshotTriggerV1::NonDeterministicCall => SnapshotTrigger::NonDeterministicCall, - } - } -} - -impl From<&'_ ArchivedJournalSnapshotTriggerV1> for SnapshotTrigger { - fn from(val: &'_ ArchivedJournalSnapshotTriggerV1) -> Self { - match val { - ArchivedJournalSnapshotTriggerV1::Idle => SnapshotTrigger::Idle, - ArchivedJournalSnapshotTriggerV1::Listen => SnapshotTrigger::FirstListen, - ArchivedJournalSnapshotTriggerV1::Environ => SnapshotTrigger::FirstEnviron, - ArchivedJournalSnapshotTriggerV1::Stdin => SnapshotTrigger::FirstStdin, - ArchivedJournalSnapshotTriggerV1::Timer => SnapshotTrigger::PeriodicInterval, - ArchivedJournalSnapshotTriggerV1::Sigint => SnapshotTrigger::Sigint, - ArchivedJournalSnapshotTriggerV1::Sigalrm => SnapshotTrigger::Sigalrm, - ArchivedJournalSnapshotTriggerV1::Sigtstp => SnapshotTrigger::Sigtstp, - ArchivedJournalSnapshotTriggerV1::Sigstop => SnapshotTrigger::Sigstop, - ArchivedJournalSnapshotTriggerV1::NonDeterministicCall => { - SnapshotTrigger::NonDeterministicCall - } - } - } -} - #[repr(C)] #[derive( Debug, @@ -1797,39 +1604,6 @@ pub enum JournalEpollCtlV1 { Unknown, } -impl From for JournalEpollCtlV1 { - fn from(val: wasi::EpollCtl) -> Self { - match val { - wasi::EpollCtl::Add => JournalEpollCtlV1::Add, - wasi::EpollCtl::Mod => JournalEpollCtlV1::Mod, - wasi::EpollCtl::Del => JournalEpollCtlV1::Del, - wasi::EpollCtl::Unknown => JournalEpollCtlV1::Unknown, - } - } -} - -impl From for wasi::EpollCtl { - fn from(val: JournalEpollCtlV1) -> Self { - match val { - JournalEpollCtlV1::Add => wasi::EpollCtl::Add, - JournalEpollCtlV1::Mod => wasi::EpollCtl::Mod, - JournalEpollCtlV1::Del => wasi::EpollCtl::Del, - JournalEpollCtlV1::Unknown => wasi::EpollCtl::Unknown, - } - } -} - -impl From<&'_ ArchivedJournalEpollCtlV1> for wasi::EpollCtl { - fn from(val: &'_ ArchivedJournalEpollCtlV1) -> Self { - match val { - ArchivedJournalEpollCtlV1::Add => wasi::EpollCtl::Add, - ArchivedJournalEpollCtlV1::Mod => wasi::EpollCtl::Mod, - ArchivedJournalEpollCtlV1::Del => wasi::EpollCtl::Del, - ArchivedJournalEpollCtlV1::Unknown => wasi::EpollCtl::Unknown, - } - } -} - #[repr(C)] #[repr(align(8))] #[derive(Debug, Clone, RkyvSerialize, RkyvDeserialize, Archive)] @@ -1842,42 +1616,6 @@ pub struct JournalEpollEventCtlV1 { pub data2: u64, } -impl From for JournalEpollEventCtlV1 { - fn from(val: EpollEventCtl) -> Self { - JournalEpollEventCtlV1 { - events: val.events.bits(), - ptr: val.ptr, - fd: val.fd, - data1: val.data1, - data2: val.data2, - } - } -} - -impl From for EpollEventCtl { - fn from(val: JournalEpollEventCtlV1) -> Self { - Self { - events: EpollType::from_bits_truncate(val.events), - ptr: val.ptr, - fd: val.fd, - data1: val.data1, - data2: val.data2, - } - } -} - -impl From<&'_ ArchivedJournalEpollEventCtlV1> for EpollEventCtl { - fn from(val: &'_ ArchivedJournalEpollEventCtlV1) -> Self { - Self { - events: EpollType::from_bits_truncate(val.events), - ptr: val.ptr, - fd: val.fd, - data1: val.data1, - data2: val.data2, - } - } -} - #[repr(C)] #[derive( Debug, @@ -1901,41 +1639,6 @@ pub enum JournalStreamSecurityV1 { Unknown, } -impl From for JournalStreamSecurityV1 { - fn from(val: StreamSecurity) -> Self { - match val { - StreamSecurity::Unencrypted => JournalStreamSecurityV1::Unencrypted, - StreamSecurity::AnyEncyption => JournalStreamSecurityV1::AnyEncryption, - StreamSecurity::ClassicEncryption => JournalStreamSecurityV1::ClassicEncryption, - StreamSecurity::DoubleEncryption => JournalStreamSecurityV1::DoubleEncryption, - } - } -} - -impl From for StreamSecurity { - fn from(val: JournalStreamSecurityV1) -> Self { - match val { - JournalStreamSecurityV1::Unencrypted => StreamSecurity::Unencrypted, - JournalStreamSecurityV1::AnyEncryption => StreamSecurity::AnyEncyption, - JournalStreamSecurityV1::ClassicEncryption => StreamSecurity::ClassicEncryption, - JournalStreamSecurityV1::DoubleEncryption => StreamSecurity::DoubleEncryption, - JournalStreamSecurityV1::Unknown => StreamSecurity::AnyEncyption, - } - } -} - -impl From<&'_ ArchivedJournalStreamSecurityV1> for StreamSecurity { - fn from(val: &'_ ArchivedJournalStreamSecurityV1) -> Self { - match val { - ArchivedJournalStreamSecurityV1::Unencrypted => StreamSecurity::Unencrypted, - ArchivedJournalStreamSecurityV1::AnyEncryption => StreamSecurity::AnyEncyption, - ArchivedJournalStreamSecurityV1::ClassicEncryption => StreamSecurity::ClassicEncryption, - ArchivedJournalStreamSecurityV1::DoubleEncryption => StreamSecurity::DoubleEncryption, - ArchivedJournalStreamSecurityV1::Unknown => StreamSecurity::AnyEncyption, - } - } -} - #[repr(C)] #[derive( Debug, @@ -1958,39 +1661,6 @@ pub enum JournalAddressfamilyV1 { Unix, } -impl From for JournalAddressfamilyV1 { - fn from(val: wasi::Addressfamily) -> Self { - match val { - wasi::Addressfamily::Unspec => JournalAddressfamilyV1::Unspec, - wasi::Addressfamily::Inet4 => JournalAddressfamilyV1::Inet4, - wasi::Addressfamily::Inet6 => JournalAddressfamilyV1::Inet6, - wasi::Addressfamily::Unix => JournalAddressfamilyV1::Unix, - } - } -} - -impl From for wasi::Addressfamily { - fn from(val: JournalAddressfamilyV1) -> Self { - match val { - JournalAddressfamilyV1::Unspec => wasi::Addressfamily::Unspec, - JournalAddressfamilyV1::Inet4 => wasi::Addressfamily::Inet4, - JournalAddressfamilyV1::Inet6 => wasi::Addressfamily::Inet6, - JournalAddressfamilyV1::Unix => wasi::Addressfamily::Unix, - } - } -} - -impl From<&'_ ArchivedJournalAddressfamilyV1> for wasi::Addressfamily { - fn from(val: &'_ ArchivedJournalAddressfamilyV1) -> Self { - match val { - ArchivedJournalAddressfamilyV1::Unspec => wasi::Addressfamily::Unspec, - ArchivedJournalAddressfamilyV1::Inet4 => wasi::Addressfamily::Inet4, - ArchivedJournalAddressfamilyV1::Inet6 => wasi::Addressfamily::Inet6, - ArchivedJournalAddressfamilyV1::Unix => wasi::Addressfamily::Unix, - } - } -} - #[repr(C)] #[derive( Debug, @@ -2014,42 +1684,6 @@ pub enum JournalSocktypeV1 { Seqpacket, } -impl From for JournalSocktypeV1 { - fn from(val: wasi::Socktype) -> Self { - match val { - wasi::Socktype::Stream => JournalSocktypeV1::Stream, - wasi::Socktype::Dgram => JournalSocktypeV1::Dgram, - wasi::Socktype::Raw => JournalSocktypeV1::Raw, - wasi::Socktype::Seqpacket => JournalSocktypeV1::Seqpacket, - wasi::Socktype::Unknown => JournalSocktypeV1::Unknown, - } - } -} - -impl From for wasi::Socktype { - fn from(val: JournalSocktypeV1) -> Self { - match val { - JournalSocktypeV1::Stream => wasi::Socktype::Stream, - JournalSocktypeV1::Dgram => wasi::Socktype::Dgram, - JournalSocktypeV1::Raw => wasi::Socktype::Raw, - JournalSocktypeV1::Seqpacket => wasi::Socktype::Seqpacket, - JournalSocktypeV1::Unknown => wasi::Socktype::Unknown, - } - } -} - -impl From<&'_ ArchivedJournalSocktypeV1> for wasi::Socktype { - fn from(val: &'_ ArchivedJournalSocktypeV1) -> Self { - match val { - ArchivedJournalSocktypeV1::Stream => wasi::Socktype::Stream, - ArchivedJournalSocktypeV1::Dgram => wasi::Socktype::Dgram, - ArchivedJournalSocktypeV1::Raw => wasi::Socktype::Raw, - ArchivedJournalSocktypeV1::Seqpacket => wasi::Socktype::Seqpacket, - ArchivedJournalSocktypeV1::Unknown => wasi::Socktype::Unknown, - } - } -} - #[repr(C)] #[derive( Debug, @@ -2095,108 +1729,6 @@ pub enum JournalSockoptionV1 { Proto, } -impl From for JournalSockoptionV1 { - fn from(val: wasi::Sockoption) -> Self { - match val { - wasi::Sockoption::Noop => JournalSockoptionV1::Noop, - wasi::Sockoption::ReusePort => JournalSockoptionV1::ReusePort, - wasi::Sockoption::ReuseAddr => JournalSockoptionV1::ReuseAddr, - wasi::Sockoption::NoDelay => JournalSockoptionV1::NoDelay, - wasi::Sockoption::DontRoute => JournalSockoptionV1::DontRoute, - wasi::Sockoption::OnlyV6 => JournalSockoptionV1::OnlyV6, - wasi::Sockoption::Broadcast => JournalSockoptionV1::Broadcast, - wasi::Sockoption::MulticastLoopV4 => JournalSockoptionV1::MulticastLoopV4, - wasi::Sockoption::MulticastLoopV6 => JournalSockoptionV1::MulticastLoopV6, - wasi::Sockoption::Promiscuous => JournalSockoptionV1::Promiscuous, - wasi::Sockoption::Listening => JournalSockoptionV1::Listening, - wasi::Sockoption::LastError => JournalSockoptionV1::LastError, - wasi::Sockoption::KeepAlive => JournalSockoptionV1::KeepAlive, - wasi::Sockoption::Linger => JournalSockoptionV1::Linger, - wasi::Sockoption::OobInline => JournalSockoptionV1::OobInline, - wasi::Sockoption::RecvBufSize => JournalSockoptionV1::RecvBufSize, - wasi::Sockoption::SendBufSize => JournalSockoptionV1::SendBufSize, - wasi::Sockoption::RecvLowat => JournalSockoptionV1::RecvLowat, - wasi::Sockoption::SendLowat => JournalSockoptionV1::SendLowat, - wasi::Sockoption::RecvTimeout => JournalSockoptionV1::RecvTimeout, - wasi::Sockoption::SendTimeout => JournalSockoptionV1::SendTimeout, - wasi::Sockoption::ConnectTimeout => JournalSockoptionV1::ConnectTimeout, - wasi::Sockoption::AcceptTimeout => JournalSockoptionV1::AcceptTimeout, - wasi::Sockoption::Ttl => JournalSockoptionV1::Ttl, - wasi::Sockoption::MulticastTtlV4 => JournalSockoptionV1::MulticastTtlV4, - wasi::Sockoption::Type => JournalSockoptionV1::Type, - wasi::Sockoption::Proto => JournalSockoptionV1::Proto, - } - } -} - -impl From for wasi::Sockoption { - fn from(val: JournalSockoptionV1) -> Self { - match val { - JournalSockoptionV1::Noop => wasi::Sockoption::Noop, - JournalSockoptionV1::ReusePort => wasi::Sockoption::ReusePort, - JournalSockoptionV1::ReuseAddr => wasi::Sockoption::ReuseAddr, - JournalSockoptionV1::NoDelay => wasi::Sockoption::NoDelay, - JournalSockoptionV1::DontRoute => wasi::Sockoption::DontRoute, - JournalSockoptionV1::OnlyV6 => wasi::Sockoption::OnlyV6, - JournalSockoptionV1::Broadcast => wasi::Sockoption::Broadcast, - JournalSockoptionV1::MulticastLoopV4 => wasi::Sockoption::MulticastLoopV4, - JournalSockoptionV1::MulticastLoopV6 => wasi::Sockoption::MulticastLoopV6, - JournalSockoptionV1::Promiscuous => wasi::Sockoption::Promiscuous, - JournalSockoptionV1::Listening => wasi::Sockoption::Listening, - JournalSockoptionV1::LastError => wasi::Sockoption::LastError, - JournalSockoptionV1::KeepAlive => wasi::Sockoption::KeepAlive, - JournalSockoptionV1::Linger => wasi::Sockoption::Linger, - JournalSockoptionV1::OobInline => wasi::Sockoption::OobInline, - JournalSockoptionV1::RecvBufSize => wasi::Sockoption::RecvBufSize, - JournalSockoptionV1::SendBufSize => wasi::Sockoption::SendBufSize, - JournalSockoptionV1::RecvLowat => wasi::Sockoption::RecvLowat, - JournalSockoptionV1::SendLowat => wasi::Sockoption::SendLowat, - JournalSockoptionV1::RecvTimeout => wasi::Sockoption::RecvTimeout, - JournalSockoptionV1::SendTimeout => wasi::Sockoption::SendTimeout, - JournalSockoptionV1::ConnectTimeout => wasi::Sockoption::ConnectTimeout, - JournalSockoptionV1::AcceptTimeout => wasi::Sockoption::AcceptTimeout, - JournalSockoptionV1::Ttl => wasi::Sockoption::Ttl, - JournalSockoptionV1::MulticastTtlV4 => wasi::Sockoption::MulticastTtlV4, - JournalSockoptionV1::Type => wasi::Sockoption::Type, - JournalSockoptionV1::Proto => wasi::Sockoption::Proto, - } - } -} - -impl From<&'_ ArchivedJournalSockoptionV1> for wasi::Sockoption { - fn from(val: &'_ ArchivedJournalSockoptionV1) -> Self { - match val { - ArchivedJournalSockoptionV1::Noop => wasi::Sockoption::Noop, - ArchivedJournalSockoptionV1::ReusePort => wasi::Sockoption::ReusePort, - ArchivedJournalSockoptionV1::ReuseAddr => wasi::Sockoption::ReuseAddr, - ArchivedJournalSockoptionV1::NoDelay => wasi::Sockoption::NoDelay, - ArchivedJournalSockoptionV1::DontRoute => wasi::Sockoption::DontRoute, - ArchivedJournalSockoptionV1::OnlyV6 => wasi::Sockoption::OnlyV6, - ArchivedJournalSockoptionV1::Broadcast => wasi::Sockoption::Broadcast, - ArchivedJournalSockoptionV1::MulticastLoopV4 => wasi::Sockoption::MulticastLoopV4, - ArchivedJournalSockoptionV1::MulticastLoopV6 => wasi::Sockoption::MulticastLoopV6, - ArchivedJournalSockoptionV1::Promiscuous => wasi::Sockoption::Promiscuous, - ArchivedJournalSockoptionV1::Listening => wasi::Sockoption::Listening, - ArchivedJournalSockoptionV1::LastError => wasi::Sockoption::LastError, - ArchivedJournalSockoptionV1::KeepAlive => wasi::Sockoption::KeepAlive, - ArchivedJournalSockoptionV1::Linger => wasi::Sockoption::Linger, - ArchivedJournalSockoptionV1::OobInline => wasi::Sockoption::OobInline, - ArchivedJournalSockoptionV1::RecvBufSize => wasi::Sockoption::RecvBufSize, - ArchivedJournalSockoptionV1::SendBufSize => wasi::Sockoption::SendBufSize, - ArchivedJournalSockoptionV1::RecvLowat => wasi::Sockoption::RecvLowat, - ArchivedJournalSockoptionV1::SendLowat => wasi::Sockoption::SendLowat, - ArchivedJournalSockoptionV1::RecvTimeout => wasi::Sockoption::RecvTimeout, - ArchivedJournalSockoptionV1::SendTimeout => wasi::Sockoption::SendTimeout, - ArchivedJournalSockoptionV1::ConnectTimeout => wasi::Sockoption::ConnectTimeout, - ArchivedJournalSockoptionV1::AcceptTimeout => wasi::Sockoption::AcceptTimeout, - ArchivedJournalSockoptionV1::Ttl => wasi::Sockoption::Ttl, - ArchivedJournalSockoptionV1::MulticastTtlV4 => wasi::Sockoption::MulticastTtlV4, - ArchivedJournalSockoptionV1::Type => wasi::Sockoption::Type, - ArchivedJournalSockoptionV1::Proto => wasi::Sockoption::Proto, - } - } -} - #[repr(C)] #[derive( Debug, @@ -2221,45 +1753,6 @@ pub enum JournalTimeTypeV1 { Linger, } -impl From for JournalTimeTypeV1 { - fn from(val: SocketOptTimeType) -> Self { - match val { - SocketOptTimeType::ReadTimeout => JournalTimeTypeV1::ReadTimeout, - SocketOptTimeType::WriteTimeout => JournalTimeTypeV1::WriteTimeout, - SocketOptTimeType::AcceptTimeout => JournalTimeTypeV1::AcceptTimeout, - SocketOptTimeType::ConnectTimeout => JournalTimeTypeV1::ConnectTimeout, - SocketOptTimeType::BindTimeout => JournalTimeTypeV1::BindTimeout, - SocketOptTimeType::Linger => JournalTimeTypeV1::Linger, - } - } -} - -impl From for SocketOptTimeType { - fn from(val: JournalTimeTypeV1) -> Self { - match val { - JournalTimeTypeV1::ReadTimeout => SocketOptTimeType::ReadTimeout, - JournalTimeTypeV1::WriteTimeout => SocketOptTimeType::WriteTimeout, - JournalTimeTypeV1::AcceptTimeout => SocketOptTimeType::AcceptTimeout, - JournalTimeTypeV1::ConnectTimeout => SocketOptTimeType::ConnectTimeout, - JournalTimeTypeV1::BindTimeout => SocketOptTimeType::BindTimeout, - JournalTimeTypeV1::Linger => SocketOptTimeType::Linger, - } - } -} - -impl From<&'_ ArchivedJournalTimeTypeV1> for SocketOptTimeType { - fn from(val: &'_ ArchivedJournalTimeTypeV1) -> Self { - match val { - ArchivedJournalTimeTypeV1::ReadTimeout => SocketOptTimeType::ReadTimeout, - ArchivedJournalTimeTypeV1::WriteTimeout => SocketOptTimeType::WriteTimeout, - ArchivedJournalTimeTypeV1::AcceptTimeout => SocketOptTimeType::AcceptTimeout, - ArchivedJournalTimeTypeV1::ConnectTimeout => SocketOptTimeType::ConnectTimeout, - ArchivedJournalTimeTypeV1::BindTimeout => SocketOptTimeType::BindTimeout, - ArchivedJournalTimeTypeV1::Linger => SocketOptTimeType::Linger, - } - } -} - #[repr(C)] #[derive( Debug, @@ -2280,1280 +1773,3 @@ pub enum JournalSocketShutdownV1 { Write, Both, } - -impl From for JournalSocketShutdownV1 { - fn from(val: SocketShutdownHow) -> Self { - match val { - SocketShutdownHow::Read => JournalSocketShutdownV1::Read, - SocketShutdownHow::Write => JournalSocketShutdownV1::Write, - SocketShutdownHow::Both => JournalSocketShutdownV1::Both, - } - } -} - -impl From for SocketShutdownHow { - fn from(val: JournalSocketShutdownV1) -> Self { - match val { - JournalSocketShutdownV1::Read => SocketShutdownHow::Read, - JournalSocketShutdownV1::Write => SocketShutdownHow::Write, - JournalSocketShutdownV1::Both => SocketShutdownHow::Both, - } - } -} - -impl From<&'_ ArchivedJournalSocketShutdownV1> for SocketShutdownHow { - fn from(val: &'_ ArchivedJournalSocketShutdownV1) -> Self { - match val { - ArchivedJournalSocketShutdownV1::Read => SocketShutdownHow::Read, - ArchivedJournalSocketShutdownV1::Write => SocketShutdownHow::Write, - ArchivedJournalSocketShutdownV1::Both => SocketShutdownHow::Both, - } - } -} - -impl<'a> TryFrom> for JournalEntry<'a> { - type Error = anyhow::Error; - - fn try_from(value: ArchivedJournalEntry<'a>) -> anyhow::Result { - Ok(match value { - ArchivedJournalEntry::InitModuleV1(ArchivedJournalEntryInitModuleV1 { wasm_hash }) => { - Self::InitModuleV1 { - wasm_hash: *wasm_hash, - } - } - ArchivedJournalEntry::UpdateMemoryRegionV1( - ArchivedJournalEntryUpdateMemoryRegionV1 { - start, - end, - compressed_data, - _padding: _, - }, - ) => Self::UpdateMemoryRegionV1 { - region: (*start)..(*end), - data: Cow::Owned(decompress_size_prepended(compressed_data.as_ref())?), - }, - ArchivedJournalEntry::ProcessExitV1(ArchivedJournalEntryProcessExitV1 { - exit_code, - _padding: _, - }) => Self::ProcessExitV1 { - exit_code: exit_code.as_ref().map(|code| code.into()), - }, - ArchivedJournalEntry::SetThreadV1(ArchivedJournalEntrySetThreadV1 { - id, - call_stack, - memory_stack, - store_data, - _padding: _, - is_64bit, - }) => Self::SetThreadV1 { - id: *id, - call_stack: call_stack.as_ref().into(), - memory_stack: memory_stack.as_ref().into(), - store_data: store_data.as_ref().into(), - is_64bit: *is_64bit, - }, - ArchivedJournalEntry::CloseThreadV1(ArchivedJournalEntryCloseThreadV1 { - id, - exit_code, - }) => Self::CloseThreadV1 { - id: *id, - exit_code: exit_code.as_ref().map(|code| code.into()), - }, - ArchivedJournalEntry::FileDescriptorWriteV1( - ArchivedJournalEntryFileDescriptorWriteV1 { - data, - fd, - offset, - is_64bit, - _padding: _, - }, - ) => Self::FileDescriptorWriteV1 { - data: data.as_ref().into(), - fd: *fd, - offset: *offset, - is_64bit: *is_64bit, - }, - ArchivedJournalEntry::FileDescriptorSeekV1( - ArchivedJournalEntryFileDescriptorSeekV1 { - fd, - offset, - ref whence, - }, - ) => Self::FileDescriptorSeekV1 { - fd: *fd, - offset: *offset, - whence: whence.into(), - }, - ArchivedJournalEntry::OpenFileDescriptorV1( - ArchivedJournalEntryOpenFileDescriptorV1 { - fd, - dirfd, - dirflags, - path, - o_flags, - fs_rights_base, - fs_rights_inheriting, - fs_flags, - _padding: _, - }, - ) => Self::OpenFileDescriptorV1 { - fd: *fd, - dirfd: *dirfd, - dirflags: *dirflags, - path: path.as_ref().into(), - o_flags: wasi::Oflags::from_bits_truncate(*o_flags), - fs_rights_base: wasi::Rights::from_bits_truncate(*fs_rights_base), - fs_rights_inheriting: wasi::Rights::from_bits_truncate(*fs_rights_inheriting), - fs_flags: wasi::Fdflags::from_bits_truncate(*fs_flags), - }, - ArchivedJournalEntry::CloseFileDescriptorV1( - ArchivedJournalEntryCloseFileDescriptorV1 { fd, _padding: _ }, - ) => Self::CloseFileDescriptorV1 { fd: *fd }, - ArchivedJournalEntry::RemoveDirectoryV1(ArchivedJournalEntryRemoveDirectoryV1 { - fd, - path, - _padding: _, - }) => Self::RemoveDirectoryV1 { - fd: *fd, - path: path.as_ref().into(), - }, - ArchivedJournalEntry::UnlinkFileV1(ArchivedJournalEntryUnlinkFileV1 { - fd, - path, - _padding: _, - }) => Self::UnlinkFileV1 { - fd: *fd, - path: path.as_ref().into(), - }, - ArchivedJournalEntry::PathRenameV1(ArchivedJournalEntryPathRenameV1 { - old_fd, - old_path, - new_fd, - new_path, - _padding: _, - }) => Self::PathRenameV1 { - old_fd: *old_fd, - old_path: old_path.as_ref().into(), - new_fd: *new_fd, - new_path: new_path.as_ref().into(), - }, - ArchivedJournalEntry::SnapshotV1(ArchivedJournalEntrySnapshotV1 { - since_epoch, - ref trigger, - }) => Self::SnapshotV1 { - when: SystemTime::UNIX_EPOCH - .checked_add((*since_epoch).try_into().unwrap()) - .unwrap_or(SystemTime::UNIX_EPOCH), - trigger: trigger.into(), - }, - ArchivedJournalEntry::SetClockTimeV1(ArchivedJournalEntrySetClockTimeV1 { - ref clock_id, - time, - }) => Self::SetClockTimeV1 { - clock_id: clock_id.into(), - time: *time, - }, - ArchivedJournalEntry::RenumberFileDescriptorV1( - ArchivedJournalEntryRenumberFileDescriptorV1 { old_fd, new_fd }, - ) => Self::RenumberFileDescriptorV1 { - old_fd: *old_fd, - new_fd: *new_fd, - }, - ArchivedJournalEntry::DuplicateFileDescriptorV1( - ArchivedJournalEntryDuplicateFileDescriptorV1 { - original_fd: old_fd, - copied_fd: new_fd, - }, - ) => Self::DuplicateFileDescriptorV1 { - original_fd: *old_fd, - copied_fd: *new_fd, - }, - ArchivedJournalEntry::CreateDirectoryV1(ArchivedJournalEntryCreateDirectoryV1 { - fd, - path, - _padding: _, - }) => Self::CreateDirectoryV1 { - fd: *fd, - path: path.as_ref().into(), - }, - ArchivedJournalEntry::PathSetTimesV1(ArchivedJournalEntryPathSetTimesV1 { - fd, - path, - flags, - st_atim, - st_mtim, - fst_flags, - _padding: _, - }) => Self::PathSetTimesV1 { - fd: *fd, - path: path.as_ref().into(), - flags: *flags, - st_atim: *st_atim, - st_mtim: *st_mtim, - fst_flags: wasi::Fstflags::from_bits_truncate(*fst_flags), - }, - ArchivedJournalEntry::FileDescriptorSetTimesV1( - ArchivedJournalEntryFileDescriptorSetTimesV1 { - fd, - st_atim, - st_mtim, - fst_flags, - }, - ) => Self::FileDescriptorSetTimesV1 { - fd: *fd, - st_atim: *st_atim, - st_mtim: *st_mtim, - fst_flags: wasi::Fstflags::from_bits_truncate(*fst_flags), - }, - ArchivedJournalEntry::FileDescriptorSetSizeV1( - ArchivedJournalEntryFileDescriptorSetSizeV1 { fd, st_size }, - ) => Self::FileDescriptorSetSizeV1 { - fd: *fd, - st_size: *st_size, - }, - ArchivedJournalEntry::FileDescriptorSetFlagsV1( - ArchivedJournalEntryFileDescriptorSetFlagsV1 { fd, flags }, - ) => Self::FileDescriptorSetFlagsV1 { - fd: *fd, - flags: Fdflags::from_bits_truncate(*flags), - }, - ArchivedJournalEntry::FileDescriptorSetRightsV1( - ArchivedJournalEntryFileDescriptorSetRightsV1 { - fd, - fs_rights_base, - fs_rights_inheriting, - }, - ) => Self::FileDescriptorSetRightsV1 { - fd: *fd, - fs_rights_base: Rights::from_bits_truncate(*fs_rights_base), - fs_rights_inheriting: Rights::from_bits_truncate(*fs_rights_inheriting), - }, - ArchivedJournalEntry::FileDescriptorAdviseV1( - ArchivedJournalEntryFileDescriptorAdviseV1 { - fd, - offset, - len, - ref advice, - }, - ) => Self::FileDescriptorAdviseV1 { - fd: *fd, - offset: *offset, - len: *len, - advice: advice.into(), - }, - ArchivedJournalEntry::FileDescriptorAllocateV1( - ArchivedJournalEntryFileDescriptorAllocateV1 { fd, offset, len }, - ) => Self::FileDescriptorAllocateV1 { - fd: *fd, - offset: *offset, - len: *len, - }, - ArchivedJournalEntry::CreateHardLinkV1(ArchivedJournalEntryCreateHardLinkV1 { - old_fd, - old_path, - old_flags, - new_fd, - new_path, - _padding: _, - }) => Self::CreateHardLinkV1 { - old_fd: *old_fd, - old_path: old_path.as_ref().into(), - old_flags: *old_flags, - new_fd: *new_fd, - new_path: new_path.as_ref().into(), - }, - ArchivedJournalEntry::CreateSymbolicLinkV1( - ArchivedJournalEntryCreateSymbolicLinkV1 { - old_path, - fd, - new_path, - _padding: _, - }, - ) => Self::CreateSymbolicLinkV1 { - old_path: old_path.as_ref().into(), - fd: *fd, - new_path: new_path.as_ref().into(), - }, - ArchivedJournalEntry::ChangeDirectoryV1(ArchivedJournalEntryChangeDirectoryV1 { - path, - }) => Self::ChangeDirectoryV1 { - path: path.as_ref().into(), - }, - ArchivedJournalEntry::EpollCreateV1(ArchivedJournalEntryEpollCreateV1 { - fd, - _padding: _, - }) => Self::EpollCreateV1 { fd: *fd }, - ArchivedJournalEntry::EpollCtlV1(ArchivedJournalEntryEpollCtlV1 { - epfd, - ref op, - fd, - ref event, - }) => Self::EpollCtlV1 { - epfd: *epfd, - op: op.into(), - fd: *fd, - event: event.as_ref().map(|e| e.into()), - }, - ArchivedJournalEntry::TtySetV1(ArchivedJournalEntryTtySetV1 { - cols, - rows, - width, - height, - stdin_tty, - stdout_tty, - stderr_tty, - echo, - line_buffered, - line_feeds, - }) => Self::TtySetV1 { - tty: wasi::Tty { - cols: *cols, - rows: *rows, - width: *width, - height: *height, - stdin_tty: *stdin_tty, - stdout_tty: *stdout_tty, - stderr_tty: *stderr_tty, - echo: *echo, - line_buffered: *line_buffered, - }, - line_feeds: *line_feeds, - }, - ArchivedJournalEntry::CreatePipeV1(ArchivedJournalEntryCreatePipeV1 { fd1, fd2 }) => { - Self::CreatePipeV1 { - fd1: *fd1, - fd2: *fd2, - } - } - ArchivedJournalEntry::PortAddAddrV1(ArchivedJournalEntryPortAddAddrV1 { cidr }) => { - Self::PortAddAddrV1 { - cidr: IpCidr { - ip: cidr.ip.as_ipaddr(), - prefix: cidr.prefix, - }, - } - } - ArchivedJournalEntry::PortDelAddrV1(ArchivedJournalEntryPortDelAddrV1 { addr }) => { - Self::PortDelAddrV1 { - addr: addr.as_ipaddr(), - } - } - ArchivedJournalEntry::PortAddrClearV1 => Self::PortAddrClearV1, - ArchivedJournalEntry::PortBridgeV1(ArchivedJournalEntryPortBridgeV1 { - network, - token, - ref security, - _padding: _, - }) => Self::PortBridgeV1 { - network: network.as_ref().into(), - token: token.as_ref().into(), - security: security.into(), - }, - ArchivedJournalEntry::PortUnbridgeV1 => Self::PortUnbridgeV1, - ArchivedJournalEntry::PortDhcpAcquireV1 => Self::PortDhcpAcquireV1, - ArchivedJournalEntry::PortGatewaySetV1(ArchivedJournalEntryPortGatewaySetV1 { ip }) => { - Self::PortGatewaySetV1 { ip: ip.as_ipaddr() } - } - ArchivedJournalEntry::PortRouteAddV1(ArchivedJournalEntryPortRouteAddV1 { - cidr, - via_router, - preferred_until, - expires_at, - }) => Self::PortRouteAddV1 { - cidr: IpCidr { - ip: cidr.ip.as_ipaddr(), - prefix: cidr.prefix, - }, - via_router: via_router.as_ipaddr(), - preferred_until: preferred_until - .as_ref() - .map(|time| (*time).try_into().unwrap()), - expires_at: expires_at.as_ref().map(|time| (*time).try_into().unwrap()), - }, - ArchivedJournalEntry::PortRouteClearV1 => Self::PortRouteClearV1, - ArchivedJournalEntry::PortRouteDelV1(ArchivedJournalEntryPortRouteDelV1 { ip }) => { - Self::PortRouteDelV1 { ip: ip.as_ipaddr() } - } - ArchivedJournalEntry::SocketOpenV1(ArchivedJournalEntrySocketOpenV1 { - ref af, - ref ty, - pt, - fd, - }) => Self::SocketOpenV1 { - af: af.into(), - ty: ty.into(), - pt: (*pt).try_into().unwrap_or(wasi::SockProto::Max), - fd: *fd, - }, - ArchivedJournalEntry::SocketListenV1(ArchivedJournalEntrySocketListenV1 { - fd, - backlog, - }) => Self::SocketListenV1 { - fd: *fd, - backlog: *backlog, - }, - ArchivedJournalEntry::SocketBindV1(ArchivedJournalEntrySocketBindV1 { fd, addr }) => { - Self::SocketBindV1 { - fd: *fd, - addr: addr.as_socket_addr(), - } - } - ArchivedJournalEntry::SocketConnectedV1(ArchivedJournalEntrySocketConnectedV1 { - fd, - addr, - }) => Self::SocketConnectedV1 { - fd: *fd, - addr: addr.as_socket_addr(), - }, - ArchivedJournalEntry::SocketAcceptedV1(ArchivedJournalEntrySocketAcceptedV1 { - listen_fd, - fd, - peer_addr, - fd_flags, - nonblocking, - }) => Self::SocketAcceptedV1 { - listen_fd: *listen_fd, - fd: *fd, - peer_addr: peer_addr.as_socket_addr(), - fd_flags: Fdflags::from_bits_truncate(*fd_flags), - non_blocking: *nonblocking, - }, - ArchivedJournalEntry::SocketJoinIpv4MulticastV1( - ArchivedJournalEntrySocketJoinIpv4MulticastV1 { - fd, - multiaddr, - iface, - }, - ) => Self::SocketJoinIpv4MulticastV1 { - fd: *fd, - multiaddr: multiaddr.as_ipv4(), - iface: iface.as_ipv4(), - }, - ArchivedJournalEntry::SocketJoinIpv6MulticastV1( - ArchivedJournalEntrySocketJoinIpv6MulticastV1 { - fd, - multiaddr, - iface, - }, - ) => Self::SocketJoinIpv6MulticastV1 { - fd: *fd, - multi_addr: multiaddr.as_ipv6(), - iface: *iface, - }, - ArchivedJournalEntry::SocketLeaveIpv4MulticastV1( - ArchivedJournalEntrySocketLeaveIpv4MulticastV1 { - fd, - multiaddr, - iface, - }, - ) => Self::SocketLeaveIpv4MulticastV1 { - fd: *fd, - multi_addr: multiaddr.as_ipv4(), - iface: iface.as_ipv4(), - }, - ArchivedJournalEntry::SocketLeaveIpv6MulticastV1( - ArchivedJournalEntrySocketLeaveIpv6MulticastV1 { - fd, - multiaddr, - iface, - }, - ) => Self::SocketLeaveIpv6MulticastV1 { - fd: *fd, - multi_addr: multiaddr.as_ipv6(), - iface: *iface, - }, - ArchivedJournalEntry::SocketSendFileV1(ArchivedJournalEntrySocketSendFileV1 { - socket_fd, - file_fd, - offset, - count, - }) => Self::SocketSendFileV1 { - socket_fd: *socket_fd, - file_fd: *file_fd, - offset: *offset, - count: *count, - }, - ArchivedJournalEntry::SocketSendToV1(ArchivedJournalEntrySocketSendToV1 { - fd, - data, - flags, - addr, - is_64bit, - _padding: _, - }) => Self::SocketSendToV1 { - fd: *fd, - data: data.as_ref().into(), - flags: *flags, - addr: addr.as_socket_addr(), - is_64bit: *is_64bit, - }, - ArchivedJournalEntry::SocketSendV1(ArchivedJournalEntrySocketSendV1 { - fd, - data, - flags, - is_64bit, - _padding: _, - }) => Self::SocketSendV1 { - fd: *fd, - data: data.as_ref().into(), - flags: *flags, - is_64bit: *is_64bit, - }, - ArchivedJournalEntry::SocketSetOptFlagV1(ArchivedJournalEntrySocketSetOptFlagV1 { - fd, - ref opt, - flag, - }) => Self::SocketSetOptFlagV1 { - fd: *fd, - opt: opt.into(), - flag: *flag, - }, - ArchivedJournalEntry::SocketSetOptSizeV1(ArchivedJournalEntrySocketSetOptSizeV1 { - fd, - ref opt, - size, - }) => Self::SocketSetOptSizeV1 { - fd: *fd, - opt: opt.into(), - size: *size, - }, - ArchivedJournalEntry::SocketSetOptTimeV1(ArchivedJournalEntrySocketSetOptTimeV1 { - fd, - ref ty, - time, - }) => Self::SocketSetOptTimeV1 { - fd: *fd, - ty: ty.into(), - time: time.as_ref().map(|time| (*time).try_into().unwrap()), - }, - ArchivedJournalEntry::SocketShutdownV1(ArchivedJournalEntrySocketShutdownV1 { - fd, - ref how, - }) => Self::SocketShutdownV1 { - fd: *fd, - how: how.into(), - }, - ArchivedJournalEntry::CreateEventV1(ArchivedJournalEntryCreateEventV1 { - initial_val, - flags, - fd, - }) => Self::CreateEventV1 { - initial_val: *initial_val, - flags: *flags, - fd: *fd, - }, - }) - } -} - -#[cfg(test)] -mod tests { - use rkyv::ser::serializers::{ - AllocScratch, CompositeSerializer, SharedSerializeMap, WriteSerializer, - }; - - use super::*; - - pub fn run_test<'a>(record: JournalEntry<'a>) { - tracing::info!("record: {:?}", record); - - // Determine the record type - let record_type = record.archive_record_type(); - tracing::info!("record_type: {:?}", record_type); - - // Serialize it - let mut buffer = Vec::new(); - let mut serializer = CompositeSerializer::new( - WriteSerializer::new(&mut buffer), - AllocScratch::default(), - SharedSerializeMap::default(), - ); - - record.clone().serialize_archive(&mut serializer).unwrap(); - let buffer = &buffer[..]; - if buffer.len() < 20 { - tracing::info!("buffer: {:x?}", buffer); - } else { - tracing::info!("buffer_len: {}", buffer.len()); - } - - // Deserialize it - let record2 = unsafe { record_type.deserialize_archive(buffer).unwrap() }; - tracing::info!("record2: {:?}", record2); - - // Check it - assert_eq!(record, record2); - - // Now make it static and check it again - let record3 = record2.into_owned(); - tracing::info!("record3: {:?}", record3); - assert_eq!(record, record3); - } - - #[tracing_test::traced_test] - #[test] - pub fn test_record_init_module() { - run_test(JournalEntry::InitModuleV1 { - wasm_hash: [13u8; 8], - }); - } - - #[tracing_test::traced_test] - #[test] - pub fn test_record_process_exit() { - run_test(JournalEntry::ProcessExitV1 { - exit_code: Some(wasi::ExitCode::Errno(wasi::Errno::Fault)), - }); - } - - #[tracing_test::traced_test] - #[test] - pub fn test_record_set_thread() { - run_test(JournalEntry::SetThreadV1 { - id: 1234u32, - call_stack: vec![1, 2, 3].into(), - memory_stack: vec![4, 5, 6, 7].into(), - store_data: vec![10, 11].into(), - is_64bit: true, - }); - } - - #[tracing_test::traced_test] - #[test] - pub fn test_record_close_thread() { - run_test(JournalEntry::CloseThreadV1 { - id: 987u32, - exit_code: Some(wasi::ExitCode::Errno(wasi::Errno::Fault)), - }); - } - - #[tracing_test::traced_test] - #[test] - pub fn test_record_descriptor_seek() { - run_test(JournalEntry::FileDescriptorSeekV1 { - fd: 765u32, - offset: 9183722450971234i64, - whence: wasi::Whence::End, - }); - } - - #[tracing_test::traced_test] - #[test] - pub fn test_record_descriptor_write() { - run_test(JournalEntry::FileDescriptorWriteV1 { - fd: 54321u32, - offset: 13897412934u64, - data: vec![74u8, 98u8, 36u8].into(), - is_64bit: true, - }); - } - - #[tracing_test::traced_test] - #[test] - pub fn test_record_update_memory() { - run_test(JournalEntry::UpdateMemoryRegionV1 { - region: 76u64..8237453u64, - data: [74u8; 40960].to_vec().into(), - }); - } - - #[tracing_test::traced_test] - #[test] - pub fn test_record_set_clock_time() { - run_test(JournalEntry::SetClockTimeV1 { - clock_id: wasi::Snapshot0Clockid::Realtime, - time: 7912837412934u64, - }); - } - - #[tracing_test::traced_test] - #[test] - pub fn test_record_open_file_descriptor() { - run_test(JournalEntry::OpenFileDescriptorV1 { - fd: 298745u32, - dirfd: 23458922u32, - dirflags: 134512345, - path: "/blah".into(), - o_flags: wasi::Oflags::all(), - fs_rights_base: wasi::Rights::all(), - fs_rights_inheriting: wasi::Rights::all(), - fs_flags: wasi::Fdflags::all(), - }); - } - - #[tracing_test::traced_test] - #[test] - pub fn test_record_close_descriptor() { - run_test(JournalEntry::CloseFileDescriptorV1 { fd: 23845732u32 }); - } - - #[tracing_test::traced_test] - #[test] - pub fn test_record_renumber_file_descriptor() { - run_test(JournalEntry::RenumberFileDescriptorV1 { - old_fd: 27834u32, - new_fd: 398452345u32, - }); - } - - #[tracing_test::traced_test] - #[test] - pub fn test_record_duplicate_file_descriptor() { - run_test(JournalEntry::DuplicateFileDescriptorV1 { - original_fd: 23482934u32, - copied_fd: 9384529u32, - }); - } - - #[tracing_test::traced_test] - #[test] - pub fn test_record_create_directory() { - run_test(JournalEntry::CreateDirectoryV1 { - fd: 238472u32, - path: "/joasjdf/asdfn".into(), - }); - } - - #[tracing_test::traced_test] - #[test] - pub fn test_record_remove_directory() { - run_test(JournalEntry::RemoveDirectoryV1 { - fd: 23894952u32, - path: "/blahblah".into(), - }); - } - - #[tracing_test::traced_test] - #[test] - pub fn test_record_path_set_times() { - run_test(JournalEntry::PathSetTimesV1 { - fd: 1238934u32, - flags: 234523, - path: "/".into(), - st_atim: 923452345, - st_mtim: 350, - fst_flags: wasi::Fstflags::all(), - }); - } - - #[tracing_test::traced_test] - #[test] - pub fn test_record_file_descriptor_set_times() { - run_test(JournalEntry::FileDescriptorSetTimesV1 { - fd: 898785u32, - st_atim: 29834952345, - st_mtim: 239845892345, - fst_flags: wasi::Fstflags::all(), - }); - } - - #[tracing_test::traced_test] - #[test] - pub fn test_record_file_descriptor_set_size() { - run_test(JournalEntry::FileDescriptorSetSizeV1 { - fd: 34958234u32, - st_size: 234958293845u64, - }); - } - - #[tracing_test::traced_test] - #[test] - pub fn test_record_file_descriptor_set_flags() { - run_test(JournalEntry::FileDescriptorSetFlagsV1 { - fd: 982348752u32, - flags: wasi::Fdflags::all(), - }); - } - - #[tracing_test::traced_test] - #[test] - pub fn test_record_file_descriptor_set_rights() { - run_test(JournalEntry::FileDescriptorSetRightsV1 { - fd: 872345u32, - fs_rights_base: wasi::Rights::all(), - fs_rights_inheriting: wasi::Rights::all(), - }); - } - - #[tracing_test::traced_test] - #[test] - pub fn test_record_file_descriptor_advise() { - run_test(JournalEntry::FileDescriptorAdviseV1 { - fd: 298434u32, - offset: 92834529092345, - len: 23485928345, - advice: wasi::Advice::Random, - }); - } - - #[tracing_test::traced_test] - #[test] - pub fn test_record_file_descriptor_allocate() { - run_test(JournalEntry::FileDescriptorAllocateV1 { - fd: 2934852, - offset: 23489582934523, - len: 9845982345, - }); - } - - #[tracing_test::traced_test] - #[test] - pub fn test_record_create_hard_link() { - run_test(JournalEntry::CreateHardLinkV1 { - old_fd: 324983845, - old_path: "/asjdfiasidfasdf".into(), - old_flags: 234857, - new_fd: 34958345, - new_path: "/ashdufnasd".into(), - }); - } - - #[tracing_test::traced_test] - #[test] - pub fn test_record_create_symbolic_link() { - run_test(JournalEntry::CreateSymbolicLinkV1 { - old_path: "/asjbndfjasdf/asdafasdf".into(), - fd: 235422345, - new_path: "/asdf".into(), - }); - } - - #[tracing_test::traced_test] - #[test] - pub fn test_record_unlink_file() { - run_test(JournalEntry::UnlinkFileV1 { - fd: 32452345, - path: "/asdfasd".into(), - }); - } - - #[tracing_test::traced_test] - #[test] - pub fn test_record_path_rename() { - run_test(JournalEntry::PathRenameV1 { - old_fd: 32451345, - old_path: "/asdfasdfas/asdfasdf".into(), - new_fd: 23452345, - new_path: "/ahgfdfghdfghdfgh".into(), - }); - } - - #[tracing_test::traced_test] - #[test] - pub fn test_record_change_directory() { - run_test(JournalEntry::ChangeDirectoryV1 { - path: "/etc".to_string().into(), - }); - } - - #[tracing_test::traced_test] - #[test] - pub fn test_record_epoll_create() { - run_test(JournalEntry::EpollCreateV1 { fd: 45384752 }); - } - - #[tracing_test::traced_test] - #[test] - pub fn test_record_epoll_ctl() { - run_test(JournalEntry::EpollCtlV1 { - epfd: 34523455, - op: wasi::EpollCtl::Unknown, - fd: 23452345, - event: Some(wasi::EpollEventCtl { - events: wasi::EpollType::all(), - ptr: 32452345, - fd: 23452345, - data1: 1235245756, - data2: 23452345, - }), - }); - } - - #[tracing_test::traced_test] - #[test] - pub fn test_record_tty_set() { - run_test(JournalEntry::TtySetV1 { - tty: wasi::Tty { - cols: 1234, - rows: 6754, - width: 4563456, - height: 345, - stdin_tty: true, - stdout_tty: false, - stderr_tty: true, - echo: true, - line_buffered: true, - }, - line_feeds: true, - }); - } - - #[tracing_test::traced_test] - #[test] - pub fn test_record_create_pipe() { - run_test(JournalEntry::CreatePipeV1 { - fd1: 3452345, - fd2: 2345163, - }); - } - - #[tracing_test::traced_test] - #[test] - pub fn test_record_create_event() { - run_test(JournalEntry::CreateEventV1 { - initial_val: 13451345, - flags: 2343, - fd: 5836544, - }); - } - - #[tracing_test::traced_test] - #[test] - pub fn test_record_port_add_addr() { - run_test(JournalEntry::PortAddAddrV1 { - cidr: IpCidr { - ip: Ipv4Addr::LOCALHOST.into(), - prefix: 24, - }, - }); - } - - #[tracing_test::traced_test] - #[test] - pub fn test_record_del_addr() { - run_test(JournalEntry::PortDelAddrV1 { - addr: Ipv6Addr::LOCALHOST.into(), - }); - } - - #[tracing_test::traced_test] - #[test] - pub fn test_record_addr_clear() { - run_test(JournalEntry::PortAddrClearV1); - } - - #[tracing_test::traced_test] - #[test] - pub fn test_record_port_bridge() { - run_test(JournalEntry::PortBridgeV1 { - network: "mynetwork".into(), - token: format!("blh blah").into(), - security: StreamSecurity::ClassicEncryption, - }); - } - - #[tracing_test::traced_test] - #[test] - pub fn test_record_unbridge() { - run_test(JournalEntry::PortUnbridgeV1); - } - - #[tracing_test::traced_test] - #[test] - pub fn test_record_dhcp_acquire() { - run_test(JournalEntry::PortDhcpAcquireV1); - } - - #[tracing_test::traced_test] - #[test] - pub fn test_record_gateway_set() { - run_test(JournalEntry::PortGatewaySetV1 { - ip: Ipv4Addr::new(12, 34, 136, 220).into(), - }); - } - - #[tracing_test::traced_test] - #[test] - pub fn test_record_route_add() { - run_test(JournalEntry::PortRouteAddV1 { - cidr: IpCidr { - ip: Ipv4Addr::LOCALHOST.into(), - prefix: 24, - }, - via_router: Ipv4Addr::LOCALHOST.into(), - preferred_until: Some(Duration::MAX), - expires_at: Some(Duration::ZERO), - }); - } - - #[tracing_test::traced_test] - #[test] - pub fn test_record_route_clear() { - run_test(JournalEntry::PortRouteClearV1); - } - - #[tracing_test::traced_test] - #[test] - pub fn test_record_route_del() { - run_test(JournalEntry::PortRouteDelV1 { - ip: Ipv4Addr::BROADCAST.into(), - }); - } - - #[tracing_test::traced_test] - #[test] - pub fn test_record_socket_open() { - run_test(JournalEntry::SocketOpenV1 { - af: wasi::Addressfamily::Inet6, - ty: wasi::Socktype::Stream, - pt: wasi::SockProto::Tcp, - fd: 23452345, - }); - } - - #[tracing_test::traced_test] - #[test] - pub fn test_record_socket_listen() { - run_test(JournalEntry::SocketListenV1 { - fd: 12341234, - backlog: 123, - }); - } - - #[tracing_test::traced_test] - #[test] - pub fn test_record_socket_bind() { - run_test(JournalEntry::SocketBindV1 { - fd: 2341234, - addr: SocketAddr::new(Ipv6Addr::UNSPECIFIED.into(), 1234), - }); - } - - #[tracing_test::traced_test] - #[test] - pub fn test_record_socket_connected() { - run_test(JournalEntry::SocketConnectedV1 { - fd: 12341, - addr: SocketAddr::new(Ipv6Addr::UNSPECIFIED.into(), 1234), - }); - } - - #[tracing_test::traced_test] - #[test] - pub fn test_record_socket_accepted() { - run_test(JournalEntry::SocketAcceptedV1 { - listen_fd: 21234, - fd: 1, - peer_addr: SocketAddr::new(Ipv6Addr::UNSPECIFIED.into(), 3452), - fd_flags: wasi::Fdflags::all(), - non_blocking: true, - }); - } - - #[tracing_test::traced_test] - #[test] - pub fn test_record_socket_join_ipv4_multicast() { - run_test(JournalEntry::SocketJoinIpv4MulticastV1 { - fd: 12, - multiaddr: Ipv4Addr::new(123, 123, 123, 123).into(), - iface: Ipv4Addr::new(128, 0, 0, 1).into(), - }); - } - - #[tracing_test::traced_test] - #[test] - pub fn test_record_socket_join_ipv6_multicast() { - run_test(JournalEntry::SocketJoinIpv6MulticastV1 { - fd: 12, - multi_addr: Ipv6Addr::new(123, 123, 123, 123, 1234, 12663, 31, 1324).into(), - iface: 23541, - }); - } - - #[tracing_test::traced_test] - #[test] - pub fn test_record_socket_leave_ipv4_multicast() { - run_test(JournalEntry::SocketLeaveIpv4MulticastV1 { - fd: 12, - multi_addr: Ipv4Addr::new(123, 123, 123, 123).into(), - iface: Ipv4Addr::new(128, 0, 0, 1).into(), - }); - } - - #[tracing_test::traced_test] - #[test] - pub fn test_record_socket_leave_ipv6_multicast() { - run_test(JournalEntry::SocketLeaveIpv6MulticastV1 { - fd: 12, - multi_addr: Ipv6Addr::new(123, 123, 123, 123, 1234, 12663, 31, 1324).into(), - iface: 23541, - }); - } - - #[tracing_test::traced_test] - #[test] - pub fn test_record_socket_send_file() { - run_test(JournalEntry::SocketSendFileV1 { - socket_fd: 22234, - file_fd: 989, - offset: 124, - count: 345673456234651234, - }); - } - - #[tracing_test::traced_test] - #[test] - pub fn test_record_socket_send_to() { - run_test(JournalEntry::SocketSendToV1 { - fd: 123, - data: [98u8; 102400].to_vec().into(), - flags: 1234, - addr: SocketAddr::new(Ipv6Addr::UNSPECIFIED.into(), 3452), - is_64bit: true, - }); - } - - #[tracing_test::traced_test] - #[test] - pub fn test_record_socket_send() { - run_test(JournalEntry::SocketSendV1 { - fd: 123, - data: [98u8; 102400].to_vec().into(), - flags: 1234, - is_64bit: true, - }); - } - - #[tracing_test::traced_test] - #[test] - pub fn test_record_socket_set_opt_flag() { - run_test(JournalEntry::SocketSetOptFlagV1 { - fd: 0, - opt: wasi::Sockoption::Linger, - flag: true, - }); - } - - #[tracing_test::traced_test] - #[test] - pub fn test_record_socket_set_opt_size() { - run_test(JournalEntry::SocketSetOptSizeV1 { - fd: 15, - opt: wasi::Sockoption::Linger, - size: 234234, - }); - } - - #[tracing_test::traced_test] - #[test] - pub fn test_record_socket_set_opt_time() { - run_test(JournalEntry::SocketSetOptTimeV1 { - fd: 0, - ty: SocketOptTimeType::AcceptTimeout, - time: Some(Duration::ZERO), - }); - } - - #[tracing_test::traced_test] - #[test] - pub fn test_record_socket_shutdown() { - run_test(JournalEntry::SocketShutdownV1 { - fd: 123, - how: SocketShutdownHow::Both, - }); - } - - #[tracing_test::traced_test] - #[test] - pub fn test_record_snapshot() { - run_test(JournalEntry::SnapshotV1 { - when: SystemTime::now(), - trigger: SnapshotTrigger::Idle, - }); - } - - #[tracing_test::traced_test] - #[test] - pub fn test_record_alignment() { - assert_eq!(std::mem::align_of::(), 8); - assert_eq!(std::mem::align_of::(), 8); - assert_eq!(std::mem::align_of::(), 8); - assert_eq!(std::mem::align_of::(), 8); - assert_eq!(std::mem::align_of::(), 8); - assert_eq!(std::mem::align_of::(), 8); - assert_eq!(std::mem::align_of::(), 8); - assert_eq!(std::mem::align_of::(), 8); - assert_eq!(std::mem::align_of::(), 8); - assert_eq!(std::mem::align_of::(), 8); - assert_eq!( - std::mem::align_of::(), - 8 - ); - assert_eq!( - std::mem::align_of::(), - 8 - ); - assert_eq!(std::mem::align_of::(), 8); - assert_eq!(std::mem::align_of::(), 8); - assert_eq!(std::mem::align_of::(), 8); - assert_eq!( - std::mem::align_of::(), - 8 - ); - assert_eq!( - std::mem::align_of::(), - 8 - ); - assert_eq!( - std::mem::align_of::(), - 8 - ); - assert_eq!( - std::mem::align_of::(), - 8 - ); - assert_eq!( - std::mem::align_of::(), - 8 - ); - assert_eq!( - std::mem::align_of::(), - 8 - ); - assert_eq!(std::mem::align_of::(), 8); - assert_eq!(std::mem::align_of::(), 8); - assert_eq!(std::mem::align_of::(), 8); - assert_eq!(std::mem::align_of::(), 8); - assert_eq!(std::mem::align_of::(), 8); - assert_eq!(std::mem::align_of::(), 8); - assert_eq!(std::mem::align_of::(), 8); - assert_eq!(std::mem::align_of::(), 8); - assert_eq!(std::mem::align_of::(), 8); - assert_eq!(std::mem::align_of::(), 8); - assert_eq!(std::mem::align_of::(), 8); - assert_eq!(std::mem::align_of::(), 8); - assert_eq!(std::mem::align_of::(), 8); - assert_eq!(std::mem::align_of::(), 8); - assert_eq!(std::mem::align_of::(), 8); - assert_eq!(std::mem::align_of::(), 8); - assert_eq!(std::mem::align_of::(), 8); - assert_eq!(std::mem::align_of::(), 8); - assert_eq!(std::mem::align_of::(), 8); - assert_eq!(std::mem::align_of::(), 8); - assert_eq!(std::mem::align_of::(), 8); - assert_eq!( - std::mem::align_of::(), - 8 - ); - assert_eq!( - std::mem::align_of::(), - 8 - ); - assert_eq!( - std::mem::align_of::(), - 8 - ); - assert_eq!( - std::mem::align_of::(), - 8 - ); - assert_eq!(std::mem::align_of::(), 8); - assert_eq!(std::mem::align_of::(), 8); - assert_eq!(std::mem::align_of::(), 8); - assert_eq!(std::mem::align_of::(), 8); - assert_eq!(std::mem::align_of::(), 8); - assert_eq!(std::mem::align_of::(), 8); - assert_eq!(std::mem::align_of::(), 8); - assert_eq!(std::mem::align_of::(), 8); - } -} diff --git a/lib/journal/src/concrete/archived_from.rs b/lib/journal/src/concrete/archived_from.rs new file mode 100644 index 00000000000..b37e53f3c26 --- /dev/null +++ b/lib/journal/src/concrete/archived_from.rs @@ -0,0 +1,1107 @@ +use lz4_flex::block::decompress_size_prepended; +use std::borrow::Cow; +use std::time::SystemTime; +use wasmer_wasix_types::wasi; + +use super::*; + +impl From for JournalSnapshot0ClockidV1 { + fn from(val: wasi::Snapshot0Clockid) -> Self { + match val { + wasi::Snapshot0Clockid::Realtime => JournalSnapshot0ClockidV1::Realtime, + wasi::Snapshot0Clockid::Monotonic => JournalSnapshot0ClockidV1::Monotonic, + wasi::Snapshot0Clockid::ProcessCputimeId => JournalSnapshot0ClockidV1::ProcessCputimeId, + wasi::Snapshot0Clockid::ThreadCputimeId => JournalSnapshot0ClockidV1::ThreadCputimeId, + wasi::Snapshot0Clockid::Unknown => JournalSnapshot0ClockidV1::Unknown, + } + } +} + +impl From for wasi::Snapshot0Clockid { + fn from(val: JournalSnapshot0ClockidV1) -> Self { + match val { + JournalSnapshot0ClockidV1::Realtime => wasi::Snapshot0Clockid::Realtime, + JournalSnapshot0ClockidV1::Monotonic => wasi::Snapshot0Clockid::Monotonic, + JournalSnapshot0ClockidV1::ProcessCputimeId => wasi::Snapshot0Clockid::ProcessCputimeId, + JournalSnapshot0ClockidV1::ThreadCputimeId => wasi::Snapshot0Clockid::ThreadCputimeId, + JournalSnapshot0ClockidV1::Unknown => wasi::Snapshot0Clockid::Unknown, + } + } +} + +impl From<&'_ ArchivedJournalSnapshot0ClockidV1> for wasi::Snapshot0Clockid { + fn from(val: &'_ ArchivedJournalSnapshot0ClockidV1) -> Self { + match val { + ArchivedJournalSnapshot0ClockidV1::Realtime => wasi::Snapshot0Clockid::Realtime, + ArchivedJournalSnapshot0ClockidV1::Monotonic => wasi::Snapshot0Clockid::Monotonic, + ArchivedJournalSnapshot0ClockidV1::ProcessCputimeId => { + wasi::Snapshot0Clockid::ProcessCputimeId + } + ArchivedJournalSnapshot0ClockidV1::ThreadCputimeId => { + wasi::Snapshot0Clockid::ThreadCputimeId + } + ArchivedJournalSnapshot0ClockidV1::Unknown => wasi::Snapshot0Clockid::Unknown, + } + } +} + +impl From for JournalWhenceV1 { + fn from(val: wasi::Whence) -> Self { + match val { + wasi::Whence::Set => JournalWhenceV1::Set, + wasi::Whence::Cur => JournalWhenceV1::Cur, + wasi::Whence::End => JournalWhenceV1::End, + wasi::Whence::Unknown => JournalWhenceV1::Unknown, + } + } +} + +impl From for wasi::Whence { + fn from(val: JournalWhenceV1) -> Self { + match val { + JournalWhenceV1::Set => wasi::Whence::Set, + JournalWhenceV1::Cur => wasi::Whence::Cur, + JournalWhenceV1::End => wasi::Whence::End, + JournalWhenceV1::Unknown => wasi::Whence::Unknown, + } + } +} + +impl From<&'_ ArchivedJournalWhenceV1> for wasi::Whence { + fn from(val: &'_ ArchivedJournalWhenceV1) -> Self { + match val { + ArchivedJournalWhenceV1::Set => wasi::Whence::Set, + ArchivedJournalWhenceV1::Cur => wasi::Whence::Cur, + ArchivedJournalWhenceV1::End => wasi::Whence::End, + ArchivedJournalWhenceV1::Unknown => wasi::Whence::Unknown, + } + } +} + +impl From for JournalAdviceV1 { + fn from(val: wasi::Advice) -> Self { + match val { + wasi::Advice::Normal => JournalAdviceV1::Normal, + wasi::Advice::Sequential => JournalAdviceV1::Sequential, + wasi::Advice::Random => JournalAdviceV1::Random, + wasi::Advice::Willneed => JournalAdviceV1::Willneed, + wasi::Advice::Dontneed => JournalAdviceV1::Dontneed, + wasi::Advice::Noreuse => JournalAdviceV1::Noreuse, + wasi::Advice::Unknown => JournalAdviceV1::Unknown, + } + } +} + +impl From for wasi::Advice { + fn from(val: JournalAdviceV1) -> Self { + match val { + JournalAdviceV1::Normal => wasi::Advice::Normal, + JournalAdviceV1::Sequential => wasi::Advice::Sequential, + JournalAdviceV1::Random => wasi::Advice::Random, + JournalAdviceV1::Willneed => wasi::Advice::Willneed, + JournalAdviceV1::Dontneed => wasi::Advice::Dontneed, + JournalAdviceV1::Noreuse => wasi::Advice::Noreuse, + JournalAdviceV1::Unknown => wasi::Advice::Unknown, + } + } +} + +impl From<&'_ ArchivedJournalAdviceV1> for wasi::Advice { + fn from(val: &'_ ArchivedJournalAdviceV1) -> Self { + match val { + ArchivedJournalAdviceV1::Normal => wasi::Advice::Normal, + ArchivedJournalAdviceV1::Sequential => wasi::Advice::Sequential, + ArchivedJournalAdviceV1::Random => wasi::Advice::Random, + ArchivedJournalAdviceV1::Willneed => wasi::Advice::Willneed, + ArchivedJournalAdviceV1::Dontneed => wasi::Advice::Dontneed, + ArchivedJournalAdviceV1::Noreuse => wasi::Advice::Noreuse, + ArchivedJournalAdviceV1::Unknown => wasi::Advice::Unknown, + } + } +} + +impl From for JournalIpCidrV1 { + fn from(value: virtual_net::IpCidr) -> Self { + Self { + ip: value.ip, + prefix: value.prefix, + } + } +} + +impl From for virtual_net::IpCidr { + fn from(value: JournalIpCidrV1) -> Self { + Self { + ip: value.ip, + prefix: value.prefix, + } + } +} + +impl From for JournalExitCodeV1 { + fn from(val: wasi::ExitCode) -> Self { + match val { + wasi::ExitCode::Errno(errno) => JournalExitCodeV1::Errno(errno as u16), + wasi::ExitCode::Other(id) => JournalExitCodeV1::Other(id), + } + } +} + +impl From for wasi::ExitCode { + fn from(val: JournalExitCodeV1) -> Self { + match val { + JournalExitCodeV1::Errno(errno) => { + wasi::ExitCode::Errno(errno.try_into().unwrap_or(wasi::Errno::Unknown)) + } + JournalExitCodeV1::Other(id) => wasi::ExitCode::Other(id), + } + } +} + +impl From<&'_ ArchivedJournalExitCodeV1> for wasi::ExitCode { + fn from(val: &'_ ArchivedJournalExitCodeV1) -> Self { + match val { + ArchivedJournalExitCodeV1::Errno(errno) => { + wasi::ExitCode::Errno((*errno).try_into().unwrap_or(wasi::Errno::Unknown)) + } + ArchivedJournalExitCodeV1::Other(id) => wasi::ExitCode::Other(*id), + } + } +} + +impl From for JournalSnapshotTriggerV1 { + fn from(val: SnapshotTrigger) -> Self { + match val { + SnapshotTrigger::Idle => JournalSnapshotTriggerV1::Idle, + SnapshotTrigger::FirstListen => JournalSnapshotTriggerV1::Listen, + SnapshotTrigger::FirstEnviron => JournalSnapshotTriggerV1::Environ, + SnapshotTrigger::FirstStdin => JournalSnapshotTriggerV1::Stdin, + SnapshotTrigger::PeriodicInterval => JournalSnapshotTriggerV1::Timer, + SnapshotTrigger::Sigint => JournalSnapshotTriggerV1::Sigint, + SnapshotTrigger::Sigalrm => JournalSnapshotTriggerV1::Sigalrm, + SnapshotTrigger::Sigtstp => JournalSnapshotTriggerV1::Sigtstp, + SnapshotTrigger::Sigstop => JournalSnapshotTriggerV1::Sigstop, + SnapshotTrigger::NonDeterministicCall => JournalSnapshotTriggerV1::NonDeterministicCall, + } + } +} + +impl From for SnapshotTrigger { + fn from(val: JournalSnapshotTriggerV1) -> Self { + match val { + JournalSnapshotTriggerV1::Idle => SnapshotTrigger::Idle, + JournalSnapshotTriggerV1::Listen => SnapshotTrigger::FirstListen, + JournalSnapshotTriggerV1::Environ => SnapshotTrigger::FirstEnviron, + JournalSnapshotTriggerV1::Stdin => SnapshotTrigger::FirstStdin, + JournalSnapshotTriggerV1::Timer => SnapshotTrigger::PeriodicInterval, + JournalSnapshotTriggerV1::Sigint => SnapshotTrigger::Sigint, + JournalSnapshotTriggerV1::Sigalrm => SnapshotTrigger::Sigalrm, + JournalSnapshotTriggerV1::Sigtstp => SnapshotTrigger::Sigtstp, + JournalSnapshotTriggerV1::Sigstop => SnapshotTrigger::Sigstop, + JournalSnapshotTriggerV1::NonDeterministicCall => SnapshotTrigger::NonDeterministicCall, + } + } +} + +impl From<&'_ ArchivedJournalSnapshotTriggerV1> for SnapshotTrigger { + fn from(val: &'_ ArchivedJournalSnapshotTriggerV1) -> Self { + match val { + ArchivedJournalSnapshotTriggerV1::Idle => SnapshotTrigger::Idle, + ArchivedJournalSnapshotTriggerV1::Listen => SnapshotTrigger::FirstListen, + ArchivedJournalSnapshotTriggerV1::Environ => SnapshotTrigger::FirstEnviron, + ArchivedJournalSnapshotTriggerV1::Stdin => SnapshotTrigger::FirstStdin, + ArchivedJournalSnapshotTriggerV1::Timer => SnapshotTrigger::PeriodicInterval, + ArchivedJournalSnapshotTriggerV1::Sigint => SnapshotTrigger::Sigint, + ArchivedJournalSnapshotTriggerV1::Sigalrm => SnapshotTrigger::Sigalrm, + ArchivedJournalSnapshotTriggerV1::Sigtstp => SnapshotTrigger::Sigtstp, + ArchivedJournalSnapshotTriggerV1::Sigstop => SnapshotTrigger::Sigstop, + ArchivedJournalSnapshotTriggerV1::NonDeterministicCall => { + SnapshotTrigger::NonDeterministicCall + } + } + } +} + +impl From for JournalEpollCtlV1 { + fn from(val: wasi::EpollCtl) -> Self { + match val { + wasi::EpollCtl::Add => JournalEpollCtlV1::Add, + wasi::EpollCtl::Mod => JournalEpollCtlV1::Mod, + wasi::EpollCtl::Del => JournalEpollCtlV1::Del, + wasi::EpollCtl::Unknown => JournalEpollCtlV1::Unknown, + } + } +} + +impl From for wasi::EpollCtl { + fn from(val: JournalEpollCtlV1) -> Self { + match val { + JournalEpollCtlV1::Add => wasi::EpollCtl::Add, + JournalEpollCtlV1::Mod => wasi::EpollCtl::Mod, + JournalEpollCtlV1::Del => wasi::EpollCtl::Del, + JournalEpollCtlV1::Unknown => wasi::EpollCtl::Unknown, + } + } +} + +impl From<&'_ ArchivedJournalEpollCtlV1> for wasi::EpollCtl { + fn from(val: &'_ ArchivedJournalEpollCtlV1) -> Self { + match val { + ArchivedJournalEpollCtlV1::Add => wasi::EpollCtl::Add, + ArchivedJournalEpollCtlV1::Mod => wasi::EpollCtl::Mod, + ArchivedJournalEpollCtlV1::Del => wasi::EpollCtl::Del, + ArchivedJournalEpollCtlV1::Unknown => wasi::EpollCtl::Unknown, + } + } +} + +impl From for JournalEpollEventCtlV1 { + fn from(val: wasi::EpollEventCtl) -> Self { + JournalEpollEventCtlV1 { + events: val.events.bits(), + ptr: val.ptr, + fd: val.fd, + data1: val.data1, + data2: val.data2, + } + } +} + +impl From for wasi::EpollEventCtl { + fn from(val: JournalEpollEventCtlV1) -> Self { + Self { + events: wasi::EpollType::from_bits_truncate(val.events), + ptr: val.ptr, + fd: val.fd, + data1: val.data1, + data2: val.data2, + } + } +} + +impl From<&'_ ArchivedJournalEpollEventCtlV1> for wasi::EpollEventCtl { + fn from(val: &'_ ArchivedJournalEpollEventCtlV1) -> Self { + Self { + events: wasi::EpollType::from_bits_truncate(val.events), + ptr: val.ptr, + fd: val.fd, + data1: val.data1, + data2: val.data2, + } + } +} + +impl From for JournalStreamSecurityV1 { + fn from(val: virtual_net::StreamSecurity) -> Self { + use virtual_net::StreamSecurity; + match val { + StreamSecurity::Unencrypted => JournalStreamSecurityV1::Unencrypted, + StreamSecurity::AnyEncyption => JournalStreamSecurityV1::AnyEncryption, + StreamSecurity::ClassicEncryption => JournalStreamSecurityV1::ClassicEncryption, + StreamSecurity::DoubleEncryption => JournalStreamSecurityV1::DoubleEncryption, + } + } +} + +impl From for virtual_net::StreamSecurity { + fn from(val: JournalStreamSecurityV1) -> Self { + use virtual_net::StreamSecurity; + match val { + JournalStreamSecurityV1::Unencrypted => StreamSecurity::Unencrypted, + JournalStreamSecurityV1::AnyEncryption => StreamSecurity::AnyEncyption, + JournalStreamSecurityV1::ClassicEncryption => StreamSecurity::ClassicEncryption, + JournalStreamSecurityV1::DoubleEncryption => StreamSecurity::DoubleEncryption, + JournalStreamSecurityV1::Unknown => StreamSecurity::AnyEncyption, + } + } +} + +impl From<&'_ ArchivedJournalStreamSecurityV1> for virtual_net::StreamSecurity { + fn from(val: &'_ ArchivedJournalStreamSecurityV1) -> Self { + use virtual_net::StreamSecurity; + match val { + ArchivedJournalStreamSecurityV1::Unencrypted => StreamSecurity::Unencrypted, + ArchivedJournalStreamSecurityV1::AnyEncryption => StreamSecurity::AnyEncyption, + ArchivedJournalStreamSecurityV1::ClassicEncryption => StreamSecurity::ClassicEncryption, + ArchivedJournalStreamSecurityV1::DoubleEncryption => StreamSecurity::DoubleEncryption, + ArchivedJournalStreamSecurityV1::Unknown => StreamSecurity::AnyEncyption, + } + } +} + +impl From for JournalAddressfamilyV1 { + fn from(val: wasi::Addressfamily) -> Self { + match val { + wasi::Addressfamily::Unspec => JournalAddressfamilyV1::Unspec, + wasi::Addressfamily::Inet4 => JournalAddressfamilyV1::Inet4, + wasi::Addressfamily::Inet6 => JournalAddressfamilyV1::Inet6, + wasi::Addressfamily::Unix => JournalAddressfamilyV1::Unix, + } + } +} + +impl From for wasi::Addressfamily { + fn from(val: JournalAddressfamilyV1) -> Self { + match val { + JournalAddressfamilyV1::Unspec => wasi::Addressfamily::Unspec, + JournalAddressfamilyV1::Inet4 => wasi::Addressfamily::Inet4, + JournalAddressfamilyV1::Inet6 => wasi::Addressfamily::Inet6, + JournalAddressfamilyV1::Unix => wasi::Addressfamily::Unix, + } + } +} + +impl From<&'_ ArchivedJournalAddressfamilyV1> for wasi::Addressfamily { + fn from(val: &'_ ArchivedJournalAddressfamilyV1) -> Self { + match val { + ArchivedJournalAddressfamilyV1::Unspec => wasi::Addressfamily::Unspec, + ArchivedJournalAddressfamilyV1::Inet4 => wasi::Addressfamily::Inet4, + ArchivedJournalAddressfamilyV1::Inet6 => wasi::Addressfamily::Inet6, + ArchivedJournalAddressfamilyV1::Unix => wasi::Addressfamily::Unix, + } + } +} + +impl From for JournalSocktypeV1 { + fn from(val: wasi::Socktype) -> Self { + match val { + wasi::Socktype::Stream => JournalSocktypeV1::Stream, + wasi::Socktype::Dgram => JournalSocktypeV1::Dgram, + wasi::Socktype::Raw => JournalSocktypeV1::Raw, + wasi::Socktype::Seqpacket => JournalSocktypeV1::Seqpacket, + wasi::Socktype::Unknown => JournalSocktypeV1::Unknown, + } + } +} + +impl From for wasi::Socktype { + fn from(val: JournalSocktypeV1) -> Self { + match val { + JournalSocktypeV1::Stream => wasi::Socktype::Stream, + JournalSocktypeV1::Dgram => wasi::Socktype::Dgram, + JournalSocktypeV1::Raw => wasi::Socktype::Raw, + JournalSocktypeV1::Seqpacket => wasi::Socktype::Seqpacket, + JournalSocktypeV1::Unknown => wasi::Socktype::Unknown, + } + } +} + +impl From<&'_ ArchivedJournalSocktypeV1> for wasi::Socktype { + fn from(val: &'_ ArchivedJournalSocktypeV1) -> Self { + match val { + ArchivedJournalSocktypeV1::Stream => wasi::Socktype::Stream, + ArchivedJournalSocktypeV1::Dgram => wasi::Socktype::Dgram, + ArchivedJournalSocktypeV1::Raw => wasi::Socktype::Raw, + ArchivedJournalSocktypeV1::Seqpacket => wasi::Socktype::Seqpacket, + ArchivedJournalSocktypeV1::Unknown => wasi::Socktype::Unknown, + } + } +} + +impl From for JournalSockoptionV1 { + fn from(val: wasi::Sockoption) -> Self { + match val { + wasi::Sockoption::Noop => JournalSockoptionV1::Noop, + wasi::Sockoption::ReusePort => JournalSockoptionV1::ReusePort, + wasi::Sockoption::ReuseAddr => JournalSockoptionV1::ReuseAddr, + wasi::Sockoption::NoDelay => JournalSockoptionV1::NoDelay, + wasi::Sockoption::DontRoute => JournalSockoptionV1::DontRoute, + wasi::Sockoption::OnlyV6 => JournalSockoptionV1::OnlyV6, + wasi::Sockoption::Broadcast => JournalSockoptionV1::Broadcast, + wasi::Sockoption::MulticastLoopV4 => JournalSockoptionV1::MulticastLoopV4, + wasi::Sockoption::MulticastLoopV6 => JournalSockoptionV1::MulticastLoopV6, + wasi::Sockoption::Promiscuous => JournalSockoptionV1::Promiscuous, + wasi::Sockoption::Listening => JournalSockoptionV1::Listening, + wasi::Sockoption::LastError => JournalSockoptionV1::LastError, + wasi::Sockoption::KeepAlive => JournalSockoptionV1::KeepAlive, + wasi::Sockoption::Linger => JournalSockoptionV1::Linger, + wasi::Sockoption::OobInline => JournalSockoptionV1::OobInline, + wasi::Sockoption::RecvBufSize => JournalSockoptionV1::RecvBufSize, + wasi::Sockoption::SendBufSize => JournalSockoptionV1::SendBufSize, + wasi::Sockoption::RecvLowat => JournalSockoptionV1::RecvLowat, + wasi::Sockoption::SendLowat => JournalSockoptionV1::SendLowat, + wasi::Sockoption::RecvTimeout => JournalSockoptionV1::RecvTimeout, + wasi::Sockoption::SendTimeout => JournalSockoptionV1::SendTimeout, + wasi::Sockoption::ConnectTimeout => JournalSockoptionV1::ConnectTimeout, + wasi::Sockoption::AcceptTimeout => JournalSockoptionV1::AcceptTimeout, + wasi::Sockoption::Ttl => JournalSockoptionV1::Ttl, + wasi::Sockoption::MulticastTtlV4 => JournalSockoptionV1::MulticastTtlV4, + wasi::Sockoption::Type => JournalSockoptionV1::Type, + wasi::Sockoption::Proto => JournalSockoptionV1::Proto, + } + } +} + +impl From for wasi::Sockoption { + fn from(val: JournalSockoptionV1) -> Self { + match val { + JournalSockoptionV1::Noop => wasi::Sockoption::Noop, + JournalSockoptionV1::ReusePort => wasi::Sockoption::ReusePort, + JournalSockoptionV1::ReuseAddr => wasi::Sockoption::ReuseAddr, + JournalSockoptionV1::NoDelay => wasi::Sockoption::NoDelay, + JournalSockoptionV1::DontRoute => wasi::Sockoption::DontRoute, + JournalSockoptionV1::OnlyV6 => wasi::Sockoption::OnlyV6, + JournalSockoptionV1::Broadcast => wasi::Sockoption::Broadcast, + JournalSockoptionV1::MulticastLoopV4 => wasi::Sockoption::MulticastLoopV4, + JournalSockoptionV1::MulticastLoopV6 => wasi::Sockoption::MulticastLoopV6, + JournalSockoptionV1::Promiscuous => wasi::Sockoption::Promiscuous, + JournalSockoptionV1::Listening => wasi::Sockoption::Listening, + JournalSockoptionV1::LastError => wasi::Sockoption::LastError, + JournalSockoptionV1::KeepAlive => wasi::Sockoption::KeepAlive, + JournalSockoptionV1::Linger => wasi::Sockoption::Linger, + JournalSockoptionV1::OobInline => wasi::Sockoption::OobInline, + JournalSockoptionV1::RecvBufSize => wasi::Sockoption::RecvBufSize, + JournalSockoptionV1::SendBufSize => wasi::Sockoption::SendBufSize, + JournalSockoptionV1::RecvLowat => wasi::Sockoption::RecvLowat, + JournalSockoptionV1::SendLowat => wasi::Sockoption::SendLowat, + JournalSockoptionV1::RecvTimeout => wasi::Sockoption::RecvTimeout, + JournalSockoptionV1::SendTimeout => wasi::Sockoption::SendTimeout, + JournalSockoptionV1::ConnectTimeout => wasi::Sockoption::ConnectTimeout, + JournalSockoptionV1::AcceptTimeout => wasi::Sockoption::AcceptTimeout, + JournalSockoptionV1::Ttl => wasi::Sockoption::Ttl, + JournalSockoptionV1::MulticastTtlV4 => wasi::Sockoption::MulticastTtlV4, + JournalSockoptionV1::Type => wasi::Sockoption::Type, + JournalSockoptionV1::Proto => wasi::Sockoption::Proto, + } + } +} + +impl From<&'_ ArchivedJournalSockoptionV1> for wasi::Sockoption { + fn from(val: &'_ ArchivedJournalSockoptionV1) -> Self { + match val { + ArchivedJournalSockoptionV1::Noop => wasi::Sockoption::Noop, + ArchivedJournalSockoptionV1::ReusePort => wasi::Sockoption::ReusePort, + ArchivedJournalSockoptionV1::ReuseAddr => wasi::Sockoption::ReuseAddr, + ArchivedJournalSockoptionV1::NoDelay => wasi::Sockoption::NoDelay, + ArchivedJournalSockoptionV1::DontRoute => wasi::Sockoption::DontRoute, + ArchivedJournalSockoptionV1::OnlyV6 => wasi::Sockoption::OnlyV6, + ArchivedJournalSockoptionV1::Broadcast => wasi::Sockoption::Broadcast, + ArchivedJournalSockoptionV1::MulticastLoopV4 => wasi::Sockoption::MulticastLoopV4, + ArchivedJournalSockoptionV1::MulticastLoopV6 => wasi::Sockoption::MulticastLoopV6, + ArchivedJournalSockoptionV1::Promiscuous => wasi::Sockoption::Promiscuous, + ArchivedJournalSockoptionV1::Listening => wasi::Sockoption::Listening, + ArchivedJournalSockoptionV1::LastError => wasi::Sockoption::LastError, + ArchivedJournalSockoptionV1::KeepAlive => wasi::Sockoption::KeepAlive, + ArchivedJournalSockoptionV1::Linger => wasi::Sockoption::Linger, + ArchivedJournalSockoptionV1::OobInline => wasi::Sockoption::OobInline, + ArchivedJournalSockoptionV1::RecvBufSize => wasi::Sockoption::RecvBufSize, + ArchivedJournalSockoptionV1::SendBufSize => wasi::Sockoption::SendBufSize, + ArchivedJournalSockoptionV1::RecvLowat => wasi::Sockoption::RecvLowat, + ArchivedJournalSockoptionV1::SendLowat => wasi::Sockoption::SendLowat, + ArchivedJournalSockoptionV1::RecvTimeout => wasi::Sockoption::RecvTimeout, + ArchivedJournalSockoptionV1::SendTimeout => wasi::Sockoption::SendTimeout, + ArchivedJournalSockoptionV1::ConnectTimeout => wasi::Sockoption::ConnectTimeout, + ArchivedJournalSockoptionV1::AcceptTimeout => wasi::Sockoption::AcceptTimeout, + ArchivedJournalSockoptionV1::Ttl => wasi::Sockoption::Ttl, + ArchivedJournalSockoptionV1::MulticastTtlV4 => wasi::Sockoption::MulticastTtlV4, + ArchivedJournalSockoptionV1::Type => wasi::Sockoption::Type, + ArchivedJournalSockoptionV1::Proto => wasi::Sockoption::Proto, + } + } +} + +impl From for JournalTimeTypeV1 { + fn from(val: SocketOptTimeType) -> Self { + match val { + SocketOptTimeType::ReadTimeout => JournalTimeTypeV1::ReadTimeout, + SocketOptTimeType::WriteTimeout => JournalTimeTypeV1::WriteTimeout, + SocketOptTimeType::AcceptTimeout => JournalTimeTypeV1::AcceptTimeout, + SocketOptTimeType::ConnectTimeout => JournalTimeTypeV1::ConnectTimeout, + SocketOptTimeType::BindTimeout => JournalTimeTypeV1::BindTimeout, + SocketOptTimeType::Linger => JournalTimeTypeV1::Linger, + } + } +} + +impl From for SocketOptTimeType { + fn from(val: JournalTimeTypeV1) -> Self { + match val { + JournalTimeTypeV1::ReadTimeout => SocketOptTimeType::ReadTimeout, + JournalTimeTypeV1::WriteTimeout => SocketOptTimeType::WriteTimeout, + JournalTimeTypeV1::AcceptTimeout => SocketOptTimeType::AcceptTimeout, + JournalTimeTypeV1::ConnectTimeout => SocketOptTimeType::ConnectTimeout, + JournalTimeTypeV1::BindTimeout => SocketOptTimeType::BindTimeout, + JournalTimeTypeV1::Linger => SocketOptTimeType::Linger, + } + } +} + +impl From<&'_ ArchivedJournalTimeTypeV1> for SocketOptTimeType { + fn from(val: &'_ ArchivedJournalTimeTypeV1) -> Self { + match val { + ArchivedJournalTimeTypeV1::ReadTimeout => SocketOptTimeType::ReadTimeout, + ArchivedJournalTimeTypeV1::WriteTimeout => SocketOptTimeType::WriteTimeout, + ArchivedJournalTimeTypeV1::AcceptTimeout => SocketOptTimeType::AcceptTimeout, + ArchivedJournalTimeTypeV1::ConnectTimeout => SocketOptTimeType::ConnectTimeout, + ArchivedJournalTimeTypeV1::BindTimeout => SocketOptTimeType::BindTimeout, + ArchivedJournalTimeTypeV1::Linger => SocketOptTimeType::Linger, + } + } +} + +impl From for JournalSocketShutdownV1 { + fn from(val: SocketShutdownHow) -> Self { + match val { + SocketShutdownHow::Read => JournalSocketShutdownV1::Read, + SocketShutdownHow::Write => JournalSocketShutdownV1::Write, + SocketShutdownHow::Both => JournalSocketShutdownV1::Both, + } + } +} + +impl From for SocketShutdownHow { + fn from(val: JournalSocketShutdownV1) -> Self { + match val { + JournalSocketShutdownV1::Read => SocketShutdownHow::Read, + JournalSocketShutdownV1::Write => SocketShutdownHow::Write, + JournalSocketShutdownV1::Both => SocketShutdownHow::Both, + } + } +} + +impl From<&'_ ArchivedJournalSocketShutdownV1> for SocketShutdownHow { + fn from(val: &'_ ArchivedJournalSocketShutdownV1) -> Self { + match val { + ArchivedJournalSocketShutdownV1::Read => SocketShutdownHow::Read, + ArchivedJournalSocketShutdownV1::Write => SocketShutdownHow::Write, + ArchivedJournalSocketShutdownV1::Both => SocketShutdownHow::Both, + } + } +} + +impl<'a> TryFrom> for JournalEntry<'a> { + type Error = anyhow::Error; + + fn try_from(value: ArchivedJournalEntry<'a>) -> anyhow::Result { + Ok(match value { + ArchivedJournalEntry::InitModuleV1(ArchivedJournalEntryInitModuleV1 { wasm_hash }) => { + Self::InitModuleV1 { + wasm_hash: *wasm_hash, + } + } + ArchivedJournalEntry::UpdateMemoryRegionV1( + ArchivedJournalEntryUpdateMemoryRegionV1 { + start, + end, + compressed_data, + _padding: _, + }, + ) => Self::UpdateMemoryRegionV1 { + region: (*start)..(*end), + data: Cow::Owned(decompress_size_prepended(compressed_data.as_ref())?), + }, + ArchivedJournalEntry::ProcessExitV1(ArchivedJournalEntryProcessExitV1 { + exit_code, + _padding: _, + }) => Self::ProcessExitV1 { + exit_code: exit_code.as_ref().map(|code| code.into()), + }, + ArchivedJournalEntry::SetThreadV1(ArchivedJournalEntrySetThreadV1 { + id, + call_stack, + memory_stack, + store_data, + _padding: _, + is_64bit, + }) => Self::SetThreadV1 { + id: *id, + call_stack: call_stack.as_ref().into(), + memory_stack: memory_stack.as_ref().into(), + store_data: store_data.as_ref().into(), + is_64bit: *is_64bit, + }, + ArchivedJournalEntry::CloseThreadV1(ArchivedJournalEntryCloseThreadV1 { + id, + exit_code, + }) => Self::CloseThreadV1 { + id: *id, + exit_code: exit_code.as_ref().map(|code| code.into()), + }, + ArchivedJournalEntry::FileDescriptorWriteV1( + ArchivedJournalEntryFileDescriptorWriteV1 { + data, + fd, + offset, + is_64bit, + _padding: _, + }, + ) => Self::FileDescriptorWriteV1 { + data: data.as_ref().into(), + fd: *fd, + offset: *offset, + is_64bit: *is_64bit, + }, + ArchivedJournalEntry::FileDescriptorSeekV1( + ArchivedJournalEntryFileDescriptorSeekV1 { + fd, + offset, + ref whence, + }, + ) => Self::FileDescriptorSeekV1 { + fd: *fd, + offset: *offset, + whence: whence.into(), + }, + ArchivedJournalEntry::OpenFileDescriptorV1( + ArchivedJournalEntryOpenFileDescriptorV1 { + fd, + dirfd, + dirflags, + path, + o_flags, + fs_rights_base, + fs_rights_inheriting, + fs_flags, + _padding: _, + }, + ) => Self::OpenFileDescriptorV1 { + fd: *fd, + dirfd: *dirfd, + dirflags: *dirflags, + path: path.as_ref().into(), + o_flags: wasi::Oflags::from_bits_truncate(*o_flags), + fs_rights_base: wasi::Rights::from_bits_truncate(*fs_rights_base), + fs_rights_inheriting: wasi::Rights::from_bits_truncate(*fs_rights_inheriting), + fs_flags: wasi::Fdflags::from_bits_truncate(*fs_flags), + }, + ArchivedJournalEntry::CloseFileDescriptorV1( + ArchivedJournalEntryCloseFileDescriptorV1 { fd, _padding: _ }, + ) => Self::CloseFileDescriptorV1 { fd: *fd }, + ArchivedJournalEntry::RemoveDirectoryV1(ArchivedJournalEntryRemoveDirectoryV1 { + fd, + path, + _padding: _, + }) => Self::RemoveDirectoryV1 { + fd: *fd, + path: path.as_ref().into(), + }, + ArchivedJournalEntry::UnlinkFileV1(ArchivedJournalEntryUnlinkFileV1 { + fd, + path, + _padding: _, + }) => Self::UnlinkFileV1 { + fd: *fd, + path: path.as_ref().into(), + }, + ArchivedJournalEntry::PathRenameV1(ArchivedJournalEntryPathRenameV1 { + old_fd, + old_path, + new_fd, + new_path, + _padding: _, + }) => Self::PathRenameV1 { + old_fd: *old_fd, + old_path: old_path.as_ref().into(), + new_fd: *new_fd, + new_path: new_path.as_ref().into(), + }, + ArchivedJournalEntry::SnapshotV1(ArchivedJournalEntrySnapshotV1 { + since_epoch, + ref trigger, + }) => Self::SnapshotV1 { + when: SystemTime::UNIX_EPOCH + .checked_add((*since_epoch).try_into().unwrap()) + .unwrap_or(SystemTime::UNIX_EPOCH), + trigger: trigger.into(), + }, + ArchivedJournalEntry::SetClockTimeV1(ArchivedJournalEntrySetClockTimeV1 { + ref clock_id, + time, + }) => Self::SetClockTimeV1 { + clock_id: clock_id.into(), + time: *time, + }, + ArchivedJournalEntry::RenumberFileDescriptorV1( + ArchivedJournalEntryRenumberFileDescriptorV1 { old_fd, new_fd }, + ) => Self::RenumberFileDescriptorV1 { + old_fd: *old_fd, + new_fd: *new_fd, + }, + ArchivedJournalEntry::DuplicateFileDescriptorV1( + ArchivedJournalEntryDuplicateFileDescriptorV1 { + original_fd: old_fd, + copied_fd: new_fd, + }, + ) => Self::DuplicateFileDescriptorV1 { + original_fd: *old_fd, + copied_fd: *new_fd, + }, + ArchivedJournalEntry::CreateDirectoryV1(ArchivedJournalEntryCreateDirectoryV1 { + fd, + path, + _padding: _, + }) => Self::CreateDirectoryV1 { + fd: *fd, + path: path.as_ref().into(), + }, + ArchivedJournalEntry::PathSetTimesV1(ArchivedJournalEntryPathSetTimesV1 { + fd, + path, + flags, + st_atim, + st_mtim, + fst_flags, + _padding: _, + }) => Self::PathSetTimesV1 { + fd: *fd, + path: path.as_ref().into(), + flags: *flags, + st_atim: *st_atim, + st_mtim: *st_mtim, + fst_flags: wasi::Fstflags::from_bits_truncate(*fst_flags), + }, + ArchivedJournalEntry::FileDescriptorSetTimesV1( + ArchivedJournalEntryFileDescriptorSetTimesV1 { + fd, + st_atim, + st_mtim, + fst_flags, + }, + ) => Self::FileDescriptorSetTimesV1 { + fd: *fd, + st_atim: *st_atim, + st_mtim: *st_mtim, + fst_flags: wasi::Fstflags::from_bits_truncate(*fst_flags), + }, + ArchivedJournalEntry::FileDescriptorSetSizeV1( + ArchivedJournalEntryFileDescriptorSetSizeV1 { fd, st_size }, + ) => Self::FileDescriptorSetSizeV1 { + fd: *fd, + st_size: *st_size, + }, + ArchivedJournalEntry::FileDescriptorSetFlagsV1( + ArchivedJournalEntryFileDescriptorSetFlagsV1 { fd, flags }, + ) => Self::FileDescriptorSetFlagsV1 { + fd: *fd, + flags: wasi::Fdflags::from_bits_truncate(*flags), + }, + ArchivedJournalEntry::FileDescriptorSetRightsV1( + ArchivedJournalEntryFileDescriptorSetRightsV1 { + fd, + fs_rights_base, + fs_rights_inheriting, + }, + ) => Self::FileDescriptorSetRightsV1 { + fd: *fd, + fs_rights_base: wasi::Rights::from_bits_truncate(*fs_rights_base), + fs_rights_inheriting: wasi::Rights::from_bits_truncate(*fs_rights_inheriting), + }, + ArchivedJournalEntry::FileDescriptorAdviseV1( + ArchivedJournalEntryFileDescriptorAdviseV1 { + fd, + offset, + len, + ref advice, + }, + ) => Self::FileDescriptorAdviseV1 { + fd: *fd, + offset: *offset, + len: *len, + advice: advice.into(), + }, + ArchivedJournalEntry::FileDescriptorAllocateV1( + ArchivedJournalEntryFileDescriptorAllocateV1 { fd, offset, len }, + ) => Self::FileDescriptorAllocateV1 { + fd: *fd, + offset: *offset, + len: *len, + }, + ArchivedJournalEntry::CreateHardLinkV1(ArchivedJournalEntryCreateHardLinkV1 { + old_fd, + old_path, + old_flags, + new_fd, + new_path, + _padding: _, + }) => Self::CreateHardLinkV1 { + old_fd: *old_fd, + old_path: old_path.as_ref().into(), + old_flags: *old_flags, + new_fd: *new_fd, + new_path: new_path.as_ref().into(), + }, + ArchivedJournalEntry::CreateSymbolicLinkV1( + ArchivedJournalEntryCreateSymbolicLinkV1 { + old_path, + fd, + new_path, + _padding: _, + }, + ) => Self::CreateSymbolicLinkV1 { + old_path: old_path.as_ref().into(), + fd: *fd, + new_path: new_path.as_ref().into(), + }, + ArchivedJournalEntry::ChangeDirectoryV1(ArchivedJournalEntryChangeDirectoryV1 { + path, + }) => Self::ChangeDirectoryV1 { + path: path.as_ref().into(), + }, + ArchivedJournalEntry::EpollCreateV1(ArchivedJournalEntryEpollCreateV1 { + fd, + _padding: _, + }) => Self::EpollCreateV1 { fd: *fd }, + ArchivedJournalEntry::EpollCtlV1(ArchivedJournalEntryEpollCtlV1 { + epfd, + ref op, + fd, + ref event, + }) => Self::EpollCtlV1 { + epfd: *epfd, + op: op.into(), + fd: *fd, + event: event.as_ref().map(|e| e.into()), + }, + ArchivedJournalEntry::TtySetV1(ArchivedJournalEntryTtySetV1 { + cols, + rows, + width, + height, + stdin_tty, + stdout_tty, + stderr_tty, + echo, + line_buffered, + line_feeds, + }) => Self::TtySetV1 { + tty: wasi::Tty { + cols: *cols, + rows: *rows, + width: *width, + height: *height, + stdin_tty: *stdin_tty, + stdout_tty: *stdout_tty, + stderr_tty: *stderr_tty, + echo: *echo, + line_buffered: *line_buffered, + }, + line_feeds: *line_feeds, + }, + ArchivedJournalEntry::CreatePipeV1(ArchivedJournalEntryCreatePipeV1 { fd1, fd2 }) => { + Self::CreatePipeV1 { + fd1: *fd1, + fd2: *fd2, + } + } + ArchivedJournalEntry::PortAddAddrV1(ArchivedJournalEntryPortAddAddrV1 { cidr }) => { + Self::PortAddAddrV1 { + cidr: JournalIpCidrV1 { + ip: cidr.ip.as_ipaddr(), + prefix: cidr.prefix, + } + .into(), + } + } + ArchivedJournalEntry::PortDelAddrV1(ArchivedJournalEntryPortDelAddrV1 { addr }) => { + Self::PortDelAddrV1 { + addr: addr.as_ipaddr(), + } + } + ArchivedJournalEntry::PortAddrClearV1 => Self::PortAddrClearV1, + ArchivedJournalEntry::PortBridgeV1(ArchivedJournalEntryPortBridgeV1 { + network, + token, + ref security, + _padding: _, + }) => Self::PortBridgeV1 { + network: network.as_ref().into(), + token: token.as_ref().into(), + security: security.into(), + }, + ArchivedJournalEntry::PortUnbridgeV1 => Self::PortUnbridgeV1, + ArchivedJournalEntry::PortDhcpAcquireV1 => Self::PortDhcpAcquireV1, + ArchivedJournalEntry::PortGatewaySetV1(ArchivedJournalEntryPortGatewaySetV1 { ip }) => { + Self::PortGatewaySetV1 { ip: ip.as_ipaddr() } + } + ArchivedJournalEntry::PortRouteAddV1(ArchivedJournalEntryPortRouteAddV1 { + cidr, + via_router, + preferred_until, + expires_at, + }) => Self::PortRouteAddV1 { + cidr: JournalIpCidrV1 { + ip: cidr.ip.as_ipaddr(), + prefix: cidr.prefix, + } + .into(), + via_router: via_router.as_ipaddr(), + preferred_until: preferred_until + .as_ref() + .map(|time| (*time).try_into().unwrap()), + expires_at: expires_at.as_ref().map(|time| (*time).try_into().unwrap()), + }, + ArchivedJournalEntry::PortRouteClearV1 => Self::PortRouteClearV1, + ArchivedJournalEntry::PortRouteDelV1(ArchivedJournalEntryPortRouteDelV1 { ip }) => { + Self::PortRouteDelV1 { ip: ip.as_ipaddr() } + } + ArchivedJournalEntry::SocketOpenV1(ArchivedJournalEntrySocketOpenV1 { + ref af, + ref ty, + pt, + fd, + }) => Self::SocketOpenV1 { + af: af.into(), + ty: ty.into(), + pt: (*pt).try_into().unwrap_or(wasi::SockProto::Max), + fd: *fd, + }, + ArchivedJournalEntry::SocketListenV1(ArchivedJournalEntrySocketListenV1 { + fd, + backlog, + }) => Self::SocketListenV1 { + fd: *fd, + backlog: *backlog, + }, + ArchivedJournalEntry::SocketBindV1(ArchivedJournalEntrySocketBindV1 { fd, addr }) => { + Self::SocketBindV1 { + fd: *fd, + addr: addr.as_socket_addr(), + } + } + ArchivedJournalEntry::SocketConnectedV1(ArchivedJournalEntrySocketConnectedV1 { + fd, + addr, + }) => Self::SocketConnectedV1 { + fd: *fd, + addr: addr.as_socket_addr(), + }, + ArchivedJournalEntry::SocketAcceptedV1(ArchivedJournalEntrySocketAcceptedV1 { + listen_fd, + fd, + peer_addr, + fd_flags, + nonblocking, + }) => Self::SocketAcceptedV1 { + listen_fd: *listen_fd, + fd: *fd, + peer_addr: peer_addr.as_socket_addr(), + fd_flags: wasi::Fdflags::from_bits_truncate(*fd_flags), + non_blocking: *nonblocking, + }, + ArchivedJournalEntry::SocketJoinIpv4MulticastV1( + ArchivedJournalEntrySocketJoinIpv4MulticastV1 { + fd, + multiaddr, + iface, + }, + ) => Self::SocketJoinIpv4MulticastV1 { + fd: *fd, + multiaddr: multiaddr.as_ipv4(), + iface: iface.as_ipv4(), + }, + ArchivedJournalEntry::SocketJoinIpv6MulticastV1( + ArchivedJournalEntrySocketJoinIpv6MulticastV1 { + fd, + multiaddr, + iface, + }, + ) => Self::SocketJoinIpv6MulticastV1 { + fd: *fd, + multi_addr: multiaddr.as_ipv6(), + iface: *iface, + }, + ArchivedJournalEntry::SocketLeaveIpv4MulticastV1( + ArchivedJournalEntrySocketLeaveIpv4MulticastV1 { + fd, + multiaddr, + iface, + }, + ) => Self::SocketLeaveIpv4MulticastV1 { + fd: *fd, + multi_addr: multiaddr.as_ipv4(), + iface: iface.as_ipv4(), + }, + ArchivedJournalEntry::SocketLeaveIpv6MulticastV1( + ArchivedJournalEntrySocketLeaveIpv6MulticastV1 { + fd, + multiaddr, + iface, + }, + ) => Self::SocketLeaveIpv6MulticastV1 { + fd: *fd, + multi_addr: multiaddr.as_ipv6(), + iface: *iface, + }, + ArchivedJournalEntry::SocketSendFileV1(ArchivedJournalEntrySocketSendFileV1 { + socket_fd, + file_fd, + offset, + count, + }) => Self::SocketSendFileV1 { + socket_fd: *socket_fd, + file_fd: *file_fd, + offset: *offset, + count: *count, + }, + ArchivedJournalEntry::SocketSendToV1(ArchivedJournalEntrySocketSendToV1 { + fd, + data, + flags, + addr, + is_64bit, + _padding: _, + }) => Self::SocketSendToV1 { + fd: *fd, + data: data.as_ref().into(), + flags: *flags, + addr: addr.as_socket_addr(), + is_64bit: *is_64bit, + }, + ArchivedJournalEntry::SocketSendV1(ArchivedJournalEntrySocketSendV1 { + fd, + data, + flags, + is_64bit, + _padding: _, + }) => Self::SocketSendV1 { + fd: *fd, + data: data.as_ref().into(), + flags: *flags, + is_64bit: *is_64bit, + }, + ArchivedJournalEntry::SocketSetOptFlagV1(ArchivedJournalEntrySocketSetOptFlagV1 { + fd, + ref opt, + flag, + }) => Self::SocketSetOptFlagV1 { + fd: *fd, + opt: opt.into(), + flag: *flag, + }, + ArchivedJournalEntry::SocketSetOptSizeV1(ArchivedJournalEntrySocketSetOptSizeV1 { + fd, + ref opt, + size, + }) => Self::SocketSetOptSizeV1 { + fd: *fd, + opt: opt.into(), + size: *size, + }, + ArchivedJournalEntry::SocketSetOptTimeV1(ArchivedJournalEntrySocketSetOptTimeV1 { + fd, + ref ty, + time, + }) => Self::SocketSetOptTimeV1 { + fd: *fd, + ty: ty.into(), + time: time.as_ref().map(|time| (*time).try_into().unwrap()), + }, + ArchivedJournalEntry::SocketShutdownV1(ArchivedJournalEntrySocketShutdownV1 { + fd, + ref how, + }) => Self::SocketShutdownV1 { + fd: *fd, + how: how.into(), + }, + ArchivedJournalEntry::CreateEventV1(ArchivedJournalEntryCreateEventV1 { + initial_val, + flags, + fd, + }) => Self::CreateEventV1 { + initial_val: *initial_val, + flags: *flags, + fd: *fd, + }, + }) + } +} diff --git a/lib/journal/src/concrete/mod.rs b/lib/journal/src/concrete/mod.rs index f6b666d8be1..9cc1abe2224 100644 --- a/lib/journal/src/concrete/mod.rs +++ b/lib/journal/src/concrete/mod.rs @@ -1,5 +1,6 @@ mod arc; mod archived; +mod archived_from; mod boxed; mod buffered; mod compacting; @@ -13,6 +14,8 @@ mod null; mod pipe; mod printing; mod recombined; +#[cfg(test)] +mod tests; mod unsupported; pub(super) use super::*; diff --git a/lib/journal/src/concrete/tests.rs b/lib/journal/src/concrete/tests.rs new file mode 100644 index 00000000000..cc48dedb4af --- /dev/null +++ b/lib/journal/src/concrete/tests.rs @@ -0,0 +1,715 @@ +use std::{ + net::{Ipv4Addr, Ipv6Addr, SocketAddr}, + time::{Duration, SystemTime}, +}; + +use super::*; +use rkyv::ser::serializers::{ + AllocScratch, CompositeSerializer, SharedSerializeMap, WriteSerializer, +}; +use wasmer_wasix_types::wasi; + +pub fn run_test<'a>(record: JournalEntry<'a>) { + tracing::info!("record: {:?}", record); + + // Determine the record type + let record_type = record.archive_record_type(); + tracing::info!("record_type: {:?}", record_type); + + // Serialize it + let mut buffer = Vec::new(); + let mut serializer = CompositeSerializer::new( + WriteSerializer::new(&mut buffer), + AllocScratch::default(), + SharedSerializeMap::default(), + ); + + record.clone().serialize_archive(&mut serializer).unwrap(); + let buffer = &buffer[..]; + if buffer.len() < 20 { + tracing::info!("buffer: {:x?}", buffer); + } else { + tracing::info!("buffer_len: {}", buffer.len()); + } + + // Deserialize it + let record2 = unsafe { record_type.deserialize_archive(buffer).unwrap() }; + tracing::info!("record2: {:?}", record2); + + // Check it + assert_eq!(record, record2); + + // Now make it static and check it again + let record3 = record2.into_owned(); + tracing::info!("record3: {:?}", record3); + assert_eq!(record, record3); +} + +#[tracing_test::traced_test] +#[test] +pub fn test_record_init_module() { + run_test(JournalEntry::InitModuleV1 { + wasm_hash: [13u8; 8], + }); +} + +#[tracing_test::traced_test] +#[test] +pub fn test_record_process_exit() { + run_test(JournalEntry::ProcessExitV1 { + exit_code: Some(wasi::ExitCode::Errno(wasi::Errno::Fault)), + }); +} + +#[tracing_test::traced_test] +#[test] +pub fn test_record_set_thread() { + run_test(JournalEntry::SetThreadV1 { + id: 1234u32, + call_stack: vec![1, 2, 3].into(), + memory_stack: vec![4, 5, 6, 7].into(), + store_data: vec![10, 11].into(), + is_64bit: true, + }); +} + +#[tracing_test::traced_test] +#[test] +pub fn test_record_close_thread() { + run_test(JournalEntry::CloseThreadV1 { + id: 987u32, + exit_code: Some(wasi::ExitCode::Errno(wasi::Errno::Fault)), + }); +} + +#[tracing_test::traced_test] +#[test] +pub fn test_record_descriptor_seek() { + run_test(JournalEntry::FileDescriptorSeekV1 { + fd: 765u32, + offset: 9183722450971234i64, + whence: wasi::Whence::End, + }); +} + +#[tracing_test::traced_test] +#[test] +pub fn test_record_descriptor_write() { + run_test(JournalEntry::FileDescriptorWriteV1 { + fd: 54321u32, + offset: 13897412934u64, + data: vec![74u8, 98u8, 36u8].into(), + is_64bit: true, + }); +} + +#[tracing_test::traced_test] +#[test] +pub fn test_record_update_memory() { + run_test(JournalEntry::UpdateMemoryRegionV1 { + region: 76u64..8237453u64, + data: [74u8; 40960].to_vec().into(), + }); +} + +#[tracing_test::traced_test] +#[test] +pub fn test_record_set_clock_time() { + run_test(JournalEntry::SetClockTimeV1 { + clock_id: wasi::Snapshot0Clockid::Realtime, + time: 7912837412934u64, + }); +} + +#[tracing_test::traced_test] +#[test] +pub fn test_record_open_file_descriptor() { + run_test(JournalEntry::OpenFileDescriptorV1 { + fd: 298745u32, + dirfd: 23458922u32, + dirflags: 134512345, + path: "/blah".into(), + o_flags: wasi::Oflags::all(), + fs_rights_base: wasi::Rights::all(), + fs_rights_inheriting: wasi::Rights::all(), + fs_flags: wasi::Fdflags::all(), + }); +} + +#[tracing_test::traced_test] +#[test] +pub fn test_record_close_descriptor() { + run_test(JournalEntry::CloseFileDescriptorV1 { fd: 23845732u32 }); +} + +#[tracing_test::traced_test] +#[test] +pub fn test_record_renumber_file_descriptor() { + run_test(JournalEntry::RenumberFileDescriptorV1 { + old_fd: 27834u32, + new_fd: 398452345u32, + }); +} + +#[tracing_test::traced_test] +#[test] +pub fn test_record_duplicate_file_descriptor() { + run_test(JournalEntry::DuplicateFileDescriptorV1 { + original_fd: 23482934u32, + copied_fd: 9384529u32, + }); +} + +#[tracing_test::traced_test] +#[test] +pub fn test_record_create_directory() { + run_test(JournalEntry::CreateDirectoryV1 { + fd: 238472u32, + path: "/joasjdf/asdfn".into(), + }); +} + +#[tracing_test::traced_test] +#[test] +pub fn test_record_remove_directory() { + run_test(JournalEntry::RemoveDirectoryV1 { + fd: 23894952u32, + path: "/blahblah".into(), + }); +} + +#[tracing_test::traced_test] +#[test] +pub fn test_record_path_set_times() { + run_test(JournalEntry::PathSetTimesV1 { + fd: 1238934u32, + flags: 234523, + path: "/".into(), + st_atim: 923452345, + st_mtim: 350, + fst_flags: wasi::Fstflags::all(), + }); +} + +#[tracing_test::traced_test] +#[test] +pub fn test_record_file_descriptor_set_times() { + run_test(JournalEntry::FileDescriptorSetTimesV1 { + fd: 898785u32, + st_atim: 29834952345, + st_mtim: 239845892345, + fst_flags: wasi::Fstflags::all(), + }); +} + +#[tracing_test::traced_test] +#[test] +pub fn test_record_file_descriptor_set_size() { + run_test(JournalEntry::FileDescriptorSetSizeV1 { + fd: 34958234u32, + st_size: 234958293845u64, + }); +} + +#[tracing_test::traced_test] +#[test] +pub fn test_record_file_descriptor_set_flags() { + run_test(JournalEntry::FileDescriptorSetFlagsV1 { + fd: 982348752u32, + flags: wasi::Fdflags::all(), + }); +} + +#[tracing_test::traced_test] +#[test] +pub fn test_record_file_descriptor_set_rights() { + run_test(JournalEntry::FileDescriptorSetRightsV1 { + fd: 872345u32, + fs_rights_base: wasi::Rights::all(), + fs_rights_inheriting: wasi::Rights::all(), + }); +} + +#[tracing_test::traced_test] +#[test] +pub fn test_record_file_descriptor_advise() { + run_test(JournalEntry::FileDescriptorAdviseV1 { + fd: 298434u32, + offset: 92834529092345, + len: 23485928345, + advice: wasi::Advice::Random, + }); +} + +#[tracing_test::traced_test] +#[test] +pub fn test_record_file_descriptor_allocate() { + run_test(JournalEntry::FileDescriptorAllocateV1 { + fd: 2934852, + offset: 23489582934523, + len: 9845982345, + }); +} + +#[tracing_test::traced_test] +#[test] +pub fn test_record_create_hard_link() { + run_test(JournalEntry::CreateHardLinkV1 { + old_fd: 324983845, + old_path: "/asjdfiasidfasdf".into(), + old_flags: 234857, + new_fd: 34958345, + new_path: "/ashdufnasd".into(), + }); +} + +#[tracing_test::traced_test] +#[test] +pub fn test_record_create_symbolic_link() { + run_test(JournalEntry::CreateSymbolicLinkV1 { + old_path: "/asjbndfjasdf/asdafasdf".into(), + fd: 235422345, + new_path: "/asdf".into(), + }); +} + +#[tracing_test::traced_test] +#[test] +pub fn test_record_unlink_file() { + run_test(JournalEntry::UnlinkFileV1 { + fd: 32452345, + path: "/asdfasd".into(), + }); +} + +#[tracing_test::traced_test] +#[test] +pub fn test_record_path_rename() { + run_test(JournalEntry::PathRenameV1 { + old_fd: 32451345, + old_path: "/asdfasdfas/asdfasdf".into(), + new_fd: 23452345, + new_path: "/ahgfdfghdfghdfgh".into(), + }); +} + +#[tracing_test::traced_test] +#[test] +pub fn test_record_change_directory() { + run_test(JournalEntry::ChangeDirectoryV1 { + path: "/etc".to_string().into(), + }); +} + +#[tracing_test::traced_test] +#[test] +pub fn test_record_epoll_create() { + run_test(JournalEntry::EpollCreateV1 { fd: 45384752 }); +} + +#[tracing_test::traced_test] +#[test] +pub fn test_record_epoll_ctl() { + run_test(JournalEntry::EpollCtlV1 { + epfd: 34523455, + op: wasi::EpollCtl::Unknown, + fd: 23452345, + event: Some(wasi::EpollEventCtl { + events: wasi::EpollType::all(), + ptr: 32452345, + fd: 23452345, + data1: 1235245756, + data2: 23452345, + }), + }); +} + +#[tracing_test::traced_test] +#[test] +pub fn test_record_tty_set() { + run_test(JournalEntry::TtySetV1 { + tty: wasi::Tty { + cols: 1234, + rows: 6754, + width: 4563456, + height: 345, + stdin_tty: true, + stdout_tty: false, + stderr_tty: true, + echo: true, + line_buffered: true, + }, + line_feeds: true, + }); +} + +#[tracing_test::traced_test] +#[test] +pub fn test_record_create_pipe() { + run_test(JournalEntry::CreatePipeV1 { + fd1: 3452345, + fd2: 2345163, + }); +} + +#[tracing_test::traced_test] +#[test] +pub fn test_record_create_event() { + run_test(JournalEntry::CreateEventV1 { + initial_val: 13451345, + flags: 2343, + fd: 5836544, + }); +} + +#[tracing_test::traced_test] +#[test] +pub fn test_record_port_add_addr() { + run_test(JournalEntry::PortAddAddrV1 { + cidr: JournalIpCidrV1 { + ip: Ipv4Addr::LOCALHOST.into(), + prefix: 24, + } + .into(), + }); +} + +#[tracing_test::traced_test] +#[test] +pub fn test_record_del_addr() { + run_test(JournalEntry::PortDelAddrV1 { + addr: Ipv6Addr::LOCALHOST.into(), + }); +} + +#[tracing_test::traced_test] +#[test] +pub fn test_record_addr_clear() { + run_test(JournalEntry::PortAddrClearV1); +} + +#[tracing_test::traced_test] +#[test] +pub fn test_record_port_bridge() { + run_test(JournalEntry::PortBridgeV1 { + network: "mynetwork".into(), + token: format!("blh blah").into(), + security: JournalStreamSecurityV1::ClassicEncryption.into(), + }); +} + +#[tracing_test::traced_test] +#[test] +pub fn test_record_unbridge() { + run_test(JournalEntry::PortUnbridgeV1); +} + +#[tracing_test::traced_test] +#[test] +pub fn test_record_dhcp_acquire() { + run_test(JournalEntry::PortDhcpAcquireV1); +} + +#[tracing_test::traced_test] +#[test] +pub fn test_record_gateway_set() { + run_test(JournalEntry::PortGatewaySetV1 { + ip: Ipv4Addr::new(12, 34, 136, 220).into(), + }); +} + +#[tracing_test::traced_test] +#[test] +pub fn test_record_route_add() { + run_test(JournalEntry::PortRouteAddV1 { + cidr: JournalIpCidrV1 { + ip: Ipv4Addr::LOCALHOST.into(), + prefix: 24, + } + .into(), + via_router: Ipv4Addr::LOCALHOST.into(), + preferred_until: Some(Duration::MAX), + expires_at: Some(Duration::ZERO), + }); +} + +#[tracing_test::traced_test] +#[test] +pub fn test_record_route_clear() { + run_test(JournalEntry::PortRouteClearV1); +} + +#[tracing_test::traced_test] +#[test] +pub fn test_record_route_del() { + run_test(JournalEntry::PortRouteDelV1 { + ip: Ipv4Addr::BROADCAST.into(), + }); +} + +#[tracing_test::traced_test] +#[test] +pub fn test_record_socket_open() { + run_test(JournalEntry::SocketOpenV1 { + af: wasi::Addressfamily::Inet6, + ty: wasi::Socktype::Stream, + pt: wasi::SockProto::Tcp, + fd: 23452345, + }); +} + +#[tracing_test::traced_test] +#[test] +pub fn test_record_socket_listen() { + run_test(JournalEntry::SocketListenV1 { + fd: 12341234, + backlog: 123, + }); +} + +#[tracing_test::traced_test] +#[test] +pub fn test_record_socket_bind() { + run_test(JournalEntry::SocketBindV1 { + fd: 2341234, + addr: SocketAddr::new(Ipv6Addr::UNSPECIFIED.into(), 1234), + }); +} + +#[tracing_test::traced_test] +#[test] +pub fn test_record_socket_connected() { + run_test(JournalEntry::SocketConnectedV1 { + fd: 12341, + addr: SocketAddr::new(Ipv6Addr::UNSPECIFIED.into(), 1234), + }); +} + +#[tracing_test::traced_test] +#[test] +pub fn test_record_socket_accepted() { + run_test(JournalEntry::SocketAcceptedV1 { + listen_fd: 21234, + fd: 1, + peer_addr: SocketAddr::new(Ipv6Addr::UNSPECIFIED.into(), 3452), + fd_flags: wasi::Fdflags::all(), + non_blocking: true, + }); +} + +#[tracing_test::traced_test] +#[test] +pub fn test_record_socket_join_ipv4_multicast() { + run_test(JournalEntry::SocketJoinIpv4MulticastV1 { + fd: 12, + multiaddr: Ipv4Addr::new(123, 123, 123, 123).into(), + iface: Ipv4Addr::new(128, 0, 0, 1).into(), + }); +} + +#[tracing_test::traced_test] +#[test] +pub fn test_record_socket_join_ipv6_multicast() { + run_test(JournalEntry::SocketJoinIpv6MulticastV1 { + fd: 12, + multi_addr: Ipv6Addr::new(123, 123, 123, 123, 1234, 12663, 31, 1324).into(), + iface: 23541, + }); +} + +#[tracing_test::traced_test] +#[test] +pub fn test_record_socket_leave_ipv4_multicast() { + run_test(JournalEntry::SocketLeaveIpv4MulticastV1 { + fd: 12, + multi_addr: Ipv4Addr::new(123, 123, 123, 123).into(), + iface: Ipv4Addr::new(128, 0, 0, 1).into(), + }); +} + +#[tracing_test::traced_test] +#[test] +pub fn test_record_socket_leave_ipv6_multicast() { + run_test(JournalEntry::SocketLeaveIpv6MulticastV1 { + fd: 12, + multi_addr: Ipv6Addr::new(123, 123, 123, 123, 1234, 12663, 31, 1324).into(), + iface: 23541, + }); +} + +#[tracing_test::traced_test] +#[test] +pub fn test_record_socket_send_file() { + run_test(JournalEntry::SocketSendFileV1 { + socket_fd: 22234, + file_fd: 989, + offset: 124, + count: 345673456234651234, + }); +} + +#[tracing_test::traced_test] +#[test] +pub fn test_record_socket_send_to() { + run_test(JournalEntry::SocketSendToV1 { + fd: 123, + data: [98u8; 102400].to_vec().into(), + flags: 1234, + addr: SocketAddr::new(Ipv6Addr::UNSPECIFIED.into(), 3452), + is_64bit: true, + }); +} + +#[tracing_test::traced_test] +#[test] +pub fn test_record_socket_send() { + run_test(JournalEntry::SocketSendV1 { + fd: 123, + data: [98u8; 102400].to_vec().into(), + flags: 1234, + is_64bit: true, + }); +} + +#[tracing_test::traced_test] +#[test] +pub fn test_record_socket_set_opt_flag() { + run_test(JournalEntry::SocketSetOptFlagV1 { + fd: 0, + opt: wasi::Sockoption::Linger, + flag: true, + }); +} + +#[tracing_test::traced_test] +#[test] +pub fn test_record_socket_set_opt_size() { + run_test(JournalEntry::SocketSetOptSizeV1 { + fd: 15, + opt: wasi::Sockoption::Linger, + size: 234234, + }); +} + +#[tracing_test::traced_test] +#[test] +pub fn test_record_socket_set_opt_time() { + run_test(JournalEntry::SocketSetOptTimeV1 { + fd: 0, + ty: SocketOptTimeType::AcceptTimeout, + time: Some(Duration::ZERO), + }); +} + +#[tracing_test::traced_test] +#[test] +pub fn test_record_socket_shutdown() { + run_test(JournalEntry::SocketShutdownV1 { + fd: 123, + how: SocketShutdownHow::Both, + }); +} + +#[tracing_test::traced_test] +#[test] +pub fn test_record_snapshot() { + run_test(JournalEntry::SnapshotV1 { + when: SystemTime::now(), + trigger: SnapshotTrigger::Idle, + }); +} + +#[tracing_test::traced_test] +#[test] +pub fn test_record_alignment() { + assert_eq!(std::mem::align_of::(), 8); + assert_eq!(std::mem::align_of::(), 8); + assert_eq!(std::mem::align_of::(), 8); + assert_eq!(std::mem::align_of::(), 8); + assert_eq!(std::mem::align_of::(), 8); + assert_eq!(std::mem::align_of::(), 8); + assert_eq!(std::mem::align_of::(), 8); + assert_eq!(std::mem::align_of::(), 8); + assert_eq!(std::mem::align_of::(), 8); + assert_eq!(std::mem::align_of::(), 8); + assert_eq!( + std::mem::align_of::(), + 8 + ); + assert_eq!( + std::mem::align_of::(), + 8 + ); + assert_eq!(std::mem::align_of::(), 8); + assert_eq!(std::mem::align_of::(), 8); + assert_eq!(std::mem::align_of::(), 8); + assert_eq!( + std::mem::align_of::(), + 8 + ); + assert_eq!( + std::mem::align_of::(), + 8 + ); + assert_eq!( + std::mem::align_of::(), + 8 + ); + assert_eq!( + std::mem::align_of::(), + 8 + ); + assert_eq!( + std::mem::align_of::(), + 8 + ); + assert_eq!( + std::mem::align_of::(), + 8 + ); + assert_eq!(std::mem::align_of::(), 8); + assert_eq!(std::mem::align_of::(), 8); + assert_eq!(std::mem::align_of::(), 8); + assert_eq!(std::mem::align_of::(), 8); + assert_eq!(std::mem::align_of::(), 8); + assert_eq!(std::mem::align_of::(), 8); + assert_eq!(std::mem::align_of::(), 8); + assert_eq!(std::mem::align_of::(), 8); + assert_eq!(std::mem::align_of::(), 8); + assert_eq!(std::mem::align_of::(), 8); + assert_eq!(std::mem::align_of::(), 8); + assert_eq!(std::mem::align_of::(), 8); + assert_eq!(std::mem::align_of::(), 8); + assert_eq!(std::mem::align_of::(), 8); + assert_eq!(std::mem::align_of::(), 8); + assert_eq!(std::mem::align_of::(), 8); + assert_eq!(std::mem::align_of::(), 8); + assert_eq!(std::mem::align_of::(), 8); + assert_eq!(std::mem::align_of::(), 8); + assert_eq!(std::mem::align_of::(), 8); + assert_eq!(std::mem::align_of::(), 8); + assert_eq!( + std::mem::align_of::(), + 8 + ); + assert_eq!( + std::mem::align_of::(), + 8 + ); + assert_eq!( + std::mem::align_of::(), + 8 + ); + assert_eq!( + std::mem::align_of::(), + 8 + ); + assert_eq!(std::mem::align_of::(), 8); + assert_eq!(std::mem::align_of::(), 8); + assert_eq!(std::mem::align_of::(), 8); + assert_eq!(std::mem::align_of::(), 8); + assert_eq!(std::mem::align_of::(), 8); + assert_eq!(std::mem::align_of::(), 8); + assert_eq!(std::mem::align_of::(), 8); + assert_eq!(std::mem::align_of::(), 8); +}