From 22cb821b6cee15c3d64f3d0c9b244d729affd0d1 Mon Sep 17 00:00:00 2001
From: Caleb Figgers <cdfiggers@gmail.com>
Date: Sun, 21 Jul 2024 20:12:33 -0400
Subject: [PATCH 01/10] Use copy of Jayson for formatter before/after Instead
 of referencing jgraph

---
 test/test-format-file-after.janet  | 300 +++++++++++++---------------
 test/test-format-file-before.janet | 310 +++++++++++++----------------
 2 files changed, 279 insertions(+), 331 deletions(-)

diff --git a/test/test-format-file-after.janet b/test/test-format-file-after.janet
index bd2b914..c1cb75f 100644
--- a/test/test-format-file-after.janet
+++ b/test/test-format-file-after.janet
@@ -1,170 +1,144 @@
-# Uncomment to use `janet-lang/spork` helper functions.
-# (use spork)
-(import jgraph)
+(use judge)
 
-(def- parse-peg
-  "Peg to parse Janet with extra information, namely comments."
-  (peg/compile
-    ~{:ws (/ (* ($) (<- (+ (set " \t\r\f\0\v"))) ($)) ,|[$1 :whitespace {:from $0 :to $2}])
-      :newline (/ (* ($) (<- "\n") ($)) ,|[$1 :newline {:from $0 :to $2}])
-      :readermac (set "';~,|")
-      :symchars (+ (range "09" "AZ" "az" "\x80\xFF") (set "!$%&*+-./:<?=>@^_"))
-      :token (some :symchars)
-      :hex (range "09" "af" "AF")
-      :escape (* "\\" (+ (set "ntrzfev0\"\\")
-                         (* "x" :hex :hex)
-                         (* "u" :hex :hex :hex :hex)
-                         (* "U" :hex :hex :hex :hex :hex :hex)
-                         (error (constant "bad hex escape"))))
-      :comment (/ (* ($) (* "#" '(any (if-not (+ "\n" -1) 1))) ($)) ,|[$1 :comment {:from $0 :to $2}])
-      :span (/ (* ($) ':token ($)) ,|[$1 :span {:from $0 :to $2}])
-      :bytes '(* "\"" (any (+ :escape (if-not "\"" 1))) "\"")
-      :string (/ (* ($) :bytes ($)) ,|[$1 :string {:from $0 :to $2}])
-      :buffer (/ (* ($) (* "@" :bytes) ($)) ,|[$1 :buffer {:from $0 :to $2}])
-      :long-bytes '{:delim (some "`")
-                    :open (capture :delim :n)
-                    :close (cmt (* (not (> -1 "`")) (-> :n) ':delim) ,=)
-                    :main (drop (* :open (any (if-not :close 1)) :close))}
-      :long-string (/ (* ($) :long-bytes ($)) ,|[$1 :string {:from $0 :to $2}])
-      :long-buffer (/ (* ($) (* "@" :long-bytes) ($)) ,|[$1 :buffer {:from $0 :to $2}])
-      :ptuple (/ (* ($) (group (* "(" (any :input) (+ ")" (error)))) ($)) ,|[$1 :ptuple {:from $0 :to $2}])
-      :btuple (/ (* ($) (group (* "[" (any :input) (+ "]" (error)))) ($)) ,|[$1 :btuple {:from $0 :to $2}])
-      :struct (/ (* ($) (group (* "{" (any :input) (+ "}" (error)))) ($)) ,|[$1 :struct {:from $0 :to $2}])
-      :parray (/ (* ($) (group (* "@(" (any :input) (+ ")" (error)))) ($)) ,|[$1 :array {:from $0 :to $2}])
-      :barray (/ (* ($) (group (* "@[" (any :input) (+ "]" (error)))) ($)) ,|[$1 :array {:from $0 :to $2}])
-      :table (/ (* ($) (group (* "@{" (any :input) (+ "}" (error)))) ($)) ,|[$1 :table {:from $0 :to $2}])
-      :rmform (/ (* ($) (group (* ':readermac
-                                  (group (any :non-form))
-                                  :form)) ($))
-                 ,|[$1 :rmform {:from $0 :to $2}])
-      :form (choice :rmform
-                    :parray :barray :ptuple :btuple :table :struct
-                    :buffer :string :long-buffer :long-string
-                    :span)
-      :non-form (choice :newline :ws :comment)
-      :input (choice :non-form :form)
-      :main (* (any :input) (+ -1 (error)))}))
-
-(defn- make-tree
-  "Turn a string of source code into a tree that will be printed"
-  [source]
-  [:top (peg/match parse-peg source)])
+(defmacro- letv [bindings & body]
+  ~(do ,;(seq [[k v] :in (partition 2 bindings)] ['var k v]) ,;body))
 
-(defn count-lines [sq] (inc (length sq)))
+(defn- read-hex [n]
+  (scan-number (string "0x" n)))
 
-(defn- calculate-coverage [ast]
-  (let [second |(get $ 1)
-        ptuples (filter |(= (second $) :ptuple) (second ast))
-        tagged-ptuples (map (fn [sexpr] [(get-in sexpr [0 0 0])
-                                         (filter |(= :newline (second $))
-                                                 (first sexpr))
-                                         sexpr]) ptuples)]
-    (map |[(first $) (count-lines (second $)) (get $ 2)] tagged-ptuples)))
+(defn- check-utf-16 [capture]
+  (let [u (read-hex capture)]
+    (if (and (>= u 0xD800)
+             (<= u 0xDBFF))
+      capture
+      false)))
 
-(defn main [& args]
-  (print "Hello, World!"))
+(def- utf-8->bytes
+  (peg/compile
+    ~{:double-u-esc (/ (* "\\u" (cmt (<- 4) ,|(check-utf-16 $)) "\\u" (<- 4))
+                       ,|(+ (blshift (- (read-hex $0) 0xD800) 10)
+                            (- (read-hex $1) 0xDC00) 0x10000))
+      :single-u-esc (/ (* "\\u" (<- 4)) ,|(read-hex $))
+      :unicode-esc (/ (+ :double-u-esc :single-u-esc)
+                      ,|(string/from-bytes
+                          ;(cond
+                             (<= $ 0x7f) [$]
+                             (<= $ 0x7ff)
+                             [(bor (band (brshift $ 6) 0x1F) 0xC0)
+                              (bor (band (brshift $ 0) 0x3F) 0x80)]
+                             (<= $ 0xffff)
+                             [(bor (band (brshift $ 12) 0x0F) 0xE0)
+                              (bor (band (brshift $ 6) 0x3F) 0x80)
+                              (bor (band (brshift $ 0) 0x3F) 0x80)]
+                             # Otherwise
+                             [(bor (band (brshift $ 18) 0x07) 0xF0)
+                              (bor (band (brshift $ 12) 0x3F) 0x80)
+                              (bor (band (brshift $ 6) 0x3F) 0x80)
+                              (bor (band (brshift $ 0) 0x3F) 0x80)])))
+      :escape (/ (* "\\" (<- (set "avbnfrt\"\\/")))
+                 ,|(get {"a" "\a" "v" "\v" "b" "\b"
+                         "n" "\n" "f" "\f" "r" "\r"
+                         "t" "\t"} $ $))
+      :main (+ (some (+ :unicode-esc :escape (<- 1))) -1)}))
 
 (comment
 
-  (def testfile (slurp "./misc/test-joule.janet.test"))
-  (def testfile (slurp "./misc/small-test.janet.test"))
-  (def testfile (slurp "./project.janet"))
-
-  (defn count-lines [sq] (inc (length sq)))
-  (def line (count-lines (string/find-all "\n" testfile)))
-  (def ast (make-tree testfile))
-
-  (:top @[(@[("declare-project" :span) ("\n" :newline)
-             (" " :whitespace) (" " :whitespace) (":name" :span) (" " :whitespace) ("\"covrj\"" :string) ("\n" :newline)
-             (" " :whitespace) (" " :whitespace) (":description" :span) (" " :whitespace) ("\"TODO: Write a cool description\"" :string)] :ptuple) (" " :whitespace) ("\n" :newline)
-          (" " :whitespace) (" " :whitespace) (" " :whitespace) ("\n" :newline)
-          (@[("declare-executable" :span) ("\n" :newline)
-             (" " :whitespace) (" " :whitespace) (":name" :span) (" " :whitespace) ("\"covrj\"" :string) ("\n" :newline)
-             (" " :whitespace) (" " :whitespace) (":entry" :span) (" " :whitespace) ("\"src/covrj.janet\"" :string) ("\n" :newline)
-             (" " :whitespace) (" " :whitespace) (" :lflags [\"-static\"]" :comment)
-             (" " :whitespace) (" " :whitespace) (":install" :span) (" " :whitespace) ("false" :span)] :ptuple)])
-
-  (:top @[(@[("declare-project" :span {:from 1 :to 16}) ("\n" :newline {:from 16 :to 17})
-             (" " :whitespace {:from 17 :to 18}) (" " :whitespace {:from 18 :to 19}) (":name" :span {:from 19 :to 24}) (" " :whitespace {:from 24 :to 25}) ("\"covrj\"" :string {:from 25 :to 34}) ("\n" :newline {:from 34 :to 35})
-             (" " :whitespace {:from 35 :to 36}) (" " :whitespace {:from 36 :to 37}) (":description" :span {:from 37 :to 49}) (" " :whitespace {:from 49 :to 50}) ("\"TODO: Write a cool description\"" :string {:from 50 :to 82})] :ptuple {:from 0 :to 83}) (" " :whitespace {:from 83 :to 84}) ("\n" :newline {:from 84 :to 85})
-          (" " :whitespace {:from 85 :to 86}) (" " :whitespace {:from 86 :to 87}) (" " :whitespace {:from 87 :to 88}) ("\n" :newline {:from 88 :to 89})
-          (@[("declare-executable" :span {:from 90 :to 108}) ("\n" :newline {:from 108 :to 109})
-             (" " :whitespace {:from 109 :to 110}) (" " :whitespace {:from 110 :to 111}) (":name" :span {:from 111 :to 116}) (" " :whitespace {:from 116 :to 117}) ("\"covrj\"" :string {:from 117 :to 126}) ("\n" :newline {:from 126 :to 127})
-             (" " :whitespace {:from 127 :to 128}) (" " :whitespace {:from 128 :to 129}) (":entry" :span {:from 129 :to 135}) (" " :whitespace {:from 135 :to 136}) ("\"src/covrj.janet\"" :string {:from 136 :to 155}) ("\n" :newline {:from 155 :to 156})
-             (" " :whitespace {:from 156 :to 157}) (" " :whitespace {:from 157 :to 158}) (" :lflags [\"-static\"]" :comment {:from 158 :to 179}) ("\n" :newline {:from 179 :to 180})
-             (" " :whitespace {:from 180 :to 181}) (" " :whitespace {:from 181 :to 182}) (":install" :span {:from 182 :to 190}) (" " :whitespace {:from 190 :to 191}) ("false" :span {:from 191 :to 196})] :ptuple {:from 89 :to 197})])
-
-  @[(@[("declare-project" :span {:from 1 :to 16}) ("\n" :newline {:from 16 :to 17})
-       (" " :whitespace {:from 17 :to 18}) (" " :whitespace {:from 18 :to 19}) (":name" :span {:from 19 :to 24}) (" " :whitespace {:from 24 :to 25}) ("\"covrj\"" :string {:from 25 :to 32}) ("\n" :newline {:from 32 :to 33})
-       (" " :whitespace {:from 33 :to 34}) (" " :whitespace {:from 34 :to 35}) (":description" :span {:from 35 :to 47}) (" " :whitespace {:from 47 :to 48}) ("\"TODO: Write a cool description\"" :string {:from 48 :to 80})] :ptuple {:from 0 :to 81})
-    (@[("declare-executable" :span {:from 88 :to 106}) ("\n" :newline {:from 106 :to 107})
-       (" " :whitespace {:from 107 :to 108}) (" " :whitespace {:from 108 :to 109}) (":name" :span {:from 109 :to 114}) (" " :whitespace {:from 114 :to 115}) ("\"covrj\"" :string {:from 115 :to 122}) ("\n" :newline {:from 122 :to 123})
-       (" " :whitespace {:from 123 :to 124}) (" " :whitespace {:from 124 :to 125}) (":entry" :span {:from 125 :to 131}) (" " :whitespace {:from 131 :to 132}) ("\"src/covrj.janet\"" :string {:from 132 :to 149}) ("\n" :newline {:from 149 :to 150})
-       (" " :whitespace {:from 150 :to 151}) (" " :whitespace {:from 151 :to 152}) (" :lflags [\"-static\"]" :comment {:from 152 :to 173}) ("\n" :newline {:from 173 :to 174})
-       (" " :whitespace {:from 174 :to 175}) (" " :whitespace {:from 175 :to 176}) (":install" :span {:from 176 :to 184}) (" " :whitespace {:from 184 :to 185}) ("false" :span {:from 185 :to 190})] :ptuple {:from 87 :to 191})]
-
-  @[("declare-project" (@[("declare-project" :span {:from 1 :to 16}) ("\n" :newline {:from 16 :to 17}) (" " :whitespace {:from 17 :to 18}) (" " :whitespace {:from 18 :to 19}) (":name" :span {:from 19 :to 24}) (" " :whitespace {:from 24 :to 25}) ("\"covrj\"" :string {:from 25 :to 32}) ("\n" :newline {:from 32 :to 33}) (" " :whitespace {:from 33 :to 34}) (" " :whitespace {:from 34 :to 35}) (":description" :span {:from 35 :to 47}) (" " :whitespace {:from 47 :to 48}) ("\"TODO: Write a cool description\"" :string {:from 48 :to 80})] :ptuple {:from 0 :to 81}))
-    ("declare-executable" (@[("declare-executable" :span {:from 88 :to 106}) ("\n" :newline {:from 106 :to 107}) (" " :whitespace {:from 107 :to 108}) (" " :whitespace {:from 108 :to 109}) (":name" :span {:from 109 :to 114}) (" " :whitespace {:from 114 :to 115}) ("\"covrj\"" :string {:from 115 :to 122}) ("\n" :newline {:from 122 :to 123}) (" " :whitespace {:from 123 :to 124}) (" " :whitespace {:from 124 :to 125}) (":entry" :span {:from 125 :to 131}) (" " :whitespace {:from 131 :to 132}) ("\"src/covrj.janet\"" :string {:from 132 :to 149}) ("\n" :newline {:from 149 :to 150}) (" " :whitespace {:from 150 :to 151}) (" " :whitespace {:from 151 :to 152}) (" :lflags [\"-static\"]" :comment {:from 152 :to 173}) ("\n" :newline {:from 173 :to 174}) (" " :whitespace {:from 174 :to 175}) (" " :whitespace {:from 175 :to 176}) (":install" :span {:from 176 :to 184}) (" " :whitespace {:from 184 :to 185}) ("false" :span {:from 185 :to 190})] :ptuple {:from 87 :to 191}))]
-
-  @[(@[("defn" :span {:from 1 :to 5}) (" " :whitespace {:from 5 :to 6}) ("test-fn" :span {:from 6 :to 13}) (" " :whitespace {:from 13 :to 14}) (@[("bool" :span {:from 15 :to 19})] :btuple {:from 14 :to 20}) ("\n" :newline {:from 20 :to 21}) (" " :whitespace {:from 21 :to 22}) (" " :whitespace {:from 22 :to 23}) (@[("if" :span {:from 24 :to 26}) (" " :whitespace {:from 26 :to 27}) ("bool" :span {:from 27 :to 31}) ("\n" :newline {:from 31 :to 32}) (" " :whitespace {:from 32 :to 33}) (" " :whitespace {:from 33 :to 34}) (" " :whitespace {:from 34 :to 35}) (" " :whitespace {:from 35 :to 36}) (@[("print" :span {:from 37 :to 42}) (" " :whitespace {:from 42 :to 43}) ("\"True!\"" :string {:from 43 :to 50})] :ptuple {:from 36 :to 51}) ("\n" :newline {:from 51 :to 52}) (" " :whitespace {:from 52 :to 53}) (" " :whitespace {:from 53 :to 54}) (" " :whitespace {:from 54 :to 55}) (" " :whitespace {:from 55 :to 56}) (@[("print" :span {:from 57 :to 62}) (" " :whitespace {:from 62 :to 63}) ("\"False!\"" :string {:from 63 :to 71})] :ptuple {:from 56 :to 72})] :ptuple {:from 23 :to 73})] :ptuple {:from 0 :to 74}) ("\n" :newline {:from 74 :to 75}) ("\n" :newline {:from 75 :to 76}) (@[("defn" :span {:from 77 :to 81}) (" " :whitespace {:from 81 :to 82}) ("main" :span {:from 82 :to 86}) (" " :whitespace {:from 86 :to 87}) (@[("&" :span {:from 88 :to 89}) (" " :whitespace {:from 89 :to 90}) ("args" :span {:from 90 :to 94})] :btuple {:from 87 :to 95}) ("\n" :newline {:from 95 :to 96}) (" " :whitespace {:from 96 :to 97}) (" " :whitespace {:from 97 :to 98}) (@[("test-fn" :span {:from 99 :to 106}) (" " :whitespace {:from 106 :to 107}) ("true" :span {:from 107 :to 111})] :ptuple {:from 98 :to 112})] :ptuple {:from 76 :to 113})]
-
-  (calculate-coverage ast)
-
-  (def ptuples (filter |(= (second $) :ptuple) (second ast)))
-  (def functions (filter |(= "defn" (get-in $ [0 0 0])) ptuples))
-  (def function-names (map |(get-in $ [0 2 0]) functions))
-  ()
-
-  (var fn-graph (jgraph/defgraph))
-
-  # (get-in $ [2 0 4 0])
-  (let [coverage (calculate-coverage ast)
-        (map |[(get $ 0) (get $ 1)] coverage)])
-
-  (case a
-    "use" :notest
-    "import" :notest
-    "def" :notest
-    "var" :notest
-    "defn" :needs-test
-    (symbols))
-
-  (do (var currently-in "")
-      (prewalk
-        (fn [n]
-          (when (= :ptuple (get n 1))
-            (when (= "defn" (get-in n [0 0 0]))
-              (let [fn-name (get-in n [0 2 0])]
-                (prewalk
-                  (fn [m]
-                    (when (= :ptuple (get m 1))
-                      (unless (= :array (type (get-in m [0 0 0])))
-                        (pp [fn-name (get-in m [0 0 0])])))
-                    m)
-                  (first n)))))
-          n)
-        (get ast 1)) !!
-      nil)
-
-  (def test-ast [[[[0 1 3]] 16 7 [3 [3 5]] 3 4] 1 [3 4]])
-
-  (defn walker
-    `Simple walker function, that prints non-sequential 
-   members of the form or prints "Sequence" and walks 
-   recursively sequential members of the tree.`
-    [form]
-    (if (or (indexed? form) (dictionary? form))
-      (do (print "Sequence")
-          (walk walker form))
-      (print form)))
-
-  (walk walker test-ast)
-
-  (defn mapper
-    [form]
-    (if (or (indexed? form) (dictionary? form))
-      (do (print "Sequence")
-          (map mapper form))
-      (print form)))
-
-  (map mapper test-ast))
+  "👎"
+  (json/encode "👎")
+  (json/decode (json/encode "👎"))
+
+  (encode "👎")
+  (decode (encode "👎")))
+
+(defn decode
+  ``
+Returns a janet object after parsing JSON. If keywords is truthy,
+string keys will be converted to keywords. If nils is truthy, null
+will become nil instead of the keyword :json/null.
+``
+  [json-source &opt keywords nils]
+
+  (def json-parser
+    {:null (if nils
+             ~(/ (<- (+ "null" "Null")) nil)
+             ~(/ (<- (+ "null" "Null")) :json/null))
+     :bool-t ~(/ (<- (+ "true")) true)
+     :bool-f ~(/ (<- (+ "false")) false)
+     :number ~(/ (<- (* (? "-") :d+ (? (* "." :d+)))) ,|(scan-number $))
+     :string ~(/ (* "\"" (<- (to (* (> -1 (not "\\")) "\"")))
+                    (* (> -1 (not "\\")) "\""))
+                 ,|(string/join (peg/match utf-8->bytes $)))
+     :array ~(/ (* "[" :s* (? (* :value (any (* :s* "," :value)))) "]") ,|(array ;$&))
+     :key-value (if keywords
+                  ~(* :s* (/ :string ,|(keyword $)) :s* ":" :value)
+                  ~(* :s* :string :s* ":" :value))
+     :object ~(/ (* "{" :s* (? (* :key-value (any (* :s* "," :key-value)))) "}")
+                 ,|(from-pairs (partition 2 $&)))
+     :value ~(* :s* (+ :null :bool-t :bool-f :number :string :array :object) :s*)
+     :unmatched ~(/ (<- (to (+ :value -1))) ,|[:unmatched $])
+     :main ~(some (+ :value "\n" :unmatched))})
+
+  (first (peg/match (peg/compile json-parser) json-source)))
+
+(def- bytes->utf-8
+  (peg/compile
+    ~{:four-byte (/ (* (<- (range "\xf0\xff")) (<- 1) (<- 1) (<- 1))
+                    ,|(bor (blshift (band (first $0) 0x07) 18)
+                           (blshift (band (first $1) 0x3F) 12)
+                           (blshift (band (first $2) 0x3F) 6)
+                           (blshift (band (first $3) 0x3F) 0)))
+      :three-byte (/ (* (<- (range "\xe0\xef")) (<- 1) (<- 1))
+                     ,|(bor (blshift (band (first $0) 0x0F) 12)
+                            (blshift (band (first $1) 0x3F) 6)
+                            (blshift (band (first $2) 0x3F) 0)))
+      :two-byte (/ (* (<- (range "\x80\xdf")) (<- 1))
+                   ,|(bor (blshift (band (first $0) 0x1F) 6)
+                          (blshift (band (first $1) 0x3F) 0)))
+      :multi-byte (/ (+ :two-byte :three-byte :four-byte)
+                     ,|(if (< $ 0x10000)
+                         (string/format "\\u%04X" $)
+                         (string/format "\\u%04X\\u%04X"
+                                        (+ (brshift (- $ 0x10000) 10) 0xD800)
+                                        (+ (band (- $ 0x10000) 0x3FF) 0xDC00))))
+      :one-byte (<- (range "\x20\x7f"))
+      :0to31 (/ (<- (range "\0\x1F"))
+                ,|(or ({"\a" "\\u0007" "\b" "\\u0008"
+                        "\t" "\\u0009" "\n" "\\u000A"
+                        "\v" "\\u000B" "\f" "\\u000C"
+                        "\r" "\\u000D"} $)
+                      (string/format "\\u%04X" (first $))))
+      :backslash (/ (<- "\\") "\\\\")
+      :quote (/ (<- "\"") "\\\"")
+      :main (some (+ :0to31 :backslash :quote :one-byte :multi-byte))}))
+
+(defn- encodeone [encoder x depth]
+  (if (> depth 1024) (error "recurred too deeply"))
+  (cond
+    (= x :json/null) "null"
+    (bytes? x) (string "\"" (string/join (peg/match bytes->utf-8 x)) "\"")
+    (indexed? x) (string "[" (string/join (map |(encodeone encoder $ (inc depth)) x) ",") "]")
+    (dictionary? x) (string "{" (string/join
+                                  (seq [[k v] :in (pairs x)]
+                                    (string (encodeone encoder k (inc depth)) ":" (encodeone encoder v (inc depth)))) ",") "}")
+    (case (type x)
+      :nil "null"
+      :boolean (string x)
+      :number (string x)
+      (error "type not supported"))))
+
+(defn encode
+  `` 
+Encodes a janet value in JSON (utf-8). `tab` and `newline` are optional byte sequence which are used 
+to format the output JSON. If `buf` is provided, the formated JSON is append to `buf` instead of a new buffer.
+Returns the modifed buffer.
+``
+  [x &opt tab newline buf]
+
+  (letv [encoder {:indent 0
+                  :buffer @""
+                  :tab tab
+                  :newline newline}
+         ret (encodeone encoder x 0)]
+        (if (and buf (buffer? buf))
+          (buffer/push ret)
+          (thaw ret))))
diff --git a/test/test-format-file-before.janet b/test/test-format-file-before.janet
index b53b118..5879fc7 100644
--- a/test/test-format-file-before.janet
+++ b/test/test-format-file-before.janet
@@ -1,171 +1,145 @@
-# Uncomment to use `janet-lang/spork` helper functions.
-# (use spork)
-(import jgraph)
-
-(def- parse-peg
-  "Peg to parse Janet with extra information, namely comments."
-  (peg/compile
-    ~{:ws (/ (* ($) (<- (+ (set " \t\r\f\0\v"))) ($)) ,|[$1 :whitespace {:from $0 :to $2}])
-      :newline (/ (* ($) (<- "\n") ($)) ,|[$1 :newline {:from $0 :to $2}])
-      :readermac (set "';~,|")
-      :symchars (+ (range "09" "AZ" "az" "\x80\xFF") (set "!$%&*+-./:<?=>@^_"))
-      :token (some :symchars)
-      :hex (range "09" "af" "AF")
-      :escape (* "\\" (+ (set "ntrzfev0\"\\")
-                         (* "x" :hex :hex)
-                         (* "u" :hex :hex :hex :hex)
-                         (* "U" :hex :hex :hex :hex :hex :hex)
-                         (error (constant "bad hex escape"))))
-      :comment (/ (* ($) (* "#" '(any (if-not (+ "\n" -1) 1))) ($)) ,|[$1 :comment {:from $0 :to $2}])
-      :span (/ (* ($) ':token ($)) ,|[$1 :span {:from $0 :to $2}])
-      :bytes '(* "\"" (any (+ :escape (if-not "\"" 1))) "\"")
-      :string (/ (* ($) :bytes ($)) ,|[$1 :string {:from $0 :to $2}])
-      :buffer (/ (* ($) (* "@" :bytes) ($)) ,|[$1 :buffer {:from $0 :to $2}])
-      :long-bytes '{:delim (some "`")
-                    :open (capture :delim :n)
-                    :close (cmt (* (not (> -1 "`")) (-> :n) ':delim) ,=)
-                    :main (drop (* :open (any (if-not :close 1)) :close))}
-      :long-string (/ (* ($) :long-bytes ($)) ,|[$1 :string {:from $0 :to $2}])
-      :long-buffer (/ (* ($) (* "@" :long-bytes) ($)) ,|[$1 :buffer {:from $0 :to $2}])
-      :ptuple (/ (* ($) (group (* "(" (any :input) (+ ")" (error)))) ($)) ,|[$1 :ptuple {:from $0 :to $2}])
-      :btuple (/ (* ($) (group (* "[" (any :input) (+ "]" (error)))) ($)) ,|[$1 :btuple {:from $0 :to $2}])
-      :struct (/ (* ($) (group (* "{" (any :input) (+ "}" (error)))) ($)) ,|[$1 :struct {:from $0 :to $2}])
-      :parray (/ (* ($) (group (* "@(" (any :input) (+ ")" (error)))) ($)) ,|[$1 :array {:from $0 :to $2}])
-      :barray (/ (* ($) (group (* "@[" (any :input) (+ "]" (error)))) ($)) ,|[$1 :array {:from $0 :to $2}])
-      :table (/ (* ($) (group (* "@{" (any :input) (+ "}" (error)))) ($)) ,|[$1 :table{:from $0 :to $2}])
-      :rmform (/ (* ($) (group (* ':readermac
-                                  (group (any :non-form))
-                                  :form)) ($))
-                 ,|[$1 :rmform {:from $0 :to $2}])
-      :form (choice :rmform
-                    :parray :barray :ptuple :btuple :table :struct
-                    :buffer :string :long-buffer :long-string
-                    :span)
-      :non-form (choice :newline :ws :comment)
-      :input (choice :non-form :form)
-      :main (* (any :input) (+ -1 (error)))}))
-
-(defn- make-tree
-  "Turn a string of source code into a tree that will be printed"
-  [source]
-  [:top (peg/match parse-peg source)])
-
-(defn count-lines [sq] (inc (length sq)))
-
-(defn- calculate-coverage [ast]
-    (let [second |(get $ 1) 
-          ptuples (filter |(= (second $) :ptuple) (second ast)) 
-          tagged-ptuples (map (fn [sexpr] [(get-in sexpr [0 0 0]) 
-                                           (filter |(= :newline (second $)) 
-                                                   (first sexpr))
-                                           sexpr]) ptuples)]
-      (map |[(first $) (count-lines (second $)) (get $ 2)] tagged-ptuples)))
-
-(defn main [& args]
-  (print "Hello, World!"))
+(use judge)
+
+(defmacro- letv [bindings & body]
+~(do ,;(seq [[k v] :in (partition 2 bindings)] ['var k v]) ,;body))
+
+(defn- read-hex [n] 
+(scan-number (string "0x" n)))
+
+(defn- check-utf-16 [capture]
+(let [u (read-hex capture)]
+(if (and (>= u 0xD800)
+(<= u 0xDBFF))
+capture 
+false)))
+
+(def- utf-8->bytes 
+(peg/compile
+~{:double-u-esc (/ (* "\\u" (cmt (<- 4) ,|(check-utf-16 $)) "\\u" (<- 4))
+,|(+ (blshift (- (read-hex $0) 0xD800) 10)
+(- (read-hex $1) 0xDC00) 0x10000))
+:single-u-esc (/ (* "\\u" (<- 4)) ,|(read-hex $))
+:unicode-esc  (/ (+ :double-u-esc :single-u-esc)
+,|(string/from-bytes
+;(cond
+(<= $ 0x7f) [$]
+(<= $ 0x7ff)
+[(bor (band (brshift $  6) 0x1F) 0xC0)
+(bor (band (brshift $  0) 0x3F) 0x80)]
+(<= $ 0xffff)
+[(bor (band (brshift $ 12) 0x0F) 0xE0)
+(bor (band (brshift $  6) 0x3F) 0x80)
+(bor (band (brshift $  0) 0x3F) 0x80)]
+# Otherwise
+[(bor (band (brshift $ 18) 0x07) 0xF0)
+(bor (band (brshift $ 12) 0x3F) 0x80)
+(bor (band (brshift $  6) 0x3F) 0x80)
+(bor (band (brshift $  0) 0x3F) 0x80)])))
+:escape       (/ (* "\\" (<- (set "avbnfrt\"\\/")))
+,|(get {"a" "\a" "v" "\v" "b" "\b"
+"n" "\n" "f" "\f" "r" "\r"
+"t" "\t"} $ $))
+:main         (+ (some (+ :unicode-esc :escape (<- 1))) -1)}))
 
 (comment
 
-  (def testfile (slurp "./misc/test-joule.janet.test"))
-  (def testfile (slurp "./misc/small-test.janet.test"))
-  (def testfile (slurp "./project.janet"))
-
-  (defn count-lines [sq] (inc (length sq)))
-  (def line (count-lines (string/find-all "\n" testfile)))
-  (def ast (make-tree testfile))
-
-  (:top @[(@[("declare-project" :span) ("\n" :newline)
-             (" " :whitespace) (" " :whitespace) (":name" :span) (" " :whitespace) ("\"covrj\"" :string) ("\n" :newline)
-             (" " :whitespace) (" " :whitespace) (":description" :span) (" " :whitespace) ("\"TODO: Write a cool description\"" :string)] :ptuple) (" " :whitespace) ("\n" :newline)
-          (" " :whitespace) (" " :whitespace) (" " :whitespace) ("\n" :newline)
-          (@[("declare-executable" :span) ("\n" :newline)
-             (" " :whitespace) (" " :whitespace) (":name" :span) (" " :whitespace) ("\"covrj\"" :string) ("\n" :newline) 
-             (" " :whitespace) (" " :whitespace) (":entry" :span) (" " :whitespace) ("\"src/covrj.janet\"" :string) ("\n" :newline)
-             (" " :whitespace) (" " :whitespace) (" :lflags [\"-static\"]" :comment) 
-             (" " :whitespace) (" " :whitespace) (":install" :span) (" " :whitespace) ("false" :span)] :ptuple)])
-  
-  (:top @[(@[("declare-project" :span {:from 1 :to 16}) ("\n" :newline {:from 16 :to 17}) 
-             (" " :whitespace {:from 17 :to 18}) (" " :whitespace {:from 18 :to 19}) (":name" :span {:from 19 :to 24}) (" " :whitespace {:from 24 :to 25}) ("\"covrj\"" :string {:from 25 :to 34}) ("\n" :newline {:from 34 :to 35}) 
-             (" " :whitespace {:from 35 :to 36}) (" " :whitespace {:from 36 :to 37}) (":description" :span {:from 37 :to 49}) (" " :whitespace {:from 49 :to 50}) ("\"TODO: Write a cool description\"" :string {:from 50 :to 82})] :ptuple {:from 0 :to 83}) (" " :whitespace {:from 83 :to 84}) ("\n" :newline {:from 84 :to 85}) 
-          (" " :whitespace {:from 85 :to 86}) (" " :whitespace {:from 86 :to 87}) (" " :whitespace {:from 87 :to 88}) ("\n" :newline {:from 88 :to 89}) 
-          (@[("declare-executable" :span {:from 90 :to 108}) ("\n" :newline {:from 108 :to 109}) 
-             (" " :whitespace {:from 109 :to 110}) (" " :whitespace {:from 110 :to 111}) (":name" :span {:from 111 :to 116}) (" " :whitespace {:from 116 :to 117}) ("\"covrj\"" :string {:from 117 :to 126}) ("\n" :newline {:from 126 :to 127}) 
-             (" " :whitespace {:from 127 :to 128}) (" " :whitespace {:from 128 :to 129}) (":entry" :span {:from 129 :to 135}) (" " :whitespace {:from 135 :to 136}) ("\"src/covrj.janet\"" :string {:from 136 :to 155}) ("\n" :newline {:from 155 :to 156}) 
-             (" " :whitespace {:from 156 :to 157}) (" " :whitespace {:from 157 :to 158}) (" :lflags [\"-static\"]" :comment {:from 158 :to 179}) ("\n" :newline {:from 179 :to 180}) 
-             (" " :whitespace {:from 180 :to 181}) (" " :whitespace {:from 181 :to 182}) (":install" :span {:from 182 :to 190}) (" " :whitespace {:from 190 :to 191}) ("false" :span {:from 191 :to 196})] :ptuple {:from 89 :to 197})])
-
-  @[(@[("declare-project" :span {:from 1 :to 16}) ("\n" :newline {:from 16 :to 17}) 
-       (" " :whitespace {:from 17 :to 18}) (" " :whitespace {:from 18 :to 19}) (":name" :span {:from 19 :to 24}) (" " :whitespace {:from 24 :to 25}) ("\"covrj\"" :string {:from 25 :to 32}) ("\n" :newline {:from 32 :to 33}) 
-       (" " :whitespace {:from 33 :to 34}) (" " :whitespace {:from 34 :to 35}) (":description" :span {:from 35 :to 47}) (" " :whitespace {:from 47 :to 48}) ("\"TODO: Write a cool description\"" :string {:from 48 :to 80})] :ptuple {:from 0 :to 81}) 
-    (@[("declare-executable" :span {:from 88 :to 106}) ("\n" :newline {:from 106 :to 107}) 
-       (" " :whitespace {:from 107 :to 108}) (" " :whitespace {:from 108 :to 109}) (":name" :span {:from 109 :to 114}) (" " :whitespace {:from 114 :to 115}) ("\"covrj\"" :string {:from 115 :to 122}) ("\n" :newline {:from 122 :to 123}) 
-       (" " :whitespace {:from 123 :to 124}) (" " :whitespace {:from 124 :to 125}) (":entry" :span {:from 125 :to 131}) (" " :whitespace {:from 131 :to 132}) ("\"src/covrj.janet\"" :string {:from 132 :to 149}) ("\n" :newline {:from 149 :to 150}) 
-       (" " :whitespace {:from 150 :to 151}) (" " :whitespace {:from 151 :to 152}) (" :lflags [\"-static\"]" :comment {:from 152 :to 173}) ("\n" :newline {:from 173 :to 174}) 
-       (" " :whitespace {:from 174 :to 175}) (" " :whitespace {:from 175 :to 176}) (":install" :span {:from 176 :to 184}) (" " :whitespace {:from 184 :to 185}) ("false" :span {:from 185 :to 190})] :ptuple {:from 87 :to 191})]
-
-  @[("declare-project" (@[("declare-project" :span {:from 1 :to 16}) ("\n" :newline {:from 16 :to 17}) (" " :whitespace {:from 17 :to 18}) (" " :whitespace {:from 18 :to 19}) (":name" :span {:from 19 :to 24}) (" " :whitespace {:from 24 :to 25}) ("\"covrj\"" :string {:from 25 :to 32}) ("\n" :newline {:from 32 :to 33}) (" " :whitespace {:from 33 :to 34}) (" " :whitespace {:from 34 :to 35}) (":description" :span {:from 35 :to 47}) (" " :whitespace {:from 47 :to 48}) ("\"TODO: Write a cool description\"" :string {:from 48 :to 80})] :ptuple {:from 0 :to 81})) 
-    ("declare-executable" (@[("declare-executable" :span {:from 88 :to 106}) ("\n" :newline {:from 106 :to 107}) (" " :whitespace {:from 107 :to 108}) (" " :whitespace {:from 108 :to 109}) (":name" :span {:from 109 :to 114}) (" " :whitespace {:from 114 :to 115}) ("\"covrj\"" :string {:from 115 :to 122}) ("\n" :newline {:from 122 :to 123}) (" " :whitespace {:from 123 :to 124}) (" " :whitespace {:from 124 :to 125}) (":entry" :span {:from 125 :to 131}) (" " :whitespace {:from 131 :to 132}) ("\"src/covrj.janet\"" :string {:from 132 :to 149}) ("\n" :newline {:from 149 :to 150}) (" " :whitespace {:from 150 :to 151}) (" " :whitespace {:from 151 :to 152}) (" :lflags [\"-static\"]" :comment {:from 152 :to 173}) ("\n" :newline {:from 173 :to 174}) (" " :whitespace {:from 174 :to 175}) (" " :whitespace {:from 175 :to 176}) (":install" :span {:from 176 :to 184}) (" " :whitespace {:from 184 :to 185}) ("false" :span {:from 185 :to 190})] :ptuple {:from 87 :to 191}))]
-
-  @[(@[("defn" :span {:from 1 :to 5}) (" " :whitespace {:from 5 :to 6}) ("test-fn" :span {:from 6 :to 13}) (" " :whitespace {:from 13 :to 14}) (@[("bool" :span {:from 15 :to 19})] :btuple {:from 14 :to 20}) ("\n" :newline {:from 20 :to 21}) (" " :whitespace {:from 21 :to 22}) (" " :whitespace {:from 22 :to 23}) (@[("if" :span {:from 24 :to 26}) (" " :whitespace {:from 26 :to 27}) ("bool" :span {:from 27 :to 31}) ("\n" :newline {:from 31 :to 32}) (" " :whitespace {:from 32 :to 33}) (" " :whitespace {:from 33 :to 34}) (" " :whitespace {:from 34 :to 35}) (" " :whitespace {:from 35 :to 36}) (@[("print" :span {:from 37 :to 42}) (" " :whitespace {:from 42 :to 43}) ("\"True!\"" :string {:from 43 :to 50})] :ptuple {:from 36 :to 51}) ("\n" :newline {:from 51 :to 52}) (" " :whitespace {:from 52 :to 53}) (" " :whitespace {:from 53 :to 54}) (" " :whitespace {:from 54 :to 55}) (" " :whitespace {:from 55 :to 56}) (@[("print" :span {:from 57 :to 62}) (" " :whitespace {:from 62 :to 63}) ("\"False!\"" :string {:from 63 :to 71})] :ptuple {:from 56 :to 72})] :ptuple {:from 23 :to 73})] :ptuple {:from 0 :to 74}) ("\n" :newline {:from 74 :to 75}) ("\n" :newline {:from 75 :to 76}) (@[("defn" :span {:from 77 :to 81}) (" " :whitespace {:from 81 :to 82}) ("main" :span {:from 82 :to 86}) (" " :whitespace {:from 86 :to 87}) (@[("&" :span {:from 88 :to 89}) (" " :whitespace {:from 89 :to 90}) ("args" :span {:from 90 :to 94})] :btuple {:from 87 :to 95}) ("\n" :newline {:from 95 :to 96}) (" " :whitespace {:from 96 :to 97}) (" " :whitespace {:from 97 :to 98}) (@[("test-fn" :span {:from 99 :to 106}) (" " :whitespace {:from 106 :to 107}) ("true" :span {:from 107 :to 111})] :ptuple {:from 98 :to 112})] :ptuple {:from 76 :to 113})]
-  
-  (calculate-coverage ast)
-
-  (def ptuples (filter |(= (second $) :ptuple) (second ast)))
-  (def functions (filter |(= "defn" (get-in $ [0 0 0])) ptuples))
-  (def function-names (map |(get-in $ [0 2 0]) functions))
-  ()
-
-  (var fn-graph (jgraph/defgraph))
-
-  # (get-in $ [2 0 4 0])
-  (let [coverage (calculate-coverage ast)
-         (map |[(get $ 0) (get $ 1)] coverage)])
-
-  (case a
-    "use" :notest
-    "import" :notest
-    "def" :notest
-    "var" :notest
-    "defn" :needs-test
-    (symbols))
-  
-  (do (var currently-in "")
-      (prewalk
-       (fn [n] 
-         (when (= :ptuple (get n 1))
-           (when (= "defn" (get-in n [0 0 0])) 
-             (let [fn-name (get-in n [0 2 0])]
-               (prewalk 
-                (fn [m]
-                  (when (= :ptuple (get m 1))
-                    (unless (= :array (type (get-in m [0 0 0]))) 
-                      (pp [fn-name (get-in m [0 0 0])])))
-                  m)
-                (first n)))))
-         n)
-       (get ast 1))!!
-      nil)
-
-  (def test-ast [[[[0 1 3]] 16 7 [3 [3 5]] 3 4] 1 [3 4]])
-  
-  (defn walker
-  `Simple walker function, that prints non-sequential 
-   members of the form or prints "Sequence" and walks 
-   recursively sequential members of the tree.` 
-  [form] 
-  (if (or (indexed? form) (dictionary? form))
-    (do (print "Sequence")
-        (walk walker form))
-    (print form)))
-
-  (walk walker test-ast)
-
-  (defn mapper 
-    [form] 
-  (if (or (indexed? form) (dictionary? form))
-    (do (print "Sequence")
-        (map mapper form))
-    (print form)))
-  
-  (map mapper test-ast)
-  )
\ No newline at end of file
+"👎"
+(json/encode "👎")
+(json/decode (json/encode "👎"))
+
+(encode "👎")
+(decode (encode "👎"))
+)
+
+(defn decode 
+``
+Returns a janet object after parsing JSON. If keywords is truthy,
+string keys will be converted to keywords. If nils is truthy, null
+will become nil instead of the keyword :json/null.
+``
+[json-source &opt keywords nils] 
+
+(def json-parser 
+{:null (if nils
+~(/ (<- (+ "null" "Null")) nil)
+~(/ (<- (+ "null" "Null")) :json/null))
+:bool-t ~(/ (<- (+ "true")) true)
+:bool-f ~(/ (<- (+ "false")) false)
+:number ~(/ (<- (* (? "-") :d+ (? (* "." :d+)))) ,|(scan-number $))
+:string ~(/ (* "\"" (<- (to (* (> -1 (not "\\")) "\"")))
+(* (> -1 (not "\\")) "\""))
+,|(string/join (peg/match utf-8->bytes $))) 
+:array ~(/ (* "[" :s* (? (* :value (any (* :s* "," :value)))) "]") ,|(array ;$&))
+:key-value (if keywords
+~(* :s* (/ :string ,|(keyword $)) :s* ":" :value)
+~(* :s* :string :s* ":" :value))
+:object ~(/ (* "{" :s* (? (* :key-value (any (* :s* "," :key-value)))) "}")
+,|(from-pairs (partition 2 $&)))
+:value ~(* :s* (+ :null :bool-t :bool-f :number :string :array :object) :s*)
+:unmatched ~(/ (<- (to (+ :value -1))) ,|[:unmatched $])
+:main ~(some (+ :value "\n" :unmatched))})
+
+(first (peg/match (peg/compile json-parser) json-source)))
+
+(def- bytes->utf-8
+(peg/compile
+~{:four-byte  (/ (* (<- (range "\xf0\xff")) (<- 1) (<- 1) (<- 1))
+,|(bor (blshift (band (first $0) 0x07) 18)
+(blshift (band (first $1) 0x3F) 12)
+(blshift (band (first $2) 0x3F) 6)
+(blshift (band (first $3) 0x3F) 0)))
+:three-byte (/ (* (<- (range "\xe0\xef")) (<- 1) (<- 1))
+,|(bor (blshift (band (first $0) 0x0F) 12)
+(blshift (band (first $1) 0x3F) 6)
+(blshift (band (first $2) 0x3F) 0)))
+:two-byte   (/ (* (<- (range "\x80\xdf")) (<- 1))
+,|(bor (blshift (band (first $0) 0x1F) 6)
+(blshift (band (first $1) 0x3F) 0)))
+:multi-byte (/ (+ :two-byte :three-byte :four-byte)
+,|(if (< $ 0x10000) 
+(string/format "\\u%04X" $)
+(string/format "\\u%04X\\u%04X" 
+(+ (brshift (- $ 0x10000) 10) 0xD800) 
+(+ (band (- $ 0x10000) 0x3FF) 0xDC00))))
+:one-byte   (<- (range "\x20\x7f"))
+:0to31      (/ (<- (range "\0\x1F"))
+,|(or ({"\a" "\\u0007" "\b" "\\u0008"
+"\t" "\\u0009" "\n" "\\u000A"
+"\v" "\\u000B" "\f" "\\u000C"
+"\r" "\\u000D"} $)
+(string/format "\\u%04X" (first $))))
+:backslash  (/ (<- "\\") "\\\\")
+:quote      (/ (<- "\"") "\\\"")
+:main       (some (+ :0to31 :backslash :quote :one-byte :multi-byte))}))
+
+(defn- encodeone [encoder x depth]
+(if (> depth 1024) (error "recurred too deeply"))
+(cond
+(= x :json/null) "null"
+(bytes? x) (string "\"" (string/join (peg/match bytes->utf-8 x)) "\"")
+(indexed? x) (string "[" (string/join (map |(encodeone encoder $ (inc depth)) x) ",") "]")
+(dictionary? x) (string "{" (string/join
+(seq [[k v] :in (pairs x)]
+(string (encodeone encoder k (inc depth)) ":" (encodeone encoder v (inc depth)))) ",") "}")
+(case (type x)
+:nil "null"
+:boolean (string x)
+:number (string x)
+(error "type not supported"))))
+
+(defn encode 
+`` 
+Encodes a janet value in JSON (utf-8). `tab` and `newline` are optional byte sequence which are used 
+to format the output JSON. If `buf` is provided, the formated JSON is append to `buf` instead of a new buffer.
+Returns the modifed buffer.
+``
+[x &opt tab newline buf]
+
+(letv [encoder {:indent 0
+:buffer @""
+:tab tab
+:newline newline}
+ret (encodeone encoder x 0)]
+(if (and buf (buffer? buf))
+(buffer/push ret)
+(thaw ret))))

From 8c3b9c2e0281cae2bda3938d2a044f2c4679f6c1 Mon Sep 17 00:00:00 2001
From: Caleb Figgers <cdfiggers@gmail.com>
Date: Sun, 21 Jul 2024 20:13:16 -0400
Subject: [PATCH 02/10] Update import of /src/main to avoid calling `main`
 during tests

---
 test/test-main.janet | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/test/test-main.janet b/test/test-main.janet
index 6fc9290..845788f 100644
--- a/test/test-main.janet
+++ b/test/test-main.janet
@@ -1,11 +1,11 @@
 (use judge)
 
-(use ../src/main)
+(import ../src/main)
 
 (deftest "parse-content-length"
-  (test (parse-content-length "000:123:456:789") 123)
-  (test (parse-content-length "123:456:789") 456)
-  (test (parse-content-length "0123:456::::789") 456))
+  (test (main/parse-content-length "000:123:456:789") 123)
+  (test (main/parse-content-length "123:456:789") 456)
+  (test (main/parse-content-length "0123:456::::789") 456))
 
 (deftest "test binding-to-lsp-item"
   (setdyn :eval-env (table/proto-flatten (make-env root-env)))
@@ -25,7 +25,7 @@
                     [@[:a 1] :array] # [(coro) :fiber]
                     ['anil :nil]])
 
-  (test (map (juxt 1 |(binding-to-lsp-item (first $))) test-cases)
+  (test (map (juxt 1 |(main/binding-to-lsp-item (first $))) test-cases)
         @[[:symbol    {:kind 12 :label hello}]
           [:boolean   {:kind 6  :label true}]
           [:function  {:kind 3  :label @%}]
@@ -43,7 +43,7 @@
           [:nil       {:kind 12 :label anil}]]))
 
 (deftest "test find-all-module-files"
-  (test (find-all-module-files (os/cwd))
+  (test (main/find-all-module-files (os/cwd))
     @["/home/caleb/projects/vscode/vscode-janet-plus-plus/janet-lsp/src/main.janet"
       "/home/caleb/projects/vscode/vscode-janet-plus-plus/janet-lsp/src/rpc.janet"
       "/home/caleb/projects/vscode/vscode-janet-plus-plus/janet-lsp/src/logging.janet"
@@ -61,7 +61,7 @@
       "/home/caleb/projects/vscode/vscode-janet-plus-plus/janet-lsp/build/janet-lsp.jimage"]))
 
 (deftest "test find-all-module-files"
-  (test (find-all-module-files (os/cwd) true)
+  (test (main/find-all-module-files (os/cwd) true)
     @["/home/caleb/projects/vscode/vscode-janet-plus-plus/janet-lsp/src/main.janet"
       "/home/caleb/projects/vscode/vscode-janet-plus-plus/janet-lsp/src/rpc.janet"
       "/home/caleb/projects/vscode/vscode-janet-plus-plus/janet-lsp/src/logging.janet"
@@ -79,15 +79,15 @@
       "/home/caleb/projects/vscode/vscode-janet-plus-plus/janet-lsp/build/janet-lsp.jimage"]))
 
 (deftest "test find-unique-paths"
-  (test (find-unique-paths (find-all-module-files (os/cwd)))
+  (test (main/find-unique-paths (main/find-all-module-files (os/cwd)))
     @["./src/:all:.janet"
       "./libs/:all:.janet"
       "./test/:all:.janet"
       "./build/:all:.jimage"]))
 
 (deftest "test find-unique-paths"
-  (test (find-unique-paths (find-all-module-files (os/cwd) true))
+  (test (main/find-unique-paths (main/find-all-module-files (os/cwd) true))
     @["./src/:all:.janet"
       "./libs/:all:.janet"
       "./test/:all:.janet"
-      "./build/:all:.jimage"]))
\ No newline at end of file
+      "./build/:all:.jimage"]))

From bb2f67ef6c09a6d7b4f54b83fa315c821c1532a5 Mon Sep 17 00:00:00 2001
From: Caleb Figgers <cdfiggers@gmail.com>
Date: Sun, 21 Jul 2024 20:13:54 -0400
Subject: [PATCH 03/10] Disable test setup (currently throwing errors)

---
 test/test-integration.janet | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/test/test-integration.janet b/test/test-integration.janet
index 8a55704..48e9055 100644
--- a/test/test-integration.janet
+++ b/test/test-integration.janet
@@ -54,13 +54,15 @@
 
 (deftest-type with-process
   :setup    (fn []
-              (start-lsp))
+              # (start-lsp)
+              )
   :reset    (fn [context]
               (printf "context is: %q" context)
-              (exit-lsp context)
-              (start-lsp))
+              # (exit-lsp context)
+              #   (start-lsp)
+              )
   :teardown (fn [context] ))
 
 (deftest: with-process "Starts and exits" [context]
   (printf "context is: %q" context)
-  (test (= true true) true))
\ No newline at end of file
+  (test (= true true) true))

From 50f4a5bbc165e6e630a1329dae42f37c00235b15 Mon Sep 17 00:00:00 2001
From: Caleb Figgers <cdfiggers@gmail.com>
Date: Mon, 29 Jul 2024 19:13:00 -0500
Subject: [PATCH 04/10] Formatting

---
 src/main.janet | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/src/main.janet b/src/main.janet
index 0f21a05..3fc128c 100644
--- a/src/main.janet
+++ b/src/main.janet
@@ -227,7 +227,7 @@
   (file/write file response)
 
   # Flush response
-  (file/flush file))
+  (file/flush file)) 
 
 (defn read-message []
   (let [input (file/read stdin :line)
@@ -236,13 +236,14 @@
     (json/decode input))) 
 
 (defn message-loop [&named state]
+  (logging/log "Loop enter")
   (let [message (read-message)] 
     (logging/log (string/format "got: %q" message))
     (match (handle-message message state)
       [:ok new-state & response] (do
-                                 (logging/log "successful rpc")
-                                 (write-response stdout (rpc/success-response (get message "id") ;response))
-                                 (message-loop :state new-state))
+                                   (logging/log "successful rpc")
+                                   (write-response stdout (rpc/success-response (get message "id") ;response))
+                                   (message-loop :state new-state))
       [:noresponse new-state] (message-loop :state new-state)
 
       [:error new-state err] (printf "unhandled error response: %m" err)

From 8cd3ff90c12564b32950d647a8ddd5c9f58ee5f6 Mon Sep 17 00:00:00 2001
From: Caleb Figgers <cdfiggers@gmail.com>
Date: Mon, 29 Jul 2024 19:13:19 -0500
Subject: [PATCH 05/10] Prelim work on jump to definition

---
 src/main.janet | 22 ++++++++++++++++++++--
 1 file changed, 20 insertions(+), 2 deletions(-)

diff --git a/src/main.janet b/src/main.janet
index 3fc128c..c8fdba0 100644
--- a/src/main.janet
+++ b/src/main.janet
@@ -147,7 +147,7 @@
   that this server provides so the client knows what it can request.
   ``
   [state params]
-  (logging/log (string/format "on-initialize called with these params: %m" params))
+  (comment (logging/log (string/format "on-initialize called with these params: %m" params)))
 
   (if-let [diagnostic? (get-in params ["capabilities" "textDocument" "diagnostic"])]
     (setdyn :push-diagnostics false)
@@ -161,7 +161,9 @@
                                                   :workspaceDiagnostics false}
                              :hoverProvider true
                              :signatureHelpProvider {:triggerCharacters [" "]}
-                             :documentFormattingProvider true}
+                             :documentFormattingProvider true
+                            #  :definitionProvider true
+                             }
               :serverInfo {:name "janet-lsp"
                            :version version}}])
 
@@ -190,6 +192,21 @@
   [state params]
   [:ok state :json/null])
 
+# (defn on-document-definition
+#   ``
+#   Called by the LSP client to request the location of a symbol's definition.
+#   ``
+#   [state params]
+#   (let [uri (get-in params ["textDocument" "uri"])
+#         content (get-in state [:documents uri :content])
+#         {"line" line "character" character} (get params "position")
+#         {:word define-word :range [start end]} (lookup/word-at {:line line :character character} content)]
+#     (if-let [[uri line col] ((dyn (symbol define-word) :source-map))]
+#       [:ok state {:uri uri 
+#                   :range {:start {:line (max 0 (dec line)) :character col}
+#                           :end {:line (max 0 (dec line)) :character col}}}]
+#       [:ok state :json/null])))
+
 (defn handle-message [message state]
   (let [id (get message "id")
         method (get message "method")
@@ -208,6 +225,7 @@
       "textDocument/signatureHelp" (on-document-signature-help state params)
       # "textDocument/references" (on-document-references state params) TODO: Implement this? See src/lsp/api.ts:103
       # "textDocument/documentSymbol" (on-document-symbols state params) TODO: Implement this? See src/lsp/api.ts:121
+      # "textDocument/definition" (on-document-definition state params)
       "janet/serverInfo" (on-janet-serverinfo state params)
       "shutdown" (on-shutdown state params)
       "exit" (on-exit state params)

From 86b00de022ab536921c165f4e5c9aef6f709e043 Mon Sep 17 00:00:00 2001
From: Caleb Figgers <cdfiggers@gmail.com>
Date: Mon, 29 Jul 2024 22:10:33 -0500
Subject: [PATCH 06/10] Expanded CHANGELOG

---
 CHANGELOG.md | 30 ++++++++++++++++++++++++++++++
 1 file changed, 30 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 2644e46..1c9efd2 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,36 @@
 All notable changes to this project will be documented in this file.
 Format for entires is <version-string> - release date.
 
+## 0.0.5 - 2024-06-14
+
+- Diagnostics
+  - Only syntax highlight function signatures in pop-up hover definitions by @CFiggers in [#23](https://github.com/CFiggers/janet-lsp/pull/23)
+  - Fix bug with publishing diagnostics (was causing last diagnostic warning to not clear when using for e.g. nvim-lsp) by @CFiggers in [#23](https://github.com/CFiggers/janet-lsp/pull/23)
+- Tests
+  - Additional tests and reorganization by @CFiggers in [#23](https://github.com/CFiggers/janet-lsp/pull/23)
+
+## 0.0.4 - 2024-01-26
+
+- Project
+  - We now install `janet-lsp` as a binscript instead of building an executable at all by @CFiggers in https://github.com/CFiggers/janet-lsp/pull/19
+- Formatting
+  - Tweak to vendored copy of `spork/fmt` by @CFiggers in https://github.com/CFiggers/janet-lsp/pull/19
+- Diagnostics
+  - `eval-buffer` now starts with a clean environment on every evaluation (resolving many consistency issues) by @CFiggers in https://github.com/CFiggers/janet-lsp/pull/19
+  - Can now push diagnostics to clients that prefer not to request by issuing `testDocument/publishDiagnostics` RPC notifications by @CFiggers in https://github.com/CFiggers/janet-lsp/pull/18
+- Completion and Hover Definitions
+  - Bugs with jpm definitions resolved by @CFiggers in https://github.com/CFiggers/janet-lsp/pull/19
+- Signature helps
+  - Fix bugs in `sexp-at` (off-by-one and crash on unparenthesized top-level forms) by @CFiggers in https://github.com/CFiggers/janet-lsp/pull/19
+- RPC
+  - Can now send properly formatted LSP notifications (in addition to responses, needed for publishing diagnostics)by @CFiggers in https://github.com/CFiggers/janet-lsp/pull/19
+- Testing
+  - Migrated Judge tests from main.janet into a separate fileby @CFiggers in https://github.com/CFiggers/janet-lsp/pull/19
+- CLI
+  - `--debug` flag now works correctlyby @CFiggers in https://github.com/CFiggers/janet-lsp/pull/19
+- Misc
+  - Source code formatting and comment cleanup by @CFiggers in https://github.com/CFiggers/janet-lsp/pull/17
+
 ## 0.0.3 - 2024-01-09
 
 - Completion

From c8f4c7cbdd97d36e10c24c8c46e875fd8b815538 Mon Sep 17 00:00:00 2001
From: Caleb Figgers <cdfiggers@gmail.com>
Date: Tue, 30 Jul 2024 19:09:53 -0500
Subject: [PATCH 07/10] Do away with `read-offset`

---
 src/main.janet | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/src/main.janet b/src/main.janet
index c8fdba0..49bf1ce 100644
--- a/src/main.janet
+++ b/src/main.janet
@@ -232,10 +232,6 @@
       [:noresponse state])))
 
 (def line-ending "\r\n\r\n")
-(def read-offset
-  (case (os/which)
-    :windows 1
-    2))
 
 (defn write-response [file response]
   # Write headers
@@ -248,9 +244,9 @@
   (file/flush file)) 
 
 (defn read-message []
-  (let [input (file/read stdin :line)
-        content-length (+ (parse-content-length input) read-offset)
-        input (file/read stdin content-length)]
+  (let [content-length-line (file/read stdin :line)
+        _ (file/read stdin :line)
+        input (file/read stdin (parse-content-length content-length-line))]
     (json/decode input))) 
 
 (defn message-loop [&named state]

From 9b4aef91ee9814184d3162d30d13f49f6ca5ea35 Mon Sep 17 00:00:00 2001
From: Caleb Figgers <cdfiggers@gmail.com>
Date: Wed, 31 Jul 2024 16:05:38 -0500
Subject: [PATCH 08/10] Handle failure to write janetlsp.log.txt

---
 src/logging.janet | 3 ++-
 src/main.janet    | 4 +++-
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/logging.janet b/src/logging.janet
index 95e0243..77435b2 100644
--- a/src/logging.janet
+++ b/src/logging.janet
@@ -20,5 +20,6 @@
     (print (:print (dyn :client) output)))
 
   (when (dyn :debug)
-    (spit "janetlsp.log.txt" (string output "\n") :a)
+    (try (spit "janetlsp.log.txt" (string output "\n") :a) 
+      ([_]))
     (file/write stderr (string output "\n"))))
diff --git a/src/main.janet b/src/main.janet
index 49bf1ce..654a5c5 100644
--- a/src/main.janet
+++ b/src/main.janet
@@ -295,7 +295,9 @@
 
 (defn start-language-server []
   (print "Starting LSP")
-  (when (dyn :debug) (spit "janetlsp.log.txt" ""))
+  (when (dyn :debug) 
+    (try (spit "janetlsp.log.txt" "") 
+      ([_] (logging/log "Tried to write to janetlsp.log txt, but couldn't"))))
 
   (merge-module root-env jpm-defs nil true)
   (setdyn :eval-env (make-env root-env))

From f4638f86ec1f20e5223e82fe836b3ac4c8595121 Mon Sep 17 00:00:00 2001
From: Caleb Figgers <cdfiggers@gmail.com>
Date: Wed, 31 Jul 2024 16:10:27 -0500
Subject: [PATCH 09/10] Bugfix: Line endings broken on Windows

---
 src/main.janet | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/main.janet b/src/main.janet
index 654a5c5..1629e6a 100644
--- a/src/main.janet
+++ b/src/main.janet
@@ -231,11 +231,10 @@
       "exit" (on-exit state params)
       [:noresponse state])))
 
-(def line-ending "\r\n\r\n")
-
 (defn write-response [file response]
   # Write headers
-  (file/write file (string "Content-Length: " (length response) line-ending))
+  (file/write file (string "Content-Length: " (length response) (case (os/which)
+                                                                  :windows "\n\n" "\r\n\r\n")))
 
   # Write response
   (file/write file response)

From 2587cec3993e56097f39b6a398f34b698b063fbc Mon Sep 17 00:00:00 2001
From: CFiggers <55862180+CFiggers@users.noreply.github.com>
Date: Wed, 31 Jul 2024 21:11:03 -0500
Subject: [PATCH 10/10] Version bump and update README and CHANGELOG for v0.0.6

---
 CHANGELOG.md   | 10 ++++++++++
 README.md      |  7 +++++--
 project.janet  |  2 +-
 src/main.janet |  2 +-
 4 files changed, 17 insertions(+), 4 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 1c9efd2..9d4a1fe 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,16 @@
 All notable changes to this project will be documented in this file.
 Format for entires is <version-string> - release date.
 
+## 0.0.6 - 2024-07-31
+
+- Core Functionality
+  - Factored out `line-ending` and `read-offset` functions/values by @CFiggers in https://github.com/CFiggers/janet-lsp/pull/25
+  - Fix bug with line endings (communication over `stdio` was broken on Widows) by @CFiggers in https://github.com/CFiggers/janet-lsp/pull/25
+- Logging
+  - Now fail gracefully when unable to write to `janetlsp.log.txt` by @CFiggers in https://github.com/CFiggers/janet-lsp/pull/25
+- Jump to Definition
+  - Preliminary work (not completed yet) by @CFiggers in https://github.com/CFiggers/janet-lsp/pull/25
+
 ## 0.0.5 - 2024-06-14
 
 - Diagnostics
diff --git a/README.md b/README.md
index 64762c1..fd287f1 100644
--- a/README.md
+++ b/README.md
@@ -36,12 +36,15 @@ Desirable, but possibly more complicated/difficult features include:
 
 ## Clients (i.e. Editors)
 
-Currently, the only editor tested and known working with Janet LSP is [Visual Studio Code](https://code.visualstudio.com/), which you can try/take advantage of by installing the [Janet++](https://github.com/CFiggers/vscode-janet-plus-plus) extension [from the VS Code marketplace](https://marketplace.visualstudio.com/items?itemName=CalebFiggers.vscode-janet-plus-plus).
+Currently, Janet LSP is being regularly tested and is expected to work out of the box with two major editors:
+
+- [Visual Studio Code](https://code.visualstudio.com/), which you can try/take advantage of by installing the [Janet++](https://github.com/CFiggers/vscode-janet-plus-plus) extension [from the VS Code marketplace](https://marketplace.visualstudio.com/items?itemName=CalebFiggers.vscode-janet-plus-plus), and
+- [Neovim](https://neovim.io/), which ships with support for LSP servers.
 
 Other editors that implement LSP client protocols, either built-in or through editor extensions, include:
 
 - Emacs
-- vim/neovim
+- Vim
 - Sublime Text
 - Helix
 - Kakoune
diff --git a/project.janet b/project.janet
index 159a09e..95a4e9b 100644
--- a/project.janet
+++ b/project.janet
@@ -1,7 +1,7 @@
 (declare-project
   :name "janet-lsp"
   :description "A Language Server (LSP) for the Janet Programming Language"
-  :version "0.0.5"
+  :version "0.0.6"
   :dependencies ["https://github.com/janet-lang/spork.git"
                  "https://github.com/CFiggers/jayson.git"
                  "https://github.com/ianthehenry/judge.git"
diff --git a/src/main.janet b/src/main.janet
index 1629e6a..10ccdd0 100644
--- a/src/main.janet
+++ b/src/main.janet
@@ -13,7 +13,7 @@
 
 (use judge)
 
-(def version "0.0.5")
+(def version "0.0.6")
 
 (def jpm-defs (require "../libs/jpm-defs"))